From 4c9009f47facb3d3c82f5e1186d257629d9d1f11 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 08:32:08 -0400 Subject: [PATCH 01/44] Record UK shared-graph contract lane baseline Source-only verification that WeightUpdate, the ordered-axis weight receipt, and the three KernelContext frame fields are all absent on 15ebde806, with the exact #901 consumer read that fixes their shape. Co-Authored-By: Claude Opus 5 (cherry picked from commit 97428cd9f4d3d3a668ee1474f6fa970b2de0ecf7) --- PROGRESS-uk-shared-graph-contracts.md | 66 +++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 PROGRESS-uk-shared-graph-contracts.md diff --git a/PROGRESS-uk-shared-graph-contracts.md b/PROGRESS-uk-shared-graph-contracts.md new file mode 100644 index 000000000..bd18785be --- /dev/null +++ b/PROGRESS-uk-shared-graph-contracts.md @@ -0,0 +1,66 @@ +# UK-enabling shared graph contracts (amendments 25 and 26) + +Lane journal. Append-only within this lane; historicize rather than +overwrite once the branch merges (CLAUDE.md, "Root journals are history"). + +## State + +Bounded, source-only slice extracting the shared graph contract that +María's UK full-build graph (#901) consumes, as numbered amendments on +current `main`. No UK graph stage, calibration science or country +kernel is added here — those stay in #901. + +- Worktree: `_worktrees/microcosm-uk-shared-graph-contracts-20260913` +- Branch: `uk-shared-graph-contracts-20260913` +- Base: `origin/main` `15ebde806cd1a262363f7217fe535c7234ff757f` +- Reviewed UK head: #901 `051fb972b19d319d58277bd63306d0d0e0947ce2` + (draft, base `microcosm-us-launch-integration-20260909`, unchanged + since 2026-09-10T21:03:10Z; re-verified via `gh` on 2026-09-13) +- Source review followed: `uk-parallel-review.md` (2026-09-12), section + "Best independent implementation slice" + +**Runtime is UNTESTED in this lane.** Instructions forbid pytest, +production imports, engine, native sources and installation here. Only +stdlib `ast`/`ruff`/CI-inventory source checks were run. A finite +invented-only runtime plan is at the end of this file for root review +*before* execution. + +## Absence verified on base 15ebde806 + +| Contract | Present on main? | Evidence | +| --- | --- | --- | +| `WeightUpdate` (same-kind weight replacement) | **absent** | `grep -rni weightupdate .` over the worktree returns nothing; `decl.py:320` carries only `WeightTransition`, whose `__post_init__` requires `to_kind` strictly later in `WEIGHT_KINDS`, and `population.py:1944` rejects a non-forward move. A same-kind update is therefore unrepresentable. | +| `weight_update_receipt` / ordered-axis evidence | **absent** | no `weight_update` module under `packages/microcosm-graph/src/microcosm/graph/`. | +| `KernelContext.frame_metadata` | **absent** | `kernel.py:361-370` lists the complete field set; no metadata field. | +| `KernelContext.frame_mass_log` | **absent** (read side) | same field list. The *write* side already exists: `population._append_frame_mass_log` (`population.py:2117`) already ingests `receipt['frame_mass_log_append']`, so only the kernel's view of the incoming log is missing. | +| `KernelContext.frame_column_order` | **absent** | same field list. The executor projects `tables[entity]` in declaration order (`executor.py:600-628`), not the population version's own column order, so a consumer cannot reconstruct the frame layout. | + +Frame-side prerequisites already on main: `Frame.mass_log`, +`Frame.metadata`, `MassChangeRecord` and `_freeze_metadata` +(`packages/microcosm-frame/src/microcosm/frame/bundle.py`). + +Interface lock on base matches the files exactly: +`decl.py ed0a859adcae12510d5ba74d51c694617201f7b448b108a3f602410f5da44876`, +`kernel.py dbf57c137330f0f12744c557ee594586a1b308b6d6adaba1938e2b6efded21ca`. + +## Actual consumer read before specifying shape + +`packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py` +at `051fb972` (SHA-256 `fc6f5b33127020f6e0529b39304715fe2028d47a6627b152b1e52e0d69f2efdc`): + +- `context_frame` (L58-80) reads `context.frame_column_order.get(entity, ...)`, + `context.weights`, `context.strata`, `getattr(context, "frame_mass_log", ())` + and `getattr(context, "frame_metadata", {})`. +- `UKSampleNormalizationKernel` (L386-412 region) declares + `WeightUpdate("household", weight_kind, "Normalize sampled source-family mass.")` + with `mass="declared"` and places `weight_update_receipt(ids)` under + `receipt["weight_update"]`. + +## Done + +- (nothing yet; baseline recorded) + +## Next + +- Amendment 25: `WeightUpdate` + ordered-axis receipt. +- Amendment 26: `KernelContext` frame metadata / mass log / column order. From cb08ee054339c01ba54be17c8a0e9d5aa8b683f7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 08:40:32 -0400 Subject: [PATCH 02/44] Declare a same-kind weight update (graph amendment 25) WeightTransition only moves a weight kind forward, so a stage that recomputes weights it already holds -- a sampling normalization, a re-solve of an existing calibration -- had no declaration at all, and the only way to express it was to misdeclare a transition. WeightUpdate(entity, kind, reason, mass) is that declaration and is deliberately narrower: incumbent, declared and returned kinds must all agree; mass is conserve or declared, never free; and reason is required, non-empty and normative. Positional replacement values are not self-describing, so a kernel binds the ordered entity axis it computed them against with weight_update_receipt, and the executor recomputes that binding from the incumbent axis. A cache hit reconstructs the KernelResult and re-applies the REWEIGHT to the current base, so replay re-enters the same check rather than a parallel rule. to_kind is a property, not a field, so the two declarations have disjoint field sets and a WeightUpdate can never canonicalize -- or serialize -- to the same bytes as a WeightTransition. The transition payload is byte-for-byte unchanged. Node gains no field, so no existing node key moves. decl.py is re-locked; kernel.py is untouched. Raised by the source review of the UK full-build graph (#901, head 051fb972), whose uk.full.normalize node is the first consumer. Its UK graph stages and calibration science stay in that branch. Runtime UNTESTED in this lane: source-only ast/ruff and CI test inventory checks only. Co-Authored-By: Claude Opus 5 (cherry picked from commit ef2dc69c32bc02d206de2857272d73cf73541abf) --- ...red-graph-contracts-weight-update.added.md | 1 + docs/graph-acceptance.md | 40 ++ docs/graph-interface.lock | 2 +- .../src/microcosm/graph/__init__.py | 7 + .../src/microcosm/graph/decl.py | 87 ++++- .../src/microcosm/graph/explain.py | 14 +- .../src/microcosm/graph/population.py | 69 +++- .../src/microcosm/graph/serialize.py | 51 ++- .../src/microcosm/graph/weight_update.py | 69 ++++ .../shared/test_graph_weight_update.py | 352 ++++++++++++++++++ 10 files changed, 675 insertions(+), 17 deletions(-) create mode 100644 changelog.d/uk-shared-graph-contracts-weight-update.added.md create mode 100644 packages/microcosm-graph/src/microcosm/graph/weight_update.py create mode 100644 packages/microcosm-graph/tests/engine_free/shared/test_graph_weight_update.py diff --git a/changelog.d/uk-shared-graph-contracts-weight-update.added.md b/changelog.d/uk-shared-graph-contracts-weight-update.added.md new file mode 100644 index 000000000..2b5340f9d --- /dev/null +++ b/changelog.d/uk-shared-graph-contracts-weight-update.added.md @@ -0,0 +1 @@ +Added `WeightUpdate`, a declared same-kind replacement of an entity's weight values, with `weight_update_receipt` binding the ordered entity axis the replacement values are positional against (graph amendment 25). diff --git a/docs/graph-acceptance.md b/docs/graph-acceptance.md index 047c83126..25944ba00 100644 --- a/docs/graph-acceptance.md +++ b/docs/graph-acceptance.md @@ -526,6 +526,46 @@ lock unchanged: lock is unchanged. Adopted 2026-09-18 for the retention seal's verifier, which reads the live population and seals its content (#950, #951). +25. **A weight update that keeps its kind is declarable.** + `WeightTransition` only ever moves a kind forward, so a stage that + recomputes weights it already holds — a sampling normalization, a + re-solve of an existing calibration — could not be declared at all, + and the only way to express it was to misdeclare a transition. + `WeightUpdate(entity, kind, reason, mass)` is that declaration and is + deliberately narrower than a transition: the incumbent kind, the + declared kind and the returned weights' kind must all be the same one; + `mass` is `conserve` or `declared` (`WEIGHT_UPDATE_MASS_POLICIES`), + because an update that neither changes kind nor bounds mass records + nothing a reader could check it against; and `reason` is required, + non-empty and normative. + + Positional replacement values are not self-describing: the same vector + is correct against one row order and silently wrong against another. + A kernel therefore binds its ordered entity axis with + `microcosm.graph.weight_update.weight_update_receipt` under + `receipt['weight_update']`, and the executor recomputes that binding + from the incumbent axis it is about to apply the values to. This is + checked on replay by construction rather than by a parallel rule: a + cache hit reconstructs the `KernelResult` with its restored weights + and receipt and re-applies the REWEIGHT to the current base, so it + re-enters the same function. A count mismatch, a missing binding and a + binding against a different axis are each a rejection. + + `WeightUpdate.to_kind` is a property, not a field, so the two + declarations have disjoint field sets (`{entity, to_kind, mass}` and + `{entity, kind, reason, mass}`) and a `WeightUpdate` can never + canonicalize, or serialize, to the same bytes as a + `WeightTransition`. Declaration JSON discriminates on those names, and + a transition's payload is byte-for-byte what it was before this + amendment, so every declaration written earlier restores unchanged. + Existing `to_kind` readers — the design-weight cap and the calibration + view — keep working through the property; the view drops the arrow + that would claim a kind moved. `Node` gains no field, so no existing + node key moves. `decl.py` is re-locked. Raised by the source review of + the UK full-build graph (#901, head `051fb972`), whose + `uk.full.normalize` node is the first consumer; its UK graph stages + and calibration science stay in that branch. + Adding a normative field with a default changes the canonical projection of every node that carries it, so node keys moved with amendments 11 and 13's sibling field `entrants`; no released artifact pins a graph key yet. diff --git a/docs/graph-interface.lock b/docs/graph-interface.lock index b9b01741c..8940b370c 100644 --- a/docs/graph-interface.lock +++ b/docs/graph-interface.lock @@ -1,2 +1,2 @@ -ed0a859adcae12510d5ba74d51c694617201f7b448b108a3f602410f5da44876 decl.py +e78c7359b48813c57eff6f697359682753b74f86f565f4e6974421e5a4048933 decl.py dbf57c137330f0f12744c557ee594586a1b308b6d6adaba1938e2b6efded21ca kernel.py diff --git a/packages/microcosm-graph/src/microcosm/graph/__init__.py b/packages/microcosm-graph/src/microcosm/graph/__init__.py index fdc4d6be5..37227aed5 100644 --- a/packages/microcosm-graph/src/microcosm/graph/__init__.py +++ b/packages/microcosm-graph/src/microcosm/graph/__init__.py @@ -16,6 +16,7 @@ PARTITION_DTYPES, ROWS_ALL, WEIGHT_KINDS, + WEIGHT_UPDATE_MASS_POLICIES, ArtifactInput, ArtifactOutput, ArtifactType, @@ -30,6 +31,7 @@ SourceRef, StructuralDelta, WeightTransition, + WeightUpdate, compile_graph, ) from .errors import ( @@ -57,6 +59,7 @@ ) from .keys import platform_fingerprint from .randomness import keyed_uniform +from .weight_update import WEIGHT_UPDATE_AXIS_SCHEMA, weight_update_receipt __all__ = [ "platform_fingerprint", @@ -117,6 +120,9 @@ "StoreUnavailable", "StructuralDelta", "WeightTransition", + "WeightUpdate", + "WEIGHT_UPDATE_AXIS_SCHEMA", + "WEIGHT_UPDATE_MASS_POLICIES", "compile_graph", "describe", "explain_html", @@ -127,6 +133,7 @@ "load_source_bytes", "run_graph", "source_hash", + "weight_update_receipt", ] _FRAME_SERIES = "0.1" diff --git a/packages/microcosm-graph/src/microcosm/graph/decl.py b/packages/microcosm-graph/src/microcosm/graph/decl.py index 16b2ba0e2..f866be049 100644 --- a/packages/microcosm-graph/src/microcosm/graph/decl.py +++ b/packages/microcosm-graph/src/microcosm/graph/decl.py @@ -59,6 +59,7 @@ "PARTITION_DTYPES", "ROWS_ALL", "WEIGHT_KINDS", + "WEIGHT_UPDATE_MASS_POLICIES", "CompiledGraph", "Graph", "GraphError", @@ -70,6 +71,7 @@ "SourceRef", "StructuralDelta", "WeightTransition", + "WeightUpdate", "compile_graph", ] @@ -97,6 +99,10 @@ #: Mass policies a weight transition or structural node may declare. MASS_POLICIES = frozenset({"conserve", "free", "declared"}) +#: Mass policies a same-kind :class:`WeightUpdate` may declare. ``free`` is +#: deliberately absent (amendment 25). +WEIGHT_UPDATE_MASS_POLICIES = frozenset({"conserve", "declared"}) + #: The dtypes a mass-partition column may have. PARTITION_DTYPES = frozenset({"int32", "int64", "string"}) @@ -348,6 +354,71 @@ def __post_init__(self) -> None: ) +@dataclass(frozen=True) +class WeightUpdate: + """A declared numerical replacement of weights that keeps their kind. + + :class:`WeightTransition` only ever moves a weight *kind* forward, so a + stage that recomputes the numbers of weights it already holds — a + sampling normalization, a re-solve of an existing calibration — cannot + be declared at all. This is that declaration, and it is deliberately + narrower than a transition (amendment 25): + + - The kind does not move. The executor checks the incumbent kind, the + declared kind and the returned weights' kind are the same one. + - Mass is ``conserve`` or ``declared``; ``free`` is not offered, + because an update that may move mass arbitrarily and does not + change kind records nothing a reader could check it against. + - ``reason`` is required, non-empty, and normative: it enters the node + key, so a node that replaces weights for a different stated purpose + is a different node. + - The kernel must bind the ordered entity axis its replacement values + are positional against, through + :func:`~microcosm.graph.weight_update.weight_update_receipt`. The + executor recomputes that binding from the incumbent axis, on cold + execution and on replay. + + Design-weight ancestry is untouched: the executor carries the original + design anchors exactly as it does for any other node. + + Attributes: + entity: The entity whose explicit weights are replaced. + kind: The unchanged weight kind; one of :data:`WEIGHT_KINDS`. + reason: Why the numbers are replaced. Normative, non-empty. + mass: ``conserve`` or ``declared``. + """ + + entity: str + kind: str + reason: str + mass: str = "declared" + + def __post_init__(self) -> None: + _name("WeightUpdate.entity", self.entity) + _nonempty("WeightUpdate.reason", self.reason) + if self.kind not in WEIGHT_KINDS: + raise GraphError( + f"WeightUpdate.kind {self.kind!r} is not one of {WEIGHT_KINDS}." + ) + if self.mass not in WEIGHT_UPDATE_MASS_POLICIES: + raise GraphError( + f"WeightUpdate.mass {self.mass!r} is not one of " + f"{sorted(WEIGHT_UPDATE_MASS_POLICIES)}; an update that keeps " + "its kind does not get unconstrained free mass." + ) + + @property + def to_kind(self) -> str: + """The unchanged kind, for shared structural-weight accounting. + + A property, not a field, so it stays out of the normative + projection and a :class:`WeightUpdate` can never canonicalize to + the same bytes as a :class:`WeightTransition`. + """ + + return self.kind + + @dataclass(frozen=True) class Node: """One unit of computation and cell ownership. @@ -376,7 +447,9 @@ class Node: base: For a structural node other than ``CREATE``: the population version it transforms. sources: Names of :class:`SourceRef` entries this node reads. - weights: A declared weight-kind transition, if any. + weights: A declared :class:`WeightTransition` (the kind moves + forward) or :class:`WeightUpdate` (the numbers are replaced + and the kind does not move), if any. mass: Mass policy for structural nodes that change rows or weights. entrants: ``EXPAND`` nodes only: the kernel may add rows that copy no base row. Such a row has null lineage, the kernel supplies @@ -396,7 +469,7 @@ class Node: structural: StructuralDelta = StructuralDelta.NONE base: str | None = None sources: tuple[str, ...] = () - weights: WeightTransition | None = None + weights: WeightTransition | WeightUpdate | None = None mass: str = "conserve" description: str = "" citation: str = "" @@ -474,6 +547,13 @@ def __post_init__(self) -> None: f"Node {self.id!r}: entrants add mass, so an entrant-admitting " "node cannot declare mass='conserve'." ) + if self.weights is not None and not isinstance( + self.weights, WeightTransition | WeightUpdate + ): + raise GraphError( + f"Node {self.id!r}: weights must be a WeightTransition or a " + "WeightUpdate, not a look-alike." + ) if self.weights is not None and self.structural is not StructuralDelta.REWEIGHT: raise GraphError( f"Node {self.id!r}: a weight transition changes the population " @@ -482,7 +562,8 @@ def __post_init__(self) -> None: if self.structural is StructuralDelta.REWEIGHT: if self.weights is None: raise GraphError( - f"Node {self.id!r}: a REWEIGHT node declares its WeightTransition." + f"Node {self.id!r}: a REWEIGHT node declares its " + "WeightTransition or WeightUpdate." ) if self.mass != self.weights.mass: raise GraphError( diff --git a/packages/microcosm-graph/src/microcosm/graph/explain.py b/packages/microcosm-graph/src/microcosm/graph/explain.py index d17d8be94..5e1d51936 100644 --- a/packages/microcosm-graph/src/microcosm/graph/explain.py +++ b/packages/microcosm-graph/src/microcosm/graph/explain.py @@ -13,7 +13,7 @@ from typing import TYPE_CHECKING from .availability import execution_state -from .decl import GATE_OUTCOMES, CompiledGraph, StructuralDelta +from .decl import GATE_OUTCOMES, CompiledGraph, StructuralDelta, WeightUpdate from .manifest import NodeReceipt, RunManifest from .population import mass_record_receipt from .view import describe @@ -1196,12 +1196,22 @@ def _render_calibration(compiled: CompiledGraph, manifest: RunManifest) -> str: mass = _mass_payload(manifest, node, receipt) transition = node.weights assert transition is not None + # A same-kind update does not move the kind, so it does not get the + # arrow that says it did (amendment 25). + kind_label = ( + f"{_escape(transition.entity)} → {_escape(transition.to_kind)}" + if not isinstance(transition, WeightUpdate) + else ( + f"{_escape(transition.entity)} · {_escape(transition.kind)} " + f"updated ({_escape(transition.reason)})" + ) + ) cards.append( '
' '
' f"

{_escape(node.id)}

" f'

{_escape(node.kernel)} · ' - f"{_escape(transition.entity)} → {_escape(transition.to_kind)}

" + f"{kind_label}

" f'{_escape(transition.mass)} mass' "

Declared targets and results

" + target_table diff --git a/packages/microcosm-graph/src/microcosm/graph/population.py b/packages/microcosm-graph/src/microcosm/graph/population.py index bdeef2b50..a23266b2a 100644 --- a/packages/microcosm-graph/src/microcosm/graph/population.py +++ b/packages/microcosm-graph/src/microcosm/graph/population.py @@ -24,9 +24,11 @@ Owned, Ownership, StructuralDelta, + WeightUpdate, ) from .kernel import KernelResult from .store import _encode_object_scalar +from .weight_update import weight_update_receipt __all__ = [ "MassRecord", @@ -1140,7 +1142,11 @@ def patch( node, transitioning=node.weights.entity, ) - frame = _apply_weight_transition(population, frame, node, result) + frame = ( + _apply_weight_update(population, frame, node, result) + if isinstance(node.weights, WeightUpdate) + else _apply_weight_transition(population, frame, node, result) + ) elif result.weights is not None: raise PopulationError( f"Node {node.id!r} returned weights without declaring a transition." @@ -1971,6 +1977,67 @@ def _apply_weight_transition( return _replace_weights(frame, transition.entity, result.weights) +def _apply_weight_update( + population: Population, frame: Frame, node: Node, result: KernelResult +) -> Frame: + """Replace an entity's weight values without moving their kind. + + The kind is checked three ways — incumbent, declaration and returned + weights must all be the same one — and the kernel's ordered-axis + binding is recomputed against the incumbent axis these values are + about to be applied to. A cached hit re-enters this function with the + restored weights and receipt, so replay is checked by the same code + rather than a parallel rule (amendment 25). + """ + + update = node.weights + assert isinstance(update, WeightUpdate) + if result.weights is None: + raise PopulationError( + f"Node {node.id!r} declares a weight update but returned no weights." + ) + if update.entity not in population.frame.weighted_entities: + raise PopulationError( + f"Node {node.id!r} cannot update inherited weights for " + f"{update.entity!r}; explicit weights are required." + ) + old = population.frame.weights_for(update.entity) + declared_kind = WeightKind(update.kind) + if old.kind is not declared_kind: + raise PopulationError( + f"Node {node.id!r} declares a same-kind weight update to " + f"{update.kind!r}, but the incumbent weights are " + f"{old.kind.value!r}; a change of kind is a WeightTransition." + ) + if result.weights.kind is not declared_kind: + raise PopulationError( + f"Node {node.id!r} declared {update.kind!r} weights but the kernel " + f"returned {result.weights.kind.value!r}." + ) + + id_column = population.frame.schema.entity_id_column(update.entity) + axis = population.frame.table(update.entity)[id_column].tolist() + if len(result.weights.values) != len(axis): + raise PopulationError( + f"Node {node.id!r} returned {len(result.weights.values)} weights for " + f"an incumbent {update.entity!r} axis of {len(axis)} rows." + ) + declared_axis = result.receipt.get("weight_update") + if not isinstance(declared_axis, Mapping): + raise PopulationError( + f"Node {node.id!r} weight update requires receipt['weight_update']; " + "positional weights without their ordered axis are unverifiable." + ) + expected = weight_update_receipt(axis) + if dict(declared_axis) != expected: + raise PopulationError( + f"Node {node.id!r} bound its replacement weights to a different " + f"{update.entity!r} axis than the incumbent population carries." + ) + + return _replace_weights(frame, update.entity, result.weights) + + def _replace_weights(frame: Frame, entity: str, replacement: Weights) -> Frame: weights = { weighted: frame.weights_for(weighted) for weighted in frame.weighted_entities diff --git a/packages/microcosm-graph/src/microcosm/graph/serialize.py b/packages/microcosm-graph/src/microcosm/graph/serialize.py index e513c3b7a..b9b06a40b 100644 --- a/packages/microcosm-graph/src/microcosm/graph/serialize.py +++ b/packages/microcosm-graph/src/microcosm/graph/serialize.py @@ -19,6 +19,7 @@ SourceRef, StructuralDelta, WeightTransition, + WeightUpdate, ) __all__ = ["graph_from_json", "graph_to_json"] @@ -149,15 +150,7 @@ def _node_payload(node: Node) -> dict[str, object]: "structural": node.structural.value, "base": node.base, "sources": list(node.sources), - "weights": ( - None - if node.weights is None - else { - "entity": node.weights.entity, - "to_kind": node.weights.to_kind, - "mass": node.weights.mass, - } - ), + "weights": _weights_payload(node.weights), "mass": node.mass, **({"entrants": True} if node.entrants else {}), "description": node.description, @@ -288,10 +281,48 @@ def _owned_from_payload(value: object, label: str) -> Owned: ) -def _weights_from_payload(value: object, label: str) -> WeightTransition | None: +def _weights_payload( + weights: WeightTransition | WeightUpdate | None, +) -> dict[str, object] | None: + """Discriminate the two weight declarations by their own field names. + + ``WeightUpdate`` exposes ``to_kind`` as a property, so projecting it + the way a transition is projected would round-trip it back as a + transition and silently change what the node means. The transition + payload is byte-for-byte what it was before amendment 25, so every + declaration serialized before it restores unchanged. + """ + + if weights is None: + return None + if isinstance(weights, WeightUpdate): + return { + "entity": weights.entity, + "kind": weights.kind, + "reason": weights.reason, + "mass": weights.mass, + } + return { + "entity": weights.entity, + "to_kind": weights.to_kind, + "mass": weights.mass, + } + + +def _weights_from_payload( + value: object, label: str +) -> WeightTransition | WeightUpdate | None: if value is None: return None payload = _mapping(value, label) + if "kind" in payload: + _exact_fields(payload, {"entity", "kind", "reason", "mass"}, label) + return WeightUpdate( + entity=_string(payload["entity"], f"{label}.entity"), + kind=_string(payload["kind"], f"{label}.kind"), + reason=_string(payload["reason"], f"{label}.reason"), + mass=_string(payload["mass"], f"{label}.mass"), + ) _exact_fields(payload, {"entity", "to_kind", "mass"}, label) return WeightTransition( entity=_string(payload["entity"], f"{label}.entity"), diff --git a/packages/microcosm-graph/src/microcosm/graph/weight_update.py b/packages/microcosm-graph/src/microcosm/graph/weight_update.py new file mode 100644 index 000000000..69c1c5699 --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/weight_update.py @@ -0,0 +1,69 @@ +"""Ordered entity-axis evidence for a declared same-kind weight update. + +A :class:`~microcosm.graph.decl.WeightUpdate` kernel returns positional +weight values. Positions alone are not evidence: the same vector is correct +against one row order and silently wrong against another, and nothing in a +replayed receipt would show the difference. A kernel therefore binds the +ordered entity axis its values were computed against, and the executor +recomputes that binding from the incumbent axis it is about to apply them +to — on cold execution and on every replay of the cached receipt +(amendment 25). + +The binding is a digest, not the ids: an axis of millions of rows does not +belong in a manifest, and the executor only ever needs to answer whether +the axis it holds is the axis the kernel used. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from numbers import Integral + +from .canonical import canonical_json, sha256_domain + +__all__ = ["WEIGHT_UPDATE_AXIS_SCHEMA", "weight_update_receipt"] + +#: The tagged schema of the axis binding. The executor authors the same +#: token when it recomputes, so a receipt written under another schema is a +#: mismatch rather than an unchecked pass. +WEIGHT_UPDATE_AXIS_SCHEMA = "microcosm.graph.weight-update-axis.v1" + + +def weight_update_receipt(entity_ids: Sequence[int | str]) -> dict[str, object]: + """Bind positional replacement weights to a unique, ordered entity axis. + + Place the result under ``KernelResult.receipt['weight_update']``. + + Args: + entity_ids: The entity ids, in the order the returned weight values + are positional against. Integers or non-empty strings, unique. + ``bool`` is refused: it is an ``Integral`` in Python, and an + axis of ``True``/``False`` is a mistake, not an id space. + + Returns: + The tagged binding: its schema, the row count, and a domain- + separated SHA-256 over the canonical JSON of the ordered ids. + + Raises: + ValueError: An id is neither an integer nor a non-empty string, or + the ids repeat. + """ + + ids: list[int | str] = [] + for value in entity_ids: + if isinstance(value, Integral) and not isinstance(value, bool): + ids.append(int(value)) + elif isinstance(value, str) and value: + ids.append(value) + else: + raise ValueError( + "Weight update ids must be integers or non-empty strings; got " + f"{value!r}." + ) + if len(set(ids)) != len(ids): + raise ValueError("Weight update ids must be unique.") + return { + "schema": WEIGHT_UPDATE_AXIS_SCHEMA, + "count": len(ids), + "entity_ids_sha256": sha256_domain("weight-update-axis", canonical_json(ids)), + } diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_weight_update.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_weight_update.py new file mode 100644 index 000000000..5fb48b2b8 --- /dev/null +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_weight_update.py @@ -0,0 +1,352 @@ +"""Amendment 25: a same-kind weight update is declarable and axis-bound. + +``WeightTransition`` only moves a kind forward, so a stage that recomputes +weights it already holds — a sampling normalization, a re-solve of an +existing calibration — had no declaration at all. These properties are +about the one that does: the kind cannot move, mass cannot be free, and +positional replacement values are refused unless the kernel binds the +ordered entity axis they were computed against. + +Everything here runs real shared graph operations over the invented toy +country. No country model, engine, or build artifact is involved. +""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +from microcosm.frame import WeightKind, Weights +from microcosm.graph import ( + WEIGHT_UPDATE_AXIS_SCHEMA, + WEIGHT_UPDATE_MASS_POLICIES, + ContentStore, + Graph, + GraphError, + KernelResult, + Node, + NodeRejectedError, + Slice, + StructuralDelta, + WeightTransition, + WeightUpdate, + graph_from_json, + graph_to_json, + weight_update_receipt, +) +from microcosm.graph.canonical import normative +from test_support.paths import paths_for + +_TEST_PATHS = paths_for("microcosm-graph") + +if "_toy" not in sys.modules: + _SPEC = importlib.util.spec_from_file_location( + "_toy", _TEST_PATHS.tests / "_toy.py" + ) + sys.modules["_toy"] = importlib.util.module_from_spec(_SPEC) + _SPEC.loader.exec_module(sys.modules["_toy"]) +toy = sys.modules["_toy"] + +UPDATE_REF = "reweight.same_kind@1" + + +class ScaleSameKind(toy.ToyKernel): + """A REWEIGHT that replaces weight values and keeps their kind. + + ``axis`` selects which ordered axis the kernel claims its positional + values belong to: the incumbent one, the incumbent one reversed, or + none at all. ``returns_kind`` lets a property separate "the incumbent + disagrees with the declaration" from "the kernel disagrees with it". + """ + + def compute(self, context): + entity = str(context.params["entity"]) + incumbent = context.weights[entity] + after = incumbent.values * float(context.params["factor"]) + returns = context.params.get("returns_kind") + kind = incumbent.kind if returns is None else WeightKind(str(returns)) + ids = context.tables[entity][toy.id_column(entity)].tolist() + receipt: dict[str, object] = { + "mass": toy._mass_record( + context, incumbent.values, after, str(context.params["policy"]) + ) + } + axis = str(context.params.get("axis", "incumbent")) + if axis == "incumbent": + receipt["weight_update"] = weight_update_receipt(ids) + elif axis == "reversed": + receipt["weight_update"] = weight_update_receipt(list(reversed(ids))) + elif axis == "short": + receipt["weight_update"] = weight_update_receipt(ids[:-1]) + elif axis != "absent": # pragma: no cover - guards the fixture itself + raise AssertionError(f"unknown axis fixture {axis!r}") + return KernelResult(weights=Weights(values=after, kind=kind), receipt=receipt) + + +def registry_with_update(): + """The toy registry plus the same-kind update kernel.""" + registry = toy.toy_registry() + registry.register(ScaleSameKind(UPDATE_REF, toy._REWEIGHT)) + return registry + + +def update_node( + *, + kind: str = "design", + reason: str = "reinstall normalized source mass", + mass: str = "declared", + factor: float = 2.0, + axis: str = "incumbent", + returns_kind: str | None = None, +) -> Node: + """One same-kind update of the toy household weights, on ``survey``.""" + return Node( + "update", + UPDATE_REF, + structural=StructuralDelta.REWEIGHT, + base="survey", + inputs=(Slice("person", ("age",)), Slice("household", ("household_size",))), + params={ + "entity": "household", + "factor": factor, + "policy": mass, + "axis": axis, + "returns_kind": returns_kind, + }, + weights=WeightUpdate("household", kind, reason, mass=mass), + mass=mass, + description="same-kind weight update", + ) + + +def update_graph(**kwargs) -> Graph: + return Graph("toy", (toy.SOURCE,), (toy.CREATE, update_node(**kwargs))) + + +def run_update(root: Path, **kwargs) -> object: + return toy.run_toy(update_graph(**kwargs), root, registry=registry_with_update()) + + +# ---------------------------------------------------------------------- +# The declaration +# ---------------------------------------------------------------------- + + +def test_weight_update_declares_a_non_empty_reason() -> None: + """The stated purpose is required, and it is normative.""" + with pytest.raises(GraphError, match="WeightUpdate.reason"): + WeightUpdate("household", "design", "") + one = WeightUpdate("household", "design", "normalize sampled mass") + other = WeightUpdate("household", "design", "re-solve the calibration") + assert normative(one) != normative(other) + + +def test_weight_update_refuses_free_mass() -> None: + """An update that neither moves kind nor bounds mass records nothing.""" + assert WEIGHT_UPDATE_MASS_POLICIES == frozenset({"conserve", "declared"}) + for mass in sorted(WEIGHT_UPDATE_MASS_POLICIES): + assert WeightUpdate("household", "design", "why", mass=mass).mass == mass + with pytest.raises(GraphError, match="free"): + WeightUpdate("household", "design", "why", mass="free") + + +def test_weight_update_refuses_an_unknown_kind() -> None: + with pytest.raises(GraphError, match="WeightUpdate.kind"): + WeightUpdate("household", "provisional", "why") + + +def test_weight_update_is_never_a_transition() -> None: + """Disjoint field sets, so the two can never canonicalize alike. + + ``to_kind`` is a property on the update, which is what keeps it out of + the normative projection. + """ + update = WeightUpdate("household", "calibrated", "re-solve", mass="declared") + transition = WeightTransition("household", "calibrated", mass="declared") + assert update.to_kind == "calibrated" + assert set(normative(update)) == {"entity", "kind", "reason", "mass"} + assert set(normative(transition)) == {"entity", "to_kind", "mass"} + assert normative(update) != normative(transition) + + +def test_reweight_node_accepts_either_declaration() -> None: + """A REWEIGHT node still needs one of them, and rejects a look-alike.""" + with pytest.raises(GraphError, match="REWEIGHT node declares"): + Node("n", UPDATE_REF, structural=StructuralDelta.REWEIGHT, base="survey") + with pytest.raises(GraphError, match="look-alike"): + Node( + "n", + UPDATE_REF, + structural=StructuralDelta.REWEIGHT, + base="survey", + weights={"entity": "household", "kind": "design", "reason": "no"}, + mass="declared", + ) + + +def test_node_mass_must_agree_with_the_update() -> None: + with pytest.raises(GraphError, match="disagrees"): + Node( + "n", + UPDATE_REF, + structural=StructuralDelta.REWEIGHT, + base="survey", + weights=WeightUpdate("household", "design", "why", mass="declared"), + mass="conserve", + ) + + +# ---------------------------------------------------------------------- +# JSON round trips +# ---------------------------------------------------------------------- + + +def test_weight_update_round_trips_as_itself() -> None: + """Not as a transition, which its ``to_kind`` property would allow.""" + graph = update_graph() + restored = graph_from_json(graph_to_json(graph)) + assert restored == graph + weights = restored.node("update").weights + assert isinstance(weights, WeightUpdate) + assert (weights.entity, weights.kind, weights.mass) == ( + "household", + "design", + "declared", + ) + assert weights.reason == "reinstall normalized source mass" + + +def test_transition_payload_is_unchanged_by_the_amendment() -> None: + """Every declaration serialized before amendment 25 restores unchanged.""" + graph = Graph("toy", (toy.SOURCE,), (toy.CREATE, toy.POOL)) + text = graph_to_json(graph) + assert '"weights":{"entity":"household","mass":"free","to_kind":"importance"}' in ( + text + ) + restored = graph_from_json(text) + assert restored == graph + assert isinstance(restored.node("pool").weights, WeightTransition) + + +def test_round_trip_refuses_a_mixed_weights_payload() -> None: + """``kind`` selects the update arm; its fields are then exact.""" + text = graph_to_json(update_graph()) + mixed = text.replace(',"reason":"reinstall normalized source mass"', "") + with pytest.raises(ValueError, match="weights"): + graph_from_json(mixed) + both = text.replace('"kind":"design"', '"kind":"design","to_kind":"design"') + with pytest.raises(ValueError, match="weights"): + graph_from_json(both) + + +def test_axis_receipt_is_a_tagged_digest() -> None: + """The binding answers one question and does not carry the axis.""" + receipt = weight_update_receipt([3, 1, 2]) + assert set(receipt) == {"schema", "count", "entity_ids_sha256"} + assert receipt["schema"] == WEIGHT_UPDATE_AXIS_SCHEMA + assert receipt["count"] == 3 + assert receipt != weight_update_receipt([1, 2, 3]) + assert receipt == weight_update_receipt([3, 1, 2]) + with pytest.raises(ValueError, match="unique"): + weight_update_receipt([1, 1]) + with pytest.raises(ValueError, match="integers or non-empty strings"): + weight_update_receipt([True]) + with pytest.raises(ValueError, match="integers or non-empty strings"): + weight_update_receipt([""]) + + +# ---------------------------------------------------------------------- +# The shared graph operation +# ---------------------------------------------------------------------- + + +def test_same_kind_update_replaces_values_and_keeps_the_kind(tmp_path: Path) -> None: + """The real property: new numbers, same kind, mass declared.""" + run = run_update(tmp_path / "run") + before = run.manifest.population("survey").weights_for("household") + after = run.manifest.population("update").weights_for("household") + assert after.kind is WeightKind.DESIGN is before.kind + assert list(after.values) == [value * 2.0 for value in before.values] + + +def test_conserving_update_is_accepted(tmp_path: Path) -> None: + run = run_update(tmp_path / "run", mass="conserve", factor=1.0) + weights = run.manifest.population("update").weights_for("household") + assert weights.kind is WeightKind.DESIGN + + +def test_update_refuses_a_kind_that_moved(tmp_path: Path) -> None: + """Declaring a different kind than the incumbent is a transition.""" + with pytest.raises(NodeRejectedError, match="a change of kind is a"): + run_update(tmp_path / "run", kind="importance", returns_kind="importance") + + +def test_update_refuses_weights_of_another_kind(tmp_path: Path) -> None: + """The kernel must also return the kind the node declared.""" + with pytest.raises(NodeRejectedError, match="the kernel returned"): + run_update(tmp_path / "run", returns_kind="importance") + + +def test_update_requires_its_ordered_axis(tmp_path: Path) -> None: + with pytest.raises(NodeRejectedError, match="unverifiable"): + run_update(tmp_path / "run", axis="absent") + + +def test_update_refuses_a_foreign_axis(tmp_path: Path) -> None: + """Same ids, other order: the values would land on the wrong rows.""" + with pytest.raises(NodeRejectedError, match="different .household. axis"): + run_update(tmp_path / "run", axis="reversed") + + +def test_update_refuses_a_short_axis(tmp_path: Path) -> None: + with pytest.raises(NodeRejectedError, match="different .household. axis"): + run_update(tmp_path / "run", axis="short") + + +# ---------------------------------------------------------------------- +# Replay +# ---------------------------------------------------------------------- + + +def test_cold_then_required_replay_revalidates_the_axis(tmp_path: Path) -> None: + """A hit re-applies the REWEIGHT, so the axis is checked again. + + The cached result is reconstructed with its stored weights and receipt + and passed back through the same application, which is why replay + needs no parallel rule. ``resume="require"`` proves the second run read + the store rather than recomputing. + """ + cold = run_update(tmp_path / "run") + assert cold.misses() == set(cold.compiled.order) + + registry = registry_with_update() + warm = toy.run_toy( + update_graph(), + tmp_path / "run", + sources=cold.sources, + registry=registry, + store=ContentStore(tmp_path / "run" / "store"), + resume="require", + ) + assert warm.misses() == set() + assert toy.total_calls(registry) == 0 + assert warm.keys() == cold.keys() + replayed = warm.manifest.population("update").weights_for("household") + original = cold.manifest.population("update").weights_for("household") + assert replayed.kind is original.kind + assert list(replayed.values) == list(original.values) + + +def test_reason_is_part_of_the_node_identity(tmp_path: Path) -> None: + """Two updates that state different purposes are different nodes.""" + first = run_update(tmp_path / "first") + second = toy.run_toy( + update_graph(reason="re-solve the calibration"), + tmp_path / "second", + registry=registry_with_update(), + ) + assert first.keys()["survey"] == second.keys()["survey"] + assert first.keys()["update"] != second.keys()["update"] From 1d4bbac7a4cebf8d788128aae6151c60fc7a5c0a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 08:47:48 -0400 Subject: [PATCH 03/44] Carry the version's metadata, mass log and column order (graph amendment 26) The executor projects each entity table in declaration order, so KernelContext.tables is not the population version's layout, and the version's Frame metadata and mass log were not reachable from a kernel at all. A kernel that has to hand its declared projection back to a legacy function as a Frame could not rebuild one without inventing the parts it could not see. KernelContext gains three read-only fields: - frame_metadata: the version's metadata. The executor passes Frame.metadata, which Frame has already deeply frozen; this class adds a read-only view and does not itself deep-freeze a mapping built some other way. This deliberately differs from the UK branch, which imports microcosm.frame.bundle._freeze_metadata into the frozen interface -- a frozen contract should not depend on another shard's private name. - frame_mass_log: the *incoming* Frame mass records. The write side already existed (receipt['frame_mass_log_append']); only the read side was missing. - frame_column_order: the version's own column order, restricted to the projected columns. An entry that is not exactly an ordering of that table's columns is refused, so an order can neither hide a column the node was given nor name one it was not -- a column name is itself information about the version. The three ride after artifacts and before tolerances, so amendment 17's statement that numerics rides at the end stays literally true, and amendment 19's that artifacts rides before the pair does too; the unit assertion of adjacency becomes the ordering amendment 19 actually claimed. Nothing here is normative: Node is untouched and no node key moves. kernel.py is re-locked; decl.py is untouched. The acceptance suite's B2 field set follows in its own commit, as amendment 19's did (a2b6dfb0b). Runtime UNTESTED in this lane: source-only ast/ruff and CI test inventory checks only. Co-Authored-By: Claude Opus 5 (cherry picked from commit f33d47cdac3d34a9df491703a1307bbd33d94b72) --- ...red-graph-contracts-frame-context.added.md | 1 + docs/graph-acceptance.md | 43 ++ docs/graph-interface.lock | 2 +- .../src/microcosm/graph/executor.py | 13 + .../src/microcosm/graph/kernel.py | 54 ++- .../shared/test_graph_frame_context.py | 438 ++++++++++++++++++ .../shared/test_graph_kernel_contract.py | 4 +- 7 files changed, 552 insertions(+), 3 deletions(-) create mode 100644 changelog.d/uk-shared-graph-contracts-frame-context.added.md create mode 100644 packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py diff --git a/changelog.d/uk-shared-graph-contracts-frame-context.added.md b/changelog.d/uk-shared-graph-contracts-frame-context.added.md new file mode 100644 index 000000000..64fe69ac1 --- /dev/null +++ b/changelog.d/uk-shared-graph-contracts-frame-context.added.md @@ -0,0 +1 @@ +Added `KernelContext.frame_metadata`, `frame_mass_log` and `frame_column_order`, so a kernel can reconstruct its population version's declared slices without seeing any column it did not declare (graph amendment 26). diff --git a/docs/graph-acceptance.md b/docs/graph-acceptance.md index 25944ba00..3ee9d651d 100644 --- a/docs/graph-acceptance.md +++ b/docs/graph-acceptance.md @@ -566,6 +566,49 @@ lock unchanged: `uk.full.normalize` node is the first consumer; its UK graph stages and calibration science stay in that branch. +26. **The context carries the version's metadata, mass log and column + order.** The executor projects each entity table in *declaration* + order, so `KernelContext.tables` is not the population version's + layout, and the version's `Frame` metadata and mass log were not + reachable from a kernel at all. A kernel that has to hand a declared + projection back to a legacy function as a `Frame` therefore could not + reconstruct one without inventing the parts it could not see. + `kernel.py` gains three read-only fields: + + - `frame_metadata` — the version's own metadata. The executor passes + `Frame.metadata`, which `Frame` has already deeply frozen; + `KernelContext` adds a read-only view over it and does **not** itself + deep-freeze a mapping built some other way. (This is a deliberate + difference from the UK branch, which imports + `microcosm.frame.bundle._freeze_metadata` into the frozen interface: + a frozen contract should not depend on another shard's private name.) + - `frame_mass_log` — the version's `Frame` mass records, in order, and + specifically the *incoming* ones. A node needing a stage's completed + records must run after that stage's structural boundary or read its + predecessor's evidence; incidental node order is not authority. The + write side already existed (`receipt['frame_mass_log_append']`); only + the read side was missing. + - `frame_column_order` — entity to the version's own column order, + restricted to the columns projected into `tables`. An entry that is + not exactly an ordering of that table's columns is refused, so an + order can neither hide a column the node was given nor name one it + was not: a column *name* is itself information about the version, and + B1's "nothing else is visible" covers names as well as values. + + The three ride after `artifacts` and before `tolerances`, so amendment + 17's statement that `numerics` rides at the end of the context stays + literally true and amendment 19's that `artifacts` rides before the + pair does too — the unit assertion of *adjacency* becomes the ordering + amendment 19 actually claimed. The acceptance suite's B2 field set + gains the three in its own commit, as amendment 19's did. Nothing here + is normative: `Node` is untouched, no canonical projection changes, and + no node key moves. The fields are rebuilt from a restored `Frame` on a + cache hit exactly as they are from a computed one, which is what + amendment 22's metadata-preserving Frame format makes possible. + `kernel.py` is re-locked. Raised by the same source review as amendment + 25; the UK full-build graph's `context_frame` helper (#901, head + `051fb972`) is the first consumer. + Adding a normative field with a default changes the canonical projection of every node that carries it, so node keys moved with amendments 11 and 13's sibling field `entrants`; no released artifact pins a graph key yet. diff --git a/docs/graph-interface.lock b/docs/graph-interface.lock index 8940b370c..f09fe7bb1 100644 --- a/docs/graph-interface.lock +++ b/docs/graph-interface.lock @@ -1,2 +1,2 @@ e78c7359b48813c57eff6f697359682753b74f86f565f4e6974421e5a4048933 decl.py -dbf57c137330f0f12744c557ee594586a1b308b6d6adaba1938e2b6efded21ca kernel.py +51f45e899ba578a6b2324636a9879253266b067812f68b648eacfcd5a184d245 kernel.py diff --git a/packages/microcosm-graph/src/microcosm/graph/executor.py b/packages/microcosm-graph/src/microcosm/graph/executor.py index 71eed6031..49fafc448 100644 --- a/packages/microcosm-graph/src/microcosm/graph/executor.py +++ b/packages/microcosm-graph/src/microcosm/graph/executor.py @@ -737,6 +737,16 @@ def _project_context( ) strata = frame.strata.loc[person_mask].copy() _freeze_series(strata) + # The projection above orders each table by declaration, not by the + # version's own layout, so a consumer rebuilding the version's tables + # needs that layout separately -- restricted to what it was given, so it + # never learns the name of a column it cannot read (amendment 26). + column_order: dict[str, tuple[str, ...]] = {} + for entity, table in tables.items(): + projected = set(table.columns) + column_order[entity] = tuple( + column for column in frame.table(entity).columns if column in projected + ) return KernelContext( node=node, tables=MappingProxyType(tables), @@ -746,6 +756,9 @@ def _project_context( rng=np.random.default_rng(seed(key)), sources=MappingProxyType({name: sources[name] for name in node.sources}), artifacts={} if artifacts is None else artifacts, + frame_metadata=frame.metadata, + frame_mass_log=frame.mass_log, + frame_column_order=MappingProxyType(column_order), tolerances=tolerances, numerics=numerics, ) diff --git a/packages/microcosm-graph/src/microcosm/graph/kernel.py b/packages/microcosm-graph/src/microcosm/graph/kernel.py index d6b16909d..528d11836 100644 --- a/packages/microcosm-graph/src/microcosm/graph/kernel.py +++ b/packages/microcosm-graph/src/microcosm/graph/kernel.py @@ -57,7 +57,7 @@ import numpy as np import pandas as pd -from microcosm.frame import Frame, Weights +from microcosm.frame import Frame, MassChangeRecord, Weights from .decl import ArtifactType, Node, Param, StructuralDelta @@ -348,6 +348,24 @@ class KernelContext: validates the versioned payload before using it, because a nominal type does not itself verify serialized data (amendment 19). + frame_metadata: The population version's own metadata. The + executor passes :attr:`microcosm.frame.Frame.metadata`, which + ``Frame`` has already deeply frozen; this class adds a + read-only view over that mapping and does not itself deep-freeze + a mapping built some other way (amendment 26). + frame_mass_log: The population version's ``Frame`` mass records, in + order. This is the *incoming* log: a node that needs a stage's + completed records must run after that stage's structural + boundary or read its predecessor's evidence, because incidental + node order is not authority (amendment 26). + frame_column_order: Entity to the population version's own column + order, restricted to the columns projected into ``tables``. + The executor projects ``tables`` in declaration order, so this + is the only way to reconstruct the version's layout. It never + names a column the node did not get: an entry that is not + exactly an ordering of that table's columns is refused, so an + undeclared column cannot be smuggled in as a name (amendment + 26). tolerances: ``(entity, column)`` of each declared input column to the :class:`Tolerance` its owning kernel declared, or ``None`` for a bitwise owner. A gate compares against these. @@ -366,6 +384,9 @@ class KernelContext: rng: np.random.Generator sources: Mapping[str, Path] = field(default_factory=dict) artifacts: Mapping[str, ArtifactValue] = field(default_factory=dict) + frame_metadata: Mapping[str, object] = field(default_factory=dict) + frame_mass_log: tuple[MassChangeRecord, ...] = () + frame_column_order: Mapping[str, tuple[str, ...]] = field(default_factory=dict) tolerances: Mapping[tuple[str, str], Tolerance | None] = field(default_factory=dict) numerics: Mapping[tuple[str, str], NumericScope] = field(default_factory=dict) @@ -382,6 +403,37 @@ def __post_init__(self) -> None: ) object.__setattr__(self, "artifacts", MappingProxyType(values)) + metadata = dict(self.frame_metadata) + if any(not isinstance(name, str) or not name for name in metadata): + raise TypeError( + "KernelContext.frame_metadata keys must be non-empty strings." + ) + object.__setattr__(self, "frame_metadata", MappingProxyType(metadata)) + + if not isinstance(self.frame_mass_log, tuple) or any( + not isinstance(record, MassChangeRecord) for record in self.frame_mass_log + ): + raise TypeError( + "KernelContext.frame_mass_log must be a tuple of MassChangeRecord." + ) + + # An order is an order *of the projected columns*, so it can neither + # name a column the node was not given nor hide one it was. + order = dict(self.frame_column_order) + for entity, columns in order.items(): + table = self.tables.get(entity) + if ( + table is None + or not isinstance(columns, tuple) + or len(set(columns)) != len(columns) + or set(columns) != set(table.columns) + ): + raise TypeError( + "KernelContext.frame_column_order must order exactly the " + f"projected columns of each entity; {entity!r} does not." + ) + object.__setattr__(self, "frame_column_order", MappingProxyType(order)) + @dataclass(frozen=True) class KernelResult: diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py new file mode 100644 index 000000000..d7026ca35 --- /dev/null +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py @@ -0,0 +1,438 @@ +"""Amendment 26: the context carries the version's metadata, mass log, order. + +The executor projects each table in *declaration* order, so a kernel that +reconstructs its population version's layout cannot do it from +``KernelContext.tables`` alone, and it cannot see the version's metadata or +its incoming ``Frame`` mass log at all. These properties are about the three +fields that close that gap, and about the one thing they must never do: +name a column the node was not given. + +Everything here runs real shared graph operations over the invented toy +country. No country model, engine, or build artifact is involved. +""" + +from __future__ import annotations + +import dataclasses +import importlib.util +import sys +from pathlib import Path + +import pandas as pd +import pytest + +from microcosm.frame import Frame, MassChangeRecord +from microcosm.graph import ( + ContentStore, + Graph, + KernelContext, + KernelResult, + Node, + Owned, + Slice, + compile_graph, + run_graph, +) +from test_support.paths import paths_for + +_TEST_PATHS = paths_for("microcosm-graph") + +if "_toy" not in sys.modules: + _SPEC = importlib.util.spec_from_file_location( + "_toy", _TEST_PATHS.tests / "_toy.py" + ) + sys.modules["_toy"] = importlib.util.module_from_spec(_SPEC) + _SPEC.loader.exec_module(sys.modules["_toy"]) +toy = sys.modules["_toy"] + +SOURCE_REF = "source.metadata@1" +PROBE_REF = "probe.frame_context@1" + +#: Metadata the CREATE kernel puts on the version, including a nested value +#: so the frozen projection is exercised rather than a flat string map. +VERSION_METADATA = {"time_period": "2024", "vintage": ("frs", "was")} + + +class MetadataSource(toy.ToyKernel): + """CREATE: the toy population, carrying declared version metadata.""" + + def compute(self, context: KernelContext) -> KernelResult: + frame = toy.read_toy_frame(context.sources["survey"]) + tables = {entity: frame.table(entity) for entity in frame.entities} + tables.update({link: frame.link(link) for link in frame.links}) + return KernelResult( + frame=Frame( + tables, + frame.schema, + { + entity: frame.weights_for(entity) + for entity in frame.weighted_entities + }, + frame.strata, + metadata=VERSION_METADATA, + ), + receipt={"persons": frame.n("person")}, + ) + + +class FrameContextProbe(toy.ToyKernel): + """Owns one column and records the frame view it was handed. + + With ``mass_reason`` set it also appends one ``Frame`` mass record + asserting the household total is unchanged, which is what the legacy + ``Frame`` contract calls an explicit conservation check. + """ + + def __init__(self, ref: str, capabilities, *, variant: str = "base") -> None: + super().__init__(ref, capabilities, variant=variant) + self.seen: list[dict[str, object]] = [] + + def compute(self, context: KernelContext) -> KernelResult: + self.seen.append( + { + "node": context.node.id, + "column_order": { + entity: tuple(columns) + for entity, columns in context.frame_column_order.items() + }, + "projected": { + entity: tuple(table.columns) + for entity, table in context.tables.items() + }, + "metadata": dict(context.frame_metadata), + "mass_log": tuple(context.frame_mass_log), + } + ) + ids = pd.Index(context.tables["person"]["person_id"], name="person_id") + result_columns = { + ("person", str(context.params["target"])): pd.Series( + 1.0, index=ids, dtype="float64" + ) + } + reason = context.params.get("mass_reason") + if reason is None: + return KernelResult(columns=result_columns) + total = float(context.weights["household"].values.sum()) + return KernelResult( + columns=result_columns, + receipt={ + "frame_mass_log_append": [ + { + "entity": "household", + "old_total": total, + "new_total": total, + "declared_factor": None, + "reason": str(reason), + } + ] + }, + ) + + +def build_registry() -> tuple[object, FrameContextProbe]: + """The toy registry with the metadata source and the probe registered.""" + registry = toy.toy_registry() + registry.register(MetadataSource(SOURCE_REF, toy._CREATE)) + probe = FrameContextProbe(PROBE_REF, toy._DETERMINISTIC) + registry.register(probe) + return registry, probe + + +CREATE = dataclasses.replace(toy.CREATE, kernel=SOURCE_REF) + + +def probe_node( + node_id: str, + *, + columns: tuple[str, ...], + target: str, + mass_reason: str | None = None, +) -> Node: + """A probe reading ``columns`` of the person entity, in that order. + + A probe that states a mass record also reads the household entity, + because the record is stated against the household total and a node + only receives weights for the entities it projects. + """ + inputs = (Slice("person", columns),) + if mass_reason is not None: + inputs = (*inputs, Slice("household", ("household_size",))) + return Node( + node_id, + PROBE_REF, + inputs=inputs, + outputs=(Owned("person", target, "float64"),), + params={"target": target, "mass_reason": mass_reason}, + population="survey", + ) + + +#: ``income`` before ``age`` is the reverse of the source table's own order, +#: so the projection and the version's layout genuinely disagree. +FIRST = probe_node( + "probe_first", + columns=("income", "age"), + target="probe_a", + mass_reason="households are unchanged by a derivation", +) +SECOND = probe_node("probe_second", columns=("probe_a",), target="probe_b") + + +def probe_graph(*extra: Node) -> Graph: + return Graph("toy", (toy.SOURCE,), (CREATE, FIRST, SECOND, *extra)) + + +def run_probe( + root: Path, + graph: Graph | None = None, + *, + registry=None, + probe=None, + store: ContentStore | None = None, + sources=None, + resume: str = "auto", + observer=None, +): + """Run ``graph`` and return ``(manifest, probe, sources, store)``.""" + if registry is None: + registry, probe = build_registry() + if sources is None: + sources = {"survey": toy.copy_source(root / "source")} + if store is None: + store = ContentStore(root / "store") + manifest = run_graph( + compile_graph(graph or probe_graph()), + sources=dict(sources), + store=store, + kernels=registry, + resume=resume, + _population_observer=observer, + ) + return manifest, probe, sources, store + + +def observation(probe: FrameContextProbe, node_id: str) -> dict[str, object]: + (found,) = [item for item in probe.seen if item["node"] == node_id] + return found + + +# ---------------------------------------------------------------------- +# The field contract +# ---------------------------------------------------------------------- + + +def test_the_three_fields_ride_before_the_amendment_13_17_pair() -> None: + """Amendment 17's "numerics rides at the end" stays literally true.""" + fields = [f.name for f in dataclasses.fields(KernelContext)] + assert fields[-2:] == ["tolerances", "numerics"] + assert fields[fields.index("artifacts") + 1 : fields.index("tolerances")] == [ + "frame_metadata", + "frame_mass_log", + "frame_column_order", + ] + + +def bare_context(**kwargs) -> KernelContext: + return KernelContext( + node=Node("n", PROBE_REF), + tables={"person": pd.DataFrame({"person_id": [1, 2], "age": [30, 40]})}, + weights={}, + strata=pd.Series(dtype="string"), + params={}, + rng=None, + **kwargs, + ) + + +def test_the_fields_default_to_an_empty_view() -> None: + context = bare_context() + assert dict(context.frame_metadata) == {} + assert context.frame_mass_log == () + assert dict(context.frame_column_order) == {} + + +def test_column_order_refuses_an_undeclared_column() -> None: + """The one thing an order must never do is name a column not given. + + A name is itself information about the version, so an order that + mentions an unprojected column is refused rather than trimmed. + """ + with pytest.raises(TypeError, match="exactly the projected columns"): + bare_context(frame_column_order={"person": ("person_id", "age", "income")}) + + +def test_column_order_refuses_a_partial_or_repeated_order() -> None: + with pytest.raises(TypeError, match="exactly the projected columns"): + bare_context(frame_column_order={"person": ("person_id",)}) + with pytest.raises(TypeError, match="exactly the projected columns"): + bare_context(frame_column_order={"person": ("person_id", "person_id", "age")}) + with pytest.raises(TypeError, match="exactly the projected columns"): + bare_context(frame_column_order={"person": ["person_id", "age"]}) + + +def test_column_order_refuses_an_unprojected_entity() -> None: + with pytest.raises(TypeError, match="exactly the projected columns"): + bare_context(frame_column_order={"household": ("household_id",)}) + + +def test_mass_log_refuses_anything_but_mass_records() -> None: + with pytest.raises(TypeError, match="tuple of MassChangeRecord"): + bare_context(frame_mass_log=({"entity": "household"},)) + with pytest.raises(TypeError, match="tuple of MassChangeRecord"): + bare_context( + frame_mass_log=[ + MassChangeRecord( + entity="household", + old_total=1.0, + new_total=1.0, + declared_factor=None, + reason="why", + ) + ] + ) + + +def test_metadata_is_a_read_only_view() -> None: + source = {"time_period": "2024"} + context = bare_context(frame_metadata=source) + assert dict(context.frame_metadata) == source + with pytest.raises(TypeError): + context.frame_metadata["time_period"] = "2025" # type: ignore[index] + source["time_period"] = "2025" + assert context.frame_metadata["time_period"] == "2024" + with pytest.raises(TypeError, match="non-empty strings"): + bare_context(frame_metadata={"": "no"}) + + +# ---------------------------------------------------------------------- +# What the executor actually supplies +# ---------------------------------------------------------------------- + + +def test_column_order_is_the_versions_own_order_not_the_projection( + tmp_path: Path, +) -> None: + """The real property: two different orders over the same column set.""" + _, probe, _, _ = run_probe(tmp_path / "run") + seen = observation(probe, "probe_first") + order = seen["column_order"]["person"] + projected = seen["projected"]["person"] + assert set(order) == set(projected) + assert order != projected + assert projected.index("income") < projected.index("age") + assert order.index("age") < order.index("income") + + +def test_column_order_names_no_column_the_node_did_not_read(tmp_path: Path) -> None: + """``receives_x`` exists on the version and is in neither view.""" + _, probe, _, _ = run_probe(tmp_path / "run") + for key in ("column_order", "projected"): + person = observation(probe, "probe_first")[key]["person"] + assert "receives_x" not in person + assert "is_adult" not in person + assert set(observation(probe, "probe_second")["column_order"]["person"]) == { + "person_id", + "person_household_id", + "person_release_id", + "probe_a", + } + + +def test_version_metadata_reaches_every_node_of_the_version(tmp_path: Path) -> None: + _, probe, _, _ = run_probe(tmp_path / "run") + for node_id in ("probe_first", "probe_second"): + metadata = observation(probe, node_id)["metadata"] + assert metadata["time_period"] == "2024" + assert tuple(metadata["vintage"]) == ("frs", "was") + + +def test_mass_log_is_the_incoming_log(tmp_path: Path) -> None: + """The appending node sees the log before its own record; its successor after.""" + _, probe, _, _ = run_probe(tmp_path / "run") + assert observation(probe, "probe_first")["mass_log"] == () + after = observation(probe, "probe_second")["mass_log"] + assert len(after) == 1 + assert after[0].entity == "household" + assert after[0].reason == "households are unchanged by a derivation" + assert after[0].old_total == after[0].new_total + + +# ---------------------------------------------------------------------- +# Replay +# ---------------------------------------------------------------------- + + +def test_required_replay_is_a_full_hit(tmp_path: Path) -> None: + """The three fields are additive: no key moves, everything replays.""" + cold_manifest, _, sources, store = run_probe(tmp_path / "run") + assert not any(receipt.hit for receipt in cold_manifest.nodes.values()) + + registry, probe = build_registry() + warm = run_graph( + compile_graph(probe_graph()), + sources=dict(sources), + store=ContentStore(tmp_path / "run" / "store"), + kernels=registry, + resume="require", + ) + assert all(receipt.hit for receipt in warm.nodes.values()) + assert probe.seen == [] + assert {n: r.key for n, r in warm.nodes.items()} == { + n: r.key for n, r in cold_manifest.nodes.items() + } + + +def test_a_new_node_over_restored_populations_sees_the_same_frame( + tmp_path: Path, +) -> None: + """The fields are rebuilt from a restored Frame, not only a computed one. + + The store round trip is where metadata and the ``Frame`` mass log could + quietly disappear, so the property is stated against a node that runs + cold on top of cache hits. + """ + _, cold_probe, sources, _ = run_probe(tmp_path / "run") + expected = observation(cold_probe, "probe_second") + + third = probe_node("probe_third", columns=("probe_a",), target="probe_c") + registry, probe = build_registry() + warm = run_graph( + compile_graph(probe_graph(third)), + sources=dict(sources), + store=ContentStore(tmp_path / "run" / "store"), + kernels=registry, + resume="auto", + ) + assert {n for n, r in warm.nodes.items() if not r.hit} == {"probe_third"} + seen = observation(probe, "probe_third") + assert seen["metadata"] == expected["metadata"] + assert seen["mass_log"] == expected["mass_log"] + assert seen["column_order"] == expected["column_order"] + + +def test_a_retained_mutating_observer_changes_nothing(tmp_path: Path) -> None: + """Amendment 24 still holds across the new fields. + + The observer keeps every snapshot and rewrites its tables and its + metadata view after the callback returns; what later nodes read through + the frame fields, and the run's identity, are unchanged. + """ + plain_manifest, plain_probe, sources, _ = run_probe(tmp_path / "plain") + retained: list[object] = [] + + def observe(node_id: str, population) -> None: + retained.append(population) + for entity in population.frame.entities: + table = population.frame.table(entity) + for column in table.columns: + table.loc[:, column] = table[column].iloc[0] + + observed_manifest, observed_probe, _, _ = run_probe( + tmp_path / "observed", sources=sources, observer=observe + ) + assert len(retained) == len(observed_manifest.nodes) + assert {n: r.key for n, r in observed_manifest.nodes.items()} == { + n: r.key for n, r in plain_manifest.nodes.items() + } + for node_id in ("probe_first", "probe_second"): + assert observation(observed_probe, node_id) == observation(plain_probe, node_id) diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_kernel_contract.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_kernel_contract.py index 885116345..fe524a25d 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_kernel_contract.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_kernel_contract.py @@ -340,7 +340,9 @@ def test_context_artifacts_default_empty_and_are_immutable() -> None: """Amendment 19: ``artifacts`` rides before the amendment-13/17 pair.""" fields = [f.name for f in dataclasses.fields(KernelContext)] assert fields[-2:] == ["tolerances", "numerics"] - assert fields[fields.index("artifacts") + 1] == "tolerances" + # Amendment 19 claims artifacts rides *before* the pair, not adjacent to + # it; amendment 26's three frame fields ride between them. + assert fields.index("artifacts") < fields.index("tolerances") node = Node("draw", "fit.draw@1") bare = KernelContext( node=node, From 5b94370182cfee76a1f342a2f584087087560842 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 08:47:57 -0400 Subject: [PATCH 04/44] B2: the kernel context carries the version's frame view (amendment 26) Co-Authored-By: Claude Opus 5 (cherry picked from commit 895aabf191f5ddd675ad6ebb9376546830eacfb4) --- .../tests/engine_free/shared/test_acceptance_b_ownership.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_acceptance_b_ownership.py b/packages/microcosm-graph/tests/engine_free/shared/test_acceptance_b_ownership.py index e9f38d01c..23076fac6 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_acceptance_b_ownership.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_acceptance_b_ownership.py @@ -120,6 +120,9 @@ def test_b2_executor_enforces_ownership(tmp_path: Path) -> None: "rng", "sources", "artifacts", # amendment 19: declared typed opaque artifact inputs + "frame_metadata", # amendment 26: the population version's metadata + "frame_mass_log", # amendment 26: the version's incoming Frame mass log + "frame_column_order", # amendment 26: the version's order, projected only "tolerances", # amendment 13: declared tolerances of the inputs' owners "numerics", # amendment 17: per-coordinate numeric class, bound, platform } From ffbc992e425e9d468bb4c1f2b66363a1bd07ec45 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 08:50:06 -0400 Subject: [PATCH 05/44] Record the UK shared-graph contract lane receipts and runtime plan Source-only receipt for amendments 25 and 26: exact heads, the absence proof on 15ebde806, the #901 consumer that fixed the contract shape, file hashes, the contract decisions that differ from #901, and the finite invented-only runtime plan left for root to execute. Runtime UNTESTED in this lane. Co-Authored-By: Claude Opus 5 (cherry picked from commit 6f4ba4ec989eba93786b5d88033631ad253fdc5c) --- PROGRESS-uk-shared-graph-contracts.md | 109 +++++++++++++++++ .../901-uk-shared-graph-contracts-receipts.md | 115 ++++++++++++++++++ 2 files changed, 224 insertions(+) create mode 100644 experiments/901-uk-shared-graph-contracts-receipts.md diff --git a/PROGRESS-uk-shared-graph-contracts.md b/PROGRESS-uk-shared-graph-contracts.md index bd18785be..0bdde5d7c 100644 --- a/PROGRESS-uk-shared-graph-contracts.md +++ b/PROGRESS-uk-shared-graph-contracts.md @@ -64,3 +64,112 @@ at `051fb972` (SHA-256 `fc6f5b33127020f6e0529b39304715fe2028d47a6627b152b1e52e0d - Amendment 25: `WeightUpdate` + ordered-axis receipt. - Amendment 26: `KernelContext` frame metadata / mass log / column order. + +--- + +## Done (2026-09-13) + +| Commit | What | +| --- | --- | +| `97428cd9f` | Lane baseline: absence proof + the exact #901 consumer read | +| `ef2dc69c3` | Amendment 25: `WeightUpdate` + `weight_update_receipt` | +| `f33d47cda` | Amendment 26: `KernelContext` frame metadata / mass log / column order | +| `895aabf19` | Acceptance suite B2 field set, isolated (as amendment 19's was, `a2b6dfb0b`) | + +The two amendments are separable: 25 touches `decl.py` (re-locked) and +leaves `kernel.py` byte-identical; 26 touches `kernel.py` (re-locked) and +leaves `decl.py` byte-identical. Either can be dropped without the other. + +## Contract decisions + +1. **`reason` is normative.** It enters the node key, so two updates that + state different purposes are different nodes. Follows #901's own + declaration; the consequence is stated in the amendment. +2. **`to_kind` stays a property, not a field.** That is what keeps the two + declarations' field sets disjoint (`{entity, to_kind, mass}` vs + `{entity, kind, reason, mass}`), so neither canonical bytes nor + declaration JSON can confuse them — and existing `to_kind` readers (the + design-weight cap, the calibration view) keep working unchanged. +3. **No `free` mass on an update.** #901 declares this too. An update that + neither moves kind nor bounds mass records nothing checkable. +4. **Replay is structural, not a parallel rule.** `_load_cached_result` + already reconstructs the `KernelResult` and re-applies REWEIGHT to the + current base, so a cache hit re-enters `_apply_weight_update`. No + executor change was needed for the axis check. +5. **The frozen interface does not import another shard's private name.** + #901's `kernel.py` imports `microcosm.frame.bundle._freeze_metadata`. + This lane does not: `Frame` has already deeply frozen the metadata the + executor passes, and `KernelContext` adds a read-only view over it. The + docstring says exactly that and claims no deep freeze of its own. +6. **The three fields ride between `artifacts` and `tolerances`**, not at + the end as in #901, so amendment 17's "numerics rides at the end of the + context" stays literally true. Only amendment 19's unit assertion of + *adjacency* relaxes, to the ordering it actually claimed. +7. **A column order may not name an unprojected column.** Set equality + with the projected columns, not a subset: a column *name* is itself + information about the version. + +## Remaining risks + +- **Runtime is UNTESTED here.** No pytest, import, engine or install was + run, per the lane's instructions. Everything below the source level is + unverified; see the runtime plan. +- The acceptance-suite commit `895aabf19` is the one change this lane made + to a file the charter assigns to the suite lane. It is isolated to one + file and follows the precedent the amendment-19 doc text states + explicitly ("the acceptance suite's B2 field set gains it in its own + commit"). If root's owner disagrees, dropping that commit leaves B2 red + and the rest intact. +- Amendment 25 changes no node key; amendment 26 changes none either. + Neither re-pins a spec digest. If a spec/seed digest moves in CI, that + is main drift, not this lane (see `[[spec-engine-attested-modules]]`). +- `_context_digest` (B4's mutation check) was **not** extended to the + three new fields. They are immutable views over an immutable `Frame`, so + there is nothing for a kernel to mutate; stated here so the omission is + a decision rather than an oversight. +- No claim is made that any UK build, native lane, calibration or release + passes. This lane read source only. + +## Next (for root, before execution) + +Finite, invented-only runtime plan — nothing below touches a country +model, engine, native source, gated microdata or the network. + +``` +uv sync --all-packages --locked +uv run pytest packages/microcosm-graph/tests/test_graph_weight_update.py \ + packages/microcosm-graph/tests/test_graph_frame_context.py +uv run pytest packages/microcosm-graph/tests/test_graph_interface_lock.py \ + packages/microcosm-graph/tests/test_acceptance_b_ownership.py \ + packages/microcosm-graph/tests/test_graph_kernel_contract.py \ + packages/microcosm-graph/tests/test_graph_serialize.py \ + packages/microcosm-graph/tests/test_graph_decl.py \ + packages/microcosm-graph/tests/test_acceptance_d_weights.py \ + packages/microcosm-graph/tests/test_graph_population.py \ + packages/microcosm-graph/tests/test_graph_executor.py \ + packages/microcosm-graph/tests/test_graph_explain.py \ + packages/microcosm-graph/tests/test_acceptance_replays.py +uv run pytest packages/microcosm-graph packages/microcosm-frame +uv run ruff check . +uv run python tools/ci_test_groups.py --verify +``` + +Expected: 19 + 19 new tests pass; the ten regression files stay green; +`ruff check` and `--verify` already pass here. The four places a source-only +lane could be wrong, in the order worth checking: + +1. `test_graph_frame_context.py::test_a_retained_mutating_observer_changes_nothing` + mutates a detached snapshot with `table.loc[:, column] = table[column].iloc[0]`. + If pandas copy-on-write makes that a no-op on the snapshot, the test + passes vacuously rather than falsely; tighten it rather than trust it. +2. The toy person column order is asserted from `fixtures/toy_country/person.csv` + (`person_id, person_household_id, person_release_id, age, income, ...`). + If that fixture changes, `test_column_order_is_the_versions_own_order_not_the_projection` + is the test that notices. +3. `_append_frame_mass_log` requires the record to bracket the real + household totals; the probe states an unchanged total from + `context.weights["household"].values.sum()`. A mismatch would surface + as `PopulationError` from `test_mass_log_is_the_incoming_log`. +4. `test_round_trip_refuses_a_mixed_weights_payload` edits canonical JSON + by string surgery and depends on lexicographic key order + (`entity, kind, mass, reason`). diff --git a/experiments/901-uk-shared-graph-contracts-receipts.md b/experiments/901-uk-shared-graph-contracts-receipts.md new file mode 100644 index 000000000..58e428d4a --- /dev/null +++ b/experiments/901-uk-shared-graph-contracts-receipts.md @@ -0,0 +1,115 @@ +# Shared graph contracts extracted for the UK full-build graph (#901) + +Source-only lane receipt, 2026-09-13. Runtime **UNTESTED** — see +`PROGRESS-uk-shared-graph-contracts.md` for the decisions, risks and the +finite invented-only runtime plan this lane leaves for root. + +## Heads + +| Thing | Exact identity | +| --- | --- | +| Worktree | `_worktrees/microcosm-uk-shared-graph-contracts-20260913` | +| Branch | `uk-shared-graph-contracts-20260913` | +| Base (reviewed main) | `15ebde806cd1a262363f7217fe535c7234ff757f` | +| #901 head (reviewed, re-verified) | `051fb972b19d319d58277bd63306d0d0e0947ce2` | +| #901 live state | OPEN, draft, `CONFLICTING`, updated 2026-09-10T21:03:10Z | +| Source review followed | `uk-parallel-review.md`, 2026-09-12 | + +`origin/main` was re-fetched after the work and is still `15ebde806`. + +## Absence, before anything was written + +`grep -rni weightupdate` over the worktree at `15ebde806` returns nothing. +`decl.py:320` carries only `WeightTransition`, which requires `to_kind` +strictly later in `WEIGHT_KINDS`; `population.py:1944` rejects a +non-forward move. `kernel.py:361-370` is the complete `KernelContext` +field list and has no metadata, mass-log or column-order field. The +interface lock matched both files exactly. So both extensions were +genuinely absent, and no duplicate patch was produced. + +The write half of the mass-log contract already existed on main +(`population._append_frame_mass_log`, `receipt['frame_mass_log_append']`); +only the kernel's view of the incoming log was missing. That is why this +lane adds a read side and no second ledger. + +## Consumer that fixed the shape + +`packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py` +at `051fb972`, SHA-256 +`fc6f5b33127020f6e0529b39304715fe2028d47a6627b152b1e52e0d69f2efdc`: + +- `context_frame` (L58-80): `context.frame_column_order.get(...)`, + `getattr(context, "frame_mass_log", ())`, + `getattr(context, "frame_metadata", {})`. +- `uk.full.normalize`: `WeightUpdate("household", weight_kind, "Normalize + sampled source-family mass.")`, `mass="declared"`, and + `receipt["weight_update"] = weight_update_receipt(ids)` where `ids` is + the household-id axis of the context frame. + +Both signatures land exactly as #901 calls them, so no UK edit is needed +to consume this. Nothing was copied from #901's executor, and +`graph.attachments._PopulationRetention` is not imported anywhere. + +## Amendments + +**25 — a same-kind weight update is declarable.** `decl.py` gains +`WeightUpdate(entity, kind, reason, mass)` and +`WEIGHT_UPDATE_MASS_POLICIES`; a new non-frozen +`microcosm/graph/weight_update.py` gains `weight_update_receipt`. The +incumbent, declared and returned kinds must agree; mass is `conserve` or +`declared`; `reason` is required, non-empty and normative. The kernel +binds its ordered entity axis and the executor recomputes that binding +from the incumbent axis, on cold execution and on replay — replay for +free, because `_load_cached_result` already re-applies REWEIGHT to the +current base through the same function. `to_kind` is a property, so the +two declarations' field sets are disjoint and declaration JSON round-trips +each as itself; the transition payload is byte-for-byte unchanged. + +**26 — the context carries the version's frame view.** `kernel.py` gains +`frame_metadata`, `frame_mass_log` and `frame_column_order`, riding after +`artifacts` and before `tolerances` so amendment 17's "numerics rides at +the end" stays literally true. A column order must be exactly an ordering +of the projected columns, so it can neither hide a column the node was +given nor name one it was not. The frozen interface does **not** import +`microcosm.frame.bundle._freeze_metadata` (as #901's does); the executor +passes `Frame.metadata`, already deeply frozen by `Frame`, and the +docstring claims only the read-only view it actually adds. + +Neither amendment adds a `Node` field, changes a canonical projection, or +moves a node key. `decl.py` is re-locked by 25, `kernel.py` by 26. + +## Identity + +| File | SHA-256 | +| --- | --- | +| `decl.py` | `e78c7359b48813c57eff6f697359682753b74f86f565f4e6974421e5a4048933` | +| `kernel.py` | `51f45e899ba578a6b2324636a9879253266b067812f68b648eacfcd5a184d245` | +| `weight_update.py` | `0ccfe6fcd257ef62b1f771b12eecc8ac5d447f5aa7d0403102e0ae290de16720` | +| `population.py` | `33d1bb7bacea22870940288bf1907fb9eb24df7c245a216ff802e7fb41f5208f` | +| `serialize.py` | `e5bf83c1082154f148626b6a36676614c6ff6c3fe0721aed94a1501da3021b1f` | +| `executor.py` | `ec4473bb033c4b1a36180c1518a42c755a46a2265d10461df1dfa00065862364` | +| `explain.py` | `734a7b0e31692c31a99528cd83d9e74d3e508a317d913c1c169724a42d69d0de` | +| `graph/__init__.py` | `697c59a37989a36124e6d43c7b07dd3b0582d965f97303c1fb02c88b41db2d48` | +| `tests/test_graph_weight_update.py` | `771ae0becbc57a4dd6198b9df229ec8c8262a5597647f335b7d5ed9be83471ff` | +| `tests/test_graph_frame_context.py` | `d58250d2194c81002be7282cd53597a7b9cff0906bb879b4b3926b4ac5b91ada` | +| `docs/graph-interface.lock` | `b42811ff0411dc179aaf9ddcf827aa270db866e08112cef8d1839df99c5a1f02` | + +The lock was re-recorded as part of each numbered amendment, never +refreshed to make a test green: amendment 25 moved only the `decl.py` +line, amendment 26 only the `kernel.py` line. + +## What stays with María + +Every UK graph stage, kernel, calibration target, geography ladder, +scorecard and release gate in #901. This lane adds no country graph, no +UK node, and no second definition of anything #901 owns. + +## Checks actually run + +`ruff check .` (clean), `ruff format --check` on every file this lane +touched (clean), `python -I -B -S` stdlib `ast` parses of all 17 changed +files, and `tools/ci_test_groups.py --verify` (`verification=ok`; both new +test files land in `fast/rest` and the engine lane beside their 27 sibling +graph tests, neither `[defaulted]`). No pytest, no import of the +production package, no engine, no install, no network beyond `gh` +metadata and public blob reads. From 22b2738cb417ad677112af47b3f3f6d8571eaad8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 09:18:12 -0400 Subject: [PATCH 06/44] File the independent adjudication and open the fix round The reviewer returned its verdict through the tool result because its session had no write tool, so the text is filed here unedited rather than restated. Records the round's scope: F1, F3-F6 accepted, F2 not. Co-Authored-By: Claude Opus 5 (cherry picked from commit 6104459f693db98ea835fb13044ef5ac1f866224) --- FABLE-REVIEW.md | 48 +++++++++++++++++++++++++++ PROGRESS-uk-shared-graph-contracts.md | 31 +++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 FABLE-REVIEW.md diff --git a/FABLE-REVIEW.md b/FABLE-REVIEW.md new file mode 100644 index 000000000..b5a4456bd --- /dev/null +++ b/FABLE-REVIEW.md @@ -0,0 +1,48 @@ + + +**Verdict: REQUEST_CHANGES** for the source proposal. Two findings are actionable design defects in the new contracts; the rest are hardening and test-strength items. I could not save FABLE-REVIEW.md: this session is in plan mode with no write tool exposed, so the full review is below for root to file verbatim. + +## Reviewed identity + +| Item | Value | +| --- | --- | +| HEAD | `6f4ba4ec989eba93786b5d88033631ad253fdc5c` (resolved from `.git/refs/heads/uk-shared-graph-contracts-20260913`) | +| Base | `15ebde806cd1a262363f7217fe535c7234ff757f` (as given; not independently resolved) | +| Method | Read-only. No git diff was available without a shell, so I reviewed the current contents of every file in the receipts' identity table plus both new test files, the B2 edit, the kernel-contract unit test, and the amendment 25/26 text. No imports, pytest, engine, or data. | + +## Ranked findings + +**1. HIGH. `frame_mass_log` leaks non-declared same-version sibling output into kernel inputs with no key binding.** +`_project_context` passes `population.frame.mass_log` at `packages/microcosm-graph/src/microcosm/graph/executor.py:678`. For an ordinary node that population is the cumulative version state, updated after every ordinary node at `executor.py:2569-2570`, and `_append_frame_mass_log` runs for ordinary nodes too at `population.py:1157`. So node X sees records appended by any earlier ordinary node A in the same version, whether or not A is an ancestor. X's key binds only `frame_key(version)` plus declared input owners (`keys.py:187-209`), never A. Result: adding, removing, or re-parameterising A changes X's visible input while X's key is unchanged, so a cache hit replays output computed against a different log. This is a new charter-A hole; before amendment 26 kernels could not see the log at all. The doc sentence "incidental node order is not authority" describes the hazard but nothing enforces it. +Exact fix, minimal: in `run_graph` record `boundary_logs[node.id] = updated.frame.mass_log` when a structural node is admitted, and pass `frame_mass_log=boundary_logs[compiled.versions[node_id]]` for ordinary nodes; structural nodes keep the full incumbent log, which their key already binds through `members` (`keys.py:221-225`). Alternative that keeps sibling visibility: track which node appended each record and reject projection when a record's author is not in `_transitive_ancestors(compiled, node_id)`. Either way `test_mass_log_is_the_incoming_log` at `test_graph_frame_context.py:346-354` must change: the successor is an ordinary node reading the appender's column, and the fix makes it see `()` unless the ancestor rule is used. + +**2. HIGH. A `WeightUpdate` on kind `design` leaves design anchors stale and, after an EXPAND, mixed.** +Anchors are captured once at CREATE (`population.py:333-338`) and only carried afterwards (`population.py:1159`, `_carry_design_weights` at `2125-2181`). `_apply_weight_update` replaces the frame's design-kind values (`population.py:2038`) without re-anchoring. Consequences: the calibrated cap at `population.py:2323-2351` and `realized_max_weight_ratio` at `2354-2375` compare against pre-update design weights, so a normalisation by factor k makes every later cap ratio off by k; an EXPAND after the update anchors entrants from the updated values (`2166-2179`) while retained rows keep original anchors. The #901 consumer, `uk.full.normalize`, is exactly a design-weight normalisation. The decl docstring at `decl.py:381-382` states "ancestry is untouched" as if deliberate, but the mixed-anchor case is not a coherent contract. +Exact fix: in `patch` after line 1159, when `isinstance(node.weights, WeightUpdate)` and the kind is `design`, set `design_weights[entity] = frame.weights_for(entity).values`, and say so in amendment 25. If root prefers the current semantics, refuse `kind="design"` in `WeightUpdate.__post_init__` instead; leaving it silent is the one option I would not accept. + +**3. MEDIUM. `_context_digest` omission is defensible but the fields share live objects.** +Actual behaviour checked: `frame_metadata` is a `MappingProxyType` over a copy whose leaves are `Frame`-frozen tuples, frozensets and `_FrozenMapping` (`bundle.py:1346-1362`); `frame_column_order` values are `tuple[str]`; `frame_mass_log` is the population's own tuple of frozen `MassChangeRecord` objects, shared by reference (`executor.py:678`). Through the public API nothing is writable, so the omission at `executor.py:459-488` does not produce false passes. But `object.__setattr__` on a shared record silently rewrites the live version's log, and unlike `tables` the executor would not notice. This matches the existing treatment of `context.node`, so it is not blocking. Cheap hardening: digest `canonical_json(column_order)`, the mass-log record fields, and `store._encode_frame_metadata(frame_metadata)` inside `_context_digest`. + +**4. MEDIUM. Two new tests are weaker than their names.** +`test_cold_then_required_replay_revalidates_the_axis` (`test_graph_weight_update.py:311-337`) only proves a hit succeeds; it never shows the axis check executes on replay. `test_a_retained_mutating_observer_changes_nothing` (`test_graph_frame_context.py:410-435`) says it rewrites the metadata view but only rewrites tables; the three new fields are untouched. + +**5. LOW. The motivating "re-solve an existing calibration" case is unreachable with the shared kernel.** `calibrate.adam@1` returns no `receipt['weight_update']` and its error text at `packages/microcosm-calibrate/src/microcosm/calibrate/kernels.py:205-209` still demands a `WeightTransition`. Declaring it as a `WeightUpdate` rejects at `population.py:2025-2030`. Not this lane's file, but amendment 25's text should not claim the case is covered. + +**6. LOW. Cosmetic.** `decl.py:568-572` says "weight transition's" for an update mismatch. `test_graph_kernel_contract.py:339-345` was also edited, so the receipts' "isolated to one test file" claim applies to the acceptance suite only; that is acceptable under the charter. + +## Reproducing tests to invent + +- **T1 leak:** probe kernel writes `float(len(context.frame_mass_log))` into `person.x`, reading only `person.age`. Run `{survey, X}` cold: x = 0. Run `{survey, A, X}` into the same store where A appends a record: X hits and reports 0 while a cold run gives 1. Same key, different truth. +- **T2 anchors:** `WeightUpdate("household","design",…)` with factor 2, then a `calibrated` transition returning weights equal to the updated design weights with `max_weight_ratio=1.5`. Currently rejected as ratio 2.0; the receipt reports realized ratio 2.0 against a frame whose own design weights it equals. +- **T3 replay:** monkeypatch `microcosm.graph.population.weight_update_receipt` to return a foreign digest during the `resume="require"` run and assert `NodeRejectedError` matching "different .household. axis". +- **T4 tamper:** a kernel that calls `object.__setattr__(context.frame_mass_log[0], "reason", "x")`; assert the next node sees the original reason. Currently the live log changes and no rejection fires. + +## Residual runtime risks + +- Nothing here was executed. Pandas copy-on-write behaviour in the observer test and the fixture column order at `fixtures/toy_country/person.csv` line 1 (age before income) are the two places a source-only read can be wrong. +- In `resume="require"`, a mismatched axis receipt fails mid-run at apply time rather than in `_preflight_require`, since preflight validates record shape only. +- Column order is bound to the key by construction (owners are ancestors, order follows depth then id, rewrites keep position); I found no leak there. diff --git a/PROGRESS-uk-shared-graph-contracts.md b/PROGRESS-uk-shared-graph-contracts.md index 0bdde5d7c..2e0519236 100644 --- a/PROGRESS-uk-shared-graph-contracts.md +++ b/PROGRESS-uk-shared-graph-contracts.md @@ -173,3 +173,34 @@ lane could be wrong, in the order worth checking: 4. `test_round_trip_refuses_a_mixed_weights_payload` edits canonical JSON by string surgery and depends on lexicographic key order (`entity, kind, mass, reason`). + +--- + +# Fix round (2026-09-13, after the independent Fable adjudication) + +## State + +The lane's four commits (`97428cd9f`..`6f4ba4ec9`) stand. An independent +read-only adjudication of exactly `6f4ba4ec9` over base `15ebde806` +returned **REQUEST_CHANGES**; its verbatim text is now filed in this +packet as `FABLE-REVIEW.md` (the reviewer ran without a write tool and +returned the review through its tool result instead). + +Root owns adjudication. Findings F1, F3, F4, F5 and F6 are accepted and +implemented in this round. **F2 is not accepted as stated** — see the +point-by-point response below and in `FIX-RESULT.md`. + +`origin/main` re-fetched before editing: still +`15ebde806cd1a262363f7217fe535c7234ff757f`, nothing new to merge, branch +is 5 commits ahead and 0 behind. + +**Runtime remains UNTESTED in this round too.** No pytest, no production +import, no engine, no install, no network. Source, stdlib `ast`, `ruff` +and the CI group inventory only. + +## Next + +- F1: project ordinary nodes from their version boundary's mass log. +- F3/F4: detach the exposed frame objects; digest all three fields. +- F2: source-backed rejection plus the anchor-invariance tests. +- F5/F6: correct the docs' motivating claim and the disclosure. From 22561bf48a72e5bec27e3bc788c51eb279644712 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 09:23:29 -0400 Subject: [PATCH 07/44] F1: a node sees the mass log its own key binds (amendment 26) The projection handed every node `population.frame.mass_log`. For an ordinary node that is the version's *cumulative* log, updated after each member, while its key binds only the version's structural boundary and the owners of the columns it declared. Adding or re-parameterising an unrelated sibling therefore changed what a node was shown without moving its key, so a cache hit could replay output computed against a different log. `run_graph` now records each version's log as that version is admitted -- cold execution and a restored hit both reach the same line -- and projects ordinary nodes from that boundary. A structural node still receives the cumulative log, because `keys.py` binds its base *and* every ordinary member of that version through `members`. The properties distinguish the two cases: an unread same-version appender is invisible to an ordinary member and moves neither its key nor its stored bytes, cold-with-sibling equals cold-without and replays across, while the same appender is visible to the structural boundary whose key it moves. Co-Authored-By: Claude Opus 5 (cherry picked from commit c0e275543374ee18febf5f1811c5fee9cfc6d32a) --- ...red-graph-contracts-frame-context.added.md | 2 +- docs/graph-acceptance.md | 28 +- docs/graph-interface.lock | 2 +- .../src/microcosm/graph/executor.py | 42 ++- .../src/microcosm/graph/kernel.py | 18 +- .../shared/test_graph_frame_context.py | 332 ++++++++++++++++-- 6 files changed, 381 insertions(+), 43 deletions(-) diff --git a/changelog.d/uk-shared-graph-contracts-frame-context.added.md b/changelog.d/uk-shared-graph-contracts-frame-context.added.md index 64fe69ac1..a66fead06 100644 --- a/changelog.d/uk-shared-graph-contracts-frame-context.added.md +++ b/changelog.d/uk-shared-graph-contracts-frame-context.added.md @@ -1 +1 @@ -Added `KernelContext.frame_metadata`, `frame_mass_log` and `frame_column_order`, so a kernel can reconstruct its population version's declared slices without seeing any column it did not declare (graph amendment 26). +Added `KernelContext.frame_metadata`, `frame_mass_log` and `frame_column_order`, so a kernel can reconstruct its population version's declared slices without seeing any column it did not declare. The mass log a node receives is the one its key binds: its version's structural boundary for an ordinary node, the base version's cumulative log for a structural one (graph amendment 26). diff --git a/docs/graph-acceptance.md b/docs/graph-acceptance.md index 3ee9d651d..bec9a1d99 100644 --- a/docs/graph-acceptance.md +++ b/docs/graph-acceptance.md @@ -582,12 +582,28 @@ lock unchanged: difference from the UK branch, which imports `microcosm.frame.bundle._freeze_metadata` into the frozen interface: a frozen contract should not depend on another shard's private name.) - - `frame_mass_log` — the version's `Frame` mass records, in order, and - specifically the *incoming* ones. A node needing a stage's completed - records must run after that stage's structural boundary or read its - predecessor's evidence; incidental node order is not authority. The - write side already existed (`receipt['frame_mass_log_append']`); only - the read side was missing. + - `frame_mass_log` — the `Frame` mass records the node's *key* binds, + in order. An ordinary node's key binds its version's structural + boundary (`population_input`) and the owners of the columns it + declared (`input_artifacts`); it does not bind the other ordinary + members of its version. Its log is therefore the version's + **boundary** log, captured when the structural node was admitted, and + a record another member appends afterwards is not visible to it. The + alternative — the cumulative log the version carries at the moment the + node runs — would be a kernel input no key binds: adding or + re-parameterising an unrelated sibling would change what the node + sees while its key, and so its cache entry, stayed put, and a hit + would replay output computed against a different log. A structural + node is given its base version's cumulative log instead, because its + key does bind it: `keys.py` binds the base's frame identity *and* + every ordinary member of that version through `members`, which + `compile_graph` fills with `members.get(base, ())`. Boundaries are + captured where the version is admitted, so cold execution and a + restored cache hit record the same one. A node needing a stage's + completed records therefore runs after that stage's structural + boundary or reads its predecessor's evidence; incidental node order is + not authority. The write side already existed + (`receipt['frame_mass_log_append']`); only the read side was missing. - `frame_column_order` — entity to the version's own column order, restricted to the columns projected into `tables`. An entry that is not exactly an ordering of that table's columns is refused, so an diff --git a/docs/graph-interface.lock b/docs/graph-interface.lock index f09fe7bb1..cb01bf926 100644 --- a/docs/graph-interface.lock +++ b/docs/graph-interface.lock @@ -1,2 +1,2 @@ e78c7359b48813c57eff6f697359682753b74f86f565f4e6974421e5a4048933 decl.py -51f45e899ba578a6b2324636a9879253266b067812f68b648eacfcd5a184d245 kernel.py +a49bb4461e7928118b05c13351a99b7585e5068a0868f6bc40634ebb057f29ca kernel.py diff --git a/packages/microcosm-graph/src/microcosm/graph/executor.py b/packages/microcosm-graph/src/microcosm/graph/executor.py index 49fafc448..614352d67 100644 --- a/packages/microcosm-graph/src/microcosm/graph/executor.py +++ b/packages/microcosm-graph/src/microcosm/graph/executor.py @@ -18,7 +18,7 @@ import numpy as np import pandas as pd -from microcosm.frame import Frame, WeightKind, Weights +from microcosm.frame import Frame, MassChangeRecord, WeightKind, Weights from . import keys as graph_keys from .artifact_edges import scope_payload, typed_contracts, value_from_descriptor @@ -643,7 +643,17 @@ def _project_context( tolerances: Mapping[tuple[str, str], Tolerance | None], numerics: Mapping[tuple[str, str], NumericScope], artifacts: Mapping[str, ArtifactValue] | None = None, + mass_log: tuple[MassChangeRecord, ...] = (), ) -> KernelContext: + """Project one node's frozen inputs out of the population it reads. + + ``mass_log`` is the ``Frame`` mass log this node's *key* binds, which is + not in general the cumulative log carried by ``population``: see the + boundary selection in :func:`run_graph`. It defaults to the empty log, + so a caller that projects a context outside the executor states no mass + history rather than inheriting one it never bound (amendment 26). + """ + if population is None: return KernelContext( node=node, @@ -757,7 +767,7 @@ def _project_context( sources=MappingProxyType({name: sources[name] for name in node.sources}), artifacts={} if artifacts is None else artifacts, frame_metadata=frame.metadata, - frame_mass_log=frame.mass_log, + frame_mass_log=mass_log, frame_column_order=MappingProxyType(column_order), tolerances=tolerances, numerics=numerics, @@ -2684,6 +2694,11 @@ def _execute_graph( _preflight_require(compiled, store, keys, implementations, kernels) populations: dict[str, Population] = {} + # Each structural version's ``Frame`` mass log as that version was + # admitted. ``populations`` cannot answer this: an ordinary member + # replaces its version's entry there, so by the time a later member runs + # the boundary state is gone. + boundary_mass_logs: dict[str, tuple[MassChangeRecord, ...]] = {} receipts: dict[str, NodeReceipt] = {} receipt_payloads: dict[str, Mapping[str, object]] = {} for node_id in compiled.order: @@ -2804,6 +2819,25 @@ def _execute_graph( ) for binding in node.artifact_inputs } + if incumbent is None: + incoming_mass_log: tuple[MassChangeRecord, ...] = () + elif node.structural is StructuralDelta.NONE: + # An ordinary node's key binds its version's structural + # boundary and the owners of the columns it declared + # (``population_input`` and ``input_artifacts`` in keys.py) -- + # never a sibling that merely ran earlier in the same version + # and appended a mass record. Projecting the cumulative log + # would hand the kernel an input its own key does not bind, so + # adding or re-parameterising that sibling would change what a + # node sees while its key, and therefore its cache entry, + # stayed put. It is projected from the boundary instead. + incoming_mass_log = boundary_mass_logs[compiled.versions[node_id]] + else: + # A structural node's key binds its base *and* every ordinary + # member of that version (``members`` in keys.py, built from + # ``compiled.predecessors``), so the base version's cumulative + # log is bound by the key that will name its cache entry. + incoming_mass_log = incumbent.frame.mass_log context = _project_context( node, incumbent, @@ -2812,6 +2846,7 @@ def _execute_graph( tolerances=input_tolerances, numerics=input_numerics, artifacts=artifact_values, + mass_log=incoming_mass_log, ) before = _context_digest(context) try: @@ -2969,6 +3004,9 @@ def _execute_graph( populations[compiled.versions[node_id]] = updated else: populations[node.id] = updated + # Captured at admission, so cold execution and a restored cache + # hit record the same boundary: both reach this line. + boundary_mass_logs[node.id] = updated.frame.mass_log if _population_observer is not None: _population_observer( diff --git a/packages/microcosm-graph/src/microcosm/graph/kernel.py b/packages/microcosm-graph/src/microcosm/graph/kernel.py index 528d11836..f718c5f48 100644 --- a/packages/microcosm-graph/src/microcosm/graph/kernel.py +++ b/packages/microcosm-graph/src/microcosm/graph/kernel.py @@ -353,11 +353,19 @@ class KernelContext: ``Frame`` has already deeply frozen; this class adds a read-only view over that mapping and does not itself deep-freeze a mapping built some other way (amendment 26). - frame_mass_log: The population version's ``Frame`` mass records, in - order. This is the *incoming* log: a node that needs a stage's - completed records must run after that stage's structural - boundary or read its predecessor's evidence, because incidental - node order is not authority (amendment 26). + frame_mass_log: The ``Frame`` mass records this node's key binds, + in order. For an ordinary node that is its population version's + *boundary* log -- the log as that version was admitted -- so a + record another member of the same version appends afterwards is + not visible here: an ordinary node's key binds its version's + structural boundary and the owners of the columns it declared, + not its siblings, and a view the key does not bind could not + survive a cache hit. A structural node receives its base + version's cumulative log, which its key does bind, through that + base and that version's members. A node that needs a stage's + completed records therefore runs after that stage's structural + boundary or reads its predecessor's evidence; incidental node + order is not authority (amendment 26). frame_column_order: Entity to the population version's own column order, restricted to the columns projected into ``tables``. The executor projects ``tables`` in declaration order, so this diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py index d7026ca35..1ade82291 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py @@ -21,7 +21,7 @@ import pandas as pd import pytest -from microcosm.frame import Frame, MassChangeRecord +from microcosm.frame import Frame, MassChangeRecord, WeightKind, Weights from microcosm.graph import ( ContentStore, Graph, @@ -30,6 +30,8 @@ Node, Owned, Slice, + StructuralDelta, + WeightTransition, compile_graph, run_graph, ) @@ -47,6 +49,7 @@ SOURCE_REF = "source.metadata@1" PROBE_REF = "probe.frame_context@1" +REWEIGHT_PROBE_REF = "probe.frame_context_reweight@1" #: Metadata the CREATE kernel puts on the version, including a nested value #: so the frozen projection is exercised rather than a flat string map. @@ -75,6 +78,22 @@ def compute(self, context: KernelContext) -> KernelResult: ) +def observed(context: KernelContext) -> dict[str, object]: + """Everything a property asserts about one node's frame view.""" + return { + "node": context.node.id, + "column_order": { + entity: tuple(columns) + for entity, columns in context.frame_column_order.items() + }, + "projected": { + entity: tuple(table.columns) for entity, table in context.tables.items() + }, + "metadata": dict(context.frame_metadata), + "mass_log": tuple(context.frame_mass_log), + } + + class FrameContextProbe(toy.ToyKernel): """Owns one column and records the frame view it was handed. @@ -88,25 +107,14 @@ def __init__(self, ref: str, capabilities, *, variant: str = "base") -> None: self.seen: list[dict[str, object]] = [] def compute(self, context: KernelContext) -> KernelResult: - self.seen.append( - { - "node": context.node.id, - "column_order": { - entity: tuple(columns) - for entity, columns in context.frame_column_order.items() - }, - "projected": { - entity: tuple(table.columns) - for entity, table in context.tables.items() - }, - "metadata": dict(context.frame_metadata), - "mass_log": tuple(context.frame_mass_log), - } - ) + self.seen.append(observed(context)) ids = pd.Index(context.tables["person"]["person_id"], name="person_id") + # The output *is* the mass-log view, so the stored bytes of a node + # differ whenever what it was shown differs. A key that survives an + # unrelated sibling therefore has to have been shown the same log. result_columns = { ("person", str(context.params["target"])): pd.Series( - 1.0, index=ids, dtype="float64" + float(len(context.frame_mass_log)), index=ids, dtype="float64" ) } reason = context.params.get("mass_reason") @@ -129,12 +137,39 @@ def compute(self, context: KernelContext) -> KernelResult: ) +class ReweightProbe(toy.ToyKernel): + """REWEIGHT: records the frame view it was handed, then scales weights. + + A structural node, unlike an ordinary one, has a key that binds its + base *and* every ordinary member of that version, so it is the node + that may be shown the version's cumulative mass log. + """ + + def __init__(self, ref: str, capabilities, *, variant: str = "base") -> None: + super().__init__(ref, capabilities, variant=variant) + self.seen: list[dict[str, object]] = [] + + def compute(self, context: KernelContext) -> KernelResult: + self.seen.append(observed(context)) + before = context.weights["household"].values + after = before * 2.0 + return KernelResult( + weights=Weights(values=after, kind=WeightKind.IMPORTANCE), + receipt={"mass": toy._mass_record(context, before, after, "free")}, + ) + + def build_registry() -> tuple[object, FrameContextProbe]: - """The toy registry with the metadata source and the probe registered.""" + """The toy registry with the metadata source and both probes registered. + + The structural probe is reachable as ``registry.get(REWEIGHT_PROBE_REF)`` + for the properties that are about a boundary rather than a member. + """ registry = toy.toy_registry() registry.register(MetadataSource(SOURCE_REF, toy._CREATE)) probe = FrameContextProbe(PROBE_REF, toy._DETERMINISTIC) registry.register(probe) + registry.register(ReweightProbe(REWEIGHT_PROBE_REF, toy._REWEIGHT)) return registry, probe @@ -147,6 +182,7 @@ def probe_node( columns: tuple[str, ...], target: str, mass_reason: str | None = None, + population: str = "survey", ) -> Node: """A probe reading ``columns`` of the person entity, in that order. @@ -163,7 +199,21 @@ def probe_node( inputs=inputs, outputs=(Owned("person", target, "float64"),), params={"target": target, "mass_reason": mass_reason}, - population="survey", + population=population, + ) + + +def reweight_probe_node(node_id: str = "boundary", *, base: str = "survey") -> Node: + """A structural boundary that observes the log it is handed.""" + return Node( + node_id, + REWEIGHT_PROBE_REF, + structural=StructuralDelta.REWEIGHT, + base=base, + inputs=(Slice("person", ("age",)), Slice("household", ("household_size",))), + weights=WeightTransition("household", "importance", mass="free"), + mass="free", + description="design -> importance, observing the frame view", ) @@ -346,15 +396,241 @@ def test_version_metadata_reaches_every_node_of_the_version(tmp_path: Path) -> N assert tuple(metadata["vintage"]) == ("frs", "was") -def test_mass_log_is_the_incoming_log(tmp_path: Path) -> None: - """The appending node sees the log before its own record; its successor after.""" - _, probe, _, _ = run_probe(tmp_path / "run") - assert observation(probe, "probe_first")["mass_log"] == () - after = observation(probe, "probe_second")["mass_log"] - assert len(after) == 1 - assert after[0].entity == "household" - assert after[0].reason == "households are unchanged by a derivation" - assert after[0].old_total == after[0].new_total +# ---------------------------------------------------------------------- +# The mass log is the one the node's key binds +# ---------------------------------------------------------------------- + + +#: One ordinary member that appends a ``Frame`` mass record, and one that +#: reads a different column of the same version and declares nothing of the +#: appender's. ``compile_graph`` orders equal-depth nodes by id, so +#: ``appender`` runs first and its record is in the version's cumulative log +#: by the time ``unrelated`` is projected. +APPEND_REASON = "households are unchanged by a derivation" + + +def sibling_graph(*, appender: bool = True, reason: str = APPEND_REASON) -> Graph: + """``unrelated`` beside an optional, unread same-version mass appender.""" + unrelated = probe_node("unrelated", columns=("income",), target="unrelated_a") + if not appender: + return Graph("toy", (toy.SOURCE,), (CREATE, unrelated)) + return Graph( + "toy", + (toy.SOURCE,), + ( + CREATE, + probe_node( + "appender", + columns=("age",), + target="appender_a", + mass_reason=reason, + ), + unrelated, + ), + ) + + +def boundary_graph() -> Graph: + """An appender, a structural boundary over it, and a member of the new version.""" + return Graph( + "toy", + (toy.SOURCE,), + ( + CREATE, + probe_node( + "appender", + columns=("age",), + target="appender_a", + mass_reason=APPEND_REASON, + ), + reweight_probe_node(), + probe_node( + "after_boundary", + columns=("age",), + target="after_a", + population="boundary", + ), + ), + ) + + +def boundary_graph_with_reason(reason: str) -> Graph: + """``boundary_graph`` with the appender stating a different purpose.""" + return Graph( + "toy", + (toy.SOURCE,), + tuple( + probe_node( + "appender", + columns=("age",), + target="appender_a", + mass_reason=reason, + ) + if node.id == "appender" + else node + for node in boundary_graph().nodes + ), + ) + + +def unrelated_bytes(store: ContentStore, manifest) -> tuple[bytes, bytes]: + """The stored bytes of ``unrelated``'s output column.""" + key = manifest.nodes["unrelated"].artifacts[("person", "unrelated_a")] + return toy.artifact_bytes(store, key) + + +def only_record(view: object) -> MassChangeRecord: + """The single mass record in an observed view, asserted to be alone.""" + log = tuple(view) # type: ignore[call-overload] + assert len(log) == 1 + return log[0] + + +def test_an_ordinary_node_sees_its_versions_boundary_log(tmp_path: Path) -> None: + """Not the cumulative log, and specifically not a sibling's record. + + ``appender`` runs first and appends one record to the ``survey`` + version. ``unrelated`` declares no column of the appender's, so its key + binds ``survey``'s boundary and nothing else; showing it the record + would be showing it an input its key does not bind. + """ + _, probe, _, _ = run_probe(tmp_path / "run", graph=sibling_graph()) + assert observation(probe, "appender")["mass_log"] == () + assert observation(probe, "unrelated")["mass_log"] == () + + +def test_the_appenders_own_record_reaches_the_version_it_leaves( + tmp_path: Path, +) -> None: + """The record is applied — it is the *view* that is bounded, not the log.""" + manifest, _, _, _ = run_probe(tmp_path / "run", graph=sibling_graph()) + record = only_record(manifest.population("survey").mass_log) + assert record.entity == "household" + assert record.reason == APPEND_REASON + assert record.old_total == record.new_total + + +def test_a_keyed_structural_predecessor_does_see_the_cumulative_log( + tmp_path: Path, +) -> None: + """A structural node's key binds its base *and* that version's members. + + That is the difference the projection turns on: the boundary may be + shown what an ordinary member may not, because re-parameterising the + appender moves the boundary's key and so cannot be replayed onto it. + """ + registry, probe = build_registry() + manifest, _, sources, _ = run_probe( + tmp_path / "run", graph=boundary_graph(), registry=registry, probe=probe + ) + boundary = registry.get(REWEIGHT_PROBE_REF) + record = only_record(observation(boundary, "boundary")["mass_log"]) + assert record.reason == APPEND_REASON + + other_registry, _ = build_registry() + moved = run_graph( + compile_graph(boundary_graph_with_reason("a different stated reason")), + sources=dict(sources), + store=ContentStore(tmp_path / "moved" / "store"), + kernels=other_registry, + ) + assert moved.nodes["boundary"].key != manifest.nodes["boundary"].key + assert moved.nodes["after_boundary"].key != manifest.nodes["after_boundary"].key + + +def test_a_member_of_the_next_version_sees_the_carried_boundary_log( + tmp_path: Path, +) -> None: + """The boundary carries the record forward, so its own members do see it.""" + registry, probe = build_registry() + run_probe(tmp_path / "run", graph=boundary_graph(), registry=registry, probe=probe) + record = only_record(observation(probe, "after_boundary")["mass_log"]) + assert record.reason == APPEND_REASON + + +def test_a_restored_boundary_log_is_the_one_a_cold_member_sees( + tmp_path: Path, +) -> None: + """The boundary survives the store round trip, not only the computed frame.""" + registry, probe = build_registry() + _, _, sources, _ = run_probe( + tmp_path / "run", graph=boundary_graph(), registry=registry, probe=probe + ) + expected = observation(probe, "after_boundary") + + later = probe_node( + "later", columns=("age",), target="later_a", population="boundary" + ) + warm_registry, warm_probe = build_registry() + warm = run_graph( + compile_graph( + Graph("toy", (toy.SOURCE,), (*boundary_graph().nodes, later)), + ), + sources=dict(sources), + store=ContentStore(tmp_path / "run" / "store"), + kernels=warm_registry, + ) + assert {n for n, r in warm.nodes.items() if not r.hit} == {"later"} + assert observation(warm_probe, "later")["mass_log"] == expected["mass_log"] + + +def test_an_unrelated_appender_changes_neither_the_key_nor_the_view( + tmp_path: Path, +) -> None: + """Same key, same truth: cold with the sibling equals cold without it. + + This is the property the boundary rule exists for. ``unrelated``'s + output column *is* its mass-log view, so equal stored bytes mean it was + shown the same log in both graphs — and its key is the same, so the + store would serve either result for the other. + """ + with_manifest, with_probe, sources, with_store = run_probe( + tmp_path / "with", graph=sibling_graph() + ) + without_manifest, without_probe, _, without_store = run_probe( + tmp_path / "without", graph=sibling_graph(appender=False), sources=sources + ) + assert ( + with_manifest.nodes["unrelated"].key == without_manifest.nodes["unrelated"].key + ) + assert observation(with_probe, "unrelated")["mass_log"] == () + assert observation(without_probe, "unrelated")["mass_log"] == () + assert unrelated_bytes(with_store, with_manifest) == unrelated_bytes( + without_store, without_manifest + ) + + +def test_required_replay_agrees_when_the_sibling_is_added(tmp_path: Path) -> None: + """A store written with the appender serves the graph without it.""" + _, _, sources, _ = run_probe(tmp_path / "run", graph=sibling_graph()) + registry, probe = build_registry() + warm = run_graph( + compile_graph(sibling_graph(appender=False)), + sources=dict(sources), + store=ContentStore(tmp_path / "run" / "store"), + kernels=registry, + resume="require", + ) + assert all(receipt.hit for receipt in warm.nodes.values()) + assert probe.seen == [] + + +def test_required_replay_agrees_when_the_sibling_is_reparameterized( + tmp_path: Path, +) -> None: + """Re-stating the appender's reason moves its key, and no other node's.""" + first_manifest, _, sources, _ = run_probe(tmp_path / "run", graph=sibling_graph()) + registry, probe = build_registry() + second = run_graph( + compile_graph(sibling_graph(reason="a different stated reason")), + sources=dict(sources), + store=ContentStore(tmp_path / "run" / "store"), + kernels=registry, + ) + assert second.nodes["appender"].key != first_manifest.nodes["appender"].key + assert second.nodes["unrelated"].key == first_manifest.nodes["unrelated"].key + assert {n for n, r in second.nodes.items() if not r.hit} == {"appender"} + assert [item["node"] for item in probe.seen] == ["appender"] # ---------------------------------------------------------------------- From f80ea7f02f9ceae730032243f1d304619757ca4d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 09:29:10 -0400 Subject: [PATCH 08/44] F3/F4: detach the frame view, and check it for mutation (amendment 26) `Frame` deeply freezes its metadata and its mass records, but a frozen dataclass still yields to `object.__setattr__`. Passing the version's own `_FrozenMapping` leaves and `MassChangeRecord`s by reference therefore made every kernel -- and anything retaining a context past its own mutation check -- a live handle on the population: one `object.__setattr__` on a shared record rewrote the version's log, and unlike a table nothing would have noticed. The projection now hands out a deep copy of the metadata and rebuilt mass records, through the same rule `_observer_snapshot` already followed (now one shared `_detached_record`). `_context_digest` binds all three fields: the metadata through the frame format's own store codec, each mass record field by field, and the projected column order. The codec is guarded, so a value `Frame` would never have admitted is reported as the mutation it is rather than raising while the comparison is computed. The properties cover both halves, and are not vacuous: the rewrite is asserted to have landed on the retained copy before the live version and the next node are asserted unchanged. The observer property now rewrites nested metadata and mass records too -- rewriting tables alone never touched these fields. Co-Authored-By: Claude Opus 5 (cherry picked from commit 6ec46d62c33ed89d8f9923326ea10b43070d5468) --- ...red-graph-contracts-frame-context.added.md | 2 +- docs/graph-acceptance.md | 15 ++ docs/graph-interface.lock | 2 +- .../src/microcosm/graph/executor.py | 94 +++++-- .../src/microcosm/graph/kernel.py | 21 +- .../shared/test_graph_frame_context.py | 254 +++++++++++++++++- 6 files changed, 355 insertions(+), 33 deletions(-) diff --git a/changelog.d/uk-shared-graph-contracts-frame-context.added.md b/changelog.d/uk-shared-graph-contracts-frame-context.added.md index a66fead06..b70127ecc 100644 --- a/changelog.d/uk-shared-graph-contracts-frame-context.added.md +++ b/changelog.d/uk-shared-graph-contracts-frame-context.added.md @@ -1 +1 @@ -Added `KernelContext.frame_metadata`, `frame_mass_log` and `frame_column_order`, so a kernel can reconstruct its population version's declared slices without seeing any column it did not declare. The mass log a node receives is the one its key binds: its version's structural boundary for an ordinary node, the base version's cumulative log for a structural one (graph amendment 26). +Added `KernelContext.frame_metadata`, `frame_mass_log` and `frame_column_order`, so a kernel can reconstruct its population version's declared slices without seeing any column it did not declare. The mass log a node receives is the one its key binds: its version's structural boundary for an ordinary node, the base version's cumulative log for a structural one. All three are detached from the live population before a kernel sees them, and all three enter the executor's input-mutation check (graph amendment 26). diff --git a/docs/graph-acceptance.md b/docs/graph-acceptance.md index bec9a1d99..9fd3a9367 100644 --- a/docs/graph-acceptance.md +++ b/docs/graph-acceptance.md @@ -611,6 +611,21 @@ lock unchanged: was not: a column *name* is itself information about the version, and B1's "nothing else is visible" covers names as well as values. + All three are detached before a kernel sees them and are covered by + B4's before/after mutation check. Detachment is not redundant with + `Frame`'s freezing: a frozen dataclass still yields to + `object.__setattr__`, so passing the version's own `_FrozenMapping` + leaves and `MassChangeRecord`s by reference would make every kernel — and + anything that retains a context past its own mutation check — a live + handle on the population. The executor therefore hands out a deep copy + of the metadata and rebuilt mass records (the rule `_observer_snapshot` + already followed, now shared), and `_context_digest` binds the metadata + through the frame format's own store codec, the mass records field by + field, and the projected column order. Neither is a substitute for the + other: the digest catches a kernel whose output stops being a function + of its declared inputs, while detachment is what stops a retained view + from rewriting the live version after that check has passed. + The three ride after `artifacts` and before `tolerances`, so amendment 17's statement that `numerics` rides at the end of the context stays literally true and amendment 19's that `artifacts` rides before the diff --git a/docs/graph-interface.lock b/docs/graph-interface.lock index cb01bf926..578315abd 100644 --- a/docs/graph-interface.lock +++ b/docs/graph-interface.lock @@ -1,2 +1,2 @@ e78c7359b48813c57eff6f697359682753b74f86f565f4e6974421e5a4048933 decl.py -a49bb4461e7928118b05c13351a99b7585e5068a0868f6bc40634ebb057f29ca kernel.py +2df6cc5b5b396adae578f885aedbd038d2c0e51f56b4a31b64c2f6b69e1b918d kernel.py diff --git a/packages/microcosm-graph/src/microcosm/graph/executor.py b/packages/microcosm-graph/src/microcosm/graph/executor.py index 614352d67..ff7f9ba19 100644 --- a/packages/microcosm-graph/src/microcosm/graph/executor.py +++ b/packages/microcosm-graph/src/microcosm/graph/executor.py @@ -79,6 +79,7 @@ StoreCorrupt, StoreMiss, StoreUnavailable, + _encode_frame_metadata, ) __all__ = ["NodeRejected", "NodeRejectedError", "run_graph"] @@ -352,6 +353,25 @@ def _freeze_frame(table: pd.DataFrame) -> pd.DataFrame: return frozen +def _detached_record(record): + """Rebuild one frozen record with independently copied field values. + + Reconstructed through the dataclass constructor rather than by a + reflective copy of its namespace, because source identities may seal + these classes. A frozen dataclass is not immune to ``object.__setattr__``, + so a record handed out by reference is a live handle on whatever holds + it; every hand-out goes through here. + """ + + return replace( + record, + **{ + field.name: deepcopy(getattr(record, field.name)) + for field in fields(record) + }, + ) + + def _observer_snapshot(population: Population) -> Population: """Detach every observation from the executable population and its cache. @@ -370,22 +390,12 @@ def _observer_snapshot(population: Population) -> Population: tables = {name: frame.table(name) for name in frame.entities} tables.update({name: frame.link(name) for name in frame.links}) tables, strata = pickle.loads(pickle.dumps((tables, frame.strata), protocol=5)) - - def copied_record(record): - return replace( - record, - **{ - field.name: deepcopy(getattr(record, field.name)) - for field in fields(record) - }, - ) - snapshot = Frame( tables, replace( frame.schema, group_entities=deepcopy(frame.schema.group_entities), - links=tuple(copied_record(link) for link in frame.schema.links), + links=tuple(_detached_record(link) for link in frame.schema.links), ), { entity: Weights( @@ -394,7 +404,9 @@ def copied_record(record): for entity in frame.weighted_entities }, strata, - mass_log=tuple(copied_record(record) for record in frame.mass_log), + mass_log=tuple(_detached_record(record) for record in frame.mass_log), + # ``Frame.__init__`` re-freezes metadata, which rebuilds every nested + # mapping, so the snapshot shares no metadata object with its parent. metadata=frame.metadata, ) return Population( @@ -402,7 +414,9 @@ def copied_record(record): population.version, dict(population.owners), dict(population.weight_kind), - mass_ledger=tuple(copied_record(record) for record in population.mass_ledger), + mass_ledger=tuple( + _detached_record(record) for record in population.mass_ledger + ), design_weights=population.design_weights, ) @@ -567,9 +581,54 @@ def _context_digest(context: KernelContext) -> bytes: ) digest.update(len(value.payload).to_bytes(8, "little")) digest.update(value.payload) + # The amendment-26 frame fields are kernel inputs like any other, so B4's + # before/after comparison covers them too. They are handed out detached + # (`_project_context`), so a kernel that rewrites one cannot reach the + # live version -- but it can still make its own node's output a function + # of something other than what it was given, and that is what this + # catches. + digest.update(b"frame-metadata\0") + digest.update(_frame_metadata_payload(context.frame_metadata)) + digest.update(b"frame-column-order\0") + for entity, columns in context.frame_column_order.items(): + _update_scalar(digest, entity) + for column in columns: + _update_scalar(digest, column) + digest.update(b"frame-mass-log\0") + for record in context.frame_mass_log: + for value in ( + getattr(record, "entity", None), + getattr(record, "old_total", None), + getattr(record, "new_total", None), + getattr(record, "declared_factor", None), + getattr(record, "reason", None), + ): + _update_scalar(digest, value) return digest.digest() +#: What a metadata tree digests to when the store codec cannot encode it. +#: Reached only after a kernel has replaced a value with something ``Frame`` +#: would never have admitted, which is itself the mutation being reported. +_UNCODABLE_FRAME_METADATA = b"" + + +def _frame_metadata_payload(metadata: Mapping[str, object]) -> bytes: + """Digest bytes for a frame-metadata view, without trusting its contents. + + ``store._encode_frame_metadata`` is the codec the frame format already + uses, so an unchanged view digests exactly as it persists. It is a codec + for *valid* metadata, though, and this runs a second time after a kernel + has held the object: it has to report a mutation rather than raise while + computing the comparison that would report it. + """ + + try: + return canonical_json(_encode_frame_metadata(metadata)) + except (TypeError, ValueError, RecursionError): + return _UNCODABLE_FRAME_METADATA + + def _structural_columns(frame: Frame, entity: str) -> list[str]: columns = [frame.schema.entity_id_column(entity)] if entity == frame.schema.person_entity: @@ -766,8 +825,13 @@ def _project_context( rng=np.random.default_rng(seed(key)), sources=MappingProxyType({name: sources[name] for name in node.sources}), artifacts={} if artifacts is None else artifacts, - frame_metadata=frame.metadata, - frame_mass_log=mass_log, + # Detached before the kernel sees them: a frozen dataclass still + # yields to ``object.__setattr__``, so passing the version's own + # ``_FrozenMapping`` leaves and mass records by reference would make + # every kernel -- and anything that retains a context past its own + # mutation check -- a live handle on the population (amendment 26). + frame_metadata=deepcopy(frame.metadata), + frame_mass_log=tuple(_detached_record(record) for record in mass_log), frame_column_order=MappingProxyType(column_order), tolerances=tolerances, numerics=numerics, diff --git a/packages/microcosm-graph/src/microcosm/graph/kernel.py b/packages/microcosm-graph/src/microcosm/graph/kernel.py index f718c5f48..a402e5b37 100644 --- a/packages/microcosm-graph/src/microcosm/graph/kernel.py +++ b/packages/microcosm-graph/src/microcosm/graph/kernel.py @@ -349,10 +349,12 @@ class KernelContext: nominal type does not itself verify serialized data (amendment 19). frame_metadata: The population version's own metadata. The - executor passes :attr:`microcosm.frame.Frame.metadata`, which - ``Frame`` has already deeply frozen; this class adds a - read-only view over that mapping and does not itself deep-freeze - a mapping built some other way (amendment 26). + executor passes a deep copy of + :attr:`microcosm.frame.Frame.metadata`, so the view is detached + from the live version as well as deeply frozen by ``Frame``; + this class adds a read-only view over that mapping and does not + itself deep-freeze or copy a mapping built some other way + (amendment 26). frame_mass_log: The ``Frame`` mass records this node's key binds, in order. For an ordinary node that is its population version's *boundary* log -- the log as that version was admitted -- so a @@ -365,15 +367,20 @@ class KernelContext: base and that version's members. A node that needs a stage's completed records therefore runs after that stage's structural boundary or reads its predecessor's evidence; incidental node - order is not authority (amendment 26). + order is not authority. The executor hands out rebuilt records, + not the version's own: a frozen dataclass still yields to + ``object.__setattr__``, so a record passed by reference would be + a live handle on the population (amendment 26). frame_column_order: Entity to the population version's own column order, restricted to the columns projected into ``tables``. The executor projects ``tables`` in declaration order, so this is the only way to reconstruct the version's layout. It never names a column the node did not get: an entry that is not exactly an ordering of that table's columns is refused, so an - undeclared column cannot be smuggled in as a name (amendment - 26). + undeclared column cannot be smuggled in as a name. All three + frame fields are ordinary inputs: the executor's before/after + comparison covers them, so rewriting one is refused exactly as + rewriting a table is (amendment 26). tolerances: ``(entity, column)`` of each declared input column to the :class:`Tolerance` its owning kernel declared, or ``None`` for a bitwise owner. A gate compares against these. diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py index 1ade82291..90fafc87f 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py @@ -17,6 +17,7 @@ import importlib.util import sys from pathlib import Path +from types import MappingProxyType import pandas as pd import pytest @@ -28,6 +29,7 @@ KernelContext, KernelResult, Node, + NodeRejectedError, Owned, Slice, StructuralDelta, @@ -50,10 +52,21 @@ SOURCE_REF = "source.metadata@1" PROBE_REF = "probe.frame_context@1" REWEIGHT_PROBE_REF = "probe.frame_context_reweight@1" +ISOLATION_REF = "probe.frame_context_isolation@1" -#: Metadata the CREATE kernel puts on the version, including a nested value -#: so the frozen projection is exercised rather than a flat string map. -VERSION_METADATA = {"time_period": "2024", "vintage": ("frs", "was")} +#: What a tampering probe or observer writes, so an assertion can tell +#: "the rewrite never happened" from "the rewrite never escaped". +TAMPERED = "tampered" + +#: Metadata the CREATE kernel puts on the version, including a nested +#: mapping and a nested sequence, so the frozen projection is exercised +#: rather than a flat string map -- and so a property can rewrite a value +#: one level below the mapping the context hands out. +VERSION_METADATA = { + "time_period": "2024", + "vintage": ("frs", "was"), + "provenance": {"source": "frs", "year": 2024}, +} class MetadataSource(toy.ToyKernel): @@ -159,6 +172,76 @@ def compute(self, context: KernelContext) -> KernelResult: ) +def rewrite(context: KernelContext, field: str) -> None: + """Rewrite one of the three frame fields of ``context``, in place. + + This is the adversary the amendment defends against, so it uses the + capability a kernel actually has rather than a public setter: a frozen + dataclass yields to ``object.__setattr__``, and the nested metadata + leaves are frozen dataclasses. The nested field is found through + ``dataclasses.fields`` rather than named, so the property survives a + rename inside ``microcosm.frame``. + """ + + if field == "metadata": + nested = context.frame_metadata["provenance"] + object.__setattr__( + nested, dataclasses.fields(nested)[0].name, (("source", TAMPERED),) + ) + elif field == "mass_record": + object.__setattr__(context.frame_mass_log[0], "reason", TAMPERED) + elif field == "column_order": + object.__setattr__( + context, + "frame_column_order", + MappingProxyType( + { + entity: tuple(reversed(columns)) + for entity, columns in context.frame_column_order.items() + } + ), + ) + else: # pragma: no cover - guards the fixture itself + raise AssertionError(f"unknown frame field {field!r}") + + +class IsolationProbe(toy.ToyKernel): + """Retains the frame view it is handed, and rewrites views on request. + + ``retain`` keeps this node's own context. ``tamper`` rewrites every + context retained so far, which happens *after* those nodes' mutation + checks have already passed -- at that point only detachment keeps the + live version intact. ``tamper_self`` rewrites one field of this node's + own context instead, which the before/after comparison has to refuse. + """ + + def __init__(self, ref: str, capabilities, *, variant: str = "base") -> None: + super().__init__(ref, capabilities, variant=variant) + self.retained: list[KernelContext] = [] + self.seen: list[dict[str, object]] = [] + + def compute(self, context: KernelContext) -> KernelResult: + if context.params.get("tamper"): + for held in self.retained: + rewrite(held, "metadata") + rewrite(held, "mass_record") + if context.params.get("retain"): + self.retained.append(context) + self.seen.append(observed(context)) + ids = pd.Index(context.tables["person"]["person_id"], name="person_id") + result = KernelResult( + columns={ + ("person", str(context.params["target"])): pd.Series( + 1.0, index=ids, dtype="float64" + ) + } + ) + tamper_self = context.params.get("tamper_self") + if tamper_self is not None: + rewrite(context, str(tamper_self)) + return result + + def build_registry() -> tuple[object, FrameContextProbe]: """The toy registry with the metadata source and both probes registered. @@ -170,6 +253,7 @@ def build_registry() -> tuple[object, FrameContextProbe]: probe = FrameContextProbe(PROBE_REF, toy._DETERMINISTIC) registry.register(probe) registry.register(ReweightProbe(REWEIGHT_PROBE_REF, toy._REWEIGHT)) + registry.register(IsolationProbe(ISOLATION_REF, toy._DETERMINISTIC)) return registry, probe @@ -633,6 +717,133 @@ def test_required_replay_agrees_when_the_sibling_is_reparameterized( assert [item["node"] for item in probe.seen] == ["appender"] +# ---------------------------------------------------------------------- +# Isolation: the fields are detached, and rewriting one is refused +# ---------------------------------------------------------------------- + + +def isolation_node( + node_id: str, + *, + target: str, + retain: bool = False, + tamper: bool = False, + tamper_self: str | None = None, +) -> Node: + """One ordinary member of the ``boundary`` version, with a tamper role.""" + return Node( + node_id, + ISOLATION_REF, + inputs=(Slice("person", ("age",)),), + outputs=(Owned("person", target, "float64"),), + params={ + "target": target, + "retain": retain, + "tamper": tamper, + "tamper_self": tamper_self, + }, + population="boundary", + ) + + +def isolation_graph(*nodes: Node) -> Graph: + """``boundary_graph``'s appender and boundary, then the given members. + + The members sit in the ``boundary`` version because that is the version + whose boundary log is non-empty: a property about rewriting a mass + record needs a node that was actually handed one. + """ + return Graph( + "toy", + (toy.SOURCE,), + ( + CREATE, + probe_node( + "appender", + columns=("age",), + target="appender_a", + mass_reason=APPEND_REASON, + ), + reweight_probe_node(), + *nodes, + ), + ) + + +def run_isolation(root: Path, *nodes: Node): + """Run ``isolation_graph`` and return ``(manifest, isolation probe)``.""" + registry, probe = build_registry() + manifest, _, _, _ = run_probe( + root, graph=isolation_graph(*nodes), registry=registry, probe=probe + ) + return manifest, registry.get(ISOLATION_REF) + + +def test_a_retained_context_cannot_rewrite_the_live_version(tmp_path: Path) -> None: + """Detachment, stated where the mutation check cannot reach. + + ``iso_b_tamper`` rewrites the nested metadata and the mass record of a + context ``iso_a_retain`` was handed, after that node finished and its + own before/after comparison passed. The rewrite lands -- the assertions + on the retained view prove the property is not vacuous -- and reaches + neither the node that runs next nor the version itself. + """ + manifest, isolation = run_isolation( + tmp_path / "run", + isolation_node("iso_a_retain", target="iso_a", retain=True), + isolation_node("iso_b_tamper", target="iso_b", tamper=True), + isolation_node("iso_c_reader", target="iso_c"), + ) + (held,) = isolation.retained + assert held.frame_mass_log[0].reason == TAMPERED + assert held.frame_metadata["provenance"]["source"] == TAMPERED + + reader = observation(isolation, "iso_c_reader") + assert only_record(reader["mass_log"]).reason == APPEND_REASON + assert reader["metadata"]["provenance"]["source"] == "frs" + for version in ("survey", "boundary"): + assert manifest.population(version).metadata["provenance"]["source"] == "frs" + assert only_record(manifest.population("boundary").mass_log).reason == APPEND_REASON + + +def test_rewriting_nested_metadata_is_refused(tmp_path: Path) -> None: + """A leaf one level below the mapping the context hands out.""" + with pytest.raises(NodeRejectedError, match="mutated its input context"): + run_isolation( + tmp_path / "run", + isolation_node("iso_tamper", target="iso_t", tamper_self="metadata"), + ) + + +def test_rewriting_a_mass_record_is_refused(tmp_path: Path) -> None: + """The record is the node's input, not a description of one.""" + with pytest.raises(NodeRejectedError, match="mutated its input context"): + run_isolation( + tmp_path / "run", + isolation_node("iso_tamper", target="iso_t", tamper_self="mass_record"), + ) + + +def test_rewriting_the_projected_column_order_is_refused(tmp_path: Path) -> None: + """Reversing an order is a different claim about the version's layout.""" + with pytest.raises(NodeRejectedError, match="mutated its input context"): + run_isolation( + tmp_path / "run", + isolation_node("iso_tamper", target="iso_t", tamper_self="column_order"), + ) + + +def test_an_untouched_frame_view_still_passes_the_mutation_check( + tmp_path: Path, +) -> None: + """The digest additions do not make an ordinary node look mutated.""" + manifest, isolation = run_isolation( + tmp_path / "run", isolation_node("iso_quiet", target="iso_q") + ) + assert [item["node"] for item in isolation.seen] == ["iso_quiet"] + assert manifest.nodes["iso_quiet"].hit is False + + # ---------------------------------------------------------------------- # Replay # ---------------------------------------------------------------------- @@ -689,11 +900,16 @@ def test_a_new_node_over_restored_populations_sees_the_same_frame( def test_a_retained_mutating_observer_changes_nothing(tmp_path: Path) -> None: """Amendment 24 still holds across the new fields. - The observer keeps every snapshot and rewrites its tables and its - metadata view after the callback returns; what later nodes read through - the frame fields, and the run's identity, are unchanged. + The observer keeps every snapshot and rewrites its tables, its nested + metadata *and* its mass records after the callback returns. Table + mutation alone would not touch the amendment-26 fields at all, so it is + the metadata and mass-record rewrites that make this property about + them; the run is over ``boundary_graph`` because that is the graph + whose snapshots carry a mass record to rewrite. """ - plain_manifest, plain_probe, sources, _ = run_probe(tmp_path / "plain") + plain_manifest, plain_probe, sources, _ = run_probe( + tmp_path / "plain", graph=boundary_graph() + ) retained: list[object] = [] def observe(node_id: str, population) -> None: @@ -702,13 +918,33 @@ def observe(node_id: str, population) -> None: table = population.frame.table(entity) for column in table.columns: table.loc[:, column] = table[column].iloc[0] + for record in population.frame.mass_log: + object.__setattr__(record, "reason", TAMPERED) + nested = population.frame.metadata["provenance"] + object.__setattr__( + nested, dataclasses.fields(nested)[0].name, (("source", TAMPERED),) + ) observed_manifest, observed_probe, _, _ = run_probe( - tmp_path / "observed", sources=sources, observer=observe + tmp_path / "observed", + graph=boundary_graph(), + sources=sources, + observer=observe, ) assert len(retained) == len(observed_manifest.nodes) assert {n: r.key for n, r in observed_manifest.nodes.items()} == { n: r.key for n, r in plain_manifest.nodes.items() } - for node_id in ("probe_first", "probe_second"): + for node_id in ("appender", "after_boundary"): assert observation(observed_probe, node_id) == observation(plain_probe, node_id) + + # The rewrites landed on the snapshots, and on nothing else. + rewritten = [record for snapshot in retained for record in snapshot.frame.mass_log] + assert rewritten and all(record.reason == TAMPERED for record in rewritten) + assert all( + snapshot.frame.metadata["provenance"]["source"] == TAMPERED + for snapshot in retained + ) + version = observed_manifest.population("boundary") + assert version.metadata["provenance"]["source"] == "frs" + assert only_record(version.mass_log).reason == APPEND_REASON From a9ec79d3b24d8c3c2ff76f1e89d1bb4f0d682521 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 09:54:10 -0400 Subject: [PATCH 09/44] Record the resumed fix round and what it re-verified Co-Authored-By: Claude Opus 5 (cherry picked from commit 09827aec81155bf889632789cd2dba123215e14c) --- PROGRESS-uk-shared-graph-contracts.md | 50 +++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/PROGRESS-uk-shared-graph-contracts.md b/PROGRESS-uk-shared-graph-contracts.md index 2e0519236..b6b8810aa 100644 --- a/PROGRESS-uk-shared-graph-contracts.md +++ b/PROGRESS-uk-shared-graph-contracts.md @@ -204,3 +204,53 @@ and the CI group inventory only. - F3/F4: detach the exposed frame objects; digest all three fields. - F2: source-backed rejection plus the anchor-invariance tests. - F5/F6: correct the docs' motivating claim and the disclosure. + +## Fix round, resumed (2026-09-13) + +The first fix-round process died on an external API DNS error after two +commits. Nothing was reset; the checkout resumed clean at `6ec46d62c`. +`origin/main` re-fetched on resume: still `15ebde806`, branch 8 ahead / 0 +behind. Restrictions unchanged — no pytest, import, engine, native source, +install, publication or disallowed log; source, stdlib `ast`, `ruff` and +the CI group inventory only. + +### Verified on resume, not assumed from the commit messages + +- `executor.py` boundary selection: `boundary_mass_logs[node.id]` is + written at exactly one place — beside `populations[node.id] = updated` + in the structural arm of the admission step (`executor.py:2669-2674`), + which both a cold run and a restored hit reach — and read at exactly one + place, the `StructuralDelta.NONE` arm that projects a context + (`executor.py:2508`). Its key set is therefore a subset of + `populations`', so the ordinary lookup cannot miss a version whose + incumbent lookup succeeded. +- Key binding re-read at source: `keys.py:206-209` binds an ordinary + node's `population_input` to `frame_key(version_key)` only, and + `keys.py:214-225` binds a structural node's `base` *and* `members`. The + projection matches that split exactly. +- `store.py:1157-1174`/`1334-1368` round-trip `Frame.mass_log`, so a + restored boundary carries the same records a computed one does. +- Detachment: `Frame.__init__` calls `_freeze_metadata` + (`bundle.py:112`), and `_freeze_metadata_value` rebuilds every nested + mapping, tuple and frozenset (`bundle.py:1382-1405`), so the + `_observer_snapshot` metadata hand-off shares no mutable-by-`setattr` + object with its parent. `_project_context` deep-copies the metadata and + rebuilds every record through `_detached_record`. +- `_context_digest` additions cannot raise while computing the comparison + that reports a mutation: `canonical_json` raises only `TypeError` / + `ValueError` (`canonical.py:_json_value`), `_encode_frame_metadata` + only `TypeError` (`store.py:1060-1082`), and both are caught alongside + `RecursionError`. +- `_project_context`'s only other caller, + `packages/microcosm-build/tests/test_uk_uc_capital_coherence.py:223`, + passes no `mass_log` and so takes the documented empty default. +- `ruff check .` clean; `ruff format --check` clean on every touched file; + stdlib `ast` parses clean. + +### Next + +- F4b: the required-replay axis property is still the weak one the review + named. Strengthen it. +- F2: source-backed rejection plus the anchor-invariance properties. +- F5/F6: the unsupported `calibrate.adam` motivating claim, the + transition-only wording, and an accurate receipts disclosure. From 38eee10e1ea48eeda91819490fbb4cac4e8a0bf3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 10:01:30 -0400 Subject: [PATCH 10/44] F2/F4/F5/F6: state what an update does to design ancestry, and prove it The review's F2 asked for a design-kind `WeightUpdate` to re-anchor `Population.design_weights`, on the grounds that a later calibration cap is otherwise "off by k" and that an `EXPAND` afterwards leaves anchors "mixed". Read against the source, the first is the declared semantics and the second is not what the code does. An anchor is the design weight a row entered the population carrying. `Population.design_weights` is captured once, at CREATE (`Population.from_frame`), and afterwards only carried by stable entity id (`_carry_design_weights`), which `patch` hands to `Population.from_frame` explicitly so the re-derive-from-the-frame default is never taken. `_carry_design_weights` maps an EXPAND's *copied* rows back to their source row's original anchor; only a row with no lineage at all reads the current frame, because it has no earlier weight to be anchored on. So the values move and no anchor does -- for retained, cloned and entrant rows alike -- which the new properties assert by running the same EXPAND over an updated and an un-updated population and comparing the anchors. `max_weight_ratio` with `weight_anchor='design'` is therefore still relative to the original design weights, which is what its refusal has always said ("original design weight"). Re-anchoring would let an unrelated normalization silently widen every cap declared upstream of it by that normalization's factor. The review's own T2 case is asserted as the intended outcome: calibrated weights equal to design weights doubled by an update are refused at a cap of 1.5 and admitted exactly at 2.0, reporting a realized ratio of 2.0. The `WeightUpdate` docstring and amendment 25 now say all of this instead of "ancestry is untouched". F4: the required-replay property only showed that a hit succeeds. The stored record's axis binding is now rewritten to the same ids in reverse, and to a short axis, and each is refused on a run that must hit the record with no kernel called -- so the check runs on the replay path and rejects there. A third property re-files the record unchanged and still replays, so the two refusals are about the binding rather than about re-filing. The weak test is renamed to the claim it does support. F5: amendment 25 claimed a re-solve of an existing calibration as a case it covers. `calibrate.adam@1` emits no `receipt['weight_update']`, so that declaration would be refused as unverifiable; it is now marked a future consumer adaptation, in the amendment and in the two docstrings that repeated it. F6: the node mass-policy mismatch message said "weight transition's" on a path both declarations reach. `decl.py` is re-locked for the amendment-25 text. Co-Authored-By: Claude Opus 5 (cherry picked from commit f5aa65d408b6c65be37e4b0e1a31dc85aa729c79) --- ...red-graph-contracts-weight-update.added.md | 2 +- docs/graph-acceptance.md | 34 ++- docs/graph-interface.lock | 2 +- .../src/microcosm/graph/decl.py | 29 ++- .../shared/test_graph_population.py | 238 ++++++++++++++++++ .../shared/test_graph_weight_update.py | 126 +++++++++- 6 files changed, 410 insertions(+), 21 deletions(-) diff --git a/changelog.d/uk-shared-graph-contracts-weight-update.added.md b/changelog.d/uk-shared-graph-contracts-weight-update.added.md index 2b5340f9d..76d148bf4 100644 --- a/changelog.d/uk-shared-graph-contracts-weight-update.added.md +++ b/changelog.d/uk-shared-graph-contracts-weight-update.added.md @@ -1 +1 @@ -Added `WeightUpdate`, a declared same-kind replacement of an entity's weight values, with `weight_update_receipt` binding the ordered entity axis the replacement values are positional against (graph amendment 25). +Added `WeightUpdate`, a declared same-kind replacement of an entity's weight values, with `weight_update_receipt` binding the ordered entity axis the replacement values are positional against, checked against the incumbent axis on cold execution and on every replay. An update replaces weight values only: design anchors, and so any `max_weight_ratio` declared against them, are unchanged (graph amendment 25). diff --git a/docs/graph-acceptance.md b/docs/graph-acceptance.md index 9fd3a9367..2304f3784 100644 --- a/docs/graph-acceptance.md +++ b/docs/graph-acceptance.md @@ -528,9 +528,9 @@ lock unchanged: 25. **A weight update that keeps its kind is declarable.** `WeightTransition` only ever moves a kind forward, so a stage that - recomputes weights it already holds — a sampling normalization, a - re-solve of an existing calibration — could not be declared at all, - and the only way to express it was to misdeclare a transition. + recomputes weights it already holds — a sampling normalization is the + case this was extracted for — could not be declared at all, and the + only way to express it was to misdeclare a transition. `WeightUpdate(entity, kind, reason, mass)` is that declaration and is deliberately narrower than a transition: the incumbent kind, the declared kind and the returned weights' kind must all be the same one; @@ -551,6 +551,34 @@ lock unchanged: re-enters the same function. A count mismatch, a missing binding and a binding against a different axis are each a rejection. + An update replaces weight *values* and does not re-anchor design + ancestry. `Population.design_weights` is captured once, at `CREATE` + (`Population.from_frame`), and afterwards only carried by stable entity + id (`_carry_design_weights`), which `patch` passes on explicitly so the + re-derive-from-the-frame default is never taken. A design-kind update + therefore leaves every existing row's anchor where it was; a later + `EXPAND`'s copied rows still inherit the anchor of the row they copy + rather than that row's current value; and `max_weight_ratio` with + `weight_anchor='design'` keeps the denominator it was written against — + the error text has always said "original design weight". A row admitted + with no ancestor is anchored on whatever design weight the `EXPAND` + installs for it, because it has no earlier weight to be anchored on; + that is the anchor definition applied to a row with no ancestry, not a + mixture. Re-anchoring on a same-kind update would instead let an + unrelated normalization silently widen every cap declared upstream of + it by that normalization's factor, which is a non-local change to an + already-declared contract. A stage that wants a cap against normalized + weights states the ratio it means. + + The shared calibration kernel does **not** consume this yet: + `calibrate.adam@1` emits no `receipt['weight_update']` + (`packages/microcosm-calibrate/src/microcosm/calibrate/kernels.py`), so + declaring a re-solve of an existing calibration as a `WeightUpdate` + would be refused by the axis check as unverifiable, and its own guard + still asks for a `WeightTransition` by name. Re-solving an existing + calibration through the shared kernel is a **future consumer + adaptation**, not a case this amendment already covers. + `WeightUpdate.to_kind` is a property, not a field, so the two declarations have disjoint field sets (`{entity, to_kind, mass}` and `{entity, kind, reason, mass}`) and a `WeightUpdate` can never diff --git a/docs/graph-interface.lock b/docs/graph-interface.lock index 578315abd..85920c480 100644 --- a/docs/graph-interface.lock +++ b/docs/graph-interface.lock @@ -1,2 +1,2 @@ -e78c7359b48813c57eff6f697359682753b74f86f565f4e6974421e5a4048933 decl.py +11c2abd77f50389c6cd0b51e26cb3ebd5cbd0770ba96e9de1b53c71e9725eaa4 decl.py 2df6cc5b5b396adae578f885aedbd038d2c0e51f56b4a31b64c2f6b69e1b918d kernel.py diff --git a/packages/microcosm-graph/src/microcosm/graph/decl.py b/packages/microcosm-graph/src/microcosm/graph/decl.py index f866be049..a443f7f9b 100644 --- a/packages/microcosm-graph/src/microcosm/graph/decl.py +++ b/packages/microcosm-graph/src/microcosm/graph/decl.py @@ -360,8 +360,8 @@ class WeightUpdate: :class:`WeightTransition` only ever moves a weight *kind* forward, so a stage that recomputes the numbers of weights it already holds — a - sampling normalization, a re-solve of an existing calibration — cannot - be declared at all. This is that declaration, and it is deliberately + sampling normalization is the case this was extracted for — cannot be + declared at all. This is that declaration, and it is deliberately narrower than a transition (amendment 25): - The kind does not move. The executor checks the incumbent kind, the @@ -378,8 +378,23 @@ class WeightUpdate: executor recomputes that binding from the incumbent axis, on cold execution and on replay. - Design-weight ancestry is untouched: the executor carries the original - design anchors exactly as it does for any other node. + An update replaces weight *values*; it does not re-anchor design + ancestry, and it is not a way to redefine a calibration cap. A row's + design anchor is the design weight it entered the population carrying: + ``Population.design_weights`` is captured once, at ``CREATE`` + (``population.Population.from_frame``), and afterwards only carried by + stable entity id (``population._carry_design_weights``), which ``patch`` + passes on explicitly so the re-derive-from-the-frame default is never + taken. So a design-kind update leaves every existing row's anchor + exactly where it was, a later ``EXPAND``'s copied rows still inherit + the anchor of the row they copy rather than that row's current value, + and ``max_weight_ratio`` with ``weight_anchor='design'`` keeps the + denominator it was written against. A row admitted with no ancestor is + anchored on the design weight the ``EXPAND`` installs for it, whatever + that is — it has no earlier weight to be anchored on. Re-anchoring here + instead would let an unrelated normalization silently widen every cap + declared upstream of it by that normalization's factor; a stage that + wants a cap against normalized weights states the ratio it means. Attributes: entity: The entity whose explicit weights are replaced. @@ -566,9 +581,11 @@ def __post_init__(self) -> None: "WeightTransition or WeightUpdate." ) if self.mass != self.weights.mass: + # Reached by a WeightTransition and a WeightUpdate alike, so + # the text names neither (amendment 25). raise GraphError( - f"Node {self.id!r}: mass policy {self.mass!r} disagrees with its " - f"weight transition's {self.weights.mass!r}." + f"Node {self.id!r}: mass policy {self.mass!r} disagrees with " + f"its declared weight change's {self.weights.mass!r}." ) declared_inputs = {(s.entity, c) for s in self.inputs for c in s.columns} for s in self.inputs: diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_population.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_population.py index 3735edda9..52c7fdfc1 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_population.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_population.py @@ -18,6 +18,7 @@ Slice, StructuralDelta, WeightTransition, + WeightUpdate, ) from microcosm.graph.kernel import KernelResult from microcosm.graph.population import ( @@ -36,6 +37,7 @@ weight_cap_receipt, ) from microcosm.graph.store import ContentStore, _encode_object_scalar +from microcosm.graph.weight_update import weight_update_receipt def _frame() -> Frame: @@ -1359,6 +1361,242 @@ def test_calibration_cap_stays_anchored_to_original_design_after_filter() -> Non ) +# ---------------------------------------------------------------------- +# Amendment 25: what a same-kind design update does to design ancestry +# ---------------------------------------------------------------------- + + +#: Households 10, 20 and 30 enter the toy population carrying these design +#: weights, so these are the anchors every cap below is stated against. +ORIGINAL_DESIGN = np.array([1.0, 2.0, 3.0]) + + +def _design_update_node(node_id: str = "normalize") -> Node: + """A REWEIGHT that replaces design weights without moving their kind.""" + return Node( + node_id, + "test@1", + structural=StructuralDelta.REWEIGHT, + base="source", + weights=WeightUpdate( + "household", + "design", + "normalize sampled source-family mass", + mass="declared", + ), + mass="declared", + ) + + +def _design_update_result(population: Population, *, factor: float) -> KernelResult: + """Scale every design weight by ``factor``, declaring the mass it moves. + + The declared totals are derived from the incumbent frame rather than + written down, because a uniform scaling multiplies every stratum's mass + by the same factor; a hand-copied number here would only be a second + chance to be wrong. + """ + frame = population.frame + before = frame.stratum_mass() + id_column = frame.schema.entity_id_column("household") + ids = frame.table("household")[id_column].tolist() + return KernelResult( + weights=Weights( + frame.weights_for("household").values * factor, WeightKind.DESIGN + ), + receipt={ + "weight_update": weight_update_receipt(ids), + "mass": { + "policy": "declared", + "before": float(before.sum()), + "after": float(before.sum()) * factor, + "stratum_before": {key: float(value) for key, value in before.items()}, + "stratum_after": { + key: float(value) * factor for key, value in before.items() + }, + }, + }, + ) + + +def _clone_and_entrant_expand_node(*, base: str) -> Node: + """An EXPAND that clones one household and admits one true entrant. + + The toy household table carries no data columns, so the entrant row + materializes nothing; ``expand_cells`` is empty exactly as it is for the + lineage EXPAND above. + """ + return Node( + "grow", + "test@1", + structural=StructuralDelta.EXPAND, + base=base, + params={ + "expand_cells": (), + "expand_weight_entity": "household", + "expand_weight_kind": "design", + }, + mass="free", + entrants=True, + ) + + +#: What the EXPAND declares for the household that has no ancestor. It is a +#: number the kernel states, not one derived from the incumbent weights, so +#: it is the same in every run below. +ENTRANT_DESIGN_WEIGHT = 7.0 + + +def _clone_and_entrant_expand_result(population: Population) -> KernelResult: + """Clone household 10 as 40 and admit 50 from nothing. + + The returned design weights are the incumbent ones with the clone's copy + of its source appended, then the entrant's declared weight. + """ + incumbent = population.frame.weights_for("household").values + return KernelResult( + expand={ + "person": pd.Series( + [], + index=pd.Index([], dtype="int64", name="person_id"), + dtype="int64", + ), + "household": pd.Series( + pd.array([10, pd.NA], dtype="Int64"), + index=pd.Index([40, 50], dtype="int64", name="household_id"), + ), + }, + weights=Weights( + np.array([*incumbent, incumbent[0], ENTRANT_DESIGN_WEIGHT]), + WeightKind.DESIGN, + ), + ) + + +def test_a_same_kind_design_update_leaves_the_original_anchors_invariant() -> None: + """New numbers, same kind, same ancestry. + + ``_apply_weight_update`` replaces the frame's design *values*; + ``_carry_design_weights`` carries ``Population.design_weights`` forward + by stable entity id, and ``patch`` passes the carried anchors to + ``Population.from_frame`` explicitly, so the default that would re-derive + anchors from the frame is never taken. An update is therefore not a + re-anchoring, and a cap declared against ``weight_anchor='design'`` + keeps the denominator it was written against. + """ + population = _population() + np.testing.assert_array_equal( + population.design_weights["household"], ORIGINAL_DESIGN + ) + + updated = patch( + population, _design_update_node(), _design_update_result(population, factor=2.0) + ) + + weights = updated.frame.weights_for("household") + assert weights.kind is WeightKind.DESIGN + np.testing.assert_array_equal(weights.values, ORIGINAL_DESIGN * 2.0) + np.testing.assert_array_equal(updated.design_weights["household"], ORIGINAL_DESIGN) + assert not updated.design_weights["household"].flags.writeable + + +def test_a_design_update_moves_no_anchor_for_retained_clone_or_entrant_rows() -> None: + """The same EXPAND over an updated and an un-updated population. + + Retained rows keep their own original anchor, a clone inherits the + anchor of the row it copies -- not that row's *current* design weight -- + and a row with no ancestor is anchored on the design weight the EXPAND + installs for it, which the kernel declares rather than derives. All + three agree across the two runs, so the update moved no anchor at all; + the frames' design weights are a factor apart, so the property is not + vacuous. + """ + plain = _population() + updated = patch( + plain, _design_update_node(), _design_update_result(plain, factor=2.0) + ) + np.testing.assert_array_equal( + updated.frame.weights_for("household").values, ORIGINAL_DESIGN * 2.0 + ) + + over_updated = patch( + updated, + _clone_and_entrant_expand_node(base="normalize"), + _clone_and_entrant_expand_result(updated), + ) + over_plain = patch( + plain, + _clone_and_entrant_expand_node(base="source"), + _clone_and_entrant_expand_result(plain), + ) + + expected = np.array([1.0, 2.0, 3.0, 1.0, ENTRANT_DESIGN_WEIGHT]) + np.testing.assert_array_equal(over_updated.design_weights["household"], expected) + np.testing.assert_array_equal(over_plain.design_weights["household"], expected) + np.testing.assert_array_equal( + over_updated.frame.table("household")["household_id"], + np.array([10, 20, 30, 40, 50]), + ) + # Not vacuous: the two versions' design *values* differ by the factor for + # every row that existed before the update, and the clone copies the + # updated value while inheriting the original anchor. + np.testing.assert_array_equal( + over_updated.frame.weights_for("household").values, + np.array([2.0, 4.0, 6.0, 2.0, ENTRANT_DESIGN_WEIGHT]), + ) + np.testing.assert_array_equal( + over_plain.frame.weights_for("household").values, + np.array([1.0, 2.0, 3.0, 1.0, ENTRANT_DESIGN_WEIGHT]), + ) + + +def test_a_design_update_does_not_move_the_calibration_cap_denominator() -> None: + """``max_weight_ratio`` stays relative to the original design weights. + + Calibrated weights equal to the *updated* design weights are twice the + original anchors, so a cap of 1.5 refuses them and a cap of 2.0 admits + them exactly at the limit. Re-anchoring on the update would instead make + the same numbers a ratio of 1.0, silently widening every cap declared + upstream of an unrelated normalization by that normalization's factor. + """ + population = _population() + updated = patch( + population, _design_update_node(), _design_update_result(population, factor=2.0) + ) + normalized = updated.frame.weights_for("household").values + + def calibrated_node(cap: float) -> Node: + return Node( + "calibrated", + "test@1", + structural=StructuralDelta.REWEIGHT, + base="normalize", + params={"max_weight_ratio": cap, "weight_anchor": "design"}, + weights=WeightTransition("household", "calibrated", mass="free"), + mass="free", + ) + + with pytest.raises(PopulationError, match="calibrated.*original design"): + patch( + updated, + calibrated_node(1.5), + KernelResult(weights=Weights(normalized, WeightKind.CALIBRATED)), + ) + + at_cap = calibrated_node(2.0) + accepted = patch( + updated, + at_cap, + KernelResult(weights=Weights(normalized, WeightKind.CALIBRATED)), + ) + np.testing.assert_array_equal(accepted.design_weights["household"], ORIGINAL_DESIGN) + assert weight_cap_receipt(accepted, at_cap) == { + "weight_anchor": "design", + "max_weight_ratio": 2.0, + "realized_max_weight_ratio": 2.0, + } + + def test_design_cap_fails_closed_when_source_has_no_design_lineage() -> None: frame = _frame() importance_frame = Frame( diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_weight_update.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_weight_update.py index 5fb48b2b8..319742715 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_weight_update.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_weight_update.py @@ -1,11 +1,15 @@ """Amendment 25: a same-kind weight update is declarable and axis-bound. ``WeightTransition`` only moves a kind forward, so a stage that recomputes -weights it already holds — a sampling normalization, a re-solve of an -existing calibration — had no declaration at all. These properties are -about the one that does: the kind cannot move, mass cannot be free, and -positional replacement values are refused unless the kernel binds the -ordered entity axis they were computed against. +weights it already holds — a sampling normalization is the case this was +extracted for — had no declaration at all. These properties are about the +one that does: the kind cannot move, mass cannot be free, and positional +replacement values are refused unless the kernel binds the ordered entity +axis they were computed against. + +What an update does *not* do is re-anchor design ancestry; those +properties live beside the other design-anchor ones in +``test_graph_population.py``. Everything here runs real shared graph operations over the invented toy country. No country model, engine, or build artifact is involved. @@ -38,6 +42,7 @@ weight_update_receipt, ) from microcosm.graph.canonical import normative +from microcosm.graph.executor import _cache_record_key from test_support.paths import paths_for _TEST_PATHS = paths_for("microcosm-graph") @@ -311,13 +316,15 @@ def test_update_refuses_a_short_axis(tmp_path: Path) -> None: # ---------------------------------------------------------------------- -def test_cold_then_required_replay_revalidates_the_axis(tmp_path: Path) -> None: - """A hit re-applies the REWEIGHT, so the axis is checked again. +def test_required_replay_reapplies_the_stored_update(tmp_path: Path) -> None: + """A hit re-applies the REWEIGHT rather than restoring its frame. The cached result is reconstructed with its stored weights and receipt - and passed back through the same application, which is why replay - needs no parallel rule. ``resume="require"`` proves the second run read - the store rather than recomputing. + and passed back through the same application, which is why replay needs + no parallel rule. ``resume="require"`` proves the second run read the + store rather than recomputing. That the axis check *rejects* on that + path is the next three properties; this one only establishes that the + path is taken and agrees. """ cold = run_update(tmp_path / "run") assert cold.misses() == set(cold.compiled.order) @@ -340,6 +347,105 @@ def test_cold_then_required_replay_revalidates_the_axis(tmp_path: Path) -> None: assert list(replayed.values) == list(original.values) +def household_axis(run: object) -> list[int]: + """The incumbent ``survey`` household axis the update is applied to.""" + household = run.manifest.population("survey").entity("household") + return household[toy.id_column("household")].tolist() + + +def rebind_cached_axis(run: object, ids: list[int]) -> dict[str, object]: + """Rewrite the stored update record's axis binding to name ``ids``. + + The cache record is filed under a key derived from the node key, not + from its own content, and its receipt body carries no digest of its + own (`_require_record_shape` validates the envelope and the execution + evidence), so this is a faithful stand-in for a record produced against + a different axis rather than a store forgery the loader would catch. + """ + key = run.keys()["update"] + record_key = _cache_record_key(key) + record = run.store.load_json(record_key) + receipt = dict(record["receipt"]) + receipt["weight_update"] = weight_update_receipt(ids) + record["receipt"] = receipt + run.store.put_json(record_key, record, node_key=key, verify_existing=False) + return record + + +def test_required_replay_refuses_a_cached_binding_against_another_axis( + tmp_path: Path, +) -> None: + """The stored binding is re-checked, not trusted, on a hit. + + The cold run's record is rewritten to bind the same household ids in + the reverse order — the one case a count check cannot catch — and the + replay is then required to hit that record. It is refused, and no + kernel runs, so the refusal came from re-applying the cached result + rather than from recomputing it. This is also where the check fires: + ``resume="require"``'s preflight validates record *shape*, so a foreign + axis surfaces at apply time, mid-run. + """ + cold = run_update(tmp_path / "run") + ids = household_axis(cold) + assert len(ids) > 1 + rebind_cached_axis(cold, list(reversed(ids))) + + registry = registry_with_update() + with pytest.raises(NodeRejectedError, match="different .household. axis"): + toy.run_toy( + update_graph(), + tmp_path / "run", + sources=cold.sources, + registry=registry, + store=ContentStore(tmp_path / "run" / "store"), + resume="require", + ) + assert toy.total_calls(registry) == 0 + + +def test_required_replay_refuses_a_cached_binding_of_the_wrong_length( + tmp_path: Path, +) -> None: + """A shortened axis is refused on replay for the same reason.""" + cold = run_update(tmp_path / "run") + rebind_cached_axis(cold, household_axis(cold)[:-1]) + + registry = registry_with_update() + with pytest.raises(NodeRejectedError, match="different .household. axis"): + toy.run_toy( + update_graph(), + tmp_path / "run", + sources=cold.sources, + registry=registry, + store=ContentStore(tmp_path / "run" / "store"), + resume="require", + ) + assert toy.total_calls(registry) == 0 + + +def test_an_untouched_cached_binding_still_replays(tmp_path: Path) -> None: + """The rewrite above is what fails, not the rewriting. + + The record is written back through the same call with the binding it + already had, so the two properties above cannot be passing because a + re-filed record is unreadable. + """ + cold = run_update(tmp_path / "run") + rebind_cached_axis(cold, household_axis(cold)) + + registry = registry_with_update() + warm = toy.run_toy( + update_graph(), + tmp_path / "run", + sources=cold.sources, + registry=registry, + store=ContentStore(tmp_path / "run" / "store"), + resume="require", + ) + assert warm.misses() == set() + assert toy.total_calls(registry) == 0 + + def test_reason_is_part_of_the_node_identity(tmp_path: Path) -> None: """Two updates that state different purposes are different nodes.""" first = run_update(tmp_path / "first") From c95f4c25f17a2d06180576fa4af8797c1eaf5216 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 10:02:17 -0400 Subject: [PATCH 11/44] Disclose the fix round's contract deltas and refresh the receipts identity Co-Authored-By: Claude Opus 5 (cherry picked from commit ecc98107ec191a353eeca1f9ce60cd5177095b4b) --- .../901-uk-shared-graph-contracts-receipts.md | 88 +++++++++++++++++-- 1 file changed, 82 insertions(+), 6 deletions(-) diff --git a/experiments/901-uk-shared-graph-contracts-receipts.md b/experiments/901-uk-shared-graph-contracts-receipts.md index 58e428d4a..3831c997b 100644 --- a/experiments/901-uk-shared-graph-contracts-receipts.md +++ b/experiments/901-uk-shared-graph-contracts-receipts.md @@ -80,19 +80,40 @@ moves a node key. `decl.py` is re-locked by 25, `kernel.py` by 26. ## Identity +Files as of the fix round (`f5aa65d40`). The four earlier rows this table +carried for `decl.py`, `executor.py` and the two new test files were the +pre-fix bytes and are superseded here. + | File | SHA-256 | | --- | --- | -| `decl.py` | `e78c7359b48813c57eff6f697359682753b74f86f565f4e6974421e5a4048933` | -| `kernel.py` | `51f45e899ba578a6b2324636a9879253266b067812f68b648eacfcd5a184d245` | +| `decl.py` | `11c2abd77f50389c6cd0b51e26cb3ebd5cbd0770ba96e9de1b53c71e9725eaa4` | +| `kernel.py` | `2df6cc5b5b396adae578f885aedbd038d2c0e51f56b4a31b64c2f6b69e1b918d` | | `weight_update.py` | `0ccfe6fcd257ef62b1f771b12eecc8ac5d447f5aa7d0403102e0ae290de16720` | | `population.py` | `33d1bb7bacea22870940288bf1907fb9eb24df7c245a216ff802e7fb41f5208f` | | `serialize.py` | `e5bf83c1082154f148626b6a36676614c6ff6c3fe0721aed94a1501da3021b1f` | -| `executor.py` | `ec4473bb033c4b1a36180c1518a42c755a46a2265d10461df1dfa00065862364` | +| `executor.py` | `5ab918e495fe4f8dd32c16155fe8c7a911e60e171cdbc8edb790626ce2d58c19` | | `explain.py` | `734a7b0e31692c31a99528cd83d9e74d3e508a317d913c1c169724a42d69d0de` | | `graph/__init__.py` | `697c59a37989a36124e6d43c7b07dd3b0582d965f97303c1fb02c88b41db2d48` | -| `tests/test_graph_weight_update.py` | `771ae0becbc57a4dd6198b9df229ec8c8262a5597647f335b7d5ed9be83471ff` | -| `tests/test_graph_frame_context.py` | `d58250d2194c81002be7282cd53597a7b9cff0906bb879b4b3926b4ac5b91ada` | -| `docs/graph-interface.lock` | `b42811ff0411dc179aaf9ddcf827aa270db866e08112cef8d1839df99c5a1f02` | +| `tests/test_graph_weight_update.py` | `6ff1296b1d769454b9664ca95bf084b91cf8bd05ada99ed58d54c8d4768751c7` | +| `tests/test_graph_frame_context.py` | `bcee6caccb7ffce47f04785557d246d2e8d5211d760ce5a344f3023cb1d3bf0c` | +| `tests/test_graph_population.py` | `b1aeabac04dbe6af8aeaf3b1691c7306a1442a142279043868500fdfe6476eec` | +| `tests/test_graph_kernel_contract.py` | `b13105cd302702eadcb30278545e794c16f3d819f571693810fb8f838473b126` | +| `tests/test_acceptance_b_ownership.py` | `a78ad49a341d08fe2a76e98e94d6dfe2fb60140ce5589101603b93f933f5bb34` | +| `docs/graph-interface.lock` | `b857403be2206156b844958cbd6abcb25ef951d05c0cc11e22554169a9343d2e` | + +### Every test file this lane touched + +An earlier draft of this receipt said the lane's test change was "isolated +to one file". That is true only of the **acceptance suite**, which the +charter assigns to the suite lane. In full: + +| File | Suite? | Change | +| --- | --- | --- | +| `test_graph_weight_update.py` | no | new (amendment 25) | +| `test_graph_frame_context.py` | no | new (amendment 26) | +| `test_graph_population.py` | no | three design-anchor properties added beside the existing ones (fix round, F2) | +| `test_graph_kernel_contract.py` | no | one assertion of *adjacency* relaxed to the ordering amendment 19 actually claims | +| `test_acceptance_b_ownership.py` | **yes** | B2's `KernelContext` field set, in its own commit (`895aabf19`), as amendment 19's was | The lock was re-recorded as part of each numbered amendment, never refreshed to make a test green: amendment 25 moved only the `decl.py` @@ -113,3 +134,58 @@ test files land in `fast/rest` and the engine lane beside their 27 sibling graph tests, neither `[defaulted]`). No pytest, no import of the production package, no engine, no install, no network beyond `gh` metadata and public blob reads. + +--- + +## Fix round (2026-09-13), after the independent adjudication + +An independent read-only review of `6f4ba4ec9` over `15ebde806` returned +REQUEST_CHANGES; its verbatim text is `FABLE-REVIEW.md` in this packet. +Root adjudicated. What changed in the contracts above: + +**Amendment 26, mass log (F1).** The projection handed every node +`population.frame.mass_log`, which for an ordinary node is its version's +*cumulative* log. An ordinary node's key binds only its version's +structural boundary and the owners of the columns it declared, so a +sibling that ran earlier in the same version and appended a record could +change what the node was shown without moving its key. `run_graph` now +records each version's log as that version is admitted — one line, which +cold execution and a restored hit both reach — and projects ordinary nodes +from that boundary. A structural node still receives the cumulative log, +which its key binds through `base` and `members`. + +**Amendment 26, isolation (F3/F4).** `Frame` deeply freezes its metadata +and its mass records, but a frozen dataclass still yields to +`object.__setattr__`, so passing those objects by reference made every +kernel a live handle on the population. The projection now hands out a +deep copy of the metadata and rebuilt records, through the rule +`_observer_snapshot` already followed (now the shared `_detached_record`), +and `_context_digest` binds all three fields — the metadata through the +frame format's own store codec, each mass record field by field, and the +projected column order. + +**Amendment 25, design ancestry (F2 — not accepted as stated).** The +review asked a design-kind update to re-anchor `Population.design_weights`. +Root refused: an anchor is the design weight a row entered carrying, it is +captured once at CREATE and afterwards only carried by stable entity id, +and `_carry_design_weights` maps an EXPAND's copied rows back to their +source's *original* anchor rather than its current value. Re-anchoring +would silently redefine the denominator of every `max_weight_ratio` +declared upstream of an unrelated normalization. The amendment and the +`WeightUpdate` docstring now state the anchor rule instead of "ancestry is +untouched", and `test_graph_population.py` asserts it. + +**Amendment 25, motivating claim (F5).** The amendment claimed a re-solve +of an existing calibration as a covered case. `calibrate.adam@1` emits no +`receipt['weight_update']`, so that declaration would be refused by the +axis check; the claim is now marked a future consumer adaptation. + +### Checks actually run in the fix round + +`ruff check .` (clean), `ruff format --check` on every touched file +(clean), stdlib `ast` parses of every touched Python file, and +`tools/ci_test_groups.py --verify` (`verification=ok`; the three touched +graph test files appear under `[fast]` and `[engine]`, none under +`[defaulted]`). Still no pytest, no import of the production package, no +engine, no install, no native source and no publication. `origin/main` +re-fetched on resume: `15ebde806`, nothing to merge. From 34fd21a70f6cdbb695d16520e4fb9c96364fde26 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 10:04:23 -0400 Subject: [PATCH 12/44] Copy the cloned household's members, as a copied group requires `_remapped_expand_memberships` aligns clone ordinals: a copied group needs the same number of copies of every incumbent member, so cloning household 10 without copying persons 1 and 2 would have been refused rather than proving anything about anchors. The clone now carries its members, which also lets the property assert the remapped memberships; the entrant household still joins with none, which is allowed. Co-Authored-By: Claude Opus 5 (cherry picked from commit c1a7920f72ff1ee5a9ea4d0430c9be9b9454e0df) --- .../shared/test_graph_population.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_population.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_population.py index 52c7fdfc1..7a79766c9 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_population.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_population.py @@ -1422,9 +1422,9 @@ def _design_update_result(population: Population, *, factor: float) -> KernelRes def _clone_and_entrant_expand_node(*, base: str) -> Node: """An EXPAND that clones one household and admits one true entrant. - The toy household table carries no data columns, so the entrant row - materializes nothing; ``expand_cells`` is empty exactly as it is for the - lineage EXPAND above. + Copied rows carry their source's storage, and the toy household table + has no column but its id, so nothing here has to be materialized; + ``expand_cells`` is empty exactly as it is for the lineage EXPAND above. """ return Node( "grow", @@ -1448,17 +1448,20 @@ def _clone_and_entrant_expand_node(*, base: str) -> Node: def _clone_and_entrant_expand_result(population: Population) -> KernelResult: - """Clone household 10 as 40 and admit 50 from nothing. + """Clone household 10 as 40, and admit household 50 from nothing. - The returned design weights are the incumbent ones with the clone's copy - of its source appended, then the entrant's declared weight. + Household 10's members are copied with it, because a copied group + requires the same number of copies of every incumbent member + (``_remapped_expand_memberships``); the entrant household joins with + none. The returned design weights are the incumbent ones, then the + clone's copy of its source's weight, then the entrant's declared one. """ incumbent = population.frame.weights_for("household").values return KernelResult( expand={ "person": pd.Series( - [], - index=pd.Index([], dtype="int64", name="person_id"), + [1, 2], + index=pd.Index([5, 6], dtype="int64", name="person_id"), dtype="int64", ), "household": pd.Series( @@ -1537,6 +1540,11 @@ def test_a_design_update_moves_no_anchor_for_retained_clone_or_entrant_rows() -> over_updated.frame.table("household")["household_id"], np.array([10, 20, 30, 40, 50]), ) + person = over_updated.frame.table("person") + np.testing.assert_array_equal(person["person_id"], np.array([1, 2, 3, 4, 5, 6])) + np.testing.assert_array_equal( + person["person_household_id"], np.array([10, 10, 20, 30, 40, 40]) + ) # Not vacuous: the two versions' design *values* differ by the factor for # every row that existed before the update, and the clone copies the # updated value while inheriting the original anchor. From 4bc5487d8cfb34b63f3a955323778616812c7376 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 10:05:32 -0400 Subject: [PATCH 13/44] Record the fix round's outcome, the F2 source reading and the runtime plan Co-Authored-By: Claude Opus 5 (cherry picked from commit 03b5cb8e2b0376a9badf1c9e4cc14cad60dd214d) --- PROGRESS-uk-shared-graph-contracts.md | 111 ++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/PROGRESS-uk-shared-graph-contracts.md b/PROGRESS-uk-shared-graph-contracts.md index b6b8810aa..436b5c399 100644 --- a/PROGRESS-uk-shared-graph-contracts.md +++ b/PROGRESS-uk-shared-graph-contracts.md @@ -120,6 +120,10 @@ leaves `decl.py` byte-identical. Either can be dropped without the other. explicitly ("the acceptance suite's B2 field set gains it in its own commit"). If root's owner disagrees, dropping that commit leaves B2 red and the rest intact. + *(2026-09-13, fix round: "isolated to one file" is true of the* + *acceptance suite only. `test_graph_kernel_contract.py` — not a suite* + *file — was also edited, and the fix round adds three properties to* + *`test_graph_population.py`. The full list is in the receipts.)* - Amendment 25 changes no node key; amendment 26 changes none either. Neither re-pins a spec digest. If a spec/seed digest moves in CI, that is main drift, not this lane (see `[[spec-engine-attested-modules]]`). @@ -127,6 +131,10 @@ leaves `decl.py` byte-identical. Either can be dropped without the other. three new fields. They are immutable views over an immutable `Frame`, so there is nothing for a kernel to mutate; stated here so the omission is a decision rather than an oversight. + *(2026-09-13, fix round: this reasoning was wrong and the decision is* + *reversed. A frozen dataclass still yields to `object.__setattr__`, so* + *the fields are now handed out detached and all three are digested —* + *commit `6ec46d62c`. Superseded; kept for the record.)* - No claim is made that any UK build, native lane, calibration or release passes. This lane read source only. @@ -254,3 +262,106 @@ the CI group inventory only. - F2: source-backed rejection plus the anchor-invariance properties. - F5/F6: the unsupported `calibrate.adam` motivating claim, the transition-only wording, and an accurate receipts disclosure. + +## Done (fix round) + +| Commit | What | +| --- | --- | +| `6104459f6` | File `FABLE-REVIEW.md` and open the fix round | +| `c0e275543` | F1: a node sees the mass log its own key binds | +| `6ec46d62c` | F3/F4: detach the frame view, and digest it for mutation | +| `09827aec8` | Record the resume and what it re-verified | +| `f5aa65d40` | F2/F4/F5/F6: design ancestry stated and proven; replay axis properties; the `calibrate.adam` claim withdrawn; `decl.py` re-locked | +| `c1a7920f7` | The clone fixture copies its household's members, as a copied group requires | +| `ecc98107e` | Receipts: fix-round deltas and refreshed identity | + +`decl.py` is now `11c2abd77f50389c6cd0b51e26cb3ebd5cbd0770ba96e9de1b53c71e9725eaa4` +and `kernel.py` `2df6cc5b5b396adae578f885aedbd038d2c0e51f56b4a31b64c2f6b69e1b918d`; +`docs/graph-interface.lock` matches both. `kernel.py` moved only in +amendment 26's commits, `decl.py` only in amendment 25's. + +## F2: why the re-anchoring was refused + +Read at source, not argued from the declaration text: + +1. `Population.design_weights` is set from the frame **once**, at CREATE + (`population.py:333-338` via `_create_population`, `executor.py:1461`). +2. Every later version gets its anchors from `_carry_design_weights` + (`population.py:2125-2181`), which aligns the incumbent anchors to the + new version by stable entity id. +3. `patch` passes those carried anchors to `Population.from_frame` + **explicitly** (`population.py:1159-1180`), so the `design_weights is + None` default that would re-derive them from the frame + (`population.py:333-338`) is never reached after CREATE. +4. `_apply_weight_update` (`population.py:2032-2040`) replaces the frame's + weight values. It touches `design_weights` not at all, and it runs + *before* step 2 in `patch`. +5. So a design-kind update moves no existing row's anchor. The EXPAND arm + of `_carry_design_weights` maps copied rows back to their **source + row's original anchor** (`population.py:2145-2157`, and the cached + twin at `941-981`), not to the source's current value, so a clone after + an update inherits the pre-update anchor too. Only a row with no + lineage reads `frame.weights_for(entity)` — and it has no earlier + weight to be anchored on. +6. The cap therefore stays what its own refusal has always called it: + `current > design * cap` against the **original** design weights + (`population.py:2333-2350`), with `realized_max_weight_ratio` reported + on the same denominator (`population.py:2354-2375`). + +The review's T2 case is real behaviour and is the intended outcome, now +asserted as such: calibrated weights equal to design weights an update +doubled are refused at `max_weight_ratio=1.5` and admitted exactly at +`2.0`, realized ratio `2.0`. Re-anchoring would make the same numbers a +ratio of `1.0` and would widen every cap declared upstream of an unrelated +normalization by that normalization's factor — a non-local change to an +already-declared contract, which is what the assignment forbade. + +No mixed-anchor defect beyond that definition was found, so nothing was +stopped. The one consequence worth naming, and now named in the +amendment: a row admitted *after* an update is anchored on whatever design +weight the EXPAND installs for it. If a kernel derives an entrant's design +weight from the updated incumbent values, that derived number is the +entrant's anchor — because it is the weight the row entered carrying. + +## Next (for root, before execution) — fix round + +Superseding the earlier plan's first block; the rest of that plan stands. + +``` +uv sync --all-packages --locked +uv run pytest packages/microcosm-graph/tests/test_graph_weight_update.py \ + packages/microcosm-graph/tests/test_graph_frame_context.py \ + packages/microcosm-graph/tests/test_graph_population.py +uv run pytest packages/microcosm-graph/tests/test_graph_interface_lock.py \ + packages/microcosm-graph/tests/test_acceptance_b_ownership.py \ + packages/microcosm-graph/tests/test_graph_kernel_contract.py \ + packages/microcosm-graph/tests/test_graph_executor.py \ + packages/microcosm-graph/tests/test_acceptance_replays.py \ + packages/microcosm-graph/tests/test_acceptance_d_weights.py +uv run pytest packages/microcosm-graph packages/microcosm-frame +uv run ruff check . +uv run python tools/ci_test_groups.py --verify +``` + +Where a source-only round could still be wrong, in the order worth +checking: + +1. `test_graph_population.py::test_a_design_update_moves_no_anchor_for_retained_clone_or_entrant_rows` + builds an EXPAND by hand. The clone-ordinal rule in + `_remapped_expand_memberships` already forced one correction here + (`c1a7920f7`); if another of `_patch_expand`'s guards fires, it will be + a `PopulationError` naming the guard, not a wrong anchor. +2. The three cached-axis properties in `test_graph_weight_update.py` + re-file a store record through `ContentStore.put_json(..., + verify_existing=False)`. That is the documented replace path, but it is + the one place these tests touch store internals; the third property + (re-file unchanged, still replays) exists to separate "the binding was + refused" from "the re-filed record was unreadable". +3. `_context_digest`'s metadata arm calls `store._encode_frame_metadata` + twice per node. It is guarded against a kernel-planted value the codec + cannot encode, but the guard's `except` list is reasoned from the + codec's source, not observed. +4. The four risks the first plan listed are unchanged, except that the + observer property named there now also rewrites nested metadata and + mass records, so a pandas copy-on-write no-op could no longer make the + whole property vacuous. From 722af66bde9dabafd4bcf5bfbcdf543bc9948aff Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 10:08:31 -0400 Subject: [PATCH 14/44] Write the fix-round result report Co-Authored-By: Claude Opus 5 (cherry picked from commit a7e61b98a04a5afe51c4e134947cfae1cbed20e6) --- FIX-RESULT.md | 352 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 352 insertions(+) create mode 100644 FIX-RESULT.md diff --git a/FIX-RESULT.md b/FIX-RESULT.md new file mode 100644 index 000000000..f7a0af8c8 --- /dev/null +++ b/FIX-RESULT.md @@ -0,0 +1,352 @@ +# Fix round result — UK-enabling shared graph contracts (amendments 25, 26) + +Source-only. **Runtime is UNTESTED.** No pytest, no import of the +production package, no engine, no country model, no native source, no +install, no gated data, no network beyond `git fetch`, no publication. The +checks that were run are stdlib `ast` parses, `ruff check` / `ruff format +--check`, `tools/ci_test_groups.py --verify`, and reading source. Every +behavioural claim below is a claim about what the source says, not about +an observed run; the bounded runtime plan root needs is at the end. + +## Identity + +| Thing | Value | +| --- | --- | +| Worktree | `_worktrees/microcosm-uk-shared-graph-contracts-20260913` | +| Branch | `uk-shared-graph-contracts-20260913` | +| Base | `origin/main` `15ebde806cd1a262363f7217fe535c7234ff757f` (re-fetched on resume; unchanged, 0 behind) | +| HEAD | `03b5cb8e2` | +| Reviewed head the adjudication ran on | `6f4ba4ec989eba93786b5d88033631ad253fdc5c` | + +### Commits, in order + +| Commit | Kind | What | +| --- | --- | --- | +| `97428cd9f` | journal | Lane baseline: absence proof, the #901 consumer read | +| `ef2dc69c3` | source+tests+docs | Amendment 25: `WeightUpdate`, `weight_update_receipt` | +| `f33d47cda` | source+tests+docs | Amendment 26: the three `KernelContext` frame fields | +| `895aabf19` | acceptance suite | B2's field set, isolated (as amendment 19's was) | +| `6f4ba4ec9` | journal+receipts | Lane receipts and the first runtime plan | +| `6104459f6` | journal | File `FABLE-REVIEW.md`; open the fix round | +| `c0e275543` | source+tests+docs | **F1** — a node sees the mass log its own key binds | +| `6ec46d62c` | source+tests+docs | **F3/F4** — detach the frame view; digest all three fields | +| `09827aec8` | journal | Record the resume and what it re-verified | +| `f5aa65d40` | source+tests+docs | **F2/F4/F5/F6** — design ancestry stated and proven; cached-axis replay properties; the `calibrate.adam` claim withdrawn; `decl.py` re-locked | +| `c1a7920f7` | tests | The clone fixture copies its household's members, as a copied group requires | +| `ecc98107e` | receipts | Fix-round contract deltas; refreshed identity; full test-file disclosure | +| `03b5cb8e2` | journal | Fix-round outcome, the F2 source reading, the runtime plan | + +The two commits the previous process had already landed (`c0e275543`, +`6ec46d62c`) were preserved and re-verified rather than trusted; what that +verification consisted of is in `PROGRESS-uk-shared-graph-contracts.md` +under "Verified on resume, not assumed from the commit messages". + +### File hashes at `03b5cb8e2` + +| File | SHA-256 | +| --- | --- | +| `graph/decl.py` | `11c2abd77f50389c6cd0b51e26cb3ebd5cbd0770ba96e9de1b53c71e9725eaa4` | +| `graph/kernel.py` | `2df6cc5b5b396adae578f885aedbd038d2c0e51f56b4a31b64c2f6b69e1b918d` | +| `graph/executor.py` | `5ab918e495fe4f8dd32c16155fe8c7a911e60e171cdbc8edb790626ce2d58c19` | +| `graph/population.py` | `33d1bb7bacea22870940288bf1907fb9eb24df7c245a216ff802e7fb41f5208f` | +| `graph/weight_update.py` | `0ccfe6fcd257ef62b1f771b12eecc8ac5d447f5aa7d0403102e0ae290de16720` | +| `graph/serialize.py` | `e5bf83c1082154f148626b6a36676614c6ff6c3fe0721aed94a1501da3021b1f` | +| `graph/explain.py` | `734a7b0e31692c31a99528cd83d9e74d3e508a317d913c1c169724a42d69d0de` | +| `graph/__init__.py` | `697c59a37989a36124e6d43c7b07dd3b0582d965f97303c1fb02c88b41db2d48` | +| `docs/graph-interface.lock` | `b857403be2206156b844958cbd6abcb25ef951d05c0cc11e22554169a9343d2e` | + +The lock file records `decl.py` and `kernel.py` at exactly the hashes +above; `test_graph_interface_lock.py` is what enforces that. The lock moved +twice in this lane and both times for a numbered amendment: +`kernel.py` in amendment 26's commits (`f33d47cda`, then the F1 and F3/F4 +docstring edits), `decl.py` in amendment 25's (`ef2dc69c3`, then +`f5aa65d40`'s semantics correction). It was never refreshed to make a test +green. + +### Test files touched, with their roles + +| File | Acceptance suite? | Change | +| --- | --- | --- | +| `test_graph_weight_update.py` | no | new (amendment 25); fix round adds three cached-axis replay properties and renames the weak one | +| `test_graph_frame_context.py` | no | new (amendment 26); fix round adds the boundary-log and isolation properties | +| `test_graph_population.py` | no | fix round adds three design-anchor properties beside the existing ones | +| `test_graph_kernel_contract.py` | no | one assertion of field *adjacency* relaxed to the ordering amendment 19 actually claims | +| `test_acceptance_b_ownership.py` | **yes** | B2's `KernelContext` field set, in its own commit `895aabf19` | + +The lane's earlier "isolated to one test file" claim was true of the +acceptance suite only. That is now stated accurately in the receipts and +historicized in the journal. + +--- + +## Point-by-point response to the adjudication + +### F1 — `frame_mass_log` leaked a same-version sibling's record. **Accepted; fixed.** (`c0e275543`) + +The finding was correct. `_project_context` passed +`population.frame.mass_log`, which for an ordinary node is the version's +cumulative log (`executor.py:2669` rewrites the version entry after every +ordinary member, and `_append_frame_mass_log` runs for ordinary nodes). +An ordinary node's key binds `frame_key(version_key)` and the owners of +the columns it declared (`keys.py:206-209`, `keys.py:181-201`) — never a +sibling — so a cache hit could replay output computed against a different +log. + +The fix is the reviewer's own minimal option. `run_graph` records +`boundary_mass_logs[node.id] = updated.frame.mass_log` where a structural +version is admitted (`executor.py:2674`), beside +`populations[node.id] = updated`, which both cold execution and a restored +hit reach; an ordinary node is projected from +`boundary_mass_logs[compiled.versions[node_id]]` (`executor.py:2511`), a +structural node from `incumbent.frame.mass_log`, whose key binds the base +*and* every ordinary member through `members` (`keys.py:214-225`). The two +maps are written at the same point, so the boundary lookup cannot miss a +version whose incumbent lookup (`executor.py:2413`, which runs first) +succeeded. + +`test_mass_log_is_the_incoming_log` was replaced, as the review said it +had to be. The properties now distinguish the two cases: an unread +same-version appender is invisible to an ordinary member and moves neither +its key nor its stored bytes; cold-with-sibling equals cold-without and +each replays into the other's store under `resume="require"`; the same +appender *is* visible to the structural boundary, whose key it moves. The +ancestor-tracking alternative was not taken: it would make a node's input a +function of graph topology the key does not bind either. + +### F2 — a design-kind `WeightUpdate` should re-anchor. **Not accepted.** Source-backed response. + +The premise that anchors become "stale and mixed" is not what the source +does, and the proposed fix would redefine a declared contract. + +1. **Anchors are captured once, at CREATE.** + `_create_population` calls `Population.from_frame(frame, node.id)` + (`executor.py:1461`), whose `design_weights is None` default reads the + frame's design weights (`population.py:333-338`). That default is + reached exactly once per graph. +2. **Afterwards they are only carried.** Every later version gets + `design_weights=_carry_design_weights(...)` passed **explicitly** + (`population.py:1159`, `population.py:1180`), so the re-derive default + never runs again. +3. **An update does not touch them.** `_apply_weight_update` + (`population.py:2032-2040`) returns a frame with replaced weight values + and nothing else, and it runs *before* the carry in `patch`. +4. **Clones are not "mixed".** The review's EXPAND case says entrants + anchor from updated values "while retained rows keep original anchors". + The code separates three cases, not two: + `_carry_design_weights` maps an EXPAND's **copied** rows back to their + source row's *original* anchor (`population.py:2145-2157`; the cached + twin at `941-981` does the same), retained rows keep their own, and only + a row with **no lineage at all** reads `frame.weights_for(entity)` — + because it has no earlier weight to be anchored on. The assignment's + framing is exactly the code's. +5. **The cap denominator is the original by declaration, not by accident.** + `_assert_design_weight_cap` compares `current > design * cap` against + the carried anchors (`population.py:2333-2350`) and its refusal has + always read "above N * original design weight"; + `realized_max_weight_ratio` uses the same denominator + (`population.py:2354-2375`), under a parameter the node must spell + `weight_anchor='design'`. +6. **Re-anchoring would be the silent change.** It would let a + normalization node inserted anywhere upstream widen every already- + declared `max_weight_ratio` by its own factor — a non-local + redefinition of a contract other nodes wrote against. + +The review's T2 case is real behaviour, and it is now asserted as the +intended outcome rather than left implicit: with design weights doubled by +an update, calibrated weights equal to them are refused at +`max_weight_ratio=1.5` and admitted exactly at `2.0`, reporting a realized +ratio of `2.0`. A stage that wants a cap against normalized weights states +the ratio it means. + +**No mixed-anchor defect beyond that definition was established**, so +nothing was stopped with a counterexample. One consequence is worth naming +and is now named in the amendment and the docstring: a row admitted +*after* an update is anchored on whatever design weight the `EXPAND` +installs for it. If a kernel derives an entrant's design weight from the +updated incumbent values, that derived number becomes the entrant's +anchor — because it is the weight the row entered carrying. That is the +anchor rule applied to a row with no ancestry, not a mixture of two rules. + +The review's alternative ("refuse `kind='design'` in +`WeightUpdate.__post_init__`") was also declined: the lane's recorded read +of #901 shows `uk.full.normalize` declaring +`WeightUpdate("household", weight_kind, ...)` with `weight_kind` a +variable, so refusing the design arm would refuse a consumer this +amendment exists for, to avoid a defect that is not there. + +What changed instead: `WeightUpdate`'s docstring no longer says "ancestry +is untouched" as a bare assertion but states the anchor rule and its three +cases; amendment 25 gains the same paragraph; and +`test_graph_population.py` gains three properties — +`test_a_same_kind_design_update_leaves_the_original_anchors_invariant`, +`test_a_design_update_moves_no_anchor_for_retained_clone_or_entrant_rows` +(the same EXPAND run over an updated and an un-updated population, with +the anchors asserted equal and the frames' design values asserted a factor +apart so it is not vacuous), and +`test_a_design_update_does_not_move_the_calibration_cap_denominator` (T2). + +### F3 — the three fields shared live objects. **Accepted; fixed.** (`6ec46d62c`) + +The review rated this non-blocking; root took it as blocking, because +"nothing writable through the public API" is not the property that matters +once a kernel holds the object: a frozen dataclass still yields to +`object.__setattr__`, and a shared `MassChangeRecord` is a live handle on +the version's log that, unlike a table, nothing would notice. + +`_project_context` now hands out `deepcopy(frame.metadata)` and records +rebuilt through `_detached_record`, the rule `_observer_snapshot` already +followed and now shares. `_context_digest` binds all three fields: the +metadata through `store._encode_frame_metadata` (the frame format's own +codec, so an unchanged view digests as it persists), each mass record +field by field, and the projected column order. The codec call is guarded — +a value `Frame` would never have admitted is reported as the mutation it +is rather than raising while the comparison that would report it is being +computed. + +Detachment and the digest are not redundant: the digest catches a kernel +whose output stops being a function of its declared inputs; detachment is +what stops a *retained* view from rewriting the live version after that +node's check has already passed. + +### F4 — two tests were weaker than their names. **Accepted; fixed.** (`6ec46d62c`, `f5aa65d40`) + +- The observer property now rewrites nested metadata and mass records as + well as tables, over the graph whose snapshots actually carry a mass + record, and asserts the rewrite landed on the snapshot before asserting + the live version and the next node are unchanged. +- `test_cold_then_required_replay_revalidates_the_axis` is renamed + `test_required_replay_reapplies_the_stored_update`, which is what it + proves. Three new properties carry the claim it did not: + `test_required_replay_refuses_a_cached_binding_against_another_axis` + rewrites the stored record's binding to the same household ids in + reverse order — the one case a count check cannot catch — and requires + the replay to hit that record; it is refused with `different 'household' + axis` and no kernel runs, so the refusal came from re-applying the + cached result. A second does the same with a short axis. A third re-files + the record *unchanged* and still replays, so the two refusals are about + the binding rather than about a record having been re-filed. The tests + also pin where the check fires: `resume="require"`'s preflight validates + record shape, so a foreign axis surfaces at apply time, mid-run — which + is the residual the review itself flagged. + +### F5 — the `calibrate.adam` motivating claim. **Accepted; withdrawn.** (`f5aa65d40`) + +Confirmed at source: `packages/microcosm-calibrate/src/` contains no +`weight_update` string at all, so `calibrate.adam@1` emits no +`receipt['weight_update']`, and `_apply_weight_update` would refuse the +declaration as unverifiable (`population.py:2025-2031`). Its own guard +also still asks for a `WeightTransition` by name. Amendment 25, the +`WeightUpdate` docstring and the test module docstring no longer offer +"a re-solve of an existing calibration" as a covered case; the amendment +now states plainly that re-solving through the shared kernel is a **future +consumer adaptation**. + +### F6 — cosmetic and disclosure. **Accepted.** (`f5aa65d40`, `ecc98107e`) + +The node mass-policy mismatch message said "weight transition's" on a path +both declarations reach; it now names neither. The receipts disclose every +test file touched and which one is the acceptance suite's, and the +journal's "isolated to one file" line is historicized in place rather than +edited away. + +--- + +## Bounded runtime plan for root + +Finite and invented-only: the toy country fixture under +`packages/microcosm-graph/tests/fixtures/toy_country/` and `tmp_path`. No +country model, engine, native source, gated microdata, credential or +network. Nothing below publishes, promotes or writes outside `tmp_path` +and the uv environment. + +``` +uv sync --all-packages --locked + +# 1. The new and changed properties. +uv run pytest packages/microcosm-graph/tests/test_graph_weight_update.py \ + packages/microcosm-graph/tests/test_graph_frame_context.py \ + packages/microcosm-graph/tests/test_graph_population.py + +# 2. Everything that reads the two frozen files, the executor, or replay. +uv run pytest packages/microcosm-graph/tests/test_graph_interface_lock.py \ + packages/microcosm-graph/tests/test_acceptance_b_ownership.py \ + packages/microcosm-graph/tests/test_graph_kernel_contract.py \ + packages/microcosm-graph/tests/test_graph_executor.py \ + packages/microcosm-graph/tests/test_graph_explain.py \ + packages/microcosm-graph/tests/test_graph_serialize.py \ + packages/microcosm-graph/tests/test_graph_decl.py \ + packages/microcosm-graph/tests/test_acceptance_replays.py \ + packages/microcosm-graph/tests/test_acceptance_d_weights.py + +# 3. Both shards whole. +uv run pytest packages/microcosm-graph packages/microcosm-frame + +# 4. Lint and the CI partition (both already pass here). +uv run ruff check . +uv run python tools/ci_test_groups.py --verify +``` + +### Helper dependencies each step needs + +| Step | Needs | Why | +| --- | --- | --- | +| 1 | `packages/microcosm-graph/tests/_toy.py`, `fixtures/toy_country/*.csv`, `schema.json` | every graph-level property runs the toy country; `test_graph_frame_context.py` also asserts the **person column order** from `person.csv`'s header, so a fixture column reorder is a genuine (and intended) failure there | +| 1 | `pandas`, `numpy` from the locked env | the anchor properties call `patch()` directly and build pandas lineage objects; no store, no executor | +| 1 | a writable `tmp_path` | the frame-context and weight-update properties build real `ContentStore`s under it | +| 2 | `docs/graph-interface.lock` | `test_graph_interface_lock.py` reads it from the repo, not the wheel | +| 3 | nothing further | | +| 4 | `tools/ci_test_groups.py` | partition authority; `--verify` must stay `verification=ok` | + +Expected: `test_graph_weight_update.py`'s 22 and +`test_graph_frame_context.py`'s 26 properties pass, `test_graph_population.py` +passes all 60 (57 of them pre-existing, 3 added this round), the +regression files stay green, `ruff check` and `--verify` stay clean, and no +node key moves anywhere (neither amendment adds a `Node` field or changes a canonical +projection, so no spec or seed digest should move; if one does, that is +main drift, not this lane). + +### Where a source-only round could still be wrong, worth checking in this order + +1. **`test_a_design_update_moves_no_anchor_for_retained_clone_or_entrant_rows` + builds an EXPAND by hand.** `_patch_expand`'s guards are many; one + already forced a correction in this round (`c1a7920f7` — a copied group + requires the same number of copies of every incumbent member, + `_remapped_expand_memberships`). A second guard firing would surface as + a `PopulationError` naming that guard, not as a wrong anchor. +2. **The three cached-axis properties re-file a store record** through + `ContentStore.put_json(..., verify_existing=False)`. That is the + documented replace path (`store._put` → + `_replace_write_only_collision`), and the record key is derived from the + node key rather than from content, so a rewritten receipt is not a + key/content mismatch. The third property (re-file unchanged, still + replays) is the control that separates "the binding was refused" from + "the re-filed record was unreadable". +3. **`_context_digest`'s metadata arm** calls `store._encode_frame_metadata` + twice per node. Its guard (`TypeError`, `ValueError`, `RecursionError`) + is reasoned from the codec's and `canonical_json`'s source, not + observed. It also costs one encode per node per side; metadata is small, + but that cost is real and unmeasured. +4. **`deepcopy(frame.metadata)` per node projection** is likewise + unmeasured. `Frame` metadata is stage-level, not row-level, so this is + expected to be negligible; it has not been timed. +5. **Pandas copy-on-write in the observer property.** The earlier plan + flagged that a table-only rewrite could make that property vacuous; it + now also rewrites nested metadata and mass records through + `object.__setattr__`, which copy-on-write does not affect, so the + property can no longer be vacuous in the way flagged — but the table arm + of it can still be. +6. **The mass-record probe** in `test_graph_frame_context.py` states an + unchanged household total from `context.weights["household"].values.sum()`; + if `_append_frame_mass_log`'s bracketing disagrees, it surfaces as a + `PopulationError` from the boundary-log properties. + +## What this round does not claim + +No UK build, calibration, native lane, release or publication was run, +prepared or authorized. No country model was imported. `#901` is +untouched: this lane adds no UK graph stage, kernel, target or gate, and +the two amendments remain separable in source — 25 touches `decl.py`, +`serialize.py`, `explain.py`, `population.py`, `graph/__init__.py` and the +new `weight_update.py`, leaving `kernel.py` byte-identical; 26 touches +`kernel.py` and `executor.py`, leaving `decl.py` byte-identical. Either can +be dropped without the other. From b38b355023be18827b6688d4bdcb79d6c7a1f76a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 11:04:22 -0400 Subject: [PATCH 15/44] Preserve graph context framing and original design ratios (cherry picked from commit c5c1fd567717dab66c925ceff2b0844df776b1cf) --- .../src/microcosm/graph/executor.py | 4 + .../src/microcosm/graph/explain.py | 38 ++++-- .../engine_free/shared/test_graph_explain.py | 116 ++++++++++++++++++ .../shared/test_graph_frame_context.py | 74 +++++++++++ .../shared/test_graph_population.py | 54 ++++++-- 5 files changed, 265 insertions(+), 21 deletions(-) diff --git a/packages/microcosm-graph/src/microcosm/graph/executor.py b/packages/microcosm-graph/src/microcosm/graph/executor.py index ff7f9ba19..310ec8dda 100644 --- a/packages/microcosm-graph/src/microcosm/graph/executor.py +++ b/packages/microcosm-graph/src/microcosm/graph/executor.py @@ -590,8 +590,12 @@ def _context_digest(context: KernelContext) -> bytes: digest.update(b"frame-metadata\0") digest.update(_frame_metadata_payload(context.frame_metadata)) digest.update(b"frame-column-order\0") + # Frame both levels: flattening entity names and columns lets a changed + # mapping reinterpret an entity name as a column without changing bytes. + digest.update(len(context.frame_column_order).to_bytes(8, "little")) for entity, columns in context.frame_column_order.items(): _update_scalar(digest, entity) + digest.update(len(columns).to_bytes(8, "little")) for column in columns: _update_scalar(digest, column) digest.update(b"frame-mass-log\0") diff --git a/packages/microcosm-graph/src/microcosm/graph/explain.py b/packages/microcosm-graph/src/microcosm/graph/explain.py index 5e1d51936..2a3b87d76 100644 --- a/packages/microcosm-graph/src/microcosm/graph/explain.py +++ b/packages/microcosm-graph/src/microcosm/graph/explain.py @@ -884,9 +884,11 @@ def _frame_ratios(frame, anchor, entity: str) -> list[float]: except (KeyError, ValueError): return [] design_by_id = dict(zip(design_ids, design_weights, strict=True)) + if any(entity_id not in design_by_id for entity_id in entity_ids): + return [] ratios = [] for entity_id, current in zip(entity_ids, values, strict=True): - design = float(design_by_id.get(entity_id, 0.0)) + design = float(design_by_id[entity_id]) current_value = float(current) if design > 0: ratios.append(current_value / design) @@ -902,6 +904,14 @@ def _population_ratios( manifest: RunManifest, node: Node, ) -> tuple[list[float], list[float]]: + """Infer ratios only when the manifest retains the original anchor. + + Manifest populations are Frames, not Populations with carried design + anchors. CREATE design weights suffice across updates and filters, but + an EXPAND needs copy/entrant ancestry not carried by those Frames. In + that case leave the fallback unavailable; explicit receipt samples + remain usable. A later design-kind update never becomes a new anchor. + """ if node.weights is None: return [], [] before = None if node.base is None else manifest.populations.get(node.base) @@ -912,17 +922,24 @@ def _population_ratios( anchor_node = node.base anchor = None while anchor_node is not None: - candidate = manifest.populations.get(anchor_node) - if candidate is not None and entity in candidate.weighted_entities: - weights = candidate.weights_for(entity) - if str(_value(weights.kind)) == "design": - anchor = candidate - break declaration = compiled.graph.node(anchor_node) + if declaration.structural is StructuralDelta.EXPAND: + return [], [] + if declaration.structural is StructuralDelta.CREATE: + candidate = manifest.populations.get(anchor_node) + if candidate is not None and entity in candidate.weighted_entities: + weights = candidate.weights_for(entity) + if str(_value(weights.kind)) == "design": + anchor = candidate + break anchor_node = declaration.base if anchor is None: return [], [] - return _frame_ratios(before, anchor, entity), _frame_ratios(after, anchor, entity) + before_ratios = _frame_ratios(before, anchor, entity) + after_ratios = _frame_ratios(after, anchor, entity) + if not before_ratios or not after_ratios: + return [], [] + return before_ratios, after_ratios def _numeric_sequence(value: object) -> list[float]: @@ -965,7 +982,10 @@ def _histogram_svg(before: Sequence[float], after: Sequence[float]) -> str: finite = [*finite_before, *finite_after] nonfinite = len(before) + len(after) - len(finite) if not finite: - return '

Weight-ratio samples are not present in this manifest.

' + return ( + '

Weight-ratio samples are not recorded or original ' + "design ancestry is unavailable in this manifest.

" + ) low, high = min(finite), max(finite) bin_count = min(12, max(4, int(math.sqrt(len(finite))))) if math.isclose(low, high): diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_explain.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_explain.py index 112bc5f92..05513c49b 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_explain.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_explain.py @@ -13,7 +13,9 @@ import pytest import microcosm.graph as graph_api +from microcosm.frame import WeightKind, Weights from microcosm.graph import describe, explain_html, graph_to_json +from microcosm.graph.explain import _population_ratios from test_support.paths import paths_for _TEST_PATHS = paths_for("microcosm-graph") @@ -243,6 +245,120 @@ def test_calibration_view_uses_targets_ratios_and_mass(explanation) -> None: assert "urban" in rendered +class _NormalizeDesign(toy.ToyKernel): + """A real same-kind update for the original-anchor explanation check.""" + + def compute(self, context): + before = context.weights["household"].values + after = before * 2.0 + return graph_api.KernelResult( + weights=Weights(after, WeightKind.DESIGN), + receipt={ + "mass": toy._mass_record(context, before, after, "declared"), + "weight_update": graph_api.weight_update_receipt( + context.tables["household"]["household_id"].tolist() + ), + }, + ) + + +@pytest.mark.parametrize("filtered", [False, True], ids=("same-axis", "filtered")) +def test_ratio_fallback_keeps_create_anchors_after_design_update_and_replay( + tmp_path: Path, filtered: bool +) -> None: + """The chart and the actual cap receipt both use the original denominator.""" + normalize = graph_api.Node( + "normalize", + "explain.normalize@1", + structural=graph_api.StructuralDelta.REWEIGHT, + base="survey", + inputs=( + graph_api.Slice("person", ("age",)), + graph_api.Slice("household", ("household_size",)), + ), + weights=graph_api.WeightUpdate( + "household", "design", "toy normalization", mass="declared" + ), + mass="declared", + ) + nodes = [toy.CREATE, normalize] + base = normalize.id + if filtered: + selection = toy.select_node("adults", base=base) + nodes.append(selection) + base = selection.id + calibrated = toy.reweight_node( + "calibrated", base=base, to_kind="calibrated", factor=1.0 + ) + calibrated = replace( + calibrated, + params={ + **dict(calibrated.params), + "max_weight_ratio": 2.0, + "weight_anchor": "design", + }, + ) + graph = graph_api.Graph("toy", (toy.SOURCE,), (*nodes, calibrated)) + + def registry(): + value = toy.toy_registry() + value.register(_NormalizeDesign(normalize.kernel, toy._REWEIGHT)) + return value + + cold = toy.run_toy(graph, tmp_path / "cold", registry=registry()) + replay = toy.run_toy( + graph, + tmp_path / "replay", + sources=cold.sources, + store=cold.store, + registry=registry(), + resume="require", + ) + assert all(receipt.hit for receipt in replay.manifest.nodes.values()) + assert toy.total_calls(replay.registry) == 0 + for run in (cold, replay): + before, after = _population_ratios(run.compiled, run.manifest, calibrated) + count = run.manifest.populations[calibrated.id].n("household") + assert before == [2.0] * count + assert after == [2.0] * count + receipt = run.manifest.nodes[calibrated.id].receipt + assert receipt["realized_max_weight_ratio"] == 2.0 + assert "weight_ratios" not in receipt + assert "Before (n=" in explain_html(run.compiled, run.manifest) + + +def test_ratio_fallback_declines_expand_ancestry_but_preserves_receipt_samples( + tmp_path: Path, +) -> None: + """Manifest Frames cannot reconstruct copied/entrant original anchors.""" + expand, claim = toy.entrant_expand_node() + calibrated = toy.reweight_node( + "calibrated", base=expand.id, to_kind="calibrated", factor=1.0 + ) + graph = graph_api.Graph( + "toy", (toy.SOURCE,), (toy.CREATE, expand, claim, calibrated) + ) + run = toy.run_toy(graph, tmp_path / "run") + + assert _population_ratios(run.compiled, run.manifest, calibrated) == ([], []) + rendered = explain_html(run.compiled, run.manifest) + assert "original design ancestry is unavailable" in rendered + original = run.manifest.nodes[calibrated.id] + updated = replace( + original, + receipt={ + **dict(original.receipt), + "weight_ratios": {"before": (1.0, 2.0), "after": (2.0, 3.0)}, + }, + ) + supplied = replace( + run.manifest, nodes={**dict(run.manifest.nodes), calibrated.id: updated} + ) + rendered = explain_html(run.compiled, supplied) + assert "Before (n=2)" in rendered and "After (n=2)" in rendered + assert "original design ancestry is unavailable" not in rendered + + def test_calibration_view_renders_partition_mass_with_deltas(tmp_path: Path) -> None: run = toy.run_toy(toy.full_graph(), tmp_path / "run") original = run.manifest.nodes["calibrated"] diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py index 90fafc87f..a26e9cad8 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py @@ -76,6 +76,11 @@ def compute(self, context: KernelContext) -> KernelResult: frame = toy.read_toy_frame(context.sources["survey"]) tables = {entity: frame.table(entity) for entity in frame.entities} tables.update({link: frame.link(link) for link in frame.links}) + mass_log = () + reason = context.params.get("source_mass_reason") + if reason is not None: + total = float(frame.weights_for("household").values.sum()) + mass_log = (MassChangeRecord("household", total, total, None, reason),) return KernelResult( frame=Frame( tables, @@ -85,6 +90,7 @@ def compute(self, context: KernelContext) -> KernelResult: for entity in frame.weighted_entities }, frame.strata, + mass_log=mass_log, metadata=VERSION_METADATA, ), receipt={"persons": frame.n("person")}, @@ -201,6 +207,28 @@ def rewrite(context: KernelContext, field: str) -> None: } ), ) + elif field in ("column_order_collapse", "column_order_regroup"): + (first_entity, first_columns), (second_entity, second_columns) = ( + context.frame_column_order.items() + ) + if field == "column_order_collapse": + changed = {first_entity: (*first_columns, second_entity, *second_columns)} + else: + # Keep two entries and the same flattened string sequence, but + # reinterpret the first entity's last column as the second entity. + changed = { + first_entity: first_columns[:-1], + first_columns[-1]: (second_entity, *second_columns), + } + assert changed != dict(context.frame_column_order) + assert [ + value for entity, columns in changed.items() for value in (entity, *columns) + ] == [ + value + for entity, columns in context.frame_column_order.items() + for value in (entity, *columns) + ] + object.__setattr__(context, "frame_column_order", MappingProxyType(changed)) else: # pragma: no cover - guards the fixture itself raise AssertionError(f"unknown frame field {field!r}") @@ -833,6 +861,28 @@ def test_rewriting_the_projected_column_order_is_refused(tmp_path: Path) -> None ) +def test_collapsing_entity_column_groups_is_refused(tmp_path: Path) -> None: + node = isolation_node( + "iso_collapse", target="iso_t", tamper_self="column_order_collapse" + ) + node = dataclasses.replace( + node, inputs=(*node.inputs, Slice("household", ("household_id",))) + ) + with pytest.raises(NodeRejectedError, match="mutated its input context"): + run_isolation(tmp_path / "run", node) + + +def test_moving_a_column_to_an_entity_boundary_is_refused(tmp_path: Path) -> None: + node = isolation_node( + "iso_regroup", target="iso_t", tamper_self="column_order_regroup" + ) + node = dataclasses.replace( + node, inputs=(*node.inputs, Slice("household", ("household_id",))) + ) + with pytest.raises(NodeRejectedError, match="mutated its input context"): + run_isolation(tmp_path / "run", node) + + def test_an_untouched_frame_view_still_passes_the_mutation_check( tmp_path: Path, ) -> None: @@ -869,6 +919,30 @@ def test_required_replay_is_a_full_hit(tmp_path: Path) -> None: } +def test_a_cached_create_preserves_its_nonempty_boundary_log(tmp_path: Path) -> None: + reason = "source weights reconciled before graph admission" + source = dataclasses.replace(CREATE, params={"source_mass_reason": reason}) + first = probe_node("first", columns=("age",), target="first_count") + graph = Graph("toy", (toy.SOURCE,), (source, first)) + cold, probe, sources, store = run_probe(tmp_path / "run", graph=graph) + expected = observation(probe, "first")["mass_log"] + assert len(expected) == 1 and expected[0].reason == reason + + registry, next_probe = build_registry() + second = probe_node("second", columns=("age",), target="second_count") + warm = run_graph( + compile_graph(Graph("toy", (toy.SOURCE,), (source, second))), + sources=dict(sources), + store=store, + kernels=registry, + ) + assert warm.nodes[source.id].hit is True + assert warm.nodes["second"].hit is False + assert warm.nodes[source.id].key == cold.nodes[source.id].key + assert observation(next_probe, "second")["mass_log"] == expected + assert warm.population(source.id).mass_log == expected + + def test_a_new_node_over_restored_populations_sees_the_same_frame( tmp_path: Path, ) -> None: diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_population.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_population.py index 7a79766c9..d6d6505c6 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_population.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_population.py @@ -1422,9 +1422,9 @@ def _design_update_result(population: Population, *, factor: float) -> KernelRes def _clone_and_entrant_expand_node(*, base: str) -> Node: """An EXPAND that clones one household and admits one true entrant. - Copied rows carry their source's storage, and the toy household table - has no column but its id, so nothing here has to be materialized; - ``expand_cells`` is empty exactly as it is for the lineage EXPAND above. + Both members of the copied household are cloned without reassignment. + A true entrant person has all carried cells and its stratum declared, + and joins the new entrant household. """ return Node( "grow", @@ -1432,7 +1432,13 @@ def _clone_and_entrant_expand_node(*, base: str) -> Node: structural=StructuralDelta.EXPAND, base=base, params={ - "expand_cells": (), + "expand_cells": ( + ("person", "person_household_id", "int64"), + ("person", "keep", "bool"), + ("person", "owned", "boolean"), + ("person", "nullable", "boolean"), + ("person", "amount", "float64"), + ), "expand_weight_entity": "household", "expand_weight_kind": "design", }, @@ -1452,27 +1458,50 @@ def _clone_and_entrant_expand_result(population: Population) -> KernelResult: Household 10's members are copied with it, because a copied group requires the same number of copies of every incumbent member - (``_remapped_expand_memberships``); the entrant household joins with - none. The returned design weights are the incumbent ones, then the - clone's copy of its source's weight, then the entrant's declared one. + (``_remapped_expand_memberships``); person 7 enters without ancestry + and belongs to household 50. The returned design weights are the + incumbent ones, the clone's source weight, then the entrant's weight. """ incumbent = population.frame.weights_for("household").values + person = population.frame.table("person") + ids = pd.Index([1, 2, 3, 4, 5, 6, 7], name="person_id") + entrant = { + "person_household_id": 50, + "keep": True, + "owned": False, + "nullable": pd.NA, + "amount": 4.0, + } + columns = { + ("person", column): pd.Series( + [*person[column], *person[column].iloc[:2], value], + index=ids, + dtype=person[column].dtype, + ) + for column, value in entrant.items() + } + columns[("person", "person_household_id")] = pd.Series( + [10, 10, 20, 30, 40, 40, 50], index=ids, dtype="int64" + ) return KernelResult( expand={ "person": pd.Series( - [1, 2], - index=pd.Index([5, 6], dtype="int64", name="person_id"), - dtype="int64", + pd.array([1, 2, pd.NA], dtype="Int64"), + index=pd.Index([5, 6, 7], dtype="int64", name="person_id"), ), "household": pd.Series( pd.array([10, pd.NA], dtype="Int64"), index=pd.Index([40, 50], dtype="int64", name="household_id"), ), }, + columns=columns, weights=Weights( np.array([*incumbent, incumbent[0], ENTRANT_DESIGN_WEIGHT]), WeightKind.DESIGN, ), + strata=pd.Series( + ["entrant"], index=pd.Index([7], name="person_id"), dtype=object + ), ) @@ -1541,10 +1570,11 @@ def test_a_design_update_moves_no_anchor_for_retained_clone_or_entrant_rows() -> np.array([10, 20, 30, 40, 50]), ) person = over_updated.frame.table("person") - np.testing.assert_array_equal(person["person_id"], np.array([1, 2, 3, 4, 5, 6])) + np.testing.assert_array_equal(person["person_id"], np.array([1, 2, 3, 4, 5, 6, 7])) np.testing.assert_array_equal( - person["person_household_id"], np.array([10, 10, 20, 30, 40, 40]) + person["person_household_id"], np.array([10, 10, 20, 30, 40, 40, 50]) ) + assert over_updated.frame.strata.iloc[-1] == "entrant" # Not vacuous: the two versions' design *values* differ by the factor for # every row that existed before the update, and the clone copies the # updated value while inheriting the original anchor. From 2e2c6c69ca6f45d31a9c56c00fa113172ddb012b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 11:12:20 -0400 Subject: [PATCH 16/44] Project a declared household field in context mutation tests (cherry picked from commit 5742c17cef27f02db40997b39d7ac7e8b4f8492e) --- .../tests/engine_free/shared/test_graph_frame_context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py index a26e9cad8..6e9043104 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py @@ -866,7 +866,7 @@ def test_collapsing_entity_column_groups_is_refused(tmp_path: Path) -> None: "iso_collapse", target="iso_t", tamper_self="column_order_collapse" ) node = dataclasses.replace( - node, inputs=(*node.inputs, Slice("household", ("household_id",))) + node, inputs=(*node.inputs, Slice("household", ("household_size",))) ) with pytest.raises(NodeRejectedError, match="mutated its input context"): run_isolation(tmp_path / "run", node) @@ -877,7 +877,7 @@ def test_moving_a_column_to_an_entity_boundary_is_refused(tmp_path: Path) -> Non "iso_regroup", target="iso_t", tamper_self="column_order_regroup" ) node = dataclasses.replace( - node, inputs=(*node.inputs, Slice("household", ("household_id",))) + node, inputs=(*node.inputs, Slice("household", ("household_size",))) ) with pytest.raises(NodeRejectedError, match="mutated its input context"): run_isolation(tmp_path / "run", node) From ed166858faad804fb7f13c82cba96294cabe74b0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 12:06:42 -0400 Subject: [PATCH 17/44] Record verified shared graph contract acceptance (cherry picked from commit 072c2e88b2a66782a351044a662d7e8ada2e9f61) --- FABLE-REVIEW.md | 5 ++ FIX-RESULT.md | 4 + PROGRESS-uk-shared-graph-contracts.md | 3 + .../901-uk-shared-graph-contracts-receipts.md | 3 + ...ared-graph-contract-acceptance-20260913.md | 73 +++++++++++++++++++ 5 files changed, 88 insertions(+) create mode 100644 experiments/uk-shared-graph-contract-acceptance-20260913.md diff --git a/FABLE-REVIEW.md b/FABLE-REVIEW.md index b5a4456bd..bebe6cef2 100644 --- a/FABLE-REVIEW.md +++ b/FABLE-REVIEW.md @@ -1,3 +1,8 @@ +> Historical review of the initial proposal, preserved verbatim. The findings +> were subsequently resolved or adjudicated. See the +> [13 September acceptance record](experiments/uk-shared-graph-contract-acceptance-20260913.md) +> for the tested revision and remaining scope. + - -**Verdict: REQUEST_CHANGES** for the source proposal. Two findings are actionable design defects in the new contracts; the rest are hardening and test-strength items. I could not save FABLE-REVIEW.md: this session is in plan mode with no write tool exposed, so the full review is below for root to file verbatim. - -## Reviewed identity - -| Item | Value | -| --- | --- | -| HEAD | `6f4ba4ec989eba93786b5d88033631ad253fdc5c` (resolved from `.git/refs/heads/uk-shared-graph-contracts-20260913`) | -| Base | `15ebde806cd1a262363f7217fe535c7234ff757f` (as given; not independently resolved) | -| Method | Read-only. No git diff was available without a shell, so I reviewed the current contents of every file in the receipts' identity table plus both new test files, the B2 edit, the kernel-contract unit test, and the amendment 25/26 text. No imports, pytest, engine, or data. | - -## Ranked findings - -**1. HIGH. `frame_mass_log` leaks non-declared same-version sibling output into kernel inputs with no key binding.** -`_project_context` passes `population.frame.mass_log` at `packages/microcosm-graph/src/microcosm/graph/executor.py:678`. For an ordinary node that population is the cumulative version state, updated after every ordinary node at `executor.py:2569-2570`, and `_append_frame_mass_log` runs for ordinary nodes too at `population.py:1157`. So node X sees records appended by any earlier ordinary node A in the same version, whether or not A is an ancestor. X's key binds only `frame_key(version)` plus declared input owners (`keys.py:187-209`), never A. Result: adding, removing, or re-parameterising A changes X's visible input while X's key is unchanged, so a cache hit replays output computed against a different log. This is a new charter-A hole; before amendment 26 kernels could not see the log at all. The doc sentence "incidental node order is not authority" describes the hazard but nothing enforces it. -Exact fix, minimal: in `run_graph` record `boundary_logs[node.id] = updated.frame.mass_log` when a structural node is admitted, and pass `frame_mass_log=boundary_logs[compiled.versions[node_id]]` for ordinary nodes; structural nodes keep the full incumbent log, which their key already binds through `members` (`keys.py:221-225`). Alternative that keeps sibling visibility: track which node appended each record and reject projection when a record's author is not in `_transitive_ancestors(compiled, node_id)`. Either way `test_mass_log_is_the_incoming_log` at `test_graph_frame_context.py:346-354` must change: the successor is an ordinary node reading the appender's column, and the fix makes it see `()` unless the ancestor rule is used. - -**2. HIGH. A `WeightUpdate` on kind `design` leaves design anchors stale and, after an EXPAND, mixed.** -Anchors are captured once at CREATE (`population.py:333-338`) and only carried afterwards (`population.py:1159`, `_carry_design_weights` at `2125-2181`). `_apply_weight_update` replaces the frame's design-kind values (`population.py:2038`) without re-anchoring. Consequences: the calibrated cap at `population.py:2323-2351` and `realized_max_weight_ratio` at `2354-2375` compare against pre-update design weights, so a normalisation by factor k makes every later cap ratio off by k; an EXPAND after the update anchors entrants from the updated values (`2166-2179`) while retained rows keep original anchors. The #901 consumer, `uk.full.normalize`, is exactly a design-weight normalisation. The decl docstring at `decl.py:381-382` states "ancestry is untouched" as if deliberate, but the mixed-anchor case is not a coherent contract. -Exact fix: in `patch` after line 1159, when `isinstance(node.weights, WeightUpdate)` and the kind is `design`, set `design_weights[entity] = frame.weights_for(entity).values`, and say so in amendment 25. If root prefers the current semantics, refuse `kind="design"` in `WeightUpdate.__post_init__` instead; leaving it silent is the one option I would not accept. - -**3. MEDIUM. `_context_digest` omission is defensible but the fields share live objects.** -Actual behaviour checked: `frame_metadata` is a `MappingProxyType` over a copy whose leaves are `Frame`-frozen tuples, frozensets and `_FrozenMapping` (`bundle.py:1346-1362`); `frame_column_order` values are `tuple[str]`; `frame_mass_log` is the population's own tuple of frozen `MassChangeRecord` objects, shared by reference (`executor.py:678`). Through the public API nothing is writable, so the omission at `executor.py:459-488` does not produce false passes. But `object.__setattr__` on a shared record silently rewrites the live version's log, and unlike `tables` the executor would not notice. This matches the existing treatment of `context.node`, so it is not blocking. Cheap hardening: digest `canonical_json(column_order)`, the mass-log record fields, and `store._encode_frame_metadata(frame_metadata)` inside `_context_digest`. - -**4. MEDIUM. Two new tests are weaker than their names.** -`test_cold_then_required_replay_revalidates_the_axis` (`test_graph_weight_update.py:311-337`) only proves a hit succeeds; it never shows the axis check executes on replay. `test_a_retained_mutating_observer_changes_nothing` (`test_graph_frame_context.py:410-435`) says it rewrites the metadata view but only rewrites tables; the three new fields are untouched. - -**5. LOW. The motivating "re-solve an existing calibration" case is unreachable with the shared kernel.** `calibrate.adam@1` returns no `receipt['weight_update']` and its error text at `packages/microcosm-calibrate/src/microcosm/calibrate/kernels.py:205-209` still demands a `WeightTransition`. Declaring it as a `WeightUpdate` rejects at `population.py:2025-2030`. Not this lane's file, but amendment 25's text should not claim the case is covered. - -**6. LOW. Cosmetic.** `decl.py:568-572` says "weight transition's" for an update mismatch. `test_graph_kernel_contract.py:339-345` was also edited, so the receipts' "isolated to one test file" claim applies to the acceptance suite only; that is acceptable under the charter. - -## Reproducing tests to invent - -- **T1 leak:** probe kernel writes `float(len(context.frame_mass_log))` into `person.x`, reading only `person.age`. Run `{survey, X}` cold: x = 0. Run `{survey, A, X}` into the same store where A appends a record: X hits and reports 0 while a cold run gives 1. Same key, different truth. -- **T2 anchors:** `WeightUpdate("household","design",…)` with factor 2, then a `calibrated` transition returning weights equal to the updated design weights with `max_weight_ratio=1.5`. Currently rejected as ratio 2.0; the receipt reports realized ratio 2.0 against a frame whose own design weights it equals. -- **T3 replay:** monkeypatch `microcosm.graph.population.weight_update_receipt` to return a foreign digest during the `resume="require"` run and assert `NodeRejectedError` matching "different .household. axis". -- **T4 tamper:** a kernel that calls `object.__setattr__(context.frame_mass_log[0], "reason", "x")`; assert the next node sees the original reason. Currently the live log changes and no rejection fires. - -## Residual runtime risks - -- Nothing here was executed. Pandas copy-on-write behaviour in the observer test and the fixture column order at `fixtures/toy_country/person.csv` line 1 (age before income) are the two places a source-only read can be wrong. -- In `resume="require"`, a mismatched axis receipt fails mid-run at apply time rather than in `_preflight_require`, since preflight validates record shape only. -- Column order is bound to the key by construction (owners are ancestors, order follows depth then id, rewrites keep position); I found no leak there. diff --git a/FIX-RESULT.md b/FIX-RESULT.md deleted file mode 100644 index 022a0ffdf..000000000 --- a/FIX-RESULT.md +++ /dev/null @@ -1,356 +0,0 @@ -# Fix round result — UK-enabling shared graph contracts (amendments 25, 26) - -> Historical source-only report. Later source corrections and the passing -> 101-test run are recorded in the -> [13 September acceptance record](experiments/uk-shared-graph-contract-acceptance-20260913.md). - -Source-only. **Runtime is UNTESTED.** No pytest, no import of the -production package, no engine, no country model, no native source, no -install, no gated data, no network beyond `git fetch`, no publication. The -checks that were run are stdlib `ast` parses, `ruff check` / `ruff format ---check`, `tools/ci_test_groups.py --verify`, and reading source. Every -behavioural claim below is a claim about what the source says, not about -an observed run; the bounded runtime plan root needs is at the end. - -## Identity - -| Thing | Value | -| --- | --- | -| Worktree | `_worktrees/microcosm-uk-shared-graph-contracts-20260913` | -| Branch | `uk-shared-graph-contracts-20260913` | -| Base | `origin/main` `15ebde806cd1a262363f7217fe535c7234ff757f` (re-fetched on resume; unchanged, 0 behind) | -| HEAD | `03b5cb8e2` | -| Reviewed head the adjudication ran on | `6f4ba4ec989eba93786b5d88033631ad253fdc5c` | - -### Commits, in order - -| Commit | Kind | What | -| --- | --- | --- | -| `97428cd9f` | journal | Lane baseline: absence proof, the #901 consumer read | -| `ef2dc69c3` | source+tests+docs | Amendment 25: `WeightUpdate`, `weight_update_receipt` | -| `f33d47cda` | source+tests+docs | Amendment 26: the three `KernelContext` frame fields | -| `895aabf19` | acceptance suite | B2's field set, isolated (as amendment 19's was) | -| `6f4ba4ec9` | journal+receipts | Lane receipts and the first runtime plan | -| `6104459f6` | journal | File `FABLE-REVIEW.md`; open the fix round | -| `c0e275543` | source+tests+docs | **F1** — a node sees the mass log its own key binds | -| `6ec46d62c` | source+tests+docs | **F3/F4** — detach the frame view; digest all three fields | -| `09827aec8` | journal | Record the resume and what it re-verified | -| `f5aa65d40` | source+tests+docs | **F2/F4/F5/F6** — design ancestry stated and proven; cached-axis replay properties; the `calibrate.adam` claim withdrawn; `decl.py` re-locked | -| `c1a7920f7` | tests | The clone fixture copies its household's members, as a copied group requires | -| `ecc98107e` | receipts | Fix-round contract deltas; refreshed identity; full test-file disclosure | -| `03b5cb8e2` | journal | Fix-round outcome, the F2 source reading, the runtime plan | - -The two commits the previous process had already landed (`c0e275543`, -`6ec46d62c`) were preserved and re-verified rather than trusted; what that -verification consisted of is in `PROGRESS-uk-shared-graph-contracts.md` -under "Verified on resume, not assumed from the commit messages". - -### File hashes at `03b5cb8e2` - -| File | SHA-256 | -| --- | --- | -| `graph/decl.py` | `11c2abd77f50389c6cd0b51e26cb3ebd5cbd0770ba96e9de1b53c71e9725eaa4` | -| `graph/kernel.py` | `2df6cc5b5b396adae578f885aedbd038d2c0e51f56b4a31b64c2f6b69e1b918d` | -| `graph/executor.py` | `5ab918e495fe4f8dd32c16155fe8c7a911e60e171cdbc8edb790626ce2d58c19` | -| `graph/population.py` | `33d1bb7bacea22870940288bf1907fb9eb24df7c245a216ff802e7fb41f5208f` | -| `graph/weight_update.py` | `0ccfe6fcd257ef62b1f771b12eecc8ac5d447f5aa7d0403102e0ae290de16720` | -| `graph/serialize.py` | `e5bf83c1082154f148626b6a36676614c6ff6c3fe0721aed94a1501da3021b1f` | -| `graph/explain.py` | `734a7b0e31692c31a99528cd83d9e74d3e508a317d913c1c169724a42d69d0de` | -| `graph/__init__.py` | `697c59a37989a36124e6d43c7b07dd3b0582d965f97303c1fb02c88b41db2d48` | -| `docs/graph-interface.lock` | `b857403be2206156b844958cbd6abcb25ef951d05c0cc11e22554169a9343d2e` | - -The lock file records `decl.py` and `kernel.py` at exactly the hashes -above; `test_graph_interface_lock.py` is what enforces that. The lock moved -twice in this lane and both times for a numbered amendment: -`kernel.py` in amendment 26's commits (`f33d47cda`, then the F1 and F3/F4 -docstring edits), `decl.py` in amendment 25's (`ef2dc69c3`, then -`f5aa65d40`'s semantics correction). It was never refreshed to make a test -green. - -### Test files touched, with their roles - -| File | Acceptance suite? | Change | -| --- | --- | --- | -| `test_graph_weight_update.py` | no | new (amendment 25); fix round adds three cached-axis replay properties and renames the weak one | -| `test_graph_frame_context.py` | no | new (amendment 26); fix round adds the boundary-log and isolation properties | -| `test_graph_population.py` | no | fix round adds three design-anchor properties beside the existing ones | -| `test_graph_kernel_contract.py` | no | one assertion of field *adjacency* relaxed to the ordering amendment 19 actually claims | -| `test_acceptance_b_ownership.py` | **yes** | B2's `KernelContext` field set, in its own commit `895aabf19` | - -The lane's earlier "isolated to one test file" claim was true of the -acceptance suite only. That is now stated accurately in the receipts and -historicized in the journal. - ---- - -## Point-by-point response to the adjudication - -### F1 — `frame_mass_log` leaked a same-version sibling's record. **Accepted; fixed.** (`c0e275543`) - -The finding was correct. `_project_context` passed -`population.frame.mass_log`, which for an ordinary node is the version's -cumulative log (`executor.py:2669` rewrites the version entry after every -ordinary member, and `_append_frame_mass_log` runs for ordinary nodes). -An ordinary node's key binds `frame_key(version_key)` and the owners of -the columns it declared (`keys.py:206-209`, `keys.py:181-201`) — never a -sibling — so a cache hit could replay output computed against a different -log. - -The fix is the reviewer's own minimal option. `run_graph` records -`boundary_mass_logs[node.id] = updated.frame.mass_log` where a structural -version is admitted (`executor.py:2674`), beside -`populations[node.id] = updated`, which both cold execution and a restored -hit reach; an ordinary node is projected from -`boundary_mass_logs[compiled.versions[node_id]]` (`executor.py:2511`), a -structural node from `incumbent.frame.mass_log`, whose key binds the base -*and* every ordinary member through `members` (`keys.py:214-225`). The two -maps are written at the same point, so the boundary lookup cannot miss a -version whose incumbent lookup (`executor.py:2413`, which runs first) -succeeded. - -`test_mass_log_is_the_incoming_log` was replaced, as the review said it -had to be. The properties now distinguish the two cases: an unread -same-version appender is invisible to an ordinary member and moves neither -its key nor its stored bytes; cold-with-sibling equals cold-without and -each replays into the other's store under `resume="require"`; the same -appender *is* visible to the structural boundary, whose key it moves. The -ancestor-tracking alternative was not taken: it would make a node's input a -function of graph topology the key does not bind either. - -### F2 — a design-kind `WeightUpdate` should re-anchor. **Not accepted.** Source-backed response. - -The premise that anchors become "stale and mixed" is not what the source -does, and the proposed fix would redefine a declared contract. - -1. **Anchors are captured once, at CREATE.** - `_create_population` calls `Population.from_frame(frame, node.id)` - (`executor.py:1461`), whose `design_weights is None` default reads the - frame's design weights (`population.py:333-338`). That default is - reached exactly once per graph. -2. **Afterwards they are only carried.** Every later version gets - `design_weights=_carry_design_weights(...)` passed **explicitly** - (`population.py:1159`, `population.py:1180`), so the re-derive default - never runs again. -3. **An update does not touch them.** `_apply_weight_update` - (`population.py:2032-2040`) returns a frame with replaced weight values - and nothing else, and it runs *before* the carry in `patch`. -4. **Clones are not "mixed".** The review's EXPAND case says entrants - anchor from updated values "while retained rows keep original anchors". - The code separates three cases, not two: - `_carry_design_weights` maps an EXPAND's **copied** rows back to their - source row's *original* anchor (`population.py:2145-2157`; the cached - twin at `941-981` does the same), retained rows keep their own, and only - a row with **no lineage at all** reads `frame.weights_for(entity)` — - because it has no earlier weight to be anchored on. The assignment's - framing is exactly the code's. -5. **The cap denominator is the original by declaration, not by accident.** - `_assert_design_weight_cap` compares `current > design * cap` against - the carried anchors (`population.py:2333-2350`) and its refusal has - always read "above N * original design weight"; - `realized_max_weight_ratio` uses the same denominator - (`population.py:2354-2375`), under a parameter the node must spell - `weight_anchor='design'`. -6. **Re-anchoring would be the silent change.** It would let a - normalization node inserted anywhere upstream widen every already- - declared `max_weight_ratio` by its own factor — a non-local - redefinition of a contract other nodes wrote against. - -The review's T2 case is real behaviour, and it is now asserted as the -intended outcome rather than left implicit: with design weights doubled by -an update, calibrated weights equal to them are refused at -`max_weight_ratio=1.5` and admitted exactly at `2.0`, reporting a realized -ratio of `2.0`. A stage that wants a cap against normalized weights states -the ratio it means. - -**No mixed-anchor defect beyond that definition was established**, so -nothing was stopped with a counterexample. One consequence is worth naming -and is now named in the amendment and the docstring: a row admitted -*after* an update is anchored on whatever design weight the `EXPAND` -installs for it. If a kernel derives an entrant's design weight from the -updated incumbent values, that derived number becomes the entrant's -anchor — because it is the weight the row entered carrying. That is the -anchor rule applied to a row with no ancestry, not a mixture of two rules. - -The review's alternative ("refuse `kind='design'` in -`WeightUpdate.__post_init__`") was also declined: the lane's recorded read -of #901 shows `uk.full.normalize` declaring -`WeightUpdate("household", weight_kind, ...)` with `weight_kind` a -variable, so refusing the design arm would refuse a consumer this -amendment exists for, to avoid a defect that is not there. - -What changed instead: `WeightUpdate`'s docstring no longer says "ancestry -is untouched" as a bare assertion but states the anchor rule and its three -cases; amendment 25 gains the same paragraph; and -`test_graph_population.py` gains three properties — -`test_a_same_kind_design_update_leaves_the_original_anchors_invariant`, -`test_a_design_update_moves_no_anchor_for_retained_clone_or_entrant_rows` -(the same EXPAND run over an updated and an un-updated population, with -the anchors asserted equal and the frames' design values asserted a factor -apart so it is not vacuous), and -`test_a_design_update_does_not_move_the_calibration_cap_denominator` (T2). - -### F3 — the three fields shared live objects. **Accepted; fixed.** (`6ec46d62c`) - -The review rated this non-blocking; root took it as blocking, because -"nothing writable through the public API" is not the property that matters -once a kernel holds the object: a frozen dataclass still yields to -`object.__setattr__`, and a shared `MassChangeRecord` is a live handle on -the version's log that, unlike a table, nothing would notice. - -`_project_context` now hands out `deepcopy(frame.metadata)` and records -rebuilt through `_detached_record`, the rule `_observer_snapshot` already -followed and now shares. `_context_digest` binds all three fields: the -metadata through `store._encode_frame_metadata` (the frame format's own -codec, so an unchanged view digests as it persists), each mass record -field by field, and the projected column order. The codec call is guarded — -a value `Frame` would never have admitted is reported as the mutation it -is rather than raising while the comparison that would report it is being -computed. - -Detachment and the digest are not redundant: the digest catches a kernel -whose output stops being a function of its declared inputs; detachment is -what stops a *retained* view from rewriting the live version after that -node's check has already passed. - -### F4 — two tests were weaker than their names. **Accepted; fixed.** (`6ec46d62c`, `f5aa65d40`) - -- The observer property now rewrites nested metadata and mass records as - well as tables, over the graph whose snapshots actually carry a mass - record, and asserts the rewrite landed on the snapshot before asserting - the live version and the next node are unchanged. -- `test_cold_then_required_replay_revalidates_the_axis` is renamed - `test_required_replay_reapplies_the_stored_update`, which is what it - proves. Three new properties carry the claim it did not: - `test_required_replay_refuses_a_cached_binding_against_another_axis` - rewrites the stored record's binding to the same household ids in - reverse order — the one case a count check cannot catch — and requires - the replay to hit that record; it is refused with `different 'household' - axis` and no kernel runs, so the refusal came from re-applying the - cached result. A second does the same with a short axis. A third re-files - the record *unchanged* and still replays, so the two refusals are about - the binding rather than about a record having been re-filed. The tests - also pin where the check fires: `resume="require"`'s preflight validates - record shape, so a foreign axis surfaces at apply time, mid-run — which - is the residual the review itself flagged. - -### F5 — the `calibrate.adam` motivating claim. **Accepted; withdrawn.** (`f5aa65d40`) - -Confirmed at source: `packages/microcosm-calibrate/src/` contains no -`weight_update` string at all, so `calibrate.adam@1` emits no -`receipt['weight_update']`, and `_apply_weight_update` would refuse the -declaration as unverifiable (`population.py:2025-2031`). Its own guard -also still asks for a `WeightTransition` by name. Amendment 25, the -`WeightUpdate` docstring and the test module docstring no longer offer -"a re-solve of an existing calibration" as a covered case; the amendment -now states plainly that re-solving through the shared kernel is a **future -consumer adaptation**. - -### F6 — cosmetic and disclosure. **Accepted.** (`f5aa65d40`, `ecc98107e`) - -The node mass-policy mismatch message said "weight transition's" on a path -both declarations reach; it now names neither. The receipts disclose every -test file touched and which one is the acceptance suite's, and the -journal's "isolated to one file" line is historicized in place rather than -edited away. - ---- - -## Bounded runtime plan for root - -Finite and invented-only: the toy country fixture under -`packages/microcosm-graph/tests/fixtures/toy_country/` and `tmp_path`. No -country model, engine, native source, gated microdata, credential or -network. Nothing below publishes, promotes or writes outside `tmp_path` -and the uv environment. - -``` -uv sync --all-packages --locked - -# 1. The new and changed properties. -uv run pytest packages/microcosm-graph/tests/test_graph_weight_update.py \ - packages/microcosm-graph/tests/test_graph_frame_context.py \ - packages/microcosm-graph/tests/test_graph_population.py - -# 2. Everything that reads the two frozen files, the executor, or replay. -uv run pytest packages/microcosm-graph/tests/test_graph_interface_lock.py \ - packages/microcosm-graph/tests/test_acceptance_b_ownership.py \ - packages/microcosm-graph/tests/test_graph_kernel_contract.py \ - packages/microcosm-graph/tests/test_graph_executor.py \ - packages/microcosm-graph/tests/test_graph_explain.py \ - packages/microcosm-graph/tests/test_graph_serialize.py \ - packages/microcosm-graph/tests/test_graph_decl.py \ - packages/microcosm-graph/tests/test_acceptance_replays.py \ - packages/microcosm-graph/tests/test_acceptance_d_weights.py - -# 3. Both shards whole. -uv run pytest packages/microcosm-graph packages/microcosm-frame - -# 4. Lint and the CI partition (both already pass here). -uv run ruff check . -uv run python tools/ci_test_groups.py --verify -``` - -### Helper dependencies each step needs - -| Step | Needs | Why | -| --- | --- | --- | -| 1 | `packages/microcosm-graph/tests/_toy.py`, `fixtures/toy_country/*.csv`, `schema.json` | every graph-level property runs the toy country; `test_graph_frame_context.py` also asserts the **person column order** from `person.csv`'s header, so a fixture column reorder is a genuine (and intended) failure there | -| 1 | `pandas`, `numpy` from the locked env | the anchor properties call `patch()` directly and build pandas lineage objects; no store, no executor | -| 1 | a writable `tmp_path` | the frame-context and weight-update properties build real `ContentStore`s under it | -| 2 | `docs/graph-interface.lock` | `test_graph_interface_lock.py` reads it from the repo, not the wheel | -| 3 | nothing further | | -| 4 | `tools/ci_test_groups.py` | partition authority; `--verify` must stay `verification=ok` | - -Expected: `test_graph_weight_update.py`'s 22 and -`test_graph_frame_context.py`'s 26 properties pass, `test_graph_population.py` -passes all 60 (57 of them pre-existing, 3 added this round), the -regression files stay green, `ruff check` and `--verify` stay clean, and no -node key moves anywhere (neither amendment adds a `Node` field or changes a canonical -projection, so no spec or seed digest should move; if one does, that is -main drift, not this lane). - -### Where a source-only round could still be wrong, worth checking in this order - -1. **`test_a_design_update_moves_no_anchor_for_retained_clone_or_entrant_rows` - builds an EXPAND by hand.** `_patch_expand`'s guards are many; one - already forced a correction in this round (`c1a7920f7` — a copied group - requires the same number of copies of every incumbent member, - `_remapped_expand_memberships`). A second guard firing would surface as - a `PopulationError` naming that guard, not as a wrong anchor. -2. **The three cached-axis properties re-file a store record** through - `ContentStore.put_json(..., verify_existing=False)`. That is the - documented replace path (`store._put` → - `_replace_write_only_collision`), and the record key is derived from the - node key rather than from content, so a rewritten receipt is not a - key/content mismatch. The third property (re-file unchanged, still - replays) is the control that separates "the binding was refused" from - "the re-filed record was unreadable". -3. **`_context_digest`'s metadata arm** calls `store._encode_frame_metadata` - twice per node. Its guard (`TypeError`, `ValueError`, `RecursionError`) - is reasoned from the codec's and `canonical_json`'s source, not - observed. It also costs one encode per node per side; metadata is small, - but that cost is real and unmeasured. -4. **`deepcopy(frame.metadata)` per node projection** is likewise - unmeasured. `Frame` metadata is stage-level, not row-level, so this is - expected to be negligible; it has not been timed. -5. **Pandas copy-on-write in the observer property.** The earlier plan - flagged that a table-only rewrite could make that property vacuous; it - now also rewrites nested metadata and mass records through - `object.__setattr__`, which copy-on-write does not affect, so the - property can no longer be vacuous in the way flagged — but the table arm - of it can still be. -6. **The mass-record probe** in `test_graph_frame_context.py` states an - unchanged household total from `context.weights["household"].values.sum()`; - if `_append_frame_mass_log`'s bracketing disagrees, it surfaces as a - `PopulationError` from the boundary-log properties. - -## What this round does not claim - -No UK build, calibration, native lane, release or publication was run, -prepared or authorized. No country model was imported. `#901` is -untouched: this lane adds no UK graph stage, kernel, target or gate, and -the two amendments remain separable in source — 25 touches `decl.py`, -`serialize.py`, `explain.py`, `population.py`, `graph/__init__.py` and the -new `weight_update.py`, leaving `kernel.py` byte-identical; 26 touches -`kernel.py` and `executor.py`, leaving `decl.py` byte-identical. Either can -be dropped without the other. diff --git a/PROGRESS-uk-shared-graph-contracts.md b/PROGRESS-uk-shared-graph-contracts.md deleted file mode 100644 index 028568a9f..000000000 --- a/PROGRESS-uk-shared-graph-contracts.md +++ /dev/null @@ -1,370 +0,0 @@ -# UK-enabling shared graph contracts (amendments 25 and 26) - -> Historical lane journal. For the final tested revision, see the -> [13 September acceptance record](experiments/uk-shared-graph-contract-acceptance-20260913.md). - -Lane journal. Append-only within this lane; historicize rather than -overwrite once the branch merges (CLAUDE.md, "Root journals are history"). - -## State - -Bounded, source-only slice extracting the shared graph contract that -María's UK full-build graph (#901) consumes, as numbered amendments on -current `main`. No UK graph stage, calibration science or country -kernel is added here — those stay in #901. - -- Worktree: `_worktrees/microcosm-uk-shared-graph-contracts-20260913` -- Branch: `uk-shared-graph-contracts-20260913` -- Base: `origin/main` `15ebde806cd1a262363f7217fe535c7234ff757f` -- Reviewed UK head: #901 `051fb972b19d319d58277bd63306d0d0e0947ce2` - (draft, base `microcosm-us-launch-integration-20260909`, unchanged - since 2026-09-10T21:03:10Z; re-verified via `gh` on 2026-09-13) -- Source review followed: `uk-parallel-review.md` (2026-09-12), section - "Best independent implementation slice" - -**Runtime is UNTESTED in this lane.** Instructions forbid pytest, -production imports, engine, native sources and installation here. Only -stdlib `ast`/`ruff`/CI-inventory source checks were run. A finite -invented-only runtime plan is at the end of this file for root review -*before* execution. - -## Absence verified on base 15ebde806 - -| Contract | Present on main? | Evidence | -| --- | --- | --- | -| `WeightUpdate` (same-kind weight replacement) | **absent** | `grep -rni weightupdate .` over the worktree returns nothing; `decl.py:320` carries only `WeightTransition`, whose `__post_init__` requires `to_kind` strictly later in `WEIGHT_KINDS`, and `population.py:1944` rejects a non-forward move. A same-kind update is therefore unrepresentable. | -| `weight_update_receipt` / ordered-axis evidence | **absent** | no `weight_update` module under `packages/microcosm-graph/src/microcosm/graph/`. | -| `KernelContext.frame_metadata` | **absent** | `kernel.py:361-370` lists the complete field set; no metadata field. | -| `KernelContext.frame_mass_log` | **absent** (read side) | same field list. The *write* side already exists: `population._append_frame_mass_log` (`population.py:2117`) already ingests `receipt['frame_mass_log_append']`, so only the kernel's view of the incoming log is missing. | -| `KernelContext.frame_column_order` | **absent** | same field list. The executor projects `tables[entity]` in declaration order (`executor.py:600-628`), not the population version's own column order, so a consumer cannot reconstruct the frame layout. | - -Frame-side prerequisites already on main: `Frame.mass_log`, -`Frame.metadata`, `MassChangeRecord` and `_freeze_metadata` -(`packages/microcosm-frame/src/microcosm/frame/bundle.py`). - -Interface lock on base matches the files exactly: -`decl.py ed0a859adcae12510d5ba74d51c694617201f7b448b108a3f602410f5da44876`, -`kernel.py dbf57c137330f0f12744c557ee594586a1b308b6d6adaba1938e2b6efded21ca`. - -## Actual consumer read before specifying shape - -`packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py` -at `051fb972` (SHA-256 `fc6f5b33127020f6e0529b39304715fe2028d47a6627b152b1e52e0d69f2efdc`): - -- `context_frame` (L58-80) reads `context.frame_column_order.get(entity, ...)`, - `context.weights`, `context.strata`, `getattr(context, "frame_mass_log", ())` - and `getattr(context, "frame_metadata", {})`. -- `UKSampleNormalizationKernel` (L386-412 region) declares - `WeightUpdate("household", weight_kind, "Normalize sampled source-family mass.")` - with `mass="declared"` and places `weight_update_receipt(ids)` under - `receipt["weight_update"]`. - -## Done - -- (nothing yet; baseline recorded) - -## Next - -- Amendment 25: `WeightUpdate` + ordered-axis receipt. -- Amendment 26: `KernelContext` frame metadata / mass log / column order. - ---- - -## Done (2026-09-13) - -| Commit | What | -| --- | --- | -| `97428cd9f` | Lane baseline: absence proof + the exact #901 consumer read | -| `ef2dc69c3` | Amendment 25: `WeightUpdate` + `weight_update_receipt` | -| `f33d47cda` | Amendment 26: `KernelContext` frame metadata / mass log / column order | -| `895aabf19` | Acceptance suite B2 field set, isolated (as amendment 19's was, `a2b6dfb0b`) | - -The two amendments are separable: 25 touches `decl.py` (re-locked) and -leaves `kernel.py` byte-identical; 26 touches `kernel.py` (re-locked) and -leaves `decl.py` byte-identical. Either can be dropped without the other. - -## Contract decisions - -1. **`reason` is normative.** It enters the node key, so two updates that - state different purposes are different nodes. Follows #901's own - declaration; the consequence is stated in the amendment. -2. **`to_kind` stays a property, not a field.** That is what keeps the two - declarations' field sets disjoint (`{entity, to_kind, mass}` vs - `{entity, kind, reason, mass}`), so neither canonical bytes nor - declaration JSON can confuse them — and existing `to_kind` readers (the - design-weight cap, the calibration view) keep working unchanged. -3. **No `free` mass on an update.** #901 declares this too. An update that - neither moves kind nor bounds mass records nothing checkable. -4. **Replay is structural, not a parallel rule.** `_load_cached_result` - already reconstructs the `KernelResult` and re-applies REWEIGHT to the - current base, so a cache hit re-enters `_apply_weight_update`. No - executor change was needed for the axis check. -5. **The frozen interface does not import another shard's private name.** - #901's `kernel.py` imports `microcosm.frame.bundle._freeze_metadata`. - This lane does not: `Frame` has already deeply frozen the metadata the - executor passes, and `KernelContext` adds a read-only view over it. The - docstring says exactly that and claims no deep freeze of its own. -6. **The three fields ride between `artifacts` and `tolerances`**, not at - the end as in #901, so amendment 17's "numerics rides at the end of the - context" stays literally true. Only amendment 19's unit assertion of - *adjacency* relaxes, to the ordering it actually claimed. -7. **A column order may not name an unprojected column.** Set equality - with the projected columns, not a subset: a column *name* is itself - information about the version. - -## Remaining risks - -- **Runtime is UNTESTED here.** No pytest, import, engine or install was - run, per the lane's instructions. Everything below the source level is - unverified; see the runtime plan. -- The acceptance-suite commit `895aabf19` is the one change this lane made - to a file the charter assigns to the suite lane. It is isolated to one - file and follows the precedent the amendment-19 doc text states - explicitly ("the acceptance suite's B2 field set gains it in its own - commit"). If root's owner disagrees, dropping that commit leaves B2 red - and the rest intact. - *(2026-09-13, fix round: "isolated to one file" is true of the* - *acceptance suite only. `test_graph_kernel_contract.py` — not a suite* - *file — was also edited, and the fix round adds three properties to* - *`test_graph_population.py`. The full list is in the receipts.)* -- Amendment 25 changes no node key; amendment 26 changes none either. - Neither re-pins a spec digest. If a spec/seed digest moves in CI, that - is main drift, not this lane (see `[[spec-engine-attested-modules]]`). -- `_context_digest` (B4's mutation check) was **not** extended to the - three new fields. They are immutable views over an immutable `Frame`, so - there is nothing for a kernel to mutate; stated here so the omission is - a decision rather than an oversight. - *(2026-09-13, fix round: this reasoning was wrong and the decision is* - *reversed. A frozen dataclass still yields to `object.__setattr__`, so* - *the fields are now handed out detached and all three are digested —* - *commit `6ec46d62c`. Superseded; kept for the record.)* -- No claim is made that any UK build, native lane, calibration or release - passes. This lane read source only. - -## Next (for root, before execution) - -Finite, invented-only runtime plan — nothing below touches a country -model, engine, native source, gated microdata or the network. - -``` -uv sync --all-packages --locked -uv run pytest packages/microcosm-graph/tests/test_graph_weight_update.py \ - packages/microcosm-graph/tests/test_graph_frame_context.py -uv run pytest packages/microcosm-graph/tests/test_graph_interface_lock.py \ - packages/microcosm-graph/tests/test_acceptance_b_ownership.py \ - packages/microcosm-graph/tests/test_graph_kernel_contract.py \ - packages/microcosm-graph/tests/test_graph_serialize.py \ - packages/microcosm-graph/tests/test_graph_decl.py \ - packages/microcosm-graph/tests/test_acceptance_d_weights.py \ - packages/microcosm-graph/tests/test_graph_population.py \ - packages/microcosm-graph/tests/test_graph_executor.py \ - packages/microcosm-graph/tests/test_graph_explain.py \ - packages/microcosm-graph/tests/test_acceptance_replays.py -uv run pytest packages/microcosm-graph packages/microcosm-frame -uv run ruff check . -uv run python tools/ci_test_groups.py --verify -``` - -Expected: 19 + 19 new tests pass; the ten regression files stay green; -`ruff check` and `--verify` already pass here. The four places a source-only -lane could be wrong, in the order worth checking: - -1. `test_graph_frame_context.py::test_a_retained_mutating_observer_changes_nothing` - mutates a detached snapshot with `table.loc[:, column] = table[column].iloc[0]`. - If pandas copy-on-write makes that a no-op on the snapshot, the test - passes vacuously rather than falsely; tighten it rather than trust it. -2. The toy person column order is asserted from `fixtures/toy_country/person.csv` - (`person_id, person_household_id, person_release_id, age, income, ...`). - If that fixture changes, `test_column_order_is_the_versions_own_order_not_the_projection` - is the test that notices. -3. `_append_frame_mass_log` requires the record to bracket the real - household totals; the probe states an unchanged total from - `context.weights["household"].values.sum()`. A mismatch would surface - as `PopulationError` from `test_mass_log_is_the_incoming_log`. -4. `test_round_trip_refuses_a_mixed_weights_payload` edits canonical JSON - by string surgery and depends on lexicographic key order - (`entity, kind, mass, reason`). - ---- - -# Fix round (2026-09-13, after the independent Fable adjudication) - -## State - -The lane's four commits (`97428cd9f`..`6f4ba4ec9`) stand. An independent -read-only adjudication of exactly `6f4ba4ec9` over base `15ebde806` -returned **REQUEST_CHANGES**; its verbatim text is now filed in this -packet as `FABLE-REVIEW.md` (the reviewer ran without a write tool and -returned the review through its tool result instead). - -Root owns adjudication. Findings F1, F3, F4, F5 and F6 are accepted and -implemented in this round. **F2 is not accepted as stated** — see the -point-by-point response below and in `FIX-RESULT.md`. - -`origin/main` re-fetched before editing: still -`15ebde806cd1a262363f7217fe535c7234ff757f`, nothing new to merge, branch -is 5 commits ahead and 0 behind. - -**Runtime remains UNTESTED in this round too.** No pytest, no production -import, no engine, no install, no network. Source, stdlib `ast`, `ruff` -and the CI group inventory only. - -## Next - -- F1: project ordinary nodes from their version boundary's mass log. -- F3/F4: detach the exposed frame objects; digest all three fields. -- F2: source-backed rejection plus the anchor-invariance tests. -- F5/F6: correct the docs' motivating claim and the disclosure. - -## Fix round, resumed (2026-09-13) - -The first fix-round process died on an external API DNS error after two -commits. Nothing was reset; the checkout resumed clean at `6ec46d62c`. -`origin/main` re-fetched on resume: still `15ebde806`, branch 8 ahead / 0 -behind. Restrictions unchanged — no pytest, import, engine, native source, -install, publication or disallowed log; source, stdlib `ast`, `ruff` and -the CI group inventory only. - -### Verified on resume, not assumed from the commit messages - -- `executor.py` boundary selection: `boundary_mass_logs[node.id]` is - written at exactly one place — beside `populations[node.id] = updated` - in the structural arm of the admission step (`executor.py:2669-2674`), - which both a cold run and a restored hit reach — and read at exactly one - place, the `StructuralDelta.NONE` arm that projects a context - (`executor.py:2508`). Its key set is therefore a subset of - `populations`', so the ordinary lookup cannot miss a version whose - incumbent lookup succeeded. -- Key binding re-read at source: `keys.py:206-209` binds an ordinary - node's `population_input` to `frame_key(version_key)` only, and - `keys.py:214-225` binds a structural node's `base` *and* `members`. The - projection matches that split exactly. -- `store.py:1157-1174`/`1334-1368` round-trip `Frame.mass_log`, so a - restored boundary carries the same records a computed one does. -- Detachment: `Frame.__init__` calls `_freeze_metadata` - (`bundle.py:112`), and `_freeze_metadata_value` rebuilds every nested - mapping, tuple and frozenset (`bundle.py:1382-1405`), so the - `_observer_snapshot` metadata hand-off shares no mutable-by-`setattr` - object with its parent. `_project_context` deep-copies the metadata and - rebuilds every record through `_detached_record`. -- `_context_digest` additions cannot raise while computing the comparison - that reports a mutation: `canonical_json` raises only `TypeError` / - `ValueError` (`canonical.py:_json_value`), `_encode_frame_metadata` - only `TypeError` (`store.py:1060-1082`), and both are caught alongside - `RecursionError`. -- `_project_context`'s only other caller, - `packages/microcosm-build/tests/test_uk_uc_capital_coherence.py:223`, - passes no `mass_log` and so takes the documented empty default. -- `ruff check .` clean; `ruff format --check` clean on every touched file; - stdlib `ast` parses clean. - -### Next - -- F4b: the required-replay axis property is still the weak one the review - named. Strengthen it. -- F2: source-backed rejection plus the anchor-invariance properties. -- F5/F6: the unsupported `calibrate.adam` motivating claim, the - transition-only wording, and an accurate receipts disclosure. - -## Done (fix round) - -| Commit | What | -| --- | --- | -| `6104459f6` | File `FABLE-REVIEW.md` and open the fix round | -| `c0e275543` | F1: a node sees the mass log its own key binds | -| `6ec46d62c` | F3/F4: detach the frame view, and digest it for mutation | -| `09827aec8` | Record the resume and what it re-verified | -| `f5aa65d40` | F2/F4/F5/F6: design ancestry stated and proven; replay axis properties; the `calibrate.adam` claim withdrawn; `decl.py` re-locked | -| `c1a7920f7` | The clone fixture copies its household's members, as a copied group requires | -| `ecc98107e` | Receipts: fix-round deltas and refreshed identity | - -`decl.py` is now `11c2abd77f50389c6cd0b51e26cb3ebd5cbd0770ba96e9de1b53c71e9725eaa4` -and `kernel.py` `2df6cc5b5b396adae578f885aedbd038d2c0e51f56b4a31b64c2f6b69e1b918d`; -`docs/graph-interface.lock` matches both. `kernel.py` moved only in -amendment 26's commits, `decl.py` only in amendment 25's. - -## F2: why the re-anchoring was refused - -Read at source, not argued from the declaration text: - -1. `Population.design_weights` is set from the frame **once**, at CREATE - (`population.py:333-338` via `_create_population`, `executor.py:1461`). -2. Every later version gets its anchors from `_carry_design_weights` - (`population.py:2125-2181`), which aligns the incumbent anchors to the - new version by stable entity id. -3. `patch` passes those carried anchors to `Population.from_frame` - **explicitly** (`population.py:1159-1180`), so the `design_weights is - None` default that would re-derive them from the frame - (`population.py:333-338`) is never reached after CREATE. -4. `_apply_weight_update` (`population.py:2032-2040`) replaces the frame's - weight values. It touches `design_weights` not at all, and it runs - *before* step 2 in `patch`. -5. So a design-kind update moves no existing row's anchor. The EXPAND arm - of `_carry_design_weights` maps copied rows back to their **source - row's original anchor** (`population.py:2145-2157`, and the cached - twin at `941-981`), not to the source's current value, so a clone after - an update inherits the pre-update anchor too. Only a row with no - lineage reads `frame.weights_for(entity)` — and it has no earlier - weight to be anchored on. -6. The cap therefore stays what its own refusal has always called it: - `current > design * cap` against the **original** design weights - (`population.py:2333-2350`), with `realized_max_weight_ratio` reported - on the same denominator (`population.py:2354-2375`). - -The review's T2 case is real behaviour and is the intended outcome, now -asserted as such: calibrated weights equal to design weights an update -doubled are refused at `max_weight_ratio=1.5` and admitted exactly at -`2.0`, realized ratio `2.0`. Re-anchoring would make the same numbers a -ratio of `1.0` and would widen every cap declared upstream of an unrelated -normalization by that normalization's factor — a non-local change to an -already-declared contract, which is what the assignment forbade. - -No mixed-anchor defect beyond that definition was found, so nothing was -stopped. The one consequence worth naming, and now named in the -amendment: a row admitted *after* an update is anchored on whatever design -weight the EXPAND installs for it. If a kernel derives an entrant's design -weight from the updated incumbent values, that derived number is the -entrant's anchor — because it is the weight the row entered carrying. - -## Next (for root, before execution) — fix round - -Superseding the earlier plan's first block; the rest of that plan stands. - -``` -uv sync --all-packages --locked -uv run pytest packages/microcosm-graph/tests/test_graph_weight_update.py \ - packages/microcosm-graph/tests/test_graph_frame_context.py \ - packages/microcosm-graph/tests/test_graph_population.py -uv run pytest packages/microcosm-graph/tests/test_graph_interface_lock.py \ - packages/microcosm-graph/tests/test_acceptance_b_ownership.py \ - packages/microcosm-graph/tests/test_graph_kernel_contract.py \ - packages/microcosm-graph/tests/test_graph_executor.py \ - packages/microcosm-graph/tests/test_acceptance_replays.py \ - packages/microcosm-graph/tests/test_acceptance_d_weights.py -uv run pytest packages/microcosm-graph packages/microcosm-frame -uv run ruff check . -uv run python tools/ci_test_groups.py --verify -``` - -Where a source-only round could still be wrong, in the order worth -checking: - -1. `test_graph_population.py::test_a_design_update_moves_no_anchor_for_retained_clone_or_entrant_rows` - builds an EXPAND by hand. The clone-ordinal rule in - `_remapped_expand_memberships` already forced one correction here - (`c1a7920f7`); if another of `_patch_expand`'s guards fires, it will be - a `PopulationError` naming the guard, not a wrong anchor. -2. The three cached-axis properties in `test_graph_weight_update.py` - re-file a store record through `ContentStore.put_json(..., - verify_existing=False)`. That is the documented replace path, but it is - the one place these tests touch store internals; the third property - (re-file unchanged, still replays) exists to separate "the binding was - refused" from "the re-filed record was unreadable". -3. `_context_digest`'s metadata arm calls `store._encode_frame_metadata` - twice per node. It is guarded against a kernel-planted value the codec - cannot encode, but the guard's `except` list is reasoned from the - codec's source, not observed. -4. The four risks the first plan listed are unchanged, except that the - observer property named there now also rewrites nested metadata and - mass records, so a pandas copy-on-write no-op could no longer make the - whole property vacuous. diff --git a/changelog.d/uk-shared-graph-contracts-frame-context.added.md b/changelog.d/uk-shared-graph-contracts-frame-context.added.md index b70127ecc..03868b4e0 100644 --- a/changelog.d/uk-shared-graph-contracts-frame-context.added.md +++ b/changelog.d/uk-shared-graph-contracts-frame-context.added.md @@ -1 +1 @@ -Added `KernelContext.frame_metadata`, `frame_mass_log` and `frame_column_order`, so a kernel can reconstruct its population version's declared slices without seeing any column it did not declare. The mass log a node receives is the one its key binds: its version's structural boundary for an ordinary node, the base version's cumulative log for a structural one. All three are detached from the live population before a kernel sees them, and all three enter the executor's input-mutation check (graph amendment 26). +Added `KernelContext.frame_metadata`, `frame_mass_log` and `frame_column_order`, so a kernel can reconstruct its population version's declared slices without seeing any column it did not declare. The mass log a node receives is the one its key binds: its version's structural boundary for an ordinary node, the base version's cumulative log for a structural one. All three are detached from the live population before a kernel sees them, and all three enter the executor's input-mutation check (graph amendment 27). diff --git a/changelog.d/uk-shared-graph-contracts-weight-update.added.md b/changelog.d/uk-shared-graph-contracts-weight-update.added.md index 76d148bf4..2aa5a6321 100644 --- a/changelog.d/uk-shared-graph-contracts-weight-update.added.md +++ b/changelog.d/uk-shared-graph-contracts-weight-update.added.md @@ -1 +1 @@ -Added `WeightUpdate`, a declared same-kind replacement of an entity's weight values, with `weight_update_receipt` binding the ordered entity axis the replacement values are positional against, checked against the incumbent axis on cold execution and on every replay. An update replaces weight values only: design anchors, and so any `max_weight_ratio` declared against them, are unchanged (graph amendment 25). +Added `WeightUpdate`, a declared same-kind replacement of an entity's weight values, with `weight_update_receipt` binding the ordered entity axis the replacement values are positional against, checked against the incumbent axis on cold execution and on every replay. An update replaces weight values only: design anchors, and so any `max_weight_ratio` declared against them, are unchanged (graph amendment 26). diff --git a/docs/graph-acceptance.md b/docs/graph-acceptance.md index 2304f3784..7d22f73e3 100644 --- a/docs/graph-acceptance.md +++ b/docs/graph-acceptance.md @@ -526,7 +526,7 @@ lock unchanged: lock is unchanged. Adopted 2026-09-18 for the retention seal's verifier, which reads the live population and seals its content (#950, #951). -25. **A weight update that keeps its kind is declarable.** +26. **A weight update that keeps its kind is declarable.** `WeightTransition` only ever moves a kind forward, so a stage that recomputes weights it already holds — a sampling normalization is the case this was extracted for — could not be declared at all, and the @@ -594,7 +594,7 @@ lock unchanged: `uk.full.normalize` node is the first consumer; its UK graph stages and calibration science stay in that branch. -26. **The context carries the version's metadata, mass log and column +27. **The context carries the version's metadata, mass log and column order.** The executor projects each entity table in *declaration* order, so `KernelContext.tables` is not the population version's layout, and the version's `Frame` metadata and mass log were not @@ -665,8 +665,16 @@ lock unchanged: cache hit exactly as they are from a computed one, which is what amendment 22's metadata-preserving Frame format makes possible. `kernel.py` is re-locked. Raised by the same source review as amendment - 25; the UK full-build graph's `context_frame` helper (#901, head - `051fb972`) is the first consumer. + 26; the UK full-build graph's `context_frame` helper (#901, head + `051fb972`) is the first consumer. The executor keeps live references in its + boundary mass logs, so under amendment 25's opt-in + (`_population_observer_detach=False`) a mutating observer can change + what a later node sees as `frame_mass_log`; that lies inside the + guarantees amendment 25 already withdraws and adds no new one. + Renumbered from 26 on the rebase onto main (2026-09-25), because main had + recorded the observer opt-in as amendment 25 in the meantime; amendment 26 + above was 25 in the same lane. + Adding a normative field with a default changes the canonical projection of every node that carries it, so node keys moved with amendments 11 and diff --git a/docs/graph-interface.lock b/docs/graph-interface.lock index 85920c480..6ba6ed147 100644 --- a/docs/graph-interface.lock +++ b/docs/graph-interface.lock @@ -1,2 +1,2 @@ -11c2abd77f50389c6cd0b51e26cb3ebd5cbd0770ba96e9de1b53c71e9725eaa4 decl.py -2df6cc5b5b396adae578f885aedbd038d2c0e51f56b4a31b64c2f6b69e1b918d kernel.py +b25ae4a62fd2777a5dffa30ecbba7d345a804b74b9d6b6e68c2a5d38f3e0a129 decl.py +24045ab2a62295874520d7940c7f7b59be9d909a3cfbf7e111a734e369b0815b kernel.py diff --git a/experiments/901-uk-shared-graph-contracts-receipts.md b/experiments/901-uk-shared-graph-contracts-receipts.md index bc3c11199..097acfdb4 100644 --- a/experiments/901-uk-shared-graph-contracts-receipts.md +++ b/experiments/901-uk-shared-graph-contracts-receipts.md @@ -1,5 +1,10 @@ # Shared graph contracts extracted for the UK full-build graph (#901) +> Renumbering note (2026-09-25, rebase onto main): the amendments this lane recorded as 25 and 26 +> are 26 and 27 on main, because main recorded the observer opt-in (#950/#951) as amendment 25 +> first. Commit hashes cited below are the pre-cherry-pick #918 hashes (Max's commits were +> cherry-picked with `-x`, so each new commit names its origin). + > Historical source-only receipt. Later fixes and runtime verification are > recorded in the [13 September acceptance record](uk-shared-graph-contract-acceptance-20260913.md). @@ -55,7 +60,7 @@ to consume this. Nothing was copied from #901's executor, and ## Amendments -**25 — a same-kind weight update is declarable.** `decl.py` gains +**26 — a same-kind weight update is declarable.** `decl.py` gains `WeightUpdate(entity, kind, reason, mass)` and `WEIGHT_UPDATE_MASS_POLICIES`; a new non-frozen `microcosm/graph/weight_update.py` gains `weight_update_receipt`. The @@ -68,7 +73,7 @@ current base through the same function. `to_kind` is a property, so the two declarations' field sets are disjoint and declaration JSON round-trips each as itself; the transition payload is byte-for-byte unchanged. -**26 — the context carries the version's frame view.** `kernel.py` gains +**27 — the context carries the version's frame view.** `kernel.py` gains `frame_metadata`, `frame_mass_log` and `frame_column_order`, riding after `artifacts` and before `tolerances` so amendment 17's "numerics rides at the end" stays literally true. A column order must be exactly an ordering @@ -112,15 +117,15 @@ charter assigns to the suite lane. In full: | File | Suite? | Change | | --- | --- | --- | -| `test_graph_weight_update.py` | no | new (amendment 25) | -| `test_graph_frame_context.py` | no | new (amendment 26) | +| `test_graph_weight_update.py` | no | new (amendment 26) | +| `test_graph_frame_context.py` | no | new (amendment 27) | | `test_graph_population.py` | no | three design-anchor properties added beside the existing ones (fix round, F2) | | `test_graph_kernel_contract.py` | no | one assertion of *adjacency* relaxed to the ordering amendment 19 actually claims | | `test_acceptance_b_ownership.py` | **yes** | B2's `KernelContext` field set, in its own commit (`895aabf19`), as amendment 19's was | The lock was re-recorded as part of each numbered amendment, never -refreshed to make a test green: amendment 25 moved only the `decl.py` -line, amendment 26 only the `kernel.py` line. +refreshed to make a test green: amendment 26 moved only the `decl.py` +line, amendment 27 only the `kernel.py` line. ## What stays with María @@ -146,7 +151,7 @@ An independent read-only review of `6f4ba4ec9` over `15ebde806` returned REQUEST_CHANGES; its verbatim text is `FABLE-REVIEW.md` in this packet. Root adjudicated. What changed in the contracts above: -**Amendment 26, mass log (F1).** The projection handed every node +**Amendment 27, mass log (F1).** The projection handed every node `population.frame.mass_log`, which for an ordinary node is its version's *cumulative* log. An ordinary node's key binds only its version's structural boundary and the owners of the columns it declared, so a @@ -157,7 +162,7 @@ cold execution and a restored hit both reach — and projects ordinary nodes from that boundary. A structural node still receives the cumulative log, which its key binds through `base` and `members`. -**Amendment 26, isolation (F3/F4).** `Frame` deeply freezes its metadata +**Amendment 27, isolation (F3/F4).** `Frame` deeply freezes its metadata and its mass records, but a frozen dataclass still yields to `object.__setattr__`, so passing those objects by reference made every kernel a live handle on the population. The projection now hands out a @@ -167,7 +172,7 @@ and `_context_digest` binds all three fields — the metadata through the frame format's own store codec, each mass record field by field, and the projected column order. -**Amendment 25, design ancestry (F2 — not accepted as stated).** The +**Amendment 26, design ancestry (F2 — not accepted as stated).** The review asked a design-kind update to re-anchor `Population.design_weights`. Root refused: an anchor is the design weight a row entered carrying, it is captured once at CREATE and afterwards only carried by stable entity id, @@ -178,7 +183,7 @@ declared upstream of an unrelated normalization. The amendment and the `WeightUpdate` docstring now state the anchor rule instead of "ancestry is untouched", and `test_graph_population.py` asserts it. -**Amendment 25, motivating claim (F5).** The amendment claimed a re-solve +**Amendment 26, motivating claim (F5).** The amendment claimed a re-solve of an existing calibration as a covered case. `calibrate.adam@1` emits no `receipt['weight_update']`, so that declaration would be refused by the axis check; the claim is now marked a future consumer adaptation. diff --git a/packages/microcosm-graph/src/microcosm/graph/decl.py b/packages/microcosm-graph/src/microcosm/graph/decl.py index a443f7f9b..24c92ab7a 100644 --- a/packages/microcosm-graph/src/microcosm/graph/decl.py +++ b/packages/microcosm-graph/src/microcosm/graph/decl.py @@ -100,7 +100,7 @@ MASS_POLICIES = frozenset({"conserve", "free", "declared"}) #: Mass policies a same-kind :class:`WeightUpdate` may declare. ``free`` is -#: deliberately absent (amendment 25). +#: deliberately absent (amendment 26). WEIGHT_UPDATE_MASS_POLICIES = frozenset({"conserve", "declared"}) #: The dtypes a mass-partition column may have. @@ -362,7 +362,7 @@ class WeightUpdate: stage that recomputes the numbers of weights it already holds — a sampling normalization is the case this was extracted for — cannot be declared at all. This is that declaration, and it is deliberately - narrower than a transition (amendment 25): + narrower than a transition (amendment 26): - The kind does not move. The executor checks the incumbent kind, the declared kind and the returned weights' kind are the same one. @@ -582,7 +582,7 @@ def __post_init__(self) -> None: ) if self.mass != self.weights.mass: # Reached by a WeightTransition and a WeightUpdate alike, so - # the text names neither (amendment 25). + # the text names neither (amendment 26). raise GraphError( f"Node {self.id!r}: mass policy {self.mass!r} disagrees with " f"its declared weight change's {self.weights.mass!r}." diff --git a/packages/microcosm-graph/src/microcosm/graph/executor.py b/packages/microcosm-graph/src/microcosm/graph/executor.py index 310ec8dda..a1e3377f1 100644 --- a/packages/microcosm-graph/src/microcosm/graph/executor.py +++ b/packages/microcosm-graph/src/microcosm/graph/executor.py @@ -581,7 +581,7 @@ def _context_digest(context: KernelContext) -> bytes: ) digest.update(len(value.payload).to_bytes(8, "little")) digest.update(value.payload) - # The amendment-26 frame fields are kernel inputs like any other, so B4's + # The amendment-27 frame fields are kernel inputs like any other, so B4's # before/after comparison covers them too. They are handed out detached # (`_project_context`), so a kernel that rewrites one cannot reach the # live version -- but it can still make its own node's output a function @@ -712,9 +712,9 @@ def _project_context( ``mass_log`` is the ``Frame`` mass log this node's *key* binds, which is not in general the cumulative log carried by ``population``: see the - boundary selection in :func:`run_graph`. It defaults to the empty log, + boundary selection in :func:`_execute_graph`. It defaults to the empty log, so a caller that projects a context outside the executor states no mass - history rather than inheriting one it never bound (amendment 26). + history rather than inheriting one it never bound (amendment 27). """ if population is None: @@ -813,7 +813,7 @@ def _project_context( # The projection above orders each table by declaration, not by the # version's own layout, so a consumer rebuilding the version's tables # needs that layout separately -- restricted to what it was given, so it - # never learns the name of a column it cannot read (amendment 26). + # never learns the name of a column it cannot read (amendment 27). column_order: dict[str, tuple[str, ...]] = {} for entity, table in tables.items(): projected = set(table.columns) @@ -833,7 +833,7 @@ def _project_context( # yields to ``object.__setattr__``, so passing the version's own # ``_FrozenMapping`` leaves and mass records by reference would make # every kernel -- and anything that retains a context past its own - # mutation check -- a live handle on the population (amendment 26). + # mutation check -- a live handle on the population (amendment 27). frame_metadata=deepcopy(frame.metadata), frame_mass_log=tuple(_detached_record(record) for record in mass_log), frame_column_order=MappingProxyType(column_order), diff --git a/packages/microcosm-graph/src/microcosm/graph/explain.py b/packages/microcosm-graph/src/microcosm/graph/explain.py index 2a3b87d76..0885cb991 100644 --- a/packages/microcosm-graph/src/microcosm/graph/explain.py +++ b/packages/microcosm-graph/src/microcosm/graph/explain.py @@ -1217,7 +1217,7 @@ def _render_calibration(compiled: CompiledGraph, manifest: RunManifest) -> str: transition = node.weights assert transition is not None # A same-kind update does not move the kind, so it does not get the - # arrow that says it did (amendment 25). + # arrow that says it did (amendment 26). kind_label = ( f"{_escape(transition.entity)} → {_escape(transition.to_kind)}" if not isinstance(transition, WeightUpdate) diff --git a/packages/microcosm-graph/src/microcosm/graph/kernel.py b/packages/microcosm-graph/src/microcosm/graph/kernel.py index a402e5b37..b42edd8c7 100644 --- a/packages/microcosm-graph/src/microcosm/graph/kernel.py +++ b/packages/microcosm-graph/src/microcosm/graph/kernel.py @@ -354,7 +354,7 @@ class KernelContext: from the live version as well as deeply frozen by ``Frame``; this class adds a read-only view over that mapping and does not itself deep-freeze or copy a mapping built some other way - (amendment 26). + (amendment 27). frame_mass_log: The ``Frame`` mass records this node's key binds, in order. For an ordinary node that is its population version's *boundary* log -- the log as that version was admitted -- so a @@ -370,7 +370,7 @@ class KernelContext: order is not authority. The executor hands out rebuilt records, not the version's own: a frozen dataclass still yields to ``object.__setattr__``, so a record passed by reference would be - a live handle on the population (amendment 26). + a live handle on the population (amendment 27). frame_column_order: Entity to the population version's own column order, restricted to the columns projected into ``tables``. The executor projects ``tables`` in declaration order, so this @@ -380,7 +380,7 @@ class KernelContext: undeclared column cannot be smuggled in as a name. All three frame fields are ordinary inputs: the executor's before/after comparison covers them, so rewriting one is refused exactly as - rewriting a table is (amendment 26). + rewriting a table is (amendment 27). tolerances: ``(entity, column)`` of each declared input column to the :class:`Tolerance` its owning kernel declared, or ``None`` for a bitwise owner. A gate compares against these. diff --git a/packages/microcosm-graph/src/microcosm/graph/population.py b/packages/microcosm-graph/src/microcosm/graph/population.py index a23266b2a..46b8f0de0 100644 --- a/packages/microcosm-graph/src/microcosm/graph/population.py +++ b/packages/microcosm-graph/src/microcosm/graph/population.py @@ -1987,7 +1987,7 @@ def _apply_weight_update( binding is recomputed against the incumbent axis these values are about to be applied to. A cached hit re-enters this function with the restored weights and receipt, so replay is checked by the same code - rather than a parallel rule (amendment 25). + rather than a parallel rule (amendment 26). """ update = node.weights diff --git a/packages/microcosm-graph/src/microcosm/graph/serialize.py b/packages/microcosm-graph/src/microcosm/graph/serialize.py index b9b06a40b..501d6e37d 100644 --- a/packages/microcosm-graph/src/microcosm/graph/serialize.py +++ b/packages/microcosm-graph/src/microcosm/graph/serialize.py @@ -289,7 +289,7 @@ def _weights_payload( ``WeightUpdate`` exposes ``to_kind`` as a property, so projecting it the way a transition is projected would round-trip it back as a transition and silently change what the node means. The transition - payload is byte-for-byte what it was before amendment 25, so every + payload is byte-for-byte what it was before amendment 26, so every declaration serialized before it restores unchanged. """ diff --git a/packages/microcosm-graph/src/microcosm/graph/weight_update.py b/packages/microcosm-graph/src/microcosm/graph/weight_update.py index 69c1c5699..b7806af4a 100644 --- a/packages/microcosm-graph/src/microcosm/graph/weight_update.py +++ b/packages/microcosm-graph/src/microcosm/graph/weight_update.py @@ -7,7 +7,7 @@ ordered entity axis its values were computed against, and the executor recomputes that binding from the incumbent axis it is about to apply them to — on cold execution and on every replay of the cached receipt -(amendment 25). +(amendment 26). The binding is a digest, not the ids: an axis of millions of rows does not belong in a manifest, and the executor only ever needs to answer whether diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_acceptance_b_ownership.py b/packages/microcosm-graph/tests/engine_free/shared/test_acceptance_b_ownership.py index 23076fac6..0b9d52701 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_acceptance_b_ownership.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_acceptance_b_ownership.py @@ -120,9 +120,9 @@ def test_b2_executor_enforces_ownership(tmp_path: Path) -> None: "rng", "sources", "artifacts", # amendment 19: declared typed opaque artifact inputs - "frame_metadata", # amendment 26: the population version's metadata - "frame_mass_log", # amendment 26: the version's incoming Frame mass log - "frame_column_order", # amendment 26: the version's order, projected only + "frame_metadata", # amendment 27: the population version's metadata + "frame_mass_log", # amendment 27: the version's incoming Frame mass log + "frame_column_order", # amendment 27: the version's order, projected only "tolerances", # amendment 13: declared tolerances of the inputs' owners "numerics", # amendment 17: per-coordinate numeric class, bound, platform } diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py index 6e9043104..3c7479832 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_frame_context.py @@ -1,4 +1,4 @@ -"""Amendment 26: the context carries the version's metadata, mass log, order. +"""Amendment 27: the context carries the version's metadata, mass log, order. The executor projects each table in *declaration* order, so a kernel that reconstructs its population version's layout cannot do it from @@ -976,7 +976,7 @@ def test_a_retained_mutating_observer_changes_nothing(tmp_path: Path) -> None: The observer keeps every snapshot and rewrites its tables, its nested metadata *and* its mass records after the callback returns. Table - mutation alone would not touch the amendment-26 fields at all, so it is + mutation alone would not touch the amendment-27 fields at all, so it is the metadata and mass-record rewrites that make this property about them; the run is over ``boundary_graph`` because that is the graph whose snapshots carry a mass record to rewrite. diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_kernel_contract.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_kernel_contract.py index fe524a25d..d4c5181ff 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_kernel_contract.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_kernel_contract.py @@ -341,7 +341,7 @@ def test_context_artifacts_default_empty_and_are_immutable() -> None: fields = [f.name for f in dataclasses.fields(KernelContext)] assert fields[-2:] == ["tolerances", "numerics"] # Amendment 19 claims artifacts rides *before* the pair, not adjacent to - # it; amendment 26's three frame fields ride between them. + # it; amendment 27's three frame fields ride between them. assert fields.index("artifacts") < fields.index("tolerances") node = Node("draw", "fit.draw@1") bare = KernelContext( diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_population.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_population.py index d6d6505c6..751af5368 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_population.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_population.py @@ -1362,7 +1362,7 @@ def test_calibration_cap_stays_anchored_to_original_design_after_filter() -> Non # ---------------------------------------------------------------------- -# Amendment 25: what a same-kind design update does to design ancestry +# Amendment 26: what a same-kind design update does to design ancestry # ---------------------------------------------------------------------- diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_weight_update.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_weight_update.py index 319742715..4af6ebf6a 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_weight_update.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_weight_update.py @@ -1,4 +1,4 @@ -"""Amendment 25: a same-kind weight update is declarable and axis-bound. +"""Amendment 26: a same-kind weight update is declarable and axis-bound. ``WeightTransition`` only moves a kind forward, so a stage that recomputes weights it already holds — a sampling normalization is the case this was @@ -225,7 +225,7 @@ def test_weight_update_round_trips_as_itself() -> None: def test_transition_payload_is_unchanged_by_the_amendment() -> None: - """Every declaration serialized before amendment 25 restores unchanged.""" + """Every declaration serialized before amendment 26 restores unchanged.""" graph = Graph("toy", (toy.SOURCE,), (toy.CREATE, toy.POOL)) text = graph_to_json(graph) assert '"weights":{"entity":"household","mass":"free","to_kind":"importance"}' in ( From 7b666ea83258d6880dd9815fde20e802d23b7145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:53:06 +0100 Subject: [PATCH 19/44] Add the shared artifact, stage-evidence and calibration-artifact helpers the UK full-build graph binds Country-agnostic pieces #901 introduced, brought onto main unchanged: artifact_files (file artifacts, byte materialisation, staged-bundle publication), stage_evidence (the typed stage-evidence artifact and its codec), gate_battery's phase-report payload codec and GateBatteryRun.record_phase, calibrate.artifacts (ordered problem, solution and calibration-result artifacts), calibrate.target_selection, and TargetSpec's to_dict/from_dict codec on the registry (the ruling of 2026-09-14). The three build-side tests are registered in the explicit shared-spec lane so they are not defaulted. The F0 contract-only kernel-id swap (load_uk_national_frame -> build_uk_frs_spine) travels with the UK spec change later in this series, because main's UK sources still bind the old id. Verified: 66 tests (new suites + test_gate_battery_contract_pins + registry) passed; tools/ci_test_groups.py --verify ok; ruff clean. Co-Authored-By: Claude Fable 5.1 --- .../src/microcosm/build/artifact_files.py | 162 +++++ .../src/microcosm/build/gate_battery.py | 98 ++- .../src/microcosm/build/stage_evidence.py | 86 +++ .../engine_free/shared/test_artifact_files.py | 91 +++ .../shared/test_gate_battery_replay.py | 76 +++ .../engine_free/shared/test_stage_evidence.py | 53 ++ .../src/microcosm/calibrate/artifacts.py | 566 ++++++++++++++++++ .../src/microcosm/calibrate/registry.py | 43 +- .../microcosm/calibrate/target_selection.py | 123 ++++ .../shared/test_ordered_artifacts.py | 140 +++++ .../tests/engine_free/shared/test_registry.py | 24 + .../shared/test_target_selection.py | 72 +++ 12 files changed, 1517 insertions(+), 17 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/artifact_files.py create mode 100644 packages/microcosm-build/src/microcosm/build/stage_evidence.py create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_artifact_files.py create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_gate_battery_replay.py create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_stage_evidence.py create mode 100644 packages/microcosm-calibrate/src/microcosm/calibrate/artifacts.py create mode 100644 packages/microcosm-calibrate/src/microcosm/calibrate/target_selection.py create mode 100644 packages/microcosm-calibrate/tests/engine_free/shared/test_ordered_artifacts.py create mode 100644 packages/microcosm-calibrate/tests/engine_free/shared/test_target_selection.py diff --git a/packages/microcosm-build/src/microcosm/build/artifact_files.py b/packages/microcosm-build/src/microcosm/build/artifact_files.py new file mode 100644 index 000000000..ff14fd7af --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/artifact_files.py @@ -0,0 +1,162 @@ +"""Atomic filesystem materialization of declared, portable build artifacts.""" + +from __future__ import annotations + +import os +import tempfile +from collections.abc import Mapping +from pathlib import Path + +from .trace import sha256_file + + +def file_artifact(path: str | Path) -> dict[str, object]: + """Bind one regular file without loading its entire payload into memory.""" + source = Path(path) + if not source.is_file(): + raise ValueError(f"Artifact is not a regular file: {source}.") + before = source.stat() + digest = sha256_file(source) + after = source.stat() + if (before.st_size, before.st_mtime_ns, before.st_ino) != ( + after.st_size, + after.st_mtime_ns, + after.st_ino, + ): + raise ValueError(f"Artifact changed while binding its identity: {source}.") + return {"filename": source.name, "sha256": digest, "size_bytes": after.st_size} + + +def materialize_bytes(payload: bytes, path: str | Path) -> dict[str, object]: + """Write deterministic bytes atomically, including after a graph cache hit.""" + if not isinstance(payload, bytes): + raise TypeError("Artifact payload must be immutable bytes.") + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp( + prefix=f".{destination.name}.", dir=destination.parent + ) + try: + with os.fdopen(fd, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + Path(temporary).replace(destination) + finally: + Path(temporary).unlink(missing_ok=True) + return file_artifact(destination) + + +def validate_file_inventory( + inventory: Mapping[str, Mapping[str, object]], *, root: str | Path +) -> None: + """Check exact bytes and sizes for each named file in a bundle directory.""" + directory = Path(root).resolve() + for role, expected in inventory.items(): + name = expected.get("filename") + if ( + not isinstance(name, str) + or not name + or Path(name).name != name + or name in {".", ".."} + ): + raise ValueError(f"Artifact {role!r} has an invalid bundle filename.") + path = directory / name + if path.resolve().parent != directory: + raise ValueError( + f"Artifact {role!r} filename escapes its bundle directory." + ) + if file_artifact(path) != dict(expected): + raise ValueError(f"Artifact {role!r} identity differs from its inventory.") + + +def publish_staged_bundle( + staged: Mapping[str, str | Path], + destinations: Mapping[str, str | Path], + *, + completion_role: str = "manifest", + expected: Mapping[str, Mapping[str, object]] | None = None, +) -> dict[str, dict[str, object]]: + """Publish a validated bundle with rollback and the completion marker last. + + The marker is absent while files change. Handled failures, including + KeyboardInterrupt, restore the prior bundle before restoring its marker. + This is a transaction over individual atomic renames, not a claim of + multi-file atomicity across process kill or power loss. + """ + import shutil + + if set(staged) != set(destinations) or completion_role not in staged: + raise ValueError( + "Staged bundle roles must match destinations and include a completion marker." + ) + source = {role: Path(path) for role, path in staged.items()} + target = {role: Path(path) for role, path in destinations.items()} + directories = {path.parent.resolve() for path in target.values()} + if len(directories) != 1 or len(set(target.values())) != len(target): + raise ValueError("Bundle destinations must be distinct files in one directory.") + directory = next(iter(directories)) + inventory = {} + for role in source: + if source[role].is_symlink() or target[role].is_symlink(): + raise ValueError("Bundle publication does not accept symlink files.") + if source[role].resolve() == target[role].resolve(): + raise ValueError("Bundle sources must be separate staging files.") + if source[role].name != target[role].name: + raise ValueError("Staged bundle filenames must match their destination.") + record = file_artifact(source[role]) + if expected is not None and ( + role not in expected + or any(expected[role].get(key) != value for key, value in record.items()) + ): + raise ValueError( + f"Staged artifact {role!r} differs from its declared identity." + ) + inventory[role] = record + created = not directory.exists() + directory.mkdir(parents=True, exist_ok=True) + backup = Path(tempfile.mkdtemp(prefix=".bundle-backup-", dir=directory.parent)) + order = tuple(role for role in staged if role != completion_role) + ( + completion_role, + ) + saved, published = [], [] + succeeded = False + try: + # Remove the old completion marker before replacing any old payload. + for role in ( + completion_role, + *[role for role in order if role != completion_role], + ): + if target[role].exists(): + if not target[role].is_file(): + raise ValueError( + f"Bundle destination is not a regular file: {target[role]}." + ) + target[role].replace(backup / target[role].name) + saved.append(role) + for role in order: + # Recheck just before moving; publication never binds mixed bytes. + if file_artifact(source[role]) != inventory[role]: + raise ValueError( + f"Staged artifact {role!r} changed during publication." + ) + source[role].replace(target[role]) + published.append(role) + succeeded = True + return inventory + finally: + if not succeeded: + for role in reversed(published): + target[role].unlink(missing_ok=True) + # The previous marker returns only after all previous payloads. + for role in ( + *[role for role in saved if role != completion_role], + *([completion_role] if completion_role in saved else []), + ): + (backup / target[role].name).replace(target[role]) + shutil.rmtree(backup) + if created and not succeeded: + try: + directory.rmdir() + except OSError: + pass diff --git a/packages/microcosm-build/src/microcosm/build/gate_battery.py b/packages/microcosm-build/src/microcosm/build/gate_battery.py index 1e22a4c31..c97070878 100644 --- a/packages/microcosm-build/src/microcosm/build/gate_battery.py +++ b/packages/microcosm-build/src/microcosm/build/gate_battery.py @@ -80,6 +80,8 @@ "GatePhaseReport", "GateStatus", "evaluate_phase", + "gate_phase_report_from_payload", + "gate_phase_report_payload", "gate_signing_key_env", ] @@ -549,6 +551,83 @@ def failures(self) -> tuple[str, ...]: # --------------------------------------------------------------------------- +def gate_phase_report_payload( + report: GatePhaseReport, *, gates: GatesManifest +) -> dict[str, object]: + """Portable numerical verdicts, bound to their exact declared policy. + + Release IDs, signing keys and filesystem paths belong to materialization, + and therefore do not enter a cached evaluation artifact. + """ + expected = tuple(entry for entry in gates.gates if entry.phase == report.phase) + if ( + report.phase not in gates.phases + or tuple(o.entry for o in report.outcomes) != expected + ): + raise ValueError("Gate phase outcomes do not match the declared manifest.") + return { + "schema_version": 1, + "gates_manifest_sha256": _canonical_sha256(_gates_manifest_payload(gates)), + "phase": report.phase, + "outcomes": [ + { + "id": outcome.entry.id, + **outcome.to_payload(), + "result_name": outcome.result.name + if outcome.result is not None + else None, + "evidence_sha256": outcome.evidence_sha256, + } + for outcome in report.outcomes + ], + } + + +def gate_phase_report_from_payload( + payload: Mapping[str, object], *, gates: GatesManifest +) -> GatePhaseReport: + """Validate stored outcomes against current entries before report replay.""" + if payload.get("schema_version") != 1: + raise ValueError("Unsupported gate phase report schema.") + if payload.get("gates_manifest_sha256") != _canonical_sha256( + _gates_manifest_payload(gates) + ): + raise ValueError("Stored gate report has a different gate manifest.") + phase = payload.get("phase") + if phase not in gates.phases: + raise ValueError("Stored gate report has an undeclared phase.") + entries = tuple(entry for entry in gates.gates if entry.phase == phase) + rows = payload.get("outcomes") + if not isinstance(rows, list) or len(rows) != len(entries): + raise ValueError("Stored gate report outcomes do not cover its phase.") + outcomes = [] + for entry, row in zip(entries, rows, strict=True): + if not isinstance(row, Mapping) or any( + row.get(key) != getattr(entry, key) + for key in ("id", "gate", "phase", "criticality") + ): + raise ValueError("Stored gate report outcomes differ from its manifest.") + status = GateStatus(row["status"]) + result = None + if status in (GateStatus.PASSED, GateStatus.FAILED): + result = GateResult( + name=str(row["result_name"]), + passed=status is GateStatus.PASSED, + failures=tuple(row["failures"]), + details=dict(row["details"]), + ) + outcomes.append( + GateOutcome( + entry=entry, + status=status, + result=result, + reason=row.get("reason"), + evidence_sha256=row.get("evidence_sha256"), + ) + ) + return GatePhaseReport(phase=str(phase), outcomes=tuple(outcomes)) + + def _evaluate_gate(name: str, evaluator: Callable[[], GateResult]) -> GateResult: """Run one evaluator, failing closed on any misbehaviour. @@ -898,10 +977,25 @@ def run_phase(self, phase: str, context: EvidenceContext) -> GatePhaseReport: f"(declared order {list(self._gates.phases)})." ) report = evaluate_phase(self._gates, phase, context, registry=self._registry) - self._phase_reports[phase] = report - self._write_report() + self.record_phase(report) return report + def record_phase(self, report: GatePhaseReport) -> None: + """Persist an evaluated or verified cached phase before enforcement. + + This is the same ordered write-then-block boundary as ``run_phase``; + it never reruns evaluators and never accepts a different gate policy. + """ + if self._blocked_at_phase is not None: + raise ValueError(f"battery blocked at phase {self._blocked_at_phase!r}.") + if report.phase != self._next_phase(): + raise ValueError( + f"phase {report.phase!r} is out of order; expected {self._next_phase()!r}." + ) + gate_phase_report_payload(report, gates=self._gates) + self._phase_reports[report.phase] = report + self._write_report() + def enforce(self, phase: str, *, mode: BlockingMode) -> bool: """Apply the phase's blocking verdict, strictly after persistence. diff --git a/packages/microcosm-build/src/microcosm/build/stage_evidence.py b/packages/microcosm-build/src/microcosm/build/stage_evidence.py new file mode 100644 index 000000000..f0373a078 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/stage_evidence.py @@ -0,0 +1,86 @@ +"""Portable evidence transport for existing stateful stage adapters. + +Capture only their declared checkpoint/evidence hooks after computation. Model +objects, closures and execution timestamps never become numerical artifacts. +Consumers need only this data contract, not a live fitting instance. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping + +from microcosm.graph import ArtifactType + +STAGE_EVIDENCE_TYPE = ArtifactType("microcosm.stage-evidence", 1) + + +def snapshot_stage_evidence(stage: str, transform: object | None) -> dict[str, object]: + """Snapshot checkpoint, fit-weight and sampling evidence without rerunning.""" + + checkpoint = None + hook = getattr(transform, "checkpoint_metadata", None) + if callable(hook): + checkpoint = dict(hook()) + evidence = checkpoint.get("evidence", checkpoint) + else: + result = getattr(transform, "last_result", None) + evidence_hook = getattr(result, "evidence", None) + evidence = ( + evidence_hook() + if callable(evidence_hook) + else result + if isinstance(result, Mapping) + else None + ) + payload: dict[str, object] = { + "schema_version": STAGE_EVIDENCE_TYPE.schema_version, + "stage": stage, + "checkpoint_metadata": checkpoint, + "evidence": evidence, + "sampling": getattr(transform, "sampling", None), + } + # Do not evaluate a raising property merely to detect whether it exists: + # missing/unreadable fitting evidence must stay visible to weight audits. + exposes_records = getattr(type(transform), "fit_weight_records", None) is not None + exposes_records |= "fit_weight_records" in getattr(transform, "__dict__", {}) + if exposes_records: + try: + records = tuple(transform.fit_weight_records or ()) + payload["fit_weight_records"] = [ + { + "fit_name": str(record.fit_name), + "weight_kind": str(record.weight_kind), + } + for record in records + ] + payload["fit_weight_records_status"] = "present" if records else "empty" + except Exception: # noqa: BLE001 - preserve the existing fail-visible audit + payload["fit_weight_records"] = [] + payload["fit_weight_records_status"] = "unreadable" + elif checkpoint is not None and "fit_weight_records" in checkpoint: + payload["fit_weight_records"] = checkpoint["fit_weight_records"] + payload["fit_weight_records_status"] = ( + "present" if checkpoint["fit_weight_records"] else "empty" + ) + return payload + + +def encode_stage_evidence(payload: Mapping[str, object]) -> bytes: + """Encode finite JSON; reject nonportable state rather than pickling it.""" + + return json.dumps( + dict(payload), sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode("utf-8") + + +def decode_stage_evidence(payload: bytes, *, stage: str) -> dict[str, object]: + document = json.loads(payload) + if not isinstance(document, dict) or document.get("schema_version") != 1: + raise ValueError("Unsupported stage evidence schema.") + if document.get("stage") != stage: + raise ValueError("Stored stage evidence has a different stage identity.") + for key in ("checkpoint_metadata", "evidence", "sampling"): + if key not in document: + raise ValueError(f"Stage evidence is missing {key!r}.") + return document diff --git a/packages/microcosm-build/tests/engine_free/shared/test_artifact_files.py b/packages/microcosm-build/tests/engine_free/shared/test_artifact_files.py new file mode 100644 index 000000000..0abe02002 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_artifact_files.py @@ -0,0 +1,91 @@ +"""Declared file artifacts are atomically materialized and checked by bytes.""" + +import pytest + +from microcosm.build.artifact_files import materialize_bytes, validate_file_inventory + + +def test_atomic_artifact_materialization_can_recreate_a_missing_output(tmp_path): + path = tmp_path / "evidence.json" + record = materialize_bytes(b'{"result":1}\n', path) + path.unlink() + assert materialize_bytes(b'{"result":1}\n', path) == record + validate_file_inventory({"evidence": record}, root=tmp_path) + path.write_bytes(b"changed") + with pytest.raises(ValueError, match="identity"): + validate_file_inventory({"evidence": record}, root=tmp_path) + + +def test_artifact_inventory_rejects_paths_outside_bundle(tmp_path): + with pytest.raises(ValueError, match="filename"): + validate_file_inventory({"escape": {"filename": "../other"}}, root=tmp_path) + + +def test_bundle_publication_replaces_existing_payloads_and_marker_last( + tmp_path, monkeypatch +): + from pathlib import Path + + from microcosm.build.artifact_files import publish_staged_bundle + + stage = tmp_path / "stage" + output = tmp_path / "output" + stage.mkdir() + output.mkdir() + sources = { + role: stage / name + for role, name in (("dataset", "full.h5"), ("manifest", "full.build.json")) + } + targets = {role: output / path.name for role, path in sources.items()} + for role in sources: + sources[role].write_bytes(("new-" + role).encode()) + targets[role].write_bytes(("old-" + role).encode()) + moves = [] + original = Path.replace + + def record(path, destination): + if Path(destination).parent == output: + moves.append(Path(destination).name) + return original(path, destination) + + monkeypatch.setattr(Path, "replace", record) + inventory = publish_staged_bundle(sources, targets) + assert moves == ["full.h5", "full.build.json"] + assert targets["manifest"].read_bytes() == b"new-manifest" + validate_file_inventory(inventory, root=output) + + +@pytest.mark.parametrize("interrupt", [RuntimeError, KeyboardInterrupt]) +def test_bundle_publication_rolls_back_old_complete_bundle( + tmp_path, monkeypatch, interrupt +): + from pathlib import Path + + from microcosm.build.artifact_files import publish_staged_bundle + + stage = tmp_path / "stage" + output = tmp_path / "output" + stage.mkdir() + output.mkdir() + sources = { + role: stage / name + for role, name in (("dataset", "full.h5"), ("manifest", "full.build.json")) + } + targets = {role: output / path.name for role, path in sources.items()} + for role in sources: + sources[role].write_bytes(("new-" + role).encode()) + targets[role].write_bytes(("old-" + role).encode()) + original = Path.replace + + def fail(path, destination): + if path == sources["manifest"]: + assert not targets["manifest"].exists() + raise interrupt("synthetic publication interruption") + return original(path, destination) + + monkeypatch.setattr(Path, "replace", fail) + with pytest.raises(interrupt): + publish_staged_bundle(sources, targets) + for role in targets: + assert targets[role].read_bytes() == ("old-" + role).encode() + assert not list(tmp_path.glob(".bundle-backup-*")) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_gate_battery_replay.py b/packages/microcosm-build/tests/engine_free/shared/test_gate_battery_replay.py new file mode 100644 index 000000000..ce4a813e3 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_gate_battery_replay.py @@ -0,0 +1,76 @@ +"""Replayed numerical gate outcomes retain the same enforcement policy.""" + +from dataclasses import replace + +import pytest + +from microcosm.build.country_spec import GateSelectionSpec, GatesManifest +from microcosm.build.gate_battery import ( + BlockingMode, + EvidenceContext, + GateBatteryBlockedError, + GateBatteryRun, + evaluate_phase, + gate_phase_report_from_payload, + gate_phase_report_payload, +) + + +def _manifest(): + return GatesManifest( + country="xx", + version=1, + policy="test", + phases=("terminal",), + gates=( + GateSelectionSpec( + id="mass", + gate="input_mass_parity", + phase="terminal", + criticality="release_blocking", + parameters={"relative_tolerance": 0.01}, + ), + ), + ) + + +def test_replayed_gate_failure_is_persisted_before_it_blocks(tmp_path): + gates = _manifest() + report = evaluate_phase( + gates, + "terminal", + EvidenceContext( + artifacts={ + "candidate_input_mass_totals": {"income": 80.0}, + "reference_input_mass_totals": {"income": 100.0}, + } + ), + ) + payload = gate_phase_report_payload(report, gates=gates) + restored = gate_phase_report_from_payload(payload, gates=gates) + run = GateBatteryRun( + gates, + release_id="replay", + report_path=tmp_path / "gates.json", + release_candidate=False, + ) + run.record_phase(restored) + with pytest.raises(GateBatteryBlockedError): + run.enforce("terminal", mode=BlockingMode.BLOCKS_ARTIFACT) + assert run.report_path.is_file() + assert run.report_payload()["blocked_at_phase"] == "terminal" + assert run.phase_report("terminal").failures == report.failures + + +def test_replay_rejects_changed_policy_and_missing_outcomes(): + gates = _manifest() + report = evaluate_phase(gates, "terminal", EvidenceContext()) + payload = gate_phase_report_payload(report, gates=gates) + changed = replace( + gates, gates=(replace(gates.gates[0], parameters={"relative_tolerance": 1.0}),) + ) + with pytest.raises(ValueError, match="manifest"): + gate_phase_report_from_payload(payload, gates=changed) + payload["outcomes"] = [] + with pytest.raises(ValueError, match="outcomes"): + gate_phase_report_from_payload(payload, gates=gates) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_stage_evidence.py b/packages/microcosm-build/tests/engine_free/shared/test_stage_evidence.py new file mode 100644 index 000000000..7832b0a4e --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_stage_evidence.py @@ -0,0 +1,53 @@ +"""Stored evidence is independent of the transform instance that produced it.""" + +import json +from types import SimpleNamespace + +import pytest + +from microcosm.build.stage_evidence import ( + decode_stage_evidence, + encode_stage_evidence, + snapshot_stage_evidence, +) + + +def test_snapshot_preserves_checkpoint_replay_and_fit_weight_records(): + class Transform: + sampling = {"fraction": 0.1, "seed": 42} + fit_weight_records = (SimpleNamespace(fit_name="wealth", weight_kind="design"),) + + def checkpoint_metadata(self): + return {"evidence": {"rows": 4}, "replay_payload": {"target": [1, 2]}} + + payload = encode_stage_evidence(snapshot_stage_evidence("wealth", Transform())) + recovered = decode_stage_evidence(payload, stage="wealth") + assert recovered["evidence"] == {"rows": 4} + assert recovered["checkpoint_metadata"]["replay_payload"] == {"target": [1, 2]} + assert recovered["fit_weight_records"] == [ + {"fit_name": "wealth", "weight_kind": "design"} + ] + assert recovered["sampling"] == {"fraction": 0.1, "seed": 42} + + +def test_unreadable_fit_records_remain_an_explicit_failed_audit_input(): + class Transform: + @property + def fit_weight_records(self): + raise RuntimeError("fit did not expose its records") + + payload = snapshot_stage_evidence("wealth", Transform()) + assert payload["fit_weight_records"] == [] + assert payload["fit_weight_records_status"] == "unreadable" + + +def test_stage_evidence_refuses_wrong_identity_and_nonportable_data(): + payload = encode_stage_evidence(snapshot_stage_evidence("one", object())) + with pytest.raises(ValueError, match="stage identity"): + decode_stage_evidence(payload, stage="two") + mutated = json.loads(payload) + mutated["schema_version"] = 2 + with pytest.raises(ValueError, match="schema"): + decode_stage_evidence(json.dumps(mutated).encode(), stage="one") + with pytest.raises(ValueError): + encode_stage_evidence({"value": float("nan")}) diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/artifacts.py b/packages/microcosm-calibrate/src/microcosm/calibrate/artifacts.py new file mode 100644 index 000000000..b7cd2783b --- /dev/null +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/artifacts.py @@ -0,0 +1,566 @@ +"""Portable ordered calibration problems and solutions, without pickles. + +The authoritative numerical input is CSR plus aligned target and entity axes. +Country adapters supply declarative row metadata and identity bindings. Python +measure closures are not serialized: ``to_target_set`` can reconstruct exact +compiled contributions, checking the consuming Frame's ordered entity IDs. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, replace +from io import BytesIO +from numbers import Integral +from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo + +import numpy as np +from scipy import sparse + +from microcosm.frame import Frame, WeightKind, Weights +from microcosm.graph import ArtifactType +from microcosm.graph.canonical import canonical_json + +from .matrix import CalibrationProblem, SkippedTarget +from .target import Target, TargetSet + +PROBLEM_TYPE = ArtifactType("microcosm.calibrate.ordered-problem", 1) +SOLUTION_TYPE = ArtifactType("microcosm.calibrate.ordered-solution", 1) +RESULT_TYPE = ArtifactType("microcosm.calibrate.calibration-result", 1) + + +def _ids(values: Sequence[int | str]) -> tuple[int | str, ...]: + if isinstance(values, str | bytes): + raise ValueError("Entity axis must be a sequence of IDs.") + result = [] + for value in values: + if isinstance(value, Integral) and not isinstance(value, bool): + result.append(int(value)) + elif isinstance(value, str) and value: + result.append(value) + else: + raise ValueError("Entity IDs must be integers or nonempty strings.") + if not result or len(set(result)) != len(result): + raise ValueError("Entity IDs must be nonempty and unique.") + return tuple(result) + + +def _sha(value: object) -> str: + if ( + not isinstance(value, str) + or len(value) != 64 + or any(c not in "0123456789abcdef" for c in value) + ): + raise ValueError("A problem binding must be a lowercase SHA-256 digest.") + return value + + +def _freeze(array: np.ndarray) -> np.ndarray: + return np.frombuffer(array.tobytes(), dtype=array.dtype).reshape(array.shape) + + +def _pack(metadata: Mapping, arrays: Mapping[str, np.ndarray]) -> bytes: + output = BytesIO() + with ZipFile(output, "w", compression=ZIP_DEFLATED) as archive: + members = { + **arrays, + "metadata": np.frombuffer(canonical_json(metadata), dtype=np.uint8), + } + for name, array in sorted(members.items()): + stream = BytesIO() + np.lib.format.write_array(stream, np.asarray(array), allow_pickle=False) + info = ZipInfo(name + ".npy", date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = ZIP_DEFLATED + archive.writestr(info, stream.getvalue()) + return output.getvalue() + + +def _unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("Duplicate calibration metadata key.") + result[key] = value + return result + + +def _unpack(payload: bytes, *, schema: str, members: set[str]) -> tuple[dict, dict]: + if type(payload) is not bytes: + raise TypeError("Calibration artifacts require immutable bytes.") + try: + with np.load(BytesIO(payload), allow_pickle=False) as archive: + if len(archive.files) != len(set(archive.files)) or set( + archive.files + ) != members | {"metadata"}: + raise ValueError("Calibration artifact members differ from its schema.") + raw = archive["metadata"] + if raw.dtype != np.uint8 or raw.ndim != 1: + raise ValueError("Calibration artifact metadata must be UTF-8 bytes.") + metadata = json.loads(raw.tobytes(), object_pairs_hook=_unique) + if ( + canonical_json(metadata) != raw.tobytes() + or metadata.get("schema") != schema + ): + raise ValueError( + "Calibration artifact schema/canonical metadata differs." + ) + arrays = {key: _freeze(archive[key]) for key in members} + return metadata, arrays + except (OSError, KeyError, TypeError, UnicodeError) as error: + raise ValueError("Malformed calibration artifact.") from error + + +def _target(target: Target) -> dict[str, object]: + return { + "name": target.name, + "entity": target.entity, + "value": target.value, + "period": target.period, + "source": target.source, + "tolerance": target.tolerance, + "metadata": dict(target.metadata), + "measure": target.measure if isinstance(target.measure, str) else None, + "filter": target.filter if isinstance(target.filter, str) else None, + "compiled_measure": callable(target.measure), + "compiled_filter": callable(target.filter), + } + + +def _read_target(raw: Mapping) -> Target: + expected = { + "name", + "entity", + "value", + "period", + "source", + "tolerance", + "metadata", + "measure", + "filter", + "compiled_measure", + "compiled_filter", + } + if not isinstance(raw, Mapping) or set(raw) != expected: + raise ValueError("Malformed calibration target descriptor.") + if ( + type(raw["compiled_measure"]) is not bool + or type(raw["compiled_filter"]) is not bool + ): + raise ValueError("Malformed compiled target flags.") + if raw["compiled_measure"] != (raw["measure"] is None): + raise ValueError("Target measure descriptor is inconsistent.") + if raw["compiled_filter"] and raw["filter"] is not None: + raise ValueError("Target filter descriptor is inconsistent.") + return Target( + **{ + key: raw[key] + for key in ( + "name", + "entity", + "value", + "period", + "source", + "tolerance", + "metadata", + ) + }, + measure=raw["measure"] or "__compiled_contribution__", + filter=raw["filter"], + ) + + +@dataclass(frozen=True) +class _CompiledRow: + entity: str + entity_ids: tuple[int | str, ...] + values: sparse.csr_array + + def __call__(self, frame: Frame) -> np.ndarray: + column = frame.schema.entity_id_column(self.entity) + if tuple(frame.table(self.entity)[column]) != self.entity_ids: + raise ValueError( + "Compiled contribution requires its exact ordered entity axis." + ) + # Keep target definitions sparse; compilation materializes one row + # at a time rather than retaining a targets × households dense array. + return self.values.toarray().ravel() + + +@dataclass(frozen=True) +class OrderedProblem: + """An identified numerical problem with explicit ordered entity IDs.""" + + problem: CalibrationProblem + entity_ids: tuple[int | str, ...] + target_metadata: tuple[Mapping, ...] + bindings: Mapping + sha256: str + + def to_target_set(self) -> TargetSet: + """Use the compiled rows in the public solver without original closures. + + Contributions already include original filters and entity aggregation. + No raw measure is recomputed; consuming a different row axis refuses. + """ + return TargetSet( + [ + replace( + target, + entity=self.problem.weight_entity, + filter=None, + measure=_CompiledRow( + self.problem.weight_entity, + self.entity_ids, + self.problem.matrix[index : index + 1], + ), + ) + for index, target in enumerate(self.problem.targets) + ] + ) + + +def encode_problem( + problem: CalibrationProblem, + *, + entity_ids: Sequence[int | str], + target_metadata: Sequence[Mapping] | None = None, + bindings: Mapping | None = None, +) -> bytes: + """Encode numerical rows, axes, skipped facts and declarative bindings.""" + ids = _ids(entity_ids) + if not isinstance(problem, CalibrationProblem): + raise TypeError("encode_problem requires CalibrationProblem.") + if len(ids) != problem.n_weights: + raise ValueError("Problem entity axis differs from the weight columns.") + matrix = sparse.csr_array(problem.matrix, dtype=np.float64, copy=True) + rows = ( + tuple({} for _ in problem.targets) + if target_metadata is None + else tuple(target_metadata) + ) + if len(rows) != problem.n_targets or any( + not isinstance(row, Mapping) for row in rows + ): + raise ValueError("Target metadata must match the ordered target axis.") + metadata = { + "schema": PROBLEM_TYPE.name + ".v1", + "shape": list(matrix.shape), + "entity_ids": ids, + "weight_entity": problem.weight_entity, + "weight_kind": problem.initial_weights.kind.value, + "names": problem.names, + "targets": [_target(target) for target in problem.targets], + "skipped": [ + {"target": _target(item.target), "reason": item.reason} + for item in problem.skipped + ], + "target_metadata": rows, + "bindings": {} if bindings is None else bindings, + } + payload = _pack( + metadata, + { + "data": matrix.data.astype(" OrderedProblem: + """Decode and validate a portable problem, preserving CSR row order.""" + metadata, arrays = _unpack( + payload, + schema=PROBLEM_TYPE.name + ".v1", + members={"data", "indices", "indptr", "targets", "weights"}, + ) + if set(metadata) != { + "schema", + "shape", + "entity_ids", + "weight_entity", + "weight_kind", + "names", + "targets", + "skipped", + "target_metadata", + "bindings", + }: + raise ValueError("Problem metadata fields differ.") + ids = _ids(metadata["entity_ids"]) + shape = metadata["shape"] + if ( + not isinstance(shape, list) + or len(shape) != 2 + or any(type(n) is not int or n < 0 for n in shape) + ): + raise ValueError("Malformed problem shape.") + if shape[1] != len(ids): + raise ValueError("Problem entity axis differs from the weight columns.") + for key, values in arrays.items(): + dtype = np.dtype("= shape[1]).any() + ): + raise ValueError("Malformed CSR index arrays.") + matrix = sparse.csr_array((data, indices, indptr), shape=tuple(shape)) + targets = tuple(_read_target(raw) for raw in metadata["targets"]) + if len({target.key for target in targets}) != len(targets) or tuple( + metadata["names"] + ) != tuple(target.row_name for target in targets): + raise ValueError("Problem target IDs/names must form a unique ordered axis.") + if arrays["targets"].tolist() != [target.value for target in targets]: + raise ValueError("Problem target values disagree with their descriptors.") + skipped = [] + for item in metadata["skipped"]: + if ( + set(item) != {"target", "reason"} + or not isinstance(item["reason"], str) + or not item["reason"] + ): + raise ValueError("Malformed skipped-target evidence.") + skipped.append(SkippedTarget(_read_target(item["target"]), item["reason"])) + rows = metadata["target_metadata"] + if ( + len(rows) != len(targets) + or any(not isinstance(row, dict) for row in rows) + or not isinstance(metadata["bindings"], dict) + ): + raise ValueError("Problem row metadata/bindings differ from their contract.") + problem = CalibrationProblem( + matrix, + arrays["targets"], + tuple(metadata["names"]), + Weights(arrays["weights"], WeightKind(metadata["weight_kind"])), + metadata["weight_entity"], + targets, + tuple(skipped), + ) + return OrderedProblem( + problem, + ids, + tuple(rows), + metadata["bindings"], + hashlib.sha256(payload).hexdigest(), + ) + + +@dataclass(frozen=True) +class OrderedSolution: + """Aligned calibrated weights bound to an exact problem artifact.""" + + weights: np.ndarray + entity_ids: tuple[int | str, ...] + problem_sha256: str + diagnostics: Mapping + sha256: str + + +def encode_solution( + weights: Sequence[float] | np.ndarray, + *, + entity_ids: Sequence[int | str], + problem_sha256: str, + diagnostics: Mapping | None = None, +) -> bytes: + """Encode a solution once; later population installation need not solve.""" + ids = _ids(entity_ids) + payload = _pack( + { + "schema": SOLUTION_TYPE.name + ".v1", + "entity_ids": ids, + "problem_sha256": _sha(problem_sha256), + "diagnostics": {} if diagnostics is None else diagnostics, + }, + {"weights": np.asarray(weights, dtype=" OrderedSolution: + """Validate solution axes, finite nonnegative weights and optional binding.""" + metadata, arrays = _unpack( + payload, schema=SOLUTION_TYPE.name + ".v1", members={"weights"} + ) + if set(metadata) != { + "schema", + "entity_ids", + "problem_sha256", + "diagnostics", + } or not isinstance(metadata["diagnostics"], dict): + raise ValueError("Solution metadata fields differ.") + ids = _ids(metadata["entity_ids"]) + digest = _sha(metadata["problem_sha256"]) + weights = arrays["weights"] + if ( + weights.dtype != np.dtype(" bytes: + """Persist the complete numerical state needed by dense/search continuation. + + A result is bound to a separately encoded ordered problem. Optimizer state + is not claimed: continuation rebuilds completed results, never mid-epoch + training. Grouped solves require a separate constraints-aware protocol. + """ + ids = _ids(entity_ids) + if len(ids) != len(result.weights): + raise ValueError("Calibration result differs from the ordered entity axis.") + if ( + "grouped_upper_bounds" in result.options + or "grouped_preserve_zeros" in result.options + ): + raise ValueError( + "Grouped calibration results require their original constraints." + ) + probabilities = result.gate_open_probabilities + return _pack( + { + "schema": RESULT_TYPE.name + ".v1", + "entity_ids": ids, + "problem_sha256": _sha(problem_sha256), + "l0_lambda": float(result.l0_lambda), + "n_nonzero": int(result.n_nonzero), + "target_loss_cap": float(result.target_loss_cap), + "closing_loss": float(result.final_loss), + "options": dict(result.options), + "has_probabilities": probabilities is not None, + }, + { + "weights": np.asarray(result.weights, dtype=" tuple[str, int | str]: """The ``(name, period)`` identity of the fact.""" return (self.name, self.period) + def to_dict(self) -> dict[str, Any]: + """The JSON-ready mapping form of the spec (``dataclasses.asdict``). + + ``hierarchy`` becomes a nested mapping; :meth:`from_dict` restores it. + This is the one encoding every artifact that carries specs uses, so a + registry JSON file and a graph artifact serialize a spec identically. + """ + return asdict(self) + + @classmethod + def from_dict(cls, raw: Mapping[str, Any]) -> TargetSpec: + """Rebuild a spec from :meth:`to_dict` output. + + ``hierarchy`` may be the nested mapping :meth:`to_dict` wrote, an + already-decoded :class:`CalibrationHierarchy`, or ``None``. Every other + field is passed through to the constructor, whose validation still + applies (including the hierarchy target-id check). + """ + fields_ = dict(raw) + hierarchy = fields_.get("hierarchy") + if isinstance(hierarchy, Mapping): + fields_["hierarchy"] = CalibrationHierarchy.from_dict(dict(hierarchy)) + return cls(**fields_) + def to_target(self) -> Target: """Compile the spec into a calibration :class:`Target`.""" return Target( @@ -251,7 +276,7 @@ def version(self) -> str: canonical = json.dumps( { "country": self._country, - "specs": [asdict(spec) for spec in self._specs], + "specs": [spec.to_dict() for spec in self._specs], }, sort_keys=True, separators=(",", ":"), @@ -309,7 +334,7 @@ def to_json(self, path: str | Path) -> Path: "country": self._country, "version": self.version, "n_specs": len(self._specs), - "specs": [asdict(spec) for spec in self._specs], + "specs": [spec.to_dict() for spec in self._specs], } path.write_text(json.dumps(payload, indent=1), encoding="utf-8") return path @@ -333,19 +358,7 @@ def from_json(cls, path: str | Path) -> TargetRegistry: f"{sorted(_READABLE_FORMAT_VERSIONS)!r})." ) raw_specs = payload["specs"] - specs = tuple( - TargetSpec( - **{ - **raw, - "hierarchy": ( - CalibrationHierarchy.from_dict(raw["hierarchy"]) - if raw.get("hierarchy") is not None - else None - ), - } - ) - for raw in raw_specs - ) + specs = tuple(TargetSpec.from_dict(raw) for raw in raw_specs) registry = cls(specs, country=payload["country"]) stored = payload.get("version") if format_version == 2: diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/target_selection.py b/packages/microcosm-calibrate/src/microcosm/calibrate/target_selection.py new file mode 100644 index 000000000..4024426de --- /dev/null +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/target_selection.py @@ -0,0 +1,123 @@ +"""Explicit target-scope selection, independent of country and pool sizing. + +Selection does not admit unsupported targets, adjudicate source authority, or +reduce validation requirements. It records only the requested geography scope. +Country adapters normalize their metadata through ``geography_resolver``. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Callable, Sequence +from dataclasses import dataclass + +from .registry import TargetRegistry, TargetSpec + + +@dataclass(frozen=True) +class TargetSelection: + """Selected ordered facts and a detached, reproducible scope receipt.""" + + registry: TargetRegistry + _receipt_json: str + + @property + def receipt(self) -> dict[str, object]: + """Return a fresh receipt; caller mutations cannot alter its identity.""" + return json.loads(self._receipt_json) + + @property + def sha256(self) -> str: + return hashlib.sha256(self.to_bytes()).hexdigest() + + def to_bytes(self) -> bytes: + """Return the canonical selection receipt bytes.""" + return self._receipt_json.encode("utf-8") + + +def _level(spec: TargetSpec) -> str: + return spec.metadata.get("geography_level", "") + + +def select_targets( + registry: TargetRegistry, + *, + geography_levels: Sequence[str] | None = None, + geography_resolver: Callable[[TargetSpec], str] | None = None, +) -> TargetSelection: + """Keep all targets by default, or explicitly select normalized levels. + + Ordering and ``(name, period)`` identity are retained. Unknown level names, + unclassified facts and an empty selected problem refuse. A resolver must + validate country aliases/ambiguities; it is never serialized as a callback. + The receipt stores its resolved result for every original fact instead. + """ + if not isinstance(registry, TargetRegistry): + raise TypeError("Target selection requires TargetRegistry.") + levels = None + if geography_levels is not None: + if isinstance(geography_levels, str | bytes): + raise ValueError("geography_levels must be a nonempty sequence of levels.") + levels = tuple(geography_levels) + if not levels or any( + not isinstance(level, str) or not level or level != level.strip() + for level in levels + ): + raise ValueError("geography_levels must contain nonempty literal names.") + if len(set(levels)) != len(levels): + raise ValueError("geography_levels must not repeat levels.") + levels = tuple(sorted(levels)) + resolve = _level if geography_resolver is None else geography_resolver + resolved = [] + for spec in registry.specs: + level = resolve(spec) + if not isinstance(level, str) or not level or level != level.strip(): + raise ValueError( + f"Target {spec.key!r} lacks normalized geography metadata." + ) + resolved.append(level) + if levels is not None and (unknown := set(levels) - set(resolved)): + raise ValueError( + f"Target selector contains unknown geography levels {sorted(unknown)}." + ) + by_key = { + spec.key: level for spec, level in zip(registry.specs, resolved, strict=True) + } + selected = registry.select( + predicate=lambda spec: levels is None or by_key[spec.key] in levels + ) + if not selected.specs: + raise ValueError("Target selection requires a nonempty selected problem.") + included, excluded = [], [] + for spec, level in zip(registry.specs, resolved, strict=True): + row = { + "name": spec.name, + "period": spec.period, + "geography_level": level, + "family": spec.family, + "source": spec.source, + } + if levels is None or level in levels: + included.append( + { + **row, + "reason": "all_geographies" + if levels is None + else "geography_selected", + } + ) + else: + excluded.append({**row, "reason": "geography_not_selected"}) + receipt = { + "schema": "microcosm.calibrate.target-selection.v1", + "source_registry_version": registry.version, + "selected_registry_version": selected.version, + "selector": {"geography_levels": levels, "explicit": levels is not None}, + "included": included, + "excluded": excluded, + } + return TargetSelection( + selected, + json.dumps(receipt, sort_keys=True, separators=(",", ":"), allow_nan=False), + ) diff --git a/packages/microcosm-calibrate/tests/engine_free/shared/test_ordered_artifacts.py b/packages/microcosm-calibrate/tests/engine_free/shared/test_ordered_artifacts.py new file mode 100644 index 000000000..285940abf --- /dev/null +++ b/packages/microcosm-calibrate/tests/engine_free/shared/test_ordered_artifacts.py @@ -0,0 +1,140 @@ +"""Portable matrix/solution artifacts retain axes and reject corrupt inputs.""" + +import numpy as np +import pandas as pd +import pytest +from scipy import sparse + +from microcosm.calibrate import CalibrationProblem, Target, build_constraint_matrix +from microcosm.calibrate.artifacts import ( + decode_calibration_result, + decode_problem, + decode_solution, + encode_calibration_result, + encode_problem, + encode_solution, +) +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights + + +def problem(): + targets = ( + Target("count", "household", lambda f: np.ones(2), value=10, period=2024), + Target("money", "household", "money", value=21, period="2025"), + ) + return CalibrationProblem( + sparse.csr_array([[1.0, 1.0], [2.0, 5.0]]), + np.array([10.0, 21.0]), + tuple(t.row_name for t in targets), + Weights(np.array([2.0, 3.0]), WeightKind.IMPORTANCE), + "household", + targets, + ) + + +def test_problem_roundtrip_is_deterministic_and_recompiles_bound_rows(): + payload = encode_problem( + problem(), entity_ids=(10, 20), bindings={"selector": "all"} + ) + assert ( + encode_problem(problem(), entity_ids=(10, 20), bindings={"selector": "all"}) + == payload + ) + restored = decode_problem(payload) + assert restored.entity_ids == (10, 20) + assert restored.problem.names == problem().names + assert restored.bindings == {"selector": "all"} + np.testing.assert_array_equal( + restored.problem.matrix.toarray(), problem().matrix.toarray() + ) + frame = Frame( + { + "person": pd.DataFrame( + {"person_id": [1, 2], "person_household_id": [10, 20]} + ), + "household": pd.DataFrame({"household_id": [10, 20]}), + }, + EntitySchema(group_entities=("household",)), + {"household": problem().initial_weights}, + pd.Series(["a", "a"]), + ) + recompiled = build_constraint_matrix( + frame, restored.to_target_set(), weight_entity="household" + ) + np.testing.assert_array_equal( + recompiled.matrix.toarray(), problem().matrix.toarray() + ) + + +def test_artifact_axes_and_solution_binding_are_strict(): + with pytest.raises(ValueError, match="unique"): + encode_problem(problem(), entity_ids=(10, 10)) + with pytest.raises(ValueError, match="axis"): + encode_problem(problem(), entity_ids=(10,)) + bound = decode_problem(encode_problem(problem(), entity_ids=(10, 20))) + payload = encode_solution( + [4.0, 6.0], + entity_ids=(10, 20), + problem_sha256=bound.sha256, + diagnostics={"converged": True}, + ) + result = decode_solution(payload, problem_sha256=bound.sha256, entity_ids=(10, 20)) + np.testing.assert_array_equal(result.weights, [4.0, 6.0]) + with pytest.raises(ValueError, match="axis"): + decode_solution(payload, entity_ids=(20, 10)) + with pytest.raises(ValueError, match="problem"): + decode_solution(payload, problem_sha256="0" * 64) + with pytest.raises(ValueError): + decode_problem(payload) + + +def test_problem_target_definitions_do_not_densify_the_whole_sparse_system(monkeypatch): + bound = decode_problem(encode_problem(problem(), entity_ids=(10, 20))) + # Country matrices can have millions of households and thousands of rows. + # Creating definitions must not materialize any dense contribution row. + monkeypatch.setattr( + sparse.csr_array, "toarray", lambda *a, **kw: pytest.fail("eager densification") + ) + assert len(bound.to_target_set().targets) == 2 + + +def test_complete_result_rebuild_preserves_dense_and_search_state(monkeypatch): + import microcosm.calibrate.solve as solve + from microcosm.calibrate import calibrate + + frame = Frame( + { + "person": pd.DataFrame( + {"person_id": [1, 2], "person_household_id": [10, 20]} + ), + "household": pd.DataFrame({"household_id": [10, 20]}), + }, + EntitySchema(group_entities=("household",)), + {"household": problem().initial_weights}, + pd.Series(["a", "a"]), + ) + bound = decode_problem(encode_problem(problem(), entity_ids=(10, 20))) + for penalty in (0.0, 0.01): + result = calibrate( + frame, + bound.to_target_set(), + epochs=3, + seed=17, + mass="free", + l0_lambda=penalty, + ) + payload = encode_calibration_result( + result, entity_ids=(10, 20), problem_sha256=bound.sha256 + ) + with monkeypatch.context() as context: + context.setattr( + solve, "_optimize", lambda *a, **kw: pytest.fail("replay optimized") + ) + restored = decode_calibration_result(payload, frame=frame, problem=bound) + np.testing.assert_array_equal(restored.weights, result.weights) + np.testing.assert_array_equal(restored.loss_trajectory, result.loss_trajectory) + assert restored.options == result.options + if penalty: + np.testing.assert_array_equal( + restored.gate_open_probabilities, result.gate_open_probabilities + ) diff --git a/packages/microcosm-calibrate/tests/engine_free/shared/test_registry.py b/packages/microcosm-calibrate/tests/engine_free/shared/test_registry.py index 8c607b073..f4b6c846d 100644 --- a/packages/microcosm-calibrate/tests/engine_free/shared/test_registry.py +++ b/packages/microcosm-calibrate/tests/engine_free/shared/test_registry.py @@ -156,6 +156,30 @@ def test_families_and_select(self) -> None: assert len(soi) == 1 and next(iter(soi)).name == "irs_soi/agi" +class TestSpecDictCodec: + def test_to_dict_from_dict_round_trip_with_and_without_hierarchy(self) -> None: + with_hierarchy = _spec(se=12.5, metadata={"kind": "x"}, hierarchy=_hierarchy()) + without = _spec(name="puf/agi", family="irs_soi", value=5e12) + for spec in (with_hierarchy, without): + raw = spec.to_dict() + assert TargetSpec.from_dict(raw) == spec + # JSON turns tuples into lists; the decode must not care. + assert TargetSpec.from_dict(json.loads(json.dumps(raw))) == spec + assert isinstance(with_hierarchy.to_dict()["hierarchy"], dict) + assert without.to_dict()["hierarchy"] is None + + def test_from_dict_accepts_an_already_decoded_hierarchy(self) -> None: + spec = _spec(hierarchy=_hierarchy()) + raw = {**spec.to_dict(), "hierarchy": spec.hierarchy} + assert TargetSpec.from_dict(raw) == spec + + def test_from_dict_keeps_the_hierarchy_target_id_check(self) -> None: + raw = _spec(hierarchy=_hierarchy()).to_dict() + raw["name"] = "another/name" + with pytest.raises(ValueError, match="hierarchy target id"): + TargetSpec.from_dict(raw) + + class TestArtifactRoundTrip: def test_round_trip_preserves_everything(self, tmp_path) -> None: registry = TargetRegistry( diff --git a/packages/microcosm-calibrate/tests/engine_free/shared/test_target_selection.py b/packages/microcosm-calibrate/tests/engine_free/shared/test_target_selection.py new file mode 100644 index 000000000..74c40fddd --- /dev/null +++ b/packages/microcosm-calibrate/tests/engine_free/shared/test_target_selection.py @@ -0,0 +1,72 @@ +"""Target scope is explicit and preserves ordered fact identities.""" + +import pytest + +from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.calibrate.target_selection import select_targets + + +def registry(): + return TargetRegistry( + [ + TargetSpec( + "population", + "person", + 20, + "one", + period=2024, + source="census", + metadata={"geography_level": "country"}, + ), + TargetSpec( + "population", + "person", + 22, + "one", + period=2025, + source="census", + metadata={"geography_level": "region"}, + ), + TargetSpec( + "local", + "person", + 5, + "one", + period=2025, + source="census", + metadata={"geography_level": "district"}, + ), + ], + country="test", + ) + + +def test_default_all_and_explicit_country_are_distinct(): + all_targets = select_targets(registry()) + assert all_targets.registry.specs == registry().specs + assert all_targets.receipt["selector"] == { + "geography_levels": None, + "explicit": False, + } + selected = select_targets(registry(), geography_levels=("country",)) + assert [spec.key for spec in selected.registry.specs] == [("population", 2024)] + assert [row["period"] for row in selected.receipt["excluded"]] == [2025, 2025] + assert all( + row["reason"] == "geography_not_selected" + for row in selected.receipt["excluded"] + ) + assert selected.receipt["source_registry_version"] == registry().version + + +def test_missing_unknown_empty_and_resolved_geographies(): + with pytest.raises(ValueError, match="unknown"): + select_targets(registry(), geography_levels=("missing",)) + with pytest.raises(ValueError, match="nonempty"): + select_targets(registry(), geography_levels=()) + missing = TargetRegistry( + [TargetSpec("x", "person", 1, "one", source="test")], country="test" + ) + with pytest.raises(ValueError, match="geography"): + select_targets(missing) + resolved = select_targets(missing, geography_resolver=lambda spec: "country") + assert resolved.receipt["included"][0]["geography_level"] == "country" From b325e8ed1834b0e3c4d291531187037f07a29ac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:19:22 +0100 Subject: [PATCH 20/44] Move the UK spine build into the package as spine_build, with graph gate nodes and stored stage evidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tools/build_uk_frs_spine.py becomes uk_runtime/spine_build.py (git mv: main's current tool is the base, so the NTS bus stage, the CGT asset-type and anchor stages, the reserved SPI band donors, LCFS without WAS, staging telemetry and delivery, the synthetic smoke mode and the fit-weight proxies all stay). Layered on it from #901: parse_uk_spine_args / prepare_uk_spine_execution / PreparedUKSpineExecution extracted from the driver body; the spine gates become graph nodes (uk_runtime/graph_evidence, new: UKSpineGateKernel with the synthetic_smoke posture, the enum-domain artifacts including capital_gains_asset_type_enum_domain, materialised gate reports); stage evidence and fit-weight records are read back from the content store through the shared stage_evidence artifact (which now walks the observation and graph-source proxies the way the driver's collector did) instead of in-memory collectors; the HMRC replay sidecar is rebuilt from the SPI stage's checkpoint metadata; the sidecar records the operation inventory and the graph manifest; graph.py declares the evidence artifacts, numerical-dependency versions, the spine endpoint and the operation inventory. The tool file is a six-line shim over spine_build.main, so the staging integration test and every runbook command keep working. UK_SPINE_EXCLUSIONS and the frs_hmrc_leaves wiring are untouched in this commit. Tests: test_uk_frs_spine.py loads spine_build by path (module copies registered in sys.modules for dataclass annotation resolution on 3.14), the replay setattr is gone, the four collector tests read through snapshot_stage_evidence; test_uk_graph.py imports the package module; test_uk_graph_evidence.py added. H2 UK spine parity fixture regenerated (oracle edb1659b…; PRODUCED_BY unchanged). Known behaviour change for review: a blocked assembled gate now fails inside run_graph, so the gate report lives in the content store rather than in spine_gates.json on that path. Verified: targeted set 82 passed; spine-uk group 2,825 passed / 21 skipped; H2 1/1; ci_test_groups --verify ok; ruff clean on edited files. Co-Authored-By: Claude Fable 5.1 --- .../src/microcosm/build/stage_evidence.py | 30 +- .../src/microcosm/build/uk_runtime/graph.py | 111 + .../build/uk_runtime/graph_evidence.py | 430 ++++ .../build/uk_runtime/graph_kernels.py | 60 +- .../microcosm/build/uk_runtime/spine_build.py | 2003 ++++++++++++++++ .../tests/engine/uk/test_uk_graph.py | 9 +- .../tests/engine_free/uk/test_uk_frs_spine.py | 85 +- .../engine_free/uk/test_uk_graph_evidence.py | 221 ++ .../fixtures/parity/uk_spine/uk_spine.json | 2 +- tools/build_uk_frs_spine.py | 2035 +---------------- 10 files changed, 2912 insertions(+), 2074 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py diff --git a/packages/microcosm-build/src/microcosm/build/stage_evidence.py b/packages/microcosm-build/src/microcosm/build/stage_evidence.py index f0373a078..2bffe3ef0 100644 --- a/packages/microcosm-build/src/microcosm/build/stage_evidence.py +++ b/packages/microcosm-build/src/microcosm/build/stage_evidence.py @@ -15,6 +15,29 @@ STAGE_EVIDENCE_TYPE = ArtifactType("microcosm.stage-evidence", 1) +def _fit_weight_hook_holder(transform: object | None) -> object | None: + """The object declaring ``fit_weight_records``, seen through the proxies. + + Detects the hook without evaluating it (a raising property must count as + a fitting stage with unreadable records, not vanish), and looks through + the wrappers a staged run puts around every stage: an observation wrapper + (telemetry) and a graph source transform both proxy attribute reads + through ``__getattr__``, which a class-level probe never consults — so a + probe on the outermost object alone silently loses the block and the + release-cut weights audit finds no evidence. + """ + + seen: set[int] = set() + candidate: object | None = transform + while candidate is not None and id(candidate) not in seen: + seen.add(id(candidate)) + declared = getattr(type(candidate), "fit_weight_records", None) is not None + if declared or "fit_weight_records" in getattr(candidate, "__dict__", {}): + return candidate + candidate = getattr(candidate, "__dict__", {}).get("transform") + return None + + def snapshot_stage_evidence(stage: str, transform: object | None) -> dict[str, object]: """Snapshot checkpoint, fit-weight and sampling evidence without rerunning.""" @@ -42,11 +65,10 @@ def snapshot_stage_evidence(stage: str, transform: object | None) -> dict[str, o } # Do not evaluate a raising property merely to detect whether it exists: # missing/unreadable fitting evidence must stay visible to weight audits. - exposes_records = getattr(type(transform), "fit_weight_records", None) is not None - exposes_records |= "fit_weight_records" in getattr(transform, "__dict__", {}) - if exposes_records: + holder = _fit_weight_hook_holder(transform) + if holder is not None: try: - records = tuple(transform.fit_weight_records or ()) + records = tuple(holder.fit_weight_records or ()) payload["fit_weight_records"] = [ { "fit_name": str(record.fit_name), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py index b2f3b1e17..ee1e3e316 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py @@ -18,8 +18,10 @@ import json from collections.abc import Iterable, Mapping from dataclasses import dataclass +from importlib import metadata from microcosm.graph import ( + ArtifactOutput, Graph, KernelRegistry, Node, @@ -27,16 +29,20 @@ Slice, SourceRef, StructuralDelta, + compile_graph, ) from ..country_spec import CountrySpec, load_country_spec +from ..stage_evidence import STAGE_EVIDENCE_TYPE from .national_sampling import UK_SAMPLE_SEED_DEFAULT __all__ = [ "UK_SPINE_EXCLUSIONS", "UK_SPINE_STRUCTURAL_STAGES", "uk_registry", + "uk_spine_endpoint", "uk_spine_graph", + "uk_spine_operation_inventory", ] @@ -965,6 +971,29 @@ def _source_refs(source_mode: str) -> tuple[SourceRef, ...]: ) +def _numerical_dependency_versions() -> tuple[tuple[str, str], ...]: + """Bind installed behavior-bearing libraries, including optional engines. + + The missing marker permits source-only declarations without engine extras; + installing the engine then produces a different identity, never a false hit. + """ + versions = [] + for name in ( + "policyengine-uk", + "policyengine-core", + "numpy", + "pandas", + "scikit-learn", + "quantile-forest", + ): + try: + version = metadata.version(name) + except metadata.PackageNotFoundError: + version = "not-installed" + versions.append((name, version)) + return tuple(versions) + + def uk_spine_graph( spec: CountrySpec | None = None, *, @@ -984,6 +1013,7 @@ def uk_spine_graph( raise ValueError("UK graph sample_seed must be non-negative.") resolved = load_country_spec("uk") if spec is None else spec stages = _manifest_stages(resolved) + dependency_versions = _numerical_dependency_versions() # The root transform loads the complete national-frame seed schema even # when a reduced hermetic manifest names only the output under test. # CREATE must declare every loaded cell, never merely the StagePlan's @@ -997,12 +1027,14 @@ def uk_spine_graph( kernel="uk.create@1", outputs=tuple(cell.owned() for cell in root_cells), structural=StructuralDelta.CREATE, + artifact_outputs=(ArtifactOutput("stage_evidence", STAGE_EVIDENCE_TYPE),), sources=_source_names("frs_spine", source_mode), params={ "time_period": "2024", "stage_contract_sha256": _stage_contract_sha256(stages[0], resolved), "sample_fraction": float(sample_fraction), "sample_seed": int(sample_seed), + "numerical_dependencies": dependency_versions, }, description="Load the source-bound UK FRS root population.", ) @@ -1054,6 +1086,7 @@ def uk_spine_graph( ), params={ "stage": stage_name, + "numerical_dependencies": dependency_versions, "time_period": "2024", "expand_cells": tuple( (cell.entity, cell.column, cell.dtype) for cell in cells @@ -1065,6 +1098,9 @@ def uk_spine_graph( ), }, structural=StructuralDelta.EXPAND, + artifact_outputs=( + ArtifactOutput("stage_evidence", STAGE_EVIDENCE_TYPE), + ), base=current_population, sources=_source_names(stage_name, source_mode), mass=_STRUCTURAL_MASS[stage_name], @@ -1138,8 +1174,12 @@ def uk_spine_graph( for cell in cells ), population=current_population, + artifact_outputs=( + ArtifactOutput("stage_evidence", STAGE_EVIDENCE_TYPE), + ), params={ "stage": stage_name, + "numerical_dependencies": dependency_versions, "time_period": "2024", "stage_contract_sha256": _stage_contract_sha256( manifest_stage, resolved @@ -1191,3 +1231,74 @@ def uk_registry( uk_spine_graph() if graph is None else graph, {} if implementations is None else implementations, ) + + +@dataclass(frozen=True) +class UKSpineEndpoint: + """The final population and complete declared cell surface for composition.""" + + population: str + inputs: tuple[Slice, ...] + stage_names: tuple[str, ...] + + +def uk_spine_endpoint(graph: Graph) -> UKSpineEndpoint: + stages = ( + "frs_spine", + *(str(node.params["stage"]) for node in graph.nodes if "stage" in node.params), + ) + live = {} + for node in graph.nodes: + for owned in node.outputs: + live[(owned.entity, owned.column)] = _Cell( + owned.entity, owned.column, owned.dtype + ) + return UKSpineEndpoint( + population=compile_graph(graph).versions[stages[-1]], + inputs=_slices(live), + stage_names=stages, + ) + + +def uk_spine_operation_inventory( + graph: Graph, spec: CountrySpec | None = None +) -> tuple[dict[str, object], ...]: + """Generate truthful operation ownership from the executable stage roster. + + Conditional fits/draw chains remain one coupled execution unit. In + particular WAS encoding observes donors and recipients jointly; its fit + is not advertised as an independently reusable donor-only artifact. + """ + resolved = load_country_spec("uk") if spec is None else spec + nodes = {node.id: node for node in graph.nodes} + rows = [] + for stage in _manifest_stages(resolved): + node_id = "create_uk_frs" if stage.stage == "frs_spine" else stage.stage + node = nodes[node_id] + rows.append( + { + "stage": stage.stage, + "node": node_id, + "kernel": node.kernel, + "operations": [ + {"kind": operation.kind, "parameters": dict(operation.parameters)} + for operation in stage.operations + ], + "execution_unit": "composite" + if len(stage.operations) > 1 + else "single", + "source_inputs": list(node.sources), + "artifact_outputs": [output.name for output in node.artifact_outputs], + "randomness": "Existing literal/child seeds and draw order are preserved inside the registered transform.", + "coupling": ( + "Donor and recipient region encoding, four segmented fit/draw chains and their child seeds remain coupled." + if stage.stage == "was_wealth" + else "Source assembly, declared household sample selection and same-kind mass normalization execute once in CREATE." + if stage.stage == "frs_spine" + else "Declared preparation, fit and application operations execute once in this stage; intermediate models are not independently cached." + if any("qrf" in operation.kind for operation in stage.operations) + else None + ), + } + ) + return tuple(rows) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py new file mode 100644 index 000000000..f1f8c36c5 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py @@ -0,0 +1,430 @@ +"""UK spine evidence and gate bindings for the shared graph/store contracts.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping, Sequence +from dataclasses import replace + +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + KernelRole, + Node, + Numeric, + SeedSource, + compile_graph, + source_hash, +) + +from .. import gate_battery +from ..country_spec import CountrySpec, GatesManifest +from ..gate_battery import ( + BlockingMode, + EvidenceContext, + GateBatteryRun, + evaluate_phase, + gate_phase_report_from_payload, + gate_phase_report_payload, +) +from ..stage_evidence import ( + STAGE_EVIDENCE_TYPE, + decode_stage_evidence, + encode_stage_evidence, +) +from . import battery_bindings +from .calibration_run import UK_SPINE_GATE_SCOPE, uk_scoped_gate_manifest +from .cgt_asset_type import CGT_ASSET_TYPE_DOMAIN +from .frs_relationships import CHRONICLE_ONS_HOUSEHOLD_TYPE_VALUE_IDS + +SPINE_GATE_REPORT_TYPE = ArtifactType("microcosm.gate-phase-report", 1) + + +def require_uk_spine_gate_admission( + context: KernelContext, *, alias: str = "spine_gate" +) -> None: + """Enforce a declared, already persisted spine phase before doing more work.""" + artifact = context.artifacts.get(alias) + if artifact is None: + if any(edge.name == alias for edge in context.node.artifact_inputs): + raise ValueError("The declared spine gate admission artifact is absent.") + return + from ..country_spec import load_country_spec + + report = gate_phase_report_from_payload( + json.loads(artifact.payload), + gates=uk_spine_gate_manifest(load_country_spec("uk")), + ) + if report.phase != context.params["spine_gate_phase"]: + raise ValueError("Spine admission report belongs to a different phase.") + blocking = report.blocking_outcomes( + release_candidate=bool(context.params["spine_gate_release_candidate"]), + synthetic_smoke=bool(context.params["spine_gate_synthetic_smoke"]), + ) + if blocking: + raise ValueError( + f"Stored {report.phase} spine gates block downstream execution: " + + ", ".join(outcome.entry.id for outcome in blocking) + ) + + +def uk_spine_gate_manifest(spec: CountrySpec) -> GatesManifest | None: + if getattr(spec, "gates", None) is None: + return None + return uk_scoped_gate_manifest( + UK_SPINE_GATE_SCOPE, + phases=("assembled", "transferred"), + policy_suffix="spine_build_scope", + source=spec.gates, + ) + + +def uk_spine_gate_artifacts(engine: object) -> dict[str, object]: + """Evidence artifacts every UK gate phase needs beside stage evidence. + + The rules engine, plus frame-only enum domains: #791 made + ``ons_household_type`` a frame column rather than an engine variable, so + its ``enum_domain`` gate takes the declared domain as an artifact, and + #725 made ``capital_gains_asset_type`` a frame column whose domain the + asset-type stage declares. One definition serves the spine-phase gate + kernel, the full-build final gates and the tests that reproduce either + report. + """ + + return { + "rules_engine": engine, + "ons_household_type_enum_domain": CHRONICLE_ONS_HOUSEHOLD_TYPE_VALUE_IDS, + "capital_gains_asset_type_enum_domain": CGT_ASSET_TYPE_DOMAIN, + } + + +def load_spine_stage_artifacts( + manifest, store, *, stage_names: Sequence[str] +) -> dict[str, dict[str, object]]: + """Read every requested stage from verified bytes, with no live transforms.""" + result = {} + for stage in stage_names: + node = "create_uk_frs" if stage == "frs_spine" else stage + receipt = manifest.nodes[node] + try: + key = receipt.opaque_artifacts["stage_evidence"] + except KeyError as error: + raise ValueError( + f"Spine stage {stage!r} has no stored evidence artifact." + ) from error + result[stage] = decode_stage_evidence(store.load_bytes(key), stage=stage) + return result + + +def spine_sidecar_evidence( + artifacts: Mapping[str, Mapping[str, object]], +) -> dict[str, object]: + """Project portable stage contracts onto the maintained sidecar schema.""" + return { + "stage_evidence": { + stage: document["evidence"] + for stage, document in artifacts.items() + if document["evidence"] is not None + }, + "fit_weight_records": { + stage: document["fit_weight_records"] + for stage, document in artifacts.items() + if "fit_weight_records" in document + }, + "sampling": artifacts["frs_spine"]["sampling"], + } + + +def add_uk_spine_gate_nodes( + graph: Graph, + *, + spec: CountrySpec, + engine_identity: str, + release_candidate: bool = False, + synthetic_smoke: bool = False, +) -> Graph: + """Attach gates to exact assembled/transferred versions and stored evidence. + + The assembled gate reads the version frozen *by* the BRMA checkpoint, + before subsequent wealth rewrites. Signing and output paths stay external. + ``synthetic_smoke`` mirrors the driver's ``GateBatteryRun`` posture: a + data-only fixture is not the population its facts describe, so + population-fact gates never block such a run. + """ + from .graph import uk_spine_endpoint + + gates = uk_spine_gate_manifest(spec) + if gates is None: + return graph + if not engine_identity: + raise ValueError("Spine gates require a declared rules-engine identity.") + if synthetic_smoke and release_candidate: + raise ValueError("a synthetic smoke build cannot be a release candidate.") + if any(node.kernel == UKSpineGateKernel.ref for node in graph.nodes): + raise ValueError("Spine gate nodes are already registered.") + compiled = compile_graph(graph) + endpoint = uk_spine_endpoint(graph) + stages = endpoint.stage_names + if "frs_brma" not in stages: + raise ValueError("Spine gates require the assembled frs_brma boundary.") + end = stages.index("frs_brma") + 1 + checkpoints = ( + ( + "assembled", + compiled.versions["frs_brma"], + graph.node("frs_brma.checkpoint").inputs, + stages[:end], + ), + ("transferred", endpoint.population, endpoint.inputs, stages), + ) + nodes = list(graph.nodes) + for phase, population, inputs, phase_stages in checkpoints: + if phase == "transferred" and end == len(stages): + continue + nodes.append( + Node( + id=f"spine.gates.{phase}", + kernel=UKSpineGateKernel.ref, + inputs=inputs, + population=population, + artifact_inputs=tuple( + ArtifactInput( + stage, + "create_uk_frs" if stage == "frs_spine" else stage, + "stage_evidence", + STAGE_EVIDENCE_TYPE, + ) + for stage in phase_stages + ) + + ( + ( + ArtifactInput( + "previous_gate", + "spine.gates.assembled", + "gate_report", + SPINE_GATE_REPORT_TYPE, + ), + ) + if phase == "transferred" + else () + ), + artifact_outputs=( + ArtifactOutput("gate_report", SPINE_GATE_REPORT_TYPE), + ), + params={ + "phase": phase, + "time_period": "2024", + "stage_names": phase_stages, + "gate_manifest": json.dumps( + gate_battery._gates_manifest_payload(gates), sort_keys=True + ), + "engine_identity": engine_identity, + "release_candidate": release_candidate, + "synthetic_smoke": synthetic_smoke, + }, + description=f"Evaluate the {phase} spine battery against its exact population and stage evidence.", + ) + ) + if end < len(stages): + # GATE receipts preserve failures but do not themselves stop kernels. + # Make the original assembled admission boundary an explicit dependency + # of the first later model, after the report has reached the store. + next_stage = stages[end] + nodes = [ + replace( + node, + artifact_inputs=( + *node.artifact_inputs, + ArtifactInput( + "spine_gate", + "spine.gates.assembled", + "gate_report", + SPINE_GATE_REPORT_TYPE, + ), + ), + params={ + **node.params, + "spine_gate_phase": "assembled", + "spine_gate_release_candidate": release_candidate, + "spine_gate_synthetic_smoke": synthetic_smoke, + }, + ) + if node.id == next_stage + else node + for node in nodes + ] + return replace(graph, nodes=tuple(nodes)) + + +class UKSpineGateKernel(KernelBase): + """Evaluate one spine gate phase from stored stage evidence. + + Node parameters: ``phase``, ``stage_names`` (the evidence artifacts to + decode), ``gate_manifest`` and ``engine_identity`` (which must match the + bound manifest and engine), ``release_candidate`` and ``synthetic_smoke``. + The last two are the battery's blocking posture, carried on the node so + the persisted verdict binds the policy it was judged under: + ``synthetic_smoke`` mirrors ``GateBatteryRun(synthetic_smoke=...)`` — on a + data-only fixture the population-fact gates are recorded but never block. + """ + + ref = "uk.spine-gates@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + seed_source=SeedSource.NONE, + role=KernelRole.GATE, + ) + + def __init__(self, *, gates: GatesManifest, engine: object, engine_identity: str): + self.gates = gates + self.engine = engine + self.engine_identity = engine_identity + + def implementation_hash(self) -> str: + from microcosm.graph.canonical import canonical_json + + from ..country_spec import load_country_spec + + return hashlib.sha256( + canonical_json( + { + "code": source_hash(type(self), gate_battery, battery_bindings), + "country_resources": load_country_spec("uk").fingerprint, + } + ) + ).hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + from microcosm.frame import Frame, MassChangeRecord + + from .graph_kernels import _minimal_frame + + expected = json.dumps( + gate_battery._gates_manifest_payload(self.gates), sort_keys=True + ) + if ( + context.params["gate_manifest"] != expected + or context.params["engine_identity"] != self.engine_identity + ): + raise ValueError( + "Spine gate binding differs from its declared manifest or engine." + ) + release_candidate = bool(context.params["release_candidate"]) + synthetic_smoke = bool(context.params["synthetic_smoke"]) + previous = context.artifacts.get("previous_gate") + if previous is not None: + previous_report = gate_phase_report_from_payload( + json.loads(previous.payload), gates=self.gates + ) + if previous_report.blocking_outcomes( + release_candidate=release_candidate, + synthetic_smoke=synthetic_smoke, + ): + report = gate_battery.GatePhaseReport( + phase=str(context.params["phase"]), + outcomes=tuple( + gate_battery.GateOutcome( + entry=entry, + status=gate_battery.GateStatus.UNREACHED, + reason="The assembled spine gate blocked this phase.", + ) + for entry in self.gates.gates + if entry.phase == context.params["phase"] + ), + ) + return KernelResult( + artifacts={ + "gate_report": encode_stage_evidence( + gate_phase_report_payload(report, gates=self.gates) + ) + }, + receipt={"outcome": "unreached", "phase": report.phase}, + ) + artifacts = { + stage: decode_stage_evidence(context.artifacts[stage].payload, stage=stage) + for stage in context.params["stage_names"] + } + evidence = spine_sidecar_evidence(artifacts) + minimal = _minimal_frame(context) + root_context = artifacts["frs_spine"]["frame_context"] + mass_records = [ + MassChangeRecord(**row) + for document in artifacts.values() + for row in document["frame_mass_log_append"] + ] + frame = Frame( + {entity: minimal.table(entity) for entity in minimal.entities}, + minimal.schema, + { + entity: minimal.weights_for(entity) + for entity in minimal.weighted_entities + }, + minimal.strata, + mass_log=tuple(mass_records), + metadata=root_context["metadata"], + ) + report = evaluate_phase( + self.gates, + str(context.params["phase"]), + EvidenceContext( + frame=frame, + artifacts={**evidence, **uk_spine_gate_artifacts(self.engine)}, + ), + registry=battery_bindings.UK_GATE_REGISTRY, + ) + blocked = report.blocking_outcomes( + release_candidate=release_candidate, + synthetic_smoke=synthetic_smoke, + ) + return KernelResult( + artifacts={ + "gate_report": encode_stage_evidence( + gate_phase_report_payload(report, gates=self.gates) + ) + }, + receipt={"outcome": "fail" if blocked else "pass", "phase": report.phase}, + ) + + +def register_spine_gate_kernel( + registry: KernelRegistry, + *, + spec: CountrySpec, + engine: object, + engine_identity: str, +) -> None: + gates = uk_spine_gate_manifest(spec) + if gates is not None: + registry.register( + UKSpineGateKernel( + gates=gates, engine=engine, engine_identity=engine_identity + ) + ) + + +def materialize_spine_gate_reports( + manifest, store, *, battery: GateBatteryRun, gates: GatesManifest +) -> None: + """Restore cached verdicts, then use the original write-before-block policy.""" + for phase in gates.phases: + node = f"spine.gates.{phase}" + if node not in manifest.nodes: + continue + key = manifest.nodes[node].opaque_artifacts.get("gate_report") + if key is None: + raise ValueError(f"Spine gate {phase!r} produced no persisted report.") + report = gate_phase_report_from_payload( + json.loads(store.load_bytes(key)), gates=gates + ) + battery.record_phase(report) + battery.enforce(phase, mode=BlockingMode.BLOCKS_ARTIFACT) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index 6609d05bd..e34993b1d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -44,6 +44,7 @@ ) from microcosm.graph.population import dtype_for_token +from .. import stage_evidence from . import bus_use_incidence, uc_relationships from .national_frame import UK_NATIONAL_SCHEMA from .rowwise_geography import id_multiplier_for_values @@ -151,13 +152,24 @@ def _stage_module(stage: str): def _implementation_hash(kernel: object, stage: str, transform: object | None) -> str: + from . import graph_evidence + # The stage module is the behavior-bearing source in every mode. Hashing # an injected transform's dynamic test wrapper would make # hermetic registries unhashable and, more importantly, would fail to bind - # production edits made elsewhere in that stage's module. - del transform + # production edits made elsewhere in that stage's module. A transform may + # still declare extra behavior-bearing sources through + # ``graph_implementation_dependencies``. + dependencies = getattr(transform, "graph_implementation_dependencies", None) return source_hash( - type(kernel), _stage_module(stage), *_STAGE_HELPER_MODULES.get(stage, ()) + type(kernel), + stage_evidence, + graph_evidence, + _stage_artifacts, + _mass_log_payload, + _stage_module(stage), + *_STAGE_HELPER_MODULES.get(stage, ()), + *(dependencies() if callable(dependencies) else ()), ) @@ -177,6 +189,29 @@ def _mass_log_payload(before: Frame, after: Frame) -> list[dict[str, object]]: ] +def _stage_artifacts( + stage: str, transform: object | None, before: Frame | None, after: Frame +) -> dict[str, bytes]: + document = stage_evidence.snapshot_stage_evidence(stage, transform) + document["frame_mass_log_append"] = ( + [ + { + "entity": record.entity, + "old_total": record.old_total, + "new_total": record.new_total, + "declared_factor": record.declared_factor, + "reason": record.reason, + } + for record in after.mass_log + ] + if before is None + else _mass_log_payload(before, after) + ) + if before is None: + document["frame_context"] = {"metadata": dict(after.metadata)} + return {"stage_evidence": stage_evidence.encode_stage_evidence(document)} + + def _invoke_transform(transform: object, frame: Frame, context: KernelContext): """Invoke a stage, giving context-bound adapters only declared sources.""" @@ -742,7 +777,10 @@ def run(self, context: KernelContext) -> KernelResult: raise TypeError( f"The UK root transform returned {type(frame).__name__}, not Frame." ) - return KernelResult(frame=_normalize_create_frame(frame, context)) + return KernelResult( + frame=_normalize_create_frame(frame, context), + artifacts=_stage_artifacts("frs_spine", self.transform, None, frame), + ) class UKIdentityKernel(KernelBase): @@ -805,6 +843,9 @@ def implementation_hash(self) -> str: return _implementation_hash(self, self.stage, self.transform) def run(self, context: KernelContext) -> KernelResult: + from .graph_evidence import require_uk_spine_gate_admission + + require_uk_spine_gate_admission(context) transform = self.transform if transform is None and self.fixture_resolver is not None: transform = self.fixture_resolver.resolve(self.stage, context) @@ -827,6 +868,7 @@ def run(self, context: KernelContext) -> KernelResult: } return KernelResult( columns=MappingProxyType(columns), + artifacts=_stage_artifacts(self.stage, transform, before, after), receipt={ "stage": self.stage, "frame_mass_log_append": _mass_log_payload(before, after), @@ -923,6 +965,9 @@ def implementation_hash(self) -> str: return _implementation_hash(self, self.stage, self.transform) def run(self, context: KernelContext) -> KernelResult: + from .graph_evidence import require_uk_spine_gate_admission + + require_uk_spine_gate_admission(context) transform = self.transform if transform is None and self.fixture_resolver is not None: transform = self.fixture_resolver.resolve(self.stage, context) @@ -984,6 +1029,7 @@ def run(self, context: KernelContext) -> KernelResult: columns=MappingProxyType(columns), expand=MappingProxyType(expand), weights=after_weights, + artifacts=_stage_artifacts(self.stage, transform, before, after), receipt=receipt, ) @@ -1070,7 +1116,11 @@ def build_uk_registry( else: registry.register(UKStageKernel(stage, transform, fixture_resolver)) - required = {node.kernel for node in graph.nodes} + # Gate bindings carry the live rules engine and are registered separately + # after population-stage construction by the composing build. + required = { + node.kernel for node in graph.nodes if node.kernel != "uk.spine-gates@1" + } if set(registry.refs()) != required: missing = sorted(required - set(registry.refs())) extra = sorted(set(registry.refs()) - required) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py new file mode 100644 index 000000000..fbd1158fe --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py @@ -0,0 +1,2003 @@ +"""Build the raw UK FRS spine Frame from pinned local tabs.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import time +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import UTC, datetime +from importlib import metadata +from pathlib import Path + +from microcosm.build.country_spec import ( + GatesManifest, + load_country_spec, +) +from microcosm.build.frame_sampling import ( + normalize_sampled_household_mass, + sample_frame_households, +) +from microcosm.build.gate_battery import BlockingMode, EvidenceContext, GateBatteryRun +from microcosm.build.logbook import canonical_json_bytes +from microcosm.build.logbook_adoption import ( + AttemptState, + append_phase, + apply_error_verdict, + atomic_write_json, + error_receipt_path, + git_code_pin, + local_artifact_reference, + preflight_digest, + record_terminal_attempt, + resolve_predecessor, + role_pins_digest, + sha256_argument, + write_error_receipt, +) +from microcosm.build.observation import ( + ObservedTransform, + StageObservation, + StageObserver, +) +from microcosm.build.plan import StageRecord +from microcosm.build.staging_cli import ( + add_staging_arguments, + validate_staging_arguments, +) +from microcosm.build.staging_v2 import ( + StagingTelemetryV2, + disabled_staging_delivery, +) +from microcosm.build.uk_runtime.age_tail import UKAgeTailStageTransform +from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY +from microcosm.build.uk_runtime.calibration_run import ( + UK_SPINE_GATE_SCOPE, + uk_scoped_gate_manifest, +) +from microcosm.build.uk_runtime.cgt_asset_type import ( + uk_cgt_asset_type_stage_transform, +) +from microcosm.build.uk_runtime.cgt_imputation import uk_cgt_spine_stage_transform +from microcosm.build.uk_runtime.cgt_structure import ( + UKCGTBandDonorStageTransform, + UKCGTIncidenceAnchorStageTransform, + UKCGTIncidenceCloneStageTransform, +) +from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity +from microcosm.build.uk_runtime.etb_services import UKETBServicesStageTransform +from microcosm.build.uk_runtime.etb_vat import UKETBVATStageTransform +from microcosm.build.uk_runtime.frs_brma import UKFRSBRMAStageTransform +from microcosm.build.uk_runtime.frs_council_tax import UKFRSCouncilTaxStageTransform +from microcosm.build.uk_runtime.frs_disability import UKFRSDisabilityStageTransform +from microcosm.build.uk_runtime.frs_education import UKFRSEducationStageTransform +from microcosm.build.uk_runtime.frs_education_grants import ( + FRS_EDUCATION_GRANT_REWRITES, + UKFRSEducationGrantSplitStageTransform, +) +from microcosm.build.uk_runtime.frs_employment import UKFRSEmploymentStageTransform +from microcosm.build.uk_runtime.frs_household_draws import ( + UKFRSHouseholdDrawsStageTransform, +) +from microcosm.build.uk_runtime.frs_legacy_proxies import ( + UKFRSLegacyProxiesStageTransform, +) +from microcosm.build.uk_runtime.frs_person_draws import UKFRSPersonDrawsStageTransform +from microcosm.build.uk_runtime.frs_relationships import ( + UKFRSRelationshipsStageTransform, +) +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release +from microcosm.build.uk_runtime.frs_spine import ( + UKFRSSpineStageTransform, + uk_frs_spine_seed_frame, +) +from microcosm.build.uk_runtime.frs_take_up import UKFRSTakeUpStageTransform +from microcosm.build.uk_runtime.graph import ( + UK_SPINE_EXCLUSIONS, + uk_registry, + uk_spine_graph, + uk_spine_operation_inventory, +) +from microcosm.build.uk_runtime.graph_evidence import ( + add_uk_spine_gate_nodes, + load_spine_stage_artifacts, + materialize_spine_gate_reports, + register_spine_gate_kernel, + spine_sidecar_evidence, +) +from microcosm.build.uk_runtime.lcfs_consumption import ( + UKLCFSConsumptionStageTransform, +) +from microcosm.build.uk_runtime.national_frame import ( + uk_household_weight_kind, + write_uk_national_frame, +) +from microcosm.build.uk_runtime.national_sampling import ( + UK_SAMPLE_RUNG_TOKENS, + UK_SAMPLE_SEED_DEFAULT, +) +from microcosm.build.uk_runtime.nts_bus_travel import UKNTSBusTravelStageTransform +from microcosm.build.uk_runtime.regional_uprating import ( + UKRegionalPropertyUpratingStageTransform, +) +from microcosm.build.uk_runtime.salary_sacrifice import UKSalarySacrificeStageTransform +from microcosm.build.uk_runtime.spi_band_donors import ( + UKSPIIncomeBandDonorStageTransform, +) +from microcosm.build.uk_runtime.spi_housing_shell import ( + UKSPIHousingShellStageTransform, +) +from microcosm.build.uk_runtime.spi_spine import ( + UKFRSHMRCSpineLeavesStageTransform, + UKSPIIncomeSpineStageTransform, + UKSPISupportChannelStageTransform, +) +from microcosm.build.uk_runtime.staging import UK_STAGING_REPOSITORY +from microcosm.build.uk_runtime.student_loans import UKStudentLoansStageTransform +from microcosm.build.uk_runtime.take_up_contract import load_uk_take_up_contract +from microcosm.build.uk_runtime.uc_capital_coherence import ( + UKUCCapitalCoherenceStageTransform, +) +from microcosm.build.uk_runtime.uc_deduction_attributes import ( + UKUCDeductionAttributesStageTransform, +) +from microcosm.build.uk_runtime.uc_reporter_redraw import ( + UKUCReporterRedrawStageTransform, +) +from microcosm.build.uk_runtime.was_wealth import UKWASWealthStageTransform +from microcosm.frame import Frame +from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine +from microcosm.graph import ContentStore, compile_graph, run_graph + +_PIPELINE = "uk-frs-spine" +_REPOSITORY = next( + ( + parent + for parent in Path(__file__).resolve().parents + if (parent / "pyproject.toml").is_file() and (parent / "packages").is_dir() + ), + Path.cwd(), +) +_RUNG_NAMED_EDGE_SIGNATURE = "The least populated classes in y have only 1 member" +_RUNG_ABORT_EXIT_CODE = 3 +#: The last stage of the assembled checkpoint: everything through the base +#: FRS mapping and the stochastic draws. A name, not an index — a position +#: standing in for a key is correct only while two independently-maintained +#: orderings happen to agree (the uk-data#468 class). +UK_SPINE_ASSEMBLED_FINAL_STAGE = "frs_brma" + + +def _uk_spine_stage_names(spec) -> tuple[str, ...]: + """Derive the runnable manifest stages from graph ownership edges.""" + + if spec.sources is None: + raise ValueError("UK country spec has no source stages.") + declared = { + stage.stage + for stage in spec.sources.stages + if stage.stage not in UK_SPINE_EXCLUSIONS + } + compiled = compile_graph(uk_spine_graph(spec)) + ordered = tuple(node_id for node_id in compiled.order if node_id in declared) + if set(ordered) != declared: + raise ValueError( + "UK spine graph and manifest stage roster disagree: " + f"graph={list(ordered)!r}, manifest={sorted(declared)!r}." + ) + return ordered + + +def _rung_sample_fraction(value: str) -> float: + """CLI rung policy (#624) over the permissive library validator.""" + + try: + fraction = float(value) + except ValueError as error: + raise argparse.ArgumentTypeError( + f"sample fraction must be a number; got {value!r}." + ) from error + if fraction not in UK_SAMPLE_RUNG_TOKENS: + raise argparse.ArgumentTypeError( + "sample fraction must be one of 0.01, 0.10, or 1.0 (the #624 rungs)." + ) + return fraction + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Build the deterministic UK FRS spine from pinned raw tabs. Every " + "stochastic stage draws identity-keyed from seeds declared in the " + "manifest, so two runs from the same inputs are payload-identical." + ) + ) + parser.add_argument( + "--frs-raw-dir", + type=Path, + help="Directory containing the 14 licensed FRS 2024-25 tab files.", + ) + parser.add_argument( + "--spine-h5", + type=Path, + required=True, + help="Output H5 path for the raw FRS spine Frame.", + ) + parser.add_argument( + "--spi-tab", + type=Path, + help="Pinned local SPI 2022-23 put2223uk.tab path.", + ) + parser.add_argument( + "--hmrc-ods", + type=Path, + help="Pinned local HMRC collated ODS path.", + ) + parser.add_argument( + "--synthetic-fixture-dir", + type=Path, + help=( + "Data-only UK spine fixture source for non-release integration testing. " + "Requires --smoke and cannot be combined with licensed input options." + ), + ) + parser.add_argument( + "--checkpoint-dir", + type=Path, + help="Optional directory for a copy of the completed spine checkpoint.", + ) + parser.add_argument( + "--sample-fraction", + type=_rung_sample_fraction, + default=1.0, + help=( + "Scale-ladder rung (#624): 0.01 smoke, 0.10 dev, or 1.0 full. " + "Below 1.0 the raw FRS spine is sampled immediately after ingest, " + "renormalized to full household mass, and treated as a receipt." + ), + ) + parser.add_argument( + "--smoke", + action="store_true", + help=( + "Mark the output as non-release and stop after spine and telemetry " + "verification. Combine with --sample-fraction 0.01 for a small " + "licensed-data run, or use the complete synthetic fixture in tests." + ), + ) + parser.add_argument( + "--release-candidate", + action="store_true", + help=( + "Evaluate the spine battery at release-candidate strictness: " + "evidence_absent gaps block instead of being tolerated. Explicit " + "by design - a full-scale developer build is not a release " + "candidate unless the caller says so." + ), + ) + parser.add_argument( + "--sample-seed", + type=int, + default=UK_SAMPLE_SEED_DEFAULT, + help=f"Raw FRS spine sampling seed (default: {UK_SAMPLE_SEED_DEFAULT}).", + ) + parser.add_argument( + "--was-tab", + type=Path, + help="Caller-supplied private WAS round-8 household tab for was_wealth.", + ) + parser.add_argument( + "--nts-household-tab", + type=Path, + help="Caller-supplied private NTS household tab for nts_bus_travel.", + ) + parser.add_argument( + "--nts-individual-tab", + type=Path, + help="Caller-supplied private NTS individual tab for nts_bus_travel.", + ) + parser.add_argument( + "--nts-trip-tab", + type=Path, + help="Caller-supplied private NTS trip tab for nts_bus_travel.", + ) + parser.add_argument( + "--nts-stage-tab", + type=Path, + help="Caller-supplied private NTS stage tab for nts_bus_travel.", + ) + parser.add_argument( + "--nts-ticket-tab", + type=Path, + help="Caller-supplied private NTS ticket tab for nts_bus_travel.", + ) + parser.add_argument( + "--lcfs-hh-tab", + type=Path, + help="Caller-supplied private LCFS 2023-24 household tab for lcfs_consumption.", + ) + parser.add_argument( + "--lcfs-person-tab", + type=Path, + help="Caller-supplied private LCFS 2023-24 person tab for lcfs_consumption.", + ) + parser.add_argument( + "--etb-tab", + type=Path, + help="Caller-supplied private ETB 1977-2024 household tab for ETB stages.", + ) + parser.add_argument( + "--emit-nonzero-shares", + type=Path, + help="Optional JSON path for unweighted per-produced-column nonzero shares.", + ) + parser.add_argument( + "--logbook-prev-row-digest", + type=sha256_argument, + help="Optional current Logbook chain head.", + ) + add_staging_arguments(parser, repository=UK_STAGING_REPOSITORY) + args = parser.parse_args(argv) + if args.sample_seed < 0: + parser.error("sample seed must be a non-negative integer.") + if args.smoke and args.release_candidate: + parser.error("non-release smoke builds refuse --release-candidate.") + if _is_sampled(args) and args.checkpoint_dir is not None: + parser.error( + "sampled spine builds refuse --checkpoint-dir; sampled artifacts " + "cannot be reused as full-scale checkpoints." + ) + production_inputs = { + "--frs-raw-dir": args.frs_raw_dir, + "--spi-tab": args.spi_tab, + "--hmrc-ods": args.hmrc_ods, + } + if args.synthetic_fixture_dir is None: + missing = [flag for flag, value in production_inputs.items() if value is None] + if missing: + parser.error(f"production builds require {', '.join(missing)}.") + else: + supplied = [ + flag for flag, value in production_inputs.items() if value is not None + ] + supplied.extend( + flag + for flag, value in ( + ("--was-tab", args.was_tab), + ("--nts-household-tab", args.nts_household_tab), + ("--nts-individual-tab", args.nts_individual_tab), + ("--nts-trip-tab", args.nts_trip_tab), + ("--nts-stage-tab", args.nts_stage_tab), + ("--nts-ticket-tab", args.nts_ticket_tab), + ("--lcfs-hh-tab", args.lcfs_hh_tab), + ("--lcfs-person-tab", args.lcfs_person_tab), + ("--etb-tab", args.etb_tab), + ) + if value is not None + ) + if supplied: + parser.error( + "--synthetic-fixture-dir cannot be combined with licensed input " + f"options: {', '.join(supplied)}." + ) + if not args.smoke: + parser.error("--synthetic-fixture-dir requires --smoke.") + validate_staging_arguments(parser, args) + return args + + +def _is_sampled(args: argparse.Namespace) -> bool: + return args.sample_fraction != 1.0 + + +def _sample_token(args: argparse.Namespace) -> str: + return UK_SAMPLE_RUNG_TOKENS[args.sample_fraction] + + +def _validate_args(args: argparse.Namespace) -> None: + if args.synthetic_fixture_dir is not None: + if not args.synthetic_fixture_dir.is_dir(): + raise ValueError( + "--synthetic-fixture-dir must be an existing directory: " + f"{args.synthetic_fixture_dir}" + ) + if not (args.synthetic_fixture_dir / "fixture.json").is_file(): + raise ValueError( + "--synthetic-fixture-dir must contain fixture.json: " + f"{args.synthetic_fixture_dir}" + ) + elif args.frs_raw_dir is None or not args.frs_raw_dir.is_dir(): + raise ValueError( + f"--frs-raw-dir must be an existing directory: {args.frs_raw_dir}" + ) + if args.spine_h5.suffix != ".h5": + raise ValueError("--spine-h5 must end with '.h5'.") + if args.synthetic_fixture_dir is None: + if args.spi_tab is None or not args.spi_tab.is_file(): + raise ValueError(f"--spi-tab must be an existing file: {args.spi_tab}") + if args.spi_tab.name != "put2223uk.tab": + raise ValueError("--spi-tab must name put2223uk.tab.") + if args.hmrc_ods is None or not args.hmrc_ods.is_file(): + raise ValueError(f"--hmrc-ods must be an existing file: {args.hmrc_ods}") + if args.hmrc_ods.suffix.lower() != ".ods": + raise ValueError("--hmrc-ods must end with '.ods'.") + paths = { + "spine_h5": args.spine_h5, + "build_sidecar": args.spine_h5.with_suffix(".build.json"), + "hmrc_replay_sidecar": args.spine_h5.with_suffix(".hmrc_replay.json"), + } + if args.emit_nonzero_shares is not None: + paths["emit_nonzero_shares"] = args.emit_nonzero_shares + resolved: dict[Path, str] = {} + for label, path in paths.items(): + target = Path(path).expanduser().resolve() + other = resolved.get(target) + if other is not None: + raise ValueError(f"{label} path collides with {other}: {target}.") + resolved[target] = label + + +def _synthetic_fixture_evidence(source: Path | None) -> dict[str, object] | None: + """Bind a non-release integration run to every file in its fixture.""" + + if source is None: + return None + root = source.resolve() + files = [] + for path in sorted(item for item in root.rglob("*") if item.is_file()): + relative = path.resolve().relative_to(root).as_posix() + files.append( + { + "path": relative, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "size_bytes": path.stat().st_size, + } + ) + descriptor = json.loads((root / "fixture.json").read_text(encoding="utf-8")) + return { + "schema_version": descriptor.get("schema_version"), + "file_count": len(files), + "digest": hashlib.sha256(canonical_json_bytes(files)).hexdigest(), + } + + +def _synthetic_graph_sources(source: Path) -> dict[str, Path]: + """Resolve every split graph source to one reviewed fixture input.""" + + root = source.resolve() + descriptor = json.loads((root / "fixture.json").read_text(encoding="utf-8")) + inputs = descriptor.get("inputs") + if not isinstance(inputs, dict): + raise ValueError("Synthetic fixture inputs must be an object.") + names = { + "was": "was", + "nts_household": "nts_household", + "nts_individual": "nts_individual", + "nts_trip": "nts_trip", + "nts_stage": "nts_stage", + "nts_ticket": "nts_ticket", + "lcfs_household": "lcfs_household", + "lcfs_person": "lcfs_person", + "etb": "etb", + "spi": "spi_donor", + "hmrc_income": "hmrc_income_targets", + } + resolved = {"frs": root} + for role, name in names.items(): + relative = inputs.get(name) + if not isinstance(relative, str) or not relative: + raise ValueError(f"Synthetic fixture inputs.{name} must be a path.") + path = (root / relative).resolve() + try: + path.relative_to(root) + except ValueError as exc: + raise ValueError( + f"Synthetic fixture inputs.{name} escapes the fixture directory." + ) from exc + if not path.exists(): + raise ValueError(f"Synthetic fixture input {name!r} does not exist.") + resolved[role] = path + return resolved + + +def _synthetic_fixture_input(source: Path, name: str) -> Path: + """Resolve one named fixture input without allowing path traversal.""" + + root = source.resolve() + descriptor = json.loads((root / "fixture.json").read_text(encoding="utf-8")) + inputs = descriptor.get("inputs") + relative = inputs.get(name) if isinstance(inputs, dict) else None + if not isinstance(relative, str) or not relative: + raise ValueError(f"Synthetic fixture inputs.{name} must be a path.") + path = (root / relative).resolve() + try: + path.relative_to(root) + except ValueError as exc: + raise ValueError( + f"Synthetic fixture inputs.{name} escapes the fixture directory." + ) from exc + if not path.exists(): + raise ValueError(f"Synthetic fixture input {name!r} does not exist.") + return path + + +def _artifact_pins(stages) -> dict[str, dict[str, object]]: + pins = {} + for stage in stages: + for artifact in stage.artifacts: + key = artifact.get("table", artifact.get("filename")) + if key is None: + continue + key = str(key) + pin = { + "locator": str(artifact["locator"]), + "sha256": str(artifact["sha256"]), + "size_bytes": int(artifact["size_bytes"]), + } + if key in pins and pins[key] != pin: + raise ValueError( + f"UK source artifact {key!r} has inconsistent pins across stages." + ) + pins[key] = pin + return dict(sorted(pins.items())) + + +def _stage_artifact_pins(stage) -> dict[str, dict[str, object]]: + return { + str(artifact.get("table", artifact.get("filename"))): { + "locator": str(artifact["locator"]), + "sha256": str(artifact["sha256"]), + "size_bytes": int(artifact["size_bytes"]), + } + for artifact in stage.artifacts + if "table" in artifact or "filename" in artifact + } + + +def _resource_pins(stages, spec) -> dict[str, str]: + """Country-package resources the selected stages declare as inputs. + + Non-tab artifacts reference committed resources by filename; their bytes + are hashed by load_country_spec, so the pin is the spec's recorded sha. + """ + + pins: dict[str, str] = {} + for stage in stages: + for artifact in stage.artifacts: + if "resource" not in artifact: + continue + resource = str(artifact["resource"]) + sha256 = spec.resource_hashes.get(resource) + if sha256 is None: + raise ValueError( + f"stage {stage.stage!r} declares resource artifact " + f"{resource!r} which is not a declared country-package " + "resource." + ) + pins[resource] = str(sha256) + return dict(sorted(pins.items())) + + +def _input_artifact_pins(stages) -> dict[str, dict[str, object]]: + """Caller-supplied private input artifacts, pinned by role. + + Non-table, non-resource artifacts (the SPI donor tab and the HMRC ODS) + carry their own sha256/size pins in the manifest. Binding them here puts + the pins in the build sidecar and the Logbook input-pins digest, so two + runs with different high-impact source inputs can never share build-side + provenance (adversarial-review finding on #717). + """ + + pins: dict[str, dict[str, object]] = {} + for stage in stages: + for artifact in stage.artifacts: + if "table" in artifact or "resource" in artifact: + continue + if "sha256" not in artifact: + continue + role = str(artifact.get("role") or artifact.get("filename") or "") + if not role: + raise ValueError( + f"stage {stage.stage!r} declares a pinned input artifact " + "without a role or filename." + ) + pin = { + "filename": str( + artifact.get("filename") or artifact.get("locator") or "" + ), + "kind": str(artifact.get("kind", "")), + "sha256": str(artifact["sha256"]), + "size_bytes": int(artifact["size_bytes"]), + } + if role in pins and pins[role] != pin: + raise ValueError( + f"input artifact role {role!r} has inconsistent pins across stages." + ) + pins[role] = pin + return dict(sorted(pins.items())) + + +def _role_pins(pins: dict[str, dict[str, object]]) -> dict[str, dict[str, object]]: + return { + table: { + "sha256": str(pin["sha256"]), + "size_bytes": int(pin["size_bytes"]), + } + for table, pin in pins.items() + } + + +def _entity_row_counts(frame) -> dict[str, int]: + return {entity: int(len(frame.table(entity))) for entity in frame.entities} + + +def _rules_engine() -> PolicyEngineUKEngine: + try: + import policyengine_uk # noqa: F401 + except ImportError as exc: + raise ImportError( + "build_uk_frs_spine requires the microcosm-build 'uk' extra " + "(policyengine-uk). Run: uv sync --all-packages --extra uk" + ) from exc + return PolicyEngineUKEngine() + + +def _rules_engine_provenance() -> dict[str, str]: + try: + version = metadata.version("policyengine-uk") + except metadata.PackageNotFoundError: + return {"package": "policyengine-uk", "version": "unavailable"} + return {"package": "policyengine-uk", "version": version} + + +def _declared_seeds(stages) -> dict[str, dict[str, int]]: + declared: dict[str, dict[str, int]] = {} + for stage in stages: + stage_seeds: dict[str, int] = {} + for operation in stage.operations: + output = operation.parameters.get("output") + seed = operation.parameters.get("seed") + if seed is None: + seed = operation.parameters.get("seed_base") + if isinstance(output, str) and isinstance(seed, int): + stage_seeds[output] = seed + elif isinstance(seed, int): + if operation.kind == "stack_zero_weight_donors": + stage_seeds["stack_zero_weight_donors"] = seed + elif operation.kind == "strict_read_private_table": + stage_seeds["donor_bootstrap"] = seed + elif operation.kind == "fit_weighted_qrf_stage1": + stage_seeds["stage1"] = seed + elif operation.kind == "fit_weighted_qrf_stage2": + stage_seeds["stage2"] = seed + elif operation.kind == "bridge_donor_column_via_qrf": + stage_seeds["bridge_donor_column_via_qrf"] = seed + elif operation.kind == "assign_binary_from_rate": + target = operation.parameters.get("target") + if isinstance(target, str): + stage_seeds[target] = seed + else: + stage_seeds["assign_binary_from_rate"] = seed + elif operation.kind == "fit_weighted_qrf_chain": + stage_seeds[stage.stage] = seed + elif operation.kind == "fit_weighted_qrf": + stage_seeds[stage.stage] = seed + elif operation.kind == "draw_capital_gains_prior_from_banded_quantiles": + stage_seeds[str(operation.parameters["salt"])] = seed + elif operation.kind == "stack_band_donor_households": + stage_seeds["stack_band_donor_households"] = seed + elif operation.kind == "stack_income_band_donor_households": + stage_seeds["stack_income_band_donor_households"] = seed + elif operation.kind == "resample_band_donor_leaves": + stage_seeds["band_donor_resample"] = seed + elif operation.kind == "impute_spi_housing_shell": + stage_seeds[stage.stage] = seed + elif operation.kind == "price_domestic_energy": + stage_seeds["gas_disconnection"] = seed + elif operation.kind == "within_band_draws": + stage_seeds["within_band_draws"] = seed + elif operation.kind in ( + "assign_residential_property_flag", + "assign_main_asset_type", + ): + stage_seeds[operation.kind] = seed + elif operation.kind == "convert_donors_to_target_stock": + stage_seeds[str(operation.parameters["salt"])] = seed + elif operation.kind == "top_up_to_stock": + stage_seeds[str(operation.parameters["salt"])] = seed + if stage_seeds: + declared[stage.stage] = stage_seeds + return declared + + +def _build_sidecar( + *, + frame, + stages, + records, + artifact_pins, + resource_pins: dict[str, str], + input_artifact_pins: dict[str, dict[str, object]], + hmrc_replay: dict[str, object], + stochastic_contract_sha256: str, + frs_vintage: str, + sampling: dict[str, object] | None, + non_release: bool = False, + release_posture: str = "development", + synthetic_fixture: Mapping[str, object] | None = None, + staging_delivery: Mapping[str, object] | None = None, + spine_gate_report: dict[str, object] | None = None, +) -> dict[str, object]: + household_weight = frame.weights_for("household") + return { + "schema_version": 2, + "pipeline": _PIPELINE, + "uk_frame_content_identity": uk_frame_content_identity(frame), + "stages": [stage.stage for stage in stages], + "time_period": str(frame.metadata["time_period"]), + "household_weight_kind": uk_household_weight_kind(frame).value, + "household_weight_total": float(household_weight.values.sum()), + "entity_row_counts": _entity_row_counts(frame), + "artifact_pins": artifact_pins, + "resource_pins": resource_pins, + "input_artifact_pins": input_artifact_pins, + "hmrc_replay": hmrc_replay, + "stage_artifact_pins": { + stage.stage: _stage_artifact_pins(stage) for stage in stages + }, + "stage_records": [ + { + "stage": record.stage, + "produced": list(record.produced), + "nonzero_share": dict(record.nonzero_share), + "seconds": record.seconds, + } + for record in records + ], + "operations": { + stage.stage: [operation.kind for operation in stage.operations] + for stage in stages + }, + "declared_seeds": _declared_seeds(stages), + "source_vintages": {"frs": frs_vintage}, + "sampling": sampling, + "non_release": non_release, + "release_posture": release_posture, + "synthetic_fixture": ( + None if synthetic_fixture is None else dict(synthetic_fixture) + ), + "staging_delivery": dict(staging_delivery or {}), + "spine_gate_report": spine_gate_report, + "stochastic_contract_sha256": stochastic_contract_sha256, + "rules_engine": _rules_engine_provenance(), + } + + +def _mark_non_release_h5(path: Path, *, build_id: str) -> None: + """Persist machine-readable refusal evidence on a bounded smoke H5.""" + + import h5py + + with h5py.File(path, mode="r+") as file: + file.attrs["populace_non_release"] = True + file.attrs["populace_release_posture"] = "smoke" + file.attrs["populace_smoke_build_id"] = build_id + + +def _nonzero_shares(frame, columns: list[str]) -> dict[str, float]: + shares: dict[str, float] = {} + for column in columns: + for entity in frame.entities: + table = frame.table(entity) + if column not in table.columns: + continue + values = table[column] + if values.dtype == object: + shares[column] = float(values.astype(str).ne("").mean()) + else: + shares[column] = float((values != 0).mean()) + break + return shares + + +def _series_nonzero_share(values) -> float: + if values.dtype == object or str(values.dtype).startswith("string"): + return float(values.fillna("").astype(str).ne("").mean()) + return float((values != 0).mean()) + + +def _graph_stage_records( + *, + manifest, + store: ContentStore, + stages, + frame, +) -> tuple[StageRecord, ...]: + """Project immediate node artifacts onto the legacy record schema. + + Entity ids and memberships are executor-carried context, not owned cells, + so the root node exposes no artifact for them although ``frs_spine`` + declares them as outputs. Their share is read from the final population + instead, which is what the legacy plan recorded (identity columns are + never zero, so the value is 1.0 on every vintage). + """ + + structural = _structural_columns(frame) + records: list[StageRecord] = [] + for stage in stages: + output_node = ( + f"{stage.stage}.owned" + if f"{stage.stage}.owned" in manifest.nodes + else stage.stage + ) + output_receipt = manifest.nodes[output_node] + shares: dict[str, float] = {} + for column in stage.outputs: + matches = [ + (coordinate, key) + for coordinate, key in output_receipt.artifacts.items() + if coordinate[1] == column + ] + if not matches and column in structural: + shares[column] = _nonzero_shares(frame, [column])[column] + continue + if len(matches) != 1: + raise RuntimeError( + f"graph stage {stage.stage!r} exposes {len(matches)} artifacts " + f"for declared output {column!r}." + ) + shares[column] = _series_nonzero_share(store.load_column(matches[0][1])) + execution_node = "create_uk_frs" if stage.stage == "frs_spine" else stage.stage + records.append( + StageRecord( + stage=stage.stage, + produced=stage.outputs, + donor_survey=stage.survey, + nonzero_share=shares, + seconds=manifest.nodes[execution_node].wall_time, + ) + ) + return tuple(records) + + +def _structural_columns(frame) -> frozenset[str]: + """Entity id and membership columns the executor carries outside owned cells.""" + + schema = frame.schema + columns = {schema.entity_id_column(entity) for entity in frame.entities} + columns.update(schema.membership_column(group) for group in schema.group_entities) + return frozenset(columns) + + +def _new_build_id(timestamp: datetime) -> str: + return f"uk-frs-spine-{timestamp.strftime('%Y%m%dT%H%M%SZ')}" + + +def _record_attempt( + *, + state: AttemptState, + started_at: float, + started_ts: datetime, + code_pin: str, + disposition: str, + predecessor: str | None, + rung: str, + spool_dir: Path, +) -> Path: + return record_terminal_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + pipeline=_PIPELINE, + rung=rung, + seed=None, + code_pin=code_pin, + disposition=disposition, + predecessor=predecessor, + spool_dir=spool_dir, + ) + + +def _sample_spine_frame( + frame, + *, + fraction: float, + seed: int, +) -> tuple[object, dict[str, object] | None]: + if fraction == 1.0: + return frame, None + household_weight = frame.weights_for("household") + pre_households = int(len(frame.table("household"))) + sampled, receipt = sample_frame_households( + frame, + fraction=fraction, + seed=seed, + source_name="UK FRS spine", + ) + normalized, factor = normalize_sampled_household_mass( + sampled, + target_mass=float(household_weight.total), + source_name="UK FRS spine", + ) + return normalized, { + "fraction": float(fraction), + "seed": int(seed), + "rung_token": UK_SAMPLE_RUNG_TOKENS[fraction], + "pre_household_count": pre_households, + "post_household_count": int(len(normalized.table("household"))), + "normalization_factor": float(factor), + "receipt": dict(receipt), + } + + +class _SampledGraphRootTransform: + """CREATE-stage adapter applying the declared sampling rung at ingest.""" + + def __init__( + self, + transform, + *, + fraction: float, + seed: int, + ) -> None: + self.transform = transform + self.fraction = fraction + self.seed = seed + self.sampling: dict[str, object] | None = None + + def _sample(self, assembled): + sampled, self.sampling = _sample_spine_frame( + assembled, + fraction=self.fraction, + seed=self.seed, + ) + # Graph populations use row positions as their internal alignment + # index. Frame sampling preserves source DataFrame indexes by design, + # so normalize those indexes at this adapter boundary. + tables = { + entity: sampled.table(entity).reset_index(drop=True) + for entity in sampled.entities + } + tables.update( + {name: sampled.link(name).reset_index(drop=True) for name in sampled.links} + ) + return Frame( + tables, + sampled.schema, + { + entity: sampled.weights_for(entity) + for entity in sampled.weighted_entities + }, + sampled.strata.reset_index(drop=True), + mass_log=sampled.mass_log, + metadata=sampled.metadata, + ) + + def effective_fraction(self) -> float: + """Return the configured input sampling fraction.""" + + return float(self.fraction) + + def __call__(self, frame): + return self._sample(self.transform(frame)) + + def run_with_sources(self, frame, sources): + runner = getattr(self.transform, "run_with_sources", None) + assembled = ( + runner(frame, sources) if callable(runner) else self.transform(frame) + ) + return self._sample(assembled) + + def checkpoint_metadata(self) -> dict[str, object]: + hook = getattr(self.transform, "checkpoint_metadata", None) + if not callable(hook): + raise RuntimeError("FRS root transform exposes no checkpoint metadata.") + return dict(hook()) + + +def _staging_stage_observer(telemetry: StagingTelemetryV2) -> StageObserver: + """Translate a shared stage observation into staging telemetry.""" + + def observe(observation: StageObservation) -> None: + telemetry.stage( + observation.stage_id, + event_status=observation.status, + elapsed_seconds=observation.elapsed_seconds, + entity_row_counts=dict(observation.entity_row_counts), + produced_column_count=observation.produced_column_count, + ) + + return observe + + +class _GraphSourceTransform: + """Build a file-reading stage from only the node's declared source paths.""" + + def __init__(self, factory) -> None: + self.factory = factory + self.transform = None + + def run_with_sources(self, frame, sources): + self.transform = self.factory(sources) + result = self.transform(frame) + if hasattr(self.transform, "fit_weight_records"): + self.fit_weight_records = self.transform.fit_weight_records + return result + + def __getattr__(self, name: str): + transform = self.__dict__.get("transform") + if transform is None: + raise AttributeError(name) + return getattr(transform, name) + + +def _run_plan_with_spine_sampling( + plan, + *, + sample_fraction: float, + sample_seed: int, + spine_battery: GateBatteryRun | None = None, + stage_evidence_provider=None, + gate_artifacts: Mapping[str, object] | None = None, +) -> tuple[object, tuple[object, ...], dict[str, object] | None]: + if not plan.stages or plan.stages[0].name != "frs_spine": + frame, records = plan.run(uk_frs_spine_seed_frame()) + return frame, records, None + + from microcosm.build.plan import StagePlan + + spine_frame, spine_records = StagePlan(plan.stages[:1]).run( + uk_frs_spine_seed_frame() + ) + spine_frame, sampling = _sample_spine_frame( + spine_frame, + fraction=sample_fraction, + seed=sample_seed, + ) + if len(plan.stages) == 1: + return spine_frame, spine_records, sampling + names = tuple(stage.name for stage in plan.stages) + if UK_SPINE_ASSEMBLED_FINAL_STAGE in names: + assembled_end = names.index(UK_SPINE_ASSEMBLED_FINAL_STAGE) + 1 + elif spine_battery is not None: + raise RuntimeError( + "spine battery is armed but the declared assembled-boundary stage " + f"{UK_SPINE_ASSEMBLED_FINAL_STAGE!r} is not in the plan; a stage " + "plan change must move the boundary declaration with it." + ) + else: + assembled_end = len(plan.stages) + frame, assembled_records = StagePlan(plan.stages[1:assembled_end]).run(spine_frame) + # Each boundary offers only the stages that have actually run: asking a + # later stage for checkpoint evidence would (correctly) raise, and the + # first licensed battery run did exactly that at the assembled boundary. + executed = tuple(stage.name for stage in plan.stages[:assembled_end]) + if spine_battery is not None: + _run_spine_gate_phase( + spine_battery, + "assembled", + frame=frame, + stage_evidence=( + stage_evidence_provider(executed) + if stage_evidence_provider is not None + else {} + ), + gate_artifacts=gate_artifacts, + ) + if assembled_end == len(plan.stages): + return frame, (*spine_records, *assembled_records), sampling + frame, tail_records = StagePlan(plan.stages[assembled_end:]).run(frame) + executed = tuple(stage.name for stage in plan.stages) + if spine_battery is not None: + _run_spine_gate_phase( + spine_battery, + "transferred", + frame=frame, + stage_evidence=( + stage_evidence_provider(executed) + if stage_evidence_provider is not None + else {} + ), + gate_artifacts=gate_artifacts, + ) + return frame, (*spine_records, *assembled_records, *tail_records), sampling + + +def _run_spine_gate_phase( + battery: GateBatteryRun, + phase: str, + *, + frame, + stage_evidence: Mapping[str, object], + gate_artifacts: Mapping[str, object] | None = None, +) -> None: + artifacts: dict[str, object] = {"stage_evidence": dict(stage_evidence)} + # The enum-domain gate resolves its domain from the live rules engine, + # exactly as the national terminal battery supplied it. + artifacts.update(dict(gate_artifacts or {})) + battery.run_phase( + phase, + EvidenceContext(frame=frame, artifacts=artifacts), + ) + battery.enforce(phase, mode=BlockingMode.BLOCKS_ARTIFACT) + + +def _spine_gate_report_path(spine_h5: Path) -> Path: + return spine_h5.with_suffix(".spine_gates.json") + + +def _spine_gate_manifest_from_spec(spec) -> GatesManifest | None: + """The spine build's scoped battery manifest, from the shared helper. + + A spec without a gates block leaves the battery unarmed (``None``), + exactly as before; when armed, the filtering runs through the one + scope-filtering implementation every scoped producer shares. The + driver passes the spec it already loaded, which is also the hermetic + tests' stub point. Digests are identical to the previous local copy + because entries, phases, and the policy suffix are unchanged. + """ + + source = getattr(spec, "gates", None) + if source is None: + return None + return uk_scoped_gate_manifest( + UK_SPINE_GATE_SCOPE, + phases=("assembled", "transferred"), + policy_suffix="spine_build_scope", + source=source, + ) + + +def _rung_abort_receipt( + args: argparse.Namespace, + *, + error: BaseException, +) -> dict[str, object]: + return { + "schema_version": 1, + "artifact_kind": "uk_frs_spine_rung_abort_receipt", + "build_kind": "uk_frs_spine", + "sampling": { + "sample_fraction": float(args.sample_fraction), + "sample_seed": int(args.sample_seed), + "rung_token": _sample_token(args), + }, + "named_edge": "spine_split_singleton_class", + "stage": "frs_spine", + "error": str(error), + "disposition": "aborted_with_receipt", + "remedy": ( + "Re-roll --sample-seed; accepted dev-scale statistical edge. " + "The computation is never altered to avoid it." + ), + } + + +def _exception_chain_contains(error: BaseException, text: str) -> bool: + """Match a named rung edge through graph execution wrappers.""" + + seen: set[int] = set() + current: BaseException | None = error + while current is not None and id(current) not in seen: + seen.add(id(current)) + if text in str(current): + return True + current = current.__cause__ or current.__context__ + return False + + +def _create_staging_telemetry( + args: argparse.Namespace, *, state: AttemptState +) -> StagingTelemetryV2 | None: + if args.no_staging: + return None + local_dir = args.staging_dir or args.spine_h5.parent / "staging" + local_only = args.staging_local_only + return StagingTelemetryV2( + run_id=args.staging_run_id or state.build_id, + country_code="GB", + operation_id="uk_frs_spine", + pipeline_id=_PIPELINE, + pipeline_version=metadata.version("microcosm-build"), + candidate_id=args.staging_candidate_id or state.build_id, + local_dir=local_dir, + run_kind="smoke" if args.smoke else "spine", + delivery_mode="local_only" if local_only else "local_and_remote", + repo_id=None if local_only else args.staging_repo_id, + upload_interval_seconds=args.staging_upload_interval_seconds, + ) + + +def _telemetry_sample( + args: argparse.Namespace, sampling: Mapping[str, object] | None +) -> dict[str, object] | None: + if args.sample_fraction == 1.0: + return {"mode": "full"} + return None + + +def _staging_delivery( + args: argparse.Namespace, telemetry: StagingTelemetryV2 | None +) -> dict[str, object]: + if telemetry is None: + return disabled_staging_delivery("--no-staging") + return telemetry.delivery_summary + + +@dataclass(frozen=True) +class PreparedUKSpineExecution: + """Declared raw-source graph and bindings, prepared without numerical execution.""" + + graph: object + kernels: object + sources: Mapping[str, Path] + spec: object + stage_names: tuple[str, ...] + stages: tuple[object, ...] + engine: object + engine_identity: str + stochastic_contract: object + frs_release: object + implementations: Mapping[str, object] + synthetic_fixture: Mapping[str, object] | None + + +def parse_uk_spine_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse the maintained raw-source request independently of execution.""" + + return _parse_args(argv) + + +def prepare_uk_spine_execution( + args: argparse.Namespace, + *, + observer: StageObserver | None = None, +) -> PreparedUKSpineExecution: + """Bind source paths and lazy transforms without fitting models or writing files. + + Everything ``main`` decides before the graph runs lives here: the stage + roster, the private-input checks, the stage implementations (licensed or + synthetic fixture), the sampling root, the optional observation wrap, the + declared graph sources, and the gate nodes bound to the rules engine. + ``observer`` is the staging telemetry's stage observer when telemetry is + enabled; ``main`` supplies it, and a caller without telemetry passes none. + """ + + _validate_args(args) + + spec = load_country_spec("uk") + if spec.sources is None: + raise ValueError("UK country spec has no source stages.") + stages_by_name = spec.sources.stage_map() + graph = uk_spine_graph( + spec, + source_mode="split", + sample_fraction=args.sample_fraction, + sample_seed=args.sample_seed, + ) + stage_names = _uk_spine_stage_names(spec) + if ( + args.synthetic_fixture_dir is None + and "was_wealth" in stage_names + and args.was_tab is None + ): + raise ValueError( + "--was-tab is required when the was_wealth stage is scheduled." + ) + if args.synthetic_fixture_dir is None and "nts_bus_travel" in stage_names: + missing_nts = [ + flag + for flag, value in ( + ("--nts-household-tab", args.nts_household_tab), + ("--nts-individual-tab", args.nts_individual_tab), + ("--nts-trip-tab", args.nts_trip_tab), + ("--nts-stage-tab", args.nts_stage_tab), + ("--nts-ticket-tab", args.nts_ticket_tab), + ) + if value is None + ] + if missing_nts: + raise ValueError( + "nts_bus_travel requires caller-supplied private inputs: " + f"{', '.join(missing_nts)}." + ) + if args.synthetic_fixture_dir is None and "lcfs_consumption" in stage_names: + missing_lcfs = [ + flag + for flag, value in ( + ("--lcfs-hh-tab", args.lcfs_hh_tab), + ("--lcfs-person-tab", args.lcfs_person_tab), + ) + if value is None + ] + if missing_lcfs: + raise ValueError( + "lcfs_consumption requires caller-supplied private inputs: " + f"{', '.join(missing_lcfs)}." + ) + if ( + args.synthetic_fixture_dir is None + and ("etb_vat" in stage_names or "etb_services" in stage_names) + and args.etb_tab is None + ): + raise ValueError( + "--etb-tab is required when etb_vat or etb_services is scheduled." + ) + stages = [stages_by_name[name] for name in stage_names] + synthetic_fixture = _synthetic_fixture_evidence(args.synthetic_fixture_dir) + engine = _rules_engine() + stochastic_contract = load_uk_take_up_contract() + frs_release = load_uk_frs_release() + hmrc_spine_transform = _GraphSourceTransform( + lambda sources: UKSPIIncomeSpineStageTransform( + sources["spi"], + sources["hmrc_income"], + stage=stages_by_name["hmrc_spi_income_spine"], + sampled_rung=_is_sampled(args), + ) + ) + implementations = { + "frs_spine": _GraphSourceTransform( + lambda sources: UKFRSSpineStageTransform( + sources["frs"], + stage=stages_by_name["frs_spine"], + ) + ), + "frs_employment": _GraphSourceTransform( + lambda sources: UKFRSEmploymentStageTransform( + sources["frs"], + stage=stages_by_name["frs_employment"], + ) + ), + "frs_council_tax": _GraphSourceTransform( + lambda sources: UKFRSCouncilTaxStageTransform( + sources["frs"], + stage=stages_by_name["frs_council_tax"], + ) + ), + "frs_disability": UKFRSDisabilityStageTransform( + stage=stages_by_name["frs_disability"], + ), + "frs_relationships": _GraphSourceTransform( + lambda sources: UKFRSRelationshipsStageTransform( + sources["frs"], + stage=stages_by_name["frs_relationships"], + ) + ), + "frs_education": _GraphSourceTransform( + lambda sources: UKFRSEducationStageTransform( + sources["frs"], + stage=stages_by_name["frs_education"], + ) + ), + "frs_legacy_proxies": _GraphSourceTransform( + lambda sources: UKFRSLegacyProxiesStageTransform( + sources["frs"], + stage=stages_by_name["frs_legacy_proxies"], + engine=engine, + ) + ), + "frs_education_grant_split": ( + UKFRSEducationGrantSplitStageTransform( + stage=stages_by_name["frs_education_grant_split"], + engine=engine, + ) + ), + "frs_take_up": UKFRSTakeUpStageTransform( + contract=stochastic_contract, + stage=stages_by_name["frs_take_up"], + ), + "frs_person_draws": UKFRSPersonDrawsStageTransform( + contract=stochastic_contract, + stage=stages_by_name["frs_person_draws"], + ), + "frs_household_draws": UKFRSHouseholdDrawsStageTransform( + contract=stochastic_contract, + stage=stages_by_name["frs_household_draws"], + ), + "frs_brma": UKFRSBRMAStageTransform( + stage=stages_by_name["frs_brma"], + engine=engine, + ), + } + if "was_wealth" in stage_names: + implementations["was_wealth"] = _GraphSourceTransform( + lambda sources: UKWASWealthStageTransform( + stage=stages_by_name["was_wealth"], + engine=engine, + was_tab_path=sources["was"], + ) + ) + if "nts_bus_travel" in stage_names: + implementations["nts_bus_travel"] = _GraphSourceTransform( + lambda sources: UKNTSBusTravelStageTransform( + stage=stages_by_name["nts_bus_travel"], + engine=engine, + nts_household_tab_path=sources["nts_household"], + nts_individual_tab_path=sources["nts_individual"], + nts_trip_tab_path=sources["nts_trip"], + nts_stage_tab_path=sources["nts_stage"], + nts_ticket_tab_path=sources["nts_ticket"], + ) + ) + if "regional_property_uprating" in stage_names: + implementations["regional_property_uprating"] = ( + UKRegionalPropertyUpratingStageTransform( + stage=stages_by_name["regional_property_uprating"], + ) + ) + if "lcfs_consumption" in stage_names: + implementations["lcfs_consumption"] = _GraphSourceTransform( + lambda sources: UKLCFSConsumptionStageTransform( + stage=stages_by_name["lcfs_consumption"], + engine=engine, + lcfs_hh_tab_path=sources["lcfs_household"], + lcfs_person_tab_path=sources["lcfs_person"], + ) + ) + if "etb_vat" in stage_names: + implementations["etb_vat"] = _GraphSourceTransform( + lambda sources: UKETBVATStageTransform( + stage=stages_by_name["etb_vat"], + engine=engine, + etb_tab_path=sources["etb"], + ) + ) + if "etb_services" in stage_names: + implementations["etb_services"] = _GraphSourceTransform( + lambda sources: UKETBServicesStageTransform( + stage=stages_by_name["etb_services"], + engine=engine, + etb_tab_path=sources["etb"], + ) + ) + implementations["frs_hmrc_spine_leaves"] = _GraphSourceTransform( + lambda sources: UKFRSHMRCSpineLeavesStageTransform( + sources["frs"], + stage=stages_by_name["frs_hmrc_spine_leaves"], + sampled_rung=_is_sampled(args), + ) + ) + implementations["spi_support_channel"] = UKSPISupportChannelStageTransform( + stage=stages_by_name["spi_support_channel"], + sample_fraction=args.sample_fraction, + ) + if "spi_income_band_donors" in stage_names: + implementations["spi_income_band_donors"] = _GraphSourceTransform( + lambda sources: UKSPIIncomeBandDonorStageTransform( + sources["spi"], + stage=stages_by_name["spi_income_band_donors"], + sample_fraction=args.sample_fraction, + ) + ) + implementations["hmrc_spi_income_spine"] = hmrc_spine_transform + if "spi_housing_shell" in stage_names: + implementations["spi_housing_shell"] = UKSPIHousingShellStageTransform( + stage=stages_by_name["spi_housing_shell"] + ) + if "uc_reporter_redraw" in stage_names: + implementations["uc_reporter_redraw"] = UKUCReporterRedrawStageTransform( + stage=stages_by_name["uc_reporter_redraw"], + engine=engine, + ) + if "uc_capital_coherence" in stage_names: + implementations["uc_capital_coherence"] = UKUCCapitalCoherenceStageTransform( + stage=stages_by_name["uc_capital_coherence"] + ) + if "uc_deduction_attributes" in stage_names: + implementations["uc_deduction_attributes"] = ( + UKUCDeductionAttributesStageTransform( + stage=stages_by_name["uc_deduction_attributes"] + ) + ) + if "cgt_incidence_clone" in stage_names: + implementations["cgt_incidence_clone"] = UKCGTIncidenceCloneStageTransform( + stage=stages_by_name["cgt_incidence_clone"] + ) + if "cgt_band_donors" in stage_names: + implementations["cgt_band_donors"] = UKCGTBandDonorStageTransform( + stage=stages_by_name["cgt_band_donors"] + ) + if "hmrc_cgt_gains_spine" in stage_names: + implementations["hmrc_cgt_gains_spine"] = uk_cgt_spine_stage_transform( + stages_by_name["hmrc_cgt_gains_spine"] + ) + if "hmrc_cgt_asset_type_spine" in stage_names: + implementations["hmrc_cgt_asset_type_spine"] = ( + uk_cgt_asset_type_stage_transform( + stages_by_name["hmrc_cgt_asset_type_spine"] + ) + ) + if "cgt_incidence_anchor" in stage_names: + implementations["cgt_incidence_anchor"] = UKCGTIncidenceAnchorStageTransform( + stage=stages_by_name["cgt_incidence_anchor"] + ) + if "salary_sacrifice" in stage_names: + implementations["salary_sacrifice"] = UKSalarySacrificeStageTransform( + stage=stages_by_name["salary_sacrifice"] + ) + if "student_loans" in stage_names: + implementations["student_loans"] = UKStudentLoansStageTransform( + stage=stages_by_name["student_loans"], + calibration_year=frs_release.calibration_year, + ) + if "age_tail" in stage_names: + implementations["age_tail"] = UKAgeTailStageTransform( + stage=stages_by_name["age_tail"] + ) + if args.synthetic_fixture_dir is not None: + from microcosm.build.uk_runtime.graph_kernels import ( + fixture_stage_plan_inputs, + ) + + fixture_stages, fixture_implementations = fixture_stage_plan_inputs( + args.synthetic_fixture_dir + ) + fixture_stage_names = tuple(stage.stage for stage in fixture_stages) + if fixture_stage_names != tuple(stage_names): + raise ValueError( + "Synthetic fixture stage order differs from the current UK spine: " + f"fixture={fixture_stage_names!r}, current={tuple(stage_names)!r}." + ) + implementations = dict(fixture_implementations) + fixture_by_name = {stage.stage: stage for stage in fixture_stages} + implementations["frs_hmrc_spine_leaves"] = UKFRSHMRCSpineLeavesStageTransform( + _synthetic_fixture_input(args.synthetic_fixture_dir, "frs_raw"), + stage=fixture_by_name["frs_hmrc_spine_leaves"], + sampled_rung=True, + ) + sampled_root = _SampledGraphRootTransform( + implementations["frs_spine"], + fraction=args.sample_fraction, + seed=args.sample_seed, + ) + implementations["frs_spine"] = sampled_root + if observer is not None: + implementations = { + stage_id: ObservedTransform( + transform, + stage_id=stage_id, + produced_column_count=len(stages_by_name[stage_id].outputs), + observer=observer, + ) + for stage_id, transform in implementations.items() + } + + if args.synthetic_fixture_dir is not None: + graph_sources = _synthetic_graph_sources(args.synthetic_fixture_dir) + else: + graph_sources = {"frs": args.frs_raw_dir} + if "was_wealth" in stage_names: + graph_sources["was"] = args.was_tab + if "nts_bus_travel" in stage_names: + graph_sources["nts_household"] = args.nts_household_tab + graph_sources["nts_individual"] = args.nts_individual_tab + graph_sources["nts_trip"] = args.nts_trip_tab + graph_sources["nts_stage"] = args.nts_stage_tab + graph_sources["nts_ticket"] = args.nts_ticket_tab + if "lcfs_consumption" in stage_names: + graph_sources["lcfs_household"] = args.lcfs_hh_tab + graph_sources["lcfs_person"] = args.lcfs_person_tab + if "etb_vat" in stage_names or "etb_services" in stage_names: + graph_sources["etb"] = args.etb_tab + if "spi_income_band_donors" in stage_names: + graph_sources["spi"] = args.spi_tab + if "hmrc_spi_income_spine" in stage_names: + graph_sources["spi"] = args.spi_tab + graph_sources["hmrc_income"] = args.hmrc_ods + engine_identity = hashlib.sha256( + canonical_json_bytes(_rules_engine_provenance()) + ).hexdigest() + graph = add_uk_spine_gate_nodes( + graph, + spec=spec, + engine_identity=engine_identity, + release_candidate=args.release_candidate, + synthetic_smoke=args.synthetic_fixture_dir is not None, + ) + kernels = uk_registry(implementations, graph=graph) + register_spine_gate_kernel( + kernels, spec=spec, engine=engine, engine_identity=engine_identity + ) + return PreparedUKSpineExecution( + graph=graph, + kernels=kernels, + sources=graph_sources, + spec=spec, + stage_names=tuple(stage_names), + stages=tuple(stages), + engine=engine, + engine_identity=engine_identity, + stochastic_contract=stochastic_contract, + frs_release=frs_release, + implementations=implementations, + synthetic_fixture=synthetic_fixture, + ) + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(argv) + rung = UK_SAMPLE_RUNG_TOKENS[args.sample_fraction] + started_at = time.perf_counter() + started_ts = datetime.now(UTC) + predecessor = resolve_predecessor(args.logbook_prev_row_digest) + digest = preflight_digest(_PIPELINE) + state = AttemptState( + build_id=_new_build_id(started_ts), + identity_digest=digest, + input_pins_digest=digest, + phases_reached=["attempt_started"], + gate_verdicts={ + "pipeline": { + "verdict": "running", + "receipt": "pending-build-scoped-spine-receipt", + } + }, + ) + code_pin = "unresolved-local-git-code-pin" + spool_dir = args.spine_h5.parent / "logbook-spool" + telemetry: StagingTelemetryV2 | None = None + try: + _validate_args(args) + # A crash between the H5 write and the sidecar writes must never + # leave a stale sidecar beside a fresh H5 (adversarial-review + # finding on #717): clear every output up front, and treat the + # build sidecar - written last, binding the replay hash - as the + # marker that the bundle is complete. + stale_outputs = [ + args.spine_h5, + args.spine_h5.with_suffix(".build.json"), + args.spine_h5.with_suffix(".hmrc_replay.json"), + _spine_gate_report_path(args.spine_h5), + args.spine_h5.with_suffix(".rung_abort.json"), + ] + if args.emit_nonzero_shares is not None: + stale_outputs.append(args.emit_nonzero_shares) + for stale in stale_outputs: + stale.unlink(missing_ok=True) + telemetry = _create_staging_telemetry(args, state=state) + if telemetry is not None: + initial_sample = _telemetry_sample(args, None) + if initial_sample is not None: + telemetry.set_sample(initial_sample) + telemetry.stage( + "configuration", + event_status="completed", + smoke=args.smoke, + sample_mode=("fraction" if args.sample_fraction != 1.0 else "full"), + ) + code_pin = git_code_pin(_REPOSITORY) + append_phase(state, "configured") + prepared = prepare_uk_spine_execution( + args, + observer=( + _staging_stage_observer(telemetry) if telemetry is not None else None + ), + ) + spec = prepared.spec + graph = prepared.graph + stage_names = prepared.stage_names + stages = list(prepared.stages) + artifact_pins = _artifact_pins(stages) + resource_pins = _resource_pins(stages, spec) + input_artifact_pins = _input_artifact_pins(stages) + overlapping_pin_roles = set(artifact_pins) & set(input_artifact_pins) + if overlapping_pin_roles: + raise ValueError( + "input artifact roles collide with FRS tab names: " + f"{sorted(overlapping_pin_roles)}." + ) + state.input_pins_digest = role_pins_digest( + _role_pins({**artifact_pins, **input_artifact_pins}) + ) + synthetic_fixture = prepared.synthetic_fixture + run_config = { + "pipeline": _PIPELINE, + "stages": list(stage_names), + "artifact_pins_digest": state.input_pins_digest, + "spine_h5": str(args.spine_h5), + "synthetic_fixture": synthetic_fixture, + } + state.identity_digest = hashlib.sha256( + canonical_json_bytes(run_config) + ).hexdigest() + append_phase(state, "inputs_pinned") + if telemetry is not None: + telemetry.stage( + "input_verification", + event_status="completed", + stage_count=len(stage_names), + input_artifact_count=len(artifact_pins) + len(input_artifact_pins), + ) + stochastic_contract = prepared.stochastic_contract + frs_release = prepared.frs_release + spine_gate_path = _spine_gate_report_path(args.spine_h5) + spine_gate_manifest = _spine_gate_manifest_from_spec(spec) + spine_battery = ( + GateBatteryRun( + spine_gate_manifest, + release_id=state.build_id, + report_path=spine_gate_path, + release_candidate=args.release_candidate, + synthetic_smoke=args.synthetic_fixture_dir is not None, + registry=UK_GATE_REGISTRY, + ) + if spine_gate_manifest is not None + else None + ) + checkpoint_root = ( + args.checkpoint_dir + if args.checkpoint_dir is not None + else args.spine_h5.parent / f".{args.spine_h5.stem}.checkpoints" + ) + compiled_graph = compile_graph(graph) + graph_store = ContentStore(checkpoint_root / "node-graph") + graph_manifest = run_graph( + compiled_graph, + sources=prepared.sources, + store=graph_store, + kernels=prepared.kernels, + resume="auto", + decisions=(), + ) + graph_manifest.save(checkpoint_root / "spine.graph.json") + final_version = compiled_graph.versions[stage_names[-1]] + frame = graph_manifest.population(final_version) + records = _graph_stage_records( + manifest=graph_manifest, + store=graph_store, + stages=stages, + frame=frame, + ) + stored_stages = load_spine_stage_artifacts( + graph_manifest, graph_store, stage_names=stage_names + ) + stored_evidence = spine_sidecar_evidence(stored_stages) + sampling = stored_evidence["sampling"] + if telemetry is not None: + sample = _telemetry_sample(args, sampling) + if sample is not None: + telemetry.set_sample(sample) + telemetry.stage( + "sampling", + event_status="completed", + realized_household_rows=( + len(frame.table("household")) + if sampling is None + else sampling.get( + "realized_household_rows", + sampling.get("post_household_count"), + ) + ), + ) + telemetry.stage("validation", event_status="started") + if spine_battery is not None: + materialize_spine_gate_reports( + graph_manifest, + graph_store, + battery=spine_battery, + gates=spine_gate_manifest, + ) + if spine_battery is not None: + append_phase(state, "spine_gates_evaluated") + if telemetry is not None: + telemetry.stage( + "validation", + event_status="completed", + entity_row_counts=_entity_row_counts(frame), + ) + append_phase(state, "spine_built") + if telemetry is not None: + telemetry.stage("spine_h5_creation", event_status="started") + output = write_uk_national_frame(frame, args.spine_h5) + if args.smoke: + _mark_non_release_h5(output, build_id=state.build_id) + if telemetry is not None: + telemetry.stage( + "spine_h5_creation", + event_status="completed", + size_bytes=output.stat().st_size, + ) + append_phase(state, "spine_written") + if args.checkpoint_dir is not None: + args.checkpoint_dir.mkdir(parents=True, exist_ok=True) + write_uk_national_frame(frame, args.checkpoint_dir / "frs_spine.h5") + append_phase(state, "checkpoint_written") + sidecar_path = output.with_suffix(".build.json") + replay_sidecar_path = output.with_suffix(".hmrc_replay.json") + replay_metadata = stored_stages["hmrc_spi_income_spine"]["checkpoint_metadata"] + if ( + not isinstance(replay_metadata, dict) + or "replay_payload" not in replay_metadata + ): + raise RuntimeError("HMRC SPI spine stage did not record replay evidence.") + atomic_write_json(replay_sidecar_path, replay_metadata["replay_payload"]) + append_phase(state, "hmrc_replay_sidecar_written") + replay_bytes = replay_sidecar_path.read_bytes() + replay_binding = { + "filename": replay_sidecar_path.name, + "report_kind": str(json.loads(replay_bytes).get("report_kind", "")), + "sha256": hashlib.sha256(replay_bytes).hexdigest(), + } + if telemetry is not None: + telemetry.stage("sidecar_creation", event_status="started") + sidecar = _build_sidecar( + frame=frame, + stages=stages, + records=records, + artifact_pins=artifact_pins, + resource_pins=resource_pins, + input_artifact_pins=input_artifact_pins, + hmrc_replay=replay_binding, + stochastic_contract_sha256=stochastic_contract.resource_sha256, + frs_vintage=frs_release.vintage, + sampling=sampling, + non_release=args.smoke, + release_posture=( + "non_release_smoke" + if args.smoke + else "release_candidate" + if args.release_candidate + else "development" + ), + synthetic_fixture=synthetic_fixture, + staging_delivery=_staging_delivery(args, telemetry), + spine_gate_report=( + { + "path": str(spine_gate_path), + "sha256": hashlib.sha256(spine_gate_path.read_bytes()).hexdigest(), + } + if spine_gate_path.is_file() + else None + ), + ) + sidecar["operation_inventory"] = list(uk_spine_operation_inventory(graph, spec)) + sidecar["graph_manifest"] = { + "path": str(checkpoint_root / "spine.graph.json"), + "key": graph_manifest.key, + } + stage_evidence = stored_evidence["stage_evidence"] + if stage_evidence: + sidecar["stage_evidence"] = stage_evidence + fit_weight_records = stored_evidence["fit_weight_records"] + if fit_weight_records: + sidecar["fit_weight_records"] = fit_weight_records + atomic_write_json(sidecar_path, sidecar) + if telemetry is not None: + telemetry.stage( + "sidecar_creation", + event_status="completed", + size_bytes=sidecar_path.stat().st_size, + ) + append_phase(state, "build_sidecar_written") + if args.emit_nonzero_shares is not None: + final_columns = list( + dict.fromkeys( + [column for record in records for column in record.produced] + + list(FRS_EDUCATION_GRANT_REWRITES) + ) + ) + atomic_write_json( + args.emit_nonzero_shares, + { + "stages": { + record.stage: dict(record.nonzero_share) for record in records + }, + "final": _nonzero_shares(frame, final_columns), + }, + ) + append_phase(state, "nonzero_shares_written") + if telemetry is not None: + telemetry.complete( + message=( + "Non-release smoke verification completed." + if args.smoke + else "UK spine staging run completed." + ) + ) + if args.staging_read_back: + telemetry.verify_remote() + telemetry.validate_local_bundle() + sidecar["staging_delivery"] = telemetry.delivery_summary + atomic_write_json(sidecar_path, sidecar) + state.artifact_location = local_artifact_reference( + output, + repository_hint=_REPOSITORY, + ) + state.gate_verdicts = { + "pipeline": { + "verdict": "passed", + "receipt": local_artifact_reference( + sidecar_path, repository_hint=_REPOSITORY + ), + } + } + if spine_gate_path.is_file(): + gate_payload = json.loads(spine_gate_path.read_text(encoding="utf-8")) + for gate_id, payload in gate_payload.get("gates", {}).items(): + state.gate_verdicts[str(gate_id)] = { + "verdict": str(payload.get("status")), + "receipt": ( + f"{local_artifact_reference(spine_gate_path, repository_hint=_REPOSITORY)}" + f"#/gates/{gate_id}" + ), + } + spool_path = _record_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + code_pin=code_pin, + disposition="iterating", + predecessor=predecessor, + rung=rung, + spool_dir=spool_dir, + ) + print(f"Wrote FRS spine H5: {output}", file=sys.stderr) + print(f"Wrote Logbook row: {spool_path}", file=sys.stderr) + return 0 + except Exception as error: + if telemetry is not None and telemetry.status == "running": + try: + telemetry.fail(error) + telemetry.validate_local_bundle() + except Exception: + pass + if _is_sampled(args) and _exception_chain_contains( + error, _RUNG_NAMED_EDGE_SIGNATURE + ): + rung_abort_path = args.spine_h5.with_suffix(".rung_abort.json") + receipt = _rung_abort_receipt(args, error=error) + atomic_write_json(rung_abort_path, receipt) + state.gate_verdicts = { + "uk_frs_spine_rung_abort": { + "verdict": "aborted", + "receipt": ( + f"{local_artifact_reference(rung_abort_path, repository_hint=_REPOSITORY)}" + "#/named_edge" + ), + } + } + append_phase(state, "rung_aborted") + _record_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + code_pin=code_pin, + disposition="discarded", + predecessor=predecessor, + rung=rung, + spool_dir=spool_dir, + ) + print(json.dumps(receipt, indent=2, sort_keys=True)) + return _RUNG_ABORT_EXIT_CODE + try: + receipt_path = write_error_receipt( + error_receipt_path(args.spine_h5.parent, build_id=state.build_id), + state=state, + pipeline=_PIPELINE, + error=error, + ) + apply_error_verdict( + state, + local_artifact_reference(receipt_path, repository_hint=_REPOSITORY), + ) + _record_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + code_pin=code_pin, + disposition="failed", + predecessor=predecessor, + rung=rung, + spool_dir=spool_dir, + ) + except Exception: + pass + print(f"UK FRS spine build failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine/uk/test_uk_graph.py index 759f2c1fd..e0146b07a 100644 --- a/packages/microcosm-build/tests/engine/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine/uk/test_uk_graph.py @@ -48,8 +48,6 @@ def test_driver_projects_a_stage_record_for_every_graph_stage_on_the_fixture( in CI's engine lane instead. """ - import importlib.util - from microcosm.build.uk_runtime.graph_kernels import fixture_stage_plan_inputs from microcosm.graph import ContentStore, run_graph @@ -57,11 +55,8 @@ def test_driver_projects_a_stage_record_for_every_graph_stage_on_the_fixture( fixture = root / "packages/microcosm-graph/tests/fixtures/parity/uk_spine" if not fixture.exists(): pytest.skip("UK spine parity fixture is not present") - spec = importlib.util.spec_from_file_location( - "build_uk_frs_spine", root / "tools" / "build_uk_frs_spine.py" - ) - driver = importlib.util.module_from_spec(spec) - spec.loader.exec_module(driver) + # The driver lives in the package; tools/build_uk_frs_spine.py is a shim. + from microcosm.build.uk_runtime import spine_build as driver country = load_country_spec("uk") stages = [ diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py index 33babb3db..d35ac7539 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py @@ -17,12 +17,14 @@ from microcosm.build.logbook import load_spool_rows from microcosm.build.observation import StageObservation from microcosm.build.source_manifest import SourceManifest, SourceStageSpec +from microcosm.build.stage_evidence import snapshot_stage_evidence from microcosm.build.staging_v2 import validate_v2_bundle from microcosm.build.uk_runtime import ( frs_disability, frs_education_grants, frs_legacy_proxies, frs_take_up, + spine_build, ) from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity from microcosm.build.uk_runtime.frs_relationships import ( @@ -47,11 +49,11 @@ validate_uk_national_frame, ) from microcosm.frame import Frame, WeightKind, engine_tables -from test_support.paths import paths_for -_TEST_PATHS = paths_for("microcosm-build") - -_TOOL_PATH = _TEST_PATHS.repository / "tools" / "build_uk_frs_spine.py" +# The driver lives in the package now; ``tools/build_uk_frs_spine.py`` is a +# shim over it. Each test still executes its own module copy so per-test +# monkeypatches never leak through the shared import. +_TOOL_PATH = Path(spine_build.__file__) def _load_tool(): @@ -59,10 +61,53 @@ def _load_tool(): assert spec is not None assert spec.loader is not None module = importlib.util.module_from_spec(spec) + # Register the copy before executing it (the documented recipe for + # importing a source file by path): ``dataclass`` resolves the driver's + # string annotations through ``sys.modules[cls.__module__]``, and each + # test's fresh copy replaces the previous one under this private name. + sys.modules[spec.name] = module spec.loader.exec_module(module) return module +def _stored_stage_evidence(*, stage_names, implementations) -> dict[str, object]: + """The sidecar's ``stage_evidence`` block from per-stage stored snapshots. + + Mirrors ``spine_sidecar_evidence`` over ``snapshot_stage_evidence``: the + driver no longer collects from live objects, it reads what each kernel + stored. Only the executed stages are consulted, exactly as each kernel + snapshots its own transform after it has run. + """ + + documents = { + stage: snapshot_stage_evidence(stage, implementations[stage]) + for stage in stage_names + if stage in implementations + } + return { + stage: document["evidence"] + for stage, document in documents.items() + if document["evidence"] is not None + } + + +def _stored_fit_weight_records( + *, stage_names, implementations +) -> dict[str, list[dict[str, str]]]: + """The sidecar's ``fit_weight_records`` block from per-stage stored snapshots.""" + + documents = { + stage: snapshot_stage_evidence(stage, implementations[stage]) + for stage in stage_names + if stage in implementations + } + return { + stage: document["fit_weight_records"] + for stage, document in documents.items() + if "fit_weight_records" in document + } + + def test_staging_stage_observer_translates_shared_observation() -> None: tool = _load_tool() calls = [] @@ -1300,18 +1345,17 @@ def __call__(self, frame: Frame) -> Frame: self.last_result = SimpleNamespace(replay_report={"report_kind": "fake"}) return result - def _write_fake_replay(report, path): - output = Path(path) - output.write_text( - json.dumps({"report_kind": "fake_spine_replay"}) + "\n", - encoding="utf-8", - ) - return output + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("Stage evidence requires completed computation.") + return { + "evidence": {"stage": self.stage.stage}, + "replay_payload": {"report_kind": "fake_spine_replay"}, + } monkeypatch.setattr(tool, "UKFRSHMRCSpineLeavesStageTransform", _FakeStageTransform) monkeypatch.setattr(tool, "UKSPISupportChannelStageTransform", _FakeStageTransform) monkeypatch.setattr(tool, "UKSPIIncomeSpineStageTransform", _FakeStageTransform) - monkeypatch.setattr(tool, "write_hmrc_replay_report", _write_fake_replay) return spi_tab, hmrc_ods @@ -2158,8 +2202,6 @@ def test_e8_manifest_seeds_all_reach_the_build_sidecar_harvester() -> None: def test_spine_sidecar_collects_stage_evidence_by_duck_type() -> None: - tool = _load_tool() - class _EvidenceResult: def __init__(self, payload: dict[str, object]) -> None: self.payload = payload @@ -2215,7 +2257,7 @@ def checkpoint_metadata(self) -> dict[str, object]: "future_stage": _CheckpointStage(new_payload), } - evidence = tool._collect_stage_evidence( + evidence = _stored_stage_evidence( stage_names=( "frs_spine", "frs_hmrc_spine_leaves", @@ -2251,8 +2293,6 @@ def checkpoint_metadata(self) -> dict[str, object]: def test_collect_fit_weight_records_is_duck_typed_and_fail_visible(): - tool = _load_tool() - class _Record: def __init__(self, fit_name, weight_kind): self.fit_name = fit_name @@ -2271,7 +2311,7 @@ def fit_weight_records(self): "etb_vat": SimpleNamespace(fit_weight_records=()), "lcfs_consumption": _Broken(), } - records = tool._collect_fit_weight_records( + records = _stored_fit_weight_records( stage_names=("frs_spine", "was_wealth", "etb_vat", "lcfs_consumption"), implementations=implementations, ) @@ -2393,8 +2433,6 @@ def test_boundary_evidence_asks_only_the_stages_that_have_run() -> None: its executed prefix — an un-run stage being consulted is the regression. """ - tool = _load_tool() - class _RefusesUntilRun: def __init__(self) -> None: self.ran = False @@ -2411,13 +2449,13 @@ def checkpoint_metadata(self) -> dict[str, object]: # The assembled-boundary call: only the executed prefix is offered, so the # un-run late stage is never consulted and nothing raises. - assembled = tool._collect_stage_evidence( + assembled = _stored_stage_evidence( stage_names=("early_stage",), implementations=implementations ) assert assembled == {} late.ran = True - transferred = tool._collect_stage_evidence( + transferred = _stored_stage_evidence( stage_names=("early_stage", "late_stage"), implementations=implementations ) assert transferred == {"late_stage": {"stage": "late_stage", "ok": True}} @@ -2457,7 +2495,6 @@ def test_collect_fit_weight_records_sees_through_the_run_proxies(): from microcosm.build.observation import ObservedTransform - tool = _load_tool() record = SimpleNamespace( fit_name="uk_was_2018_20_wealth:savings", weight_kind="design" ) @@ -2492,7 +2529,7 @@ def observed(transform, stage_id): "was_wealth": observed(_Fit(), "was_wealth"), "etb_vat": observed(_GraphLike(_Fit()), "etb_vat"), } - records = tool._collect_fit_weight_records( + records = _stored_fit_weight_records( stage_names=("frs_spine", "was_wealth", "etb_vat"), implementations=implementations, ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py new file mode 100644 index 000000000..cb7332b4e --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py @@ -0,0 +1,221 @@ +"""Spine evidence survives cached execution and a completely fresh process.""" + +import json +import subprocess +import sys +from dataclasses import replace +from types import SimpleNamespace + +import pytest + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.uk_runtime.graph import ( + uk_registry, + uk_spine_endpoint, + uk_spine_graph, + uk_spine_operation_inventory, +) +from microcosm.build.uk_runtime.graph_evidence import ( + add_uk_spine_gate_nodes, + load_spine_stage_artifacts, + spine_sidecar_evidence, +) +from microcosm.graph import ContentStore, compile_graph, load_source, run_graph +from test_support.paths import paths_for + + +def test_spine_evidence_replays_without_instantiating_original_transform(tmp_path): + country = load_country_spec("uk") + country = replace( + country, sources=replace(country.sources, stages=country.sources.stages[:1]) + ) + graph = uk_spine_graph(country) + compiled = compile_graph(graph) + source = ( + paths_for("microcosm-graph").tests + / "fixtures" + / "parity" + / "uk_spine" + / "sources" + ) + + class Root: + sampling = {"fraction": 0.1, "seed": 7} + fit_weight_records = ( + SimpleNamespace(fit_name="fixture-fit", weight_kind="design"), + ) + + def run_with_sources(self, frame, sources): + return load_source("csv-tables", sources["frs"]) + + def checkpoint_metadata(self): + return { + "evidence": {"rows": 8}, + "replay_payload": {"classification": "fixture"}, + } + + store = ContentStore(tmp_path / "store") + cold = run_graph( + compiled, + sources={"frs": source}, + store=store, + kernels=uk_registry({"frs_spine": Root()}, graph=graph), + resume="forbid", + ) + expected = load_spine_stage_artifacts(cold, store, stage_names=("frs_spine",)) + warm = run_graph( + compiled, + sources={"frs": source}, + store=store, + kernels=uk_registry(graph=graph), + resume="require", + ) + assert all(receipt.store_hit for receipt in warm.nodes.values()) + assert ( + load_spine_stage_artifacts(warm, store, stage_names=("frs_spine",)) == expected + ) + path = tmp_path / "manifest.json" + warm.save(path) + script = """ +import json, sys +from microcosm.graph import ContentStore, RunManifest +from microcosm.build.uk_runtime.graph_evidence import load_spine_stage_artifacts +store = ContentStore(sys.argv[2]) +manifest = RunManifest.load(sys.argv[1], store) +print(json.dumps(load_spine_stage_artifacts(manifest, store, stage_names=('frs_spine',)), sort_keys=True)) +""" + result = subprocess.run( + [sys.executable, "-c", script, str(path), str(store.root)], + check=True, + text=True, + capture_output=True, + ) + assert json.loads(result.stdout) == expected + sidecar = spine_sidecar_evidence(expected) + assert sidecar["sampling"] == {"fraction": 0.1, "seed": 7} + assert sidecar["fit_weight_records"]["frs_spine"][0]["weight_kind"] == "design" + + +def test_spine_inventory_is_roster_derived_and_gates_bind_checkpoint_versions(): + country = load_country_spec("uk") + spine = uk_spine_graph(country) + endpoint = uk_spine_endpoint(spine) + inventory = uk_spine_operation_inventory(spine, country) + assert tuple(row["stage"] for row in inventory) == endpoint.stage_names + assert inventory[0]["node"] == "create_uk_frs" + assert "normalization" in inventory[0]["coupling"] + wealth = next(row for row in inventory if row["stage"] == "was_wealth") + assert "Donor and recipient" in wealth["coupling"] + graph = add_uk_spine_gate_nodes(spine, spec=country, engine_identity="test-engine") + compiled = compile_graph(graph) + assembled = graph.node("spine.gates.assembled") + transferred = graph.node("spine.gates.transferred") + assert assembled.population == compiled.versions["frs_brma"] + assert assembled.population != compiled.versions["was_wealth"] + assert transferred.population == endpoint.population + assert {edge.producer for edge in transferred.artifact_inputs} >= { + "spine.gates.assembled", + "was_wealth", + } + # The gate admits whichever stage follows the BRMA checkpoint in the roster + # (the SPI block since #1012 moved it ahead of was_wealth), as main's tool + # does with UK_SPINE_ASSEMBLED_FINAL_STAGE. + stages = endpoint.stage_names + admitted = stages[stages.index("frs_brma") + 1] + assert admitted == "frs_hmrc_spine_leaves" + assert "spine.gates.assembled" in compiled.predecessors["frs_brma.checkpoint"] + assert "spine.gates.assembled" in compiled.predecessors[admitted] + assert graph.node(admitted).params["spine_gate_phase"] == "assembled" + assert "spine_gate_phase" not in graph.node("was_wealth").params + # The blocking posture travels with the gate and the admission it guards. + assert assembled.params["synthetic_smoke"] is False + assert graph.node(admitted).params["spine_gate_synthetic_smoke"] is False + smoke = add_uk_spine_gate_nodes( + spine, spec=country, engine_identity="test-engine", synthetic_smoke=True + ) + assert smoke.node("spine.gates.assembled").params["synthetic_smoke"] is True + assert smoke.node(admitted).params["spine_gate_synthetic_smoke"] is True + with pytest.raises(ValueError, match="release candidate"): + add_uk_spine_gate_nodes( + spine, + spec=country, + engine_identity="test-engine", + release_candidate=True, + synthetic_smoke=True, + ) + + +def _assembled_report(status): + from microcosm.build.gate_battery import ( + GateOutcome, + GatePhaseReport, + GateStatus, + gate_phase_report_payload, + ) + from microcosm.build.gates import GateResult + from microcosm.build.uk_runtime.graph_evidence import uk_spine_gate_manifest + + gates = uk_spine_gate_manifest(load_country_spec("uk")) + entries = tuple(entry for entry in gates.gates if entry.phase == "assembled") + selected = next( + entry + for entry in entries + if entry.criticality == "release_blocking" and not entry.evidence_absent_blocks + ) + outcomes = [] + for entry in entries: + state = status if entry == selected else GateStatus.PASSED + evaluated = state in (GateStatus.PASSED, GateStatus.FAILED) + outcomes.append( + GateOutcome( + entry, + state, + result=GateResult( + entry.gate, + state is GateStatus.PASSED, + () if state is GateStatus.PASSED else ("fixture failure",), + ) + if evaluated + else None, + reason=None if evaluated else "fixture missing reference", + ) + ) + return gate_phase_report_payload( + GatePhaseReport("assembled", tuple(outcomes)), gates=gates + ), selected.id + + +def test_assembled_admission_refuses_before_model_and_preserves_development_policy( + tmp_path, +): + from microcosm.build.gate_battery import GateStatus + from microcosm.build.uk_runtime.graph_evidence import ( + require_uk_spine_gate_admission, + ) + from microcosm.build.uk_runtime.graph_kernels import UKStageKernel + + report, failed = _assembled_report(GateStatus.FAILED) + path = tmp_path / "stored-phase.json" + path.write_text(json.dumps(report)) + context = SimpleNamespace( + artifacts={"spine_gate": SimpleNamespace(payload=path.read_bytes())}, + params={ + "spine_gate_phase": "assembled", + "spine_gate_release_candidate": False, + "spine_gate_synthetic_smoke": False, + }, + ) + + class UntouchedModel: + def __call__(self, frame): + raise AssertionError("A blocked assembled spine must not run a donor model") + + with pytest.raises(ValueError, match=failed): + UKStageKernel("was_wealth", UntouchedModel()).run(context) + assert json.loads(path.read_bytes()) == report + report, _ = _assembled_report(GateStatus.EVIDENCE_ABSENT) + context.artifacts["spine_gate"].payload = json.dumps(report).encode() + require_uk_spine_gate_admission(context) + context.params["spine_gate_release_candidate"] = True + with pytest.raises(ValueError, match="block downstream"): + require_uk_spine_gate_admission(context) diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json index 571492005..5da526f85 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json @@ -1 +1 @@ -{"country":"uk","nodes":[{"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_savings_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"047d30937efb21e39259fdbed5597751e4f5a0c341ea583c7446e4c9c11eba48","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tax_free_savings_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"frs_relationships","stage_contract_sha256":"30628808d06d9829ebcbcf460cd355a01261e78c7c73e6006c75bec2e3d60c33","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_employment","stage_contract_sha256":"ddbefaf05b44788d794a6e4b0e8926c14318d66e50d2f11f50ca549538bbf60c","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","sic_industry_division"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"frs_council_tax","stage_contract_sha256":"3381cd9f7a736514d5073c57480e07f5098890f3ca9ed3865392043594771eb9","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_education","stage_contract_sha256":"836cb0f5a582fee1425190e96c9cb81bdef859bd236c8b6bc27660b6e3d0c2f8","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"stage":"frs_take_up","stage_contract_sha256":"067d31b7fbe733d1d182588705e9ce79df0c8e5f6e8f519314765af49b1b00d7","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_person_draws","stage_contract_sha256":"7e0af955541ec5669dcdf1de4e1e6ca642853e48b775ec4e25238b41491df324","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"frs_brma","stage_contract_sha256":"5b8f0b361310c7cd3efbaae3762b347649b7d4afb9e9943131d9bef9266fc3c8","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employee_pension_contributions"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"2bb3d068003489b47cc8676ac26c203ce3c39a9b6e92f3ecd0a3c06acd6addc4","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","stage":"spi_support_channel","stage_contract_sha256":"4b10f1a4a215cdf2c406c9974de2d1e580eb3cd3a25ed34973c1b63abf4a54bb","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"free","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","stage":"spi_income_band_donors","stage_contract_sha256":"a0470c786427d0a4230a162e7a0177b07dbab15016a1f408321748de5a9dbefd","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tax_free_savings_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"hmrc_spi_income_spine","stage_contract_sha256":"038505263ed7821e028773e041d9e923131df001ad31439d78d7eba5aa1ea64f","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"spi_housing_shell","stage_contract_sha256":"809f19fd33ff79ceda005e8cc3fa89ae40b297dceca26a7658b6259267987890","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"was_wealth","stage_contract_sha256":"366e019c41ee2a28bcd61bbaa62a024ed0bb4b30b3ffc4d87ab23b340b4076bf","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"regional_property_uprating","stage_contract_sha256":"acb342e8ca1770556c5682a9411c5ead5d61dd1420cfe6b0305ed14a3407fb25","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"nts_bus_travel","stage_contract_sha256":"4a2d98e0722359b4597555b803b07dbf6362b6e261503e8712a8eca53f44ca97","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"lcfs_consumption","stage_contract_sha256":"a206bc9533fc78645ec100733350675db2732667995c83a10a168d81a899ae86","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"etb_vat","stage_contract_sha256":"5a2b777d88315bcccb72477c088a136c634131e0bc1a339b7287217ee5cfd8d3","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"etb_services","stage_contract_sha256":"6ed39560f89ef8f16845ca92e28aee484c1256c6e4988db32cca485f26801afe","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"regional_property_uprating.boundary","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"uc_reporter_redraw","stage_contract_sha256":"7e94e034cad29c5bae566ffb875c42cddf3a1025c15496907e783ceff1144f8b","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_benunit_head","is_parent","person_support_channel","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"uc_capital_coherence","stage_contract_sha256":"059f27ee687ee15385b246d43068b17aeecbe3645ccc41fd52fcad06a006a081","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","stage":"cgt_incidence_clone","stage_contract_sha256":"ee30278543cc0297a5366d855b7753aa04af5518971594c193c52be36bf8a7b4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Run structural UK stage cgt_band_donors.","id":"cgt_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_band_donors@1","mass":"free","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_band_donor","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","stage":"cgt_band_donors","stage_contract_sha256":"dfeefedb6c5b186aa94e5afb247cae51d8d6b2b180054d653ab65c8ca2199788","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_band_donors.","id":"cgt_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_band_donor"]},"population":"cgt_band_donors","sources":[],"structural":"none","weights":null},{"base":"cgt_band_donors","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_capital_gains_clone","household_is_cgt_band_donor"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported","tax_free_savings_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"4f6112180d00a5d64dc963379f92c88ec70188aed7842e24294e0e8b0ec878ea","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"04aca90943cf2dea4df66116b45c099f223f77c933f7a7c878ac26b328be430f","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","household_is_cgt_band_donor","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","employment_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","stage":"cgt_incidence_anchor","stage_contract_sha256":"00fbe9edc8bd1f31fd79b05cf20b3d5519f9a5b91ded0da35fae1316c1cbb9e0","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_capital_gains_clone","household_is_cgt_band_donor"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","capital_gains_asset_type","capital_gains_residential_property"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"salary_sacrifice","stage_contract_sha256":"a4af925af21cc6766eb7c8855b6743a3aa4eeeb38df12cfc83063d7ad6890539","time_period":"2024"},"population":"cgt_incidence_anchor","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","current_education","employee_pension_contributions","highest_education","student_loan_repayments","student_loans"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"student_loans","stage_contract_sha256":"7c4c76398c2e80b41c7f941a5f06246b57d89b5eb0c36d42e3116ac35e82e203","time_period":"2024"},"population":"cgt_incidence_anchor","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} +{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_savings_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"047d30937efb21e39259fdbed5597751e4f5a0c341ea583c7446e4c9c11eba48","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tax_free_savings_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"30628808d06d9829ebcbcf460cd355a01261e78c7c73e6006c75bec2e3d60c33","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"ddbefaf05b44788d794a6e4b0e8926c14318d66e50d2f11f50ca549538bbf60c","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","sic_industry_division"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"3381cd9f7a736514d5073c57480e07f5098890f3ca9ed3865392043594771eb9","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"836cb0f5a582fee1425190e96c9cb81bdef859bd236c8b6bc27660b6e3d0c2f8","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"067d31b7fbe733d1d182588705e9ce79df0c8e5f6e8f519314765af49b1b00d7","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"7e0af955541ec5669dcdf1de4e1e6ca642853e48b775ec4e25238b41491df324","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"5b8f0b361310c7cd3efbaae3762b347649b7d4afb9e9943131d9bef9266fc3c8","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employee_pension_contributions"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"2bb3d068003489b47cc8676ac26c203ce3c39a9b6e92f3ecd0a3c06acd6addc4","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"4b10f1a4a215cdf2c406c9974de2d1e580eb3cd3a25ed34973c1b63abf4a54bb","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"free","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"a0470c786427d0a4230a162e7a0177b07dbab15016a1f408321748de5a9dbefd","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tax_free_savings_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"038505263ed7821e028773e041d9e923131df001ad31439d78d7eba5aa1ea64f","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"809f19fd33ff79ceda005e8cc3fa89ae40b297dceca26a7658b6259267987890","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"366e019c41ee2a28bcd61bbaa62a024ed0bb4b30b3ffc4d87ab23b340b4076bf","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"acb342e8ca1770556c5682a9411c5ead5d61dd1420cfe6b0305ed14a3407fb25","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"4a2d98e0722359b4597555b803b07dbf6362b6e261503e8712a8eca53f44ca97","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"a206bc9533fc78645ec100733350675db2732667995c83a10a168d81a899ae86","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"5a2b777d88315bcccb72477c088a136c634131e0bc1a339b7287217ee5cfd8d3","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"6ed39560f89ef8f16845ca92e28aee484c1256c6e4988db32cca485f26801afe","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"regional_property_uprating.boundary","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"7e94e034cad29c5bae566ffb875c42cddf3a1025c15496907e783ceff1144f8b","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_benunit_head","is_parent","person_support_channel","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"059f27ee687ee15385b246d43068b17aeecbe3645ccc41fd52fcad06a006a081","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"uc_capital_coherence.boundary","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"ee30278543cc0297a5366d855b7753aa04af5518971594c193c52be36bf8a7b4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_clone","citation":"","description":"Run structural UK stage cgt_band_donors.","id":"cgt_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_band_donors@1","mass":"free","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_band_donor","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_band_donors","stage_contract_sha256":"dfeefedb6c5b186aa94e5afb247cae51d8d6b2b180054d653ab65c8ca2199788","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_band_donors.","id":"cgt_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_band_donor"]},"population":"cgt_band_donors","sources":[],"structural":"none","weights":null},{"base":"cgt_band_donors","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_capital_gains_clone","household_is_cgt_band_donor"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported","tax_free_savings_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"4f6112180d00a5d64dc963379f92c88ec70188aed7842e24294e0e8b0ec878ea","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"04aca90943cf2dea4df66116b45c099f223f77c933f7a7c878ac26b328be430f","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","household_is_cgt_band_donor","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","employment_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"00fbe9edc8bd1f31fd79b05cf20b3d5519f9a5b91ded0da35fae1316c1cbb9e0","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_capital_gains_clone","household_is_cgt_band_donor"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","student_loan_balance","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","capital_gains_asset_type","capital_gains_residential_property"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"a4af925af21cc6766eb7c8855b6743a3aa4eeeb38df12cfc83063d7ad6890539","time_period":"2024"},"population":"cgt_incidence_anchor","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","current_education","employee_pension_contributions","highest_education","student_loan_repayments","student_loans"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.100.0"],["policyengine-core","3.32.5"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"7c4c76398c2e80b41c7f941a5f06246b57d89b5eb0c36d42e3116ac35e82e203","time_period":"2024"},"population":"cgt_incidence_anchor","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index a2079ff58..493b09350 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -1,2037 +1,6 @@ -"""Build the raw UK FRS spine Frame from pinned local tabs.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import sys -import time -from collections.abc import Mapping, Sequence -from datetime import UTC, datetime -from importlib import metadata -from pathlib import Path - -from microcosm.build.country_spec import ( - GatesManifest, - load_country_spec, -) -from microcosm.build.frame_sampling import ( - normalize_sampled_household_mass, - sample_frame_households, -) -from microcosm.build.gate_battery import BlockingMode, EvidenceContext, GateBatteryRun -from microcosm.build.logbook import canonical_json_bytes -from microcosm.build.logbook_adoption import ( - AttemptState, - append_phase, - apply_error_verdict, - atomic_write_json, - error_receipt_path, - git_code_pin, - local_artifact_reference, - preflight_digest, - record_terminal_attempt, - resolve_predecessor, - role_pins_digest, - sha256_argument, - write_error_receipt, -) -from microcosm.build.observation import ( - ObservedTransform, - StageObservation, - StageObserver, -) -from microcosm.build.plan import StageRecord -from microcosm.build.staging_cli import ( - add_staging_arguments, - validate_staging_arguments, -) -from microcosm.build.staging_v2 import ( - StagingTelemetryV2, - disabled_staging_delivery, -) -from microcosm.build.uk_runtime.age_tail import UKAgeTailStageTransform -from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY -from microcosm.build.uk_runtime.calibration_run import ( - UK_SPINE_GATE_SCOPE, - uk_scoped_gate_manifest, -) -from microcosm.build.uk_runtime.cgt_asset_type import ( - CGT_ASSET_TYPE_DOMAIN, - uk_cgt_asset_type_stage_transform, -) -from microcosm.build.uk_runtime.cgt_imputation import uk_cgt_spine_stage_transform -from microcosm.build.uk_runtime.cgt_structure import ( - UKCGTBandDonorStageTransform, - UKCGTIncidenceAnchorStageTransform, - UKCGTIncidenceCloneStageTransform, -) -from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity -from microcosm.build.uk_runtime.etb_services import UKETBServicesStageTransform -from microcosm.build.uk_runtime.etb_vat import UKETBVATStageTransform -from microcosm.build.uk_runtime.frs_brma import UKFRSBRMAStageTransform -from microcosm.build.uk_runtime.frs_council_tax import UKFRSCouncilTaxStageTransform -from microcosm.build.uk_runtime.frs_disability import UKFRSDisabilityStageTransform -from microcosm.build.uk_runtime.frs_education import UKFRSEducationStageTransform -from microcosm.build.uk_runtime.frs_education_grants import ( - FRS_EDUCATION_GRANT_REWRITES, - UKFRSEducationGrantSplitStageTransform, -) -from microcosm.build.uk_runtime.frs_employment import UKFRSEmploymentStageTransform -from microcosm.build.uk_runtime.frs_household_draws import ( - UKFRSHouseholdDrawsStageTransform, -) -from microcosm.build.uk_runtime.frs_legacy_proxies import ( - UKFRSLegacyProxiesStageTransform, -) -from microcosm.build.uk_runtime.frs_person_draws import UKFRSPersonDrawsStageTransform -from microcosm.build.uk_runtime.frs_relationships import ( - CHRONICLE_ONS_HOUSEHOLD_TYPE_VALUE_IDS, - UKFRSRelationshipsStageTransform, -) -from microcosm.build.uk_runtime.frs_release import load_uk_frs_release -from microcosm.build.uk_runtime.frs_spine import ( - UKFRSSpineStageTransform, - uk_frs_spine_seed_frame, -) -from microcosm.build.uk_runtime.frs_take_up import UKFRSTakeUpStageTransform -from microcosm.build.uk_runtime.graph import ( - UK_SPINE_EXCLUSIONS, - uk_registry, - uk_spine_graph, -) -from microcosm.build.uk_runtime.hmrc_replay import write_hmrc_replay_report -from microcosm.build.uk_runtime.lcfs_consumption import ( - UKLCFSConsumptionStageTransform, -) -from microcosm.build.uk_runtime.national_frame import ( - uk_household_weight_kind, - write_uk_national_frame, -) -from microcosm.build.uk_runtime.national_sampling import ( - UK_SAMPLE_RUNG_TOKENS, - UK_SAMPLE_SEED_DEFAULT, -) -from microcosm.build.uk_runtime.nts_bus_travel import UKNTSBusTravelStageTransform -from microcosm.build.uk_runtime.regional_uprating import ( - UKRegionalPropertyUpratingStageTransform, -) -from microcosm.build.uk_runtime.salary_sacrifice import UKSalarySacrificeStageTransform -from microcosm.build.uk_runtime.spi_band_donors import ( - UKSPIIncomeBandDonorStageTransform, -) -from microcosm.build.uk_runtime.spi_housing_shell import ( - UKSPIHousingShellStageTransform, -) -from microcosm.build.uk_runtime.spi_spine import ( - UKFRSHMRCSpineLeavesStageTransform, - UKSPIIncomeSpineStageTransform, - UKSPISupportChannelStageTransform, -) -from microcosm.build.uk_runtime.staging import UK_STAGING_REPOSITORY -from microcosm.build.uk_runtime.student_loans import UKStudentLoansStageTransform -from microcosm.build.uk_runtime.take_up_contract import load_uk_take_up_contract -from microcosm.build.uk_runtime.uc_capital_coherence import ( - UKUCCapitalCoherenceStageTransform, -) -from microcosm.build.uk_runtime.uc_deduction_attributes import ( - UKUCDeductionAttributesStageTransform, -) -from microcosm.build.uk_runtime.uc_reporter_redraw import ( - UKUCReporterRedrawStageTransform, -) -from microcosm.build.uk_runtime.was_wealth import UKWASWealthStageTransform -from microcosm.frame import Frame -from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine -from microcosm.graph import ContentStore, compile_graph, run_graph - -_PIPELINE = "uk-frs-spine" -_REPOSITORY = Path(__file__).resolve().parents[1] -_RUNG_NAMED_EDGE_SIGNATURE = "The least populated classes in y have only 1 member" -_RUNG_ABORT_EXIT_CODE = 3 -#: The last stage of the assembled checkpoint: everything through the base -#: FRS mapping and the stochastic draws. A name, not an index — a position -#: standing in for a key is correct only while two independently-maintained -#: orderings happen to agree (the uk-data#468 class). -UK_SPINE_ASSEMBLED_FINAL_STAGE = "frs_brma" - - -def _uk_spine_stage_names(spec) -> tuple[str, ...]: - """Derive the runnable manifest stages from graph ownership edges.""" - - if spec.sources is None: - raise ValueError("UK country spec has no source stages.") - declared = { - stage.stage - for stage in spec.sources.stages - if stage.stage not in UK_SPINE_EXCLUSIONS - } - compiled = compile_graph(uk_spine_graph(spec)) - ordered = tuple(node_id for node_id in compiled.order if node_id in declared) - if set(ordered) != declared: - raise ValueError( - "UK spine graph and manifest stage roster disagree: " - f"graph={list(ordered)!r}, manifest={sorted(declared)!r}." - ) - return ordered - - -def _rung_sample_fraction(value: str) -> float: - """CLI rung policy (#624) over the permissive library validator.""" - - try: - fraction = float(value) - except ValueError as error: - raise argparse.ArgumentTypeError( - f"sample fraction must be a number; got {value!r}." - ) from error - if fraction not in UK_SAMPLE_RUNG_TOKENS: - raise argparse.ArgumentTypeError( - "sample fraction must be one of 0.01, 0.10, or 1.0 (the #624 rungs)." - ) - return fraction - - -def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=( - "Build the deterministic UK FRS spine from pinned raw tabs. Every " - "stochastic stage draws identity-keyed from seeds declared in the " - "manifest, so two runs from the same inputs are payload-identical." - ) - ) - parser.add_argument( - "--frs-raw-dir", - type=Path, - help="Directory containing the 14 licensed FRS 2024-25 tab files.", - ) - parser.add_argument( - "--spine-h5", - type=Path, - required=True, - help="Output H5 path for the raw FRS spine Frame.", - ) - parser.add_argument( - "--spi-tab", - type=Path, - help="Pinned local SPI 2022-23 put2223uk.tab path.", - ) - parser.add_argument( - "--hmrc-ods", - type=Path, - help="Pinned local HMRC collated ODS path.", - ) - parser.add_argument( - "--synthetic-fixture-dir", - type=Path, - help=( - "Data-only UK spine fixture source for non-release integration testing. " - "Requires --smoke and cannot be combined with licensed input options." - ), - ) - parser.add_argument( - "--checkpoint-dir", - type=Path, - help="Optional directory for a copy of the completed spine checkpoint.", - ) - parser.add_argument( - "--sample-fraction", - type=_rung_sample_fraction, - default=1.0, - help=( - "Scale-ladder rung (#624): 0.01 smoke, 0.10 dev, or 1.0 full. " - "Below 1.0 the raw FRS spine is sampled immediately after ingest, " - "renormalized to full household mass, and treated as a receipt." - ), - ) - parser.add_argument( - "--smoke", - action="store_true", - help=( - "Mark the output as non-release and stop after spine and telemetry " - "verification. Combine with --sample-fraction 0.01 for a small " - "licensed-data run, or use the complete synthetic fixture in tests." - ), - ) - parser.add_argument( - "--release-candidate", - action="store_true", - help=( - "Evaluate the spine battery at release-candidate strictness: " - "evidence_absent gaps block instead of being tolerated. Explicit " - "by design - a full-scale developer build is not a release " - "candidate unless the caller says so." - ), - ) - parser.add_argument( - "--sample-seed", - type=int, - default=UK_SAMPLE_SEED_DEFAULT, - help=f"Raw FRS spine sampling seed (default: {UK_SAMPLE_SEED_DEFAULT}).", - ) - parser.add_argument( - "--was-tab", - type=Path, - help="Caller-supplied private WAS round-8 household tab for was_wealth.", - ) - parser.add_argument( - "--nts-household-tab", - type=Path, - help="Caller-supplied private NTS household tab for nts_bus_travel.", - ) - parser.add_argument( - "--nts-individual-tab", - type=Path, - help="Caller-supplied private NTS individual tab for nts_bus_travel.", - ) - parser.add_argument( - "--nts-trip-tab", - type=Path, - help="Caller-supplied private NTS trip tab for nts_bus_travel.", - ) - parser.add_argument( - "--nts-stage-tab", - type=Path, - help="Caller-supplied private NTS stage tab for nts_bus_travel.", - ) - parser.add_argument( - "--nts-ticket-tab", - type=Path, - help="Caller-supplied private NTS ticket tab for nts_bus_travel.", - ) - parser.add_argument( - "--lcfs-hh-tab", - type=Path, - help="Caller-supplied private LCFS 2023-24 household tab for lcfs_consumption.", - ) - parser.add_argument( - "--lcfs-person-tab", - type=Path, - help="Caller-supplied private LCFS 2023-24 person tab for lcfs_consumption.", - ) - parser.add_argument( - "--etb-tab", - type=Path, - help="Caller-supplied private ETB 1977-2024 household tab for ETB stages.", - ) - parser.add_argument( - "--emit-nonzero-shares", - type=Path, - help="Optional JSON path for unweighted per-produced-column nonzero shares.", - ) - parser.add_argument( - "--logbook-prev-row-digest", - type=sha256_argument, - help="Optional current Logbook chain head.", - ) - add_staging_arguments(parser, repository=UK_STAGING_REPOSITORY) - args = parser.parse_args(argv) - if args.sample_seed < 0: - parser.error("sample seed must be a non-negative integer.") - if args.smoke and args.release_candidate: - parser.error("non-release smoke builds refuse --release-candidate.") - if _is_sampled(args) and args.checkpoint_dir is not None: - parser.error( - "sampled spine builds refuse --checkpoint-dir; sampled artifacts " - "cannot be reused as full-scale checkpoints." - ) - production_inputs = { - "--frs-raw-dir": args.frs_raw_dir, - "--spi-tab": args.spi_tab, - "--hmrc-ods": args.hmrc_ods, - } - if args.synthetic_fixture_dir is None: - missing = [flag for flag, value in production_inputs.items() if value is None] - if missing: - parser.error(f"production builds require {', '.join(missing)}.") - else: - supplied = [ - flag for flag, value in production_inputs.items() if value is not None - ] - supplied.extend( - flag - for flag, value in ( - ("--was-tab", args.was_tab), - ("--nts-household-tab", args.nts_household_tab), - ("--nts-individual-tab", args.nts_individual_tab), - ("--nts-trip-tab", args.nts_trip_tab), - ("--nts-stage-tab", args.nts_stage_tab), - ("--nts-ticket-tab", args.nts_ticket_tab), - ("--lcfs-hh-tab", args.lcfs_hh_tab), - ("--lcfs-person-tab", args.lcfs_person_tab), - ("--etb-tab", args.etb_tab), - ) - if value is not None - ) - if supplied: - parser.error( - "--synthetic-fixture-dir cannot be combined with licensed input " - f"options: {', '.join(supplied)}." - ) - if not args.smoke: - parser.error("--synthetic-fixture-dir requires --smoke.") - validate_staging_arguments(parser, args) - return args - - -def _is_sampled(args: argparse.Namespace) -> bool: - return args.sample_fraction != 1.0 - - -def _sample_token(args: argparse.Namespace) -> str: - return UK_SAMPLE_RUNG_TOKENS[args.sample_fraction] - - -def _validate_args(args: argparse.Namespace) -> None: - if args.synthetic_fixture_dir is not None: - if not args.synthetic_fixture_dir.is_dir(): - raise ValueError( - "--synthetic-fixture-dir must be an existing directory: " - f"{args.synthetic_fixture_dir}" - ) - if not (args.synthetic_fixture_dir / "fixture.json").is_file(): - raise ValueError( - "--synthetic-fixture-dir must contain fixture.json: " - f"{args.synthetic_fixture_dir}" - ) - elif args.frs_raw_dir is None or not args.frs_raw_dir.is_dir(): - raise ValueError( - f"--frs-raw-dir must be an existing directory: {args.frs_raw_dir}" - ) - if args.spine_h5.suffix != ".h5": - raise ValueError("--spine-h5 must end with '.h5'.") - if args.synthetic_fixture_dir is None: - if args.spi_tab is None or not args.spi_tab.is_file(): - raise ValueError(f"--spi-tab must be an existing file: {args.spi_tab}") - if args.spi_tab.name != "put2223uk.tab": - raise ValueError("--spi-tab must name put2223uk.tab.") - if args.hmrc_ods is None or not args.hmrc_ods.is_file(): - raise ValueError(f"--hmrc-ods must be an existing file: {args.hmrc_ods}") - if args.hmrc_ods.suffix.lower() != ".ods": - raise ValueError("--hmrc-ods must end with '.ods'.") - paths = { - "spine_h5": args.spine_h5, - "build_sidecar": args.spine_h5.with_suffix(".build.json"), - "hmrc_replay_sidecar": args.spine_h5.with_suffix(".hmrc_replay.json"), - } - if args.emit_nonzero_shares is not None: - paths["emit_nonzero_shares"] = args.emit_nonzero_shares - resolved: dict[Path, str] = {} - for label, path in paths.items(): - target = Path(path).expanduser().resolve() - other = resolved.get(target) - if other is not None: - raise ValueError(f"{label} path collides with {other}: {target}.") - resolved[target] = label - - -def _synthetic_fixture_evidence(source: Path | None) -> dict[str, object] | None: - """Bind a non-release integration run to every file in its fixture.""" - - if source is None: - return None - root = source.resolve() - files = [] - for path in sorted(item for item in root.rglob("*") if item.is_file()): - relative = path.resolve().relative_to(root).as_posix() - files.append( - { - "path": relative, - "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), - "size_bytes": path.stat().st_size, - } - ) - descriptor = json.loads((root / "fixture.json").read_text(encoding="utf-8")) - return { - "schema_version": descriptor.get("schema_version"), - "file_count": len(files), - "digest": hashlib.sha256(canonical_json_bytes(files)).hexdigest(), - } - - -def _synthetic_graph_sources(source: Path) -> dict[str, Path]: - """Resolve every split graph source to one reviewed fixture input.""" - - root = source.resolve() - descriptor = json.loads((root / "fixture.json").read_text(encoding="utf-8")) - inputs = descriptor.get("inputs") - if not isinstance(inputs, dict): - raise ValueError("Synthetic fixture inputs must be an object.") - names = { - "was": "was", - "nts_household": "nts_household", - "nts_individual": "nts_individual", - "nts_trip": "nts_trip", - "nts_stage": "nts_stage", - "nts_ticket": "nts_ticket", - "lcfs_household": "lcfs_household", - "lcfs_person": "lcfs_person", - "etb": "etb", - "spi": "spi_donor", - "hmrc_income": "hmrc_income_targets", - } - resolved = {"frs": root} - for role, name in names.items(): - relative = inputs.get(name) - if not isinstance(relative, str) or not relative: - raise ValueError(f"Synthetic fixture inputs.{name} must be a path.") - path = (root / relative).resolve() - try: - path.relative_to(root) - except ValueError as exc: - raise ValueError( - f"Synthetic fixture inputs.{name} escapes the fixture directory." - ) from exc - if not path.exists(): - raise ValueError(f"Synthetic fixture input {name!r} does not exist.") - resolved[role] = path - return resolved - - -def _synthetic_fixture_input(source: Path, name: str) -> Path: - """Resolve one named fixture input without allowing path traversal.""" - - root = source.resolve() - descriptor = json.loads((root / "fixture.json").read_text(encoding="utf-8")) - inputs = descriptor.get("inputs") - relative = inputs.get(name) if isinstance(inputs, dict) else None - if not isinstance(relative, str) or not relative: - raise ValueError(f"Synthetic fixture inputs.{name} must be a path.") - path = (root / relative).resolve() - try: - path.relative_to(root) - except ValueError as exc: - raise ValueError( - f"Synthetic fixture inputs.{name} escapes the fixture directory." - ) from exc - if not path.exists(): - raise ValueError(f"Synthetic fixture input {name!r} does not exist.") - return path - - -def _artifact_pins(stages) -> dict[str, dict[str, object]]: - pins = {} - for stage in stages: - for artifact in stage.artifacts: - key = artifact.get("table", artifact.get("filename")) - if key is None: - continue - key = str(key) - pin = { - "locator": str(artifact["locator"]), - "sha256": str(artifact["sha256"]), - "size_bytes": int(artifact["size_bytes"]), - } - if key in pins and pins[key] != pin: - raise ValueError( - f"UK source artifact {key!r} has inconsistent pins across stages." - ) - pins[key] = pin - return dict(sorted(pins.items())) - - -def _stage_artifact_pins(stage) -> dict[str, dict[str, object]]: - return { - str(artifact.get("table", artifact.get("filename"))): { - "locator": str(artifact["locator"]), - "sha256": str(artifact["sha256"]), - "size_bytes": int(artifact["size_bytes"]), - } - for artifact in stage.artifacts - if "table" in artifact or "filename" in artifact - } - - -def _resource_pins(stages, spec) -> dict[str, str]: - """Country-package resources the selected stages declare as inputs. - - Non-tab artifacts reference committed resources by filename; their bytes - are hashed by load_country_spec, so the pin is the spec's recorded sha. - """ - - pins: dict[str, str] = {} - for stage in stages: - for artifact in stage.artifacts: - if "resource" not in artifact: - continue - resource = str(artifact["resource"]) - sha256 = spec.resource_hashes.get(resource) - if sha256 is None: - raise ValueError( - f"stage {stage.stage!r} declares resource artifact " - f"{resource!r} which is not a declared country-package " - "resource." - ) - pins[resource] = str(sha256) - return dict(sorted(pins.items())) - - -def _input_artifact_pins(stages) -> dict[str, dict[str, object]]: - """Caller-supplied private input artifacts, pinned by role. - - Non-table, non-resource artifacts (the SPI donor tab and the HMRC ODS) - carry their own sha256/size pins in the manifest. Binding them here puts - the pins in the build sidecar and the Logbook input-pins digest, so two - runs with different high-impact source inputs can never share build-side - provenance (adversarial-review finding on #717). - """ - - pins: dict[str, dict[str, object]] = {} - for stage in stages: - for artifact in stage.artifacts: - if "table" in artifact or "resource" in artifact: - continue - if "sha256" not in artifact: - continue - role = str(artifact.get("role") or artifact.get("filename") or "") - if not role: - raise ValueError( - f"stage {stage.stage!r} declares a pinned input artifact " - "without a role or filename." - ) - pin = { - "filename": str( - artifact.get("filename") or artifact.get("locator") or "" - ), - "kind": str(artifact.get("kind", "")), - "sha256": str(artifact["sha256"]), - "size_bytes": int(artifact["size_bytes"]), - } - if role in pins and pins[role] != pin: - raise ValueError( - f"input artifact role {role!r} has inconsistent pins across stages." - ) - pins[role] = pin - return dict(sorted(pins.items())) - - -def _role_pins(pins: dict[str, dict[str, object]]) -> dict[str, dict[str, object]]: - return { - table: { - "sha256": str(pin["sha256"]), - "size_bytes": int(pin["size_bytes"]), - } - for table, pin in pins.items() - } - - -def _entity_row_counts(frame) -> dict[str, int]: - return {entity: int(len(frame.table(entity))) for entity in frame.entities} - - -def _rules_engine() -> PolicyEngineUKEngine: - try: - import policyengine_uk # noqa: F401 - except ImportError as exc: - raise ImportError( - "build_uk_frs_spine requires the microcosm-build 'uk' extra " - "(policyengine-uk). Run: uv sync --all-packages --extra uk" - ) from exc - return PolicyEngineUKEngine() - - -def _rules_engine_provenance() -> dict[str, str]: - try: - version = metadata.version("policyengine-uk") - except metadata.PackageNotFoundError: - return {"package": "policyengine-uk", "version": "unavailable"} - return {"package": "policyengine-uk", "version": version} - - -def _declared_seeds(stages) -> dict[str, dict[str, int]]: - declared: dict[str, dict[str, int]] = {} - for stage in stages: - stage_seeds: dict[str, int] = {} - for operation in stage.operations: - output = operation.parameters.get("output") - seed = operation.parameters.get("seed") - if seed is None: - seed = operation.parameters.get("seed_base") - if isinstance(output, str) and isinstance(seed, int): - stage_seeds[output] = seed - elif isinstance(seed, int): - if operation.kind == "stack_zero_weight_donors": - stage_seeds["stack_zero_weight_donors"] = seed - elif operation.kind == "strict_read_private_table": - stage_seeds["donor_bootstrap"] = seed - elif operation.kind == "fit_weighted_qrf_stage1": - stage_seeds["stage1"] = seed - elif operation.kind == "fit_weighted_qrf_stage2": - stage_seeds["stage2"] = seed - elif operation.kind == "assign_binary_from_rate": - target = operation.parameters.get("target") - if isinstance(target, str): - stage_seeds[target] = seed - else: - stage_seeds["assign_binary_from_rate"] = seed - elif operation.kind == "fit_weighted_qrf_chain": - stage_seeds[stage.stage] = seed - elif operation.kind == "fit_weighted_qrf": - stage_seeds[stage.stage] = seed - elif operation.kind == "draw_capital_gains_prior_from_banded_quantiles": - stage_seeds[str(operation.parameters["salt"])] = seed - elif operation.kind == "stack_band_donor_households": - stage_seeds["stack_band_donor_households"] = seed - elif operation.kind == "stack_income_band_donor_households": - stage_seeds["stack_income_band_donor_households"] = seed - elif operation.kind == "resample_band_donor_leaves": - stage_seeds["band_donor_resample"] = seed - elif operation.kind == "impute_spi_housing_shell": - stage_seeds[stage.stage] = seed - elif operation.kind == "price_domestic_energy": - stage_seeds["gas_disconnection"] = seed - elif operation.kind == "within_band_draws": - stage_seeds["within_band_draws"] = seed - elif operation.kind in ( - "assign_residential_property_flag", - "assign_main_asset_type", - ): - stage_seeds[operation.kind] = seed - elif operation.kind == "convert_donors_to_target_stock": - stage_seeds[str(operation.parameters["salt"])] = seed - elif operation.kind == "top_up_to_stock": - stage_seeds[str(operation.parameters["salt"])] = seed - if stage_seeds: - declared[stage.stage] = stage_seeds - return declared - - -def _result_evidence(result: object) -> object: - if isinstance(result, dict): - return result - evidence = getattr(result, "evidence", None) - if callable(evidence): - return evidence() - return None - - -def _collect_stage_evidence( - *, - stage_names: Sequence[str], - implementations: Mapping[str, object], -) -> dict[str, object]: - evidence_by_stage: dict[str, object] = {} - for stage_name in stage_names: - implementation = implementations.get(stage_name) - if implementation is None: - continue - metadata = None - metadata_hook = getattr(implementation, "checkpoint_metadata", None) - if callable(metadata_hook): - metadata = dict(metadata_hook()) - evidence = metadata.get("evidence", metadata) - else: - evidence = _result_evidence(getattr(implementation, "last_result", None)) - if evidence is not None: - evidence_by_stage[stage_name] = evidence - return evidence_by_stage - - -def _fit_weight_hook_holder(implementation: object) -> object | None: - """The object declaring ``fit_weight_records``, seen through the proxies. - - Detects the hook without evaluating it (a raising property must count as - a fitting stage with unreadable records, not vanish), and looks through - the wrappers a staged run puts around every stage: ``ObservedTransform`` - (telemetry) and the graph's source transform both proxy attribute reads - through ``__getattr__``, which a class-level probe never consults — so - every telemetry-enabled spine since the graph driver silently lost the - block and the release-cut weights audit found no evidence. - """ - - seen: set[int] = set() - candidate: object | None = implementation - while candidate is not None and id(candidate) not in seen: - seen.add(id(candidate)) - declared = getattr(type(candidate), "fit_weight_records", None) is not None - if declared or "fit_weight_records" in getattr(candidate, "__dict__", {}): - return candidate - candidate = getattr(candidate, "__dict__", {}).get("transform") - return None - - -def _collect_fit_weight_records( - *, - stage_names: Sequence[str], - implementations: Mapping[str, object], -) -> dict[str, list[dict[str, str]]]: - """Persist each fitting stage's resolved weight kinds into the sidecar. - - The terminal weights audit (``uk_weights_audit``) consumes - :class:`FitWeightRecord` evidence that only exists on live stage - objects; the release-cut certification producer runs in a later - process, so the sidecar carries the records across the run boundary. - Duck-typed like ``stage_evidence``: every stage whose transform exposes - ``fit_weight_records`` contributes, in stage order. A fitting stage - whose records are missing, unreadable, or empty records an empty list — - the audit binding fails an empty record set, so the gap stays visible - rather than vanishing from the sidecar. - """ - - records_by_stage: dict[str, list[dict[str, str]]] = {} - for stage_name in stage_names: - implementation = implementations.get(stage_name) - if implementation is None: - continue - holder = _fit_weight_hook_holder(implementation) - if holder is None: - continue - try: - records = tuple(holder.fit_weight_records or ()) - except Exception: # noqa: BLE001 - unreadable records fail the audit - records_by_stage[stage_name] = [] - continue - records_by_stage[stage_name] = [ - { - "fit_name": str(record.fit_name), - "weight_kind": str(record.weight_kind), - } - for record in records - ] - return records_by_stage - - -def _build_sidecar( - *, - frame, - stages, - records, - artifact_pins, - resource_pins: dict[str, str], - input_artifact_pins: dict[str, dict[str, object]], - hmrc_replay: dict[str, object], - stochastic_contract_sha256: str, - frs_vintage: str, - sampling: dict[str, object] | None, - non_release: bool = False, - release_posture: str = "development", - synthetic_fixture: Mapping[str, object] | None = None, - staging_delivery: Mapping[str, object] | None = None, - spine_gate_report: dict[str, object] | None = None, -) -> dict[str, object]: - household_weight = frame.weights_for("household") - return { - "schema_version": 2, - "pipeline": _PIPELINE, - "uk_frame_content_identity": uk_frame_content_identity(frame), - "stages": [stage.stage for stage in stages], - "time_period": str(frame.metadata["time_period"]), - "household_weight_kind": uk_household_weight_kind(frame).value, - "household_weight_total": float(household_weight.values.sum()), - "entity_row_counts": _entity_row_counts(frame), - "artifact_pins": artifact_pins, - "resource_pins": resource_pins, - "input_artifact_pins": input_artifact_pins, - "hmrc_replay": hmrc_replay, - "stage_artifact_pins": { - stage.stage: _stage_artifact_pins(stage) for stage in stages - }, - "stage_records": [ - { - "stage": record.stage, - "produced": list(record.produced), - "nonzero_share": dict(record.nonzero_share), - "seconds": record.seconds, - } - for record in records - ], - "operations": { - stage.stage: [operation.kind for operation in stage.operations] - for stage in stages - }, - "declared_seeds": _declared_seeds(stages), - "source_vintages": {"frs": frs_vintage}, - "sampling": sampling, - "non_release": non_release, - "release_posture": release_posture, - "synthetic_fixture": ( - None if synthetic_fixture is None else dict(synthetic_fixture) - ), - "staging_delivery": dict(staging_delivery or {}), - "spine_gate_report": spine_gate_report, - "stochastic_contract_sha256": stochastic_contract_sha256, - "rules_engine": _rules_engine_provenance(), - } - - -def _mark_non_release_h5(path: Path, *, build_id: str) -> None: - """Persist machine-readable refusal evidence on a bounded smoke H5.""" - - import h5py - - with h5py.File(path, mode="r+") as file: - file.attrs["populace_non_release"] = True - file.attrs["populace_release_posture"] = "smoke" - file.attrs["populace_smoke_build_id"] = build_id - - -def _nonzero_shares(frame, columns: list[str]) -> dict[str, float]: - shares: dict[str, float] = {} - for column in columns: - for entity in frame.entities: - table = frame.table(entity) - if column not in table.columns: - continue - values = table[column] - if values.dtype == object: - shares[column] = float(values.astype(str).ne("").mean()) - else: - shares[column] = float((values != 0).mean()) - break - return shares - - -def _series_nonzero_share(values) -> float: - if values.dtype == object or str(values.dtype).startswith("string"): - return float(values.fillna("").astype(str).ne("").mean()) - return float((values != 0).mean()) - - -def _graph_stage_records( - *, - manifest, - store: ContentStore, - stages, - frame, -) -> tuple[StageRecord, ...]: - """Project immediate node artifacts onto the legacy record schema. - - Entity ids and memberships are executor-carried context, not owned cells, - so the root node exposes no artifact for them although ``frs_spine`` - declares them as outputs. Their share is read from the final population - instead, which is what the legacy plan recorded (identity columns are - never zero, so the value is 1.0 on every vintage). - """ - - structural = _structural_columns(frame) - records: list[StageRecord] = [] - for stage in stages: - output_node = ( - f"{stage.stage}.owned" - if f"{stage.stage}.owned" in manifest.nodes - else stage.stage - ) - output_receipt = manifest.nodes[output_node] - shares: dict[str, float] = {} - for column in stage.outputs: - matches = [ - (coordinate, key) - for coordinate, key in output_receipt.artifacts.items() - if coordinate[1] == column - ] - if not matches and column in structural: - shares[column] = _nonzero_shares(frame, [column])[column] - continue - if len(matches) != 1: - raise RuntimeError( - f"graph stage {stage.stage!r} exposes {len(matches)} artifacts " - f"for declared output {column!r}." - ) - shares[column] = _series_nonzero_share(store.load_column(matches[0][1])) - execution_node = "create_uk_frs" if stage.stage == "frs_spine" else stage.stage - records.append( - StageRecord( - stage=stage.stage, - produced=stage.outputs, - donor_survey=stage.survey, - nonzero_share=shares, - seconds=manifest.nodes[execution_node].wall_time, - ) - ) - return tuple(records) - - -def _structural_columns(frame) -> frozenset[str]: - """Entity id and membership columns the executor carries outside owned cells.""" - - schema = frame.schema - columns = {schema.entity_id_column(entity) for entity in frame.entities} - columns.update(schema.membership_column(group) for group in schema.group_entities) - return frozenset(columns) - - -def _new_build_id(timestamp: datetime) -> str: - return f"uk-frs-spine-{timestamp.strftime('%Y%m%dT%H%M%SZ')}" - - -def _record_attempt( - *, - state: AttemptState, - started_at: float, - started_ts: datetime, - code_pin: str, - disposition: str, - predecessor: str | None, - rung: str, - spool_dir: Path, -) -> Path: - return record_terminal_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - pipeline=_PIPELINE, - rung=rung, - seed=None, - code_pin=code_pin, - disposition=disposition, - predecessor=predecessor, - spool_dir=spool_dir, - ) - - -def _sample_spine_frame( - frame, - *, - fraction: float, - seed: int, -) -> tuple[object, dict[str, object] | None]: - if fraction == 1.0: - return frame, None - household_weight = frame.weights_for("household") - pre_households = int(len(frame.table("household"))) - sampled, receipt = sample_frame_households( - frame, - fraction=fraction, - seed=seed, - source_name="UK FRS spine", - ) - normalized, factor = normalize_sampled_household_mass( - sampled, - target_mass=float(household_weight.total), - source_name="UK FRS spine", - ) - return normalized, { - "fraction": float(fraction), - "seed": int(seed), - "rung_token": UK_SAMPLE_RUNG_TOKENS[fraction], - "pre_household_count": pre_households, - "post_household_count": int(len(normalized.table("household"))), - "normalization_factor": float(factor), - "receipt": dict(receipt), - } - - -class _SampledGraphRootTransform: - """CREATE-stage adapter applying the declared sampling rung at ingest.""" - - def __init__( - self, - transform, - *, - fraction: float, - seed: int, - ) -> None: - self.transform = transform - self.fraction = fraction - self.seed = seed - self.sampling: dict[str, object] | None = None - - def _sample(self, assembled): - sampled, self.sampling = _sample_spine_frame( - assembled, - fraction=self.fraction, - seed=self.seed, - ) - # Graph populations use row positions as their internal alignment - # index. Frame sampling preserves source DataFrame indexes by design, - # so normalize those indexes at this adapter boundary. - tables = { - entity: sampled.table(entity).reset_index(drop=True) - for entity in sampled.entities - } - tables.update( - {name: sampled.link(name).reset_index(drop=True) for name in sampled.links} - ) - return Frame( - tables, - sampled.schema, - { - entity: sampled.weights_for(entity) - for entity in sampled.weighted_entities - }, - sampled.strata.reset_index(drop=True), - mass_log=sampled.mass_log, - metadata=sampled.metadata, - ) - - def effective_fraction(self) -> float: - """Return the configured input sampling fraction.""" - - return float(self.fraction) - - def __call__(self, frame): - return self._sample(self.transform(frame)) - - def run_with_sources(self, frame, sources): - runner = getattr(self.transform, "run_with_sources", None) - assembled = ( - runner(frame, sources) if callable(runner) else self.transform(frame) - ) - return self._sample(assembled) - - def checkpoint_metadata(self) -> dict[str, object]: - hook = getattr(self.transform, "checkpoint_metadata", None) - if not callable(hook): - raise RuntimeError("FRS root transform exposes no checkpoint metadata.") - return dict(hook()) - - -def _staging_stage_observer(telemetry: StagingTelemetryV2) -> StageObserver: - """Translate a shared stage observation into staging telemetry.""" - - def observe(observation: StageObservation) -> None: - telemetry.stage( - observation.stage_id, - event_status=observation.status, - elapsed_seconds=observation.elapsed_seconds, - entity_row_counts=dict(observation.entity_row_counts), - produced_column_count=observation.produced_column_count, - ) - - return observe - - -class _GraphSourceTransform: - """Build a file-reading stage from only the node's declared source paths.""" - - def __init__(self, factory) -> None: - self.factory = factory - self.transform = None - - def run_with_sources(self, frame, sources): - self.transform = self.factory(sources) - result = self.transform(frame) - if hasattr(self.transform, "fit_weight_records"): - self.fit_weight_records = self.transform.fit_weight_records - return result - - def __getattr__(self, name: str): - transform = self.__dict__.get("transform") - if transform is None: - raise AttributeError(name) - return getattr(transform, name) - - -def _run_plan_with_spine_sampling( - plan, - *, - sample_fraction: float, - sample_seed: int, - spine_battery: GateBatteryRun | None = None, - stage_evidence_provider=None, - gate_artifacts: Mapping[str, object] | None = None, -) -> tuple[object, tuple[object, ...], dict[str, object] | None]: - if not plan.stages or plan.stages[0].name != "frs_spine": - frame, records = plan.run(uk_frs_spine_seed_frame()) - return frame, records, None - - from microcosm.build.plan import StagePlan - - spine_frame, spine_records = StagePlan(plan.stages[:1]).run( - uk_frs_spine_seed_frame() - ) - spine_frame, sampling = _sample_spine_frame( - spine_frame, - fraction=sample_fraction, - seed=sample_seed, - ) - if len(plan.stages) == 1: - return spine_frame, spine_records, sampling - names = tuple(stage.name for stage in plan.stages) - if UK_SPINE_ASSEMBLED_FINAL_STAGE in names: - assembled_end = names.index(UK_SPINE_ASSEMBLED_FINAL_STAGE) + 1 - elif spine_battery is not None: - raise RuntimeError( - "spine battery is armed but the declared assembled-boundary stage " - f"{UK_SPINE_ASSEMBLED_FINAL_STAGE!r} is not in the plan; a stage " - "plan change must move the boundary declaration with it." - ) - else: - assembled_end = len(plan.stages) - frame, assembled_records = StagePlan(plan.stages[1:assembled_end]).run(spine_frame) - # Each boundary offers only the stages that have actually run: asking a - # later stage for checkpoint evidence would (correctly) raise, and the - # first licensed battery run did exactly that at the assembled boundary. - executed = tuple(stage.name for stage in plan.stages[:assembled_end]) - if spine_battery is not None: - _run_spine_gate_phase( - spine_battery, - "assembled", - frame=frame, - stage_evidence=( - stage_evidence_provider(executed) - if stage_evidence_provider is not None - else {} - ), - gate_artifacts=gate_artifacts, - ) - if assembled_end == len(plan.stages): - return frame, (*spine_records, *assembled_records), sampling - frame, tail_records = StagePlan(plan.stages[assembled_end:]).run(frame) - executed = tuple(stage.name for stage in plan.stages) - if spine_battery is not None: - _run_spine_gate_phase( - spine_battery, - "transferred", - frame=frame, - stage_evidence=( - stage_evidence_provider(executed) - if stage_evidence_provider is not None - else {} - ), - gate_artifacts=gate_artifacts, - ) - return frame, (*spine_records, *assembled_records, *tail_records), sampling - - -def _spine_gate_artifacts(engine: object) -> dict[str, object]: - """Evidence artifacts for spine-phase gates: the engine plus frame-only enum domains.""" - - return { - "rules_engine": engine, - # #791: ons_household_type is a frame column, not an engine variable, - # so its enum_domain gate takes the declared domain as an artifact. - "ons_household_type_enum_domain": CHRONICLE_ONS_HOUSEHOLD_TYPE_VALUE_IDS, - # #725: capital_gains_asset_type is likewise a frame column whose - # domain the asset-type stage declares. - "capital_gains_asset_type_enum_domain": CGT_ASSET_TYPE_DOMAIN, - } - - -def _run_spine_gate_phase( - battery: GateBatteryRun, - phase: str, - *, - frame, - stage_evidence: Mapping[str, object], - gate_artifacts: Mapping[str, object] | None = None, -) -> None: - artifacts: dict[str, object] = {"stage_evidence": dict(stage_evidence)} - # The enum-domain gate resolves its domain from the live rules engine, - # exactly as the national terminal battery supplied it. - artifacts.update(dict(gate_artifacts or {})) - battery.run_phase( - phase, - EvidenceContext(frame=frame, artifacts=artifacts), - ) - battery.enforce(phase, mode=BlockingMode.BLOCKS_ARTIFACT) - - -def _spine_gate_report_path(spine_h5: Path) -> Path: - return spine_h5.with_suffix(".spine_gates.json") - - -def _spine_gate_manifest_from_spec(spec) -> GatesManifest | None: - """The spine build's scoped battery manifest, from the shared helper. - - A spec without a gates block leaves the battery unarmed (``None``), - exactly as before; when armed, the filtering runs through the one - scope-filtering implementation every scoped producer shares. The - driver passes the spec it already loaded, which is also the hermetic - tests' stub point. Digests are identical to the previous local copy - because entries, phases, and the policy suffix are unchanged. - """ - - source = getattr(spec, "gates", None) - if source is None: - return None - return uk_scoped_gate_manifest( - UK_SPINE_GATE_SCOPE, - phases=("assembled", "transferred"), - policy_suffix="spine_build_scope", - source=source, - ) - - -def _rung_abort_receipt( - args: argparse.Namespace, - *, - error: BaseException, -) -> dict[str, object]: - return { - "schema_version": 1, - "artifact_kind": "uk_frs_spine_rung_abort_receipt", - "build_kind": "uk_frs_spine", - "sampling": { - "sample_fraction": float(args.sample_fraction), - "sample_seed": int(args.sample_seed), - "rung_token": _sample_token(args), - }, - "named_edge": "spine_split_singleton_class", - "stage": "frs_spine", - "error": str(error), - "disposition": "aborted_with_receipt", - "remedy": ( - "Re-roll --sample-seed; accepted dev-scale statistical edge. " - "The computation is never altered to avoid it." - ), - } - - -def _exception_chain_contains(error: BaseException, text: str) -> bool: - """Match a named rung edge through graph execution wrappers.""" - - seen: set[int] = set() - current: BaseException | None = error - while current is not None and id(current) not in seen: - seen.add(id(current)) - if text in str(current): - return True - current = current.__cause__ or current.__context__ - return False - - -def _create_staging_telemetry( - args: argparse.Namespace, *, state: AttemptState -) -> StagingTelemetryV2 | None: - if args.no_staging: - return None - local_dir = args.staging_dir or args.spine_h5.parent / "staging" - local_only = args.staging_local_only - return StagingTelemetryV2( - run_id=args.staging_run_id or state.build_id, - country_code="GB", - operation_id="uk_frs_spine", - pipeline_id=_PIPELINE, - pipeline_version=metadata.version("microcosm-build"), - candidate_id=args.staging_candidate_id or state.build_id, - local_dir=local_dir, - run_kind="smoke" if args.smoke else "spine", - delivery_mode="local_only" if local_only else "local_and_remote", - repo_id=None if local_only else args.staging_repo_id, - upload_interval_seconds=args.staging_upload_interval_seconds, - ) - - -def _telemetry_sample( - args: argparse.Namespace, sampling: Mapping[str, object] | None -) -> dict[str, object] | None: - if args.sample_fraction == 1.0: - return {"mode": "full"} - return None - - -def _staging_delivery( - args: argparse.Namespace, telemetry: StagingTelemetryV2 | None -) -> dict[str, object]: - if telemetry is None: - return disabled_staging_delivery("--no-staging") - return telemetry.delivery_summary - - -def main(argv: list[str] | None = None) -> int: - args = _parse_args(argv) - rung = UK_SAMPLE_RUNG_TOKENS[args.sample_fraction] - started_at = time.perf_counter() - started_ts = datetime.now(UTC) - predecessor = resolve_predecessor(args.logbook_prev_row_digest) - digest = preflight_digest(_PIPELINE) - state = AttemptState( - build_id=_new_build_id(started_ts), - identity_digest=digest, - input_pins_digest=digest, - phases_reached=["attempt_started"], - gate_verdicts={ - "pipeline": { - "verdict": "running", - "receipt": "pending-build-scoped-spine-receipt", - } - }, - ) - code_pin = "unresolved-local-git-code-pin" - spool_dir = args.spine_h5.parent / "logbook-spool" - telemetry: StagingTelemetryV2 | None = None - try: - _validate_args(args) - # A crash between the H5 write and the sidecar writes must never - # leave a stale sidecar beside a fresh H5 (adversarial-review - # finding on #717): clear every output up front, and treat the - # build sidecar - written last, binding the replay hash - as the - # marker that the bundle is complete. - stale_outputs = [ - args.spine_h5, - args.spine_h5.with_suffix(".build.json"), - args.spine_h5.with_suffix(".hmrc_replay.json"), - _spine_gate_report_path(args.spine_h5), - args.spine_h5.with_suffix(".rung_abort.json"), - ] - if args.emit_nonzero_shares is not None: - stale_outputs.append(args.emit_nonzero_shares) - for stale in stale_outputs: - stale.unlink(missing_ok=True) - telemetry = _create_staging_telemetry(args, state=state) - if telemetry is not None: - initial_sample = _telemetry_sample(args, None) - if initial_sample is not None: - telemetry.set_sample(initial_sample) - telemetry.stage( - "configuration", - event_status="completed", - smoke=args.smoke, - sample_mode=("fraction" if args.sample_fraction != 1.0 else "full"), - ) - code_pin = git_code_pin(_REPOSITORY) - append_phase(state, "configured") - spec = load_country_spec("uk") - if spec.sources is None: - raise ValueError("UK country spec has no source stages.") - stages_by_name = spec.sources.stage_map() - graph = uk_spine_graph( - spec, - source_mode="split", - sample_fraction=args.sample_fraction, - sample_seed=args.sample_seed, - ) - compiled_graph = compile_graph(graph) - stage_names = _uk_spine_stage_names(spec) - if ( - args.synthetic_fixture_dir is None - and "was_wealth" in stage_names - and args.was_tab is None - ): - raise ValueError( - "--was-tab is required when the was_wealth stage is scheduled." - ) - if args.synthetic_fixture_dir is None and "nts_bus_travel" in stage_names: - missing_nts = [ - flag - for flag, value in ( - ("--nts-household-tab", args.nts_household_tab), - ("--nts-individual-tab", args.nts_individual_tab), - ("--nts-trip-tab", args.nts_trip_tab), - ("--nts-stage-tab", args.nts_stage_tab), - ("--nts-ticket-tab", args.nts_ticket_tab), - ) - if value is None - ] - if missing_nts: - raise ValueError( - "nts_bus_travel requires caller-supplied private inputs: " - f"{', '.join(missing_nts)}." - ) - if args.synthetic_fixture_dir is None and "lcfs_consumption" in stage_names: - missing_lcfs = [ - flag - for flag, value in ( - ("--lcfs-hh-tab", args.lcfs_hh_tab), - ("--lcfs-person-tab", args.lcfs_person_tab), - ) - if value is None - ] - if missing_lcfs: - raise ValueError( - "lcfs_consumption requires caller-supplied private inputs: " - f"{', '.join(missing_lcfs)}." - ) - if ( - args.synthetic_fixture_dir is None - and ("etb_vat" in stage_names or "etb_services" in stage_names) - and args.etb_tab is None - ): - raise ValueError( - "--etb-tab is required when etb_vat or etb_services is scheduled." - ) - stages = [stages_by_name[name] for name in stage_names] - artifact_pins = _artifact_pins(stages) - resource_pins = _resource_pins(stages, spec) - input_artifact_pins = _input_artifact_pins(stages) - overlapping_pin_roles = set(artifact_pins) & set(input_artifact_pins) - if overlapping_pin_roles: - raise ValueError( - "input artifact roles collide with FRS tab names: " - f"{sorted(overlapping_pin_roles)}." - ) - state.input_pins_digest = role_pins_digest( - _role_pins({**artifact_pins, **input_artifact_pins}) - ) - synthetic_fixture = _synthetic_fixture_evidence(args.synthetic_fixture_dir) - run_config = { - "pipeline": _PIPELINE, - "stages": list(stage_names), - "artifact_pins_digest": state.input_pins_digest, - "spine_h5": str(args.spine_h5), - "synthetic_fixture": synthetic_fixture, - } - state.identity_digest = hashlib.sha256( - canonical_json_bytes(run_config) - ).hexdigest() - append_phase(state, "inputs_pinned") - if telemetry is not None: - telemetry.stage( - "input_verification", - event_status="completed", - stage_count=len(stage_names), - input_artifact_count=len(artifact_pins) + len(input_artifact_pins), - ) - engine = _rules_engine() - stochastic_contract = load_uk_take_up_contract() - frs_release = load_uk_frs_release() - hmrc_spine_transform = _GraphSourceTransform( - lambda sources: UKSPIIncomeSpineStageTransform( - sources["spi"], - sources["hmrc_income"], - stage=stages_by_name["hmrc_spi_income_spine"], - sampled_rung=_is_sampled(args), - ) - ) - implementations = { - "frs_spine": _GraphSourceTransform( - lambda sources: UKFRSSpineStageTransform( - sources["frs"], - stage=stages_by_name["frs_spine"], - ) - ), - "frs_employment": _GraphSourceTransform( - lambda sources: UKFRSEmploymentStageTransform( - sources["frs"], - stage=stages_by_name["frs_employment"], - ) - ), - "frs_council_tax": _GraphSourceTransform( - lambda sources: UKFRSCouncilTaxStageTransform( - sources["frs"], - stage=stages_by_name["frs_council_tax"], - ) - ), - "frs_disability": UKFRSDisabilityStageTransform( - stage=stages_by_name["frs_disability"], - ), - "frs_relationships": _GraphSourceTransform( - lambda sources: UKFRSRelationshipsStageTransform( - sources["frs"], - stage=stages_by_name["frs_relationships"], - ) - ), - "frs_education": _GraphSourceTransform( - lambda sources: UKFRSEducationStageTransform( - sources["frs"], - stage=stages_by_name["frs_education"], - ) - ), - "frs_legacy_proxies": _GraphSourceTransform( - lambda sources: UKFRSLegacyProxiesStageTransform( - sources["frs"], - stage=stages_by_name["frs_legacy_proxies"], - engine=engine, - ) - ), - "frs_education_grant_split": ( - UKFRSEducationGrantSplitStageTransform( - stage=stages_by_name["frs_education_grant_split"], - engine=engine, - ) - ), - "frs_take_up": UKFRSTakeUpStageTransform( - contract=stochastic_contract, - stage=stages_by_name["frs_take_up"], - ), - "frs_person_draws": UKFRSPersonDrawsStageTransform( - contract=stochastic_contract, - stage=stages_by_name["frs_person_draws"], - ), - "frs_household_draws": UKFRSHouseholdDrawsStageTransform( - contract=stochastic_contract, - stage=stages_by_name["frs_household_draws"], - ), - "frs_brma": UKFRSBRMAStageTransform( - stage=stages_by_name["frs_brma"], - engine=engine, - ), - } - if "was_wealth" in stage_names: - implementations["was_wealth"] = _GraphSourceTransform( - lambda sources: UKWASWealthStageTransform( - stage=stages_by_name["was_wealth"], - engine=engine, - was_tab_path=sources["was"], - ) - ) - if "nts_bus_travel" in stage_names: - implementations["nts_bus_travel"] = _GraphSourceTransform( - lambda sources: UKNTSBusTravelStageTransform( - stage=stages_by_name["nts_bus_travel"], - engine=engine, - nts_household_tab_path=sources["nts_household"], - nts_individual_tab_path=sources["nts_individual"], - nts_trip_tab_path=sources["nts_trip"], - nts_stage_tab_path=sources["nts_stage"], - nts_ticket_tab_path=sources["nts_ticket"], - ) - ) - if "regional_property_uprating" in stage_names: - implementations["regional_property_uprating"] = ( - UKRegionalPropertyUpratingStageTransform( - stage=stages_by_name["regional_property_uprating"], - ) - ) - if "lcfs_consumption" in stage_names: - implementations["lcfs_consumption"] = _GraphSourceTransform( - lambda sources: UKLCFSConsumptionStageTransform( - stage=stages_by_name["lcfs_consumption"], - engine=engine, - lcfs_hh_tab_path=sources["lcfs_household"], - lcfs_person_tab_path=sources["lcfs_person"], - ) - ) - if "etb_vat" in stage_names: - implementations["etb_vat"] = _GraphSourceTransform( - lambda sources: UKETBVATStageTransform( - stage=stages_by_name["etb_vat"], - engine=engine, - etb_tab_path=sources["etb"], - ) - ) - if "etb_services" in stage_names: - implementations["etb_services"] = _GraphSourceTransform( - lambda sources: UKETBServicesStageTransform( - stage=stages_by_name["etb_services"], - engine=engine, - etb_tab_path=sources["etb"], - ) - ) - implementations["frs_hmrc_spine_leaves"] = _GraphSourceTransform( - lambda sources: UKFRSHMRCSpineLeavesStageTransform( - sources["frs"], - stage=stages_by_name["frs_hmrc_spine_leaves"], - sampled_rung=_is_sampled(args), - ) - ) - implementations["spi_support_channel"] = UKSPISupportChannelStageTransform( - stage=stages_by_name["spi_support_channel"], - sample_fraction=args.sample_fraction, - ) - if "spi_income_band_donors" in stage_names: - implementations["spi_income_band_donors"] = _GraphSourceTransform( - lambda sources: UKSPIIncomeBandDonorStageTransform( - sources["spi"], - stage=stages_by_name["spi_income_band_donors"], - sample_fraction=args.sample_fraction, - ) - ) - implementations["hmrc_spi_income_spine"] = hmrc_spine_transform - if "spi_housing_shell" in stage_names: - implementations["spi_housing_shell"] = UKSPIHousingShellStageTransform( - stage=stages_by_name["spi_housing_shell"] - ) - if "uc_reporter_redraw" in stage_names: - implementations["uc_reporter_redraw"] = UKUCReporterRedrawStageTransform( - stage=stages_by_name["uc_reporter_redraw"], - engine=engine, - ) - if "uc_capital_coherence" in stage_names: - implementations["uc_capital_coherence"] = ( - UKUCCapitalCoherenceStageTransform( - stage=stages_by_name["uc_capital_coherence"] - ) - ) - if "uc_deduction_attributes" in stage_names: - implementations["uc_deduction_attributes"] = ( - UKUCDeductionAttributesStageTransform( - stage=stages_by_name["uc_deduction_attributes"] - ) - ) - if "cgt_incidence_clone" in stage_names: - implementations["cgt_incidence_clone"] = UKCGTIncidenceCloneStageTransform( - stage=stages_by_name["cgt_incidence_clone"] - ) - if "cgt_band_donors" in stage_names: - implementations["cgt_band_donors"] = UKCGTBandDonorStageTransform( - stage=stages_by_name["cgt_band_donors"] - ) - if "hmrc_cgt_gains_spine" in stage_names: - implementations["hmrc_cgt_gains_spine"] = uk_cgt_spine_stage_transform( - stages_by_name["hmrc_cgt_gains_spine"] - ) - if "hmrc_cgt_asset_type_spine" in stage_names: - implementations["hmrc_cgt_asset_type_spine"] = ( - uk_cgt_asset_type_stage_transform( - stages_by_name["hmrc_cgt_asset_type_spine"] - ) - ) - if "cgt_incidence_anchor" in stage_names: - implementations["cgt_incidence_anchor"] = ( - UKCGTIncidenceAnchorStageTransform( - stage=stages_by_name["cgt_incidence_anchor"] - ) - ) - if "salary_sacrifice" in stage_names: - implementations["salary_sacrifice"] = UKSalarySacrificeStageTransform( - stage=stages_by_name["salary_sacrifice"] - ) - if "student_loans" in stage_names: - implementations["student_loans"] = UKStudentLoansStageTransform( - stage=stages_by_name["student_loans"], - calibration_year=frs_release.calibration_year, - ) - if "age_tail" in stage_names: - implementations["age_tail"] = UKAgeTailStageTransform( - stage=stages_by_name["age_tail"] - ) - if args.synthetic_fixture_dir is not None: - from microcosm.build.uk_runtime.graph_kernels import ( - fixture_stage_plan_inputs, - ) - - fixture_stages, fixture_implementations = fixture_stage_plan_inputs( - args.synthetic_fixture_dir - ) - fixture_stage_names = tuple(stage.stage for stage in fixture_stages) - if fixture_stage_names != tuple(stage_names): - raise ValueError( - "Synthetic fixture stage order differs from the current UK spine: " - f"fixture={fixture_stage_names!r}, current={tuple(stage_names)!r}." - ) - implementations = dict(fixture_implementations) - fixture_by_name = {stage.stage: stage for stage in fixture_stages} - implementations["frs_hmrc_spine_leaves"] = ( - UKFRSHMRCSpineLeavesStageTransform( - _synthetic_fixture_input(args.synthetic_fixture_dir, "frs_raw"), - stage=fixture_by_name["frs_hmrc_spine_leaves"], - sampled_rung=True, - ) - ) - hmrc_spine_transform = implementations["hmrc_spi_income_spine"] - sampled_root = _SampledGraphRootTransform( - implementations["frs_spine"], - fraction=args.sample_fraction, - seed=args.sample_seed, - ) - implementations["frs_spine"] = sampled_root - if telemetry is not None: - stage_observer = _staging_stage_observer(telemetry) - implementations = { - stage_id: ObservedTransform( - transform, - stage_id=stage_id, - produced_column_count=len(stages_by_name[stage_id].outputs), - observer=stage_observer, - ) - for stage_id, transform in implementations.items() - } - spine_gate_path = _spine_gate_report_path(args.spine_h5) - spine_gate_manifest = _spine_gate_manifest_from_spec(spec) - spine_battery = ( - GateBatteryRun( - spine_gate_manifest, - release_id=state.build_id, - report_path=spine_gate_path, - release_candidate=args.release_candidate, - synthetic_smoke=args.synthetic_fixture_dir is not None, - registry=UK_GATE_REGISTRY, - ) - if spine_gate_manifest is not None - else None - ) - checkpoint_root = ( - args.checkpoint_dir - if args.checkpoint_dir is not None - else args.spine_h5.parent / f".{args.spine_h5.stem}.checkpoints" - ) - if args.synthetic_fixture_dir is not None: - graph_sources = _synthetic_graph_sources(args.synthetic_fixture_dir) - else: - graph_sources = {"frs": args.frs_raw_dir} - if "was_wealth" in stage_names: - graph_sources["was"] = args.was_tab - if "nts_bus_travel" in stage_names: - graph_sources["nts_household"] = args.nts_household_tab - graph_sources["nts_individual"] = args.nts_individual_tab - graph_sources["nts_trip"] = args.nts_trip_tab - graph_sources["nts_stage"] = args.nts_stage_tab - graph_sources["nts_ticket"] = args.nts_ticket_tab - if "lcfs_consumption" in stage_names: - graph_sources["lcfs_household"] = args.lcfs_hh_tab - graph_sources["lcfs_person"] = args.lcfs_person_tab - if "etb_vat" in stage_names or "etb_services" in stage_names: - graph_sources["etb"] = args.etb_tab - if "spi_income_band_donors" in stage_names: - graph_sources["spi"] = args.spi_tab - if "hmrc_spi_income_spine" in stage_names: - graph_sources["spi"] = args.spi_tab - graph_sources["hmrc_income"] = args.hmrc_ods - graph_store = ContentStore(checkpoint_root / "node-graph") - graph_manifest = run_graph( - compiled_graph, - sources=graph_sources, - store=graph_store, - kernels=uk_registry(implementations, graph=graph), - resume="forbid", - decisions=(), - ) - final_version = compiled_graph.versions[compiled_graph.order[-1]] - frame = graph_manifest.population(final_version) - records = _graph_stage_records( - manifest=graph_manifest, - store=graph_store, - stages=stages, - frame=frame, - ) - sampling = sampled_root.sampling - if telemetry is not None: - sample = _telemetry_sample(args, sampling) - if sample is not None: - telemetry.set_sample(sample) - telemetry.stage( - "sampling", - event_status="completed", - realized_household_rows=( - len(frame.table("household")) - if sampling is None - else sampling.get( - "realized_household_rows", - sampling.get("post_household_count"), - ) - ), - ) - telemetry.stage("validation", event_status="started") - if spine_battery is not None: - if UK_SPINE_ASSEMBLED_FINAL_STAGE not in stage_names: - raise RuntimeError( - "spine battery is armed but the graph has no assembled " - f"boundary stage {UK_SPINE_ASSEMBLED_FINAL_STAGE!r}." - ) - assembled_index = stage_names.index(UK_SPINE_ASSEMBLED_FINAL_STAGE) + 1 - assembled_version = compiled_graph.versions[UK_SPINE_ASSEMBLED_FINAL_STAGE] - _run_spine_gate_phase( - spine_battery, - "assembled", - frame=graph_manifest.population(assembled_version), - stage_evidence=_collect_stage_evidence( - stage_names=stage_names[:assembled_index], - implementations=implementations, - ), - gate_artifacts=_spine_gate_artifacts(engine), - ) - if assembled_index < len(stage_names): - _run_spine_gate_phase( - spine_battery, - "transferred", - frame=frame, - stage_evidence=_collect_stage_evidence( - stage_names=stage_names, - implementations=implementations, - ), - gate_artifacts=_spine_gate_artifacts(engine), - ) - if spine_battery is not None: - append_phase(state, "spine_gates_evaluated") - if telemetry is not None: - telemetry.stage( - "validation", - event_status="completed", - entity_row_counts=_entity_row_counts(frame), - ) - append_phase(state, "spine_built") - if telemetry is not None: - telemetry.stage("spine_h5_creation", event_status="started") - output = write_uk_national_frame(frame, args.spine_h5) - if args.smoke: - _mark_non_release_h5(output, build_id=state.build_id) - if telemetry is not None: - telemetry.stage( - "spine_h5_creation", - event_status="completed", - size_bytes=output.stat().st_size, - ) - append_phase(state, "spine_written") - if args.checkpoint_dir is not None: - args.checkpoint_dir.mkdir(parents=True, exist_ok=True) - write_uk_national_frame(frame, args.checkpoint_dir / "frs_spine.h5") - append_phase(state, "checkpoint_written") - sidecar_path = output.with_suffix(".build.json") - replay_sidecar_path = output.with_suffix(".hmrc_replay.json") - if hmrc_spine_transform.last_result is None: - raise RuntimeError("HMRC SPI spine stage did not record replay evidence.") - write_hmrc_replay_report( - hmrc_spine_transform.last_result.replay_report, - replay_sidecar_path, - ) - append_phase(state, "hmrc_replay_sidecar_written") - replay_bytes = replay_sidecar_path.read_bytes() - replay_binding = { - "filename": replay_sidecar_path.name, - "report_kind": str(json.loads(replay_bytes).get("report_kind", "")), - "sha256": hashlib.sha256(replay_bytes).hexdigest(), - } - if telemetry is not None: - telemetry.stage("sidecar_creation", event_status="started") - sidecar = _build_sidecar( - frame=frame, - stages=stages, - records=records, - artifact_pins=artifact_pins, - resource_pins=resource_pins, - input_artifact_pins=input_artifact_pins, - hmrc_replay=replay_binding, - stochastic_contract_sha256=stochastic_contract.resource_sha256, - frs_vintage=frs_release.vintage, - sampling=sampling, - non_release=args.smoke, - release_posture=( - "non_release_smoke" - if args.smoke - else "release_candidate" - if args.release_candidate - else "development" - ), - synthetic_fixture=synthetic_fixture, - staging_delivery=_staging_delivery(args, telemetry), - spine_gate_report=( - { - "path": str(spine_gate_path), - "sha256": hashlib.sha256(spine_gate_path.read_bytes()).hexdigest(), - } - if spine_gate_path.is_file() - else None - ), - ) - stage_evidence = _collect_stage_evidence( - stage_names=stage_names, - implementations=implementations, - ) - if stage_evidence: - sidecar["stage_evidence"] = stage_evidence - fit_weight_records = _collect_fit_weight_records( - stage_names=stage_names, - implementations=implementations, - ) - if fit_weight_records: - sidecar["fit_weight_records"] = fit_weight_records - atomic_write_json(sidecar_path, sidecar) - if telemetry is not None: - telemetry.stage( - "sidecar_creation", - event_status="completed", - size_bytes=sidecar_path.stat().st_size, - ) - append_phase(state, "build_sidecar_written") - if args.emit_nonzero_shares is not None: - final_columns = list( - dict.fromkeys( - [column for record in records for column in record.produced] - + list(FRS_EDUCATION_GRANT_REWRITES) - ) - ) - atomic_write_json( - args.emit_nonzero_shares, - { - "stages": { - record.stage: dict(record.nonzero_share) for record in records - }, - "final": _nonzero_shares(frame, final_columns), - }, - ) - append_phase(state, "nonzero_shares_written") - if telemetry is not None: - telemetry.complete( - message=( - "Non-release smoke verification completed." - if args.smoke - else "UK spine staging run completed." - ) - ) - if args.staging_read_back: - telemetry.verify_remote() - telemetry.validate_local_bundle() - sidecar["staging_delivery"] = telemetry.delivery_summary - atomic_write_json(sidecar_path, sidecar) - state.artifact_location = local_artifact_reference( - output, - repository_hint=_REPOSITORY, - ) - state.gate_verdicts = { - "pipeline": { - "verdict": "passed", - "receipt": local_artifact_reference( - sidecar_path, repository_hint=_REPOSITORY - ), - } - } - if spine_gate_path.is_file(): - gate_payload = json.loads(spine_gate_path.read_text(encoding="utf-8")) - for gate_id, payload in gate_payload.get("gates", {}).items(): - state.gate_verdicts[str(gate_id)] = { - "verdict": str(payload.get("status")), - "receipt": ( - f"{local_artifact_reference(spine_gate_path, repository_hint=_REPOSITORY)}" - f"#/gates/{gate_id}" - ), - } - spool_path = _record_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - code_pin=code_pin, - disposition="iterating", - predecessor=predecessor, - rung=rung, - spool_dir=spool_dir, - ) - print(f"Wrote FRS spine H5: {output}", file=sys.stderr) - print(f"Wrote Logbook row: {spool_path}", file=sys.stderr) - return 0 - except Exception as error: - if telemetry is not None and telemetry.status == "running": - try: - telemetry.fail(error) - telemetry.validate_local_bundle() - except Exception: - pass - if _is_sampled(args) and _exception_chain_contains( - error, _RUNG_NAMED_EDGE_SIGNATURE - ): - rung_abort_path = args.spine_h5.with_suffix(".rung_abort.json") - receipt = _rung_abort_receipt(args, error=error) - atomic_write_json(rung_abort_path, receipt) - state.gate_verdicts = { - "uk_frs_spine_rung_abort": { - "verdict": "aborted", - "receipt": ( - f"{local_artifact_reference(rung_abort_path, repository_hint=_REPOSITORY)}" - "#/named_edge" - ), - } - } - append_phase(state, "rung_aborted") - _record_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - code_pin=code_pin, - disposition="discarded", - predecessor=predecessor, - rung=rung, - spool_dir=spool_dir, - ) - print(json.dumps(receipt, indent=2, sort_keys=True)) - return _RUNG_ABORT_EXIT_CODE - try: - receipt_path = write_error_receipt( - error_receipt_path(args.spine_h5.parent, build_id=state.build_id), - state=state, - pipeline=_PIPELINE, - error=error, - ) - apply_error_verdict( - state, - local_artifact_reference(receipt_path, repository_hint=_REPOSITORY), - ) - _record_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - code_pin=code_pin, - disposition="failed", - predecessor=predecessor, - rung=rung, - spool_dir=spool_dir, - ) - except Exception: - pass - print(f"UK FRS spine build failed: {error}", file=sys.stderr) - return 1 +"""Build the UK raw-source spine using the installed package driver.""" +from microcosm.build.uk_runtime.spine_build import main if __name__ == "__main__": raise SystemExit(main()) From 4bf7af406ff5935285e0ba844074cb1b1ec5afb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 25 Sep 2026 19:10:49 +0100 Subject: [PATCH 21/44] Register the UK full-build graph beside the current drivers: population, targets, calibration, terminal, driver The graph full build from #901 now lives on main's code: uk_runtime/graph_build (the one-graph composition over the bound spine checkpoint), graph_population (sample, normalize, K-clone expansion, ladder locations, geography mapping and gate), graph_targets (target compilation, selection, measures, the joint problem), graph_calibration (dense reference solve, size search and draw, refit, holdout), graph_terminal (export, diagnostics, candidate and certification receipts), the full_* kernels, the country adapter that validates the UK source projection, the driver full_build_cli with tools/build_uk_full.py and the microcosm-build-uk script. Nothing of main's is removed: the seam engine, the release-role driver and every tool body stay; the graph build sits beside them with dense defaults and no release role yet (that is the next commit). Main's newer API is honoured inside the new modules rather than shimmed: the chronicle feed loader and pin (#904), area_region_codes on the local target surface (#906), baseline_pi_floor threaded as a node parameter of the size-refit and holdout nodes (#921), local_authority on the ladder columns, the geography gate's engine-domain check and the export (#971), the region-level SPI contract in fixtures (#934). The co-modified modules take #901's additions only: the strict spine-checkpoint helpers in calibration_run, the draw/derive split in geography_ladder, the geographic pool in rowwise_dataset, the prepare/solve/finish split in local_rowwise with #921 and #946 re-threaded, the size draw in dataset_size, the checkpoint loader in size_checkpoint, and eager exports for the 14 new names. UK spec: the spine's sources are declared as the 14 FRS 2024-25 raw tables with the build_uk_frs_spine contract-only kernel (added to the F0 allow-list beside the retained load_uk_national_frame), target period 2025 = the release's calibration year; the resolved UK spec digest moves (main pins none). Packaging: microcosm-data in the uk extra, the build script, uv.lock relocked, APPROVED_UV_LOCK_SHA256 recomputed, pytest pythonpath for sibling-test imports. Flagged for review: _normalise_uk_local_bound_families no longer refuses an empty declaration (the country-only scope passes none); the driver refuses every existing acceptance spine at the strict checkpoint gate because this branch's gate declarations moved, so a licensed full build needs a spine built by this branch. Verified: targeted 367 + spot-check 203 passed; whole uk group 2,599 passed / 20 skipped; worker-identity lock test 24 passed; ci_test_groups --verify ok; ruff clean. Co-Authored-By: Claude Fable 5.1 --- packages/microcosm-build/pyproject.toml | 3 +- .../microcosm/build/spec_engine/resolver.py | 4 + .../src/microcosm/build/uk/spec/bundle.yaml | 7 +- .../src/microcosm/build/uk/spec/catalogs.yaml | 8 +- .../src/microcosm/build/uk/spec/sources.yaml | 107 +- .../src/microcosm/build/uk/spec/spine.yaml | 16 +- .../src/microcosm/build/uk/spec/vintages.yaml | 12 +- .../microcosm/build/uk_runtime/__init__.py | 28 + .../build/uk_runtime/calibration_run.py | 113 ++ .../build/uk_runtime/country_adapter.py | 142 +++ .../build/uk_runtime/dataset_size.py | 126 +- .../build/uk_runtime/full_build_cli.py | 849 +++++++++++++ .../build/uk_runtime/full_certification.py | 524 ++++++++ .../microcosm/build/uk_runtime/full_gates.py | 343 +++++ .../build/uk_runtime/full_measure.py | 283 +++++ .../build/uk_runtime/full_problem.py | 216 ++++ .../build/uk_runtime/full_targets.py | 186 +++ .../build/uk_runtime/geography_ladder.py | 83 +- .../microcosm/build/uk_runtime/graph_build.py | 345 +++++ .../build/uk_runtime/graph_calibration.py | 862 +++++++++++++ .../build/uk_runtime/graph_population.py | 564 +++++++++ .../build/uk_runtime/graph_targets.py | 650 ++++++++++ .../build/uk_runtime/graph_terminal.py | 1117 +++++++++++++++++ .../build/uk_runtime/local_rowwise.py | 329 +++-- .../build/uk_runtime/rowwise_dataset.py | 153 ++- .../build/uk_runtime/size_checkpoint.py | 22 +- .../build/us_runtime/worker_identity.py | 2 +- .../engine/uk/test_uk_country_adapter.py | 17 + .../engine/uk/test_uk_full_graph_admission.py | 205 +++ .../engine/uk/test_uk_full_target_graph.py | 196 +++ .../tests/engine/uk/test_uk_graph_terminal.py | 22 + .../test_spec_engine_country_bundles.py | 3 +- .../engine_free/uk/test_uk_calibration_run.py | 68 + .../engine_free/uk/test_uk_country_adapter.py | 36 + .../engine_free/uk/test_uk_full_build_cli.py | 463 +++++++ .../uk/test_uk_full_build_preparation.py | 112 ++ .../uk/test_uk_full_calibration_graph.py | 434 +++++++ .../uk/test_uk_full_certification.py | 288 +++++ .../engine_free/uk/test_uk_full_gates.py | 317 +++++ .../engine_free/uk/test_uk_full_measure.py | 274 ++++ .../uk/test_uk_full_population_graph.py | 72 ++ .../uk/test_uk_full_solve_scope.py | 82 ++ .../uk/test_uk_full_target_graph.py | 98 ++ .../engine_free/uk/test_uk_full_targets.py | 246 ++++ .../engine_free/uk/test_uk_graph_terminal.py | 678 ++++++++++ .../engine_free/uk/test_uk_local_rowwise.py | 26 +- .../microcosm_build/uk_calibration_run.py | 30 + .../uk_full_calibration_graph.py | 64 + .../uk_full_population_graph.py | 113 ++ .../microcosm_build/uk_full_target_graph.py | 314 +++++ .../microcosm_build/uk_graph_terminal.py | 60 + .../microcosm_build/uk_hierarchy_fixtures.py | 39 + .../microcosm_build/uk_local_rowwise.py | 38 + tools/build_uk_full.py | 6 + uv.lock | 2 + 55 files changed, 11193 insertions(+), 204 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/full_problem.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py create mode 100644 packages/microcosm-build/tests/engine/uk/test_uk_country_adapter.py create mode 100644 packages/microcosm-build/tests/engine/uk/test_uk_full_graph_admission.py create mode 100644 packages/microcosm-build/tests/engine/uk/test_uk_full_target_graph.py create mode 100644 packages/microcosm-build/tests/engine/uk/test_uk_graph_terminal.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_country_adapter.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_preparation.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_full_calibration_graph.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_full_certification.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_full_gates.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_full_measure.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_full_population_graph.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_full_solve_scope.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_full_target_graph.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py create mode 100644 test_support/microcosm_build/uk_full_calibration_graph.py create mode 100644 test_support/microcosm_build/uk_full_population_graph.py create mode 100644 test_support/microcosm_build/uk_full_target_graph.py create mode 100644 test_support/microcosm_build/uk_graph_terminal.py create mode 100644 test_support/microcosm_build/uk_hierarchy_fixtures.py create mode 100644 test_support/microcosm_build/uk_local_rowwise.py create mode 100644 tools/build_uk_full.py diff --git a/packages/microcosm-build/pyproject.toml b/packages/microcosm-build/pyproject.toml index 75b92d812..51d6b154e 100644 --- a/packages/microcosm-build/pyproject.toml +++ b/packages/microcosm-build/pyproject.toml @@ -40,9 +40,10 @@ us = [ # The UK extra adds the rules engine for local metric generation from a # Microcosm UK H5. Target tables remain explicit inputs, and the base package # still does not import policyengine-uk at import time. -uk = ["policyengine-uk>=2.100.0", "h5py>=3", "tables>=3", "openpyxl>=3.1"] +uk = ["policyengine-uk>=2.100.0", "microcosm-data>=0.1,<0.2", "h5py>=3", "tables>=3", "openpyxl>=3.1"] [project.scripts] +microcosm-build-uk = "microcosm.build.uk_runtime.full_build_cli:main" microcosm-export-us-l0-refit-h5 = "microcosm.build.us_runtime.l0_refit_export:main" [project.urls] diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/resolver.py b/packages/microcosm-build/src/microcosm/build/spec_engine/resolver.py index eba46efc8..9c802277b 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/resolver.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/resolver.py @@ -162,6 +162,10 @@ def contract_only_ids(self) -> frozenset[str]: "assign_am_marz", "assign_uk_geography_ladder", "be_commune_geography_gate", + # microcosm#901: the UK spec projects the raw FRS tables through the + # canonical spine build; load_uk_national_frame stays registered for + # the seam's candidate projection until the national role moves over. + "build_uk_frs_spine", "clone_assign_communities", "clone_assign_communes", "load_populace_us_support_pool", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/bundle.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/bundle.yaml index 8ce5fec5f..f6425656a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/bundle.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/bundle.yaml @@ -2,7 +2,8 @@ country: uk identity_generation: 1 seed_protocol: legacy-v1 dataset_run: - target_period: 2023 + target_period: 2025 status: >- - Compiler walking skeleton over the pinned UK national candidate. It does not - replace the generation-0 national, HMRC, calibration, or release drivers. + Raw FRS and canonical source-stage declarations consumed by the UK full-build + country adapter. The shared F0 IR remains a static schema/identity projection; + executable population and calibration stages use uk_full_graph. diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/catalogs.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/catalogs.yaml index 247da2415..0a69d09fa 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/catalogs.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/catalogs.yaml @@ -5,7 +5,7 @@ columns: dtype: int64 unit: count definition_period: eternity - vintage: vintage:uk_target_2023 + vintage: vintage:uk_frs_2024_25 nullable: false domain: frame_identity public_stability: internal @@ -15,7 +15,7 @@ columns: dtype: int64 unit: count definition_period: eternity - vintage: vintage:uk_target_2023 + vintage: vintage:uk_frs_2024_25 nullable: false domain: frame_identity public_stability: internal @@ -25,7 +25,7 @@ columns: dtype: category unit: categorical definition_period: year - vintage: vintage:uk_target_2023 + vintage: vintage:uk_frs_2024_25 nullable: false domain: observed_geography public_stability: internal @@ -35,7 +35,7 @@ columns: dtype: int64 unit: count definition_period: eternity - vintage: vintage:uk_target_2023 + vintage: vintage:uk_frs_2024_25 nullable: false domain: frame_identity public_stability: internal diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 42e8b2c45..807ceb643 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -1,15 +1,106 @@ sources: -- id: uk_national_candidate_2023 - role: uk_national_candidate - sha256: f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833 - byte_size: 1315880118 - loader: kernel:load_uk_national_frame +- id: frs_accounts + role: frs_raw_table + sha256: fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f + byte_size: 1812923 + loader: kernel:build_uk_frs_spine vintages: - - vintage:uk_candidate_2023 + - vintage:uk_frs_2024_25 +- id: frs_adult + role: frs_raw_table + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + byte_size: 34885825 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 vintage_authorities: - - id: uk_candidate_2023 + - id: uk_frs_2024_25 kind: survey_period - value: 2023 + value: 2024 +- id: frs_benefits + role: frs_raw_table + sha256: f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3 + byte_size: 2362329 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_benunit + role: frs_raw_table + sha256: 66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a + byte_size: 13986782 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_child + role: frs_raw_table + sha256: 88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5 + byte_size: 2753961 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_chldcare + role: frs_raw_table + sha256: 7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023 + byte_size: 275878 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_extchild + role: frs_raw_table + sha256: c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4 + byte_size: 15150 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_househol + role: frs_raw_table + sha256: 2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5 + byte_size: 12108606 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_job + role: frs_raw_table + sha256: eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2 + byte_size: 10518760 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_maint + role: frs_raw_table + sha256: e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d + byte_size: 13993 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_mortgage + role: frs_raw_table + sha256: 6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0 + byte_size: 600552 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_oddjob + role: frs_raw_table + sha256: dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead + byte_size: 5339 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_penprov + role: frs_raw_table + sha256: 9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1 + byte_size: 513614 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_pension + role: frs_raw_table + sha256: 2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24 + byte_size: 1232411 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 stage_manifest: version: 1 country: uk diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/spine.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/spine.yaml index 930073bda..f42eda37b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/spine.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/spine.yaml @@ -1,6 +1,20 @@ channels: - id: frs - source: uk_national_candidate_2023 + source: + - frs_accounts + - frs_adult + - frs_benefits + - frs_benunit + - frs_child + - frs_chldcare + - frs_extchild + - frs_househol + - frs_job + - frs_maint + - frs_mortgage + - frs_oddjob + - frs_penprov + - frs_pension observed_geography: region assembly: mass_anchor_channel: frs diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/vintages.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/vintages.yaml index 28d16a395..5431701d5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/vintages.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/vintages.yaml @@ -1,16 +1,16 @@ records: -- id: uk_candidate_2023 +- id: uk_frs_2024_25 kind: survey_period_ref authority_ref: kind: source_record - source: source:uk_national_candidate_2023 - authority: uk_candidate_2023 + source: source:frs_adult + authority: uk_frs_2024_25 compatible_with: - - vintage:uk_target_2023 -- id: uk_target_2023 + - vintage:uk_target_2025 +- id: uk_target_2025 kind: target_period_ref authority_ref: kind: dataset_run pointer: /dataset_run/target_period compatible_with: - - vintage:uk_candidate_2023 + - vintage:uk_frs_2024_25 diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index 0473ee95c..8cd7bae7a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -14,9 +14,12 @@ UKGateBinding, ) from microcosm.build.uk_runtime.calibration_run import ( + load_bound_spine_checkpoint, load_bound_spine_sidecar, runtime_provenance, spine_provenance_from_sidecar, + strict_spine_provenance_from_sidecar, + uk_spine_checkpoint_gate_digests, ) from microcosm.build.uk_runtime.cgt_calibration import ( UK_CGT_ANNUAL_EXEMPT_AMOUNTS, @@ -159,6 +162,8 @@ UK_OA_LADDER_SCHEMA_VERSION, UkOaLadder, assign_uk_geography_ladder, + derive_uk_ladder_locations, + draw_uk_ladder_locations, expected_uk_ladder_area_support, load_uk_oa_ladder, region_tier_by_area, @@ -323,16 +328,21 @@ UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE, UK_LOCAL_HOLDOUT_FOLDS, UK_LOCAL_HOLDOUT_SEED, + UKPreparedFullSolve, UKRowwiseDoctrineSolve, UKRowwiseLocalMatrix, UKRowwiseNationalRows, build_uk_rowwise_local_matrix, build_uk_rowwise_local_surface_matrix, + empty_uk_local_problem, + finish_uk_full_solve, past_cap_census, + prepare_uk_full_solve, require_adjudicated_uk_local_binding, rotated_uk_local_holdout, rowwise_area_support_summary, rowwise_calibration_mass_reason, + solve_uk_dense_reference, solve_uk_rowwise_weights_under_doctrine, uk_area_support_summary, uk_ladder_area_support_summary, @@ -445,16 +455,20 @@ PERSON_ID_COLUMNS, POOL_SOURCE_LINEAGE_COLUMN, UK_SINGLE_YEAR_TABLES, + UKGeographicPool, UKLadderRowwiseDatasetResult, UKRowwiseDatasetResult, apply_uk_source_lineage_modulus, + assign_uk_geographic_pool, clone_uk_dataset_tables_with_ladder_geography, clone_uk_dataset_tables_with_rowwise_geography, clone_uk_dataset_with_ladder_geography, clone_uk_dataset_with_rowwise_geography, + expand_uk_geographic_pool, ladder_clone_index_column, load_uk_rowwise_dataset, read_uk_single_year_weight_metadata, + validate_uk_geographic_pool, validate_uk_ladder_rowwise_dataset_tables, validate_uk_rowwise_dataset_tables, write_uk_rowwise_dataset, @@ -1053,4 +1067,18 @@ "resolve_local_authority_engine_keys", "verify_lad23_names_bytes", "verify_local_authority_engine_domain", + "load_bound_spine_checkpoint", + "strict_spine_provenance_from_sidecar", + "uk_spine_checkpoint_gate_digests", + "derive_uk_ladder_locations", + "draw_uk_ladder_locations", + "UKGeographicPool", + "assign_uk_geographic_pool", + "expand_uk_geographic_pool", + "validate_uk_geographic_pool", + "UKPreparedFullSolve", + "empty_uk_local_problem", + "finish_uk_full_solve", + "prepare_uk_full_solve", + "solve_uk_dense_reference", ] diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index d98ca21fc..902715b9d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -844,6 +844,119 @@ def load_bound_spine_sidecar(path: Path, frame: Frame) -> dict[str, object]: return sidecar +def load_bound_spine_checkpoint( + path: Path, + frame: Frame, + *, + gate_report_path: Path | None = None, +) -> dict[str, object]: + """Authenticate a canonical graph checkpoint, without historical bypasses.""" + from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity + + path = Path(path) + if not path.is_file(): + raise ValueError(f"input H5 build sidecar absent: {path}") + try: + sidecar = json.loads(path.read_bytes()) + except (json.JSONDecodeError, UnicodeError) as exc: + raise ValueError(f"input H5 build sidecar is invalid JSON: {path}") from exc + if not isinstance(sidecar, dict): + raise ValueError(f"input H5 build sidecar must be a JSON object: {path}") + _assert_spine_sidecar_binds_frame(sidecar, frame) + identity = sidecar.get("uk_frame_content_identity") + if not isinstance(identity, str) or not identity: + raise ValueError("Unbound spine checkpoint: no uk_frame_content_identity.") + if identity != uk_frame_content_identity(frame): + raise ValueError("Spine checkpoint uk_frame_content_identity mismatch.") + _strict_spine_gate_report(path, sidecar, gate_report_path=gate_report_path) + return sidecar + + +def _strict_spine_gate_report( + path: Path, + sidecar: Mapping[str, object], + *, + gate_report_path: Path | None = None, +) -> tuple[Path, dict[str, object]]: + if sidecar.get("spine_gate_bypass") is not None: + raise ValueError("Canonical spine checkpoints do not accept spine_gate_bypass.") + report_path = ( + _spine_gate_report_path(Path(path)) + if gate_report_path is None + else Path(gate_report_path) + ) + binding = sidecar.get("spine_gate_report") + if not isinstance(binding, Mapping) or not binding.get("sha256"): + raise ValueError("Unbound spine checkpoint: no spine gate report SHA-256.") + if not report_path.is_file(): + raise ValueError(f"input H5 spine gate report absent: {report_path}") + report_bytes = report_path.read_bytes() + if hashlib.sha256(report_bytes).hexdigest() != binding["sha256"]: + raise ValueError("Spine checkpoint gate report SHA-256 mismatch.") + _assert_spine_gate_report_passed(report_path, sidecar) + report = json.loads(report_bytes) + for field, expected in uk_spine_checkpoint_gate_digests().items(): + if report.get(field) != expected: + raise ValueError( + f"Spine checkpoint gate report {field} differs from current declarations." + ) + gates = report["gates"] + expected = set(UK_SPINE_GATE_SCOPE) + if set(gates) != expected or any( + not isinstance(gates[gate_id], Mapping) for gate_id in gates + ): + raise ValueError( + "Spine checkpoint gate report differs from the declared spine scope." + ) + declared = { + entry.id: entry + for entry in load_country_spec("uk").gates.gates + if entry.id in expected + } + for gate_id, entry in declared.items(): + outcome = gates[gate_id] + if outcome.get("criticality") != entry.criticality: + raise ValueError(f"Spine checkpoint gate {gate_id} criticality mismatch.") + if ( + entry.criticality == "release_blocking" + and outcome.get("status") != "passed" + ): + raise ValueError(f"Spine checkpoint gate {gate_id} did not pass.") + return report_path, report + + +def uk_spine_checkpoint_gate_digests() -> dict[str, str]: + """Declare the current spine gate policy as part of checkpoint identity.""" + from microcosm.build.uk_runtime.release_certification import _scoped_digests + + return _scoped_digests( + frozenset(UK_SPINE_GATE_SCOPE), + phases=("assembled", "transferred"), + policy_suffix="spine_build_scope", + ) + + +def strict_spine_provenance_from_sidecar( + path: Path, + sidecar: Mapping[str, object], + *, + gate_report_path: Path | None = None, +) -> dict[str, object]: + """Retain exact gate bytes and fit records after strict checkpoint loading.""" + report_path, report = _strict_spine_gate_report( + path, sidecar, gate_report_path=gate_report_path + ) + provenance = spine_provenance_from_sidecar(path, sidecar) + provenance["uk_frame_content_identity"] = sidecar["uk_frame_content_identity"] + provenance["fit_weight_records"] = dict(sidecar.get("fit_weight_records", {})) + provenance["spine_gate_report"] = { + "path": str(report_path), + "sha256": sidecar["spine_gate_report"]["sha256"], + "payload": report, + } + return provenance + + def _assert_spine_sidecar_binds_frame( sidecar: Mapping[str, object], frame: Frame, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py new file mode 100644 index 000000000..9a41de5e1 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py @@ -0,0 +1,142 @@ +"""Bridge the UK country declarations to the canonical executable full graph. + +The F0 compiler remains a static schema/identity projection. This adapter +validates its source projection, then uses the existing UK graph composition; +it never loads a historical candidate H5 or creates a second execution graph. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import replace +from datetime import date +from importlib import metadata +from typing import TYPE_CHECKING + +from microcosm.build.country_spec import CountrySpec, load_country_spec +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release + +if TYPE_CHECKING: + from .graph_build import UKFullBuildConfig, UKFullGraph + + +def validate_uk_country_source_projection(spec: CountrySpec) -> None: + """Keep raw input declarations and their executable stage pins identical.""" + if spec.country != "uk" or spec.resolved_spec is None: + raise ValueError( + "The UK full-build adapter requires a resolved UK country spec." + ) + resolved = spec.resolved_spec + sources = resolved.resource("sources").domain.to_wire() + spine = resolved.resource("spine").domain.to_wire() + bundle = resolved.resource("bundle").domain.to_wire() + release = load_uk_frs_release() + root = next(stage for stage in spec.sources.stages if stage.stage == "frs_spine") + vintage = f"uk_frs_{release.vintage}" + expected = [ + { + "id": f"frs_{artifact['table']}", + "role": "frs_raw_table", + "sha256": artifact["sha256"], + "byte_size": artifact["size_bytes"], + "loader": "kernel:build_uk_frs_spine", + "vintages": [f"vintage:{vintage}"], + **( + { + "vintage_authorities": [ + { + "id": vintage, + "kind": "survey_period", + "value": release.survey_year, + } + ] + } + if artifact["table"] == "adult" + else {} + ), + } + for artifact in root.artifacts + if artifact["role"] == "frs_table" + ] + if sources["sources"] != expected: + raise ValueError( + "UK country raw-source pins differ from the canonical FRS spine stage." + ) + if spine["channels"] != [ + { + "id": "frs", + "source": [row["id"] for row in expected], + "observed_geography": "region", + } + ]: + raise ValueError("UK FRS channel must consume the declared raw FRS tables.") + if bundle["dataset_run"]["target_period"] != release.calibration_year: + raise ValueError( + "UK country target period differs from the FRS release calibration year." + ) + + +def build_uk_country_graph( + config: UKFullBuildConfig | None = None, + *, + spec: CountrySpec | None = None, + engine_identity: str | None = None, + review_date: date | None = None, + release_candidate: bool = False, + skip_holdout: bool = False, +) -> UKFullGraph: + """Compile the canonical full graph; default target scope is all geographies.""" + from microcosm.graph import compile_graph + from microcosm.graph.canonical import canonical_json + + from .graph import UK_SPINE_EXCLUSIONS, uk_spine_graph + from .graph_build import UKFullBuildConfig, uk_full_graph + from .graph_evidence import add_uk_spine_gate_nodes + from .graph_terminal import append_uk_full_gate_nodes + + spec = load_country_spec("uk") if spec is None else spec + validate_uk_country_source_projection(spec) + release = load_uk_frs_release() + if config is None: + config = UKFullBuildConfig( + calibration_year=release.calibration_year, + time_period=release.time_period, + source_year=release.survey_year, + ) + if engine_identity is None: + engine_identity = hashlib.sha256( + canonical_json( + { + "package": "policyengine-uk", + "version": metadata.version("policyengine-uk"), + } + ) + ).hexdigest() + review_date = date.today() if review_date is None else review_date + spine = add_uk_spine_gate_nodes( + uk_spine_graph(spec, source_mode="split"), + spec=spec, + engine_identity=engine_identity, + release_candidate=release_candidate, + ) + full = uk_full_graph(config, spine=spine, review_date=review_date.isoformat()) + graph = append_uk_full_gate_nodes( + full.graph, + calibration=full.calibration, + # The same roster uk_spine_graph builds: main still declares the + # retired HMRC pair in the manifest and keeps it out of the spine + # through UK_SPINE_EXCLUSIONS, so the preflight binds evidence only + # from stages the spine graph actually runs. + spine_stage_names=tuple( + stage.stage + for stage in spec.sources.stages + if stage.stage not in UK_SPINE_EXCLUSIONS + ), + engine_identity=engine_identity, + review_date=review_date, + sample_fraction=config.effective_sample_fraction, + release_candidate=release_candidate, + skip_holdout=skip_holdout, + ) + compile_graph(graph) + return replace(full, graph=graph) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/dataset_size.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/dataset_size.py index 6d39f70a8..d0e72b863 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/dataset_size.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/dataset_size.py @@ -1,5 +1,6 @@ """Exact household-count UK candidates on a fixed, materialized target surface.""" +import hashlib import json from collections.abc import Callable from dataclasses import dataclass, replace @@ -53,6 +54,80 @@ class UKSizeSelection: search_pi_hi: float +@dataclass(frozen=True) +class UKSizeDraw: + """An executed exact-count draw, independently reusable by the refit.""" + + support: np.ndarray + sampling: dict[str, object] + inclusion_probabilities: np.ndarray + feasibility: dict[str, object] + seed: int + pi_hi: float + probabilities_sha256: str + + +def _probability_digest(probabilities: np.ndarray) -> str: + return hashlib.sha256(np.asarray(probabilities, dtype=" UKSizeDraw: + """Execute only the existing exact-count draw, without search or refit.""" + n = _check_size_inputs(frame, dense, households) + pi_hi = _check_pi_hi(pi_hi) + probabilities = selection.selection.gate_open_probabilities + if ( + selection.households != households + or selection.seed != seed + or probabilities is None + or len(probabilities) != n + or len(selection.protected) != n + ): + raise ValueError("Exact-count draw requires its aligned selection and seed.") + search_result = selection.selection + feasibility = selection_feasibility( + probabilities, + households, + protected=selection.protected, + n_nonzero=int(search_result.n_nonzero), + l0_lambda=float(search_result.l0_lambda), + requested_pi_hi=pi_hi, + budget_search=search_result.options.get("budget_search"), + search_pi_hi=selection.search_pi_hi, + ) + try: + support, sampling, q = select_exact_k( + probabilities, households, pi_hi=pi_hi, seed=seed + ) + except ValueError as error: + # The draw refuses rather than clamps; carry the measured gate mass + # with the refusal so the ruling it needs can be made from the receipt. + raise ValueError( + f"{error} Selection feasibility (requested pi_hi={pi_hi:g}): " + f"{json.dumps(feasibility, sort_keys=True)}" + ) from error + support = assert_exact_k_support(support, households, pool_size=n) + if not np.isin(np.flatnonzero(selection.protected), support).all(): + raise RuntimeError("exact-count selection lost a protected carrier.") + return UKSizeDraw( + support, + sampling, + q, + feasibility, + seed, + pi_hi, + _probability_digest(probabilities), + ) + + ProgressCallback = Callable[[dict[str, object]], None] @@ -214,6 +289,7 @@ def refit_uk_dataset_size( pi_hi: float = 1.0, baseline_pi_floor: float = 0.0, selection: UKSizeSelection | None = None, + draw: UKSizeDraw | None = None, progress_callback: ProgressCallback | None = None, ) -> UKDatasetSize: """Run informed L0, a fixed-size draw, and refit under the dense doctrine. @@ -246,6 +322,8 @@ def refit_uk_dataset_size( :class:`UKSizeSelection` (a checkpoint restored by :mod:`microcosm.build.uk_runtime.size_checkpoint`); it must have been searched for the same size, epochs, learning rate and seed on this pool. + ``draw`` additionally reuses an authenticated completed draw without + consuming its random stream again; the probability binding must match. """ n = _check_size_inputs(frame, dense, households) pi_hi = _check_pi_hi(pi_hi) @@ -310,28 +388,34 @@ def refit_uk_dataset_size( probabilities = selection.selection.gate_open_probabilities assert probabilities is not None search_result = selection.selection - feasibility = selection_feasibility( - probabilities, - households, - protected=init_protected, - n_nonzero=int(search_result.n_nonzero), - l0_lambda=float(search_result.l0_lambda), - requested_pi_hi=pi_hi, - budget_search=search_result.options.get("budget_search"), - search_pi_hi=selection.search_pi_hi, - ) - try: - support, sampling, q = select_exact_k( - probabilities, households, pi_hi=pi_hi, seed=seed + if draw is None: + draw = draw_uk_dataset_size( + frame, + dense, + selection=selection, + households=households, + seed=seed, + pi_hi=pi_hi, ) - except ValueError as error: - # The draw refuses rather than clamps; carry the measured gate mass - # with the refusal so the ruling it needs can be made from the receipt. - raise ValueError( - f"{error} Selection feasibility (requested pi_hi={pi_hi:g}): " - f"{json.dumps(feasibility, sort_keys=True)}" - ) from error - support = assert_exact_k_support(support, households, pool_size=n) + if ( + draw.seed != seed + or draw.pi_hi != pi_hi + or draw.probabilities_sha256 != _probability_digest(probabilities) + ): + raise ValueError("Reused exact-count draw differs from its selection or seed.") + support = assert_exact_k_support(draw.support, households, pool_size=n) + sampling, q, feasibility = ( + draw.sampling, + draw.inclusion_probabilities, + draw.feasibility, + ) + if ( + np.asarray(q).shape != (households,) + or not np.isfinite(q).all() + or (q <= 0).any() + or (q > 1).any() + ): + raise ValueError("Reused exact-count draw has invalid inclusion probabilities.") if not np.isin(np.flatnonzero(init_protected), support).all(): raise RuntimeError("exact-count selection lost a protected carrier.") frozen = _frozen_targets(frame, dense, support) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py new file mode 100644 index 000000000..a5f210b19 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py @@ -0,0 +1,849 @@ +"""Execute the single UK full build; all geographies are selected by default. + +Numerical operations and verdicts belong to the composed graph. This module +resolves requests, executes graph endpoints and atomically materializes their +stored artifacts. Publication and signing remain explicit external services. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import tempfile +import uuid +from dataclasses import asdict, dataclass, replace +from datetime import date +from pathlib import Path + +from microcosm.build.artifact_files import file_artifact, materialize_bytes +from microcosm.graph import ( + ArtifactInput, + ContentStore, + Graph, + KernelRegistry, + SourceRef, + compile_graph, + graph_to_json, + run_graph, +) +from microcosm.graph.canonical import canonical_json + +from .chronicle_feed import load_uk_chronicle_feed +from .frs_release import load_uk_frs_release +from .full_certification import ( + append_uk_full_certification_node, + register_uk_full_certification_kernel, +) +from .graph_build import ( + SPINE_PROVENANCE_TYPE, + UKFullBuildConfig, + UKFullGraph, + bound_spine_graph, + register_uk_full_kernels, + uk_full_graph, +) +from .graph_calibration import UKGraphCalibrationConfig +from .graph_targets import TARGET_SELECTION_TYPE +from .graph_terminal import ( + FULL_DIAGNOSTICS_CSV_TYPE, + FULL_DIAGNOSTICS_TYPE, + FULL_GATE_REPORT_TYPE, + FULL_HOLDOUT_TYPE, + FULL_SUPPORT_CSV_TYPE, + add_uk_export_continuation, + add_uk_export_preparation, + append_uk_full_gate_nodes, + decode_full_gate_report, + materialize_uk_export, + materialize_uk_terminal_artifacts, + register_uk_full_gate_kernels, + register_uk_terminal_kernels, +) +from .local_doctrine import ( + UK_LOCAL_CLONE_COUNT, + UK_LOCAL_MAX_WEIGHT_RATIO, + UK_LOCAL_SOLVE_DOCTRINE, + UK_LOCAL_SOLVE_EPOCHS, + UK_LOCAL_TARGET_LOSS_CAP, +) +from .national_frame import load_uk_national_frame +from .national_sampling import UK_SAMPLE_SEED_DEFAULT + + +def _target_geographies(value: str) -> tuple[str, ...] | None: + if value == "all": + return None + levels = tuple(value.split(",")) + if ( + not levels + or len(levels) != len(set(levels)) + or set(levels) - {"country", "region", "constituency", "la"} + ): + raise argparse.ArgumentTypeError( + "Use all or a comma-separated subset of country,region,constituency,la." + ) + return levels + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + population = parser.add_mutually_exclusive_group(required=True) + population.add_argument( + "--input-h5", + type=Path, + help="Canonical spine checkpoint with bound build and gate sidecars.", + ) + population.add_argument( + "--spine-request", + type=Path, + help="JSON array of raw FRS spine arguments; these stages execute in the same graph.", + ) + parser.add_argument("--input-sidecar", type=Path) + parser.add_argument("--input-spine-gates", type=Path) + parser.add_argument("--input-sha256") + parser.add_argument("--ladder", type=Path, required=True) + parser.add_argument("--ladder-sha256") + parser.add_argument( + "--ledger-facts", + type=Path, + required=True, + help="Complete Chronicle artifact directory matching the committed feed pins.", + ) + parser.add_argument("--ledger-facts-sha256") + parser.add_argument("--ledger-manifest-sha256") + parser.add_argument("--measure-exclusions", type=Path) + parser.add_argument("--register-json", type=Path) + parser.add_argument("--input-mass-reference", type=Path) + parser.add_argument( + "--native-scorecard", + type=Path, + help="Measured incumbent-surface comparison bound to immutable candidate.json and its exact output bytes.", + ) + parser.add_argument( + "--matched-size-scorecard", + type=Path, + help="Additional measured comparison with both populations at requested k.", + ) + parser.add_argument( + "--target-geographies", + type=_target_geographies, + default=None, + metavar="all|country,...", + help="Default all: calibrate all applicable geographies together. country is an explicit filter in this same build.", + ) + parser.add_argument( + "--n-clones", + type=int, + default=UK_LOCAL_CLONE_COUNT, + help="Geographic pool copies K, independent of target scope and exported size k.", + ) + parser.add_argument( + "--dataset-households", + type=int, + help="Exact exported household count k via informed L0, draw and refit.", + ) + parser.add_argument( + "--sample-fraction", + type=float, + default=1.0, + help="Optional pool sampling before cloning. Cannot resample an already sampled source spine.", + ) + parser.add_argument("--sample-seed", type=int, default=UK_SAMPLE_SEED_DEFAULT) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--selection-seed", type=int) + parser.add_argument("--selection-pi-hi", type=float, default=1.0) + parser.add_argument("--epochs", type=int, default=UK_LOCAL_SOLVE_EPOCHS) + parser.add_argument("--learning-rate", type=float, default=0.15) + parser.add_argument( + "--target-weight-rule", + choices=("uniform", "grain_equal"), + default=UK_LOCAL_SOLVE_DOCTRINE.target_weight_rule, + ) + parser.add_argument("--engine-blocks", type=int, default=1) + parser.add_argument("--source-year", type=int) + parser.add_argument("--calibration-year", type=int) + parser.add_argument("--source-lineage-modulus", type=int) + parser.add_argument("--expected-constituency-vintage", default="2024_pcon") + parser.add_argument("--skip-holdout", action="store_true") + parser.add_argument("--release-candidate", action="store_true") + parser.add_argument("--review-date", type=date.fromisoformat, default=date.today()) + parser.add_argument( + "--resume-size-checkpoint", + type=Path, + help="Import an identity-verified historical size search, skipping its dense solve and search.", + ) + parser.add_argument( + "--graph-store", + type=Path, + help="Persistent shared graph store; default /.graph-store.", + ) + parser.add_argument( + "--resume", choices=("auto", "require", "forbid"), default="auto" + ) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument( + "--dry-run", + action="store_true", + help="Validate the request and print the compiled operation inventory without fitting or writing files.", + ) + args = parser.parse_args(argv) + if args.input_h5 is None and any( + (args.input_sidecar, args.input_spine_gates, args.input_sha256) + ): + parser.error("Input H5 sidecar/pin options require --input-h5.") + if args.dataset_households is None and ( + args.selection_seed is not None + or args.selection_pi_hi != 1.0 + or args.resume_size_checkpoint + ): + parser.error("Selection options require --dataset-households.") + if args.matched_size_scorecard is not None and args.dataset_households is None: + parser.error("A matched-size scorecard requires --dataset-households.") + if args.release_candidate and args.skip_holdout: + parser.error("A release candidate must evaluate applicable holdouts.") + if args.release_candidate: + if args.dataset_households is not None: + parser.error( + "Exact-count builds need separate matched-size evidence before release promotion." + ) + if ( + args.epochs != UK_LOCAL_SOLVE_EPOCHS + or args.n_clones != UK_LOCAL_CLONE_COUNT + or args.target_weight_rule != UK_LOCAL_SOLVE_DOCTRINE.target_weight_rule + or args.engine_blocks != 1 + or args.measure_exclusions is not None + ): + parser.error( + "A release candidate must use the maintained solve doctrine, pool count, single engine and reviewed exclusions." + ) + if args.ladder_sha256 is None or ( + args.input_h5 is not None and args.input_sha256 is None + ): + parser.error( + "A release candidate requires explicit input H5 and ladder digest pins." + ) + if args.resume_size_checkpoint and args.input_h5 is None: + parser.error( + "Historical size checkpoints bind an input H5; raw builds resume using --graph-store." + ) + return args + + +def _pin(path: Path, expected: str | None = None) -> dict: + record = file_artifact(path) + if expected is not None and expected != record["sha256"]: + raise ValueError(f"Input digest differs from the requested pin: {path}.") + return {"sha256": record["sha256"], "size_bytes": record["size_bytes"]} + + +def _checkpoint_identity(args, config, pins) -> dict | None: + if args.resume_size_checkpoint is None: + return None + # Match the existing checkpoint schema exactly. The import kernel also + # verifies ordered targets, weights, household axis and recomputed losses. + return { + "dataset_pin": pins["dataset"], + "ladder_pin": pins["ladder"], + "ledger_facts_sha256": args.ledger_facts_sha256, + "ledger_manifest_sha256": args.ledger_manifest_sha256, + "seed": args.seed, + "selection_seed": args.seed + if args.selection_seed is None + else args.selection_seed, + "n_clones": args.n_clones, + "dataset_households": args.dataset_households, + "epochs": args.epochs, + "learning_rate": args.learning_rate, + "sample_fraction": args.sample_fraction, + "sample_seed": args.sample_seed, + "source_year": config.source_year, + "source_lineage_modulus": args.source_lineage_modulus, + "calibration_year": config.calibration_year, + "target_weight_rule": args.target_weight_rule, + "engine_blocks": args.engine_blocks, + "measure_exclusions": None + if args.measure_exclusions is None + else str(args.measure_exclusions), + "doctrine": { + "target_loss_cap": float(UK_LOCAL_TARGET_LOSS_CAP), + "max_weight_ratio": float(UK_LOCAL_MAX_WEIGHT_RATIO), + "scale_rule": UK_LOCAL_SOLVE_DOCTRINE.scale_rule, + "target_weight_rule": UK_LOCAL_SOLVE_DOCTRINE.target_weight_rule, + "solve_epochs": int(UK_LOCAL_SOLVE_EPOCHS), + "clone_count": int(UK_LOCAL_CLONE_COUNT), + }, + } + + +@dataclass(frozen=True) +class PreparedUKFullBuild: + full: UKFullGraph + kernels: KernelRegistry + sources: dict[str, Path] + bindings: dict + spine_provenance: ArtifactInput | None = None + comparison_sources: dict[str, Path] | None = None + + +def prepare_full_build(args: argparse.Namespace) -> PreparedUKFullBuild: + from .calibration_run import load_bound_spine_checkpoint + from .graph import uk_spine_endpoint + from .spine_build import ( + _rules_engine, + _rules_engine_provenance, + parse_uk_spine_args, + prepare_uk_spine_execution, + ) + + release = load_uk_frs_release() + pins = {"ladder": _pin(args.ladder, args.ladder_sha256)} + sources = {"uk_ladder": args.ladder, "uk_ledger_facts": args.ledger_facts} + provenance = None + if args.input_h5 is not None: + pins["dataset"] = _pin(args.input_h5, args.input_sha256) + frame, _ = load_uk_national_frame(args.input_h5) + sidecar_path = args.input_sidecar or args.input_h5.with_suffix(".build.json") + gates_path = args.input_spine_gates or args.input_h5.with_suffix( + ".spine_gates.json" + ) + sidecar = load_bound_spine_checkpoint( + sidecar_path, frame, gate_report_path=gates_path + ) + spine = bound_spine_graph(frame) + endpoint = "uk.full.spine_checkpoint" + weight_kind = frame.weights_for("household").kind.value + time_period = str(frame.metadata["time_period"]) + source_fraction = float((sidecar.get("sampling") or {}).get("fraction", 1.0)) + stages = tuple(sidecar["stages"]) + engine = _rules_engine() + engine_identity = hashlib.sha256( + canonical_json(_rules_engine_provenance()) + ).hexdigest() + kernels = KernelRegistry() + sources.update( + uk_spine=args.input_h5, + uk_spine_evidence=sidecar_path, + uk_spine_gates=gates_path, + ) + provenance = ArtifactInput( + "spine_provenance", endpoint, "spine_provenance", SPINE_PROVENANCE_TYPE + ) + else: + raw_arguments = json.loads(args.spine_request.read_text()) + if not isinstance(raw_arguments, list) or not all( + isinstance(x, str) for x in raw_arguments + ): + raise ValueError( + "The spine request must be a JSON array of command arguments." + ) + # --spine-h5 is an output control of the checkpoint command. Preparation + # uses it only for path configuration; the full graph writes its own H5. + if "--spine-h5" not in raw_arguments and not any( + x.startswith("--spine-h5=") for x in raw_arguments + ): + raw_arguments += ["--spine-h5", str(args.out / "spine.h5")] + raw = parse_uk_spine_args(raw_arguments) + if args.release_candidate: + raw.release_candidate = True + prepared = prepare_uk_spine_execution(raw) + spine, kernels = prepared.graph, prepared.kernels + sources.update(prepared.sources) + endpoint = uk_spine_endpoint(spine).population + weight_kind = "importance" + time_period = prepared.frs_release.time_period + source_fraction = raw.sample_fraction + stages = prepared.stage_names + engine, engine_identity = prepared.engine, prepared.engine_identity + config = UKFullBuildConfig( + calibration_year=args.calibration_year or release.calibration_year, + time_period=time_period, + source_year=args.source_year + if args.source_year is not None + else int(time_period), + geography_levels=args.target_geographies, + n_clones=args.n_clones, + sample_fraction=args.sample_fraction, + source_sample_fraction=source_fraction, + sample_seed=args.sample_seed, + seed=args.seed, + engine_blocks=args.engine_blocks, + constituency_vintage=args.expected_constituency_vintage, + source_lineage_modulus=args.source_lineage_modulus, + calibration=UKGraphCalibrationConfig( + epochs=args.epochs, + learning_rate=args.learning_rate, + seed=args.seed, + dataset_households=args.dataset_households, + selection_seed=args.selection_seed, + selection_pi_hi=args.selection_pi_hi, + target_weight_rule=args.target_weight_rule, + ), + ) + if args.release_candidate and config.effective_sample_fraction != 1.0: + raise ValueError( + "Sampled builds cannot request release-candidate certification." + ) + feed = load_uk_chronicle_feed() + for supplied, committed in ( + (args.ledger_facts_sha256, feed.facts_sha256), + (args.ledger_manifest_sha256, feed.manifest_sha256), + ): + if supplied is not None and supplied != committed: + raise ValueError( + "Requested Chronicle pin differs from the committed full-build feed." + ) + optional = [] + for name, path in ( + ("uk_measure_exclusions", args.measure_exclusions), + ("uk_frozen_register", args.register_json), + ): + if path is not None: + sources[name] = path + optional.append(name) + if args.input_mass_reference is not None: + sources["uk_input_mass_reference"] = args.input_mass_reference + spine = replace( + spine, + sources=( + *spine.sources, + SourceRef("uk_input_mass_reference", "raw-bytes-v1"), + ), + ) + full = uk_full_graph( + config, + spine=spine, + spine_population=endpoint, + spine_weight_kind=weight_kind, + optional_target_sources=tuple(optional), + review_date=args.review_date.isoformat(), + checkpoint_identity=_checkpoint_identity(args, config, pins), + ) + if args.resume_size_checkpoint: + from .size_checkpoint import ( + SIZE_CHECKPOINT_ARRAYS_FILENAME, + SIZE_CHECKPOINT_MANIFEST_FILENAME, + ) + + sources["uk_size_checkpoint_manifest"] = ( + args.resume_size_checkpoint / SIZE_CHECKPOINT_MANIFEST_FILENAME + ) + sources["uk_size_checkpoint_arrays"] = ( + args.resume_size_checkpoint / SIZE_CHECKPOINT_ARRAYS_FILENAME + ) + graph = append_uk_full_gate_nodes( + full.graph, + calibration=full.calibration, + spine_stage_names=stages, + engine_identity=engine_identity, + review_date=args.review_date, + sample_fraction=config.effective_sample_fraction, + release_candidate=args.release_candidate, + spine_provenance=provenance, + skip_holdout=args.skip_holdout, + ) + bindings = { + "schema": "microcosm.uk.full-build-request.v1", + "configuration": asdict(config), + "target_scope": "all" + if config.geography_levels is None + else list(config.geography_levels), + "review_date": args.review_date.isoformat(), + "engine_identity": engine_identity, + "release_candidate": args.release_candidate, + "skip_holdout": args.skip_holdout, + "targets": { + "chronicle": { + "facts_sha256": feed.facts_sha256, + "manifest_sha256": feed.manifest_sha256, + }, + "paired_ladder_sha256": pins["ladder"]["sha256"], + }, + } + graph = add_uk_export_preparation( + graph, + population=full.population, + bindings=bindings, + artifact_inputs=( + ArtifactInput( + "full_gates", + "uk.full.gates.calibrated", + "gate_report", + FULL_GATE_REPORT_TYPE, + ), + ), + ) + register_uk_full_kernels(kernels) + register_uk_full_gate_kernels( + kernels, coverage_engine=engine, engine_identity=engine_identity + ) + register_uk_terminal_kernels(kernels) + register_uk_full_certification_kernel(kernels) + full = replace(full, graph=graph) + compile_graph(graph) + comparisons = { + name: path + for name, path in ( + ("uk_native_scorecard", args.native_scorecard), + ("uk_matched_size_scorecard", args.matched_size_scorecard), + ) + if path is not None + } + return PreparedUKFullBuild( + full, kernels, sources, bindings, provenance, comparisons + ) + + +def _through(graph: Graph, endpoint: str) -> Graph: + """Execute an actual ancestor-closed checkpoint of the one declared graph.""" + compiled = compile_graph(graph) + needed, pending = {endpoint}, [endpoint] + while pending: + for parent in compiled.predecessors[pending.pop()]: + if parent not in needed: + needed.add(parent) + pending.append(parent) + return replace( + graph, nodes=tuple(node for node in graph.nodes if node.id in needed) + ) + + +def _payload(manifest, store, node: str, artifact: str) -> bytes: + return store.load_bytes(manifest.nodes[node].opaque_artifacts[artifact]) + + +def _materialize_evidence(manifest, store, out: Path) -> dict: + inventory = {} + for node_id, receipt in manifest.nodes.items(): + for name, key in receipt.opaque_artifacts.items(): + payload = store.load_bytes(key) + suffix = ".json" if payload.startswith((b"{", b"[")) else ".artifact" + filename = f"{node_id}.{name}{suffix}" + if Path(filename).name != filename: + raise ValueError( + "Graph artifact name cannot be materialized as a bundle filename." + ) + inventory[f"{node_id}/{name}"] = { + "key": key, + **materialize_bytes(payload, out / filename), + } + materialize_bytes(canonical_json(inventory), out / "evidence-index.json") + return inventory + + +def _persist_checkpoint(manifest, store, args, phase: str) -> None: + """Keep receipts and small evidence durable if a later kernel refuses.""" + payload = manifest.to_json_bytes() + materialize_bytes(payload, args.out / f"{phase}.graph.json") + attempt = Path(args.attempt_evidence) + materialize_bytes(payload, attempt / f"{phase}.graph.json") + evidence = {} + for node_id, receipt in manifest.nodes.items(): + for name, key in receipt.opaque_artifacts.items(): + if name not in {"gate_report", "stage_evidence", "spine_provenance"}: + continue + filename = f"{node_id}.{name}.json" + if Path(filename).name != filename: + raise ValueError("Checkpoint artifact names must be simple filenames.") + evidence[f"{node_id}/{name}"] = { + "key": key, + **materialize_bytes(store.load_bytes(key), attempt / filename), + } + materialize_bytes( + canonical_json({"phase": phase, "artifacts": evidence}), + attempt / "evidence-index.json", + ) + + +def _output_locations(prepared: PreparedUKFullBuild, args: argparse.Namespace): + output = args.out.resolve() + graph_store = (args.graph_store or output / ".graph-store").resolve() + for source in {**prepared.sources, **(prepared.comparison_sources or {})}.values(): + source = source.resolve() + if source.is_relative_to(output): + raise ValueError( + "Full-build output directory must not contain an input source." + ) + if source.is_dir() and ( + output.is_relative_to(source) or graph_store.is_relative_to(source) + ): + raise ValueError( + "Output files and graph store cannot alter a declared source directory." + ) + return output, graph_store + + +def execute_full_build(prepared: PreparedUKFullBuild, args: argparse.Namespace) -> int: + """Stage complete files, then publish their completion marker last.""" + if args.dry_run: + return _execute_full_build(prepared, args) + from microcosm.build.artifact_files import publish_staged_bundle + + output, graph_store = _output_locations(prepared, args) + output.parent.mkdir(parents=True, exist_ok=True) + # Operational attempt identity never enters a scientific node/cache key. + args.attempt_evidence = graph_store / "uk-full-attempts" / uuid.uuid4().hex + with tempfile.TemporaryDirectory( + prefix=f".{output.name}.full-build-", dir=output.parent + ) as temporary: + staged_args = argparse.Namespace(**vars(args)) + staged_args.out = Path(temporary) + staged_args.graph_store = graph_store + status = _execute_full_build(prepared, staged_args) + if not (staged_args.out / "build.json").exists(): + materialize_bytes( + canonical_json( + { + "schema_version": 1, + "kind": "uk_full_build_refused", + "request": prepared.bindings, + "release_authorized": False, + "artifact_permitted": False, + } + ), + staged_args.out / "build.json", + ) + staged = { + ("manifest" if path.name == "build.json" else path.name): path + for path in staged_args.out.iterdir() + if path.is_file() + } + destinations = {role: output / path.name for role, path in staged.items()} + publish_staged_bundle(staged, destinations, completion_role="manifest") + if status == 0: + print( + f"UK full build: {output / f'microcosm_uk_{prepared.full.config.calibration_year}.h5'}; " + f"target scope {prepared.bindings['target_scope']}." + ) + return status + + +def _execute_full_build(prepared: PreparedUKFullBuild, args: argparse.Namespace) -> int: + full, kernels, sources = prepared.full, prepared.kernels, prepared.sources + if args.dry_run: + print(json.dumps(full.operation_inventory(), indent=2)) + return 0 + args.out.mkdir(parents=True, exist_ok=True) + store = ContentStore(args.graph_store or args.out / ".graph-store") + graph = full.graph + materialize_bytes(graph_to_json(graph).encode(), args.out / "graph.json") + materialize_bytes( + canonical_json(full.operation_inventory()), args.out / "operations.json" + ) + materialize_bytes( + canonical_json(prepared.bindings), args.attempt_evidence / "request.json" + ) + # Preserve source evidence before downstream transforms can raise. Each is + # an ancestor-closed checkpoint of this graph, with the same node keys/RNG. + resume = args.resume + node_ids = {node.id for node in graph.nodes} + for endpoint in ( + "spine.gates.assembled", + "spine.gates.transferred", + "uk.full.spine_checkpoint", + ): + if endpoint not in node_ids: + continue + checkpoint = run_graph( + compile_graph(_through(graph, endpoint)), + sources=sources, + store=store, + kernels=kernels, + resume=resume, + ) + _persist_checkpoint(checkpoint, store, args, endpoint) + resume = "require" if args.resume == "require" else "auto" + # Persist preflight outcomes before any solver can reject them. + preflight_graph = _through(graph, "uk.full.gates.preflight") + preflight = run_graph( + compile_graph(preflight_graph), + sources=sources, + store=store, + kernels=kernels, + resume=resume, + ) + _persist_checkpoint(preflight, store, args, "preflight") + _materialize_evidence(preflight, store, args.out) + _, admission = decode_full_gate_report( + _payload(preflight, store, "uk.full.gates.preflight", "gate_report") + ) + if not admission["artifact_permitted"]: + return 1 + manifest = run_graph( + compile_graph(_through(graph, "uk.full.gates.calibrated")), + sources=sources, + store=store, + kernels=kernels, + resume="require" if args.resume == "require" else "auto", + ) + _persist_checkpoint(manifest, store, args, "numerical") + _materialize_evidence(manifest, store, args.out) + terminal_files = materialize_uk_terminal_artifacts( + manifest, + store, + directory=args.out, + stem=f"microcosm_uk_{full.config.calibration_year}", + ) + _, enforcement = decode_full_gate_report( + _payload(manifest, store, "uk.full.gates.calibrated", "gate_report") + ) + if not enforcement["artifact_permitted"]: + return 1 + manifest = run_graph( + compile_graph(graph), + sources=sources, + store=store, + kernels=kernels, + resume="require" if args.resume == "require" else "auto", + ) + descriptor = json.loads( + _payload(manifest, store, "uk.full.export.prepare", "export_descriptor") + ) + dataset = args.out / f"microcosm_uk_{full.config.calibration_year}.h5" + materialize_uk_export(manifest.population(full.population), descriptor, dataset) + graph = add_uk_export_continuation( + graph, + population=full.population, + manifest_binding={ + "graph_sha256": hashlib.sha256(graph_to_json(graph).encode()).hexdigest() + }, + artifact_inputs=( + ArtifactInput( + "full_gates", + "uk.full.gates.calibrated", + "gate_report", + FULL_GATE_REPORT_TYPE, + ), + ArtifactInput( + "diagnostics", + "uk.full.gates.calibrated", + "calibration_diagnostics", + FULL_DIAGNOSTICS_TYPE, + ), + ArtifactInput("holdout", "uk.full.holdout", "holdout", FULL_HOLDOUT_TYPE), + ArtifactInput( + "target_diagnostics", + "uk.full.gates.calibrated", + "target_diagnostics_csv", + FULL_DIAGNOSTICS_CSV_TYPE, + ), + ArtifactInput( + "area_support", + "uk.full.gates.calibrated", + "area_support_csv", + FULL_SUPPORT_CSV_TYPE, + ), + ArtifactInput( + "target_registry", + "uk.full.target_selection", + "selection", + TARGET_SELECTION_TYPE, + ), + ), + evidence_files={ + "full_gates": "uk.full.gates.calibrated.gate_report.json", + "diagnostics": terminal_files["calibration_diagnostics"]["filename"], + "holdout": terminal_files["holdout"]["filename"], + "target_diagnostics": terminal_files["target_diagnostics"]["filename"], + "area_support": terminal_files["area_support"]["filename"], + "target_registry": terminal_files["target_registry"]["filename"], + }, + ) + evidence_sources = { + "exported_evidence_full_gates": args.out + / "uk.full.gates.calibrated.gate_report.json", + "exported_evidence_diagnostics": args.out + / terminal_files["calibration_diagnostics"]["filename"], + "exported_evidence_holdout": args.out / terminal_files["holdout"]["filename"], + "exported_evidence_target_diagnostics": args.out + / terminal_files["target_diagnostics"]["filename"], + "exported_evidence_area_support": args.out + / terminal_files["area_support"]["filename"], + "exported_evidence_target_registry": args.out + / terminal_files["target_registry"]["filename"], + } + comparisons = prepared.comparison_sources or {} + graph = append_uk_full_certification_node( + graph, + population=full.population, + spine_provenance=prepared.spine_provenance, + comparison_sources=tuple(comparisons), + ) + final = run_graph( + compile_graph(graph), + sources={ + **sources, + "exported_dataset": dataset, + **evidence_sources, + **comparisons, + }, + store=store, + kernels=kernels, + resume="auto", + ) + final.save(args.out / "build.graph.json") + materialize_bytes(graph_to_json(graph).encode(), args.out / "graph.json") + _materialize_evidence(final, store, args.out) + package = json.loads(_payload(final, store, "uk.full.package", "package_inventory")) + candidate_file = materialize_bytes( + canonical_json(package), args.out / "candidate.json" + ) + certification_payload = _payload( + final, store, "uk.full.certification", "certification_readiness" + ) + certification_file = materialize_bytes( + certification_payload, args.out / "certification.json" + ) + completion = { + **package, + "kind": "uk_full_build_completion", + "candidate_manifest": candidate_file, + "certification": { + **certification_file, + "graph_artifact_key": final.nodes["uk.full.certification"].opaque_artifacts[ + "certification_readiness" + ], + }, + } + materialize_bytes(canonical_json(completion), args.out / "build.json") + return ( + 0 if package["readback_passed"] and not enforcement["enforced_blocking"] else 1 + ) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + prepared = None + try: + prepared = prepare_full_build(args) + return execute_full_build(prepared, args) + except Exception as error: + safe_output = False + if prepared is not None: + try: + _output_locations(prepared, args) + safe_output = True + except ValueError: + pass + if not args.dry_run and safe_output: + materialize_bytes( + canonical_json( + { + "schema": "microcosm.uk.full-build-failure.v1", + "error_type": type(error).__name__, + "message": str(error), + "evidence_directory": str(args.attempt_evidence) + if hasattr(args, "attempt_evidence") + else None, + "release_authorized": False, + } + ), + args.out / "failure.json", + ) + print(f"UK full build failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py new file mode 100644 index 000000000..8393023ee --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py @@ -0,0 +1,524 @@ +"""Unsigned certification readiness from one UK graph's identified artifacts. + +This consumer does not rerun a battery, solve weights or authorize publication. +Historical split-lane signatures remain verifiable in release_certification; +current builds bind one complete gate roster and the selected target scope. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import sys +from collections.abc import Mapping +from dataclasses import replace +from pathlib import Path + +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + RunManifest, + SeedSource, + SourceRef, + source_hash, +) +from microcosm.graph.canonical import canonical_json + +from ..artifact_files import file_artifact, materialize_bytes +from ..country_spec import load_country_spec +from ..gate_battery import GateStatus, gate_phase_report_from_payload +from .full_gates import uk_full_gate_manifest, uk_full_gate_scope_receipt +from .graph_evidence import SPINE_GATE_REPORT_TYPE, uk_spine_gate_manifest +from .graph_targets import TARGET_SELECTION_TYPE, TARGET_SURFACE_TYPE +from .graph_terminal import ( + EXPORT_DESCRIPTOR_TYPE, + EXPORT_READBACK_TYPE, + FULL_DIAGNOSTICS_TYPE, + FULL_GATE_REPORT_TYPE, + FULL_HOLDOUT_TYPE, + PACKAGE_INVENTORY_TYPE, + decode_full_gate_report, +) + +FULL_CERTIFICATION_TYPE = ArtifactType("microcosm.uk.full-certification-readiness", 1) +_REQUIRED = frozenset( + { + "package", + "export_descriptor", + "export_readback", + "preflight", + "full_gates", + "diagnostics", + "holdout", + "selection", + "surface", + } +) +_COMPARISON_SOURCES = { + "native_scorecard": "uk_native_scorecard", + "matched_size_scorecard": "uk_matched_size_scorecard", +} + + +def _object(payload: bytes) -> dict: + value = json.loads(payload) + if not isinstance(value, dict): + raise ValueError("UK certification input must be a JSON object.") + return value + + +def _equal(actual, expected, label: str) -> None: + if actual != expected: + raise ValueError(f"UK certification {label} differs from its graph artifact.") + + +def _scorecard_status(payload, *, expected_identity, matched_households=None): + """Use the existing release-quality assessment, preserving incomplete evidence.""" + from microcosm.data.contract import _check_uk_incumbent_surface_evaluation + + if payload is None: + return { + "status": "evidence_absent", + "failures": ["No declared scorecard source."], + } + failures = [] + _check_uk_incumbent_surface_evaluation( + payload, failures, expected_identity=expected_identity + ) + if matched_households is not None: + comparison = payload.get("comparison", {}) + if ( + not isinstance(comparison, Mapping) + or comparison.get("kind") != "matched_size" + ): + failures.append( + "Matched-size scorecard needs an explicit matched_size comparison identity." + ) + elif any( + type(comparison.get(field)) is not int + or comparison[field] != matched_households + for field in ("candidate_households", "incumbent_households") + ): + failures.append( + "Matched-size scorecard population counts must both equal the exported k." + ) + return {"status": "passed" if not failures else "failed", "failures": failures} + + +def compose_uk_full_certification_readiness( + artifacts: Mapping[str, tuple[str, bytes]], + *, + comparison_sources: Mapping[str, tuple[Mapping, bytes]] | None = None, +) -> dict: + """Verify one graph's byte joins and report readiness for external review.""" + missing = _REQUIRED - set(artifacts) + if missing: + raise ValueError( + f"UK certification is missing graph artifacts {sorted(missing)}." + ) + documents = {name: _object(payload) for name, (_, payload) in artifacts.items()} + keys = {name: key for name, (key, _) in artifacts.items()} + package, readback = documents["package"], documents["export_readback"] + descriptor = documents["export_descriptor"] + if ( + package.get("kind") != "uk_full_build_package" + or package.get("schema_version") != 1 + or package.get("readback_passed") is not True + or package.get("release_authorized") is not False + ): + raise ValueError("UK certification requires the full graph package inventory.") + if ( + descriptor.get("kind") != "uk_full_build_export" + or descriptor.get("schema_version") != 1 + ): + raise ValueError("UK certification requires a typed export descriptor.") + if ( + readback.get("kind") != "uk_full_build_export_readback" + or readback.get("passed") is not True + ): + raise ValueError("UK certification requires passing exported-byte readback.") + _equal(package["dataset"], readback["dataset"], "candidate bytes") + _equal(package["content_sha256"], readback["content_sha256"], "candidate contents") + _equal( + readback["content_sha256"], descriptor["content_sha256"], "export descriptor" + ) + _equal(readback["bindings"], descriptor["bindings"], "export bindings") + _equal(package["build_bindings"], readback["bindings"], "package bindings") + for name in ("full_gates", "diagnostics", "holdout"): + _equal(package["artifacts"].get(name), keys[name], f"packaged {name}") + _equal( + package["artifacts"].get("export_readback"), + keys["export_readback"], + "packaged readback", + ) + selection = documents["selection"]["receipt"] + scope = uk_full_gate_scope_receipt(selection) + manifest = uk_full_gate_manifest(selection) + outcomes = {} + phase_names = [] + for name in ("preflight", "full_gates"): + document = documents[name] + report, _ = decode_full_gate_report(document) + _equal(document["selection_receipt"], selection, f"{name} target selection") + _equal(document["scope"], scope, f"{name} declared scope") + for dependency in ("selection", "surface"): + _equal( + document["artifacts"].get(dependency), + keys[dependency], + f"{name} {dependency}", + ) + phase_names.append(report.phase) + for outcome in report.outcomes: + if outcome.entry.id in outcomes: + raise ValueError("UK certification has duplicate gate outcomes.") + outcomes[outcome.entry.id] = outcome.to_payload() + final = documents["full_gates"] + _equal( + final["artifacts"].get("preflight"), keys["preflight"], "calibrated preflight" + ) + _equal(final["artifacts"].get("holdout"), keys["holdout"], "calibrated holdout") + _equal( + final["sample_fraction"], + documents["preflight"]["sample_fraction"], + "sample fraction", + ) + _equal( + final["release_candidate"], + documents["preflight"]["release_candidate"], + "release posture", + ) + if "spine_provenance" in documents: + if {"spine_assembled", "spine_transferred"} & set(documents): + raise ValueError( + "Use raw spine reports or bound checkpoint provenance, not both." + ) + provenance = documents["spine_provenance"] + _equal( + final["artifacts"].get("spine_provenance"), + keys["spine_provenance"], + "spine checkpoint provenance", + ) + report = provenance["spine_gate_report"]["payload"] + if not provenance.get("uk_frame_content_identity") or not provenance[ + "spine_gate_report" + ].get("sha256"): + raise ValueError("UK certification requires strict bound spine provenance.") + spine_gates = uk_spine_gate_manifest(load_country_spec("uk")) + expected = {entry.id: entry for entry in spine_gates.gates} + _equal(set(report["gates"]), set(expected), "checkpoint spine scope") + _equal(report.get("blocked_at_phase"), None, "checkpoint spine gate completion") + for gate_id, entry in expected.items(): + outcome = report["gates"][gate_id] + _equal( + outcome.get("criticality"), entry.criticality, f"{gate_id} criticality" + ) + _equal(outcome.get("phase"), entry.phase, f"{gate_id} phase") + outcomes[gate_id] = outcome + phase_names.extend(spine_gates.phases) + else: + spine_gates = uk_spine_gate_manifest(load_country_spec("uk")) + for name, phase in ( + ("spine_assembled", "assembled"), + ("spine_transferred", "transferred"), + ): + if name not in documents: + raise ValueError( + f"UK certification needs {name} or strict bound spine provenance." + ) + report = gate_phase_report_from_payload(documents[name], gates=spine_gates) + _equal(report.phase, phase, "spine phase") + phase_names.append(report.phase) + for outcome in report.outcomes: + if outcome.entry.id in outcomes: + raise ValueError( + "UK certification has duplicate spine gate outcomes." + ) + outcomes[outcome.entry.id] = outcome.to_payload() + _equal( + set(outcomes), {entry.id for entry in manifest.gates}, "complete gate roster" + ) + _equal(set(phase_names), set(manifest.phases), "complete gate phases") + reviewed_inapplicable = { + entry.id for entry in manifest.gates if entry.not_applicable is not None + } + failed_gates = sorted( + gate_id + for gate_id, outcome in outcomes.items() + if outcome["criticality"] == "release_blocking" + and outcome["status"] != GateStatus.PASSED.value + and not ( + gate_id in reviewed_inapplicable + and outcome["status"] == GateStatus.NOT_APPLICABLE.value + ) + ) + validation = documents["surface"]["source_validation"] + ledger = validation["ledger_provenance"] + expected_identity = { + "candidate_dataset_sha256": package["dataset"]["sha256"], + "candidate_manifest_sha256": hashlib.sha256( + artifacts["package"][1] + ).hexdigest(), + "candidate_diagnostics_sha256": hashlib.sha256( + artifacts["diagnostics"][1] + ).hexdigest(), + "ledger_facts_sha256": ledger["facts_sha256"], + "ledger_manifest_sha256": ledger["manifest_sha256"], + } + config = package["build_bindings"].get("configuration", {}) + requested_k = config.get("calibration", {}).get("dataset_households") + actual_k = descriptor["tables"]["household"]["rows"] + if requested_k is not None: + _equal(actual_k, requested_k, "exported exact household count") + comparisons = {} + comparison_sources = {} if comparison_sources is None else comparison_sources + if set(comparison_sources) - set(_COMPARISON_SOURCES): + raise ValueError("Unknown UK certification scorecard source.") + for role in _COMPARISON_SOURCES: + source = comparison_sources.get(role) + if role == "matched_size_scorecard" and requested_k is None: + comparisons[role] = { + "status": "not_required", + "reason": "No exported exact-count k was requested.", + } + if source is not None: + raise ValueError( + "Matched-size scorecard supplied to a build without requested k." + ) + continue + payload = None if source is None else _object(source[1]) + comparisons[role] = _scorecard_status( + payload, + expected_identity=expected_identity, + matched_households=actual_k if role == "matched_size_scorecard" else None, + ) + if source is not None: + file = dict(source[0]) + _equal( + file["sha256"], + hashlib.sha256(source[1]).hexdigest(), + f"{role} source bytes", + ) + _equal(file["size_bytes"], len(source[1]), f"{role} source length") + comparisons[role]["source"] = file + reasons = [] + if failed_gates: + reasons.append("Non-passing release-blocking gates: " + ", ".join(failed_gates)) + if final["sample_fraction"] != 1.0: + reasons.append( + "Development sample fraction does not establish native population readiness." + ) + holdout = documents["holdout"] + if scope["local_fit_claim"] and ( + holdout.get("skipped") + or holdout.get("method") != "rotated_folds" + or holdout.get("n_folds") != 5 + or len(holdout.get("folds", ())) != 5 + or any( + isinstance(holdout.get(field), bool) + or not isinstance(holdout.get(field), int | float) + or not math.isfinite(holdout[field]) + for field in ("mean_holdout_loss", "worst_holdout_loss") + ) + ): + reasons.append("The declared five-fold local holdout is skipped or incomplete.") + reasons.extend( + f"{role}: {record['status']}" + for role, record in comparisons.items() + if record["status"] not in {"passed", "not_required"} + ) + return { + "schema_version": 1, + "kind": "uk_full_build_certification_readiness", + "candidate": package["dataset"], + "content_sha256": package["content_sha256"], + "target_scope": scope, + "gate_coverage": { + "declared": sorted(outcomes), + "phases": sorted(phase_names), + "scope_exclusions": scope["scope_exclusions"], + }, + "gate_outcomes": outcomes, + "source_validation": validation, + "comparisons": comparisons, + "artifacts": { + name: { + "graph_artifact_key": key, + "sha256": hashlib.sha256(payload).hexdigest(), + "size_bytes": len(payload), + } + for name, (key, payload) in sorted(artifacts.items()) + }, + "ready_for_external_review": not reasons, + "readiness_failures": reasons, + "subnational_fit_certified": False, + "release_authorized": False, + "signing": "external", + } + + +class UKFullCertificationKernel(KernelBase): + ref = "uk.full.certification-readiness@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, numeric=Numeric.BITWISE, seed_source=SeedSource.NONE + ) + + def implementation_hash(self): + from microcosm.data import contract + + from . import full_gates, graph_terminal + + return source_hash(sys.modules[__name__], contract, full_gates, graph_terminal) + + def run(self, context: KernelContext) -> KernelResult: + sources = {} + for role, source_name in _COMPARISON_SOURCES.items(): + if source_name in context.sources: + path = Path(context.sources[source_name]) + payload = path.read_bytes() + sources[role] = (file_artifact(path), payload) + report = compose_uk_full_certification_readiness( + { + name: (value.key, value.payload) + for name, value in context.artifacts.items() + }, + comparison_sources=sources, + ) + return KernelResult( + artifacts={"certification_readiness": canonical_json(report)} + ) + + +def append_uk_full_certification_node( + graph: Graph, + *, + population: str, + spine_provenance: ArtifactInput | None = None, + comparison_sources: tuple[str, ...] = (), +) -> Graph: + """Append readiness after packaging; comparison sources are explicit inputs.""" + if set(comparison_sources) - set(_COMPARISON_SOURCES.values()): + raise ValueError("Unknown full-build comparison source.") + inputs = [ + ArtifactInput( + "package", "uk.full.package", "package_inventory", PACKAGE_INVENTORY_TYPE + ), + ArtifactInput( + "export_descriptor", + "uk.full.export.prepare", + "export_descriptor", + EXPORT_DESCRIPTOR_TYPE, + ), + ArtifactInput( + "export_readback", + "uk.full.export.readback", + "export_readback", + EXPORT_READBACK_TYPE, + ), + ArtifactInput( + "preflight", "uk.full.gates.preflight", "gate_report", FULL_GATE_REPORT_TYPE + ), + ArtifactInput( + "full_gates", + "uk.full.gates.calibrated", + "gate_report", + FULL_GATE_REPORT_TYPE, + ), + ArtifactInput( + "diagnostics", + "uk.full.gates.calibrated", + "calibration_diagnostics", + FULL_DIAGNOSTICS_TYPE, + ), + ArtifactInput("holdout", "uk.full.holdout", "holdout", FULL_HOLDOUT_TYPE), + ArtifactInput( + "selection", "uk.full.target_selection", "selection", TARGET_SELECTION_TYPE + ), + ArtifactInput( + "surface", "uk.full.target_compilation", "surface", TARGET_SURFACE_TYPE + ), + ] + if spine_provenance is None: + inputs.extend( + ArtifactInput( + f"spine_{phase}", + f"spine.gates.{phase}", + "gate_report", + SPINE_GATE_REPORT_TYPE, + ) + for phase in ("assembled", "transferred") + ) + else: + inputs.append(replace(spine_provenance, name="spine_provenance")) + existing = {source.name for source in graph.sources} + return replace( + graph, + sources=( + *graph.sources, + *( + SourceRef(name, "raw-bytes-v1") + for name in comparison_sources + if name not in existing + ), + ), + nodes=( + *graph.nodes, + Node( + "uk.full.certification", + UKFullCertificationKernel.ref, + population=population, + sources=comparison_sources, + artifact_inputs=tuple(inputs), + artifact_outputs=( + ArtifactOutput("certification_readiness", FULL_CERTIFICATION_TYPE), + ), + description="Bind complete selected-scope gates, exact exported bytes and native/size comparison readiness; publication and signing remain external.", + ), + ), + ) + + +def register_uk_full_certification_kernel(registry: KernelRegistry) -> None: + registry.register(UKFullCertificationKernel()) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Materialize one full UK graph's unsigned certification readiness; historical split-lane inputs are retired." + ) + parser.add_argument("--graph-manifest", required=True, type=Path) + parser.add_argument("--graph-store", required=True, type=Path) + parser.add_argument("--candidate-h5", required=True, type=Path) + parser.add_argument("--certification-json", required=True, type=Path) + args = parser.parse_args(argv) + store = ContentStore(args.graph_store) + manifest = RunManifest.load(args.graph_manifest, store) + key = manifest.nodes["uk.full.certification"].opaque_artifacts[ + "certification_readiness" + ] + payload = store.load_bytes(key) + report = _object(payload) + if ( + report.get("kind") != "uk_full_build_certification_readiness" + or report.get("release_authorized") is not False + ): + raise ValueError( + "Expected an unsigned full-graph certification readiness artifact." + ) + _equal( + file_artifact(args.candidate_h5), report["candidate"], "current candidate bytes" + ) + materialize_bytes(payload, args.certification_json) + return 0 if report["ready_for_external_review"] else 1 diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py new file mode 100644 index 000000000..c9e799c73 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py @@ -0,0 +1,343 @@ +"""Complete UK gate ownership for one full build and explicit target filters. + +Country-only target selection removes local fit claims, while geographic +integrity, source coverage, register completeness and population checks remain. +Gate evaluation and persistence use the shared battery and graph artifacts. +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Mapping +from dataclasses import replace +from types import SimpleNamespace +from typing import Any + +import numpy as np +from scipy import sparse + +from microcosm.build.country_spec import GatesManifest, load_country_spec +from microcosm.build.gate_battery import EvidenceContext +from microcosm.build.uk_runtime.calibration_run import uk_aggregate_admin_totals +from microcosm.build.uk_runtime.graph_evidence import uk_spine_gate_artifacts +from microcosm.build.uk_runtime.parity_reference import load_efrs_parity_reference +from microcosm.calibrate import TargetRegistry +from microcosm.calibrate.artifacts import OrderedProblem, OrderedSolution +from microcosm.calibrate.solve import CalibrationResult, _build_diagnostics +from microcosm.frame import Frame +from microcosm.graph.canonical import canonical_json + +_LOCAL_FIT_GATES = frozenset( + {"uk_local_target_fit", "uk_local_per_family_fit", "uk_local_area_support"} +) +_LOCAL_LEVELS = frozenset({"constituency", "local_authority", "la"}) + + +def _scope(selection_receipt: Mapping[str, Any] | None) -> tuple[bool, dict[str, str]]: + if selection_receipt is None: + return True, {} + if selection_receipt.get("schema") != "microcosm.calibrate.target-selection.v1": + raise ValueError("UK gate scope requires a versioned target-selection receipt.") + included = selection_receipt.get("included") + if not isinstance(included, list) or not included: + raise ValueError("UK gate scope requires nonempty selected targets.") + levels = {str(row["geography_level"]) for row in included} + unknown = levels - (_LOCAL_LEVELS | {"country", "region"}) + if unknown: + raise ValueError( + f"UK gate scope has unknown geography levels {sorted(unknown)}." + ) + has_local = bool(levels & _LOCAL_LEVELS) + selector = selection_receipt.get("selector", {}) + if selector.get("geography_levels") is None: + # An unfiltered source that omits all local targets is a completeness + # failure, not permission to declare national-only scope. + if not has_local: + raise ValueError("An unfiltered UK full build must include local targets.") + return True, {} + excluded = ( + {} + if has_local + else { + gate_id: "No local targets were selected; local fit and per-area fit-support claims are inapplicable." + for gate_id in sorted(_LOCAL_FIT_GATES) + } + ) + return has_local, excluded + + +def uk_full_gate_manifest( + selection_receipt: Mapping[str, Any] | None = None, + *, + source: GatesManifest | None = None, +) -> GatesManifest: + """Return every declared UK gate, except explicitly inapplicable fit checks.""" + source = load_country_spec("uk").gates if source is None else source + _, exclusions = _scope(selection_receipt) + return replace( + source, + policy=f"{source.policy}; full_build_scope", + gates=tuple(entry for entry in source.gates if entry.id not in exclusions), + ) + + +def uk_full_gate_scope_receipt( + selection_receipt: Mapping[str, Any] | None = None, +) -> dict[str, object]: + """Describe selected-fit scope without changing invariant gate ownership.""" + has_local, exclusions = _scope(selection_receipt) + return { + "schema": "microcosm.build.uk.full-gate-scope.v1", + "posture": "full_build" if has_local else "full_build_filtered_targets", + "local_fit_claim": has_local, + "scope_exclusions": exclusions, + "target_selection": None + if selection_receipt is None + else dict(selection_receipt), + } + + +def build_full_gate_context( + frame: Frame, + *, + ordered_problem: OrderedProblem, + solution: OrderedSolution, + selection_receipt: Mapping[str, Any], + stage_evidence: Mapping[str, Mapping[str, Any]], + supporting_evidence: Mapping[str, Any], +) -> EvidenceContext: + """Reconstruct gate evidence from identified rows and the installed solution. + + Supporting source/validation artifacts are forwarded under their existing + gate-binding names. Missing artifacts remain missing so shared evaluation + records an evidence failure; measured quantities are never filled with zero. + """ + _scope(selection_receipt) + bindings = ordered_problem.bindings + embedded = bindings.get("target_selection") + digest = bindings.get("target_selection_sha256") + if embedded is None and digest is None: + raise ValueError("Gate problem has no target-selection binding.") + if embedded is not None and canonical_json(embedded) != canonical_json( + selection_receipt + ): + raise ValueError("Gate target selection differs from the problem binding.") + if ( + digest is not None + and digest != hashlib.sha256(canonical_json(selection_receipt)).hexdigest() + ): + raise ValueError( + "Gate target-selection digest differs from the problem binding." + ) + if solution.problem_sha256 != ordered_problem.sha256: + raise ValueError("Gate solution belongs to a different ordered problem.") + if solution.entity_ids != ordered_problem.entity_ids: + raise ValueError("Gate solution and problem have different household axes.") + entity = ordered_problem.problem.weight_entity + ids = tuple(frame.table(entity)[frame.schema.entity_id_column(entity)]) + if ids != ordered_problem.entity_ids: + raise ValueError( + "Gate population differs from the ordered problem household axis." + ) + if not np.array_equal(frame.weights_for(entity).values, solution.weights): + raise ValueError("Gate population does not carry the bound solution weights.") + if ordered_problem.problem.skipped: + raise ValueError( + "Full-build gate context cannot omit skipped selected targets." + ) + problem = ordered_problem.problem + if len(ordered_problem.target_metadata) != len(problem.targets): + raise ValueError("Gate target metadata does not align with the ordered matrix.") + calibration_result = supporting_evidence.get("calibration_result") + if calibration_result is None: + solved_rows = _build_diagnostics( + problem, frame, problem.initial_weights.values, solution.weights + ) + else: + if not isinstance(calibration_result, CalibrationResult): + raise TypeError( + "Gate calibration_result must be a decoded CalibrationResult." + ) + result_problem = calibration_result.problem + result_ids = tuple( + calibration_result.frame.table(entity)[ + calibration_result.frame.schema.entity_id_column(entity) + ] + ) + if ( + calibration_result.weight_entity != entity + or result_ids != ordered_problem.entity_ids + or not np.array_equal(calibration_result.weights, solution.weights) + or not np.array_equal( + calibration_result.initial_weights, problem.initial_weights.values + ) + or result_problem.names != problem.names + or result_problem.matrix.shape != problem.matrix.shape + or ( + sparse.csr_array(result_problem.matrix) + != sparse.csr_array(problem.matrix) + ).nnz + or not np.array_equal(result_problem.target_vector, problem.target_vector) + or calibration_result.skipped + ): + raise ValueError( + "Gate calibration_result differs from the bound numerical problem/solution." + ) + solved_rows = calibration_result.diagnostics + if tuple( + row.name for row in solved_rows + ) != problem.names or not np.array_equal( + [row.target for row in solved_rows], problem.target_vector + ): + raise ValueError( + "Gate calibration_result diagnostics differ from the target axis." + ) + national_errors: dict[str, float] = {} + local_rows = [] + diagnostics = [] + for index, (target, metadata) in enumerate( + zip(problem.targets, ordered_problem.target_metadata, strict=True) + ): + level = str(metadata.get("geography_level", "")) + materialization = metadata.get("materialization") + if level not in _LOCAL_LEVELS | {"country", "region"}: + raise ValueError(f"Target {target.row_name!r} lacks classified geography.") + if materialization not in {"uk_local_surface", "uk_national_measure"}: + raise ValueError( + f"Target {target.row_name!r} lacks a materialization owner." + ) + row = { + "name": target.row_name, + "target_name": target.row_name, + "target": float(target.value), + "estimate": float(solved_rows[index].final_estimate), + "relative_error": float(solved_rows[index].relative_error), + "abs_relative_error": float(abs(solved_rows[index].relative_error)), + "family": str(metadata.get("family", "")), + "geography_level": level, + "area_type": "local_authority" if level == "la" else level, + "area_code": str(metadata.get("geography_id", "")), + "metric": str( + metadata.get("metric", metadata.get("contract_target_id", target.name)) + ), + } + if not row["family"]: + raise ValueError(f"Target {target.row_name!r} lacks a diagnostic family.") + diagnostics.append(row) + if materialization == "uk_local_surface": + local_rows.append(row) + else: + national_errors[target.row_name] = float(solved_rows[index].relative_error) + reference_registry = supporting_evidence.get("reference_registry") + if not isinstance(reference_registry, TargetRegistry): + raise ValueError( + "Full-build gates require the complete approved national reference registry." + ) + selected = { + (str(row["name"]), row["period"]) for row in selection_receipt["included"] + } + national_reference = { + spec.to_target().row_name + for spec in reference_registry.specs + if (spec.name, spec.period) in selected + } + reference = load_efrs_parity_reference() + manifest = uk_full_gate_manifest(selection_receipt) + admin_totals, admin_receipt = uk_aggregate_admin_totals(frame, manifest) + artifacts = dict(supporting_evidence) + artifacts.update( + { + "stage_evidence": dict(stage_evidence), + "build_stage_names": tuple(stage_evidence), + "national_calibration": { + "activated_reference_count": len(selection_receipt["included"]), + "resolved_reference_count": len(problem.targets), + "matrix_target_count": len(problem.names), + }, + "parity_evidence": SimpleNamespace( + candidate_columns={ + f"{entity}.{column}" + for entity in frame.entities + for column in frame.table(entity).columns + }, + reference_columns={ + f"{entity}.{name}" + for name, entity in reference.input_entities.items() + }, + candidate_targets=set(national_errors), + reference_targets=national_reference, + target_relative_errors=national_errors, + ), + "local_target_diagnostics": local_rows, + "target_diagnostics": diagnostics, + "aggregate_admin": admin_totals, + "aggregate_admin_measurement": admin_receipt, + "full_gate_scope": uk_full_gate_scope_receipt(selection_receipt), + } + ) + if "rules_engine" not in artifacts and "coverage_engine" in artifacts: + artifacts["rules_engine"] = artifacts["coverage_engine"] + if "rules_engine" in artifacts: + for name, value in uk_spine_gate_artifacts(artifacts["rules_engine"]).items(): + artifacts.setdefault(name, value) + return EvidenceContext(frame=frame, artifacts=artifacts) + + +def classify_full_gate_outcomes( + report, + *, + sample_fraction: float, + release_candidate: bool, +) -> dict[str, object]: + """Preserve declaration enforcement and the existing local export exception. + + The combined driver exports a diagnostic candidate after failures of its + five local fit/support/weight checks, but stops on a non-passing geography + ladder. Below f100 those five checks are recorded without enforcement. + All newly included national/source checks keep the shared battery's own + BLOCKS_ARTIFACT policy, including its declared missing-evidence rules. + """ + from microcosm.build.gate_battery import GatePhaseReport, GateStatus + from microcosm.build.uk_runtime.calibration_run import UK_LOCAL_GATE_SCOPE + + if not isinstance(report, GatePhaseReport): + raise TypeError("Classify a validated GatePhaseReport, not unbound JSON.") + if not 0.0 < sample_fraction <= 1.0: + raise ValueError("sample_fraction must be in (0, 1].") + geography = "uk_local_geography_ladder_post_calibration" + local_export_exception = set(UK_LOCAL_GATE_SCOPE) - {geography} + blocking = { + outcome.entry.id + for outcome in report.blocking_outcomes(release_candidate=release_candidate) + } + stop, exported_failures, unenforced, diagnostic = [], [], [], [] + for outcome in report.outcomes: + gate_id = outcome.entry.id + if gate_id == geography and outcome.status is not GateStatus.PASSED: + stop.append(gate_id) + continue + if outcome.status not in {GateStatus.FAILED, GateStatus.EVIDENCE_ABSENT}: + continue + if outcome.entry.criticality == "diagnostic": + diagnostic.append(gate_id) + elif gate_id in local_export_exception: + if gate_id in blocking and sample_fraction == 1.0: + exported_failures.append(gate_id) + else: + unenforced.append(gate_id) + elif gate_id in blocking: + stop.append(gate_id) + else: + unenforced.append(gate_id) + return { + "schema": "microcosm.build.uk.full-gate-enforcement.v1", + "sample_fraction": sample_fraction, + "release_candidate": release_candidate, + "structural_failures": stop, + "enforced_blocking": stop + exported_failures, + "exportable_blocking": exported_failures, + "unenforced_release_failures": unenforced, + "diagnostic_failures": diagnostic, + "artifact_permitted": not stop, + "release_blocking_gates_passed": not (stop or exported_failures or unenforced), + } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py new file mode 100644 index 000000000..a2ec27aac --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py @@ -0,0 +1,283 @@ +"""Rules-engine evaluation for the selected full-build target surface. + +Temporary inputs and prepared columns remain inside this operation. Graph +consumers receive compiled numerical contributions, never injected engine state. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.target_materialization import resolve_target_measures +from microcosm.build.uk_runtime import ( + CalibrationFrameAdapter, + UKRowwiseNationalRows, + compute_household_metrics, + drop_injected_measure_inputs, + inject_measure_inputs, + ladder_clone_index_column, + materialize_uk_ledger_targets, +) +from microcosm.build.uk_runtime.measure_simulation import UKMeasureResolver +from microcosm.frame import Frame, MassChangeRecord + +UK_BLOCK_SENSITIVE_MEASURE_COLUMNS = ( + "ons/corporate_land_value", + "ons/land_value", + "slc/student_loan_repayment/england", +) + + +def resolve_uk_full_measures( + frame, + national_registry, + *, + period: int, + scratch_dir: Path, + band_edge_registry=None, + resolver_factory=UKMeasureResolver, + blocks: int = 1, + local_grains: tuple[str, ...] = ("constituency", "la"), +) -> tuple[Any, Any, UKRowwiseNationalRows, dict[str, pd.DataFrame], dict[str, Any]]: + """Resolve national inputs and local metrics on the cloned frame. + + ``blocks=1`` uses one scratch-mode engine for the whole clone. The + reviewed escape hatch ``blocks=K`` resolves each clone index separately, + then rejoins every entity-level prepared column by its stable entity id so + the full-frame target materialization and single solve retain frame order. + """ + + household = frame.table("household") + if blocks < 1: + raise ValueError("engine resolution blocks must be positive.") + if blocks == 1: + block_frames = [(None, frame)] + else: + clone_column = ladder_clone_index_column("household") + if clone_column not in household.columns: + raise ValueError(f"per-clone engine resolution requires {clone_column}.") + clone_indices = tuple(sorted(household[clone_column].unique().tolist())) + if len(clone_indices) != blocks: + raise ValueError( + "engine resolution blocks must match the realized clone indices: " + f"requested {blocks}, found {clone_indices}." + ) + person = frame.table("person") + block_frames = [] + for clone_index in clone_indices: + household_ids = set( + household.loc[ + household[clone_column] == clone_index, + "household_id", + ].tolist() + ) + person_mask = person["person_household_id"].isin(household_ids) + block = frame.select(person_mask) + # The block carries a K-th of the cloned mass while its log still + # ends on the full-clone record, and the scratch export validates + # the chain. Declare the subset explicitly: old = the cloned + # total, new = the block total, reason naming the block. The block + # frame is engine scratch and is discarded after resolution. + block_weights = block.weights_for("household") + full_total = float(frame.weights_for("household").total) + block_total = float(block_weights.total) + subset_record = MassChangeRecord( + entity="household", + old_total=full_total, + new_total=block_total, + declared_factor=block_total / full_total, + reason=( + f"engine resolution block {clone_index} of {blocks}: " + "scratch subset of the cloned frame for measure " + "resolution only, discarded after resolution" + ), + ) + block = Frame( + { + **{name: block.table(name) for name in block.entities}, + **{name: block.link(name) for name in block.links}, + }, + block.schema, + { + entity: block.weights_for(entity) + for entity in block.weighted_entities + }, + block.strata, + mass_log=(*block.mass_log, subset_record), + metadata=block.metadata, + ) + block_frames.append((clone_index, block)) + + measure_parts: dict[tuple[str, str], list[pd.Series]] = {} + metric_parts: dict[str, list[pd.DataFrame]] = {grain: [] for grain in local_grains} + resolver_receipts: list[Mapping[str, Any]] = [] + national_input_keys: set[tuple[str, str]] | None = None + for clone_index, block_frame in block_frames: + block_scratch = ( + scratch_dir if clone_index is None else scratch_dir / f"clone-{clone_index}" + ) + resolver = resolver_factory( + simulation_source=None, + scratch_dir=block_scratch, + year=period, + frame=block_frame, + ) + resolution = resolve_target_measures( + lambda block_frame=block_frame: CalibrationFrameAdapter(block_frame), + national_registry, + resolver, + period=period, + ) + keys = set(resolution.measure_inputs) + if national_input_keys is None: + national_input_keys = keys + elif keys != national_input_keys: + raise RuntimeError( + "per-clone engine resolution returned inconsistent national inputs." + ) + for (entity, variable), values in resolution.measure_inputs.items(): + entity_table = block_frame.table(entity) + entity_id = f"{entity}_id" + measure_parts.setdefault((entity, variable), []).append( + pd.Series( + np.asarray(values), + index=entity_table[entity_id].tolist(), + ) + ) + block_household_ids = block_frame.table("household")["household_id"].tolist() + for area_type in metric_parts: + metric_parts[area_type].append( + compute_household_metrics( + resolver.simulation, + area_type, + period=period, + household_ids=block_household_ids, + ) + ) + resolver_receipts.append(resolver.receipt()) + del resolver + simulation_input = block_scratch / "simulation-input.h5" + simulation_input.unlink(missing_ok=True) + try: + block_scratch.rmdir() + except OSError: + pass + + measure_inputs: dict[tuple[str, str], np.ndarray] = {} + for (entity, variable), parts in measure_parts.items(): + combined = pd.concat(parts) + if combined.index.has_duplicates: + raise RuntimeError( + f"per-clone engine resolution duplicated {entity} ids for {variable}." + ) + ordered_ids = frame.table(entity)[f"{entity}_id"] + ordered = combined.reindex(ordered_ids.tolist()) + if ordered.isna().any(): + raise RuntimeError( + f"per-clone engine resolution missed {entity} rows for {variable}." + ) + measure_inputs[(entity, variable)] = ordered.to_numpy() + + full_household_ids = household["household_id"].tolist() + local_metrics = {} + for area_type, parts in metric_parts.items(): + combined = pd.concat(parts) + if combined.index.has_duplicates: + raise RuntimeError( + f"per-clone engine resolution duplicated {area_type} household ids." + ) + ordered = combined.reindex(full_household_ids) + if ordered.isna().any().any(): + raise RuntimeError( + f"per-clone engine resolution missed {area_type} household rows." + ) + local_metrics[area_type] = ordered + + adapter = CalibrationFrameAdapter(frame) + # Injected engine inputs are scratch state for materialization only: + # they must be dropped before the prepared frame is assembled, or the + # flattening rule refuses columns that now exist on two entities + # (region, esa_* on the live spine). Same lifecycle as the national stage. + original_columns = { + entity: set(table.columns) for entity, table in adapter.tables.items() + } + inject_measure_inputs(adapter, measure_inputs) + materialized = materialize_uk_ledger_targets( + adapter, + national_registry, + period=period, + band_edge_registry=( + national_registry if band_edge_registry is None else band_edge_registry + ), + ) + if materialized.skipped: + raise RuntimeError( + "candidate national target materialization skipped row(s): " + f"{[skip.__dict__ for skip in materialized.skipped]}." + ) + modes = {receipt.get("mode") for receipt in resolver_receipts} + versions = {receipt.get("policyengine_uk_version") for receipt in resolver_receipts} + if len(modes) != 1 or len(versions) != 1: + raise RuntimeError("per-clone engine resolver provenance is inconsistent.") + cgt_period_contract = resolver_receipts[0].get("cgt_period_contract") + if any( + block_receipt.get("cgt_period_contract") != cgt_period_contract + for block_receipt in resolver_receipts[1:] + ): + raise RuntimeError("per-clone CGT period contract is inconsistent.") + receipt = { + "mode": next(iter(modes)), + "engine_version": next(iter(versions)), + "households": len(frame.table("household")), + "persons": len(frame.table("person")), + "benunits": len(frame.table("benunit")), + "national_inputs": len(measure_inputs), + "local_metrics": { + area_type: len(metrics.columns) + for area_type, metrics in local_metrics.items() + }, + "blocks": blocks, + } + if cgt_period_contract is not None: + receipt["cgt_period_contract"] = cgt_period_contract + if blocks > 1: + receipt["deviation"] = "per_clone_block_engine_resolution" + present = sorted( + column + for column in UK_BLOCK_SENSITIVE_MEASURE_COLUMNS + if column in {variable for _, variable in measure_inputs} + ) + receipt["block_sensitivity"] = { + "known_population_normalised_measures": list( + UK_BLOCK_SENSITIVE_MEASURE_COLUMNS + ), + "present_in_this_run": present, + "caveat": ( + "per-block engine resolution mis-measures population-normalised " + "formulas (each block reproduces a national aggregate); rows " + "on these measures are not evidence for adjudication from this " + "run. Resolve in a single block before ruling on them." + ), + } + try: + scratch_dir.rmdir() + except OSError: + pass + drop_injected_measure_inputs(adapter, measure_inputs, original_columns) + national_rows = UKRowwiseNationalRows( + targets=national_registry.to_target_set(), + registry=national_registry, + families=tuple(sorted({spec.family for spec in national_registry.specs})), + ) + return ( + adapter.prepared_frame(), + adapter.restore, + national_rows, + local_metrics, + receipt, + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_problem.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_problem.py new file mode 100644 index 000000000..067e2bd6f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_problem.py @@ -0,0 +1,216 @@ +"""Compile selected geographic contribution rows in the full UK build.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.uk_runtime import ( + UKRowwiseLocalMatrix, + uk_area_region_codes, + uk_local_target_surface, +) +from microcosm.build.uk_runtime.ledger_targets import _spec_geography +from microcosm.build.uk_runtime.local_rowwise import ( + build_uk_rowwise_local_surface_matrix, + empty_uk_local_problem, +) +from microcosm.calibrate import TargetRegistry + + +def _national_contract_target_ids(registry: TargetRegistry) -> tuple[str, ...]: + return tuple( + sorted( + { + str(spec.metadata.get("contract_target_id", spec.name)) + for spec in registry.specs + if _spec_geography(spec)[0] == "country" + } + ) + ) + + +def _joint_surface_registry( + local_registry: TargetRegistry, + national_registry: TargetRegistry, +) -> TargetRegistry: + """Put country controls beside local cells for declared reconciliation. + + Regional constraints stay in the national solve registry. They are outside + the country/constituency/LA reconciliation rule and cannot be passed as + country controls or silently assigned a new reconciliation policy. + """ + + return TargetRegistry( + [ + *local_registry.specs, + *( + spec + for spec in national_registry.specs + if _spec_geography(spec)[0] == "country" + ), + ], + country="uk", + ) + + +def build_uk_full_local_problem( + assignment: Any, + *, + local_registry: TargetRegistry, + national_registry: TargetRegistry, + local_metrics: Mapping[str, pd.DataFrame], + period: int, + sample_fraction: float, + reviewed_unbound_higher_targets: Mapping[str, Mapping[str, object]], + census_household_uprating: Mapping[str, Any] | None = None, + selected_surface: pd.DataFrame | None = None, + surface_receipt: Mapping[str, Any] | None = None, +) -> tuple[ + pd.DataFrame, + UKRowwiseLocalMatrix, + dict[str, Any], + tuple[str, ...], + dict[str, Any], +]: + household = assignment.result.frame.table("household").reset_index(drop=True) + household_index = pd.Index(household["household_id"], name="household_id") + metrics = { + grain: frame.set_axis(household_index, axis="index") + for grain, frame in local_metrics.items() + } + assigned = { + "constituency": pd.Series( + household["constituency_code"].astype(str).to_numpy(), + index=household_index, + ), + "la": pd.Series( + household["local_authority_code"].astype(str).to_numpy(), + index=household_index, + ), + } + assigned = {grain: assigned[grain] for grain in metrics} + national_ids = _national_contract_target_ids(national_registry) + if selected_surface is None: + surface, cross_grain = uk_local_target_surface( + _joint_surface_registry(local_registry, national_registry), + bound_national_target_ids=national_ids, + period=period, + reviewed_unbound_higher_targets=reviewed_unbound_higher_targets, + census_household_uprating=census_household_uprating, + # #906: the run's ladder membership resolves English areas to their + # region-tier leg, as the rowwise driver does. + area_region_codes=uk_area_region_codes(assignment.ladder), + ) + else: + surface = selected_surface.copy() + cross_grain = dict(surface_receipt or {}) + covered = { + grain: set(values.astype(str).tolist()) for grain, values in assigned.items() + } + covered_mask = pd.Series( + [ + str(row.area_code) in covered[str(row.area_type)] + for row in surface.itertuples(index=False) + ], + index=surface.index, + dtype=bool, + ) + dropped = surface.loc[~covered_mask] + if sample_fraction < 1.0: + surface = surface.loc[covered_mask].reset_index(drop=True) + # Below f100 a covered area can still carry a nonzero cell with no metric + # support in the sample (no self-employed household among three drawn + # rows). The builder refuses such a cell at every rung; at development + # rungs the cell is dropped here and receipted instead. f100 stays strict. + unreachable = surface.iloc[0:0] + if sample_fraction < 1.0 and len(surface): + nonzero_by_grain = { + grain: (metrics[grain] != 0).groupby(assigned[grain]).sum() + for grain in metrics + } + unreachable_mask = pd.Series( + [ + float(row.value) != 0.0 + and str(row.metric) in nonzero_by_grain[str(row.area_type)].columns + and str(row.area_code) in nonzero_by_grain[str(row.area_type)].index + and int( + nonzero_by_grain[str(row.area_type)].loc[ + str(row.area_code), str(row.metric) + ] + ) + == 0 + for row in surface.itertuples(index=False) + ], + index=surface.index, + dtype=bool, + ) + unreachable = surface.loc[unreachable_mask] + surface = surface.loc[~unreachable_mask].reset_index(drop=True) + rung_surface = { + "dropped_unreachable_cells": int(len(unreachable)), + "dropped_unreachable_by_grain": { + str(key): int(value) + for key, value in unreachable.groupby("area_type").size().items() + }, + "dropped_unreachable_by_family": { + str(key): int(value) + for key, value in unreachable.groupby("family").size().items() + }, + "fraction": float(sample_fraction), + "dropped_cells": int(len(dropped) if sample_fraction < 1.0 else 0), + "dropped_by_grain": ( + { + str(key): int(value) + for key, value in dropped.groupby("area_type").size().items() + } + if sample_fraction < 1.0 + else {} + ), + "dropped_by_family": ( + { + str(key): int(value) + for key, value in dropped.groupby("family").size().items() + } + if sample_fraction < 1.0 + else {} + ), + } + rosters = { + "constituency": tuple(map(str, np.unique(assignment.ladder.constituency_code))), + "la": tuple(map(str, np.unique(assignment.ladder.local_authority_code))), + } + if surface.empty: + problem = empty_uk_local_problem(household_index) + else: + problem = build_uk_rowwise_local_surface_matrix( + metrics, + assigned, + surface, + area_codes_by_grain={grain: rosters[grain] for grain in metrics}, + require_every_assigned_area_covered=(sample_fraction == 1.0), + ) + local_bound = tuple( + sorted( + { + f"{row.family}/{row.area_type}" + for row in surface[["family", "area_type"]] + .drop_duplicates() + .itertuples(index=False) + } + ) + ) + national_bound = tuple( + f"national/{family}" + for family in sorted({spec.family for spec in national_registry.specs}) + ) + return ( + household, + problem, + cross_grain, + (*local_bound, *national_bound), + rung_surface, + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py new file mode 100644 index 000000000..46e81f077 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py @@ -0,0 +1,186 @@ +"""Pinned national and local target inputs for the single UK full build. + +Compilation and approved measure exclusions precede geography selection. The +unreduced national register remains available for band edges, and independent +reference-period compilations remain available for release validation. +""" + +from __future__ import annotations + +from datetime import date +from pathlib import Path +from typing import Any + +from microcosm.build.ledger_artifact import load_ledger_consumer_artifact +from microcosm.build.uk_runtime.calibration_run import ( + _ledger_provenance, + _validate_band_edge_registry, +) +from microcosm.build.uk_runtime.chronicle_feed import load_uk_chronicle_feed +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release +from microcosm.build.uk_runtime.ledger_targets import ( + compile_uk_local_target_registry, + compile_uk_target_registry, + load_uk_local_area_crosswalk, +) +from microcosm.build.uk_runtime.local_target_census import _LEDGER_FACT_FEED_PIN +from microcosm.build.uk_runtime.measure_simulation import ( + apply_uk_calibration_measure_exclusions, + load_uk_calibration_measure_exclusions, +) +from microcosm.build.uk_runtime.weighted_integrity import exclusion_evaluation_date +from microcosm.calibrate import TargetRegistry + +CHRONICLE_SOURCE_CODEC = "chronicle-consumer-facts-v1" + + +def load_chronicle_source_bytes(path: Path, *, store=None) -> bytes: + """Read facts from a manifest-verified consumer artifact or a bare feed. + + Graph source identity binds every file in an artifact directory, including + the manifest. The target compiler independently enforces reviewed UK pins. + """ + + del store + load_ledger_consumer_artifact(path) + path = Path(path) + return (path / "consumer_facts.jsonl" if path.is_dir() else path).read_bytes() + + +def load_uk_local_chronicle_pin() -> dict[str, Any]: + """Return the independently reviewed local target census feed identity.""" + + return dict(_LEDGER_FACT_FEED_PIN) + + +def load_uk_full_target_inputs( + facts_path: str | Path, + *, + expected_facts_sha256: str | None = None, + expected_manifest_sha256: str | None = None, + measure_exclusions: str | Path | None = None, + register_json: str | Path | None = None, + calibration_year: int | None = None, + exclusions_evaluated_on: date | None = None, +) -> dict[str, Any]: + """Compile the full target surface with both reviewed source contracts. + + Default hashes are the reviewed national feed pins. Explicit hashes must + agree with those pins as well: a target-scope filter does not authorize a + different source. The optional frozen register compares the complete, + pre-exclusion national register, matching its completeness role. + """ + pin = load_uk_chronicle_feed() + local_pin = load_uk_local_chronicle_pin() + for field in ("facts_sha256", "manifest_sha256", "fact_row_count"): + if local_pin[field] != getattr(pin, field): + raise ValueError( + "UK full-build independently reviewed national and local feed " + f"pins disagree on {field}." + ) + for label, supplied, committed in ( + ("facts", expected_facts_sha256, pin.facts_sha256), + ("manifest", expected_manifest_sha256, pin.manifest_sha256), + ): + if supplied is not None and supplied != committed: + raise ValueError( + f"UK full-build {label} SHA differs from the committed national feed pin." + ) + artifact = load_ledger_consumer_artifact( + Path(facts_path), + expected_facts_sha256=pin.facts_sha256, + expected_manifest_sha256=pin.manifest_sha256, + ) + if ( + artifact.facts_sha256 != pin.facts_sha256 + or artifact.manifest_sha256 != pin.manifest_sha256 + ): + raise ValueError( + "UK full-build Ledger artifact differs from the national feed pin." + ) + if len(artifact.facts) != pin.fact_row_count: + raise ValueError( + "UK full-build Chronicle fact row count differs from the reviewed " + "national and local feed pins." + ) + year = ( + load_uk_frs_release().calibration_year + if calibration_year is None + else calibration_year + ) + if type(year) is not int or year <= 0: + raise ValueError("calibration_year must be a positive integer.") + evaluated_on = exclusion_evaluation_date(exclusions_evaluated_on) + crosswalk = load_uk_local_area_crosswalk() + national_registries = {} + local_registries = {} + for period in sorted({2023, 2025, year}): + compilation = compile_uk_target_registry(artifact.facts, target_period=period) + if compilation.unsupported: + raise ValueError( + f"UK national target references failed to compile for {period}: " + f"{compilation.unsupported!r}." + ) + national_registries[period] = compilation.registry + for period in sorted({2025, year}): + compilation = compile_uk_local_target_registry( + artifact.facts, target_period=period, crosswalk=crosswalk + ) + if compilation.unsupported: + raise ValueError( + f"UK local target references failed to compile for {period}: " + f"{compilation.unsupported!r}." + ) + local_registries[period] = compilation.registry + band_edges = national_registries[year] + frozen_version = None + if register_json is not None: + frozen = TargetRegistry.from_json(Path(register_json)) + frozen_version = frozen.version + if frozen.version != band_edges.version: + raise ValueError( + "Re-derived full national register differs from the frozen scoring register: " + f"{band_edges.version} vs {frozen.version}." + ) + exclusions = load_uk_calibration_measure_exclusions( + None if measure_exclusions is None else Path(measure_exclusions) + ) + national_registry, exclusion_receipt = apply_uk_calibration_measure_exclusions( + band_edges, exclusions, now=evaluated_on + ) + _validate_band_edge_registry( + register_registry=national_registry, + band_edge_registry=band_edges, + exclusion_receipt=exclusion_receipt, + ) + by_name = {spec.name: spec for spec in band_edges.specs} + reviewed_unbound = { + str(by_name[name].metadata.get("contract_target_id", name)): record + for name, record in exclusion_receipt.items() + } + return { + "artifact": artifact, + "calibration_year": year, + "national_registry": national_registry, + "band_edge_registry": band_edges, + "local_registry": local_registries[year], + "measure_exclusions": exclusion_receipt, + "reviewed_unbound_higher_targets": reviewed_unbound, + "national_source_pin": pin.to_dict(), + "local_source_pin": local_pin, + "ledger_provenance": _ledger_provenance(artifact), + "register_completeness": { + "compiled_registry_version": band_edges.version, + "approved_registry_version": national_registry.version, + "frozen_registry_version": frozen_version, + "compiled_reference_count": len(band_edges.specs), + "approved_reference_count": len(national_registry.specs), + "measure_exclusion_count": len(exclusion_receipt), + "exclusions_evaluated_on": evaluated_on.isoformat(), + "band_edge_registry_reconciled": True, + "compiled_local_reference_count": len(local_registries[year].specs), + "local_registry_version": local_registries[year].version, + }, + "uk_ledger_compiled_registries": national_registries, + "uk_ledger_compiled_local_registries": local_registries, + } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py index fb64b2917..c7062ad6c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py @@ -405,27 +405,19 @@ def load_uk_oa_ladder(path: str | Path) -> UkOaLadder: ) -def assign_uk_geography_ladder( +def draw_uk_ladder_locations( household: pd.DataFrame, ladder: UkOaLadder, *, seed: int = 0, expected_constituency_vintage: str | None = None, region_column: str = "region", -) -> pd.DataFrame: - """Assign each household one OA and the derived ladder columns. - - Two seeded draws under the build's seed discipline: a 2024 constituency is - sampled within the household's calibrated region proportional to - constituency household counts, then an OA is sampled within that - constituency proportional to 2021 Census OA population. Every finer and - coarser layer then derives from the OA, so the calibrated region marginal - is preserved exactly while every grain becomes filterable. +) -> np.ndarray: + """Draw the atomic locations using the existing sequential two-stage RNG. - Requires region assignment to have run first (the FRS carries it). A - household region absent from the ladder is an error, never a silent partial - join — for an England-&-Wales ladder that is exactly how a Scottish or - Northern Irish household is refused until those rungs are pinned. + Carries the vintage refusals (constituency on request, local authority + always; microcosm#953) so a graph node that draws locations refuses before + any derived column is written. """ if region_column not in household.columns: @@ -459,12 +451,35 @@ def assign_uk_geography_ladder( region_column=region_column, ) - assigned_index = _sample_oa_indices( + return _sample_oa_indices( region_codes.to_numpy(), ladder=ladder, seed=seed, ) + +def derive_uk_ladder_locations( + household: pd.DataFrame, + ladder: UkOaLadder, + assigned_index: np.ndarray, + *, + region_column: str = "region", +) -> pd.DataFrame: + """Derive every geography from a validated, already drawn atomic location.""" + + assigned_index = np.asarray(assigned_index) + if ( + assigned_index.shape != (len(household),) + or assigned_index.dtype.kind not in "iu" + or (assigned_index < 0).any() + or (assigned_index >= len(ladder)).any() + ): + raise ValueError("location indices must align with households and the ladder.") + region_codes = _validated_household_ladder_region_codes( + household, + ladder, + region_column=region_column, + ) assigned_region = ladder.region_code[assigned_index] mismatched = assigned_region != region_codes.to_numpy() if mismatched.any(): @@ -506,6 +521,44 @@ def assign_uk_geography_ladder( return assigned +def assign_uk_geography_ladder( + household: pd.DataFrame, + ladder: UkOaLadder, + *, + seed: int = 0, + expected_constituency_vintage: str | None = None, + region_column: str = "region", +) -> pd.DataFrame: + """Assign each household one OA and the derived ladder columns. + + Two seeded draws under the build's seed discipline: a 2024 constituency is + sampled within the household's calibrated region proportional to + constituency household counts, then an OA is sampled within that + constituency proportional to 2021 Census OA population. Every finer and + coarser layer then derives from the OA, so the calibrated region marginal + is preserved exactly while every grain becomes filterable. + + Requires region assignment to have run first (the FRS carries it). A + household region absent from the ladder is an error, never a silent partial + join — for an England-&-Wales ladder that is exactly how a Scottish or + Northern Irish household is refused until those rungs are pinned. + """ + + assigned_index = draw_uk_ladder_locations( + household, + ladder, + seed=seed, + expected_constituency_vintage=expected_constituency_vintage, + region_column=region_column, + ) + return derive_uk_ladder_locations( + household, + ladder, + assigned_index, + region_column=region_column, + ) + + def expected_uk_ladder_area_support( household: pd.DataFrame, ladder: UkOaLadder, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py new file mode 100644 index 000000000..808582ae2 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py @@ -0,0 +1,345 @@ +"""Compose the canonical UK full build on the shared executable graph. + +The same graph handles all targets (the default), explicit target filters, +dense exports and exact-count exports. A bound spine checkpoint resumes this +composition; an arbitrary historical uk-data H5 is not a build source. +""" + +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass, field, replace +from pathlib import Path + +import pandas as pd + +from microcosm.frame import Frame +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import SOURCE_CODECS + +from . import calibration_run, national_frame, release_certification +from .frs_release import load_uk_frs_release +from .graph import uk_spine_endpoint, uk_spine_graph +from .graph_calibration import ( + UKCalibrationNodes, + UKGraphCalibrationConfig, + register_uk_calibration_kernels, + uk_calibration_nodes, +) +from .graph_kernels import UKClaimKernel, UKIdentityKernel, _normalize_create_frame +from .graph_population import ( + append_uk_population_nodes, + population_columns, + population_slices, + register_uk_population_kernels, +) +from .graph_targets import append_uk_target_nodes, register_uk_target_kernels +from .local_doctrine import UK_LOCAL_CLONE_COUNT +from .national_sampling import UK_SAMPLE_SEED_DEFAULT + +SPINE_PROVENANCE_TYPE = ArtifactType("microcosm.uk.bound-spine-provenance", 1) + + +@dataclass(frozen=True) +class UKFullBuildConfig: + """Three independent controls: geography target scope, pool K, export k.""" + + calibration_year: int + time_period: str = field(default_factory=lambda: load_uk_frs_release().time_period) + source_year: int = field(default_factory=lambda: load_uk_frs_release().survey_year) + geography_levels: tuple[str, ...] | None = None + n_clones: int = UK_LOCAL_CLONE_COUNT + sample_fraction: float = 1.0 + source_sample_fraction: float = 1.0 + sample_seed: int = UK_SAMPLE_SEED_DEFAULT + seed: int = 42 + engine_blocks: int = 1 + constituency_vintage: str = "2024_pcon" + source_lineage_modulus: int | None = None + calibration: UKGraphCalibrationConfig = UKGraphCalibrationConfig() + + def __post_init__(self) -> None: + from .national_sampling import validate_sample_fraction + + validate_sample_fraction(self.sample_fraction, label="UK full pool") + validate_sample_fraction(self.source_sample_fraction, label="UK source spine") + if self.sample_fraction != 1.0 and self.source_sample_fraction != 1.0: + raise ValueError( + "A sampled spine cannot be sampled a second time in the full build." + ) + if type(self.n_clones) is not int or self.n_clones < 1: + raise ValueError("Geographic pool K must be a positive integer.") + if self.engine_blocks not in {1, self.n_clones}: + raise ValueError( + "Engine blocks must be one or equal the geographic pool K." + ) + if self.geography_levels is not None: + if not self.geography_levels or set(self.geography_levels) - { + "country", + "region", + "constituency", + "la", + }: + raise ValueError( + "Use explicit supported geography levels or omit the selector for all." + ) + if len(set(self.geography_levels)) != len(self.geography_levels): + raise ValueError("Geographic target levels must not repeat.") + if self.seed != self.calibration.seed: + raise ValueError( + "Pool and dense solve share the existing build seed; selection_seed is separate." + ) + + @property + def effective_sample_fraction(self) -> float: + return self.sample_fraction * self.source_sample_fraction + + +@dataclass(frozen=True) +class UKFullGraph: + graph: Graph + calibration: UKCalibrationNodes + config: UKFullBuildConfig + + @property + def population(self) -> str: + return self.calibration.population + + def operation_inventory(self) -> dict: + compiled = compile_graph(self.graph) + return { + "schema": "microcosm.uk.full-build-operations.v1", + "default_scope": "all_geographies", + "configuration": asdict(self.config), + "nodes": [ + { + "id": node_id, + "kernel": self.graph.node(node_id).kernel, + "description": self.graph.node(node_id).description, + "population": compiled.versions[node_id], + "dependencies": list(compiled.predecessors[node_id]), + "artifacts": [ + o.name for o in self.graph.node(node_id).artifact_outputs + ], + } + for node_id in compiled.order + ], + } + + +def uk_full_graph( + config: UKFullBuildConfig, + *, + spine: Graph | None = None, + spine_population: str | None = None, + spine_weight_kind: str = "importance", + optional_target_sources: tuple[str, ...] = (), + checkpoint_identity: dict | None = None, + review_date: str | None = None, +) -> UKFullGraph: + """Append the full build to the existing source-owned UK spine graph.""" + + initial = uk_spine_graph(source_mode="split") if spine is None else spine + endpoint = ( + uk_spine_endpoint(initial).population + if spine_population is None + else spine_population + ) + graph = append_uk_population_nodes( + initial, + population=endpoint, + time_period=config.time_period, + weight_kind=spine_weight_kind, + sample_fraction=config.sample_fraction, + sample_seed=config.sample_seed, + n_clones=config.n_clones, + seed=config.seed, + source_year=config.source_year, + constituency_vintage=config.constituency_vintage, + source_lineage_modulus=config.source_lineage_modulus, + ) + graph = append_uk_target_nodes( + graph, + calibration_year=config.calibration_year, + time_period=config.time_period, + geography_levels=config.geography_levels, + engine_blocks=config.engine_blocks, + sample_fraction=config.effective_sample_fraction, + target_weight_rule=config.calibration.target_weight_rule, + optional_sources=optional_target_sources, + review_date=review_date, + ) + cells = population_columns(graph, "uk.full.expand") + # This structural checkpoint depends on every pool operation, including + # evidence-only gates and the contribution problem. It binds the complete + # incoming ledger before downstream nodes reconstruct any Frame slices. + graph = replace( + graph, + nodes=( + *graph.nodes, + Node( + "uk.full.pool", + UKIdentityKernel.ref, + structural=StructuralDelta.FILTER, + base="uk.full.expand", + inputs=population_slices(cells), + description="Checkpoint the complete geographic pool and selected-target problem.", + ), + ), + ) + calibration = uk_calibration_nodes( + base="uk.full.pool", + columns=cells, + problem_producer="uk.full.problem", + config=config.calibration, + checkpoint_identity=checkpoint_identity, + ) + checkpoint_sources = ( + () + if checkpoint_identity is None + else ( + SourceRef("uk_size_checkpoint_manifest", "raw-bytes-v1"), + SourceRef("uk_size_checkpoint_arrays", "raw-bytes-v1"), + ) + ) + graph = replace( + graph, + nodes=(*graph.nodes, *calibration.nodes), + sources=(*graph.sources, *checkpoint_sources), + ) + compile_graph(graph) + return UKFullGraph(graph, calibration, config) + + +def _load_spine_h5(path: Path) -> Frame: + return national_frame.load_uk_national_frame(path)[0] + + +SOURCE_CODECS.register("uk-spine-h5-v1", _load_spine_h5) + + +class UKBoundSpineKernel(KernelBase): + ref = "uk.full.bound_spine@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def implementation_hash(self) -> str: + return source_hash( + type(self), national_frame, calibration_run, release_certification + ) + + def run(self, context: KernelContext) -> KernelResult: + if ( + json.loads(context.params["spine_gate_digests"]) + != calibration_run.uk_spine_checkpoint_gate_digests() + ): + raise ValueError( + "Bound spine gate declarations differ from the compiled checkpoint request." + ) + frame = _load_spine_h5(context.sources["uk_spine"]) + sidecar_path = context.sources["uk_spine_evidence"] + sidecar = calibration_run.load_bound_spine_checkpoint( + sidecar_path, frame, gate_report_path=context.sources["uk_spine_gates"] + ) + provenance = calibration_run.strict_spine_provenance_from_sidecar( + sidecar_path, sidecar, gate_report_path=context.sources["uk_spine_gates"] + ) + return KernelResult( + frame=_normalize_create_frame(frame, context), + artifacts={"spine_provenance": canonical_json(provenance)}, + ) + + +def bound_spine_graph(frame: Frame) -> Graph: + """Declare a checkpoint schema; the CREATE kernel verifies bound evidence.""" + + structural = { + "person_id", + "person_household_id", + "person_benunit_id", + "household_id", + "benunit_id", + } + + def token(dtype): + if isinstance(dtype, pd.StringDtype) or dtype.kind in "OUS": + return "string" + return str(dtype) + + outputs = tuple( + Owned(entity, str(column), token(frame.table(entity)[column].dtype)) + for entity in frame.entities + for column in frame.table(entity).columns + if column not in structural + ) + return Graph( + "uk", + ( + SourceRef( + "uk_spine", + "uk-spine-h5-v1", + "Bound canonical Microcosm spine checkpoint.", + ), + SourceRef( + "uk_spine_evidence", + "raw-bytes-v1", + "Exact source lineage, graph and gate evidence for the checkpoint.", + ), + SourceRef( + "uk_spine_gates", + "raw-bytes-v1", + "Exact gate report bound by the canonical spine sidecar.", + ), + ), + ( + Node( + "uk.full.spine_checkpoint", + UKBoundSpineKernel.ref, + structural=StructuralDelta.CREATE, + sources=("uk_spine", "uk_spine_evidence", "uk_spine_gates"), + params={ + "spine_gate_digests": canonical_json( + calibration_run.uk_spine_checkpoint_gate_digests() + ).decode() + }, + outputs=outputs, + artifact_outputs=( + ArtifactOutput("spine_provenance", SPINE_PROVENANCE_TYPE), + ), + description="Resume a canonical spine with its bound lineage and source evidence.", + ), + ), + ) + + +def register_uk_full_kernels(registry: KernelRegistry) -> KernelRegistry: + """Extend the existing UK source/stage registry with the full build.""" + + # Source graph registries already contain these primitive kernels. + for kernel in (UKBoundSpineKernel(), UKIdentityKernel(), UKClaimKernel()): + try: + registry.get(kernel.ref) + except KeyError: + registry.register(kernel) + register_uk_population_kernels(registry) + register_uk_target_kernels(registry) + register_uk_calibration_kernels(registry) + return registry diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py new file mode 100644 index 000000000..e0793606d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py @@ -0,0 +1,862 @@ +"""UK solver bindings over shared, ordered calibration artifacts. + +Every solver consumes the original pool. Dense weights are not a selection +prior; filtering and installing the completed solution are separate structural +operations. Completed search and draw artifacts resume without repeating RNG. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import asdict, dataclass, replace + +import numpy as np +import pandas as pd + +from microcosm.calibrate import artifacts as calibration_artifacts +from microcosm.calibrate import calibrate +from microcosm.calibrate.artifacts import ( + PROBLEM_TYPE, + RESULT_TYPE, + SOLUTION_TYPE, + decode_calibration_result, + decode_problem, + decode_solution, + encode_calibration_result, + encode_problem, + encode_solution, +) +from microcosm.calibrate.exact_k import select_exact_k +from microcosm.calibrate.gates import HardConcrete +from microcosm.calibrate.initialization import contribution_initialization +from microcosm.calibrate.kernels import CalibrateAdamKernel +from microcosm.frame import Frame, MassChangeRecord, WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + SeedSource, + StructuralDelta, + WeightTransition, + source_hash, +) +from microcosm.graph.canonical import canonical_json + +from . import dataset_size, size_checkpoint +from .dataset_size import UKSizeDraw, UKSizeSelection +from .graph_population import context_frame, population_slices +from .local_doctrine import UK_LOCAL_SOLVE_EPOCHS + +SIZE_SEARCH_TYPE = ArtifactType("microcosm.uk.size-search", 1) +SIZE_DRAW_TYPE = ArtifactType("microcosm.uk.size-draw", 1) +SIZE_RECEIPT_TYPE = ArtifactType("microcosm.uk.size-receipt", 1) +_DEPENDENCIES = ("numpy", "pandas", "scipy", "torch") + + +@dataclass(frozen=True) +class UKGraphCalibrationConfig: + """Solve and sizing settings, each a parameter of the node that uses it. + + ``baseline_pi_floor`` (#921) trims the refit's Horvitz–Thompson baseline + (see :func:`~microcosm.build.uk_runtime.dataset_size.refit_uk_dataset_size`); + it is a parameter of the size-refit node and of the rotated holdout that + resizes with it, never of the search or the draw, so a resumed selection + may refit under a different floor. ``0.0`` is the untrimmed default. + """ + + epochs: int = UK_LOCAL_SOLVE_EPOCHS + learning_rate: float = 0.15 + seed: int = 42 + dataset_households: int | None = None + selection_seed: int | None = None + selection_pi_hi: float = 1.0 + baseline_pi_floor: float = 0.0 + target_weight_rule: str = "uniform" + + def __post_init__(self): + if type(self.epochs) is not int or self.epochs < 1: + raise ValueError("Calibration epochs must be a positive integer.") + if not np.isfinite(self.learning_rate) or self.learning_rate <= 0: + raise ValueError("Calibration learning rate must be positive and finite.") + for seed in (self.seed, self.selection_seed): + if seed is not None and (type(seed) is not int or seed < 0): + raise ValueError("Calibration seeds must be nonnegative integers.") + if self.dataset_households is not None and ( + type(self.dataset_households) is not int or self.dataset_households < 1 + ): + raise ValueError("Dataset household count must be a positive integer.") + dataset_size._check_pi_hi(self.selection_pi_hi) + dataset_size._check_baseline_pi_floor(self.baseline_pi_floor) + + +@dataclass(frozen=True) +class UKCalibrationNodes: + nodes: tuple[Node, ...] + population: str + result_producer: str + problem_producer: str + solution_producer: str + size_producer: str | None + dense_producer: str + + +def _axis(frame: Frame) -> list: + return frame.table("household")["household_id"].tolist() + + +def _inputs(context: KernelContext): + frame = context_frame(context) + problem = decode_problem(context.artifacts["problem"].payload) + if ( + tuple(_axis(frame)) != problem.entity_ids + or problem.problem.weight_entity != "household" + ): + raise ValueError("UK solve requires its exact original household axis.") + weights = frame.weights_for("household") + if weights.kind != problem.problem.initial_weights.kind or not np.array_equal( + weights.values, problem.problem.initial_weights.values + ): + raise ValueError("UK solve requires the original pool weights.") + if problem.problem.skipped: + raise ValueError("UK solve cannot silently omit selected targets.") + return frame, problem + + +def _dense(context, frame, problem): + return decode_calibration_result( + context.artifacts["dense"].payload, frame=frame, problem=problem + ) + + +def _solution(result, frame, problem): + if result.frame.mass_log[: len(frame.mass_log)] != frame.mass_log: + raise ValueError("Calibration replaced the original pool mass ledger.") + return encode_solution( + result.weights, + entity_ids=_axis(result.frame), + problem_sha256=problem.sha256, + diagnostics={ + "frame_mass_log_append": [ + asdict(record) + for record in result.frame.mass_log[len(frame.mass_log) :] + ] + }, + ) + + +def restore_uk_graph_result( + pool: Frame, + *, + problem_payload: bytes, + result_payload: bytes, + solution_payload: bytes, + original_problem_payload: bytes | None = None, +): + """Rebuild completed diagnostics and exact legacy mass evidence, never solve. + + A compact result has its own HT-normalized initial weights and matrix. + Its original problem is required to authenticate the supplied full pool. + ``solution_payload`` is the original-bound installation solution. + """ + problem = decode_problem(problem_payload) + original = ( + problem + if original_problem_payload is None + else decode_problem(original_problem_payload) + ) + if ( + tuple(_axis(pool)) != original.entity_ids + or not np.array_equal( + pool.weights_for("household").values, + original.problem.initial_weights.values, + ) + or pool.weights_for("household").kind != original.problem.initial_weights.kind + ): + raise ValueError("Completed result requires its authenticated original pool.") + if ( + problem.sha256 != original.sha256 + and problem.bindings.get("original_problem_sha256") != original.sha256 + ): + raise ValueError( + "Compact result is not bound to the supplied original problem." + ) + solution = decode_solution( + solution_payload, problem_sha256=original.sha256, entity_ids=problem.entity_ids + ) + ids = set(problem.entity_ids) + selected = pool.select( + pool.table("person")["person_household_id"].isin(ids).to_numpy() + ) + if tuple(_axis(selected)) != problem.entity_ids: + raise ValueError("Completed result is not an ordered subset of the pool.") + initial_frame = Frame( + {e: selected.table(e) for e in selected.entities}, + selected.schema, + {"household": problem.problem.initial_weights}, + selected.strata, + mass_log=pool.mass_log, + metadata=pool.metadata, + ) + result = decode_calibration_result( + result_payload, frame=initial_frame, problem=problem + ) + if not np.array_equal(result.weights, solution.weights): + raise ValueError("Completed result disagrees with its installed solution.") + records = tuple( + MassChangeRecord(**dict(row)) + for row in solution.diagnostics["frame_mass_log_append"] + ) + if records and ( + not np.isclose(records[0].old_total, selected.weights_for("household").total) + or not np.isclose(records[-1].new_total, float(result.weights.sum())) + ): + raise ValueError("Completed result mass evidence differs from its boundary.") + final_frame = Frame( + {e: selected.table(e) for e in selected.entities}, + selected.schema, + {"household": result.frame.weights_for("household")}, + selected.strata, + mass_log=(*pool.mass_log, *records), + metadata=pool.metadata, + ) + return replace(result, frame=final_frame) + + +class _CalibrationKernel(KernelBase): + def implementation_hash(self): + return hashlib.sha256( + canonical_json( + { + "solver": CalibrateAdamKernel().implementation_hash(), + "adapter": source_hash( + type(self), + dataset_size, + size_checkpoint, + calibration_artifacts, + context_frame, + select_exact_k, + HardConcrete, + contribution_initialization, + dependencies=self.capabilities.dependencies, + ), + } + ) + ).hexdigest() + + +class UKSizeCheckpointImportKernel(_CalibrationKernel): + """Authenticate existing external checkpoints as explicit graph sources.""" + + ref = "uk.full.size_checkpoint_import@1" + capabilities = Capabilities(Determinism.DETERMINISTIC, dependencies=_DEPENDENCIES) + + def run(self, context): + frame, problem = _inputs(context) + manifest_path = context.sources[context.params["manifest_source"]] + arrays_path = context.sources[context.params["arrays_source"]] + identity = json.loads(context.params["identity_json"]) + stored = json.loads(manifest_path.read_bytes()) + # The historical loader accepts requested subsets. Graph import must + # authenticate every stored caller pin rather than leave old source, + # selector or K settings outside the comparison. + if set(identity) != set(stored.get("identity", {})): + raise ValueError( + "Size checkpoint import requires every original identity field." + ) + restored = size_checkpoint.load_uk_size_checkpoint_files( + manifest_path, + arrays_path, + frame=frame, + target_set=problem.to_target_set(), + identity=identity, + ) + dense, selection = restored.dense, restored.selection + for key in ("epochs", "learning_rate"): + if dense.options[key] != context.params[key]: + raise ValueError(f"Imported dense solve has different {key}.") + if dense.options["seed"] != context.params["dense_seed"]: + raise ValueError("Imported dense solve has a different seed.") + for key in ("households", "epochs", "learning_rate", "seed"): + if getattr(selection, key) != context.params[key]: + raise ValueError(f"Imported size search has different {key}.") + binding = problem.bindings + if ( + dense.options["mass"] != "free" + or dense.options["mass_reason"] != binding["mass_reason"] + or dense.options["max_weight_ratio"] != binding["max_weight_ratio"] + or dense.target_loss_cap != binding["target_loss_cap"] + or not np.array_equal( + dense.target_loss_weights, binding["target_loss_weights"] + ) + ): + raise ValueError("Imported dense solve has a different solve doctrine.") + metadata = { + "method": "contribution_informed_l0", + "problem_sha256": problem.sha256, + "protected": selection.protected.tolist(), + "households": selection.households, + "epochs": selection.epochs, + "learning_rate": selection.learning_rate, + "seed": selection.seed, + "pi_hi": selection.search_pi_hi, + } + return KernelResult( + artifacts={ + "dense": encode_calibration_result( + dense, entity_ids=problem.entity_ids, problem_sha256=problem.sha256 + ), + "search": encode_calibration_result( + selection.selection, + entity_ids=problem.entity_ids, + problem_sha256=problem.sha256, + ), + "selection": canonical_json(metadata), + } + ) + + +class UKDenseSolveKernel(_CalibrationKernel): + ref = "uk.full.dense@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + seed_source=SeedSource.PARAM, + dependencies=_DEPENDENCIES, + ) + + def run(self, context): + from .graph_terminal import decode_full_gate_report + + if "preflight" not in context.artifacts: + raise ValueError( + "Dense calibration requires its source preflight artifact." + ) + report, classification = decode_full_gate_report( + context.artifacts["preflight"].payload + ) + if report.phase != "preflight": + raise ValueError("Dense calibration requires a preflight phase report.") + if not classification["artifact_permitted"]: + raise ValueError("Dense calibration refused by the source preflight.") + frame, problem = _inputs(context) + if "imported_dense" in context.artifacts: + result = decode_calibration_result( + context.artifacts["imported_dense"].payload, + frame=frame, + problem=problem, + ) + return KernelResult( + artifacts={ + "result": context.artifacts["imported_dense"].payload, + "solution": _solution(result, frame, problem), + } + ) + binding = problem.bindings + # These are the maintained full-build doctrine values carried by the + # selected problem, regardless of the geographic scope of its rows. + required = { + "mass_reason", + "max_weight_ratio", + "target_loss_weights", + "target_loss_cap", + } + if not required <= set(binding): + raise ValueError("UK ordered problem is missing its solve doctrine.") + result = calibrate( + frame, + problem.to_target_set(), + weight_entity="household", + epochs=context.params["epochs"], + learning_rate=context.params["learning_rate"], + seed=context.params["seed"], + mass="free", + mass_reason=binding["mass_reason"], + max_weight_ratio=binding["max_weight_ratio"], + target_loss_weights=np.asarray( + binding["target_loss_weights"], dtype=np.float64 + ), + target_loss_cap=binding["target_loss_cap"], + ) + return KernelResult( + artifacts={ + "result": encode_calibration_result( + result, entity_ids=problem.entity_ids, problem_sha256=problem.sha256 + ), + "solution": _solution(result, frame, problem), + } + ) + + +def _full_pool(frame, context): + k = context.params["households"] + if k > frame.n("household"): + raise ValueError( + "Requested dataset size exceeds the original pool; never clamped." + ) + return k == frame.n("household") + + +class UKSizeSearchKernel(_CalibrationKernel): + ref = "uk.full.size_search@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + seed_source=SeedSource.PARAM, + dependencies=_DEPENDENCIES, + ) + + def run(self, context): + frame, problem = _inputs(context) + dense = _dense(context, frame, problem) + if "search" in context.artifacts: + _selection(context, frame, problem) + return KernelResult( + artifacts={ + "result": context.artifacts["search"].payload, + "selection": context.artifacts["selection"].payload, + } + ) + if _full_pool(frame, context): + metadata = { + "method": "full_pool", + "problem_sha256": problem.sha256, + **dict(context.params), + } + result_payload = context.artifacts["dense"].payload + else: + selection = dataset_size.select_uk_dataset_size( + frame, dense, **dict(context.params) + ) + metadata = { + "method": "contribution_informed_l0", + "problem_sha256": problem.sha256, + "protected": selection.protected.tolist(), + **dict(context.params), + } + result_payload = encode_calibration_result( + selection.selection, + entity_ids=problem.entity_ids, + problem_sha256=problem.sha256, + ) + return KernelResult( + artifacts={"result": result_payload, "selection": canonical_json(metadata)} + ) + + +def _selection(context, frame, problem): + metadata = json.loads(context.artifacts["selection"].payload) + if metadata["problem_sha256"] != problem.sha256: + raise ValueError("Size search belongs to a different ordered problem.") + for key in ("households", "epochs", "learning_rate", "seed"): + if metadata[key] != context.params[key]: + raise ValueError(f"Size search has a different {key}.") + if metadata["method"] == "full_pool": + if not _full_pool(frame, context): + raise ValueError("Full-pool receipt used for a compact request.") + return None + if metadata["method"] != "contribution_informed_l0": + raise ValueError("Unknown UK size-search method.") + raw_protected = np.asarray(metadata["protected"]) + if raw_protected.dtype != np.bool_ or raw_protected.shape != ( + frame.n("household"), + ): + raise ValueError("Size search protected mask is not aligned.") + return UKSizeSelection( + decode_calibration_result( + context.artifacts["search"].payload, frame=frame, problem=problem + ), + raw_protected, + metadata["households"], + metadata["epochs"], + metadata["learning_rate"], + metadata["seed"], + metadata["pi_hi"], + ) + + +class UKSizeDrawKernel(_CalibrationKernel): + ref = "uk.full.size_draw@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + seed_source=SeedSource.PARAM, + dependencies=_DEPENDENCIES, + ) + + def run(self, context): + frame, problem = _inputs(context) + dense = _dense(context, frame, problem) + selection = _selection(context, frame, problem) + if selection is None: + draw = {"method": "full_pool", "support": list(range(frame.n("household")))} + else: + result = dataset_size.draw_uk_dataset_size( + frame, + dense, + selection=selection, + households=context.params["households"], + seed=context.params["seed"], + pi_hi=context.params["pi_hi"], + ) + draw = {"method": "exact_count", **asdict(result)} + for key in ("support", "inclusion_probabilities"): + draw[key] = draw[key].tolist() + return KernelResult( + artifacts={ + "draw": canonical_json({"problem_sha256": problem.sha256, **draw}) + } + ) + + +class UKSizeRefitKernel(_CalibrationKernel): + ref = "uk.full.size_refit@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + seed_source=SeedSource.PARAM, + dependencies=_DEPENDENCIES, + ) + + def run(self, context): + frame, problem = _inputs(context) + dense = _dense(context, frame, problem) + selection = _selection(context, frame, problem) + draw = json.loads(context.artifacts["draw"].payload) + if draw.pop("problem_sha256") != problem.sha256: + raise ValueError("Exact-count draw belongs to another ordered problem.") + method = draw.pop("method") + if selection is None: + if method != "full_pool" or draw["support"] != list( + range(frame.n("household")) + ): + raise ValueError("Full-pool draw has a different support.") + cached_draw = None + else: + if method != "exact_count": + raise ValueError("Compact refit requires a completed exact-count draw.") + cached_draw = UKSizeDraw( + **{ + **draw, + "support": np.asarray(draw["support"]), + "inclusion_probabilities": np.asarray( + draw["inclusion_probabilities"], dtype=np.float64 + ), + } + ) + compact = dataset_size.refit_uk_dataset_size( + frame, dense, selection=selection, draw=cached_draw, **dict(context.params) + ) + result = compact.result + ids = _axis(result.frame) + if selection is None: + compact_payload = context.artifacts["problem"].payload + result_payload = context.artifacts["dense"].payload + compact_problem = problem + else: + compact_payload = encode_problem( + result.problem, + entity_ids=ids, + target_metadata=problem.target_metadata, + bindings={ + **dict(problem.bindings), + "original_problem_sha256": problem.sha256, + }, + ) + compact_problem = decode_problem(compact_payload) + result_payload = encode_calibration_result( + result, entity_ids=ids, problem_sha256=compact_problem.sha256 + ) + return KernelResult( + artifacts={ + "problem": compact_payload, + "result": result_payload, + "solution": _solution(result, frame, problem), + "refit_solution": _solution(result, frame, compact_problem), + "size": canonical_json( + { + **compact.receipt, + "problem_sha256": problem.sha256, + "household_ids": ids, + } + ), + } + ) + + +class UKSizeFilterKernel(_CalibrationKernel): + ref = "uk.full.selected@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.FILTER, + dependencies=_DEPENDENCIES, + ) + + def run(self, context): + frame = context_frame(context) + problem = decode_problem(context.artifacts["problem"].payload) + solution = decode_solution( + context.artifacts["solution"].payload, problem_sha256=problem.sha256 + ) + selected = set(solution.entity_ids) + if [i for i in _axis(frame) if i in selected] != list(solution.entity_ids): + raise ValueError("Selected household IDs are not an ordered pool subset.") + person = frame.table("person") + keep = pd.Series( + person["person_household_id"].isin(selected).to_numpy(), + index=pd.Index(person["person_id"], name="person_id"), + dtype=bool, + ) + return KernelResult(keep=keep) + + +class UKInstallCalibrationKernel(_CalibrationKernel): + ref = "uk.full.calibrated@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.REWEIGHT, + dependencies=_DEPENDENCIES, + ) + + def run(self, context): + frame = context_frame(context) + problem = decode_problem(context.artifacts["problem"].payload) + solution = decode_solution( + context.artifacts["solution"].payload, + problem_sha256=problem.sha256, + entity_ids=_axis(frame), + ) + return KernelResult( + weights=Weights(solution.weights, kind=WeightKind.CALIBRATED), + receipt={ + "frame_mass_log_append": solution.diagnostics["frame_mass_log_append"] + }, + ) + + +def uk_calibration_nodes( + *, + base: str, + columns: Mapping[tuple[str, str], str], + problem_producer: str, + problem_artifact: str = "problem", + prefix: str = "uk.full", + config: UKGraphCalibrationConfig | None = None, + checkpoint_identity: Mapping | None = None, + checkpoint_sources: tuple[str, str] = ( + "uk_size_checkpoint_manifest", + "uk_size_checkpoint_arrays", + ), +) -> UKCalibrationNodes: + """Compose the same numerical route for every selected target scope.""" + config = UKGraphCalibrationConfig() if config is None else config + inputs = population_slices(columns) + problem_input = ArtifactInput( + "problem", problem_producer, problem_artifact, PROBLEM_TYPE + ) + dense_id = f"{prefix}.dense" + checkpoint_id = f"{prefix}.size_checkpoint_import" + imported_dense = ( + () + if checkpoint_identity is None + else (ArtifactInput("imported_dense", checkpoint_id, "dense", RESULT_TYPE),) + ) + nodes = [ + Node( + id=dense_id, + kernel=UKDenseSolveKernel.ref, + population=base, + inputs=inputs, + params={ + "epochs": config.epochs, + "learning_rate": config.learning_rate, + "seed": config.seed, + }, + artifact_inputs=(problem_input, *imported_dense), + artifact_outputs=( + ArtifactOutput("result", RESULT_TYPE), + ArtifactOutput("solution", SOLUTION_TYPE), + ), + ) + ] + if checkpoint_identity is not None: + if config.dataset_households is None: + raise ValueError( + "Importing a size checkpoint requires an explicit dataset size." + ) + nodes.insert( + 0, + Node( + id=checkpoint_id, + kernel=UKSizeCheckpointImportKernel.ref, + population=base, + inputs=inputs, + sources=checkpoint_sources, + params={ + "identity_json": canonical_json(checkpoint_identity).decode(), + "manifest_source": checkpoint_sources[0], + "arrays_source": checkpoint_sources[1], + "epochs": config.epochs, + "learning_rate": config.learning_rate, + "dense_seed": config.seed, + "seed": config.seed + if config.selection_seed is None + else config.selection_seed, + "households": config.dataset_households, + }, + artifact_inputs=(problem_input,), + artifact_outputs=( + ArtifactOutput("dense", RESULT_TYPE), + ArtifactOutput("search", RESULT_TYPE), + ArtifactOutput("selection", SIZE_SEARCH_TYPE), + ), + ), + ) + solution_producer = result_producer = dense_id + result_problem_producer = problem_producer + size_producer = None + final_base = base + if config.dataset_households is not None: + params = { + "epochs": config.epochs, + "learning_rate": config.learning_rate, + "seed": config.seed + if config.selection_seed is None + else config.selection_seed, + "households": config.dataset_households, + "pi_hi": config.selection_pi_hi, + } + dense_input = ArtifactInput("dense", dense_id, "result", RESULT_TYPE) + search_id, draw_id, refit_id = ( + f"{prefix}.{s}" for s in ("size_search", "size_draw", "size_refit") + ) + nodes.append( + Node( + id=search_id, + kernel=UKSizeSearchKernel.ref, + population=base, + inputs=inputs, + params=params, + artifact_inputs=( + problem_input, + dense_input, + *( + () + if checkpoint_identity is None + else ( + ArtifactInput( + "search", checkpoint_id, "search", RESULT_TYPE + ), + ArtifactInput( + "selection", + checkpoint_id, + "selection", + SIZE_SEARCH_TYPE, + ), + ) + ), + ), + artifact_outputs=( + ArtifactOutput("result", RESULT_TYPE), + ArtifactOutput("selection", SIZE_SEARCH_TYPE), + ), + ) + ) + search_inputs = ( + problem_input, + dense_input, + ArtifactInput("search", search_id, "result", RESULT_TYPE), + ArtifactInput("selection", search_id, "selection", SIZE_SEARCH_TYPE), + ) + nodes.append( + Node( + id=draw_id, + kernel=UKSizeDrawKernel.ref, + population=base, + inputs=inputs, + params=params, + artifact_inputs=search_inputs, + artifact_outputs=(ArtifactOutput("draw", SIZE_DRAW_TYPE),), + ) + ) + nodes.append( + Node( + id=refit_id, + kernel=UKSizeRefitKernel.ref, + population=base, + inputs=inputs, + params={**params, "baseline_pi_floor": config.baseline_pi_floor}, + artifact_inputs=( + *search_inputs, + ArtifactInput("draw", draw_id, "draw", SIZE_DRAW_TYPE), + ), + artifact_outputs=( + ArtifactOutput("result", RESULT_TYPE), + ArtifactOutput("problem", PROBLEM_TYPE), + ArtifactOutput("solution", SOLUTION_TYPE), + ArtifactOutput("refit_solution", SOLUTION_TYPE), + ArtifactOutput("size", SIZE_RECEIPT_TYPE), + ), + ) + ) + solution_producer = result_producer = result_problem_producer = ( + size_producer + ) = refit_id + final_base = f"{prefix}.selected" + nodes.append( + Node( + id=final_base, + kernel=UKSizeFilterKernel.ref, + base=base, + inputs=inputs, + structural=StructuralDelta.FILTER, + mass="free", + artifact_inputs=( + problem_input, + ArtifactInput("solution", refit_id, "solution", SOLUTION_TYPE), + ), + ) + ) + calibrated = f"{prefix}.calibrated" + nodes.append( + Node( + id=calibrated, + kernel=UKInstallCalibrationKernel.ref, + base=final_base, + inputs=inputs, + structural=StructuralDelta.REWEIGHT, + mass="free", + weights=WeightTransition("household", "calibrated", mass="free"), + artifact_inputs=( + problem_input, + ArtifactInput("solution", solution_producer, "solution", SOLUTION_TYPE), + ), + ) + ) + return UKCalibrationNodes( + tuple(nodes), + calibrated, + result_producer, + result_problem_producer, + solution_producer, + size_producer, + dense_id, + ) + + +def register_uk_calibration_kernels(registry: KernelRegistry) -> KernelRegistry: + for kernel in ( + UKSizeCheckpointImportKernel, + UKDenseSolveKernel, + UKSizeSearchKernel, + UKSizeDrawKernel, + UKSizeRefitKernel, + UKSizeFilterKernel, + UKInstallCalibrationKernel, + ): + registry.register(kernel()) + return registry diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py new file mode 100644 index 000000000..683092dc3 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py @@ -0,0 +1,564 @@ +"""Population operations in the single UK full-build graph. + +Source interpretation stays in UK adapters. Selection, row ancestry, typed +weights, content storage and replay are enforced by the shared graph runtime. +The legacy numerical functions remain the only implementations of the draws. +""" + +from __future__ import annotations + +import importlib.util +import json +from collections.abc import Mapping +from dataclasses import asdict, replace + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Owned, + SeedSource, + Slice, + SourceRef, + StructuralDelta, + WeightUpdate, + compile_graph, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import dtype_for_token +from microcosm.graph.weight_update import weight_update_receipt + +from . import geography_ladder, national_sampling, rowwise_dataset +from .geography_ladder import ( + UK_GEOGRAPHY_LADDER_COLUMNS, + derive_uk_ladder_locations, + draw_uk_ladder_locations, + load_uk_oa_ladder, +) +from .local_authority_input import verify_local_authority_engine_domain +from .national_frame import UK_NATIONAL_SCHEMA +from .rowwise_dataset import expand_uk_geographic_pool, ladder_clone_index_column + +POPULATION_RECEIPT_TYPE = ArtifactType("microcosm.uk.population-receipt", 1) +LOCATION_DRAW_TYPE = ArtifactType("microcosm.uk.ladder-location-draw", 1) +GEOGRAPHY_GATE_TYPE = ArtifactType("microcosm.uk.geography-gate", 1) + + +def context_frame(context: KernelContext) -> Frame: + """Rebuild only declared slices, with the shared immutable weight context.""" + + return Frame( + { + entity: context.tables[entity] + .loc[ + :, + list( + context.frame_column_order.get( + entity, tuple(context.tables[entity].columns) + ) + ), + ] + .copy(deep=True) + for entity in UK_NATIONAL_SCHEMA.entities + }, + UK_NATIONAL_SCHEMA, + {"household": context.weights["household"]}, + context.strata.copy(deep=True), + mass_log=getattr(context, "frame_mass_log", ()), + metadata=getattr(context, "frame_metadata", {}) + or {"time_period": str(context.params["time_period"])}, + ) + + +def population_columns(graph: Graph, population: str) -> dict[tuple[str, str], str]: + """Resolve the carried and newly owned columns in a compiled version.""" + + compiled = compile_graph(graph) + holder = graph.node(population) + cells = {} if holder.base is None else population_columns(graph, holder.base) + for node in graph.nodes: + if compiled.versions[node.id] == population: + cells.update({(o.entity, o.column): o.dtype for o in node.outputs}) + return cells + + +def population_slices(cells: Mapping[tuple[str, str], str]) -> tuple[Slice, ...]: + return tuple( + Slice(entity, tuple(sorted(c for e, c in cells if e == entity))) + for entity in sorted({e for e, _ in cells}) + ) + + +def _series(frame: Frame, entity: str, column: str, dtype: str) -> pd.Series: + table = frame.table(entity) + ids = pd.Index( + table[frame.schema.entity_id_column(entity)], + name=frame.schema.entity_id_column(entity), + ) + return pd.Series(table[column].array, index=ids, name=column).astype( + dtype_for_token(dtype) + ) + + +def _mass_records(before: Frame, after: Frame) -> list[dict]: + if after.mass_log[: len(before.mass_log)] != before.mass_log: + raise ValueError( + "A graph population operation replaced its incoming mass ledger." + ) + return [asdict(record) for record in after.mass_log[len(before.mass_log) :]] + + +def _declared_mass(before: Frame, after: Frame) -> dict: + old, new = before.stratum_mass(), after.stratum_mass() + return { + "policy": "declared", + "before": float(old.sum()), + "after": float(new.sum()), + "stratum_before": {k: float(v) for k, v in old.items()}, + "stratum_after": {k: float(v) for k, v in new.items()}, + } + + +class _PopulationKernel(KernelBase): + def implementation_hash(self) -> str: + from . import graph_evidence + + return source_hash( + type(self), + geography_ladder, + national_sampling, + rowwise_dataset, + graph_evidence, + ) + + +class UKFamilySampleKernel(_PopulationKernel): + ref = "uk.full.sample@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + seed_source=SeedSource.PARAM, + structural=StructuralDelta.FILTER, + ) + + def run(self, context: KernelContext) -> KernelResult: + from .graph_evidence import require_uk_spine_gate_admission + + require_uk_spine_gate_admission(context) + before = context_frame(context) + fraction = float(context.params["fraction"]) + seed = int(context.params["seed"]) + if fraction == 1.0: + after = before + receipt = { + "fraction": fraction, + "seed": seed, + "sampled": False, + "pre_household_count": before.n("household"), + "post_household_count": before.n("household"), + "rung_token": "f100", + } + else: + after, receipt = national_sampling.sample_uk_spine_frame( + before, fraction=fraction, seed=seed + ) + receipt = {"sampled": True, **receipt} + ids = before.table("person")["person_id"] + keep = pd.Series( + ids.isin(after.table("person")["person_id"]).to_numpy(), + index=pd.Index(ids, name="person_id"), + dtype=bool, + ) + # The sampling function computes the historical normalization once. + # Installing its weights is a separate declared same-kind operation. + payload = { + "receipt": receipt, + "household_ids": after.table("household")["household_id"].tolist(), + "weights": after.weights_for("household").values.tolist(), + "mass_log_append": _mass_records(before, after), + } + return KernelResult(keep=keep, artifacts={"sampling": canonical_json(payload)}) + + +class UKSampleNormalizationKernel(_PopulationKernel): + ref = "uk.full.normalize@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.REWEIGHT + ) + + def run(self, context: KernelContext) -> KernelResult: + frame = context_frame(context) + payload = json.loads(context.artifacts["sampling"].payload) + ids = frame.table("household")["household_id"].tolist() + if ids != payload["household_ids"]: + raise ValueError("Sample normalization household axis changed.") + weights = Weights( + np.asarray(payload["weights"], dtype=np.float64), + kind=context.weights["household"].kind, + ) + after = Frame( + {e: frame.table(e) for e in frame.entities}, + frame.schema, + {"household": weights}, + frame.strata, + metadata=frame.metadata, + ) + return KernelResult( + weights=weights, + receipt={ + "weight_update": weight_update_receipt(ids), + "mass": _declared_mass(frame, after), + "frame_mass_log_append": payload["mass_log_append"], + }, + ) + + +class UKGeographicExpansionKernel(_PopulationKernel): + ref = "uk.full.expand@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.EXPAND + ) + + def run(self, context: KernelContext) -> KernelResult: + before = context_frame(context) + household = before.table("household").copy() + household["household_weight"] = before.weights_for("household").values + pool = expand_uk_geographic_pool( + person=before.table("person"), + benunit=before.table("benunit"), + household=household, + n_clones=int(context.params["n_clones"]), + source_year=int(context.params["source_year"]), + time_period=str(context.params["time_period"]), + household_weight_kind=before.weights_for("household").kind, + mass_log=before.mass_log, + source_lineage_modulus=context.params.get("source_lineage_modulus"), + ) + after = pool.frame + ancestry = {} + for entity in before.entities: + table = after.table(entity) + id_column = before.schema.entity_id_column(entity) + added = table.iloc[before.n(entity) :] + source_ids = ( + added[id_column].to_numpy() + - added[ladder_clone_index_column(entity)].to_numpy() + * pool.id_multiplier + ) + ancestry[entity] = pd.Series( + source_ids, index=pd.Index(added[id_column], name=id_column) + ) + columns = { + (e, c): _series(after, e, c, dtype) + for e, c, dtype in context.params["expand_cells"] + } + return KernelResult( + expand=ancestry, + columns=columns, + weights=after.weights_for("household"), + artifacts={ + "expansion": canonical_json( + {"n_clones": pool.n_clones, "id_multiplier": pool.id_multiplier} + ) + }, + receipt={"frame_mass_log_append": _mass_records(before, after)}, + ) + + +class UKLocationDrawKernel(_PopulationKernel): + ref = "uk.full.locations@1" + capabilities = Capabilities(Determinism.DETERMINISTIC, seed_source=SeedSource.PARAM) + + def run(self, context: KernelContext) -> KernelResult: + ladder = load_uk_oa_ladder(context.sources["uk_ladder"]) + household = context.tables["household"] + indices = draw_uk_ladder_locations( + household, + ladder, + seed=int(context.params["seed"]), + expected_constituency_vintage=str(context.params["constituency_vintage"]), + ) + return KernelResult( + artifacts={ + "locations": canonical_json( + { + "household_ids": household["household_id"].tolist(), + "indices": indices.tolist(), + "seed": int(context.params["seed"]), + "layer_vintages": ladder.layer_vintages, + "constituency_sampling_basis": ladder.metadata[ + "constituency_sampling_basis" + ], + "oa_sampling_basis": ladder.metadata["oa_sampling_basis"], + } + ) + } + ) + + +class UKGeographyMappingKernel(_PopulationKernel): + ref = "uk.full.geography_mapping@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def run(self, context: KernelContext) -> KernelResult: + payload = json.loads(context.artifacts["locations"].payload) + household = context.tables["household"] + if household["household_id"].tolist() != payload["household_ids"]: + raise ValueError("Location draw is bound to a different household axis.") + assigned = derive_uk_ladder_locations( + household, + load_uk_oa_ladder(context.sources["uk_ladder"]), + np.asarray(payload["indices"], dtype=np.int64), + ) + index = pd.Index(household["household_id"], name="household_id") + return KernelResult( + columns={ + (owned.entity, owned.column): pd.Series( + assigned[owned.column].array, index=index, name=owned.column + ).astype(dtype_for_token(owned.dtype)) + for owned in context.node.outputs + } + ) + + +class UKGeographyGateKernel(_PopulationKernel): + ref = "uk.full.geography_gate@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def run(self, context: KernelContext) -> KernelResult: + frame = context_frame(context) + household = frame.table("household") + gate = geography_ladder.uk_geography_ladder_gate( + household, frame.weights_for("household").values + ) + # Fail closed against the installed engine (microcosm#953): every + # assigned local_authority member name must exist in this pin's + # LocalAuthority enum. Only assigned keys are checked so a toy ladder + # stays evaluable and an engine-free lane never imports the engine. + if gate.passed and importlib.util.find_spec("policyengine_uk") is not None: + verify_local_authority_engine_domain(household["local_authority"].unique()) + # Persist the failure. Downstream target materialization explicitly + # refuses this outcome; cached failure must never turn into a pass. + payload = asdict(gate) + return KernelResult( + artifacts={"gate": canonical_json(payload)}, + receipt={"outcome": "pass" if gate.passed else "fail", "evidence": payload}, + ) + + +def append_uk_population_nodes( + graph: Graph, + *, + population: str, + time_period: str, + weight_kind: str, + sample_fraction: float = 1.0, + sample_seed: int = national_sampling.UK_SAMPLE_SEED_DEFAULT, + n_clones: int = 1, + seed: int = 42, + source_year: int = 2024, + constituency_vintage: str = "2024_pcon", + source_lineage_modulus: int | None = None, +) -> Graph: + """Compose sampling → replication → location draw → derivation → gate.""" + + from .graph_kernels import UKClaimKernel + + cells = population_columns(graph, population) + if type(n_clones) is not int or n_clones < 1: + raise ValueError("Geographic replicate count K must be a positive integer.") + national_sampling.validate_sample_fraction(sample_fraction, label="UK full build") + national_sampling.validate_sample_seed(sample_seed, label="UK full build") + common = {"time_period": str(time_period)} + spine_gate_inputs = () + spine_gate_params = {} + if any(node.id == "spine.gates.transferred" for node in graph.nodes): + from .graph_evidence import SPINE_GATE_REPORT_TYPE + + gate = graph.node("spine.gates.transferred") + spine_gate_inputs = ( + ArtifactInput("spine_gate", gate.id, "gate_report", SPINE_GATE_REPORT_TYPE), + ) + spine_gate_params = { + "spine_gate_phase": "transferred", + "spine_gate_release_candidate": bool(gate.params["release_candidate"]), + } + nodes = list(graph.nodes) + nodes.append( + Node( + id="uk.full.sample", + kernel=UKFamilySampleKernel.ref, + inputs=population_slices(cells), + base=population, + structural=StructuralDelta.FILTER, + mass="free", + params={ + **common, + "fraction": float(sample_fraction), + "seed": sample_seed, + **spine_gate_params, + }, + artifact_inputs=spine_gate_inputs, + artifact_outputs=(ArtifactOutput("sampling", POPULATION_RECEIPT_TYPE),), + description="Select intact source families; calculate their historical mass normalization.", + ) + ) + nodes.append( + Node( + id="uk.full.normalize", + kernel=UKSampleNormalizationKernel.ref, + inputs=population_slices(cells), + base="uk.full.sample", + structural=StructuralDelta.REWEIGHT, + mass="declared", + weights=WeightUpdate( + "household", weight_kind, "Normalize sampled source-family mass." + ), + artifact_inputs=( + ArtifactInput( + "sampling", "uk.full.sample", "sampling", POPULATION_RECEIPT_TYPE + ), + ), + params=common, + description="Install normalized weights without changing their kind.", + ) + ) + expansion_cells = { + ("household", "source_household_id"): "int64", + ("household", "source_year"): "int64", + ("household", "source_household_key"): "string", + **{ + (entity, ladder_clone_index_column(entity)): "int64" + for entity in UK_NATIONAL_SCHEMA.entities + }, + } + # Existing source lineage is carried unchanged, except the explicitly + # requested historical modulus conversion, which owns its declarations. + if source_lineage_modulus is not None: + expansion_cells.update( + {("household", rowwise_dataset.POOL_SOURCE_LINEAGE_COLUMN): "int64"} + ) + expansion_cells = { + key: cells.get(key, dtype) for key, dtype in expansion_cells.items() + } + nodes.append( + Node( + id="uk.full.expand", + kernel=UKGeographicExpansionKernel.ref, + inputs=population_slices(cells), + structural=StructuralDelta.EXPAND, + base="uk.full.normalize", + mass="conserve", + params={ + **common, + "source_year": source_year, + "n_clones": n_clones, + "source_lineage_modulus": source_lineage_modulus, + "expand_weight_entity": "household", + "expand_weight_kind": weight_kind, + "expand_cells": tuple( + (e, c, dtype) for (e, c), dtype in sorted(expansion_cells.items()) + ), + }, + artifact_outputs=(ArtifactOutput("expansion", POPULATION_RECEIPT_TYPE),), + description="Prepare source lineage and expand linked entities into K geographic copies.", + ) + ) + nodes.append( + Node( + id="uk.full.expand.owned", + kernel=UKClaimKernel.ref, + population="uk.full.expand", + outputs=tuple( + Owned(e, c, dtype, rewrite=(e, c) in cells) + for (e, c), dtype in sorted(expansion_cells.items()) + ), + params={ + "materialized_expand_outputs": tuple( + f"{e}.{c}" for e, c in expansion_cells if (e, c) not in cells + ) + }, + description="Declare the lineage and replicate columns materialized by expansion.", + ) + ) + cells.update(expansion_cells) + nodes.append( + Node( + id="uk.full.locations", + kernel=UKLocationDrawKernel.ref, + population="uk.full.expand", + inputs=(Slice("household", ("region",)),), + sources=("uk_ladder",), + params={"seed": seed, "constituency_vintage": constituency_vintage}, + artifact_outputs=(ArtifactOutput("locations", LOCATION_DRAW_TYPE),), + description="Draw constituency then atomic area with the current sequential RNG.", + ) + ) + nodes.append( + Node( + id="uk.full.geography_mapping", + kernel=UKGeographyMappingKernel.ref, + population="uk.full.expand", + inputs=(Slice("household", ("region",)),), + sources=("uk_ladder",), + artifact_inputs=( + ArtifactInput( + "locations", "uk.full.locations", "locations", LOCATION_DRAW_TYPE + ), + ), + outputs=tuple( + Owned("household", col, "string", rewrite=("household", col) in cells) + for col in UK_GEOGRAPHY_LADDER_COLUMNS + ), + description="Derive every geography from the drawn atomic-area index.", + ) + ) + cells.update({("household", col): "string" for col in UK_GEOGRAPHY_LADDER_COLUMNS}) + nodes.append( + Node( + id="uk.full.geography_gate", + kernel=UKGeographyGateKernel.ref, + population="uk.full.expand", + inputs=population_slices(cells), + params=common, + artifact_outputs=(ArtifactOutput("gate", GEOGRAPHY_GATE_TYPE),), + description="Validate geography integrity before selected-target contributions.", + ) + ) + sources = ( + graph.sources + if any(s.name == "uk_ladder" for s in graph.sources) + else ( + *graph.sources, + SourceRef( + "uk_ladder", + "raw-bytes-v1", + "Full UK atomic-area ladder with pinned vintages.", + ), + ) + ) + return replace(graph, nodes=tuple(nodes), sources=tuple(sources)) + + +def register_uk_population_kernels(registry: KernelRegistry) -> None: + for kernel in ( + UKFamilySampleKernel(), + UKSampleNormalizationKernel(), + UKGeographicExpansionKernel(), + UKLocationDrawKernel(), + UKGeographyMappingKernel(), + UKGeographyGateKernel(), + ): + registry.register(kernel) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py new file mode 100644 index 000000000..c60106d58 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py @@ -0,0 +1,650 @@ +"""One full UK target surface: source compilation, selection and contributions.""" + +from __future__ import annotations + +import hashlib +import json +import tempfile +from collections.abc import Mapping +from dataclasses import asdict, dataclass, replace +from datetime import date +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd + +from microcosm.build import cross_grain +from microcosm.build.country_spec import load_country_spec +from microcosm.calibrate import CalibrationHierarchy, TargetRegistry, TargetSpec +from microcosm.calibrate.artifacts import ( + PROBLEM_TYPE, + _pack, + _unpack, + decode_problem, + encode_problem, +) +from microcosm.calibrate.matrix import build_constraint_matrix +from microcosm.calibrate.target_selection import select_targets +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + SourceRef, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import SOURCE_CODECS + +from . import full_measure, full_problem, ladder_targets, ledger_targets, local_doctrine +from .full_measure import resolve_uk_full_measures +from .full_problem import build_uk_full_local_problem +from .full_targets import CHRONICLE_SOURCE_CODEC, load_chronicle_source_bytes +from .geography_ladder import load_uk_oa_ladder, uk_area_region_codes +from .graph_population import ( + GEOGRAPHY_GATE_TYPE, + context_frame, + population_columns, + population_slices, +) +from .ladder_targets import ladder_vs_chronicle_household_dispersion +from .ledger_targets import uk_census_household_uprating, uk_ledger_households_total +from .local_rowwise import UKRowwiseNationalRows, prepare_uk_full_solve + +TARGET_SURFACE_TYPE = ArtifactType("microcosm.uk.full-target-surface", 1) +TARGET_SELECTION_TYPE = ArtifactType("microcosm.uk.full-target-selection", 1) +MEASURE_TYPE = ArtifactType("microcosm.uk.full-measured-contributions", 1) + +SOURCE_CODECS.register_bytes(CHRONICLE_SOURCE_CODEC, load_chronicle_source_bytes) + + +def registry_payload(registry: TargetRegistry) -> dict: + return {"country": "uk", "specs": [spec.to_dict() for spec in registry.specs]} + + +def registry_from_payload(payload: Mapping) -> TargetRegistry: + if payload.get("country") != "uk" or not isinstance(payload.get("specs"), list): + raise ValueError("Invalid UK target registry artifact.") + return TargetRegistry( + [TargetSpec.from_dict(spec) for spec in payload["specs"]], country="uk" + ) + + +def target_geography(spec: TargetSpec) -> str: + """Resolve explicit metadata without interpreting legacy registry buckets.""" + + metadata = spec.metadata + local = metadata.get("geography_level") + ledger = metadata.get("ledger_geography_level") + if local and ledger and local != ledger: + raise ValueError(f"Target {spec.name!r} has contradictory geography levels.") + level = local or ledger + if level not in {"country", "region", "constituency", "la"}: + raise ValueError( + f"Target {spec.name!r} has unsupported geography level {level!r}." + ) + return str(level) + + +def _surface_records(frame: pd.DataFrame) -> list[dict]: + def native(value): + if isinstance(value, np.generic): + value = value.item() + if value is pd.NA or (isinstance(value, float) and np.isnan(value)): + return None + return value + + def encode(key, value): + # #855: the local surface carries each spec's CalibrationHierarchy; + # store it exactly as TargetSpec.to_dict does. + if key == "hierarchy": + return None if value is None else asdict(value) + return native(value) + + # Preserve the exact binary float values; decimal-rounding table codecs + # can silently change target values on their first graph registration. + return [ + {key: encode(key, value) for key, value in row.items()} + for row in frame.to_dict(orient="records") + ] + + +def _local_specs(surface: pd.DataFrame) -> list[TargetSpec]: + return [ + TargetSpec( + name=str(row["target_name"]), + entity="household", + measure=str(row["metric"]), + value=float(row["value"]), + period=row.get("period", 0), + source=str(row.get("source", "uk_rowwise_local_surface")), + family=str(row["family"]), + # #855: schema-8 diagnostics need the hierarchy on every + # registry-backed target; decode it as TargetSpec.from_dict does. + hierarchy=( + CalibrationHierarchy.from_dict(row["hierarchy"]) + if row.get("hierarchy") is not None + else None + ), + metadata={ + **{key: value for key, value in row.items() if key != "hierarchy"}, + "geography_level": str(row["area_type"]), + "geography_id": str(row["area_code"]), + "materialization": "uk_local_surface", + }, + ) + for row in _surface_records(surface) + ] + + +def _selected_local_surface(surface: pd.DataFrame, specs) -> pd.DataFrame: + """Keep exact selected facts, including repeated names in different years.""" + keys = {spec.key for spec in specs} + periods = surface["period"] if "period" in surface else [0] * len(surface) + keep = [ + (str(name), period) in keys + for name, period in zip(surface["target_name"], periods, strict=True) + ] + return surface.loc[keep].reset_index(drop=True) + + +class _TargetKernel(KernelBase): + capabilities = Capabilities( + Determinism.DETERMINISTIC, dependencies=("policyengine-uk",) + ) + + def implementation_hash(self) -> str: + from . import full_targets + + implementation = source_hash( + type(self), + full_measure, + full_problem, + full_targets, + cross_grain, + ladder_targets, + ledger_targets, + local_doctrine, + ) + return hashlib.sha256( + canonical_json( + { + "implementation": implementation, + "country_spec": load_country_spec("uk").fingerprint, + } + ) + ).hexdigest() + + +class UKFullTargetCompilationKernel(_TargetKernel): + ref = "uk.full.target_compilation@1" + + def run(self, context: KernelContext) -> KernelResult: + from .full_targets import load_uk_full_target_inputs + from .ledger_targets import uk_local_target_surface + + inputs = load_uk_full_target_inputs( + context.sources["uk_ledger_facts"], + measure_exclusions=context.sources.get("uk_measure_exclusions"), + register_json=context.sources.get("uk_frozen_register"), + calibration_year=int(context.params["calibration_year"]), + exclusions_evaluated_on=date.fromisoformat( + str(context.params["review_date"]) + ), + ) + ladder = load_uk_oa_ladder(context.sources["uk_ladder"]) + period = int(inputs["calibration_year"]) + national = inputs["national_registry"] + local = inputs["local_registry"] + uprating = uk_census_household_uprating( + local, + uk_ledger_households_total(inputs["artifact"].facts, period=period), + period=period, + ) + dispersion = ladder_vs_chronicle_household_dispersion(ladder, local.specs) + surface, reconciliation = uk_local_target_surface( + full_problem._joint_surface_registry(local, national), + bound_national_target_ids=full_problem._national_contract_target_ids( + national + ), + period=period, + reviewed_unbound_higher_targets=inputs["reviewed_unbound_higher_targets"], + census_household_uprating=uprating, + area_region_codes=uk_area_region_codes(ladder), + ) + full = TargetRegistry( + [ + *_local_specs(surface), + *[ + replace( + spec, + metadata={ + **spec.metadata, + "materialization": "uk_national_measure", + "geography_level": target_geography(spec), + }, + ) + for spec in national.specs + ], + ], + country="uk", + ) + with Path(context.sources["uk_ladder"]).open("rb") as stream: + ladder_sha256 = hashlib.file_digest(stream, "sha256").hexdigest() + payload = { + "registry": registry_payload(full), + "local_registry": registry_payload(local), + "national_registry": registry_payload(national), + "band_edge_registry": registry_payload(inputs["band_edge_registry"]), + "surface": _surface_records(surface), + "surface_columns": surface.columns.tolist(), + "cross_geography": reconciliation, + "census_household_uprating": reconciliation["census_household_uprating"], + "household_dispersion": dispersion, + "measure_exclusions": inputs["measure_exclusions"], + "reviewed_unbound_higher_targets": inputs[ + "reviewed_unbound_higher_targets" + ], + "source_validation": { + "national_source_pin": inputs["national_source_pin"], + "local_source_pin": inputs["local_source_pin"], + "register_completeness": inputs["register_completeness"], + "ledger_provenance": inputs["ledger_provenance"], + "targets": { + "chronicle": inputs["ledger_provenance"], + "paired_ladder_sha256": ladder_sha256, + }, + }, + "uk_ledger_compiled_registries": { + str(period): registry_payload(registry) + for period, registry in inputs["uk_ledger_compiled_registries"].items() + }, + "uk_ledger_compiled_local_registries": { + str(period): registry_payload(registry) + for period, registry in inputs[ + "uk_ledger_compiled_local_registries" + ].items() + }, + "calibration_year": period, + } + return KernelResult(artifacts={"surface": canonical_json(payload)}) + + +class UKFullTargetSelectionKernel(_TargetKernel): + ref = "uk.full.target_selection@1" + + def run(self, context: KernelContext) -> KernelResult: + full = json.loads(context.artifacts["surface"].payload) + registry = registry_from_payload(full["registry"]) + levels = context.params.get("geography_levels") + selected = select_targets( + registry, geography_levels=levels, geography_resolver=target_geography + ) + if not len(selected.registry): + raise ValueError("Full build target selection contains no constraints.") + payload = { + "registry": registry_payload(selected.registry), + "receipt": selected.receipt, + } + return KernelResult(artifacts={"selection": canonical_json(payload)}) + + +class UKFullMeasureKernel(_TargetKernel): + ref = "uk.full.measures@1" + + def run(self, context: KernelContext) -> KernelResult: + gate = json.loads(context.artifacts["geography_gate"].payload) + if not gate["passed"]: + raise ValueError( + "Geography integrity gate failed: " + "; ".join(gate["failures"]) + ) + frame = context_frame(context) + full = json.loads(context.artifacts["surface"].payload) + selected = registry_from_payload( + json.loads(context.artifacts["selection"].payload)["registry"] + ) + national = selected.select( + predicate=lambda spec: ( + spec.metadata["materialization"] == "uk_national_measure" + ) + ) + grains = tuple( + sorted( + { + target_geography(spec) + for spec in selected + if spec.metadata["materialization"] == "uk_local_surface" + } + ) + ) + with tempfile.TemporaryDirectory( + prefix="microcosm-uk-full-measures-" + ) as scratch: + prepared, restore, rows, metrics, evidence = resolve_uk_full_measures( + frame, + national, + period=int(full["calibration_year"]), + scratch_dir=Path(scratch), + band_edge_registry=registry_from_payload(full["band_edge_registry"]), + blocks=int(context.params["engine_blocks"]), + local_grains=grains, + ) + # Compile national measures while temporary columns exist. The + # immutable pool, rather than that evaluation Frame, goes forward. + national_problem = ( + build_constraint_matrix(prepared, rows.targets, "household") + if len(rows.targets) + else None + ) + if national_problem is not None and national_problem.skipped: + failures = "; ".join( + f"{item.target.key}: {item.reason}" + for item in national_problem.skipped + ) + raise ValueError( + "Selected national constraints failed to compile: " + failures + ) + clean = restore(prepared) + for entity in frame.entities: + pd.testing.assert_frame_equal(clean.table(entity), frame.table(entity)) + arrays = { + f"metrics_{grain}": metrics[grain].to_numpy(dtype=np.float64) + for grain in grains + } + if national_problem is not None: + arrays["national_problem"] = np.frombuffer( + encode_problem( + national_problem, + entity_ids=frame.table("household")["household_id"].tolist(), + ), + dtype=np.uint8, + ) + metadata = { + "schema": MEASURE_TYPE.name + ".v1", + "household_ids": frame.table("household")["household_id"].tolist(), + "grains": {grain: metrics[grain].columns.tolist() for grain in grains}, + "has_national": national_problem is not None, + "evidence": evidence, + } + return KernelResult(artifacts={"measures": _pack(metadata, arrays)}) + + +def decode_measures(payload: bytes, *, selected: TargetRegistry) -> tuple[dict, dict]: + grains = { + target_geography(spec) + for spec in selected + if spec.metadata["materialization"] == "uk_local_surface" + } + has_national = any( + spec.metadata["materialization"] == "uk_national_measure" for spec in selected + ) + members = {f"metrics_{grain}" for grain in grains} + if has_national: + members.add("national_problem") + return _unpack(payload, schema=MEASURE_TYPE.name + ".v1", members=members) + + +@dataclass(frozen=True) +class UKFullProblemInputs: + """The exact admitted problem inputs shared by solve and holdout branches.""" + + frame: object + local_problem: object + national_rows: UKRowwiseNationalRows | None + bound_families: tuple[str, ...] + metadata: dict + full: dict + selection: dict + cross: dict + rung: object + national: TargetRegistry + selected: TargetRegistry + + +def reconstruct_uk_full_problem_inputs(context: KernelContext) -> UKFullProblemInputs: + """Reconstruct stored contributions without evaluating engine measures again.""" + frame = context_frame(context) + full = json.loads(context.artifacts["surface"].payload) + selection = json.loads(context.artifacts["selection"].payload) + selected = registry_from_payload(selection["registry"]) + national = selected.select( + predicate=lambda spec: spec.metadata["materialization"] == "uk_national_measure" + ) + local_specs = [ + spec + for spec in selected + if spec.metadata["materialization"] == "uk_local_surface" + ] + metadata, arrays = decode_measures( + context.artifacts["measures"].payload, selected=selected + ) + ids = frame.table("household")["household_id"].tolist() + if ids != metadata["household_ids"]: + raise ValueError("Measured contributions belong to a different household axis.") + metrics = { + grain: pd.DataFrame(arrays[f"metrics_{grain}"], columns=columns, index=ids) + for grain, columns in metadata["grains"].items() + } + surface = pd.DataFrame(full["surface"], columns=full["surface_columns"]) + surface = _selected_local_surface(surface, local_specs) + ladder = load_uk_oa_ladder(context.sources["uk_ladder"]) + _, local_problem, cross, bound_families, rung = build_uk_full_local_problem( + SimpleNamespace(result=SimpleNamespace(frame=frame), ladder=ladder), + local_registry=registry_from_payload(full["local_registry"]), + national_registry=national, + local_metrics=metrics, + period=int(full["calibration_year"]), + sample_fraction=float(context.params["sample_fraction"]), + reviewed_unbound_higher_targets=full["reviewed_unbound_higher_targets"], + selected_surface=surface, + surface_receipt=full["cross_geography"], + ) + national_rows = None + if metadata["has_national"]: + national_problem = decode_problem(arrays["national_problem"].tobytes()) + national_rows = UKRowwiseNationalRows( + national_problem.to_target_set(), + national, + tuple(sorted({spec.family for spec in national})), + ) + return UKFullProblemInputs( + frame, + local_problem, + national_rows, + tuple(bound_families), + metadata, + full, + selection, + cross, + rung, + national, + selected, + ) + + +class UKFullProblemKernel(_TargetKernel): + ref = "uk.full.problem@1" + + def run(self, context: KernelContext) -> KernelResult: + inputs = reconstruct_uk_full_problem_inputs(context) + frame, local_problem = inputs.frame, inputs.local_problem + national_rows, bound_families = inputs.national_rows, inputs.bound_families + selection, selected = inputs.selection, inputs.selected + metadata, full = inputs.metadata, inputs.full + cross, rung = inputs.cross, inputs.rung + ids = frame.table("household")["household_id"].tolist() + prepared = prepare_uk_full_solve( + frame, + local_problem, + bound_families=bound_families, + national_rows=national_rows, + target_weight_rule=str(context.params["target_weight_rule"]), + ) + problem = build_constraint_matrix(frame, prepared.target_set, "household") + if problem.skipped: + raise ValueError("Selected full-build constraints failed to compile.") + by_key = {spec.key: spec for spec in selected} + target_metadata = [] + for target in problem.targets: + spec = by_key[target.key] + target_metadata.append( + { + **spec.metadata, + "family": spec.family, + "source": spec.source, + "geography_level": target_geography(spec), + } + ) + doctrine = local_doctrine.UK_LOCAL_SOLVE_DOCTRINE + payload = encode_problem( + problem, + entity_ids=ids, + target_metadata=target_metadata, + bindings={ + "target_selection": selection["receipt"], + "target_selection_sha256": hashlib.sha256( + canonical_json(selection["receipt"]) + ).hexdigest(), + "source_surface_sha256": context.artifacts["surface"].key, + "target_loss_weights": ( + np.ones(problem.n_targets, dtype=np.float64) + if prepared.target_loss_weights is None + else prepared.target_loss_weights + ).tolist(), + "mass_reason": prepared.mass_reason, + "target_loss_cap": doctrine.target_loss_cap, + "max_weight_ratio": doctrine.max_weight_ratio, + "binding_adjudications": prepared.binding_adjudications, + "rung_surface": rung, + "bound_families": list(bound_families), + "measure_resolution": metadata["evidence"], + "cross_geography": cross, + "measure_exclusions": full["measure_exclusions"], + "calibration_year": full["calibration_year"], + }, + ) + return KernelResult(artifacts={"problem": payload}) + + +def append_uk_target_nodes( + graph: Graph, + *, + population: str = "uk.full.expand", + calibration_year: int, + time_period: str, + geography_levels: tuple[str, ...] | None = None, + engine_blocks: int = 1, + sample_fraction: float = 1.0, + target_weight_rule: str = "uniform", + optional_sources: tuple[str, ...] = (), + review_date: str | None = None, +) -> Graph: + """Default all geographies. Scope is independent of K, k and solver outcomes.""" + + if geography_levels is not None and not geography_levels: + raise ValueError("An explicit geography selector must contain levels.") + cells = population_columns(graph, population) + slices = population_slices(cells) + compile_sources = ("uk_ladder", "uk_ledger_facts", *optional_sources) + common = {"time_period": time_period} + review_date = date.today().isoformat() if review_date is None else review_date + date.fromisoformat(review_date) + contract_identity = load_country_spec("uk").fingerprint + surface = ArtifactInput( + "surface", "uk.full.target_compilation", "surface", TARGET_SURFACE_TYPE + ) + selection = ArtifactInput( + "selection", "uk.full.target_selection", "selection", TARGET_SELECTION_TYPE + ) + nodes = ( + Node( + "uk.full.target_compilation", + UKFullTargetCompilationKernel.ref, + population=population, + sources=compile_sources, + params={ + "calibration_year": calibration_year, + "review_date": review_date, + "country_contract_sha256": contract_identity, + }, + artifact_outputs=(ArtifactOutput("surface", TARGET_SURFACE_TYPE),), + description="Compile Chronicle national and local targets and validate the paired assignment ladder.", + ), + Node( + "uk.full.target_selection", + UKFullTargetSelectionKernel.ref, + population=population, + params={"geography_levels": geography_levels}, + artifact_inputs=(surface,), + artifact_outputs=(ArtifactOutput("selection", TARGET_SELECTION_TYPE),), + description="Select all geographies unless a target filter is explicitly requested.", + ), + Node( + "uk.full.measures", + UKFullMeasureKernel.ref, + population=population, + inputs=slices, + params={**common, "engine_blocks": engine_blocks}, + artifact_inputs=( + surface, + selection, + ArtifactInput( + "geography_gate", + "uk.full.geography_gate", + "gate", + GEOGRAPHY_GATE_TYPE, + ), + ), + artifact_outputs=(ArtifactOutput("measures", MEASURE_TYPE),), + description="Evaluate selected measures and compile temporary national contributions.", + ), + Node( + "uk.full.problem", + UKFullProblemKernel.ref, + population=population, + inputs=slices, + params={ + **common, + "sample_fraction": float(sample_fraction), + "target_weight_rule": target_weight_rule, + }, + sources=("uk_ladder",), + artifact_inputs=( + surface, + selection, + ArtifactInput("measures", "uk.full.measures", "measures", MEASURE_TYPE), + ), + artifact_outputs=(ArtifactOutput("problem", PROBLEM_TYPE),), + description="Build the one ordered selected-target problem and admission receipts.", + ), + ) + existing = {source.name for source in graph.sources} + sources = tuple( + SourceRef( + name, + CHRONICLE_SOURCE_CODEC if name == "uk_ledger_facts" else "raw-bytes-v1", + ) + for name in compile_sources + if name not in existing + ) + return replace( + graph, nodes=(*graph.nodes, *nodes), sources=(*graph.sources, *sources) + ) + + +def register_uk_target_kernels(registry: KernelRegistry) -> None: + for kernel in ( + UKFullTargetCompilationKernel(), + UKFullTargetSelectionKernel(), + UKFullMeasureKernel(), + UKFullProblemKernel(), + ): + registry.register(kernel) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py new file mode 100644 index 000000000..fc367dc51 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py @@ -0,0 +1,1117 @@ +"""Full-build terminal artifacts, with streamed H5 materialization and readback. + +Graph kernels describe and validate the exact numerical export. Atomic disk +materialization is an outer service, so a cache hit cannot silently skip a +required file write. The resulting H5 is a content-bound source to the graph +continuation; large H5 payloads are never duplicated in a byte artifact. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from collections.abc import Mapping +from dataclasses import asdict, replace +from importlib import metadata +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame, engine_tables +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + KernelRole, + Node, + Numeric, + SeedSource, + SourceRef, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import SOURCE_CODECS + +from ..artifact_files import file_artifact +from . import geography_ladder, national_frame +from .geography_ladder import uk_geography_ladder_gate +from .graph_population import context_frame, population_columns, population_slices +from .national_frame import ( + _read_uk_national_tables, + _write_uk_single_year_tables, + uk_household_weight_kind, + uk_time_period, + validate_uk_national_frame, +) +from .rowwise_dataset import ( + ARTIFACT_CLONE_INDEX_COLUMN, + ladder_clone_index_column, + load_uk_rowwise_dataset, +) + +EXPORT_DESCRIPTOR_TYPE = ArtifactType("microcosm.uk.full-export-descriptor", 1) +EXPORT_READBACK_TYPE = ArtifactType("microcosm.uk.full-export-readback", 1) +PACKAGE_INVENTORY_TYPE = ArtifactType("microcosm.full-package-inventory", 1) +EXPORT_SOURCE_CODEC = "uk-single-year-h5@1" + + +def _tables(frame: Frame) -> dict[str, pd.DataFrame]: + tables = engine_tables(frame, weighted_entities=("household",)) + renamed = {} + for entity in ("person", "benunit", "household"): + column = ladder_clone_index_column(entity) + if column not in tables[entity]: + raise ValueError( + f"Full-build export lacks {entity} geographic replicate lineage {column!r}." + ) + renamed[entity] = tables[entity].rename( + columns={column: ARTIFACT_CLONE_INDEX_COLUMN} + ) + return renamed + + +def _table_description(table: pd.DataFrame) -> dict[str, object]: + descriptor = { + "rows": len(table), + "columns": list(table.columns), + "dtypes": [str(dtype) for dtype in table.dtypes], + "index_dtype": str(table.index.dtype), + } + digest = hashlib.sha256(canonical_json(descriptor)) + digest.update( + np.ascontiguousarray( + pd.util.hash_pandas_object(table, index=True).to_numpy() + ).tobytes() + ) + return {**descriptor, "content_sha256": digest.hexdigest()} + + +def _content_descriptor( + tables, *, time_period, weight_kind, mass_log +) -> dict[str, object]: + descriptor = { + "tables": { + entity: _table_description(tables[entity]) + for entity in ("person", "benunit", "household") + }, + "time_period": str(time_period), + "weight_kind": weight_kind.value, + "mass_log": [asdict(record) for record in mass_log], + "hash_environment": { + "pandas": metadata.version("pandas"), + "numpy": metadata.version("numpy"), + }, + } + return { + **descriptor, + "content_sha256": hashlib.sha256(canonical_json(descriptor)).hexdigest(), + } + + +def describe_uk_export( + frame: Frame, *, bindings: Mapping[str, object] +) -> dict[str, object]: + """Validate and describe the maintained H5 layout without serializing it.""" + validate_uk_national_frame(frame) + tables = _tables(frame) + gate = uk_geography_ladder_gate( + tables["household"], frame.weights_for("household").values + ) + if not gate.passed: + raise ValueError( + "UK export geography integrity failed: " + "; ".join(gate.failures) + ) + return { + "schema_version": 1, + "kind": "uk_full_build_export", + **_content_descriptor( + tables, + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + mass_log=frame.mass_log, + ), + "bindings": dict(bindings), + "geography_integrity": {"passed": gate.passed, "failures": list(gate.failures)}, + "graph_only_metadata": [ + "strata", + "metadata_other_than_time_period", + "structural_ancestry", + ], + } + + +def _check_descriptor(descriptor: Mapping[str, object]) -> None: + if ( + descriptor.get("schema_version") != 1 + or descriptor.get("kind") != "uk_full_build_export" + ): + raise ValueError("Unsupported UK export descriptor.") + fields = ("tables", "time_period", "weight_kind", "mass_log", "hash_environment") + payload = {key: descriptor[key] for key in fields} + if hashlib.sha256(canonical_json(payload)).hexdigest() != descriptor.get( + "content_sha256" + ): + raise ValueError("UK export descriptor content identity is inconsistent.") + + +def materialize_uk_export( + frame: Frame, descriptor: Mapping[str, object], path: str | Path +) -> dict[str, object]: + """Perform only the declared serialization, recreating files on cache hits.""" + _check_descriptor(descriptor) + tables = _tables(frame) + current = _content_descriptor( + tables, + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + mass_log=frame.mass_log, + ) + if current["content_sha256"] != descriptor["content_sha256"]: + raise ValueError("UK export population differs from its graph descriptor.") + destination = Path(path) + if destination.suffix != ".h5": + raise ValueError("UK full-build dataset must have an .h5 filename.") + _write_uk_single_year_tables( + **tables, + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + mass_log=frame.mass_log, + path=destination, + ) + return file_artifact(destination) + + +def validate_uk_export( + path: str | Path, descriptor: Mapping[str, object] +) -> dict[str, object]: + """Compare actual written table values/dtypes/weights/periods to the request.""" + _check_descriptor(descriptor) + dataset = file_artifact(path) + payload, _, _ = _read_uk_national_tables(path) + actual = _content_descriptor( + payload, + time_period=payload["time_period"], + weight_kind=payload["household_weight_kind"], + mass_log=payload["mass_log"], + ) + failures = [ + f"Exported {entity} table differs from its graph descriptor." + for entity in ("person", "benunit", "household") + if actual["tables"][entity] != descriptor["tables"][entity] + ] + for key in ("time_period", "weight_kind", "mass_log", "hash_environment"): + if actual[key] != descriptor[key]: + failures.append(f"Exported {key} differs from its graph descriptor.") + if file_artifact(path) != dataset: + raise ValueError("UK exported file changed during graph readback validation.") + return { + "schema_version": 1, + "kind": "uk_full_build_export_readback", + "passed": not failures, + "failures": failures, + "dataset": dataset, + "content_sha256": actual["content_sha256"], + "expected_content_sha256": descriptor["content_sha256"], + "bindings": dict(descriptor["bindings"]), + } + + +class UKExportPrepareKernel(KernelBase): + ref = "uk.full-export.prepare@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, numeric=Numeric.BITWISE, seed_source=SeedSource.NONE + ) + + def implementation_hash(self) -> str: + return source_hash(sys.modules[__name__], national_frame, geography_ladder) + + def run(self, context: KernelContext) -> KernelResult: + for value in context.artifacts.values(): + if value.type == FULL_GATE_REPORT_TYPE: + _, enforcement = decode_full_gate_report(value.payload) + if not enforcement["artifact_permitted"]: + raise ValueError( + "UK export preparation refused by a structural full-build gate." + ) + bindings = json.loads(str(context.params["bindings"])) + bindings["artifacts"] = { + name: value.key for name, value in context.artifacts.items() + } + descriptor = describe_uk_export(context_frame(context), bindings=bindings) + return KernelResult(artifacts={"export_descriptor": canonical_json(descriptor)}) + + +class UKExportReadbackKernel(KernelBase): + ref = "uk.full-export.readback@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + seed_source=SeedSource.NONE, + role=KernelRole.GATE, + ) + + def implementation_hash(self) -> str: + return source_hash(sys.modules[__name__], national_frame) + + def run(self, context: KernelContext) -> KernelResult: + descriptor = json.loads(context.artifacts["export_descriptor"].payload) + report = validate_uk_export(context.sources["exported_dataset"], descriptor) + return KernelResult( + artifacts={"export_readback": canonical_json(report)}, + receipt={"outcome": "pass" if report["passed"] else "fail"}, + ) + + +class UKPackageInventoryKernel(KernelBase): + ref = "uk.full-export.package@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, numeric=Numeric.BITWISE, seed_source=SeedSource.NONE + ) + + def run(self, context: KernelContext) -> KernelResult: + report = json.loads(context.artifacts["export_readback"].payload) + if ( + report.get("kind") != "uk_full_build_export_readback" + or report.get("schema_version") != 1 + ): + raise ValueError( + "Package inventory requires a typed export readback report." + ) + if not report["passed"]: + raise ValueError("UK package refused because exported H5 readback failed.") + manifest = json.loads(str(context.params["manifest_binding"])) + files = {} + for alias, filename in json.loads( + str(context.params.get("evidence_files", "{}")) + ).items(): + record = file_artifact(context.sources["exported_evidence_" + alias]) + payload = context.artifacts[alias].payload + if ( + record["filename"] != filename + or record["sha256"] != hashlib.sha256(payload).hexdigest() + or record["size_bytes"] != len(payload) + ): + raise ValueError( + f"Materialized UK evidence {alias!r} differs from its graph artifact." + ) + files[alias] = { + **record, + "graph_artifact_key": context.artifacts[alias].key, + } + return KernelResult( + artifacts={ + "package_inventory": canonical_json( + { + "schema_version": 1, + "kind": "uk_full_build_package", + "readback_passed": report["passed"], + "dataset": report["dataset"], + "content_sha256": report["content_sha256"], + "build_bindings": report["bindings"], + "numerical_graph": manifest, + "artifacts": { + name: value.key for name, value in context.artifacts.items() + }, + "evidence_files": files, + "release_authorized": False, + } + ) + } + ) + + +def add_uk_export_preparation( + graph: Graph, + *, + population: str, + bindings: Mapping[str, object], + artifact_inputs: tuple[ArtifactInput, ...] = (), +) -> Graph: + node = Node( + id="uk.full.export.prepare", + kernel=UKExportPrepareKernel.ref, + population=population, + inputs=population_slices(population_columns(graph, population)), + params={"bindings": canonical_json(dict(bindings)).decode()}, + artifact_inputs=artifact_inputs, + artifact_outputs=(ArtifactOutput("export_descriptor", EXPORT_DESCRIPTOR_TYPE),), + description="Validate and describe exact H5 values, schema, weights and geographic lineage.", + ) + return replace(graph, nodes=(*graph.nodes, node)) + + +def add_uk_export_continuation( + graph: Graph, + *, + population: str, + manifest_binding: Mapping[str, object], + artifact_inputs: tuple[ArtifactInput, ...] = (), + evidence_files: Mapping[str, str] | None = None, +) -> Graph: + """Continue the same graph after its declared H5 materialization boundary.""" + evidence_files = {} if evidence_files is None else dict(evidence_files) + aliases = {item.name for item in artifact_inputs} + for alias, filename in evidence_files.items(): + if alias not in aliases or not alias.replace("_", "").isalnum(): + raise ValueError( + "Materialized evidence must name a declared simple artifact alias." + ) + if not filename or Path(filename).name != filename: + raise ValueError( + "Materialized evidence filenames must be simple path components." + ) + evidence_sources = tuple("exported_evidence_" + alias for alias in evidence_files) + readback = Node( + id="uk.full.export.readback", + kernel=UKExportReadbackKernel.ref, + population=population, + sources=("exported_dataset",), + artifact_inputs=( + ArtifactInput( + "export_descriptor", + "uk.full.export.prepare", + "export_descriptor", + EXPORT_DESCRIPTOR_TYPE, + ), + ), + artifact_outputs=(ArtifactOutput("export_readback", EXPORT_READBACK_TYPE),), + description="Read the written H5 and compare exact exported tables to the graph descriptor.", + ) + package = Node( + id="uk.full.package", + sources=evidence_sources, + kernel=UKPackageInventoryKernel.ref, + population=population, + artifact_inputs=( + ArtifactInput( + "export_readback", readback.id, "export_readback", EXPORT_READBACK_TYPE + ), + *artifact_inputs, + ), + artifact_outputs=(ArtifactOutput("package_inventory", PACKAGE_INVENTORY_TYPE),), + params={ + "manifest_binding": canonical_json(dict(manifest_binding)).decode(), + "evidence_files": canonical_json(evidence_files).decode(), + }, + description="Bind output bytes, numerical graph identity, scope/sizing and terminal evidence.", + ) + return replace( + graph, + sources=( + *graph.sources, + *(SourceRef(name, "raw-bytes-v1") for name in evidence_sources), + SourceRef( + "exported_dataset", + EXPORT_SOURCE_CODEC, + "Materialized UK H5; streamed identity and graph-owned readback.", + ), + ), + nodes=(*graph.nodes, readback, package), + ) + + +def _load_export_frame(path: Path) -> Frame: + return load_uk_rowwise_dataset(path)[0] + + +def register_uk_terminal_kernels(registry: KernelRegistry) -> None: + SOURCE_CODECS.register(EXPORT_SOURCE_CODEC, _load_export_frame) + for kernel in ( + UKExportPrepareKernel(), + UKExportReadbackKernel(), + UKPackageInventoryKernel(), + ): + registry.register(kernel) + + +FULL_GATE_REPORT_TYPE = ArtifactType("microcosm.uk.full-gate-report", 1) +FULL_DIAGNOSTICS_TYPE = ArtifactType("microcosm.uk.full-calibration-diagnostics", 1) +FULL_DIAGNOSTICS_CSV_TYPE = ArtifactType("microcosm.uk.full-target-diagnostics-csv", 1) +FULL_SUPPORT_CSV_TYPE = ArtifactType("microcosm.uk.full-area-support-csv", 1) +FULL_HOLDOUT_TYPE = ArtifactType("microcosm.uk.full-rotated-holdout", 1) + + +def _full_gate_enforcement(document: Mapping, report): + """Carry earlier phase policy forward without reevaluating its population.""" + from ..country_spec import load_country_spec + from ..gate_battery import gate_phase_report_from_payload + from .full_gates import classify_full_gate_outcomes, uk_full_gate_manifest + from .graph_evidence import uk_spine_gate_manifest + + upstream = document.get("upstream_phase_reports", {}) + allowed = {"spine_assembled": "assembled", "spine_transferred": "transferred"} + if report.phase == "terminal": + allowed["full_preflight"] = "preflight" + if not isinstance(upstream, Mapping) or set(upstream) - set(allowed): + raise ValueError("Unexpected upstream phase in the full gate artifact.") + reports = [] + for name, payload in upstream.items(): + gates = ( + uk_full_gate_manifest(document["selection_receipt"]) + if name == "full_preflight" + else uk_spine_gate_manifest(load_country_spec("uk")) + ) + previous = gate_phase_report_from_payload(payload, gates=gates) + if previous.phase != allowed[name]: + raise ValueError("Upstream full gate report has a different phase.") + reports.append(previous) + reports.append(report) + classifications = [ + classify_full_gate_outcomes( + item, + sample_fraction=document["sample_fraction"], + release_candidate=document["release_candidate"], + ) + for item in reports + ] + enforcement = dict(classifications[-1]) + for key in ( + "structural_failures", + "enforced_blocking", + "exportable_blocking", + "unenforced_release_failures", + "diagnostic_failures", + ): + enforcement[key] = list( + dict.fromkeys(value for item in classifications for value in item[key]) + ) + enforcement["artifact_permitted"] = all( + item["artifact_permitted"] for item in classifications + ) + enforcement["release_blocking_gates_passed"] = all( + item["release_blocking_gates_passed"] for item in classifications + ) + return enforcement + + +def decode_full_gate_report(payload: bytes | Mapping): + """Restore a phase report only after checking its declared gate scope.""" + from ..gate_battery import gate_phase_report_from_payload + from .full_gates import uk_full_gate_manifest + + document = json.loads(payload) if isinstance(payload, bytes) else dict(payload) + if ( + document.get("schema_version") != 1 + or document.get("kind") != "uk_full_gate_report" + ): + raise ValueError("Unsupported UK full gate artifact.") + gates = uk_full_gate_manifest(document["selection_receipt"]) + report = gate_phase_report_from_payload(document["report"], gates=gates) + enforcement = _full_gate_enforcement(document, report) + if enforcement != document["enforcement"]: + raise ValueError("UK gate enforcement differs from its bound phase report.") + return report, enforcement + + +def _spine_gate_evidence(context: KernelContext): + from ..stage_evidence import decode_stage_evidence + + names = tuple(context.params["spine_stage_names"]) + if "spine_provenance" in context.artifacts: + provenance = json.loads(context.artifacts["spine_provenance"].payload) + if tuple(provenance["stages"]) != names: + raise ValueError( + "Bound spine stage roster differs from its declared gate input." + ) + evidence = { + name: provenance.get("stage_evidence", {}).get(name) for name in names + } + records = provenance.get("fit_weight_records") + return evidence, records + documents = { + name: decode_stage_evidence(context.artifacts[name].payload, stage=name) + for name in names + } + return ( + {name: document["evidence"] for name, document in documents.items()}, + { + name: document["fit_weight_records"] + for name, document in documents.items() + if "fit_weight_records" in document + }, + ) + + +def _source_gate_evidence(context: KernelContext, engine): + from datetime import date + + from .graph_targets import registry_from_payload + + surface = json.loads(context.artifacts["surface"].payload) + return { + "coverage_engine": engine, + "build_stage_names": tuple(context.params["spine_stage_names"]), + "reference_registry": registry_from_payload(surface["national_registry"]), + "uk_ledger_compiled_registries": { + int(period): registry_from_payload(registry) + for period, registry in surface["uk_ledger_compiled_registries"].items() + }, + "uk_ledger_compiled_local_registries": { + int(period): registry_from_payload(registry) + for period, registry in surface[ + "uk_ledger_compiled_local_registries" + ].items() + }, + "exclusions_evaluated_on": date.fromisoformat( + str(context.params["review_date"]) + ), + } + + +class UKFullGateKernel(KernelBase): + ref = "uk.full-gates@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + seed_source=SeedSource.NONE, + role=KernelRole.GATE, + dependencies=("policyengine-uk",), + ) + + def __init__(self, *, coverage_engine, engine_identity: str): + self.engine = coverage_engine + self.engine_identity = engine_identity + + def implementation_hash(self) -> str: + from .. import gate_battery + from ..country_spec import load_country_spec + from . import ( + battery_bindings, + diagnostics, + full_gates, + local_rowwise, + weighted_integrity, + ) + + return hashlib.sha256( + canonical_json( + { + "code": source_hash( + sys.modules[__name__], + gate_battery, + battery_bindings, + full_gates, + local_rowwise, + weighted_integrity, + diagnostics, + ), + "country_resources": load_country_spec("uk").fingerprint, + } + ) + ).hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + from microcosm.calibrate.artifacts import decode_problem, decode_solution + + from ..gate_battery import ( + EvidenceContext, + evaluate_phase, + gate_phase_report_payload, + ) + from ..stage_evidence import encode_stage_evidence + from .battery_bindings import UK_GATE_REGISTRY + from .full_gates import ( + build_full_gate_context, + uk_full_gate_manifest, + uk_full_gate_scope_receipt, + ) + from .geography_ladder import load_uk_oa_ladder + from .local_rowwise import uk_ladder_area_support_summary + from .release_certification import rehydrate_uk_fit_weight_records + from .weighted_integrity import load_uk_input_mass_reference + + if context.params["engine_identity"] != self.engine_identity: + raise ValueError( + "UK full gate engine identity differs from its declared binding." + ) + selection = json.loads(context.artifacts["selection"].payload)["receipt"] + gates = uk_full_gate_manifest(selection) + from ..gate_battery import _gates_manifest_payload, _json_safe + + if ( + canonical_json(_gates_manifest_payload(uk_full_gate_manifest())).decode() + != context.params["gate_manifest"] + ): + raise ValueError("UK full gate manifest differs from its declared binding.") + stage_evidence, fit_weight_records = _spine_gate_evidence(context) + supporting = _source_gate_evidence(context, self.engine) + phase = str(context.params["phase"]) + diagnostics = [] + support = [] + output_artifacts = {} + upstream_reports = { + name: json.loads(context.artifacts[name].payload) + for name in ("spine_assembled", "spine_transferred") + if name in context.artifacts + } + if phase == "preflight": + # These source/reference and roster checks have no final-weight or + # contribution-matrix dependency. They run before dense calibration. + evidence = EvidenceContext(artifacts=supporting) + elif phase == "terminal": + preflight_document = json.loads(context.artifacts["preflight"].payload) + _, preflight = decode_full_gate_report(preflight_document) + if not preflight["artifact_permitted"]: + raise ValueError( + "Full calibration reached terminal gates despite a blocking source preflight." + ) + upstream_reports = { + **preflight_document.get("upstream_phase_reports", {}), + "full_preflight": preflight_document["report"], + } + frame = context_frame(context) + problem = decode_problem(context.artifacts["problem"].payload) + solution = decode_solution(context.artifacts["solution"].payload) + from microcosm.calibrate.artifacts import decode_calibration_result + + # The ordered result binds its initial weights. Rehydrate against + # the same selected table axis, then install the actual graph Frame + # so scoring uses the graph-owned mass ledger and final population. + initial_frame = Frame( + {entity: frame.table(entity) for entity in frame.entities}, + frame.schema, + {"household": problem.problem.initial_weights}, + frame.strata, + metadata=frame.metadata, + ) + result = decode_calibration_result( + context.artifacts["result"].payload, + frame=initial_frame, + problem=problem, + ) + if not np.array_equal(result.weights, solution.weights): + raise ValueError( + "Final diagnostics result differs from the installed solution." + ) + result = replace(result, frame=frame) + supporting["calibration_result"] = result + fit_records = rehydrate_uk_fit_weight_records( + {"fit_weight_records": fit_weight_records} + ) + if fit_records is not None: + supporting["fit_weight_records"] = fit_records + if "uk_input_mass_reference" in context.sources: + supporting["input_mass_reference"] = load_uk_input_mass_reference( + context.sources["uk_input_mass_reference"] + ) + household = frame.table("household").copy() + household["household_weight"] = frame.weights_for("household").values + summaries = uk_ladder_area_support_summary( + household, load_uk_oa_ladder(context.sources["uk_ladder"]) + ) + support_frame = pd.concat( + ( + summaries["constituency"].assign(geography_level="constituency"), + summaries["la"].assign(geography_level="local_authority"), + ), + ignore_index=True, + ) + supporting["uk_area_support_summary"] = support_frame + evidence = build_full_gate_context( + frame, + ordered_problem=problem, + solution=solution, + selection_receipt=selection, + stage_evidence=stage_evidence, + supporting_evidence=supporting, + ) + diagnostics = evidence.artifacts["target_diagnostics"] + support = support_frame.to_dict(orient="records") + from .diagnostics import uk_calibration_diagnostics_payload + from .graph_targets import registry_from_payload + + selected_registry = registry_from_payload( + json.loads(context.artifacts["selection"].payload)["registry"] + ) + holdout = json.loads(context.artifacts["holdout"].payload) + complete_diagnostics = uk_calibration_diagnostics_payload( + result, + frame, + target_geography_levels={ + target.row_name: str(row["geography_level"]) + for target, row in zip( + problem.problem.targets, problem.target_metadata, strict=True + ) + }, + target_registry=selected_registry, + local_area_support=support_frame, + rotated_holdout=holdout, + build={ + "build_kind": "uk_full_build", + "target_scope": selection["selector"], + }, + ) + output_artifacts.update( + { + "calibration_diagnostics": encode_stage_evidence( + _json_safe(complete_diagnostics) + ), + "target_diagnostics_csv": pd.DataFrame(diagnostics) + .to_csv(index=False) + .encode(), + "area_support_csv": support_frame.to_csv(index=False).encode(), + } + ) + else: + raise ValueError(f"Unknown full gate phase {phase!r}.") + report = evaluate_phase( + gates, phase=phase, context=evidence, registry=UK_GATE_REGISTRY + ) + payload = { + "schema_version": 1, + "kind": "uk_full_gate_report", + "selection_receipt": selection, + "sample_fraction": float(context.params["sample_fraction"]), + "release_candidate": bool(context.params["release_candidate"]), + "scope": uk_full_gate_scope_receipt(selection), + "report": gate_phase_report_payload(report, gates=gates), + "upstream_phase_reports": upstream_reports, + "target_diagnostics": _json_safe(diagnostics), + "area_support": _json_safe(support), + "artifacts": {name: value.key for name, value in context.artifacts.items()}, + } + enforcement = _full_gate_enforcement(payload, report) + payload["enforcement"] = enforcement + return KernelResult( + artifacts={ + "gate_report": encode_stage_evidence(payload), + **output_artifacts, + }, + receipt={ + "outcome": "pass" if enforcement["artifact_permitted"] else "fail" + }, + ) + + +def append_uk_full_gate_nodes( + graph: Graph, + *, + calibration, + spine_stage_names: tuple[str, ...], + engine_identity: str, + review_date, + sample_fraction: float = 1.0, + release_candidate: bool = False, + spine_provenance: ArtifactInput | None = None, + input_population: str = "uk.full.pool", + skip_holdout: bool = False, +) -> Graph: + """Own source preflight before solve and final diagnostics after installation.""" + from microcosm.calibrate.artifacts import PROBLEM_TYPE, RESULT_TYPE, SOLUTION_TYPE + + from ..gate_battery import _gates_manifest_payload + from ..stage_evidence import STAGE_EVIDENCE_TYPE + from .full_gates import uk_full_gate_manifest + from .graph_evidence import SPINE_GATE_REPORT_TYPE + from .graph_targets import TARGET_SELECTION_TYPE, TARGET_SURFACE_TYPE + + if not engine_identity: + raise ValueError("Full-build gates require a declared engine identity.") + if any(node.id == "uk.full.gates.preflight" for node in graph.nodes): + raise ValueError("Full gate nodes are already registered.") + params = { + "spine_stage_names": tuple(spine_stage_names), + "engine_identity": engine_identity, + "review_date": str(review_date), + "sample_fraction": sample_fraction, + "release_candidate": release_candidate, + "gate_manifest": canonical_json( + _gates_manifest_payload(uk_full_gate_manifest()) + ).decode(), + } + provenance = ( + (replace(spine_provenance, name="spine_provenance"),) + if spine_provenance + else tuple( + ArtifactInput( + name, + "create_uk_frs" if name == "frs_spine" else name, + "stage_evidence", + STAGE_EVIDENCE_TYPE, + ) + for name in spine_stage_names + ) + ) + common = ( + ArtifactInput( + "surface", "uk.full.target_compilation", "surface", TARGET_SURFACE_TYPE + ), + ArtifactInput( + "selection", "uk.full.target_selection", "selection", TARGET_SELECTION_TYPE + ), + *provenance, + *( + ArtifactInput( + f"spine_{phase}", + f"spine.gates.{phase}", + "gate_report", + SPINE_GATE_REPORT_TYPE, + ) + for phase in ("assembled", "transferred") + if any(node.id == f"spine.gates.{phase}" for node in graph.nodes) + ), + ) + preflight = Node( + "uk.full.gates.preflight", + UKFullGateKernel.ref, + population=input_population, + params={**params, "phase": "preflight"}, + artifact_inputs=common, + artifact_outputs=(ArtifactOutput("gate_report", FULL_GATE_REPORT_TYPE),), + description="Validate complete source/reference registries and spine stage ownership before dense calibration.", + ) + prerequisite = ArtifactInput( + "preflight", preflight.id, "gate_report", FULL_GATE_REPORT_TYPE + ) + nodes = tuple( + replace(node, artifact_inputs=(*node.artifact_inputs, prerequisite)) + if node.id == calibration.dense_producer + else node + for node in graph.nodes + ) + sources = ("uk_ladder",) + ( + ("uk_input_mass_reference",) + if any(source.name == "uk_input_mass_reference" for source in graph.sources) + else () + ) + final = Node( + "uk.full.gates.calibrated", + UKFullGateKernel.ref, + population=calibration.population, + inputs=population_slices(population_columns(graph, calibration.population)), + sources=sources, + params={**params, "phase": "terminal"}, + artifact_inputs=( + *common, + prerequisite, + ArtifactInput( + "problem", calibration.problem_producer, "problem", PROBLEM_TYPE + ), + ArtifactInput( + "solution", + calibration.solution_producer, + "refit_solution" if calibration.size_producer else "solution", + SOLUTION_TYPE, + ), + ), + artifact_outputs=(ArtifactOutput("gate_report", FULL_GATE_REPORT_TYPE),), + description="Compute final identified-row diagnostics once and evaluate all applicable national/local release gates.", + ) + holdout = uk_full_holdout_node( + graph, + calibration=calibration, + input_population=input_population, + skip_holdout=skip_holdout, + ) + final = replace( + final, + artifact_inputs=( + *final.artifact_inputs, + ArtifactInput("result", calibration.result_producer, "result", RESULT_TYPE), + ArtifactInput("holdout", holdout.id, "holdout", FULL_HOLDOUT_TYPE), + ), + artifact_outputs=( + *final.artifact_outputs, + ArtifactOutput("calibration_diagnostics", FULL_DIAGNOSTICS_TYPE), + ArtifactOutput("target_diagnostics_csv", FULL_DIAGNOSTICS_CSV_TYPE), + ArtifactOutput("area_support_csv", FULL_SUPPORT_CSV_TYPE), + ), + ) + return replace(graph, nodes=(*nodes, preflight, holdout, final)) + + +def register_uk_full_gate_kernels( + registry: KernelRegistry, *, coverage_engine, engine_identity: str +) -> None: + registry.register( + UKFullGateKernel( + coverage_engine=coverage_engine, engine_identity=engine_identity + ) + ) + + registry.register(UKFullHoldoutKernel()) + + +class UKFullHoldoutKernel(KernelBase): + ref = "uk.full.rotated-holdout@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.PARAM, + dependencies=("policyengine-uk", "torch"), + ) + + def implementation_hash(self) -> str: + from ..country_spec import load_country_spec + from . import dataset_size, graph_targets, local_rowwise + + return hashlib.sha256( + canonical_json( + { + "code": source_hash( + sys.modules[__name__], + graph_targets, + local_rowwise, + dataset_size, + dependencies=self.capabilities.dependencies, + ), + "country_resources": load_country_spec("uk").fingerprint, + } + ) + ).hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + from microcosm.calibrate.artifacts import decode_problem + + from ..gate_battery import _json_safe + from ..stage_evidence import encode_stage_evidence + from .graph_targets import reconstruct_uk_full_problem_inputs + from .local_rowwise import rotated_uk_local_holdout + + _, preflight = decode_full_gate_report(context.artifacts["preflight"].payload) + if not preflight["artifact_permitted"]: + raise ValueError( + "Rotated holdout cannot run after a blocking source preflight." + ) + if context.params["skip_holdout"]: + report = { + "report_only": True, + "skipped": True, + "reason": "Explicit development request; no holdout claim.", + } + else: + inputs = reconstruct_uk_full_problem_inputs(context) + original = decode_problem(context.artifacts["problem"].payload) + if original.entity_ids != tuple( + inputs.frame.table("household")["household_id"] + ): + raise ValueError( + "Holdout original pool differs from its bound problem axis." + ) + report = rotated_uk_local_holdout( + inputs.frame, + inputs.local_problem, + bound_families=inputs.bound_families, + national_rows=inputs.national_rows, + target_weight_rule=str(context.params["target_weight_rule"]), + epochs=int(context.params["epochs"]), + learning_rate=float(context.params["learning_rate"]), + conserve_mass=False, + target_records=None, + l0_lambda=0.0, + budget_iters=10, + dataset_households=context.params.get("dataset_households"), + solve_seed=int(context.params["seed"]), + selection_seed=context.params.get("selection_seed"), + selection_pi_hi=float(context.params["selection_pi_hi"]), + baseline_pi_floor=float(context.params["baseline_pi_floor"]), + ) + report = { + **report, + "graph_binding": { + "original_problem_artifact": context.artifacts["problem"].key, + "artifacts": { + name: value.key for name, value in context.artifacts.items() + }, + }, + } + return KernelResult( + artifacts={"holdout": encode_stage_evidence(_json_safe(report))} + ) + + +def uk_full_holdout_node( + graph: Graph, *, calibration, input_population: str, skip_holdout: bool +) -> Node: + from microcosm.calibrate.artifacts import PROBLEM_TYPE + + original = graph.node("uk.full.problem") + dense = graph.node(calibration.dense_producer) + size = ( + None + if calibration.size_producer is None + else graph.node(calibration.size_producer) + ) + return Node( + "uk.full.holdout", + UKFullHoldoutKernel.ref, + population=input_population, + inputs=population_slices(population_columns(graph, input_population)), + sources=original.sources, + params={ + **original.params, + **dense.params, + "skip_holdout": skip_holdout, + "dataset_households": None if size is None else size.params["households"], + "selection_seed": None if size is None else size.params["seed"], + "selection_pi_hi": 1.0 if size is None else size.params["pi_hi"], + "baseline_pi_floor": ( + 0.0 if size is None else size.params["baseline_pi_floor"] + ), + }, + artifact_inputs=( + *original.artifact_inputs, + ArtifactInput("problem", original.id, "problem", PROBLEM_TYPE), + ArtifactInput( + "preflight", + "uk.full.gates.preflight", + "gate_report", + FULL_GATE_REPORT_TYPE, + ), + ), + artifact_outputs=(ArtifactOutput("holdout", FULL_HOLDOUT_TYPE),), + description="Preserve five rotated local-target holdouts with national constraints fixed in training and unchanged sizing doctrine.", + ) + + +def materialize_uk_terminal_artifacts( + manifest, store, *, directory: str | Path, stem: str +) -> dict[str, dict[str, object]]: + """Write graph-produced diagnostic bytes atomically, including after cache hits.""" + from ..artifact_files import materialize_bytes + + if not stem or Path(stem).name != stem: + raise ValueError( + "UK terminal artifact stem must be a simple filename component." + ) + root = Path(directory) + artifacts = { + "calibration_diagnostics": ( + "uk.full.gates.calibrated", + "calibration_diagnostics", + ".diagnostics.json", + ), + "target_diagnostics": ( + "uk.full.gates.calibrated", + "target_diagnostics_csv", + ".targets.csv", + ), + "area_support": ( + "uk.full.gates.calibrated", + "area_support_csv", + ".area_support.csv", + ), + "holdout": ("uk.full.holdout", "holdout", ".holdout.json"), + "target_registry": ( + "uk.full.target_selection", + "selection", + ".target_selection.json", + ), + } + inventory = {} + for role, (node, output, suffix) in artifacts.items(): + key = manifest.nodes[node].opaque_artifacts[output] + inventory[role] = { + **materialize_bytes(store.load_bytes(key), root / (stem + suffix)), + "graph_artifact_key": key, + } + return inventory diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py index cd9056208..e189eb8cd 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py @@ -359,6 +359,42 @@ def build_uk_rowwise_local_matrix( ) +def empty_uk_local_problem(household_ids: Sequence[Any]) -> UKRowwiseLocalMatrix: + """An explicitly filtered local surface on the full pool household axis. + + Composition must still supply at least one selected target. This does + not manufacture local constraints or imply that local fit was assessed. + """ + + ids = tuple(household_ids) + if len(set(ids)) != len(ids): + raise ValueError("household IDs must be unique.") + return UKRowwiseLocalMatrix( + matrix=sp.csr_matrix((0, len(ids)), dtype=np.float64), + targets=np.empty(0, dtype=np.float64), + target_frame=pd.DataFrame( + columns=[ + "target_index", + "area_type", + "area_code", + "metric", + "value", + "target_name", + "family", + ] + ), + area_codes=(), + metric_names=(), + household_ids=ids, + assigned_areas=(), + metric_values=np.empty((len(ids), 0), dtype=np.float64), + area_codes_by_grain={}, + metric_names_by_grain={}, + assigned_areas_by_grain={}, + metric_values_by_grain={}, + ) + + def build_uk_rowwise_local_surface_matrix( metrics_by_grain: Mapping[str, pd.DataFrame], assigned_by_grain: Mapping[str, pd.Series | Sequence[str]], @@ -851,11 +887,6 @@ def _normalise_uk_local_bound_families( "sequence of family/area_type strings, not one string." ) declared = tuple(str(name) for name in bound_families) - if not declared: - raise ValueError( - "UK local binding declarations: bound_families must name at " - "least one family/area_type pair." - ) blanks = [name for name in declared if not name.strip()] if blanks: raise ValueError( @@ -1067,76 +1098,34 @@ def callback(event: dict[str, object]) -> None: return callback -def solve_uk_rowwise_weights_under_doctrine( +@dataclass(frozen=True) +class UKPreparedFullSolve: + """Declared target problem before any weight or size operation.""" + + frame: Frame + problem: UKRowwiseLocalMatrix + national_rows: UKRowwiseNationalRows | None + target_set: TargetSet + target_loss_weights: np.ndarray + binding_adjudications: Mapping[str, Any] + mass_reason: str + + +def prepare_uk_full_solve( frame: Frame, problem: UKRowwiseLocalMatrix, *, bound_families: Sequence[str], national_rows: UKRowwiseNationalRows | None = None, target_weight_rule: str = "uniform", - restore: Callable[[Frame], Frame] | None = None, - epochs: int = 512, - learning_rate: float = 0.15, - conserve_mass: bool = False, - target_records: int | None = None, - dataset_households: int | None = None, - l0_lambda: float = 0.0, - budget_iters: int = 10, - seed: int = 0, - selection_seed: int | None = None, - selection_pi_hi: float = 1.0, - baseline_pi_floor: float = 0.0, - size_checkpoint_dir: Path | None = None, - resume_size_checkpoint: Path | None = None, - checkpoint_identity: Mapping[str, Any] | None = None, - checkpoint_provenance: Mapping[str, Any] | None = None, - progress: Callable[[str], None] | None = None, - progress_events: Callable[[dict[str, object]], None] | None = None, -) -> UKRowwiseDoctrineSolve: - """Solve rowwise household weights under the reviewed doctrine. - - ``progress`` receives readable lines (every 100 epochs, each probe, the - search stop); ``progress_events`` receives every raw calibrator event as a - dict (``calibration_epoch``, ``budget_probe``, ``budget_search_done``), - phase-tagged by the size machinery, so a build driver can publish staging - telemetry without changing the lines a log reader follows. - - ``selection_seed`` (default ``seed``) seeds only the size selection — - the informed L0 search, the exact-count draw and the refit — so two - selections can be compared on one pool and one dense reference. +) -> UKPreparedFullSolve: + """Validate the selected surface with one doctrine for every geography.""" - ``size_checkpoint_dir`` persists the dense solve and the informed L0 - search of a ``dataset_households`` solve before the exact-count draw - (:mod:`microcosm.build.uk_runtime.size_checkpoint`), stamped with - ``checkpoint_identity``; ``resume_size_checkpoint`` restores such a - checkpoint instead of solving and searching again, refusing when the - identity, the pool or the target surface differ. The draw's threshold - (``selection_pi_hi``) may differ from the one the search stopped on; the - size receipt records both. ``baseline_pi_floor`` trims the refit's - Horvitz–Thompson baseline (see - :func:`~microcosm.build.uk_runtime.dataset_size.refit_uk_dataset_size`); - it is a refit setting, so a resumed checkpoint may use a different one. - - ``progress`` receives one readable line per hundred epochs of the dense - solve, of every budget probe and of the refit, one line per finished - probe with its drawability verdict, and one when the search stops - (:func:`~microcosm.build.uk_runtime.solve_progress.uk_solve_progress_callback`). - - Structurally knob-free like before the ``calibrate()`` migration: no - per-target parameters and no doctrine parameter — the bounds always come - from :data:`UK_LOCAL_SOLVE_DOCTRINE` and ride into the public front door - as explicit arguments. Initial weights are the frame's typed household - weights directly (a rowwise household exists in exactly one area, so - nothing is split); zero weights are refused — a dead row must be dropped - or revived upstream with a recorded mass change, never resurrected by a - solver floor. The kernel enforces the ``CALIBRATED`` kind transition and - mints the mass record (reason from - :func:`rowwise_calibration_mass_reason`); the returned frame carries - both, with the persisted ``household_weight`` column refreshed. - """ - - doctrine = UK_LOCAL_SOLVE_DOCTRINE _require_uniform_target_surface(problem) + if not len(problem.targets) and ( + national_rows is None or not len(national_rows.targets) + ): + raise ValueError("full calibration requires at least one selected target.") local_bound_families = tuple( family for family in bound_families if not str(family).startswith("national/") ) @@ -1224,6 +1213,8 @@ def solve_uk_rowwise_weights_under_doctrine( *(() if national_rows is None else national_rows.targets.targets), ] ) + if not len(target_set): + raise ValueError("full calibration requires at least one selected target.") local_count = len(local_target_set) national_count = len(target_set) - local_count grain_labels = [ @@ -1234,6 +1225,128 @@ def solve_uk_rowwise_weights_under_doctrine( grain_labels, rule=target_weight_rule, ) + return UKPreparedFullSolve( + frame=frame, + problem=problem, + national_rows=national_rows, + target_set=target_set, + target_loss_weights=target_loss_weights, + binding_adjudications=binding_adjudications, + mass_reason=mass_reason, + ) + + +def solve_uk_dense_reference( + prepared: UKPreparedFullSolve, + *, + epochs: int = 512, + learning_rate: float = 0.15, + conserve_mass: bool = False, + target_records: int | None = None, + l0_lambda: float = 0.0, + budget_iters: int = 10, + seed: int = 0, + progress_callback: Callable | None = None, +) -> CalibrationResult: + """Execute the shared solver on the original pool, before size selection.""" + + doctrine = UK_LOCAL_SOLVE_DOCTRINE + return calibrate( + prepared.frame, + prepared.target_set, + weight_entity="household", + epochs=epochs, + learning_rate=learning_rate, + mass=CONSERVE_MASS if conserve_mass else FREE_MASS, + mass_reason=None if conserve_mass else prepared.mass_reason, + max_weight_ratio=doctrine.max_weight_ratio, + target_records=target_records, + l0_lambda=l0_lambda, + budget_iters=budget_iters, + seed=seed, + target_loss_weights=prepared.target_loss_weights, + target_loss_cap=doctrine.target_loss_cap, + progress_callback=progress_callback, + ) + + +def solve_uk_rowwise_weights_under_doctrine( + frame: Frame, + problem: UKRowwiseLocalMatrix, + *, + bound_families: Sequence[str], + national_rows: UKRowwiseNationalRows | None = None, + target_weight_rule: str = "uniform", + restore: Callable[[Frame], Frame] | None = None, + epochs: int = 512, + learning_rate: float = 0.15, + conserve_mass: bool = False, + target_records: int | None = None, + dataset_households: int | None = None, + l0_lambda: float = 0.0, + budget_iters: int = 10, + seed: int = 0, + selection_seed: int | None = None, + selection_pi_hi: float = 1.0, + baseline_pi_floor: float = 0.0, + size_checkpoint_dir: Path | None = None, + resume_size_checkpoint: Path | None = None, + checkpoint_identity: Mapping[str, Any] | None = None, + checkpoint_provenance: Mapping[str, Any] | None = None, + progress: Callable[[str], None] | None = None, + progress_events: Callable[[dict[str, object]], None] | None = None, +) -> UKRowwiseDoctrineSolve: + """Solve rowwise household weights under the reviewed doctrine. + + ``progress`` receives readable lines (every 100 epochs, each probe, the + search stop); ``progress_events`` receives every raw calibrator event as a + dict (``calibration_epoch``, ``budget_probe``, ``budget_search_done``), + phase-tagged by the size machinery, so a build driver can publish staging + telemetry without changing the lines a log reader follows. + + ``selection_seed`` (default ``seed``) seeds only the size selection — + the informed L0 search, the exact-count draw and the refit — so two + selections can be compared on one pool and one dense reference. + + ``size_checkpoint_dir`` persists the dense solve and the informed L0 + search of a ``dataset_households`` solve before the exact-count draw + (:mod:`microcosm.build.uk_runtime.size_checkpoint`), stamped with + ``checkpoint_identity``; ``resume_size_checkpoint`` restores such a + checkpoint instead of solving and searching again, refusing when the + identity, the pool or the target surface differ. The draw's threshold + (``selection_pi_hi``) may differ from the one the search stopped on; the + size receipt records both. ``baseline_pi_floor`` trims the refit's + Horvitz–Thompson baseline (see + :func:`~microcosm.build.uk_runtime.dataset_size.refit_uk_dataset_size`); + it is a refit setting, so a resumed checkpoint may use a different one. + + ``progress`` receives one readable line per hundred epochs of the dense + solve, of every budget probe and of the refit, one line per finished + probe with its drawability verdict, and one when the search stops + (:func:`~microcosm.build.uk_runtime.solve_progress.uk_solve_progress_callback`). + + Structurally knob-free like before the ``calibrate()`` migration: no + per-target parameters and no doctrine parameter — the bounds always come + from :data:`UK_LOCAL_SOLVE_DOCTRINE` and ride into the public front door + as explicit arguments. Initial weights are the frame's typed household + weights directly (a rowwise household exists in exactly one area, so + nothing is split); zero weights are refused — a dead row must be dropped + or revived upstream with a recorded mass change, never resurrected by a + solver floor. The kernel enforces the ``CALIBRATED`` kind transition and + mints the mass record (reason from + :func:`rowwise_calibration_mass_reason`); the returned frame carries + both, with the persisted ``household_weight`` column refreshed. + """ + + prepared = prepare_uk_full_solve( + frame, + problem, + bound_families=bound_families, + national_rows=national_rows, + target_weight_rule=target_weight_rule, + ) + doctrine = UK_LOCAL_SOLVE_DOCTRINE + target_set = prepared.target_set if (size_checkpoint_dir is not None or resume_size_checkpoint is not None) and ( dataset_households is None ): @@ -1275,21 +1388,15 @@ def solve_uk_rowwise_weights_under_doctrine( + "." ) else: - result = calibrate( - frame, - target_set, - weight_entity="household", + result = solve_uk_dense_reference( + prepared, epochs=epochs, learning_rate=learning_rate, - mass=CONSERVE_MASS if conserve_mass else FREE_MASS, - mass_reason=None if conserve_mass else mass_reason, - max_weight_ratio=doctrine.max_weight_ratio, + conserve_mass=conserve_mass, target_records=target_records, l0_lambda=l0_lambda, budget_iters=budget_iters, seed=seed, - target_loss_weights=target_loss_weights, - target_loss_cap=doctrine.target_loss_cap, progress_callback=progress_callback, ) selected_support = None @@ -1358,6 +1465,35 @@ def solve_uk_rowwise_weights_under_doctrine( # always hands the refit the selection it just searched or restored. size_receipt["selection_reused"] = restored is not None size_receipt["checkpoint"] = checkpoint_receipt + return finish_uk_full_solve( + prepared, + result, + restore=restore, + selected_support=selected_support, + size_receipt=size_receipt, + dense_result=dense_result, + ) + + +def finish_uk_full_solve( + prepared: UKPreparedFullSolve, + result: CalibrationResult, + *, + restore: Callable[[Frame], Frame] | None = None, + selected_support: np.ndarray | None = None, + size_receipt: Mapping[str, Any] | None = None, + dense_result: CalibrationResult | None = None, +) -> UKRowwiseDoctrineSolve: + """Restore clean inputs and label evidence after the one selected solve.""" + + frame = prepared.frame + problem = prepared.problem + national_rows = prepared.national_rows + target_set = prepared.target_set + target_loss_weights = prepared.target_loss_weights + binding_adjudications = prepared.binding_adjudications + local_count = len(problem.targets) + doctrine = UK_LOCAL_SOLVE_DOCTRINE evidence = _doctrine_solve_evidence( result, target_set=target_set, @@ -1572,10 +1708,19 @@ def _doctrine_solve_evidence( ) scales = default_target_loss_scales(targets_vec) - local_targets_vec = targets_vec[:local_count] - local_scales = scales[:local_count] - local_initial = initial_estimates[:local_count] - local_final = final_estimates[:local_count] + # Identity joins avoid making local-prefix/national-suffix ordering a + # public diagnostic contract. Compilation order is still checked above. + row_positions = { + diagnostic.name: i for i, diagnostic in enumerate(result.diagnostics) + } + local_positions = np.asarray( + [row_positions[target.row_name] for target in _rowwise_target_set(problem)], + dtype=np.int64, + ) + local_targets_vec = targets_vec[local_positions] + local_scales = scales[local_positions] + local_initial = initial_estimates[local_positions] + local_final = final_estimates[local_positions] diagnostics = problem.target_frame.copy() diagnostics["target"] = local_targets_vec diagnostics["initial_estimate"] = local_initial @@ -1596,10 +1741,14 @@ def _doctrine_solve_evidence( target_frame=problem.target_frame, ) national_specs = () if national_rows is None else national_rows.registry.specs - national_initial = initial_estimates[local_count:] - national_final = final_estimates[local_count:] - national_targets_vec = targets_vec[local_count:] - national_scales = scales[local_count:] + national_positions = np.asarray( + [row_positions[spec.to_target().row_name] for spec in national_specs], + dtype=np.int64, + ) + national_initial = initial_estimates[national_positions] + national_final = final_estimates[national_positions] + national_targets_vec = targets_vec[national_positions] + national_scales = scales[national_positions] national_diagnostics = pd.DataFrame( { "name": [spec.to_target().row_name for spec in national_specs], @@ -1697,6 +1846,18 @@ def rotated_uk_local_holdout( ) -> dict[str, object]: """Run five local-row rotations with national rows fixed in training.""" + if not len(problem.targets): + return { + "report_only": True, + "method": "rotated_folds", + "outcome": "not_applicable", + "reason": "No local targets were selected for calibration.", + "n_folds": 0, + "folds": [], + "training_national_rows": ( + 0 if national_rows is None else len(national_rows.targets) + ), + } folds = rotated_folds( len(problem.targets), n_folds=UK_LOCAL_HOLDOUT_FOLDS, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py index b8bcc0c29..2358fe9b3 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py @@ -301,27 +301,36 @@ class UKLadderRowwiseDatasetResult: output_path: Path | None = None -def clone_uk_dataset_tables_with_ladder_geography( +@dataclass(frozen=True) +class UKGeographicPool: + """Linked geographic copies before any location draw. + + ``n_clones`` is the total copies K, independent of output household size. + Source IDs, replicate IDs and the original SPI/CGT ancestry stay distinct. + """ + + frame: Frame + n_clones: int + id_multiplier: int + + +def expand_uk_geographic_pool( *, person: pd.DataFrame, benunit: pd.DataFrame, household: pd.DataFrame, - ladder: UkOaLadder, n_clones: int = 1, - seed: int = 42, time_period: int | str | None = None, source_year: int | None = None, id_multiplier: int | None = None, - expected_constituency_vintage: str | None = None, - region_column: str = "region", household_weight_kind: WeightKind = WeightKind.DESIGN, mass_log: tuple[MassChangeRecord, ...] = (), source_lineage_modulus: int | None = None, -) -> UKLadderRowwiseDatasetResult: - """Clone UK tables and assign geography through the OA ladder. +) -> UKGeographicPool: + """Prepare lineage and expand linked entities using shared clone operations. - The result carries a validated UK national frame; clone indices land on - the canonical per-entity :func:`ladder_clone_index_column` names. + The existing clone-major row order, ID multiplier and weight split are + preserved. This operation consumes no randomness. """ _validate_weight_metadata(household_weight_kind, mass_log) @@ -415,16 +424,8 @@ def clone_uk_dataset_tables_with_ladder_geography( _assert_clone_link_alignment(cloned_person, cloned_household) - assigned = assign_uk_geography_ladder( - cloned_household, - ladder, - seed=seed, - expected_constituency_vintage=expected_constituency_vintage, - region_column=region_column, - ).reset_index(drop=True) - output_total = float( - np.asarray(assigned["household_weight"], dtype=np.float64).sum() + np.asarray(cloned_household["household_weight"], dtype=np.float64).sum() ) _assert_household_mass_conserved(input_total, output_total) clone_record = MassChangeRecord( @@ -439,9 +440,60 @@ def clone_uk_dataset_tables_with_ladder_geography( ), ) + return UKGeographicPool( + frame=uk_national_frame( + person=cloned_person, + benunit=cloned_benunit, + household=cloned_household, + time_period=_normalise_time_period(time_period, source_year=source_year), + weight_kind=household_weight_kind, + mass_log=(*mass_log, clone_record), + ), + n_clones=n_clones, + id_multiplier=id_multiplier, + ) + + +def assign_uk_geographic_pool( + pool: UKGeographicPool, + ladder: UkOaLadder, + *, + seed: int = 42, + expected_constituency_vintage: str | None = None, + region_column: str = "region", +) -> Frame: + """Draw and derive current ladder geography, retaining legacy RNG order.""" + + frame = pool.frame + assigned = assign_uk_geography_ladder( + frame.table("household"), + ladder, + seed=seed, + expected_constituency_vintage=expected_constituency_vintage, + region_column=region_column, + ).reset_index(drop=True) + return uk_national_frame( + person=frame.table("person"), + benunit=frame.table("benunit"), + household=assigned, + time_period=uk_time_period(frame), + weight_kind=frame.weights_for("household").kind, + household_weights=frame.weights_for("household").values, + mass_log=frame.mass_log, + ) + + +def validate_uk_geographic_pool( + frame: Frame, + *, + region_column: str = "region", +) -> GateResult: + """Check assigned geography and entity links before contributions compile.""" + + household = frame.table("household") gate = uk_geography_ladder_gate( - assigned, - np.asarray(assigned["household_weight"], dtype=np.float64), + household, + frame.weights_for("household").values, region_column=region_column, ) if not gate.passed: @@ -455,24 +507,61 @@ def clone_uk_dataset_tables_with_ladder_geography( # engine-free unit lane is not asked to import the engine; the whole # roster is asserted by test_uk_local_authority_input.py (microcosm#953). if importlib.util.find_spec("policyengine_uk") is not None: - verify_local_authority_engine_domain(assigned["local_authority"].unique()) + verify_local_authority_engine_domain(household["local_authority"].unique()) + validate_uk_ladder_rowwise_dataset_tables( + frame.table("person"), frame.table("benunit"), household + ) + return gate - validate_uk_ladder_rowwise_dataset_tables(cloned_person, cloned_benunit, assigned) - # The frame construction re-runs linkage validation and binds the typed - # household weights, the mass log, and the time period to the carrier. - frame = uk_national_frame( - person=cloned_person, - benunit=cloned_benunit, - household=assigned, - time_period=_normalise_time_period(time_period, source_year=source_year), - weight_kind=household_weight_kind, - mass_log=(*mass_log, clone_record), + +def clone_uk_dataset_tables_with_ladder_geography( + *, + person: pd.DataFrame, + benunit: pd.DataFrame, + household: pd.DataFrame, + ladder: UkOaLadder, + n_clones: int = 1, + seed: int = 42, + time_period: int | str | None = None, + source_year: int | None = None, + id_multiplier: int | None = None, + expected_constituency_vintage: str | None = None, + region_column: str = "region", + household_weight_kind: WeightKind = WeightKind.DESIGN, + mass_log: tuple[MassChangeRecord, ...] = (), + source_lineage_modulus: int | None = None, +) -> UKLadderRowwiseDatasetResult: + """Clone UK tables and assign geography through the OA ladder. + + The result carries a validated UK national frame; clone indices land on + the canonical per-entity :func:`ladder_clone_index_column` names. + """ + + pool = expand_uk_geographic_pool( + person=person, + benunit=benunit, + household=household, + n_clones=n_clones, + time_period=time_period, + source_year=source_year, + id_multiplier=id_multiplier, + household_weight_kind=household_weight_kind, + mass_log=mass_log, + source_lineage_modulus=source_lineage_modulus, ) + frame = assign_uk_geographic_pool( + pool, + ladder, + seed=seed, + expected_constituency_vintage=expected_constituency_vintage, + region_column=region_column, + ) + gate = validate_uk_geographic_pool(frame, region_column=region_column) return UKLadderRowwiseDatasetResult( frame=frame, gate=gate, n_clones=n_clones, - id_multiplier=id_multiplier, + id_multiplier=pool.id_multiplier, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/size_checkpoint.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/size_checkpoint.py index 6f659849d..f588be885 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/size_checkpoint.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/size_checkpoint.py @@ -232,8 +232,26 @@ def load_uk_size_checkpoint( with the losses recomputed on the compiled system. """ directory = Path(directory) - arrays_path = directory / SIZE_CHECKPOINT_ARRAYS_FILENAME - manifest_path = directory / SIZE_CHECKPOINT_MANIFEST_FILENAME + return load_uk_size_checkpoint_files( + directory / SIZE_CHECKPOINT_MANIFEST_FILENAME, + directory / SIZE_CHECKPOINT_ARRAYS_FILENAME, + frame=frame, + target_set=target_set, + identity=identity, + ) + + +def load_uk_size_checkpoint_files( + manifest_path: Path, + arrays_path: Path, + *, + frame: Frame, + target_set: TargetSet, + identity: Mapping[str, Any], +) -> UKSizeCheckpointRestore: + """Read the same checkpoint from separately content-verified source paths.""" + manifest_path, arrays_path = Path(manifest_path), Path(arrays_path) + directory = manifest_path.parent for required in (arrays_path, manifest_path): if not required.is_file(): raise FileNotFoundError(f"size checkpoint file missing: {required}.") diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py index 9f596580b..d2ca1c9dd 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py @@ -28,7 +28,7 @@ PRIMARY_QRF_WORKER_MODULE = "microcosm.build.us_runtime.puf_qrf_worker" PRIMARY_QRF_INTERPRETER_PLACEHOLDER = "{python_interpreter}" APPROVED_UV_LOCK_SHA256 = ( - "52e3d128869efa029f9a7557abf44d6b18861c01237abce082fdb0a72489603a" + "696ea49c875ae8e811a9d5e624e14b199252017df2b6eeb4161fcedf5578a554" ) LEGACY_CAMPAIGN_UV_LOCK_SHA256 = ( "27f47e385cfa35e2644a37410d1804b361ad9aee123577551c8421547bda65ee" diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_country_adapter.py b/packages/microcosm-build/tests/engine/uk/test_uk_country_adapter.py new file mode 100644 index 000000000..f7f5220e2 --- /dev/null +++ b/packages/microcosm-build/tests/engine/uk/test_uk_country_adapter.py @@ -0,0 +1,17 @@ +"""The UK country adapter compiles the canonical full graph on the engine.""" + +from microcosm.build.uk_runtime.country_adapter import build_uk_country_graph +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release + + +def test_country_adapter_compiles_the_same_full_graph_with_all_targets_default(): + built = build_uk_country_graph() + release = load_uk_frs_release() + assert built.config.geography_levels is None + assert built.config.calibration_year == release.calibration_year + assert built.config.source_year == release.survey_year + kernels = {node.kernel for node in built.graph.nodes} + assert {"uk.create@1", "uk.full.target_compilation@1", "uk.full.dense@1"} <= kernels + assert not any( + "national_candidate" in source.name for source in built.graph.sources + ) diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_full_graph_admission.py b/packages/microcosm-build/tests/engine/uk/test_uk_full_graph_admission.py new file mode 100644 index 000000000..4a9134441 --- /dev/null +++ b/packages/microcosm-build/tests/engine/uk/test_uk_full_graph_admission.py @@ -0,0 +1,205 @@ +"""Real full-graph gates refuse synthetic evidence before any solver or export.""" + +import json +from dataclasses import replace + +import pytest + +from microcosm.build.uk_runtime import graph_calibration +from microcosm.build.uk_runtime.full_build_cli import _through +from microcosm.build.uk_runtime.full_certification import ( + append_uk_full_certification_node, + register_uk_full_certification_kernel, +) +from microcosm.build.uk_runtime.graph_build import ( + SPINE_PROVENANCE_TYPE, + UKFullBuildConfig, + register_uk_full_kernels, + uk_full_graph, +) +from microcosm.build.uk_runtime.graph_calibration import UKGraphCalibrationConfig +from microcosm.build.uk_runtime.graph_terminal import ( + FULL_GATE_REPORT_TYPE, + add_uk_export_continuation, + add_uk_export_preparation, + append_uk_full_gate_nodes, + decode_full_gate_report, + register_uk_full_gate_kernels, + register_uk_terminal_kernels, +) +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ContentStore, + Graph, + KernelRegistry, + KernelResult, + NodeRejectedError, + compile_graph, + run_graph, +) +from microcosm.graph.canonical import canonical_json +from test_support.microcosm_build.uk_full_population_graph import ( + Source, + graph_and_registry, +) +from test_support.microcosm_build.uk_full_target_graph import ( + target_inputs as target_inputs, +) +from test_support.microcosm_build.uk_ladder_rowwise_clone import ( + toy_ladder as toy_ladder, +) + + +class IncompleteSource(Source): + ref = "uk.test.full-incomplete-source@1" + + def run(self, context): + return KernelResult( + frame=super().run(context).frame, + artifacts={ + "provenance": canonical_json( + { + "stages": ["synthetic_fixture"], + "stage_evidence": {}, + "fit_weight_records": {}, + } + ) + }, + ) + + +def test_real_full_graph_preflight_replays_and_blocks_dense_export_and_certification( + target_inputs, toy_ladder, tmp_path, monkeypatch +): + """Source adapters are synthetic; gate, solver and terminal owners are real. + + A tiny fixture lacks the approved national/local reference surface and + canonical source roster. Its proper scientific outcome is refusal, never + fabricated passing release evidence to drive a successful filesystem test. + """ + _, ladder_path = toy_ladder + primitive, _ = graph_and_registry(1) + source = replace( + primitive.node("source"), + kernel=IncompleteSource.ref, + artifact_outputs=(ArtifactOutput("provenance", SPINE_PROVENANCE_TYPE),), + ) + base = Graph( + "uk", + tuple(item for item in primitive.sources if item.name == "fixture"), + (source,), + ) + full = uk_full_graph( + UKFullBuildConfig( + calibration_year=2026, + time_period="2023", + source_year=2023, + seed=7, + calibration=UKGraphCalibrationConfig(epochs=2, seed=7), + ), + spine=base, + spine_population="source", + ) + # Keep maintained K so the tiny fixture can reach every selected area; + # zero-support targets still refuse before gates at deliberately smaller K. + provenance = ArtifactInput( + "spine_provenance", "source", "provenance", SPINE_PROVENANCE_TYPE + ) + graph = append_uk_full_gate_nodes( + full.graph, + calibration=full.calibration, + spine_stage_names=("synthetic_fixture",), + spine_provenance=provenance, + engine_identity="fixture-engine", + review_date="2026-09-10", + ) + final_gate = ArtifactInput( + "full_gates", "uk.full.gates.calibrated", "gate_report", FULL_GATE_REPORT_TYPE + ) + graph = add_uk_export_preparation( + graph, + population=full.calibration.population, + bindings={"target_scope": "all", "n_clones": full.config.n_clones}, + artifact_inputs=(final_gate,), + ) + graph = add_uk_export_continuation( + graph, + population=full.calibration.population, + manifest_binding={"kind": "fixture", "target_scope": "all"}, + artifact_inputs=(final_gate,), + ) + graph = append_uk_full_certification_node( + graph, population=full.calibration.population, spine_provenance=provenance + ) + compiled = compile_graph(graph) + assert "uk.full.package" in compiled.predecessors["uk.full.certification"] + assert "uk.full.gates.calibrated" in compiled.predecessors["uk.full.export.prepare"] + assert "uk.full.gates.preflight" in compiled.predecessors["uk.full.dense"] + + def registry(): + result = KernelRegistry() + result.register(IncompleteSource()) + register_uk_full_kernels(result) + register_uk_full_gate_kernels( + result, coverage_engine=object(), engine_identity="fixture-engine" + ) + register_uk_terminal_kernels(result) + register_uk_full_certification_kernel(result) + return result + + sources = { + "fixture": ladder_path, + "uk_ladder": ladder_path, + "uk_ledger_facts": ladder_path, + } + store = ContentStore(tmp_path / "store") + checkpoint = compile_graph(_through(graph, "uk.full.gates.preflight")) + cold = run_graph(checkpoint, sources=sources, store=store, kernels=registry()) + key = cold.nodes["uk.full.gates.preflight"].opaque_artifacts["gate_report"] + payload = store.load_bytes(key) + report, enforcement = decode_full_gate_report(payload) + assert report.phase == "preflight" + assert not enforcement["artifact_permitted"] + assert "uk_release_family_build_stages" in enforcement["structural_failures"] + selection = json.loads(payload)["selection_receipt"] + assert selection["selector"]["geography_levels"] is None + assert {row["geography_level"] for row in selection["included"]} >= { + "country", + "constituency", + "la", + } + warm = run_graph( + checkpoint, + sources=sources, + store=store, + kernels=registry(), + resume="require", + ) + assert all(receipt.hit for receipt in warm.nodes.values()) + assert warm.nodes["uk.full.gates.preflight"].opaque_artifacts["gate_report"] == key + + monkeypatch.setattr( + graph_calibration, + "calibrate", + lambda *args, **kwargs: pytest.fail( + "A failed source preflight reached the solver" + ), + ) + with pytest.raises( + NodeRejectedError, match="Dense calibration refused by the source preflight" + ): + run_graph( + compile_graph(_through(graph, "uk.full.dense")), + sources=sources, + store=store, + kernels=registry(), + ) + # The authenticated failure survives the rejected continuation. No later + # gate, export, or certification result has been created by this attempt. + assert store.load_bytes(key) == payload + assert not any( + json.loads(path.read_text()).get("node_id") + in {"uk.full.export.prepare", "uk.full.package", "uk.full.certification"} + for path in (store.root / "objects").glob("*/*/payload.json") + ) diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_full_target_graph.py b/packages/microcosm-build/tests/engine/uk/test_uk_full_target_graph.py new file mode 100644 index 000000000..82c1fc140 --- /dev/null +++ b/packages/microcosm-build/tests/engine/uk/test_uk_full_target_graph.py @@ -0,0 +1,196 @@ +"""Actual full graph on synthetic adapters: the engine-backed target paths.""" + +# ruff: noqa: F403, F405 +from test_support.microcosm_build.uk_full_target_graph import * + + +def test_explicit_country_filter_runs_same_full_graph_without_local_constraints( + target_inputs, toy_ladder, tmp_path +): + _, path = toy_ladder + full, manifest, problem = build(tmp_path, path, ("country",)) + assert problem.problem.names == ("country_households@2026",) + assert {m["geography_level"] for m in problem.target_metadata} == {"country"} + assert manifest.population(full.population).n("household") == 4 + assert len(problem.bindings["target_selection"]["excluded"]) > 0 + assert "uk.full.dense" in manifest.nodes + assert "uk.full.locations" in manifest.nodes + surface = json.loads( + ContentStore(tmp_path / "store").load_bytes( + manifest.nodes["uk.full.target_compilation"].opaque_artifacts["surface"] + ) + ) + assert len(surface["surface"]) == 10 + assert len(surface["household_dispersion"]["cells"]) == 10 + assert surface["census_household_uprating"]["household_cells"]["cells"] == 10 + assert surface["source_validation"]["targets"] == { + "chronicle": {"fixture": True}, + "paired_ladder_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + + +def test_default_all_has_direct_matrix_and_solver_parity_and_replays( + target_inputs, toy_ladder, tmp_path +): + from microcosm.build.uk_runtime.full_problem import build_uk_full_local_problem + from microcosm.build.uk_runtime.local_rowwise import ( + prepare_uk_full_solve, + solve_uk_dense_reference, + ) + from microcosm.build.uk_runtime.rowwise_dataset import ( + clone_uk_dataset_with_ladder_geography, + ) + from test_support.microcosm_build.uk_full_population_graph import source_frame + + ladder, path = toy_ladder + default, default_run, default_problem = build( + tmp_path / "default", path, None, n_clones=10 + ) + explicit, explicit_run, explicit_problem = build( + tmp_path / "explicit", + path, + ("country", "region", "constituency", "la"), + n_clones=10, + ) + assert {row["geography_level"] for row in default_problem.target_metadata} == { + "country", + "region", + "constituency", + "la", + } + assert default_problem.problem.n_targets == 12 + by_grain = {} + for value, metadata in zip( + default_problem.problem.target_vector, + default_problem.target_metadata, + strict=True, + ): + if metadata["materialization"] == "uk_local_surface": + assert metadata["contract_target_id"] == "ons.census.households" + by_grain.setdefault(metadata["geography_level"], []).append(value) + assert sum(by_grain["constituency"]) == pytest.approx(33.0) + assert sum(by_grain["la"]) == pytest.approx(33.0) + assert len(set(by_grain["constituency"])) == 1 + assert len(set(by_grain["la"])) > 1 + census_receipt = default_problem.bindings["cross_geography"][ + "census_household_uprating" + ] + assert census_receipt["grains"]["constituency"]["factor"] == 33.0 / 200.0 + assert census_receipt["grains"]["local_authority"]["factor"] == 33.0 / 195.0 + assert default_problem.problem.names == explicit_problem.problem.names + np.testing.assert_array_equal( + default_problem.problem.matrix.toarray(), + explicit_problem.problem.matrix.toarray(), + ) + np.testing.assert_array_equal( + default_problem.problem.target_vector, explicit_problem.problem.target_vector + ) + np.testing.assert_array_equal( + default_run.population(default.population).weights_for("household").values, + explicit_run.population(explicit.population).weights_for("household").values, + ) + assert default_problem.bindings["target_selection"]["selector"]["explicit"] is False + assert explicit_problem.bindings["target_selection"]["selector"]["explicit"] is True + + # Independently execute the maintained pre-graph numerical helpers on the + # same original spine, legacy location draw, target rows and solver options. + assignment = clone_uk_dataset_with_ladder_geography( + source_frame(), + ladder, + n_clones=10, + seed=7, + source_year=2023, + expected_constituency_vintage="2024_pcon", + ) + prepared_frame, _, national_rows, metrics, _ = target_inputs["measures"]( + assignment.frame, target_inputs["national"], local_grains=("constituency", "la") + ) + surface, cross = target_inputs["surface"]() + _, local, _, families, _ = build_uk_full_local_problem( + SimpleNamespace(result=SimpleNamespace(frame=prepared_frame), ladder=ladder), + local_registry=target_inputs["local"], + national_registry=target_inputs["national"], + local_metrics=metrics, + period=2026, + sample_fraction=1.0, + reviewed_unbound_higher_targets={}, + selected_surface=surface, + surface_receipt=cross, + ) + prepared = prepare_uk_full_solve( + prepared_frame, + local, + bound_families=families, + national_rows=national_rows, + target_weight_rule="uniform", + ) + direct = solve_uk_dense_reference(prepared, epochs=8, seed=7) + np.testing.assert_array_equal( + default_problem.problem.matrix.toarray(), direct.problem.matrix.toarray() + ) + np.testing.assert_array_equal( + default_run.population(default.population).weights_for("household").values, + direct.weights, + ) + + _, replay, _ = build( + tmp_path / "default", + path, + None, + n_clones=10, + resume="require", + forbid_execution=True, + ) + assert all(receipt.hit for receipt in replay.nodes.values()) + np.testing.assert_array_equal( + replay.population(default.population).weights_for("household").values, + direct.weights, + ) + + +def test_unsupported_default_all_refuses_without_narrowing( + target_inputs, toy_ladder, tmp_path +): + from microcosm.graph.errors import NodeRejectedError + + _, path = toy_ladder + # At K=1 London's only household cannot occupy both positive area cells. + with pytest.raises(NodeRejectedError, match="[Ss]upport|unassigned|positive"): + build(tmp_path / "all", path, None, n_clones=1) + country, result, problem = build( + tmp_path / "country", path, ("country",), n_clones=1 + ) + assert problem.problem.names == ("country_households@2026",) + assert result.population(country.population).n("household") == 4 + + +@pytest.mark.parametrize("levels", [None, ("country",)]) +def test_source_census_validation_precedes_any_geography_filter( + target_inputs, toy_ladder, tmp_path, levels +): + from microcosm.graph.errors import NodeRejectedError + + local = target_inputs["local"] + # A malformed NI mapping must fail even when no local constraint is selected. + target_inputs["inputs"]["local_registry"] = TargetRegistry( + [ + replace(spec, value=2.0) + if spec.metadata["geography_id"] == "N05000001" + else spec + for spec in local + ], + country="uk", + ) + with pytest.raises(NodeRejectedError, match="dispersion exceeds"): + build(tmp_path, toy_ladder[1], levels) + + +def test_target_kernel_identity_binds_country_reference_resources(monkeypatch): + kernel = graph_targets.UKFullProblemKernel() + original = kernel.implementation_hash() + monkeypatch.setattr( + graph_targets, + "load_country_spec", + lambda _country: SimpleNamespace(fingerprint="changed-reference-resource"), + ) + assert kernel.implementation_hash() != original diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_graph_terminal.py b/packages/microcosm-build/tests/engine/uk/test_uk_graph_terminal.py new file mode 100644 index 000000000..203aab510 --- /dev/null +++ b/packages/microcosm-build/tests/engine/uk/test_uk_graph_terminal.py @@ -0,0 +1,22 @@ +"""Terminal-node identity that binds the country reference resources.""" + + +def test_gate_cache_identity_includes_country_reference_resource_bytes(monkeypatch): + from types import SimpleNamespace + + from microcosm.build import country_spec + from microcosm.build.uk_runtime.graph_terminal import UKFullGateKernel + + kernel = UKFullGateKernel(coverage_engine=object(), engine_identity="fixture") + monkeypatch.setattr( + country_spec, + "load_country_spec", + lambda country: SimpleNamespace(fingerprint="a" * 64), + ) + first = kernel.implementation_hash() + monkeypatch.setattr( + country_spec, + "load_country_spec", + lambda country: SimpleNamespace(fingerprint="b" * 64), + ) + assert kernel.implementation_hash() != first diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_country_bundles.py index 6752d1651..8a0397375 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_country_bundles.py @@ -34,7 +34,7 @@ ), ( "uk", - 2023, + 2025, { "benunit.benunit_id", "household.household_id", @@ -115,6 +115,7 @@ def test_country_kernel_contract_ids_are_closed_in_the_compiler_registry() -> No "clone_assign_communes", "be_commune_geography_gate", "load_uk_national_frame", + "build_uk_frs_spine", "assign_uk_geography_ladder", "uk_geography_ladder_gate", } diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_calibration_run.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_calibration_run.py index 87266a477..9e294ecc2 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_calibration_run.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_calibration_run.py @@ -4,6 +4,74 @@ from test_support.microcosm_build.uk_calibration_run import * +def test_strict_checkpoint_binds_contents_and_retains_gate_payload(tmp_path): + frame = _frame() + path, gate_path, _ = _bound_checkpoint(tmp_path, frame) + sidecar = calibration_run.load_bound_spine_checkpoint(path, frame) + provenance = calibration_run.strict_spine_provenance_from_sidecar(path, sidecar) + assert provenance["fit_weight_records"] == sidecar["fit_weight_records"] + assert provenance["spine_gate_report"]["payload"] == json.loads( + gate_path.read_bytes() + ) + + +@pytest.mark.parametrize( + "mutation", + [ + "missing_identity", + "wrong_identity", + "bypass", + "gate_bytes", + "missing_gate_binding", + "gate_roster", + "gate_policy", + ], +) +def test_strict_checkpoint_rejects_unbound_or_changed_evidence(tmp_path, mutation): + frame = _frame() + path, gate_path, sidecar = _bound_checkpoint(tmp_path, frame) + if mutation == "missing_identity": + sidecar.pop("uk_frame_content_identity") + elif mutation == "wrong_identity": + sidecar["uk_frame_content_identity"] = "f" * 64 + elif mutation == "bypass": + sidecar["spine_gate_bypass"] = {"reviewed": True, "reason": "historical"} + elif mutation == "gate_bytes": + gate_path.write_text(gate_path.read_text() + "\n") + elif mutation == "missing_gate_binding": + sidecar.pop("spine_gate_report") + elif mutation == "gate_policy": + report = json.loads(gate_path.read_bytes()) + report["policy_sha256"] = "f" * 64 + gate_path.write_text(json.dumps(report)) + sidecar["spine_gate_report"]["sha256"] = hashlib.sha256( + gate_path.read_bytes() + ).hexdigest() + else: + report = json.loads(gate_path.read_bytes()) + report["gates"].pop(next(iter(report["gates"]))) + gate_path.write_text(json.dumps(report)) + sidecar["spine_gate_report"]["sha256"] = hashlib.sha256( + gate_path.read_bytes() + ).hexdigest() + path.write_text(json.dumps(sidecar)) + with pytest.raises(ValueError): + calibration_run.load_bound_spine_checkpoint(path, frame) + + +def test_strict_checkpoint_accepts_explicit_declared_gate_path(tmp_path): + frame = _frame() + path, gate_path, _ = _bound_checkpoint(tmp_path, frame) + moved = gate_path.rename(tmp_path / "declared-gates.json") + sidecar = calibration_run.load_bound_spine_checkpoint( + path, frame, gate_report_path=moved + ) + provenance = calibration_run.strict_spine_provenance_from_sidecar( + path, sidecar, gate_report_path=moved + ) + assert provenance["spine_gate_report"]["path"] == str(moved) + + def test_gate_scope_classifies_every_uk_gate(): all_ids = {entry.id for entry in load_country_spec("uk").gates.gates} assert ( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_country_adapter.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_country_adapter.py new file mode 100644 index 000000000..96e531fbb --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_country_adapter.py @@ -0,0 +1,36 @@ +"""The UK country adapter uses raw FRS sources and the canonical full graph.""" + +import shutil + +import pytest + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.uk_runtime.country_adapter import ( + validate_uk_country_source_projection, +) +from test_support.paths import paths_for + + +def test_country_raw_source_projection_is_current(): + spec = load_country_spec("uk") + validate_uk_country_source_projection(spec) + sources = spec.resolved_spec.resource("sources").domain.to_wire()["sources"] + assert all(row["role"] == "frs_raw_table" for row in sources) + assert all(row["loader"] == "kernel:build_uk_frs_spine" for row in sources) + assert {"frs_adult", "frs_benefits", "frs_child", "frs_househol"} <= { + row["id"] for row in sources + } + assert "uk_national_candidate_2023" not in str(sources) + + +def test_country_source_projection_refuses_divergent_header_pin(tmp_path): + source = paths_for("microcosm-build").package / "src/microcosm/build/uk" + destination = tmp_path / "uk" + shutil.copytree(source, destination) + path = destination / "spec/sources.yaml" + text = path.read_text() + first_sha = text.split(" sha256: ", 1)[1].splitlines()[0] + path.write_text(text.replace(first_sha, "f" * 64, 1)) + spec = load_country_spec(destination) + with pytest.raises(ValueError, match="raw-source pins differ"): + validate_uk_country_source_projection(spec) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py new file mode 100644 index 000000000..2052c6858 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py @@ -0,0 +1,463 @@ +"""The canonical CLI restores declared files and preserves failure/scope semantics.""" + +import hashlib +import json +from dataclasses import replace +from pathlib import Path + +import pytest + +from microcosm.build.gate_battery import ( + GateOutcome, + GatePhaseReport, + GateStatus, + gate_phase_report_payload, +) +from microcosm.build.gates import GateResult +from microcosm.build.uk_runtime import full_build_cli as cli +from microcosm.build.uk_runtime.full_certification import FULL_CERTIFICATION_TYPE +from microcosm.build.uk_runtime.full_gates import ( + classify_full_gate_outcomes, + uk_full_gate_manifest, +) +from microcosm.build.uk_runtime.graph_build import UKFullBuildConfig, UKFullGraph +from microcosm.build.uk_runtime.graph_calibration import UKCalibrationNodes +from microcosm.build.uk_runtime.graph_targets import TARGET_SELECTION_TYPE +from microcosm.build.uk_runtime.graph_terminal import ( + FULL_DIAGNOSTICS_CSV_TYPE, + FULL_DIAGNOSTICS_TYPE, + FULL_GATE_REPORT_TYPE, + FULL_HOLDOUT_TYPE, + FULL_SUPPORT_CSV_TYPE, + add_uk_export_preparation, + register_uk_terminal_kernels, +) +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json +from test_support.microcosm_build.uk_graph_terminal import _frame + + +def arguments(tmp_path, *extra): + return cli.parse_args( + [ + "--input-h5", + str(tmp_path / "spine.h5"), + "--ladder", + str(tmp_path / "ladder.npz"), + "--ledger-facts", + str(tmp_path / "ledger"), + "--out", + str(tmp_path / "out"), + *extra, + ] + ) + + +def test_every_scope_and_size_control_keeps_default_all(tmp_path): + for extra in ( + (), + ("--target-geographies", "all"), + ("--n-clones", "1"), + ("--dataset-households", "10"), + ("--n-clones", "1", "--dataset-households", "10"), + ): + assert arguments(tmp_path, *extra).target_geographies is None + assert arguments( + tmp_path, "--target-geographies", "country" + ).target_geographies == ("country",) + with pytest.raises(SystemExit): + arguments(tmp_path, "--target-geographies", "national") + + +def test_source_sampling_cannot_be_reapplied_as_pool_sampling(): + config = UKFullBuildConfig(calibration_year=2025, source_sample_fraction=0.1) + assert config.sample_fraction == 1.0 + assert config.effective_sample_fraction == 0.1 + with pytest.raises(ValueError, match="second time"): + replace(config, sample_fraction=0.1) + + +def gate_payload(phase, failed=None): + selection = { + "schema": "microcosm.calibrate.target-selection.v1", + "selector": {"geography_levels": None, "explicit": False}, + "included": [ + {"name": "count", "period": 2025, "geography_level": "country"}, + {"name": "local", "period": 2025, "geography_level": "constituency"}, + ], + "excluded": [], + } + gates = uk_full_gate_manifest(selection) + report = GatePhaseReport( + phase, + tuple( + GateOutcome( + entry, + GateStatus.FAILED if entry.id == failed else GateStatus.PASSED, + GateResult( + name=entry.id, + passed=entry.id != failed, + details={}, + failures=("synthetic failure",) if entry.id == failed else (), + ), + ) + for entry in gates.gates + if entry.phase == phase + ), + ) + return canonical_json( + { + "schema_version": 1, + "kind": "uk_full_gate_report", + "selection_receipt": selection, + "sample_fraction": 1.0, + "release_candidate": False, + "report": gate_phase_report_payload(report, gates=gates), + "enforcement": classify_full_gate_outcomes( + report, sample_fraction=1.0, release_candidate=False + ), + } + ) + + +class Fixture(KernelBase): + ref = "uk.test.cli-frame@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def implementation_hash(self): + return hashlib.sha256(self.ref.encode()).hexdigest() + + def run(self, context): + return KernelResult(frame=_frame()) + + +class Evidence(KernelBase): + ref = "uk.test.cli-evidence@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def implementation_hash(self): + return hashlib.sha256(self.ref.encode()).hexdigest() + + def run(self, context): + phase = context.params["phase"] + artifacts = {"gate_report": gate_payload(phase, context.params.get("failed"))} + if phase == "terminal": + artifacts.update( + calibration_diagnostics=b'{"fixture":true}', + target_diagnostics_csv=b"name,actual\ncount,100\n", + area_support_csv=b"area,households\nfixture,2\n", + ) + return KernelResult(artifacts=artifacts) + + +class Holdout(Evidence): + ref = "uk.test.cli-holdout@1" + + def run(self, context): + return KernelResult( + artifacts={"holdout": b'{"fixture":true}', "selection": b'{"fixture":true}'} + ) + + +class Certification(Evidence): + ref = "uk.test.cli-certification@1" + + def run(self, context): + return KernelResult( + artifacts={ + "certification_readiness": b'{"fixture":true,"release_authorized":false}' + } + ) + + +@pytest.fixture(autouse=True) +def certification_service_fixture(monkeypatch): + # Scientific certification validation has its own graph-artifact tests. + # This suite tests the filesystem/execution service with synthetic evidence. + def append(graph, *, population, **kwargs): + return replace( + graph, + nodes=( + *graph.nodes, + Node( + "uk.full.certification", + Certification.ref, + population=population, + artifact_outputs=( + ArtifactOutput( + "certification_readiness", FULL_CERTIFICATION_TYPE + ), + ), + ), + ), + ) + + monkeypatch.setattr(cli, "append_uk_full_certification_node", append) + + +def prepared(tmp_path, failed=None): + frame = _frame() + fixture = tmp_path / "fixture.txt" + fixture.write_text("constant source") + identifiers = { + "person_id", + "person_household_id", + "person_benunit_id", + "household_id", + "benunit_id", + } + root = Node( + "uk.full.calibrated", + Fixture.ref, + structural=StructuralDelta.CREATE, + sources=("fixture",), + outputs=tuple( + Owned( + e, + str(c), + "string" + if frame.table(e)[c].dtype.kind in "OUS" + else str(frame.table(e)[c].dtype), + ) + for e in frame.entities + for c in frame.table(e).columns + if c not in identifiers + ), + ) + nodes = [ + root, + Node( + "uk.full.gates.preflight", + Evidence.ref, + population=root.id, + params={"phase": "preflight", "failed": failed}, + artifact_outputs=(ArtifactOutput("gate_report", FULL_GATE_REPORT_TYPE),), + ), + Node( + "uk.full.gates.calibrated", + Evidence.ref, + population=root.id, + params={"phase": "terminal", "failed": failed}, + artifact_outputs=( + ArtifactOutput("gate_report", FULL_GATE_REPORT_TYPE), + ArtifactOutput("calibration_diagnostics", FULL_DIAGNOSTICS_TYPE), + ArtifactOutput("target_diagnostics_csv", FULL_DIAGNOSTICS_CSV_TYPE), + ArtifactOutput("area_support_csv", FULL_SUPPORT_CSV_TYPE), + ), + ), + Node( + "uk.full.holdout", + Holdout.ref, + population=root.id, + artifact_outputs=( + ArtifactOutput("holdout", FULL_HOLDOUT_TYPE), + ArtifactOutput("selection", TARGET_SELECTION_TYPE), + ), + ), + ] + # CLI materialization consumes the public target-selection endpoint. + nodes.append( + Node( + "uk.full.target_selection", + Holdout.ref, + population=root.id, + artifact_outputs=( + ArtifactOutput("holdout", FULL_HOLDOUT_TYPE), + ArtifactOutput("selection", TARGET_SELECTION_TYPE), + ), + ) + ) + nodes = [ + replace( + node, + artifact_inputs=( + ArtifactInput( + "preflight", + "uk.full.gates.preflight", + "gate_report", + FULL_GATE_REPORT_TYPE, + ), + ArtifactInput( + "holdout", "uk.full.holdout", "holdout", FULL_HOLDOUT_TYPE + ), + ArtifactInput( + "selection", + "uk.full.target_selection", + "selection", + TARGET_SELECTION_TYPE, + ), + ), + ) + if node.id == "uk.full.gates.calibrated" + else node + for node in nodes + ] + graph = Graph("uk", (SourceRef("fixture", "raw-bytes-v1"),), tuple(nodes)) + graph = add_uk_export_preparation( + graph, + population=root.id, + bindings={"target_scope": "all"}, + artifact_inputs=( + ArtifactInput( + "gates", + "uk.full.gates.calibrated", + "gate_report", + FULL_GATE_REPORT_TYPE, + ), + ), + ) + calibration = UKCalibrationNodes( + (), root.id, "unused", "unused", "unused", None, "unused" + ) + full = UKFullGraph(graph, calibration, UKFullBuildConfig(calibration_year=2025)) + kernels = KernelRegistry() + for kernel in (Fixture(), Evidence(), Holdout(), Certification()): + kernels.register(kernel) + register_uk_terminal_kernels(kernels) + return cli.PreparedUKFullBuild( + full, kernels, {"fixture": fixture}, {"target_scope": "all"} + ) + + +def test_cli_cold_and_required_replay_recreate_dataset_and_sidecars( + tmp_path, monkeypatch +): + pytest.importorskip("tables") + args = arguments(tmp_path) + first = prepared(tmp_path) + assert cli.execute_full_build(first, args) == 0 + out = args.out + expected = json.loads((out / "build.json").read_text()) + assert expected["readback_passed"] is True + assert expected["release_authorized"] is False + assert (out / "microcosm_uk_2025.targets.csv").read_text().startswith("name,actual") + for path in out.iterdir(): + if path.is_file(): + path.unlink() + + def forbidden(*args): + raise AssertionError( + "Required replay repeated a completed numerical/evidence node" + ) + + for kernel in (Fixture, Evidence, Holdout): + monkeypatch.setattr(kernel, "run", forbidden) + args.resume = "require" + assert cli.execute_full_build(prepared(tmp_path), args) == 0 + actual = json.loads((out / "build.json").read_text()) + assert actual["content_sha256"] == expected["content_sha256"] + assert (out / "microcosm_uk_2025.h5").is_file() + assert (out / "microcosm_uk_2025.holdout.json").is_file() + + +@pytest.mark.parametrize( + "failure,exported", + [ + ("uk_local_geography_ladder_post_calibration", False), + ("uk_local_target_fit", True), + ], +) +def test_cli_retains_failed_evidence_and_correct_status(tmp_path, failure, exported): + if exported: + pytest.importorskip("tables") + args = arguments(tmp_path) + build = prepared(tmp_path, failure) + assert cli.execute_full_build(build, args) == 1 + assert (args.out / "uk.full.gates.calibrated.gate_report.json").is_file() + assert (args.out / "microcosm_uk_2025.h5").exists() == exported + + +def test_dry_run_has_no_files_or_kernel_execution(tmp_path, monkeypatch, capsys): + args = arguments(tmp_path, "--dry-run") + build = prepared(tmp_path) + monkeypatch.setattr( + cli, "run_graph", lambda *a, **k: pytest.fail("dry run executed graph") + ) + assert cli.execute_full_build(build, args) == 0 + assert json.loads(capsys.readouterr().out)["default_scope"] == "all_geographies" + assert not args.out.exists() + + +def test_rejected_output_inside_source_never_writes_failure_sidecar( + tmp_path, monkeypatch +): + args = arguments(tmp_path) + build = prepared(tmp_path) + build = replace(build, sources={"fixture": tmp_path}) + monkeypatch.setattr(cli, "parse_args", lambda argv: args) + monkeypatch.setattr(cli, "prepare_full_build", lambda args: build) + assert cli.main([]) == 1 + assert not args.out.exists() + + +@pytest.mark.parametrize("resume", ["auto", "require"]) +def test_source_phase_evidence_survives_later_exception(tmp_path, monkeypatch, resume): + args = arguments(tmp_path) + build = prepared(tmp_path) + assembled = Node( + "spine.gates.assembled", + Evidence.ref, + population="uk.full.calibrated", + params={"phase": "preflight", "failed": None}, + artifact_outputs=(ArtifactOutput("gate_report", FULL_GATE_REPORT_TYPE),), + ) + transferred = replace(assembled, id="spine.gates.transferred") + graph = replace( + build.full.graph, nodes=(*build.full.graph.nodes, assembled, transferred) + ) + build = replace(build, full=replace(build.full, graph=graph)) + args.out.mkdir() + previous = b'{"kind":"previous-complete-build"}' + (args.out / "build.json").write_bytes(previous) + args.resume = resume + original_run = cli.run_graph + if resume == "require": + original_run( + cli.compile_graph(cli._through(graph, assembled.id)), + sources=build.sources, + store=cli.ContentStore(args.out / ".graph-store"), + kernels=build.kernels, + ) + monkeypatch.setattr( + Fixture, "run", lambda *a: pytest.fail("Checkpoint replay reran the source") + ) + monkeypatch.setattr( + Evidence, "run", lambda *a: pytest.fail("Checkpoint replay reran the gate") + ) + + def refuse_transferred(compiled, **kwargs): + if transferred.id in {node.id for node in compiled.graph.nodes}: + raise RuntimeError("downstream source donor bin is empty") + return original_run(compiled, **kwargs) + + monkeypatch.setattr(cli, "run_graph", refuse_transferred) + monkeypatch.setattr(cli, "parse_args", lambda argv: args) + monkeypatch.setattr(cli, "prepare_full_build", lambda args: build) + assert cli.main([]) == 1 + assert (args.out / "build.json").read_bytes() == previous + failure = json.loads((args.out / "failure.json").read_bytes()) + durable = Path(failure["evidence_directory"]) + assert (durable / "spine.gates.assembled.graph.json").is_file() + assert not (durable / "spine.gates.transferred.graph.json").exists() + gate_bytes = (durable / "spine.gates.assembled.gate_report.json").read_bytes() + assert gate_bytes == gate_payload("preflight") + index = json.loads((durable / "evidence-index.json").read_bytes()) + assert ( + index["artifacts"]["spine.gates.assembled/gate_report"]["sha256"] + == hashlib.sha256(gate_bytes).hexdigest() + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_preparation.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_preparation.py new file mode 100644 index 000000000..80e3766c3 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_preparation.py @@ -0,0 +1,112 @@ +"""Canonical CLI preparation authenticates real H5 checkpoint boundaries.""" + +import json + +import pytest + +from microcosm.build.uk_runtime import full_build_cli as cli +from microcosm.build.uk_runtime import spine_build +from microcosm.build.uk_runtime.national_frame import ( + load_uk_national_frame, + write_uk_national_frame, +) +from microcosm.graph import ContentStore, compile_graph, run_graph +from test_support.microcosm_build.uk_calibration_run import _bound_checkpoint +from test_support.microcosm_build.uk_full_population_graph import source_frame +from test_support.microcosm_build.uk_ladder_rowwise_clone import ( + toy_ladder as toy_ladder, +) + + +@pytest.fixture +def checkpoint_request(tmp_path, toy_ladder, monkeypatch): + pytest.importorskip("tables") + _, ladder_path = toy_ladder + path = write_uk_national_frame(source_frame(), tmp_path / "spine.h5") + frame, _ = load_uk_national_frame(path) + sidecar_path, gates_path, sidecar = _bound_checkpoint(tmp_path, frame) + sidecar["stages"] = ["frs_spine"] + sidecar["sampling"] = {"fraction": 1.0, "seed": 7} + sidecar_path.write_text(json.dumps(sidecar)) + ledger = tmp_path / "ledger" + ledger.mkdir() + (ledger / "facts.csv").write_text("fixture-only; unused during preparation") + monkeypatch.setattr(spine_build, "_rules_engine", lambda: object()) + monkeypatch.setattr( + spine_build, "_rules_engine_provenance", lambda: {"version": "fixture"} + ) + args = cli.parse_args( + [ + "--input-h5", + str(path), + "--input-sidecar", + str(sidecar_path), + "--input-spine-gates", + str(gates_path), + "--ladder", + str(ladder_path), + "--ledger-facts", + str(ledger), + "--out", + str(tmp_path / "out"), + "--n-clones", + "1", + "--epochs", + "8", + ] + ) + return args, sidecar_path, gates_path + + +def test_prepare_real_checkpoint_preserves_source_year_and_wires_preflight( + checkpoint_request, tmp_path +): + args, _, _ = checkpoint_request + prepared = cli.prepare_full_build(args) + assert prepared.full.config.source_year == 2023 + assert prepared.full.config.geography_levels is None + dense = prepared.full.graph.node("uk.full.dense") + assert any( + a.name == "preflight" and a.producer == "uk.full.gates.preflight" + for a in dense.artifact_inputs + ) + endpoint = cli._through(prepared.full.graph, "uk.full.spine_checkpoint") + store = ContentStore(tmp_path / "store") + first = run_graph( + compile_graph(endpoint), + sources=prepared.sources, + store=store, + kernels=prepared.kernels, + ) + provenance_key = first.nodes["uk.full.spine_checkpoint"].opaque_artifacts[ + "spine_provenance" + ] + provenance = json.loads(store.load_bytes(provenance_key)) + assert provenance["stages"] == ["frs_spine"] + assert provenance["fit_weight_records"] == {"model": {"fit_weights_used": True}} + fresh = cli.prepare_full_build(args) + replay = run_graph( + compile_graph(cli._through(fresh.full.graph, "uk.full.spine_checkpoint")), + sources=fresh.sources, + store=store, + kernels=fresh.kernels, + resume="require", + ) + assert replay.nodes["uk.full.spine_checkpoint"].hit + assert not args.out.exists() + + +@pytest.mark.parametrize("damage", ["identity", "gate_bytes"]) +def test_prepare_refuses_checkpoint_drift_before_registering_a_full_build( + checkpoint_request, damage +): + args, sidecar_path, gates_path = checkpoint_request + if damage == "identity": + value = json.loads(sidecar_path.read_text()) + value["uk_frame_content_identity"] = "f" * 64 + sidecar_path.write_text(json.dumps(value)) + else: + gates_path.write_text(gates_path.read_text() + "\n") + with pytest.raises(ValueError, match="identity mismatch|SHA-256 mismatch"): + cli.prepare_full_build(args) + assert not args.out.exists() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_calibration_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_calibration_graph.py new file mode 100644 index 000000000..f97b481e9 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_calibration_graph.py @@ -0,0 +1,434 @@ +"""The full graph preserves dense and exact-count numerical paths and replay.""" + +import hashlib +import json +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime import dataset_size +from microcosm.build.uk_runtime.graph_calibration import ( + UKGraphCalibrationConfig, + register_uk_calibration_kernels, + restore_uk_graph_result, + uk_calibration_nodes, +) +from microcosm.build.uk_runtime.graph_terminal import FULL_GATE_REPORT_TYPE +from microcosm.calibrate import Target, TargetSet, build_constraint_matrix, calibrate +from microcosm.calibrate.artifacts import ( + PROBLEM_TYPE, + decode_calibration_result, + decode_problem, + encode_problem, +) +from microcosm.frame import Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from test_support.microcosm_build.uk_full_calibration_graph import ( + preflight_payload, +) +from test_support.microcosm_build.uk_local_rowwise import _clone_frame + + +def source_frame(): + frame = _clone_frame() + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + tables["household"]["marker"] = [5, 7, 9] + tables["person"]["age"] = [30, 40, 50] + tables["benunit"]["eligible"] = [True, True, False] + return Frame( + tables, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def targets(): + return TargetSet( + [Target("count", "household", lambda f: np.ones(f.n("household")), 3)] + ) + + +def binding(): + return { + "mass_reason": "Fixture selected constraints", + "max_weight_ratio": 10.0, + "target_loss_weights": [1.0], + "target_loss_cap": 10.0, + "selector": "all", + } + + +def problem_payload(): + frame = source_frame() + return encode_problem( + build_constraint_matrix(frame, targets(), weight_entity="household"), + entity_ids=frame.table("household")["household_id"].tolist(), + bindings=binding(), + ) + + +class Source(KernelBase): + ref = "uk.test.solver-source@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def implementation_hash(self): + return hashlib.sha256(self.ref.encode()).hexdigest() + + def run(self, context): + return KernelResult( + frame=source_frame(), + artifacts={ + "problem": problem_payload(), + "preflight": preflight_payload(context.params["preflight_passed"]), + }, + ) + + +def compiled(k, checkpoint_identity=None, preflight_passed=True): + calibration = uk_calibration_nodes( + base="pool", + columns={ + ("household", "marker"): "int64", + ("person", "age"): "int64", + ("benunit", "eligible"): "bool", + }, + problem_producer="pool", + config=UKGraphCalibrationConfig( + epochs=2, learning_rate=0.02, seed=7, dataset_households=k + ), + checkpoint_identity=checkpoint_identity, + ) + graph = Graph( + "uk", + ( + SourceRef("fixture", "raw-bytes-v1"), + *( + () + if checkpoint_identity is None + else ( + SourceRef("uk_size_checkpoint_manifest", "raw-bytes-v1"), + SourceRef("uk_size_checkpoint_arrays", "raw-bytes-v1"), + ) + ), + ), + ( + Node( + "pool", + Source.ref, + sources=("fixture",), + params={"preflight_passed": preflight_passed}, + structural=StructuralDelta.CREATE, + outputs=( + Owned("household", "marker", "int64"), + Owned("person", "age", "int64"), + Owned("benunit", "eligible", "bool"), + ), + artifact_outputs=( + ArtifactOutput("problem", PROBLEM_TYPE), + ArtifactOutput("preflight", FULL_GATE_REPORT_TYPE), + ), + ), + *( + replace( + node, + artifact_inputs=( + *node.artifact_inputs, + ArtifactInput( + "preflight", "pool", "preflight", FULL_GATE_REPORT_TYPE + ), + ), + ) + if node.id == calibration.dense_producer + else node + for node in calibration.nodes + ), + ), + ) + return compile_graph(graph), calibration + + +def registry(): + kernels = KernelRegistry() + kernels.register(Source()) + return register_uk_calibration_kernels(kernels) + + +@pytest.mark.parametrize("k", [None, 2, 3]) +def test_graph_preserves_numerical_path_and_complete_resume(k, tmp_path, monkeypatch): + frame = source_frame() + dense = calibrate( + frame, + targets(), + weight_entity="household", + epochs=2, + learning_rate=0.02, + seed=7, + mass="free", + **binding_solver(), + ) + expected = ( + dense + if k is None + else dataset_size.refit_uk_dataset_size( + frame, dense, households=k, epochs=2, learning_rate=0.02, seed=7 + ).result + ) + graph, endpoints = compiled(k) + fixture = tmp_path / "fixture" + fixture.write_bytes(b"fixture") + store = ContentStore(tmp_path / "store") + first = run_graph( + graph, sources={"fixture": fixture}, store=store, kernels=registry() + ) + actual = first.population(endpoints.population) + for entity in frame.entities: + pd.testing.assert_frame_equal( + actual.table(entity), expected.frame.table(entity) + ) + np.testing.assert_array_equal( + actual.weights_for("household").values, expected.weights + ) + assert actual.mass_log == expected.frame.mass_log + assert actual.weights_for("household").kind.value == "calibrated" + + def payload(producer, name): + return store.load_bytes(first.node(producer).opaque_artifacts[name]) + + restored = restore_uk_graph_result( + frame, + problem_payload=payload(endpoints.problem_producer, "problem"), + result_payload=payload(endpoints.result_producer, "result"), + solution_payload=payload(endpoints.solution_producer, "solution"), + original_problem_payload=problem_payload(), + ) + assert restored.frame.mass_log == expected.frame.mass_log + np.testing.assert_array_equal(restored.initial_weights, expected.initial_weights) + np.testing.assert_array_equal(restored.loss_trajectory, expected.loss_trajectory) + assert restored.options == expected.options + if k == 2: + assert graph.versions["uk.full.size_search"] == "pool" + assert graph.versions["uk.full.size_refit"] == "pool" + assert graph.graph.node("uk.full.selected").structural is StructuralDelta.FILTER + if k == 3: + receipt = json.loads( + store.load_bytes(first.node("uk.full.size_refit").opaque_artifacts["size"]) + ) + assert receipt["method"] == "full_pool" + # Identity is source-authored; prohibit execution while retaining the + # original kernel source identity in this simulated fresh registry. + original_registry = registry() + for kernel in original_registry.as_mapping().values(): + monkeypatch.setattr( + kernel, "run", lambda *a, **kw: pytest.fail("replay executed") + ) + replay = run_graph( + graph, + sources={"fixture": fixture}, + store=store, + kernels=original_registry, + resume="require", + ) + np.testing.assert_array_equal( + replay.population(endpoints.population).weights_for("household").values, + expected.weights, + ) + + +def binding_solver(): + value = binding() + value.pop("selector") + return value + + +def test_reused_draw_skips_rng_and_rejects_changed_binding(monkeypatch): + frame = source_frame() + dense = calibrate(frame, targets(), epochs=2) + options = dict(households=2, epochs=2, learning_rate=0.02, seed=7) + selection = dataset_size.select_uk_dataset_size(frame, dense, **options) + draw = dataset_size.draw_uk_dataset_size( + frame, dense, selection=selection, households=2, seed=7 + ) + expected = dataset_size.refit_uk_dataset_size( + frame, dense, selection=selection, draw=draw, **options + ) + monkeypatch.setattr( + dataset_size, "select_exact_k", lambda *a, **kw: pytest.fail("draw repeated") + ) + again = dataset_size.refit_uk_dataset_size( + frame, dense, selection=selection, draw=draw, **options + ) + np.testing.assert_array_equal(expected.result.weights, again.result.weights) + from dataclasses import replace + + with pytest.raises(ValueError, match="differs from its selection"): + dataset_size.refit_uk_dataset_size( + frame, + dense, + selection=selection, + draw=replace(draw, probabilities_sha256="0" * 64), + **options, + ) + + +def test_completed_result_can_rebuild_without_optimizer(): + # A separately serialized result is also independently inspectable; the + # graph cache is not the only way to obtain diagnostics on resume. + from microcosm.calibrate.artifacts import encode_calibration_result + + frame = source_frame() + problem = decode_problem(problem_payload()) + result = calibrate(frame, problem.to_target_set(), epochs=2) + replay = decode_calibration_result( + encode_calibration_result( + result, entity_ids=problem.entity_ids, problem_sha256=problem.sha256 + ), + frame=frame, + problem=problem, + ) + assert replay.problem.names == result.problem.names + np.testing.assert_array_equal(replay.initial_weights, result.initial_weights) + + +def test_external_search_checkpoint_is_imported_without_repeating_solves( + tmp_path, monkeypatch +): + from microcosm.build.uk_runtime import graph_calibration, size_checkpoint + from microcosm.graph.errors import NodeRejectedError + + frame = source_frame() + dense = calibrate( + frame, + targets(), + weight_entity="household", + epochs=2, + learning_rate=0.02, + seed=7, + mass="free", + **binding_solver(), + ) + selection = dataset_size.select_uk_dataset_size( + frame, dense, households=2, epochs=2, learning_rate=0.02, seed=7 + ) + expected = dataset_size.refit_uk_dataset_size( + frame, + dense, + selection=selection, + households=2, + epochs=2, + learning_rate=0.02, + seed=7, + ) + identity = {"pool": "fixture", "K": 1, "selector": "all", "k": 2} + directory = tmp_path / "legacy" + size_checkpoint.write_uk_size_checkpoint( + directory, frame=frame, dense=dense, selection=selection, identity=identity + ) + fixture = tmp_path / "fixture" + fixture.write_bytes(b"fixture") + sources = { + "fixture": fixture, + "uk_size_checkpoint_manifest": directory + / size_checkpoint.SIZE_CHECKPOINT_MANIFEST_FILENAME, + "uk_size_checkpoint_arrays": directory + / size_checkpoint.SIZE_CHECKPOINT_ARRAYS_FILENAME, + } + monkeypatch.setattr( + graph_calibration, + "calibrate", + lambda *a, **kw: pytest.fail("dense solve repeated"), + ) + monkeypatch.setattr( + dataset_size, + "select_uk_dataset_size", + lambda *a, **kw: pytest.fail("search repeated"), + ) + graph, endpoints = compiled(2, checkpoint_identity=identity) + result = run_graph( + graph, + sources=sources, + store=ContentStore(tmp_path / "store"), + kernels=registry(), + ) + np.testing.assert_array_equal( + result.population(endpoints.population).weights_for("household").values, + expected.result.weights, + ) + assert "uk.full.size_checkpoint_import" in graph.predecessors["uk.full.dense"] + assert "uk.full.size_checkpoint_import" in graph.predecessors["uk.full.size_search"] + for changed in ({**identity, "K": 2}, {"K": 1}): + drift, _ = compiled(2, checkpoint_identity=changed) + with pytest.raises(NodeRejectedError, match="identity"): + run_graph( + drift, + sources=sources, + store=ContentStore(tmp_path / "drift"), + kernels=registry(), + ) + + +def test_changing_k_reuses_dense_but_source_bytes_invalidate_it(tmp_path, monkeypatch): + fixture = tmp_path / "fixture" + fixture.write_bytes(b"first source") + store = ContentStore(tmp_path / "store") + first_graph, _ = compiled(2) + run_graph( + first_graph, sources={"fixture": fixture}, store=store, kernels=registry() + ) + second_graph, _ = compiled(3) + kernels = registry() + for ref in (Source.ref, "uk.full.dense@1"): + monkeypatch.setattr( + kernels.get(ref), + "run", + lambda *a, **kw: pytest.fail("k reran upstream work"), + ) + resized = run_graph( + second_graph, sources={"fixture": fixture}, store=store, kernels=kernels + ) + assert resized.node("pool").hit + assert resized.node("uk.full.dense").hit + assert not resized.node("uk.full.size_search").hit + fixture.write_bytes(b"changed source") + rebuilt = run_graph( + second_graph, sources={"fixture": fixture}, store=store, kernels=registry() + ) + assert not rebuilt.node("pool").hit + assert not rebuilt.node("uk.full.dense").hit + + +@pytest.mark.parametrize("imported", [False, True]) +def test_blocking_preflight_refuses_before_dense_or_checkpoint_work(imported): + from types import SimpleNamespace + + from microcosm.build.uk_runtime.graph_calibration import UKDenseSolveKernel + + artifacts = {"preflight": SimpleNamespace(payload=preflight_payload(False))} + if imported: + # Deliberately unreadable result: the preflight must refuse before + # decoding an imported solution, just as it must before optimization. + artifacts["imported_dense"] = SimpleNamespace(payload=b"must not read") + with pytest.raises(ValueError, match="refused by the source preflight"): + UKDenseSolveKernel().run(SimpleNamespace(artifacts=artifacts)) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_certification.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_certification.py new file mode 100644 index 000000000..aef69f2ae --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_certification.py @@ -0,0 +1,288 @@ +"""One full graph certifies its own scope and bytes without signed lane joins.""" + +import hashlib +import json +import sys + +import pytest + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.gate_battery import ( + GateOutcome, + GatePhaseReport, + GateStatus, + gate_phase_report_payload, +) +from microcosm.build.gates import GateResult +from microcosm.build.uk_runtime import full_certification as runtime +from microcosm.build.uk_runtime.full_gates import ( + classify_full_gate_outcomes, + uk_full_gate_manifest, + uk_full_gate_scope_receipt, +) +from microcosm.build.uk_runtime.graph_evidence import uk_spine_gate_manifest +from microcosm.graph.canonical import canonical_json + + +@pytest.fixture(scope="module", autouse=True) +def _one_validated_country_spec(): + """These artifact tests share immutable declarations; source loading is separate.""" + from microcosm.build.uk_runtime import full_gates + + spec = load_country_spec("uk") + with pytest.MonkeyPatch.context() as patch: + for module in (runtime, full_gates, sys.modules[__name__]): + patch.setattr(module, "load_country_spec", lambda country: spec) + yield + + +def _passed(gates, phase): + return GatePhaseReport( + phase, + tuple( + GateOutcome(entry, GateStatus.PASSED, GateResult(entry.gate, True, (), {})) + for entry in gates.gates + if entry.phase == phase + ), + ) + + +def _fixture(*, country=False, k=None): + selection = { + "schema": "microcosm.calibrate.target-selection.v1", + "selector": {"geography_levels": ["country"] if country else None}, + "included": [ + {"name": "national", "period": 2025, "geography_level": "country"} + ], + "excluded": [], + } + if not country: + selection["included"].append( + {"name": "local", "period": 2025, "geography_level": "constituency"} + ) + names = set(runtime._REQUIRED) | {"spine_assembled", "spine_transferred"} + keys = {name: hashlib.sha256(name.encode()).hexdigest() for name in names} + gate_manifest = uk_full_gate_manifest(selection) + documents = { + "selection": {"receipt": selection}, + "diagnostics": {"targets": []}, + "holdout": { + "skipped": False, + "method": "rotated_folds", + "n_folds": 5, + "folds": [{"holdout_loss": 0.01}] * 5, + "mean_holdout_loss": 0.01, + "worst_holdout_loss": 0.01, + }, + } + for role, phase in (("preflight", "preflight"), ("full_gates", "terminal")): + report = _passed(gate_manifest, phase) + documents[role] = { + "schema_version": 1, + "kind": "uk_full_gate_report", + "selection_receipt": selection, + "sample_fraction": 1.0, + "release_candidate": False, + "scope": uk_full_gate_scope_receipt(selection), + "report": gate_phase_report_payload(report, gates=gate_manifest), + "enforcement": classify_full_gate_outcomes( + report, sample_fraction=1.0, release_candidate=False + ), + "artifacts": { + name: keys[name] + for name in ("surface", "selection", "preflight", "holdout") + }, + } + spine = uk_spine_gate_manifest(load_country_spec("uk")) + for phase in ("assembled", "transferred"): + documents[f"spine_{phase}"] = gate_phase_report_payload( + _passed(spine, phase), gates=spine + ) + bindings = {"configuration": {"calibration": {"dataset_households": k}}} + dataset = {"filename": "candidate.h5", "sha256": "d" * 64, "size_bytes": 123} + documents["export_descriptor"] = { + "kind": "uk_full_build_export", + "schema_version": 1, + "content_sha256": "a" * 64, + "bindings": bindings, + "tables": {"household": {"rows": k or 12}}, + } + documents["export_readback"] = { + "kind": "uk_full_build_export_readback", + "passed": True, + "dataset": dataset, + "content_sha256": "a" * 64, + "bindings": bindings, + } + documents["package"] = { + "kind": "uk_full_build_package", + "readback_passed": True, + "release_authorized": False, + "schema_version": 1, + "dataset": dataset, + "content_sha256": "a" * 64, + "build_bindings": bindings, + "artifacts": { + name: keys[name] + for name in ("full_gates", "diagnostics", "holdout", "export_readback") + }, + } + documents["surface"] = { + "source_validation": { + "ledger_provenance": {"facts_sha256": "b" * 64, "manifest_sha256": "c" * 64} + } + } + return { + name: (keys[name], canonical_json(document)) + for name, document in documents.items() + } + + +def _rewrite(artifacts, name, mutate): + key, payload = artifacts[name] + document = json.loads(payload) + mutate(document) + artifacts[name] = (key, canonical_json(document)) + + +def _scores(artifacts, monkeypatch, *, k=None): + def validate(payload, failures, *, expected_identity): + if payload.get("identity") != expected_identity: + failures.append("scorecard candidate identity mismatch") + + monkeypatch.setattr( + "microcosm.data.contract._check_uk_incumbent_surface_evaluation", validate + ) + identity = { + "candidate_dataset_sha256": "d" * 64, + "candidate_manifest_sha256": hashlib.sha256( + artifacts["package"][1] + ).hexdigest(), + "candidate_diagnostics_sha256": hashlib.sha256( + artifacts["diagnostics"][1] + ).hexdigest(), + "ledger_facts_sha256": "b" * 64, + "ledger_manifest_sha256": "c" * 64, + } + sources = {} + for role in ( + ("native_scorecard", "matched_size_scorecard") if k else ("native_scorecard",) + ): + document = {"identity": identity} + if role == "matched_size_scorecard": + document["comparison"] = { + "kind": "matched_size", + "candidate_households": k, + "incumbent_households": k, + } + payload = canonical_json(document) + sources[role] = ( + { + "filename": role + ".json", + "sha256": hashlib.sha256(payload).hexdigest(), + "size_bytes": len(payload), + }, + payload, + ) + return sources + + +def test_complete_graph_retains_explicit_missing_native_evidence(): + report = runtime.compose_uk_full_certification_readiness(_fixture()) + assert report["comparisons"]["native_scorecard"]["status"] == "evidence_absent" + assert report["comparisons"]["matched_size_scorecard"]["status"] == "not_required" + assert not report["ready_for_external_review"] + assert not report["release_authorized"] + assert set(report["gate_coverage"]["declared"]) == { + entry.id for entry in load_country_spec("uk").gates.gates + } + + +@pytest.mark.parametrize( + "country,k", [(False, None), (True, None), (False, 5), (True, 5)] +) +def test_bound_comparisons_can_complete_readiness_without_publication( + monkeypatch, country, k +): + artifacts = _fixture(country=country, k=k) + report = runtime.compose_uk_full_certification_readiness( + artifacts, comparison_sources=_scores(artifacts, monkeypatch, k=k) + ) + assert report["ready_for_external_review"] + assert not report["release_authorized"] + assert not report["subnational_fit_certified"] + assert report["target_scope"]["local_fit_claim"] is (not country) + + +@pytest.mark.parametrize( + "name,field", + [ + ("package", "content_sha256"), + ("export_readback", "content_sha256"), + ("full_gates", "selection_receipt"), + ], +) +def test_certification_rejects_foreign_graph_identity(name, field): + artifacts = _fixture() + _rewrite(artifacts, name, lambda document: document.__setitem__(field, "foreign")) + with pytest.raises((ValueError, AttributeError)): + runtime.compose_uk_full_certification_readiness(artifacts) + + +def test_actual_surface_validator_refuses_old_score_summary(): + artifacts = _fixture() + payload = canonical_json({"candidate_train_loss": 0.01}) + sources = { + "native_scorecard": ( + {"sha256": hashlib.sha256(payload).hexdigest(), "size_bytes": len(payload)}, + payload, + ) + } + report = runtime.compose_uk_full_certification_readiness( + artifacts, comparison_sources=sources + ) + assert report["comparisons"]["native_scorecard"]["status"] == "failed" + assert any( + "schema 2" in failure + for failure in report["comparisons"]["native_scorecard"]["failures"] + ) + + +def test_exact_size_requires_its_own_bound_comparison(monkeypatch): + artifacts = _fixture(k=5) + sources = _scores(artifacts, monkeypatch, k=5) + sources.pop("matched_size_scorecard") + report = runtime.compose_uk_full_certification_readiness( + artifacts, comparison_sources=sources + ) + assert ( + report["comparisons"]["matched_size_scorecard"]["status"] == "evidence_absent" + ) + assert not report["ready_for_external_review"] + + +def test_bound_spine_provenance_replaces_raw_spine_reports(): + artifacts = _fixture() + gates = uk_spine_gate_manifest(load_country_spec("uk")) + report = { + "blocked_at_phase": None, + "gates": { + outcome.entry.id: outcome.to_payload() + for phase in gates.phases + for outcome in _passed(gates, phase).outcomes + }, + } + provenance = { + "uk_frame_content_identity": "e" * 64, + "spine_gate_report": {"sha256": "f" * 64, "payload": report}, + } + artifacts["spine_provenance"] = ("1" * 64, canonical_json(provenance)) + for phase in ("assembled", "transferred"): + artifacts.pop(f"spine_{phase}") + _rewrite( + artifacts, + "full_gates", + lambda doc: doc["artifacts"].__setitem__("spine_provenance", "1" * 64), + ) + result = runtime.compose_uk_full_certification_readiness(artifacts) + assert result["artifacts"]["spine_provenance"]["graph_artifact_key"] == "1" * 64 diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_gates.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_gates.py new file mode 100644 index 000000000..c11c462f0 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_gates.py @@ -0,0 +1,317 @@ +"""Scope and population identity checks for the complete UK gate context.""" + +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from scipy import sparse + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.uk_runtime import full_gates as runtime +from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.calibrate.artifacts import OrderedProblem, OrderedSolution +from microcosm.calibrate.matrix import CalibrationProblem +from microcosm.calibrate.solve import CalibrationResult +from microcosm.frame import WeightKind, Weights + + +def _selection(levels=None): + rows = [{"name": "national", "period": 2024, "geography_level": "country"}] + if levels is None: + rows.append( + {"name": "local", "period": 2024, "geography_level": "constituency"} + ) + return { + "schema": "microcosm.calibrate.target-selection.v1", + "selector": {"geography_levels": levels, "explicit": levels is not None}, + "included": rows, + "excluded": [], + } + + +def test_default_full_gate_scope_owns_every_country_gate(): + assert {gate.id for gate in runtime.uk_full_gate_manifest().gates} == { + gate.id for gate in load_country_spec("uk").gates.gates + } + assert runtime.uk_full_gate_scope_receipt()["scope_exclusions"] == {} + + +def test_country_filter_retains_integrity_and_registry_checks_without_local_fit_claims(): + selected = _selection(["country"]) + receipt = runtime.uk_full_gate_scope_receipt(selected) + gates = {gate.id for gate in runtime.uk_full_gate_manifest(selected).gates} + assert receipt["local_fit_claim"] is False + assert set(receipt["scope_exclusions"]) == { + "uk_local_target_fit", + "uk_local_per_family_fit", + "uk_local_area_support", + } + assert { + "uk_local_geography_ladder_post_calibration", + "uk_weight_ess", + "uk_release_family_build_stages", + "uk_ledger_compile_parity_local_incumbent_2025", + "uk_target_surface_local_default_2025", + } <= gates + + +def test_unfiltered_build_cannot_silently_become_national_only(): + selection = _selection(["country"]) + selection["selector"]["geography_levels"] = None + with pytest.raises(ValueError, match="unfiltered UK full build"): + runtime.uk_full_gate_manifest(selection) + + +@pytest.fixture +def evidence(monkeypatch): + specs = [ + TargetSpec( + name="national", + entity="household", + value=0.5, + measure="income", + period=2024, + source="test", + family="income", + ), + TargetSpec( + name="local", + entity="household", + value=2.0, + measure="income", + period=2024, + source="test", + family="population", + ), + ] + targets = tuple(spec.to_target() for spec in specs) + weights = Weights(np.ones(2), WeightKind.CALIBRATED) + problem = CalibrationProblem( + sparse.csr_array([[0.1, 0.2], [1.0, 1.0]]), + np.array([0.5, 2.0]), + tuple(target.row_name for target in targets), + Weights(np.ones(2), WeightKind.IMPORTANCE), + "household", + targets, + ) + metadata = ( + { + "geography_level": "country", + "geography_id": "UK", + "family": "income", + "materialization": "uk_national_measure", + }, + { + "geography_level": "constituency", + "geography_id": "E14000001", + "family": "population", + "materialization": "uk_local_surface", + }, + ) + ordered = OrderedProblem( + problem, (1, 2), metadata, {"target_selection": _selection()}, "a" * 64 + ) + solution = OrderedSolution(np.ones(2), (1, 2), "a" * 64, {}, "b" * 64) + table = pd.DataFrame({"household_id": [1, 2], "income": [1.0, 1.0]}) + frame = SimpleNamespace( + entities=("household",), + table=lambda entity: table, + schema=SimpleNamespace(entity_id_column=lambda entity: "household_id"), + weights_for=lambda entity: weights, + ) + monkeypatch.setattr( + runtime, + "load_efrs_parity_reference", + lambda: SimpleNamespace(input_entities={"income": "household"}), + ) + monkeypatch.setattr( + runtime, + "uk_aggregate_admin_totals", + lambda frame, manifest: ({"admin": 2.0}, [{"measured": 2.0}]), + ) + return ( + frame, + ordered, + solution, + { + "reference_registry": TargetRegistry([specs[0]], country="uk"), + "coverage_engine": object(), + }, + ) + + +def _context(evidence, **overrides): + frame, ordered, solution, supporting = evidence + return runtime.build_full_gate_context( + frame, + ordered_problem=ordered, + solution=overrides.get("solution", solution), + selection_receipt=_selection(), + stage_evidence={"frs_spine": {}}, + supporting_evidence=supporting, + ) + + +def test_gate_diagnostics_use_bound_matrix_and_row_identity(evidence): + context = _context(evidence) + assert context.artifacts["parity_evidence"].target_relative_errors == pytest.approx( + {"national@2024": -0.4} + ) + + assert context.artifacts["parity_evidence"].reference_targets == {"national@2024"} + assert context.artifacts["local_target_diagnostics"][0]["area_code"] == "E14000001" + assert context.artifacts["local_target_diagnostics"][0]["relative_error"] == 0.0 + assert context.artifacts["national_calibration"]["matrix_target_count"] == 2 + assert context.artifacts["rules_engine"] is context.artifacts["coverage_engine"] + + +def _completed_result(evidence): + frame, ordered, solution, _ = evidence + problem = ordered.problem + return CalibrationResult( + frame=frame, + weight_entity="household", + weights=solution.weights, + initial_weights=problem.initial_weights.values, + diagnostics=runtime._build_diagnostics( + problem, frame, problem.initial_weights.values, solution.weights + ), + loss_trajectory=np.array([0.2]), + skipped=(), + problem=problem, + l0_lambda=0.0, + n_nonzero=2, + closing_loss=0.2, + target_loss_weights=np.ones(2), + target_loss_scales=np.ones(2), + target_loss_cap=10.0, + ) + + +def test_gate_context_reuses_authenticated_final_diagnostics(evidence, monkeypatch): + evidence[3]["calibration_result"] = _completed_result(evidence) + monkeypatch.setattr( + runtime, + "_build_diagnostics", + lambda *args: pytest.fail("diagnostics computed twice"), + ) + assert _context(evidence).artifacts["parity_evidence"].target_relative_errors[ + "national@2024" + ] == pytest.approx(-0.4) + + +@pytest.mark.parametrize("mutation", ["weights", "problem", "diagnostic_axis"]) +def test_gate_context_rejects_foreign_completed_result(evidence, mutation): + result = _completed_result(evidence) + if mutation == "weights": + result = replace(result, weights=np.array([1.5, 0.5])) + elif mutation == "problem": + result = replace( + result, problem=replace(result.problem, matrix=2 * result.problem.matrix) + ) + else: + result = replace(result, diagnostics=result.diagnostics[::-1]) + evidence[3]["calibration_result"] = result + with pytest.raises(ValueError, match="calibration_result"): + _context(evidence) + + +@pytest.mark.parametrize( + "change,match", + [ + ({"problem_sha256": "c" * 64}, "different ordered problem"), + ({"entity_ids": (2, 1)}, "different household axes"), + ({"weights": np.array([2.0, 1.0])}, "bound solution weights"), + ], +) +def test_gate_context_rejects_wrong_solution_identity(evidence, change, match): + with pytest.raises(ValueError, match=match): + _context(evidence, solution=replace(evidence[2], **change)) + + +def test_gate_context_rejects_selection_binding_drift(evidence): + frame, ordered, solution, supporting = evidence + ordered = replace(ordered, bindings={"target_selection_sha256": "c" * 64}) + with pytest.raises(ValueError, match="target-selection digest"): + _context((frame, ordered, solution, supporting)) + + +def _phase_failure(gate_id, *, absent=False): + from microcosm.build.gate_battery import GateOutcome, GatePhaseReport, GateStatus + from microcosm.build.gates import GateResult + + entry = next(g for g in load_country_spec("uk").gates.gates if g.id == gate_id) + return GatePhaseReport( + entry.phase, + ( + GateOutcome( + entry=entry, + status=GateStatus.EVIDENCE_ABSENT if absent else GateStatus.FAILED, + result=None + if absent + else GateResult( + name=entry.gate, passed=False, failures=("test failure",) + ), + reason="missing evidence" if absent else None, + ), + ), + ) + + +def test_existing_local_statistical_failure_can_export_but_blocks_full_release(): + report = _phase_failure("uk_local_target_fit") + full = runtime.classify_full_gate_outcomes( + report, sample_fraction=1.0, release_candidate=True + ) + assert full["artifact_permitted"] is True + assert full["exportable_blocking"] == ["uk_local_target_fit"] + assert full["release_blocking_gates_passed"] is False + rung = runtime.classify_full_gate_outcomes( + report, sample_fraction=0.1, release_candidate=False + ) + assert rung["enforced_blocking"] == [] + assert rung["unenforced_release_failures"] == ["uk_local_target_fit"] + + +@pytest.mark.parametrize( + "gate_id", + [ + "uk_local_geography_ladder_post_calibration", + "uk_nonnegative_columns", + "uk_input_mass_parity", + "uk_target_fit", + "uk_calibration_reference_coverage", + ], +) +def test_geography_and_migrated_national_gates_keep_artifact_enforcement(gate_id): + for fraction in (0.1, 1.0): + result = runtime.classify_full_gate_outcomes( + _phase_failure(gate_id), sample_fraction=fraction, release_candidate=False + ) + assert result["artifact_permitted"] is False + assert result["structural_failures"] == [gate_id] + + +def test_missing_evidence_uses_declared_development_policy(): + ordinary = _phase_failure("uk_input_mass_parity", absent=True) + assert ( + runtime.classify_full_gate_outcomes( + ordinary, sample_fraction=1.0, release_candidate=False + )["artifact_permitted"] + is True + ) + assert ( + runtime.classify_full_gate_outcomes( + ordinary, sample_fraction=1.0, release_candidate=True + )["artifact_permitted"] + is False + ) + strict = _phase_failure("uk_weights_audit", absent=True) + assert ( + runtime.classify_full_gate_outcomes( + strict, sample_fraction=1.0, release_candidate=False + )["artifact_permitted"] + is False + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_measure.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_measure.py new file mode 100644 index 000000000..55ee9587d --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_measure.py @@ -0,0 +1,274 @@ +"""Shared full-build measure evaluation retains legacy engine/RNG boundaries.""" + +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime import full_measure +from microcosm.build.uk_runtime.rowwise_dataset import ( + clone_uk_dataset_with_ladder_geography, +) +from microcosm.calibrate import TargetRegistry, TargetSpec +from test_support.microcosm_build.uk_full_population_graph import source_frame +from test_support.microcosm_build.uk_ladder_rowwise_clone import ( + toy_ladder as toy_ladder, +) + + +def test_full_measure_reuses_one_resolver( + monkeypatch, + tmp_path, +) -> None: + frame = source_frame() + constructions = [] + cgt_period_contract = { + "version": "uk-cgt-measurement-v2", + "input_period": "2024", + "calibration_period": 2025, + "bound_measurements": { + "cgt_2024_gains": { + "model_variable": "capital_gains", + "measurement_period": 2024, + } + }, + } + + class StubResolver: + def __init__(self, **kwargs): + constructions.append(kwargs) + self.simulation = object() + self.contract_targets = {} + + def receipt(self): + return { + "mode": "stub", + "policyengine_uk_version": "test", + "cgt_period_contract": cgt_period_contract, + } + + monkeypatch.setattr( + full_measure, + "compute_household_metrics", + lambda _simulation, area_type, *, household_ids, **_kwargs: pd.DataFrame( + {f"{area_type}_metric": np.ones(len(household_ids))}, + index=household_ids, + ), + ) + registry = TargetRegistry([], country="uk") + prepared, restore, national, local_metrics, receipt = ( + full_measure.resolve_uk_full_measures( + frame, + registry, + period=2025, + scratch_dir=tmp_path / "scratch", + resolver_factory=StubResolver, + ) + ) + + assert len(constructions) == 1 + assert receipt == { + "mode": "stub", + "engine_version": "test", + "households": frame.n("household"), + "persons": frame.n("person"), + "benunits": frame.n("benunit"), + "national_inputs": 0, + "local_metrics": {"constituency": 1, "la": 1}, + "blocks": 1, + "cgt_period_contract": cgt_period_contract, + } + assert set(local_metrics) == {"constituency", "la"} + assert len(national.targets) == 0 + assert restore(prepared).table("household").equals(frame.table("household")) + + +@pytest.mark.parametrize("second_cgt_period", [2024, 2025, None]) +def test_full_measure_resolves_real_per_clone_blocks( + monkeypatch, + tmp_path, + second_cgt_period, + toy_ladder, +) -> None: + frame = source_frame() + ladder, _ = toy_ladder + clone = clone_uk_dataset_with_ladder_geography( + frame, + ladder, + n_clones=2, + seed=7, + source_year=2023, + expected_constituency_vintage="2024_pcon", + source_lineage_modulus=None, + ) + constructions = [] + + class StubResolver: + def __init__(self, **kwargs): + constructions.append(kwargs) + self.simulation = object() + self.contract_targets = {} + self.cgt_period = 2024 if len(constructions) == 1 else second_cgt_period + + def receipt(self): + receipt = {"mode": "stub", "policyengine_uk_version": "test"} + if self.cgt_period is not None: + receipt["cgt_period_contract"] = { + "version": "uk-cgt-measurement-v2", + "input_period": "2024", + "calibration_period": 2025, + "bound_measurements": { + "cgt_2024_gains": { + "model_variable": "capital_gains", + "measurement_period": self.cgt_period, + } + }, + } + return receipt + + monkeypatch.setattr( + full_measure, + "compute_household_metrics", + lambda _simulation, area_type, *, household_ids, **_kwargs: pd.DataFrame( + {f"{area_type}_metric": np.arange(len(household_ids), dtype=float)}, + index=household_ids, + ), + ) + + def resolve(): + return full_measure.resolve_uk_full_measures( + clone.frame, + TargetRegistry([], country="uk"), + period=2025, + scratch_dir=tmp_path / "block-scratch", + resolver_factory=StubResolver, + blocks=2, + ) + + if second_cgt_period != 2024: + with pytest.raises(RuntimeError, match="CGT period contract is inconsistent"): + resolve() + return + + prepared, restore, _, metrics, receipt = resolve() + + assert len(constructions) == 2 + assert [len(call["frame"].table("household")) for call in constructions] == [ + frame.n("household"), + frame.n("household"), + ] + assert receipt["blocks"] == 2 + assert receipt["cgt_period_contract"]["bound_measurements"] == { + "cgt_2024_gains": { + "model_variable": "capital_gains", + "measurement_period": 2024, + } + } + assert receipt["deviation"] == "per_clone_block_engine_resolution" + sensitivity = receipt["block_sensitivity"] + assert ( + "ons/corporate_land_value" + in sensitivity["known_population_normalised_measures"] + ) + assert set(sensitivity["present_in_this_run"]) <= set( + sensitivity["known_population_normalised_measures"] + ) + assert "not evidence for adjudication" in sensitivity["caveat"] + assert ( + metrics["constituency"].index.tolist() + == clone.frame.table("household")["household_id"].tolist() + ) + assert restore(prepared).table("household").equals(clone.frame.table("household")) + + +@pytest.mark.parametrize("blocks", [1, 2]) +def test_prepared_measures_preserve_ids_and_remove_duplicate_scratch_inputs( + monkeypatch, tmp_path, toy_ladder, blocks +): + frame = source_frame() + if blocks == 2: + frame = clone_uk_dataset_with_ladder_geography( + frame, + toy_ladder[0], + n_clones=2, + seed=7, + source_year=2023, + expected_constituency_vintage="2024_pcon", + ).frame + registry = TargetRegistry( + [ + TargetSpec( + name="national", + entity="household", + measure="prepared_count", + value=33.0, + period=2025, + family="fixture", + source="fixture", + ) + ], + country="uk", + ) + + class Resolver: + def __init__(self, *, frame, **kwargs): + self.frame = frame + self.simulation = object() + + def receipt(self): + return {"mode": "stub", "policyengine_uk_version": "test"} + + monkeypatch.setattr( + full_measure, + "resolve_target_measures", + lambda _factory, _registry, provider, **kwargs: SimpleNamespace( + measure_inputs={ + ("person", "region"): np.zeros(provider.frame.n("person")), + ("household", "raw_engine_input"): provider.frame.table("household")[ + "household_id" + ].to_numpy(), + ("household", "ons/corporate_land_value"): np.ones( + provider.frame.n("household") + ), + } + ), + ) + + def materialize(adapter, _registry, **kwargs): + # The duplicate scratch region must be usable while materializing, + # then removed before assembling the flattened prepared Frame. + assert "region" in adapter.tables["person"] + table = adapter.tables["household"] + table["prepared_count"] = table["raw_engine_input"].to_numpy() + return SimpleNamespace(skipped=()) + + monkeypatch.setattr(full_measure, "materialize_uk_ledger_targets", materialize) + prepared, restore, national, metrics, receipt = ( + full_measure.resolve_uk_full_measures( + frame, + registry, + period=2025, + scratch_dir=tmp_path / "scratch", + resolver_factory=Resolver, + blocks=blocks, + local_grains=(), + ) + ) + assert "region" not in prepared.table("person") + assert "raw_engine_input" not in prepared.table("household") + np.testing.assert_array_equal( + prepared.table("household")["prepared_count"], + frame.table("household")["household_id"], + ) + for entity in frame.entities: + pd.testing.assert_frame_equal( + restore(prepared).table(entity), frame.table(entity) + ) + assert len(national.targets) == 1 + assert metrics == {} + assert receipt["national_inputs"] == 3 + if blocks == 2: + assert receipt["block_sensitivity"]["present_in_this_run"] == [ + "ons/corporate_land_value" + ] diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_population_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_population_graph.py new file mode 100644 index 000000000..5f71b612b --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_population_graph.py @@ -0,0 +1,72 @@ +"""Full-build population graph preserves the maintained geography operation.""" + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime.rowwise_dataset import ( + clone_uk_dataset_with_ladder_geography, +) +from microcosm.graph import ContentStore, compile_graph, run_graph +from test_support.microcosm_build.uk_full_population_graph import ( + graph_and_registry, + source_frame, +) +from test_support.microcosm_build.uk_ladder_rowwise_clone import ( + toy_ladder as toy_ladder, +) + + +@pytest.mark.parametrize("k", [1, 2, 5]) +def test_population_graph_preserves_rows_weights_geography_and_replays( + k, toy_ladder, tmp_path +): + ladder, path = toy_ladder + expected = clone_uk_dataset_with_ladder_geography( + source_frame(), + ladder, + n_clones=k, + seed=7, + source_year=2023, + expected_constituency_vintage="2024_pcon", + ).frame + graph, registry = graph_and_registry(k) + store = ContentStore(tmp_path / "store") + compiled = compile_graph(graph) + assert "uk.full.locations" in compiled.predecessors["uk.full.geography_mapping"] + first = run_graph( + compiled, + sources={"uk_ladder": path, "fixture": path}, + store=store, + kernels=registry, + ) + actual = first.population("uk.full.expand") + for entity in expected.entities: + pd.testing.assert_frame_equal( + actual.table(entity)[expected.table(entity).columns], + expected.table(entity), + check_dtype=False, + ) + np.testing.assert_array_equal( + actual.weights_for("household").values, expected.weights_for("household").values + ) + assert actual.mass_log == expected.mass_log + # A new registry cannot obtain results from mutable objects of the cold run. + _, fresh_registry = graph_and_registry(k) + replay = run_graph( + compiled, + sources={"uk_ladder": path, "fixture": path}, + store=store, + kernels=fresh_registry, + resume="require", + ) + assert replay.population("uk.full.expand").mass_log == actual.mass_log + + +def test_k_changes_expansion_but_never_sampling(): + first, _ = graph_and_registry(1) + second, _ = graph_and_registry(3) + assert first.node("uk.full.sample") == second.node("uk.full.sample") + assert first.node("uk.full.normalize") == second.node("uk.full.normalize") + assert first.node("uk.full.expand").params["n_clones"] == 1 + assert second.node("uk.full.expand").params["n_clones"] == 3 diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_solve_scope.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_solve_scope.py new file mode 100644 index 000000000..98c454d8a --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_solve_scope.py @@ -0,0 +1,82 @@ +"""A filtered full problem has zero local rows and uses the same solver.""" + +import numpy as np +import pytest + +from microcosm.build.uk_runtime.local_rowwise import ( + UKRowwiseNationalRows, + empty_uk_local_problem, + finish_uk_full_solve, + prepare_uk_full_solve, + rotated_uk_local_holdout, + solve_uk_dense_reference, + solve_uk_rowwise_weights_under_doctrine, +) +from microcosm.calibrate import Target, TargetRegistry, TargetSet, TargetSpec +from test_support.microcosm_build.uk_local_rowwise import _clone_frame + + +def national_rows(): + registry = TargetRegistry( + [ + TargetSpec( + name="households", + entity="household", + value=6.0, + measure="household_id", + period=2026, + family="fixture", + source="fixture", + ) + ], + country="uk", + ) + return UKRowwiseNationalRows( + TargetSet( + [ + Target( + name="households", + entity="household", + value=6.0, + measure=lambda frame: np.ones(frame.n("household")), + period=2026, + ) + ] + ), + registry, + ("fixture",), + ) + + +def test_zero_local_scope_uses_same_solver_and_has_no_fake_holdout(): + frame = _clone_frame() + local = empty_uk_local_problem(frame.table("household")["household_id"]) + rows = national_rows() + prepared = prepare_uk_full_solve( + frame, local, bound_families=("national/fixture",), national_rows=rows + ) + dense = solve_uk_dense_reference(prepared, epochs=8, seed=17) + finished = finish_uk_full_solve(prepared, dense) + existing = solve_uk_rowwise_weights_under_doctrine( + frame, + local, + bound_families=("national/fixture",), + national_rows=rows, + epochs=8, + seed=17, + ) + np.testing.assert_array_equal(finished.weights, existing.weights) + assert finished.diagnostics.empty + assert len(finished.national_diagnostics) == 1 + assert dense.problem.n_targets == 1 + holdout = rotated_uk_local_holdout(frame, local, national_rows=rows) + assert holdout["outcome"] == "not_applicable" + assert holdout["n_folds"] == 0 + assert holdout["folds"] == [] + + +def test_empty_total_surface_refuses_instead_of_manufacturing_constraints(): + frame = _clone_frame() + local = empty_uk_local_problem(frame.table("household")["household_id"]) + with pytest.raises(ValueError, match="at least one selected target"): + prepare_uk_full_solve(frame, local, bound_families=()) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_target_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_target_graph.py new file mode 100644 index 000000000..5c6d58cd1 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_target_graph.py @@ -0,0 +1,98 @@ +"""Actual full graph on synthetic target and engine-source adapters.""" + +# ruff: noqa: F403, F405 +from test_support.microcosm_build.uk_full_target_graph import * + + +def test_default_scope_contains_all_levels_and_never_depends_on_k_or_k_small(): + default = UKFullBuildConfig(calibration_year=2026) + for config in ( + default, + replace(default, n_clones=1), + replace( + default, calibration=replace(default.calibration, dataset_households=20) + ), + ): + graph = uk_full_graph(config).graph + assert graph.node("uk.full.target_selection").params["geography_levels"] is None + + +def test_local_surface_selection_keeps_exact_target_periods(): + rows = pd.DataFrame( + { + "target_name": ["same", "same", "different"], + "period": [2025, 2026, 2026], + "value": [2.0, 3.0, 4.0], + } + ) + selected = [ + TargetSpec( + name="same", + entity="household", + measure="count", + value=3.0, + period=2026, + source="fixture", + family="fixture", + ) + ] + actual = graph_targets._selected_local_surface(rows, selected) + assert actual.to_dict(orient="records") == [ + {"target_name": "same", "period": 2026, "value": 3.0} + ] + + +def test_local_problem_targets_come_from_chronicle_with_assignment_rosters( + target_inputs, toy_ladder +): + from microcosm.build.uk_runtime.full_problem import build_uk_full_local_problem + from microcosm.build.uk_runtime.rowwise_dataset import ( + clone_uk_dataset_with_ladder_geography, + ) + from test_support.microcosm_build.uk_full_population_graph import source_frame + + ladder, _ = toy_ladder + clone = clone_uk_dataset_with_ladder_geography( + source_frame(), + ladder, + n_clones=10, + seed=7, + source_year=2023, + expected_constituency_vintage="2024_pcon", + ) + ids = clone.frame.table("household")["household_id"] + metrics = { + grain: pd.DataFrame({"households": np.ones(len(ids))}, index=ids) + for grain in ("constituency", "la") + } + local = target_inputs["local"] + uprating = ledger_targets.uk_census_household_uprating( + local, {"value": 33.0, "period": 2026}, period=2026 + ) + _, problem, receipt, _, _ = build_uk_full_local_problem( + SimpleNamespace(result=clone, ladder=ladder), + local_registry=local, + national_registry=TargetRegistry([], country="uk"), + local_metrics=metrics, + period=2026, + sample_fraction=1.0, + reviewed_unbound_higher_targets={}, + census_household_uprating=uprating, + ) + expected = target_inputs["surface"]()[0].set_index("target_name")["value"] + actual = problem.target_frame.set_index("target_name")["value"] + pd.testing.assert_series_equal(actual.sort_index(), expected.sort_index()) + assert set(actual.index) == {spec.name for spec in local} + assert receipt["census_household_uprating"]["household_cells"]["cells"] == 10 + + +def test_target_kernel_identity_includes_reconciliation_and_ladder_diagnostics( + monkeypatch, +): + hashed = [] + monkeypatch.setattr( + graph_targets, "source_hash", lambda *objects: hashed.extend(objects) or "test" + ) + graph_targets.UKFullTargetCompilationKernel().implementation_hash() + assert graph_targets.cross_grain in hashed + assert graph_targets.ladder_targets in hashed diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py new file mode 100644 index 000000000..fe8e979d2 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py @@ -0,0 +1,246 @@ +"""Source pins and unreduced-register ownership in the full UK target path.""" + +import hashlib +import json +from datetime import date +from types import SimpleNamespace + +import pytest + +from microcosm.build.ledger_artifact import CONSUMER_ARTIFACT_SCHEMA_VERSION +from microcosm.build.uk_runtime import full_targets as runtime +from microcosm.calibrate import TargetRegistry, TargetSpec + + +def _registry(*names): + return TargetRegistry( + [ + TargetSpec( + name=name, + entity="person", + value=1.0, + measure="age", + period=2024, + source="test", + family="population", + metadata={"contract_target_id": name}, + ) + for name in names + ], + country="uk", + ) + + +@pytest.fixture +def prepared(monkeypatch): + national = _registry("retained", "excluded") + approved = _registry("retained") + local = _registry("local") + calls = [] + artifact = SimpleNamespace( + facts=({"test": True},), facts_sha256="a" * 64, manifest_sha256="b" * 64 + ) + pin = SimpleNamespace( + facts_sha256=artifact.facts_sha256, + manifest_sha256=artifact.manifest_sha256, + fact_row_count=1, + to_dict=lambda: { + "facts_sha256": artifact.facts_sha256, + "manifest_sha256": artifact.manifest_sha256, + }, + ) + monkeypatch.setattr(runtime, "load_uk_chronicle_feed", lambda: pin) + monkeypatch.setattr( + runtime, + "load_uk_local_chronicle_pin", + lambda: { + "facts_sha256": "a" * 64, + "manifest_sha256": "b" * 64, + "fact_row_count": 1, + }, + ) + monkeypatch.setattr( + runtime, "load_ledger_consumer_artifact", lambda *a, **kw: artifact + ) + monkeypatch.setattr(runtime, "load_uk_local_area_crosswalk", lambda: {}) + + def compile_national(facts, *, target_period): + calls.append(("national", target_period)) + return SimpleNamespace(registry=national, unsupported=()) + + def compile_local(facts, *, target_period, crosswalk): + calls.append(("local", target_period)) + return SimpleNamespace(registry=local, unsupported=()) + + monkeypatch.setattr(runtime, "compile_uk_target_registry", compile_national) + monkeypatch.setattr(runtime, "compile_uk_local_target_registry", compile_local) + monkeypatch.setattr( + runtime, "load_uk_calibration_measure_exclusions", lambda path: () + ) + monkeypatch.setattr( + runtime, + "apply_uk_calibration_measure_exclusions", + lambda registry, exclusions, now: ( + approved, + {"excluded": {"reason": "reviewed"}}, + ), + ) + return national, approved, local, artifact, calls + + +def _load(**kwargs): + return runtime.load_uk_full_target_inputs( + "facts.jsonl", + calibration_year=2024, + exclusions_evaluated_on=date(2026, 9, 10), + **kwargs, + ) + + +def test_full_inputs_preserve_band_edges_and_validation_periods(prepared): + national, approved, local, artifact, calls = prepared + result = _load() + assert result["artifact"] is artifact + assert result["band_edge_registry"] is national + assert result["national_registry"] is approved + assert result["local_registry"] is local + assert calls == [ + ("national", 2023), + ("national", 2024), + ("national", 2025), + ("local", 2024), + ("local", 2025), + ] + assert result["register_completeness"]["compiled_reference_count"] == 2 + assert result["register_completeness"]["approved_reference_count"] == 1 + assert result["register_completeness"]["compiled_local_reference_count"] == 1 + assert result["local_source_pin"]["fact_row_count"] == 1 + assert result["reviewed_unbound_higher_targets"] == { + "excluded": {"reason": "reviewed"} + } + + +@pytest.mark.parametrize("field", ["facts", "manifest"]) +def test_source_pin_mismatch_refuses_before_read(prepared, monkeypatch, field): + monkeypatch.setattr( + runtime, + "load_ledger_consumer_artifact", + lambda *a, **kw: pytest.fail("source read before pin agreement"), + ) + with pytest.raises(ValueError, match="committed national feed"): + _load(**{f"expected_{field}_sha256": "c" * 64}) + + +def test_loaded_source_mismatch_refuses(prepared): + prepared[3].manifest_sha256 = "c" * 64 + with pytest.raises(ValueError, match="Ledger artifact"): + _load() + assert prepared[4] == [] + + +def test_omitted_explicit_pins_still_bind_both_reviewed_source_hashes( + prepared, monkeypatch +): + calls = [] + monkeypatch.setattr( + runtime, + "load_ledger_consumer_artifact", + lambda path, **kwargs: calls.append(kwargs) or prepared[3], + ) + _load() + assert calls == [ + {"expected_facts_sha256": "a" * 64, "expected_manifest_sha256": "b" * 64} + ] + + +def test_local_review_is_required_before_read_or_compilation(prepared, monkeypatch): + monkeypatch.setattr( + runtime, + "load_uk_local_chronicle_pin", + lambda: {"facts_sha256": "c" * 64}, + ) + monkeypatch.setattr( + runtime, + "load_ledger_consumer_artifact", + lambda *a, **kw: pytest.fail("source read before local review"), + ) + with pytest.raises(ValueError, match="national and local feed pins disagree"): + _load() + assert prepared[4] == [] + + +def test_wrong_fact_count_refuses_before_either_compiler(prepared): + prepared[3].facts = () + with pytest.raises(ValueError, match="fact row count"): + _load() + assert prepared[4] == [] + + +def test_current_national_and_local_pins_share_one_reviewed_identity(): + # main (#904) reads one committed pin, ``uk/chronicle_feed.json``, for the + # national feed and the local census; the full build checks they agree. + national = runtime.load_uk_chronicle_feed() + local = runtime.load_uk_local_chronicle_pin() + assert ( + national.facts_sha256 + == local["facts_sha256"] + == ("e5baacd48ab32be197268ef3ef25dfb3a04b4a03d394ca3a6f22b8f15fa9e8d6") + ) + assert ( + national.manifest_sha256 + == local["manifest_sha256"] + == ("c479426724efa0a79ad2ebaf4ca0b5487fc307f0499d6150dacdc4e6d665cdff") + ) + assert ( + national.source_commit + == local["source_commit"] + == "5324aa27a7698eef97b38a7924a36b3a1f97c137" + ) + assert national.fact_row_count == local["fact_row_count"] == 287024 + + +def test_chronicle_source_codec_validates_directory_manifest(tmp_path): + from microcosm.build.uk_runtime import graph_targets # noqa: F401 + from microcosm.graph.codecs import SOURCE_CODECS + + payload = b'{"value": 1.0}\n' + (tmp_path / "consumer_facts.jsonl").write_bytes(payload) + manifest = tmp_path / "manifest.json" + manifest.write_text( + json.dumps( + { + "schema_version": CONSUMER_ARTIFACT_SCHEMA_VERSION, + "facts_sha256": hashlib.sha256(payload).hexdigest(), + } + ) + ) + assert SOURCE_CODECS.load_bytes(runtime.CHRONICLE_SOURCE_CODEC, tmp_path) == payload + (tmp_path / "consumer_facts.jsonl").write_bytes(b'{"value": 2.0}\n') + with pytest.raises(ValueError, match="manifest hash"): + SOURCE_CODECS.load_bytes(runtime.CHRONICLE_SOURCE_CODEC, tmp_path) + + +def test_frozen_register_compares_complete_not_measure_pruned_surface( + prepared, tmp_path +): + path = tmp_path / "register.json" + prepared[0].to_json(path) + assert ( + _load(register_json=path)["register_completeness"]["frozen_registry_version"] + == prepared[0].version + ) + prepared[1].to_json(path) + with pytest.raises(ValueError, match="full national register differs"): + _load(register_json=path) + + +def test_validation_reference_compilation_is_fail_closed(prepared, monkeypatch): + monkeypatch.setattr( + runtime, + "compile_uk_target_registry", + lambda *a, **kw: SimpleNamespace( + registry=prepared[0], unsupported=({"target": "missing"},) + ), + ) + with pytest.raises(ValueError, match="failed to compile for 2023"): + _load() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py new file mode 100644 index 000000000..bcdba256e --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py @@ -0,0 +1,678 @@ +"""Export artifacts bind exact H5 content and survive filesystem recreation.""" + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime.graph_terminal import ( + add_uk_export_continuation, + add_uk_export_preparation, + describe_uk_export, + materialize_uk_export, + register_uk_terminal_kernels, + validate_uk_export, +) +from microcosm.frame import WeightKind +from test_support.microcosm_build.uk_graph_terminal import _frame +from test_support.microcosm_build.uk_hierarchy_fixtures import ( + uk_fixture_hierarchy, +) + + +def test_export_roundtrip_preserves_dtype_weights_lineage_period_and_gate(tmp_path): + pytest.importorskip("tables") + frame = _frame() + descriptor = describe_uk_export( + frame, + bindings={"pool_replicates": 1, "dataset_households": 2, "target_scope": "all"}, + ) + path = tmp_path / "full.h5" + record = materialize_uk_export(frame, descriptor, path) + report = validate_uk_export(path, descriptor) + assert report["passed"] is True + assert record == report["dataset"] + assert descriptor["tables"]["person"]["dtypes"][-1] == "float32" + assert descriptor["time_period"] == "2024" + assert descriptor["weight_kind"] == "calibrated" + path.unlink() + materialize_uk_export(frame, descriptor, path) + assert validate_uk_export(path, descriptor)["passed"] is True + + +def test_export_refuses_population_changed_after_descriptor(tmp_path): + frame = _frame() + descriptor = describe_uk_export(frame, bindings={}) + frame.table("person").loc[0, "income"] = 99.0 + with pytest.raises(ValueError, match="descriptor"): + materialize_uk_export(frame, descriptor, tmp_path / "changed.h5") + + +def test_export_readback_reports_changed_stored_values(tmp_path): + pytest.importorskip("tables") + frame = _frame() + descriptor = describe_uk_export(frame, bindings={}) + path = tmp_path / "full.h5" + materialize_uk_export(frame, descriptor, path) + with pd.HDFStore(path) as store: + person = store["person"] + person.loc[0, "income"] = np.float32(77.0) + store.put("person", person, format="table") + report = validate_uk_export(path, descriptor) + assert report["passed"] is False + assert any("person" in failure for failure in report["failures"]) + + +def test_graph_export_continuation_reuses_numerics_and_validates_recreated_file( + tmp_path, +): + import json + + from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, + ) + + pytest.importorskip("tables") + + class Create(KernelBase): + ref = "fixture.create@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + return KernelResult(frame=_frame()) + + frame = _frame() + identifiers = { + "person_id", + "benunit_id", + "household_id", + "person_household_id", + "person_benunit_id", + } + source = tmp_path / "fixture.txt" + source.write_text("deterministic export fixture") + graph = Graph( + "uk", + (SourceRef("fixture", "raw-bytes-v1"),), + ( + Node( + id="root", + kernel=Create.ref, + structural=StructuralDelta.CREATE, + sources=("fixture",), + outputs=tuple( + Owned(entity, column, str(table[column].dtype)) + for entity in frame.entities + for table in (frame.table(entity),) + for column in table.columns + if column not in identifiers + ), + ), + ), + ) + graph = add_uk_export_preparation( + graph, + population="root", + bindings={"pool_replicates": 1, "dataset_households": 2, "target_scope": "all"}, + ) + registry = KernelRegistry() + registry.register(Create()) + register_uk_terminal_kernels(registry) + store = ContentStore(tmp_path / "store") + numerical = run_graph( + compile_graph(graph), sources={"fixture": source}, store=store, kernels=registry + ) + key = numerical.nodes["uk.full.export.prepare"].opaque_artifacts[ + "export_descriptor" + ] + descriptor = json.loads(store.load_bytes(key)) + path = tmp_path / "full.h5" + materialize_uk_export(numerical.population("root"), descriptor, path) + continued = add_uk_export_continuation( + graph, population="root", manifest_binding={"key": numerical.key} + ) + terminal = run_graph( + compile_graph(continued), + sources={"fixture": source, "exported_dataset": path}, + store=store, + kernels=registry, + ) + assert terminal.nodes["root"].store_hit + assert terminal.nodes["uk.full.export.prepare"].store_hit + assert terminal.nodes["uk.full.export.readback"].receipt["outcome"] == "pass" + inventory = json.loads( + store.load_bytes( + terminal.nodes["uk.full.package"].opaque_artifacts["package_inventory"] + ) + ) + assert inventory["numerical_graph"]["key"] == numerical.key + assert inventory["build_bindings"]["target_scope"] == "all" + path.unlink() + materialize_uk_export(numerical.population("root"), descriptor, path) + replay = run_graph( + compile_graph(continued), + sources={"fixture": source, "exported_dataset": path}, + store=store, + kernels=registry, + ) + assert replay.nodes["root"].store_hit + assert replay.nodes["uk.full.export.readback"].receipt["outcome"] == "pass" + + +@pytest.mark.parametrize("households", [None, 2]) +def test_full_gate_nodes_precede_dense_and_bind_final_problem_axis(households): + from microcosm.build.uk_runtime.graph import uk_spine_endpoint, uk_spine_graph + from microcosm.build.uk_runtime.graph_build import UKFullBuildConfig, uk_full_graph + from microcosm.build.uk_runtime.graph_calibration import UKGraphCalibrationConfig + from microcosm.build.uk_runtime.graph_terminal import append_uk_full_gate_nodes + from microcosm.graph import compile_graph + + raw = uk_spine_graph(source_mode="split") + full = uk_full_graph( + UKFullBuildConfig( + calibration_year=2025, + calibration=UKGraphCalibrationConfig(dataset_households=households), + ), + spine=raw, + ) + graph = append_uk_full_gate_nodes( + full.graph, + calibration=full.calibration, + spine_stage_names=uk_spine_endpoint(raw).stage_names, + engine_identity="fixture-engine", + review_date="2026-09-10", + ) + compiled = compile_graph(graph) + assert ( + "uk.full.gates.preflight" + in compiled.predecessors[full.calibration.dense_producer] + ) + preflight = graph.node("uk.full.gates.preflight") + assert not preflight.inputs + assert not {"problem", "solution"} & { + item.name for item in preflight.artifact_inputs + } + final = graph.node("uk.full.gates.calibrated") + assert final.population == full.calibration.population + inputs = {item.name: item for item in final.artifact_inputs} + assert inputs["problem"].producer == full.calibration.problem_producer + assert inputs["solution"].artifact == ( + "solution" if households is None else "refit_solution" + ) + + +def test_full_preflight_persists_real_source_failures_without_matrix_diagnostics( + monkeypatch, +): + import json + from types import SimpleNamespace + + from microcosm.build.gate_battery import _gates_manifest_payload + from microcosm.build.uk_runtime import full_gates + from microcosm.build.uk_runtime.graph_terminal import ( + UKFullGateKernel, + decode_full_gate_report, + ) + from microcosm.graph.canonical import canonical_json + + selection = { + "schema": "microcosm.calibrate.target-selection.v1", + "selector": {"geography_levels": None}, + "included": [ + {"name": "n", "period": 2025, "geography_level": "country"}, + {"name": "l", "period": 2025, "geography_level": "constituency"}, + ], + "excluded": [], + } + empty_registry = {"country": "uk", "specs": []} + + def artifact(payload): + return SimpleNamespace(payload=canonical_json(payload), key="a" * 64) + + context = SimpleNamespace( + params={ + "engine_identity": "fixture", + "phase": "preflight", + "gate_manifest": canonical_json( + _gates_manifest_payload(full_gates.uk_full_gate_manifest()) + ).decode(), + "spine_stage_names": ("frs_spine",), + "review_date": "2026-09-10", + "sample_fraction": 1.0, + "release_candidate": True, + }, + artifacts={ + "selection": artifact({"receipt": selection}), + "surface": artifact( + { + "national_registry": empty_registry, + "uk_ledger_compiled_registries": { + "2023": empty_registry, + "2025": empty_registry, + }, + "uk_ledger_compiled_local_registries": {"2025": empty_registry}, + } + ), + "spine_provenance": artifact( + {"stages": ["frs_spine"], "stage_evidence": {}} + ), + }, + ) + + def forbidden(*args, **kwargs): + raise AssertionError( + "Source preflight must not compute final matrix diagnostics" + ) + + monkeypatch.setattr(full_gates, "build_full_gate_context", forbidden) + result = UKFullGateKernel(coverage_engine=object(), engine_identity="fixture").run( + context + ) + report, enforcement = decode_full_gate_report(result.artifacts["gate_report"]) + assert report.phase == "preflight" + assert len(report.outcomes) == 6 + assert enforcement["artifact_permitted"] is False + assert "uk_release_family_build_stages" in enforcement["structural_failures"] + document = json.loads(result.artifacts["gate_report"]) + assert document["target_diagnostics"] == [] + document["enforcement"]["artifact_permitted"] = True + with pytest.raises(ValueError, match="enforcement"): + decode_full_gate_report(document) + + +def test_terminal_byte_materialization_recreates_exact_files(tmp_path): + import hashlib + from types import SimpleNamespace + + from microcosm.build.uk_runtime.graph_terminal import ( + materialize_uk_terminal_artifacts, + ) + from microcosm.graph import ContentStore + + payloads = { + "calibration_diagnostics": b'{"score":1}', + "target_diagnostics_csv": b"target,estimate\na,1\n", + "area_support_csv": b"area,rows\na,2\n", + "holdout": b'{"report_only":true}', + "selection": b'{"registry":{}}', + } + store = ContentStore(tmp_path / "store") + keys = {} + for name, payload in payloads.items(): + key = hashlib.sha256(payload).hexdigest() + store.put_bytes(key, payload) + keys[name] = key + manifest = SimpleNamespace( + nodes={ + "uk.full.gates.calibrated": SimpleNamespace(opaque_artifacts=keys), + "uk.full.holdout": SimpleNamespace(opaque_artifacts=keys), + "uk.full.target_selection": SimpleNamespace(opaque_artifacts=keys), + } + ) + first = materialize_uk_terminal_artifacts( + manifest, store, directory=tmp_path, stem="full" + ) + for record in first.values(): + (tmp_path / record["filename"]).unlink() + assert ( + materialize_uk_terminal_artifacts( + manifest, store, directory=tmp_path, stem="full" + ) + == first + ) + + +def test_holdout_kernel_preserves_existing_rotations_and_seed_settings(monkeypatch): + import json + from types import SimpleNamespace + + from microcosm.build.uk_runtime import graph_targets, local_rowwise + from microcosm.build.uk_runtime.graph_terminal import UKFullHoldoutKernel + from microcosm.calibrate import Target, TargetSet, build_constraint_matrix + from microcosm.calibrate.artifacts import encode_problem + from test_support.microcosm_build.uk_full_calibration_graph import ( + preflight_payload, + ) + from test_support.microcosm_build.uk_local_rowwise import ( + _assigned, + _clone_frame, + ) + + frame = _clone_frame() + names = ("households", "tenure/social_rent", "tenure/private_rent") + problem = local_rowwise.build_uk_rowwise_local_matrix( + pd.DataFrame({name: [1.0, 2.0, 3.0] for name in names}, index=[101, 102, 103]), + _assigned(), + pd.DataFrame( + {"code": ["E001", "S001"], **{name: [3.0, 3.0] for name in names}} + ), + ) + calls = [] + + def solve(frame, training, **kwargs): + calls.append( + (kwargs["seed"], kwargs["dataset_households"], kwargs["selection_seed"]) + ) + return SimpleNamespace(weights=np.ones(2), selected_support=np.array([0, 2])) + + monkeypatch.setattr(local_rowwise, "solve_uk_rowwise_weights_under_doctrine", solve) + monkeypatch.setattr( + local_rowwise, + "_derive_uk_local_bound_families_from_target_frame", + lambda *a, **k: (), + ) + monkeypatch.setattr( + graph_targets, + "reconstruct_uk_full_problem_inputs", + lambda context: SimpleNamespace( + frame=frame, local_problem=problem, national_rows=None, bound_families=() + ), + ) + original = encode_problem( + build_constraint_matrix( + frame, + TargetSet([Target("count", "household", lambda f: np.ones(3), 3.0)]), + "household", + ), + entity_ids=[101, 102, 103], + ) + context = SimpleNamespace( + params={ + "skip_holdout": False, + "target_weight_rule": "uniform", + "epochs": 1, + "learning_rate": 0.1, + "dataset_households": 2, + "seed": 42, + "selection_seed": 17, + "selection_pi_hi": 1.0, + "baseline_pi_floor": 0.0, + }, + artifacts={ + "preflight": SimpleNamespace(payload=preflight_payload(), key="a" * 64), + "problem": SimpleNamespace(payload=original, key="b" * 64), + }, + ) + result = json.loads(UKFullHoldoutKernel().run(context).artifacts["holdout"]) + expected = local_rowwise.rotated_uk_local_holdout( + frame, + problem, + bound_families=(), + epochs=1, + learning_rate=0.1, + dataset_households=2, + solve_seed=42, + selection_seed=17, + ) + assert { + key: value for key, value in result.items() if key != "graph_binding" + } == expected + assert calls == [(42, 2, 17)] * 10 + + +def test_final_gate_kernel_owns_complete_diagnostics_and_reuses_decoded_result( + monkeypatch, +): + import json + from dataclasses import replace + from types import SimpleNamespace + + from microcosm.build.gate_battery import _gates_manifest_payload + from microcosm.build.uk_runtime import full_gates, geography_ladder + from microcosm.build.uk_runtime.graph_targets import registry_payload + from microcosm.build.uk_runtime.graph_terminal import ( + UKFullGateKernel, + decode_full_gate_report, + ) + from microcosm.calibrate import ( + TargetRegistry, + TargetSet, + TargetSpec, + build_constraint_matrix, + calibrate, + ) + from microcosm.calibrate.artifacts import ( + decode_problem, + encode_calibration_result, + encode_problem, + encode_solution, + ) + from microcosm.frame import Frame, Weights + from microcosm.graph.canonical import canonical_json + from test_support.microcosm_build.uk_full_calibration_graph import ( + preflight_payload, + ) + + original = _frame() + tables = {entity: original.table(entity).copy() for entity in original.entities} + tables["household"]["source_household_id"] = [1, 2] + tables["household"]["household_is_spi_synthetic"] = False + tables["household"]["household_is_capital_gains_clone"] = False + initial = Frame( + tables, + original.schema, + {"household": Weights(np.array([13.0, 87.0]), WeightKind.IMPORTANCE)}, + original.strata, + metadata=original.metadata, + ) + specs = [ + TargetSpec( + name="national", + entity="household", + value=100.0, + measure="household_id", + period=2024, + source="fixture", + family="households", + hierarchy=uk_fixture_hierarchy( + "national", level="country", geography_id="UK" + ), + metadata={ + "geography_level": "country", + "geography_id": "UK", + "materialization": "uk_national_measure", + }, + ), + TargetSpec( + name="local", + entity="household", + value=13.0, + measure="household_id", + period=2024, + source="fixture", + family="census_households", + hierarchy=uk_fixture_hierarchy( + "local", level="constituency", geography_id="E14000001" + ), + metadata={ + "geography_level": "constituency", + "geography_id": "E14000001", + "materialization": "uk_local_surface", + "area_type": "constituency", + "area_code": "E14000001", + "metric": "households", + }, + ), + ] + targets = TargetSet( + [ + replace(specs[0].to_target(), measure=lambda f: np.ones(2)), + replace(specs[1].to_target(), measure=lambda f: np.array([1.0, 0.0])), + ] + ) + selection = { + "schema": "microcosm.calibrate.target-selection.v1", + "selector": {"geography_levels": None}, + "included": [ + { + "name": spec.name, + "period": 2024, + "geography_level": spec.metadata["geography_level"], + } + for spec in specs + ], + "excluded": [], + } + problem = build_constraint_matrix(initial, targets, "household") + problem_bytes = encode_problem( + problem, + entity_ids=[1, 2], + target_metadata=[{**spec.metadata, "family": spec.family} for spec in specs], + bindings={"target_selection": selection}, + ) + ordered = decode_problem(problem_bytes) + result = calibrate(initial, targets, weight_entity="household", epochs=1, seed=42) + frame = result.frame + + def artifact(payload, raw=False): + return SimpleNamespace( + payload=payload if raw else canonical_json(payload), key="a" * 64 + ) + + empty = {"country": "uk", "specs": []} + artifacts = { + "problem": artifact(problem_bytes, True), + "result": artifact( + encode_calibration_result( + result, entity_ids=[1, 2], problem_sha256=ordered.sha256 + ), + True, + ), + "solution": artifact( + encode_solution( + result.weights, entity_ids=[1, 2], problem_sha256=ordered.sha256 + ), + True, + ), + "selection": artifact( + { + "receipt": selection, + "registry": registry_payload(TargetRegistry(specs, country="uk")), + } + ), + "preflight": artifact(preflight_payload(selection=selection), True), + "spine_provenance": artifact( + {"stages": ["frs_spine"], "stage_evidence": {}, "fit_weight_records": {}} + ), + "surface": artifact( + { + "national_registry": registry_payload( + TargetRegistry(specs[:1], country="uk") + ), + "uk_ledger_compiled_registries": {"2023": empty, "2025": empty}, + "uk_ledger_compiled_local_registries": {"2025": empty}, + } + ), + "holdout": artifact({"report_only": True, "outcome": "fixture"}), + } + monkeypatch.setattr( + geography_ladder, + "load_uk_oa_ladder", + lambda path: SimpleNamespace( + constituency_code=np.array(["E14000001", "E14000002"]), + local_authority_code=np.array(["E09000001", "E07000002"]), + ), + ) + monkeypatch.setattr( + full_gates, + "load_efrs_parity_reference", + lambda: SimpleNamespace(input_entities={}), + ) + monkeypatch.setattr( + full_gates, "uk_aggregate_admin_totals", lambda frame, gates: ({}, []) + ) + + def forbidden(*args, **kwargs): + raise AssertionError("The decoded result diagnostics must be reused") + + monkeypatch.setattr(full_gates, "_build_diagnostics", forbidden) + context = SimpleNamespace( + tables={entity: frame.table(entity) for entity in frame.entities}, + weights={"household": frame.weights_for("household")}, + strata=frame.strata, + frame_metadata=frame.metadata, + frame_mass_log=frame.mass_log, + frame_column_order={}, + params={ + "phase": "terminal", + "engine_identity": "fixture", + "review_date": "2026-09-10", + "sample_fraction": 1.0, + "release_candidate": False, + "spine_stage_names": ("frs_spine",), + "gate_manifest": canonical_json( + _gates_manifest_payload(full_gates.uk_full_gate_manifest()) + ).decode(), + }, + artifacts=artifacts, + sources={"uk_ladder": "fixture"}, + ) + stored = UKFullGateKernel(coverage_engine=object(), engine_identity="fixture").run( + context + ) + phase, _ = decode_full_gate_report(stored.artifacts["gate_report"]) + assert phase.phase == "terminal" + document = json.loads(stored.artifacts["calibration_diagnostics"]) + assert document["uk_diagnostics"]["rotated_holdout"]["outcome"] == "fixture" + assert len(document["targets"]) == 2 + assert ( + len(pd.read_csv(__import__("io").BytesIO(stored.artifacts["area_support_csv"]))) + == 4 + ) + + +def test_package_validates_materialized_evidence_against_graph_bytes(tmp_path): + import json + from types import SimpleNamespace + + from microcosm.build.uk_runtime.graph_terminal import UKPackageInventoryKernel + from microcosm.graph.canonical import canonical_json + + payload = b'{"diagnostics":"graph-owned"}' + evidence = tmp_path / "full.diagnostics.json" + evidence.write_bytes(payload) + readback = { + "schema_version": 1, + "kind": "uk_full_build_export_readback", + "passed": True, + "dataset": {"filename": "full.h5", "sha256": "b" * 64, "size_bytes": 10}, + "content_sha256": "c" * 64, + "bindings": {"K": 20, "k": 2, "scope": "all"}, + } + context = SimpleNamespace( + params={ + "manifest_binding": "{}", + "evidence_files": json.dumps({"diagnostics": evidence.name}), + }, + sources={"exported_evidence_diagnostics": evidence}, + artifacts={ + "export_readback": SimpleNamespace( + payload=canonical_json(readback), key="d" * 64 + ), + "diagnostics": SimpleNamespace(payload=payload, key="a" * 64), + }, + ) + document = json.loads( + UKPackageInventoryKernel().run(context).artifacts["package_inventory"] + ) + assert document["evidence_files"]["diagnostics"]["graph_artifact_key"] == "a" * 64 + evidence.write_bytes(b"tampered") + with pytest.raises(ValueError, match="differs from its graph artifact"): + UKPackageInventoryKernel().run(context) + readback["passed"] = False + context.artifacts["export_readback"].payload = canonical_json(readback) + with pytest.raises(ValueError, match="H5 readback failed"): + UKPackageInventoryKernel().run(context) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_local_rowwise.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_local_rowwise.py index 4ad6e604a..8ac31204b 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_local_rowwise.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_local_rowwise.py @@ -43,32 +43,12 @@ from microcosm.calibrate.registry import TargetRegistry, TargetSpec from microcosm.calibrate.target import TargetSet from microcosm.frame import WeightKind +from test_support.microcosm_build.uk_local_rowwise import _assigned, _clone_frame from test_support.paths import paths_for _TEST_PATHS = paths_for("microcosm-build") -def _clone_frame(weights=(1.0, 1.0, 1.0)): - return uk_national_frame( - person=pd.DataFrame( - { - "person_id": [1, 2, 3], - "person_household_id": [101, 102, 103], - "person_benunit_id": [11, 12, 13], - } - ), - benunit=pd.DataFrame({"benunit_id": [11, 12, 13]}), - household=pd.DataFrame( - { - "household_id": [101, 102, 103], - "household_weight": list(weights), - } - ), - time_period="2023", - weight_kind=WeightKind.IMPORTANCE, - ) - - def _metrics() -> pd.DataFrame: return pd.DataFrame( { @@ -79,10 +59,6 @@ def _metrics() -> pd.DataFrame: ) -def _assigned() -> pd.Series: - return pd.Series(["E001", "E001", "S001"], index=[101, 102, 103]) - - def _targets() -> pd.DataFrame: return pd.DataFrame( { diff --git a/test_support/microcosm_build/uk_calibration_run.py b/test_support/microcosm_build/uk_calibration_run.py index d743d0075..bd32084b8 100644 --- a/test_support/microcosm_build/uk_calibration_run.py +++ b/test_support/microcosm_build/uk_calibration_run.py @@ -31,6 +31,7 @@ UKCalibrationRunPaths, run_uk_calibration, ) +from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity from microcosm.build.uk_runtime.etb_services import ( UK_NHS_SPENDING_COMPONENT_COLUMNS, ) @@ -390,4 +391,33 @@ def _mixed_epoch_artifact_dir(tmp_path: Path) -> Path: return artifact_dir +def _bound_checkpoint(tmp_path, frame): + report_path = tmp_path / "spine.spine_gates.json" + report = { + **calibration_run.uk_spine_checkpoint_gate_digests(), + "blocked_at_phase": None, + "gates": { + entry.id: {"status": "passed", "criticality": entry.criticality} + for entry in load_country_spec("uk").gates.gates + if entry.id in calibration_run.UK_SPINE_GATE_SCOPE + }, + } + report_path.write_text(json.dumps(report)) + sidecar = { + "entity_row_counts": { + entity: len(frame.table(entity)) for entity in frame.entities + }, + "household_weight_kind": frame.weights_for("household").kind.value, + "household_weight_total": float(frame.weights_for("household").values.sum()), + "uk_frame_content_identity": uk_frame_content_identity(frame), + "spine_gate_report": { + "sha256": hashlib.sha256(report_path.read_bytes()).hexdigest() + }, + "fit_weight_records": {"model": {"fit_weights_used": True}}, + } + sidecar_path = tmp_path / "spine.build.json" + sidecar_path.write_text(json.dumps(sidecar)) + return sidecar_path, report_path, sidecar + + __all__ = [name for name in globals() if not name.startswith("__")] diff --git a/test_support/microcosm_build/uk_full_calibration_graph.py b/test_support/microcosm_build/uk_full_calibration_graph.py new file mode 100644 index 000000000..30077b236 --- /dev/null +++ b/test_support/microcosm_build/uk_full_calibration_graph.py @@ -0,0 +1,64 @@ +"""A synthetic preflight gate report on the full-gate manifest, shared +by the calibration-graph and target-graph tests.""" + +from __future__ import annotations + +from microcosm.graph.canonical import canonical_json + + +def preflight_payload(passed=True, selection=None): + from microcosm.build.gate_battery import ( + GateOutcome, + GatePhaseReport, + GateStatus, + gate_phase_report_payload, + ) + from microcosm.build.gates import GateResult + from microcosm.build.uk_runtime.full_gates import ( + classify_full_gate_outcomes, + uk_full_gate_manifest, + ) + + selection = ( + selection + if selection is not None + else { + "schema": "microcosm.calibrate.target-selection.v1", + "selector": {"geography_levels": ["country"], "explicit": True}, + "included": [{"name": "count", "period": 0, "geography_level": "country"}], + "excluded": [], + } + ) + gates = uk_full_gate_manifest(selection) + report = GatePhaseReport( + "preflight", + tuple( + GateOutcome( + entry=entry, + status=GateStatus.PASSED if passed else GateStatus.FAILED, + result=GateResult( + name=entry.gate, + passed=passed, + failures=() if passed else ("fixture blocked",), + ), + ) + for entry in gates.gates + if entry.phase == "preflight" + ), + ) + return canonical_json( + { + "schema_version": 1, + "kind": "uk_full_gate_report", + "selection_receipt": selection, + "sample_fraction": 1.0, + "release_candidate": True, + "report": gate_phase_report_payload(report, gates=gates), + "enforcement": classify_full_gate_outcomes( + report, sample_fraction=1.0, release_candidate=True + ), + } + ) + + +__all__ = [name for name in globals() if not name.startswith("__")] diff --git a/test_support/microcosm_build/uk_full_population_graph.py b/test_support/microcosm_build/uk_full_population_graph.py new file mode 100644 index 000000000..ffc958b80 --- /dev/null +++ b/test_support/microcosm_build/uk_full_population_graph.py @@ -0,0 +1,113 @@ +"""Synthetic full-build population graph: the seam-frame source kernel and +the graph/registry pair the UK full-graph tests build on.""" + +# ruff: noqa: F401 + +import hashlib + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime.graph_kernels import UKClaimKernel +from microcosm.build.uk_runtime.graph_population import ( + append_uk_population_nodes, + register_uk_population_kernels, +) +from microcosm.build.uk_runtime.rowwise_dataset import ( + clone_uk_dataset_with_ladder_geography, +) +from microcosm.frame import Frame +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from test_support.microcosm_build.uk_ladder_rowwise_clone import _seam_frame +from test_support.microcosm_build.uk_ladder_rowwise_clone import ( + toy_ladder as toy_ladder, +) + + +def source_frame(): + original = _seam_frame() + tables = {e: original.table(e).copy() for e in original.entities} + tables["person"]["age"] = 40 + tables["benunit"]["would_claim_uc"] = True + tables["household"]["region"] = tables["household"]["region"].astype("string") + return Frame( + tables, + original.schema, + {"household": original.weights_for("household")}, + original.strata, + mass_log=original.mass_log, + metadata=original.metadata, + ) + + +class Source(KernelBase): + ref = "uk.test.full-source@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def implementation_hash(self): + return hashlib.sha256(self.ref.encode()).hexdigest() + + def run(self, context): + return KernelResult(frame=source_frame()) + + +def graph_and_registry(k): + frame = source_frame() + source = Node( + "source", + Source.ref, + structural=StructuralDelta.CREATE, + sources=("fixture",), + outputs=tuple( + Owned( + entity, + col, + "string" if table[col].dtype.kind in "OUS" else str(table[col].dtype), + ) + for entity in frame.entities + for table in [frame.table(entity)] + for col in table.columns + if col + not in { + "person_id", + "person_household_id", + "person_benunit_id", + "household_id", + "benunit_id", + } + ), + ) + graph = append_uk_population_nodes( + Graph("uk", (SourceRef("fixture", "raw-bytes-v1"),), (source,)), + population="source", + time_period="2023", + weight_kind="importance", + n_clones=k, + seed=7, + source_year=2023, + ) + registry = KernelRegistry() + registry.register(Source()) + registry.register(UKClaimKernel()) + register_uk_population_kernels(registry) + return graph, registry + + +__all__ = [name for name in globals() if not name.startswith("__")] diff --git a/test_support/microcosm_build/uk_full_target_graph.py b/test_support/microcosm_build/uk_full_target_graph.py new file mode 100644 index 000000000..650133841 --- /dev/null +++ b/test_support/microcosm_build/uk_full_target_graph.py @@ -0,0 +1,314 @@ +"""Synthetic target inputs, the preflight kernel and the full-graph build +helper shared by the engine-free and engine halves of the target-graph +tests.""" + +# ruff: noqa: F401 + +import hashlib +import json +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime import ( + full_targets, + geography_ladder, + graph_targets, + ledger_targets, +) +from microcosm.build.uk_runtime.graph_build import ( + UKFullBuildConfig, + register_uk_full_kernels, + uk_full_graph, +) +from microcosm.build.uk_runtime.graph_calibration import UKGraphCalibrationConfig +from microcosm.build.uk_runtime.graph_terminal import FULL_GATE_REPORT_TYPE +from microcosm.build.uk_runtime.local_rowwise import UKRowwiseNationalRows +from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.calibrate.artifacts import decode_problem +from microcosm.frame import Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + compile_graph, + run_graph, +) +from test_support.microcosm_build.uk_full_calibration_graph import ( + preflight_payload, +) +from test_support.microcosm_build.uk_full_population_graph import ( + Source, + graph_and_registry, +) +from test_support.microcosm_build.uk_ladder_rowwise_clone import ( + toy_ladder as toy_ladder, +) + + +@pytest.fixture +def target_inputs(monkeypatch, toy_ladder): + national = TargetRegistry( + [ + TargetSpec( + name="country_households", + entity="household", + measure="test_ones", + value=34.0, + period=2026, + family="fixture", + source="fixture", + metadata={ + "geography_level": "country", + "geography_id": "UK", + "contract_target_id": "obr.vat", + }, + ), + TargetSpec( + # Repeated names across periods must retain distinct metadata. + name="country_households", + entity="household", + measure="test_ones", + value=4.0, + period=2025, + family="fixture", + source="fixture", + filter="test_london", + metadata={ + "geography_level": "region", + "geography_id": "LONDON", + # A region-level contract target on main (#905 SPI region + # facts); #934 moved council-tax stock to local-authority + # publisher contracts, so the old voa.* id no longer exists. + "contract_target_id": ( + "hmrc.spi_region.income_tax_by_region_50000_70000" + ), + }, + ), + ], + country="uk", + ) + ladder, _ = toy_ladder + local = TargetRegistry( + [ + TargetSpec( + name=f"ons.census.households@{level}:{code}", + entity="household", + measure="household_count", + value=35.0 if level == "local_authority" and i == 0 else 40.0, + period=2026, + family="census_households", + source="chronicle_fixture", + metadata={ + "contract_target_id": "ons.census.households", + "geography_level": level, + "geography_id": str(code), + "uprating_from_period": 2022 if str(code).startswith("S") else 2021, + "uprating_to_period": 2026, + }, + ) + for level, codes in ( + ("constituency", ladder.constituency_code), + ("local_authority", ladder.local_authority_code), + ) + for i, code in enumerate(codes) + ], + country="uk", + ) + inputs = { + "national_registry": national, + "band_edge_registry": national, + "local_registry": local, + "artifact": SimpleNamespace(facts=()), + "calibration_year": 2026, + "measure_exclusions": {}, + "reviewed_unbound_higher_targets": {}, + "national_source_pin": {"fixture": True}, + "local_source_pin": {"fixture": True}, + "register_completeness": {"fixture": True}, + "ledger_provenance": {"fixture": True}, + "uk_ledger_compiled_registries": {2026: national}, + "uk_ledger_compiled_local_registries": {2026: local}, + } + monkeypatch.setattr( + full_targets, "load_uk_full_target_inputs", lambda *args, **kwargs: inputs + ) + reference = {"value": 33.0, "period": 2026} + monkeypatch.setattr( + graph_targets, "uk_ledger_households_total", lambda *args, **kwargs: reference + ) + + def surface(): + return ledger_targets.uk_local_target_surface( + graph_targets.full_problem._joint_surface_registry(local, national), + bound_national_target_ids=graph_targets.full_problem._national_contract_target_ids( + national + ), + period=2026, + # #906: the toy ladder's membership, as the graph and the driver pass. + area_region_codes=geography_ladder.uk_area_region_codes(ladder), + census_household_uprating=ledger_targets.uk_census_household_uprating( + local, reference, period=2026 + ), + ) + + def measures(frame, national_registry, *, local_grains, **kwargs): + tables = {e: frame.table(e).copy() for e in frame.entities} + tables["household"]["test_ones"] = 1.0 + tables["household"]["test_london"] = ( + tables["household"]["region"] == "LONDON" + ).astype(float) + prepared = Frame( + tables, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + return ( + prepared, + lambda _: frame, + UKRowwiseNationalRows( + national_registry.to_target_set(), national_registry, ("fixture",) + ), + { + g: pd.DataFrame( + {"households": np.ones(frame.n("household"))}, + index=frame.table("household")["household_id"], + ) + for g in local_grains + }, + {"fixture": True}, + ) + + monkeypatch.setattr(graph_targets, "resolve_uk_full_measures", measures) + return { + "national": national, + "local": local, + "inputs": inputs, + "surface": surface, + "measures": measures, + } + + +class Preflight(KernelBase): + """Synthetic source verdict: tests below exercise numerical graph ownership.""" + + ref = "uk.test.target-preflight@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def run(self, context): + selection = json.loads(context.artifacts["selection"].payload)["receipt"] + return KernelResult( + artifacts={"preflight": preflight_payload(selection=selection)} + ) + + +def build( + tmp_path, + ladder_path, + levels, + *, + n_clones=1, + seed=7, + dataset_households=None, + resume="auto", + forbid_execution=False, +): + primitive, _ = graph_and_registry(1) + base = Graph( + "uk", + tuple(s for s in primitive.sources if s.name == "fixture"), + (primitive.node("source"),), + ) + config = UKFullBuildConfig( + calibration_year=2026, + time_period="2023", + source_year=2023, + n_clones=n_clones, + geography_levels=levels, + seed=seed, + calibration=UKGraphCalibrationConfig( + epochs=8, seed=seed, dataset_households=dataset_households + ), + ) + full = uk_full_graph(config, spine=base, spine_population="source") + preflight = Node( + "fixture.preflight", + Preflight.ref, + population="uk.full.pool", + artifact_inputs=( + ArtifactInput( + "selection", + "uk.full.target_selection", + "selection", + graph_targets.TARGET_SELECTION_TYPE, + ), + ), + artifact_outputs=(ArtifactOutput("preflight", FULL_GATE_REPORT_TYPE),), + ) + full = replace( + full, + graph=replace( + full.graph, + nodes=( + *( + replace( + node, + artifact_inputs=( + *node.artifact_inputs, + ArtifactInput( + "preflight", + preflight.id, + "preflight", + FULL_GATE_REPORT_TYPE, + ), + ), + ) + if node.id == full.calibration.dense_producer + else node + for node in full.graph.nodes + ), + preflight, + ), + ), + ) + registry = KernelRegistry() + registry.register(Source()) + registry.register(Preflight()) + register_uk_full_kernels(registry) + if forbid_execution: + for kernel in registry.as_mapping().values(): + kernel.run = lambda *args, **kwargs: pytest.fail( + "cached full graph executed" + ) + store = ContentStore(tmp_path / "store") + manifest = run_graph( + compile_graph(full.graph), + sources={ + "fixture": ladder_path, + "uk_ladder": ladder_path, + "uk_ledger_facts": ladder_path, + }, + store=store, + kernels=registry, + resume=resume, + ) + problem = decode_problem( + store.load_bytes(manifest.nodes["uk.full.problem"].opaque_artifacts["problem"]) + ) + return full, manifest, problem + + +__all__ = [name for name in globals() if not name.startswith("__")] diff --git a/test_support/microcosm_build/uk_graph_terminal.py b/test_support/microcosm_build/uk_graph_terminal.py new file mode 100644 index 000000000..35ad86c21 --- /dev/null +++ b/test_support/microcosm_build/uk_graph_terminal.py @@ -0,0 +1,60 @@ +"""A two-household calibrated national frame carrying the full ladder +geography, shared by the terminal-node and full-build CLI tests.""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from microcosm.build.uk_runtime.local_authority_input import ( + resolve_local_authority_engine_keys, +) +from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.frame import MassChangeRecord, WeightKind + + +def _frame(): + household = pd.DataFrame( + { + "household_id": [1, 2], + "household_clone_index": [0, 0], + "region": ["LONDON", "SOUTH_EAST"], + "oa_code": ["E00000001", "E00000002"], + "lsoa_code": ["E01000001", "E01000002"], + "msoa_code": ["E02000001", "E02000002"], + # April 2023 roster codes: the ladder writes the engine's + # ``local_authority`` member name beside the code (microcosm#953). + "local_authority_code": ["E09000001", "E07000008"], + "ward_code": ["E05000001", "E05000002"], + "constituency_code": ["E14000001", "E14000002"], + "region_code": ["E12000007", "E12000008"], + "itl3_code": ["TLI31", "TLJ31"], + "itl2_code": ["TLI3", "TLJ3"], + "itl1_code": ["TLI", "TLJ"], + } + ) + household["local_authority"] = resolve_local_authority_engine_keys( + household["local_authority_code"] + ) + for column in household.select_dtypes(include=["str", "object"]).columns: + household[column] = household[column].astype("string") + return uk_national_frame( + person=pd.DataFrame( + { + "person_id": [1, 2], + "person_household_id": [1, 2], + "person_benunit_id": [1, 2], + "person_clone_index": [0, 0], + "income": pd.Series([10.5, 20.5], dtype="float32"), + } + ), + benunit=pd.DataFrame({"benunit_id": [1, 2], "benunit_clone_index": [0, 0]}), + household=household, + time_period="2024", + weight_kind=WeightKind.CALIBRATED, + household_weights=np.array([13.0, 87.0]), + mass_log=(MassChangeRecord("household", 100.0, 100.0, 1.0, "calibration"),), + ) + + +__all__ = [name for name in globals() if not name.startswith("__")] diff --git a/test_support/microcosm_build/uk_hierarchy_fixtures.py b/test_support/microcosm_build/uk_hierarchy_fixtures.py new file mode 100644 index 000000000..44f253e7f --- /dev/null +++ b/test_support/microcosm_build/uk_hierarchy_fixtures.py @@ -0,0 +1,39 @@ +"""Invented schema-8 calibration hierarchies for UK test registries. + +#855 requires a ``CalibrationHierarchy`` on every registry-backed target before +diagnostics can be published. Tests that invent ``TargetSpec`` rows use this +one builder instead of repeating the nested constructors; the shape follows the +``_uc_hierarchy`` fixture that #855 added to the retired national-driver tests. +""" + +from __future__ import annotations + +from microcosm.calibrate import ( + CalibrationHierarchy, + HierarchyCategory, + HierarchyGeography, + HierarchyNode, +) + + +def uk_fixture_hierarchy( + name: str, + *, + level: str, + geography_id: str, + provider_id: str = "ons", + provider_label: str = "Office for National Statistics", + category_id: str = "ons.households", + category_label: str = "Households", +) -> CalibrationHierarchy: + """A complete hierarchy whose target id equals the spec ``name``.""" + + return CalibrationHierarchy( + provider=HierarchyNode(id=provider_id, label=provider_label), + category=HierarchyCategory( + id=category_id, label=category_label, provider_id=provider_id + ), + geography=HierarchyGeography(id=geography_id, label=geography_id, level=level), + dimensions=(), + target=HierarchyNode(id=name, label=name), + ) diff --git a/test_support/microcosm_build/uk_local_rowwise.py b/test_support/microcosm_build/uk_local_rowwise.py new file mode 100644 index 000000000..82abff6bd --- /dev/null +++ b/test_support/microcosm_build/uk_local_rowwise.py @@ -0,0 +1,38 @@ +"""The three-household importance-weighted clone frame and its area +assignment, shared by the rowwise local solve tests, the full-graph +calibration tests and the terminal-node tests.""" + +from __future__ import annotations + +import pandas as pd + +from microcosm.build.uk_runtime import uk_national_frame +from microcosm.frame import WeightKind + + +def _clone_frame(weights=(1.0, 1.0, 1.0)): + return uk_national_frame( + person=pd.DataFrame( + { + "person_id": [1, 2, 3], + "person_household_id": [101, 102, 103], + "person_benunit_id": [11, 12, 13], + } + ), + benunit=pd.DataFrame({"benunit_id": [11, 12, 13]}), + household=pd.DataFrame( + { + "household_id": [101, 102, 103], + "household_weight": list(weights), + } + ), + time_period="2023", + weight_kind=WeightKind.IMPORTANCE, + ) + + +def _assigned() -> pd.Series: + return pd.Series(["E001", "E001", "S001"], index=[101, 102, 103]) + + +__all__ = [name for name in globals() if not name.startswith("__")] diff --git a/tools/build_uk_full.py b/tools/build_uk_full.py new file mode 100644 index 000000000..8d60eee8a --- /dev/null +++ b/tools/build_uk_full.py @@ -0,0 +1,6 @@ +"""Canonical UK full build: all applicable geographies calibrated together.""" + +from microcosm.build.uk_runtime.full_build_cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/uv.lock b/uv.lock index fd7044fde..37d480654 100644 --- a/uv.lock +++ b/uv.lock @@ -761,6 +761,7 @@ dependencies = [ [package.optional-dependencies] uk = [ { name = "h5py" }, + { name = "microcosm-data" }, { name = "openpyxl" }, { name = "policyengine-uk" }, { name = "tables" }, @@ -785,6 +786,7 @@ requires-dist = [ { name = "huggingface-hub", specifier = ">=0.20" }, { name = "jsonschema", specifier = ">=4.23,<5" }, { name = "microcosm-calibrate", editable = "packages/microcosm-calibrate" }, + { name = "microcosm-data", marker = "extra == 'uk'", editable = "packages/microcosm-data" }, { name = "microcosm-data", marker = "extra == 'us'", editable = "packages/microcosm-data" }, { name = "microcosm-diagnostics", editable = "packages/microcosm-diagnostics" }, { name = "microcosm-fit", editable = "packages/microcosm-fit" }, From 430187fbcb858cfff10707b8cd288bc7f323020a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 25 Sep 2026 20:03:13 +0100 Subject: [PATCH 22/44] Serve the dense release role through the graph driver with main's posture, refusals, manifest, Logbook and staging --release-role reaches microcosm-build-uk. The driver's role machinery is moved, not copied, out of tools/build_uk_rowwise_candidate.py into two package modules that both drivers import: uk_runtime/rowwise_cli (role-defaulted arguments and their resolution, the validation and the two refusal tables, the candidate build-id and Logbook recording, the parameters and output-path helpers, the release verdict helpers and the output filename constants) and uk_runtime/rowwise_staging (staging telemetry creation, stage events, delivery, the staged-dataset publication, the Hub seams, the thinned epoch rows and the fit summary). The size-checkpoint identity, which already carries the release role and the posture's doctrine bounds, moves to uk_runtime/size_checkpoint. The tool shrinks by 1,277 lines and keeps working; its tests read the moved names through the tool module. The graph driver now: requires --release-role and resolves the seven posture-defaulted values (epochs, learning rate, seed, clones, weight rule, constituency vintage, selection); validates with main's role-aware table (--release-candidate included; the --input-sha256 requirement applies only to --input-h5 builds); names its outputs from the posture (microcosm_uk_2024_25_local.h5 and siblings, the gate document as .local_gates.json); writes rowwise_candidate_manifest.json in main's schema-4 shape, projected from the stored artifacts by graph_terminal .rowwise_candidate_manifest_from_graph (new UK code: main's writer reads live objects), which the dense preflight and assembler accept; records the uk-local-candidate Logbook row with --logbook-prev-row-digest; runs staging telemetry around each graph phase and stages the dataset; and gains --baseline-pi-floor, --no-size-checkpoint, --candidate-clone-counts (dry-run inventory per K), --households-only (a target-selection family filter). Per-epoch calibration_progress rows come from the dense solve only; the manifest says so. --release-role national is validated and refused with an exit-2 message until the next commit dispatches it. Tests: the role and pure-CLI tests in test_uk_rowwise_candidate run against both drivers (79 cases); test_uk_full_build_cli covers posture defaults, refusals, stems, the manifest projection, blocked manifests and Logbook rows; the dense preflight and assembler each gain a case fed a graph-built bundle. Verified: whole uk group 2,654 passed / 20 skipped; role files 214 passed on spot-check; ci_test_groups --verify ok; ruff clean; both drivers' --help exit 0. Co-Authored-By: Claude Fable 5.1 --- .../build/uk_runtime/full_build_cli.py | 902 +++++++++-- .../microcosm/build/uk_runtime/graph_build.py | 22 +- .../build/uk_runtime/graph_calibration.py | 15 +- .../build/uk_runtime/graph_targets.py | 28 +- .../build/uk_runtime/graph_terminal.py | 469 ++++++ .../microcosm/build/uk_runtime/rowwise_cli.py | 796 ++++++++++ .../build/uk_runtime/rowwise_staging.py | 717 +++++++++ .../build/uk_runtime/size_checkpoint.py | 50 + .../uk/test_uk_dense_release_assembler.py | 121 ++ .../engine_free/uk/test_uk_full_build_cli.py | 626 ++++---- .../uk/test_uk_full_build_preparation.py | 2 + .../uk/test_uk_local_release_preflight.py | 53 + .../uk/test_uk_rowwise_candidate.py | 223 ++- .../uk/test_uk_rowwise_national_role.py | 6 +- .../microcosm_build/uk_full_build_cli.py | 712 +++++++++ .../microcosm_build/uk_rowwise_candidate.py | 21 +- tools/build_uk_rowwise_candidate.py | 1340 +---------------- 17 files changed, 4336 insertions(+), 1767 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py create mode 100644 test_support/microcosm_build/uk_full_build_cli.py diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py index a5f210b19..6cdd5152c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py @@ -3,6 +3,27 @@ Numerical operations and verdicts belong to the composed graph. This module resolves requests, executes graph endpoints and atomically materializes their stored artifacts. Publication and signing remain explicit external services. + +``--release-role`` declares which UK dataset line the run builds and is +required (microcosm#823): ``dense`` is the K-clone joint national + local +surface under the local doctrine, built here through the graph; ``national`` +is parsed and validated by the same posture-aware validator but is served by +the retained calibration seam (``tools/build_uk_rowwise_candidate.py``) until +the graph dispatch lands in the next commit. The role supplies every unset +solve default and refuses the other role's flags through +:mod:`microcosm.build.uk_runtime.rowwise_cli`, so the graph driver and the +rowwise tool parse, default and refuse identically. + +A non-dry dense run is wrapped in the rowwise tool's operational envelope: +the Logbook attempt (a spooled row under ``/logbook-spool`` on every +terminal outcome, an error receipt on failure), the version 2 staging +telemetry with stage events around each graph phase and per-epoch rows from +the dense solve, and the staged-dataset delivery of the published bundle. +The bundle carries ``rowwise_candidate_manifest.json`` projected from the +graph's stored artifacts +(:func:`~microcosm.build.uk_runtime.graph_terminal.rowwise_candidate_manifest_from_graph`), +so the dense release pre-flight and assembler read a graph build as they +read a rowwise-tool build. """ from __future__ import annotations @@ -12,11 +33,14 @@ import json import sys import tempfile +import time import uuid from dataclasses import asdict, dataclass, replace -from datetime import date +from datetime import UTC, date, datetime from pathlib import Path +import numpy as np + from microcosm.build.artifact_files import file_artifact, materialize_bytes from microcosm.graph import ( ArtifactInput, @@ -30,6 +54,24 @@ ) from microcosm.graph.canonical import canonical_json +from ..logbook_adoption import ( + AttemptState, + append_phase, + git_code_pin, + local_artifact_reference, + preflight_digest, + resolve_predecessor, + role_pins_digest, + sha256_argument, +) +from ..staging_cli import ( + add_staged_dataset_arguments, + add_staging_arguments, + validate_staged_dataset_arguments, + validate_staging_arguments, +) +from ..staging_dataset import SHA256SUMS_FILENAME, refresh_sha256sums_entry +from .calibration_run import runtime_provenance from .chronicle_feed import load_uk_chronicle_feed from .frs_release import load_uk_frs_release from .full_certification import ( @@ -60,16 +102,61 @@ materialize_uk_terminal_artifacts, register_uk_full_gate_kernels, register_uk_terminal_kernels, -) -from .local_doctrine import ( - UK_LOCAL_CLONE_COUNT, - UK_LOCAL_MAX_WEIGHT_RATIO, - UK_LOCAL_SOLVE_DOCTRINE, - UK_LOCAL_SOLVE_EPOCHS, - UK_LOCAL_TARGET_LOSS_CAP, + rowwise_candidate_manifest_from_graph, ) from .national_frame import load_uk_national_frame -from .national_sampling import UK_SAMPLE_SEED_DEFAULT +from .national_sampling import UK_SAMPLE_RUNG_TOKENS +from .rowwise_cli import ( + MANIFEST_FILENAME, + REPOSITORY, + candidate_clone_counts_argument, + candidate_identity_digest, + git_commit, + git_dirty, + json_text, + new_candidate_build_id, + posture_of, + record_candidate_attempt, + record_candidate_error, + refuse_national_role_arguments, + resolve_role_arguments, + validate_cli_args, +) +from .rowwise_posture import ( + UK_ROWWISE_DENSE_POSTURE, + UK_ROWWISE_RELEASE_ROLES, + uk_rowwise_posture, +) +from .rowwise_staging import ( + STAGED_DATASET_PHASES, + STAGING_UPLOAD_INTERVAL_SECONDS, + create_staging_telemetry, + fail_staging_telemetry, + finalize_staging_telemetry, + gate_statuses, + preflight_staged_dataset, + replace_manifest, + stage, + stage_dataset, + staging_delivery, + staging_epoch_every, + thinned_epochs, +) +from .size_checkpoint import uk_size_checkpoint_identity +from .staging import UK_STAGED_DATASET_REPOSITORY, UK_STAGING_REPOSITORY + +# The names the rowwise tool's tests call on either driver. +_validate_cli_args = validate_cli_args +_refuse_national_role_arguments = refuse_national_role_arguments +__all__ = [ + "UK_ROWWISE_DENSE_POSTURE", + "PreparedUKFullBuild", + "execute_full_build", + "main", + "parse_args", + "prepare_full_build", + "uk_rowwise_posture", +] def _target_geographies(value: str) -> tuple[str, ...] | None: @@ -88,7 +175,28 @@ def _target_geographies(value: str) -> tuple[str, ...] | None: def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse the request and bind the declared role's defaults. + + Value checks that need the whole namespace (the refusal tables, the + size-selection rules, the mandatory Ledger pins) run in + :func:`~microcosm.build.uk_runtime.rowwise_cli.validate_cli_args` from + :func:`main`, as on the rowwise tool, so the role tests can parse and + validate in two steps. + """ parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--release-role", + choices=UK_ROWWISE_RELEASE_ROLES, + required=True, + help=( + "Which UK dataset line this run builds: 'dense' (the K-clone joint " + "national + local surface under the local doctrine, built through " + "the graph) or 'national' (validated here; served by the retained " + "calibration seam until the graph dispatch lands). The role " + "supplies every unset solve default and refuses the other role's " + "flags." + ), + ) population = parser.add_mutually_exclusive_group(required=True) population.add_argument( "--input-h5", @@ -102,17 +210,24 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: ) parser.add_argument("--input-sidecar", type=Path) parser.add_argument("--input-spine-gates", type=Path) - parser.add_argument("--input-sha256") - parser.add_argument("--ladder", type=Path, required=True) - parser.add_argument("--ladder-sha256") + parser.add_argument( + "--input-sha256", + type=sha256_argument, + help="Pinned SHA-256 of --input-h5 (required with --input-h5).", + ) + parser.add_argument( + "--ladder", + type=Path, + help="Full-UK OA geography ladder NPZ (required by the dense role).", + ) + parser.add_argument("--ladder-sha256", type=sha256_argument) parser.add_argument( "--ledger-facts", type=Path, - required=True, help="Complete Chronicle artifact directory matching the committed feed pins.", ) - parser.add_argument("--ledger-facts-sha256") - parser.add_argument("--ledger-manifest-sha256") + parser.add_argument("--ledger-facts-sha256", type=sha256_argument) + parser.add_argument("--ledger-manifest-sha256", type=sha256_argument) parser.add_argument("--measure-exclusions", type=Path) parser.add_argument("--register-json", type=Path) parser.add_argument("--input-mass-reference", type=Path) @@ -133,11 +248,20 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: metavar="all|country,...", help="Default all: calibrate all applicable geographies together. country is an explicit filter in this same build.", ) + parser.add_argument( + "--households-only", + action="store_true", + help="Bind only Chronicle census-household constituency targets.", + ) parser.add_argument( "--n-clones", type=int, - default=UK_LOCAL_CLONE_COUNT, - help="Geographic pool copies K, independent of target scope and exported size k.", + help="Geographic pool copies K; defaults to the role's doctrine clone count.", + ) + parser.add_argument( + "--candidate-clone-counts", + type=candidate_clone_counts_argument, + help="Dry-run only comma-separated candidate clone counts.", ) parser.add_argument( "--dataset-households", @@ -150,22 +274,50 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: default=1.0, help="Optional pool sampling before cloning. Cannot resample an already sampled source spine.", ) - parser.add_argument("--sample-seed", type=int, default=UK_SAMPLE_SEED_DEFAULT) - parser.add_argument("--seed", type=int, default=42) + parser.add_argument( + "--sample-seed", type=int, help="Dense role only; defaults to the spine seed." + ) + parser.add_argument("--seed", type=int, help="Defaults to the role's seed.") parser.add_argument("--selection-seed", type=int) parser.add_argument("--selection-pi-hi", type=float, default=1.0) - parser.add_argument("--epochs", type=int, default=UK_LOCAL_SOLVE_EPOCHS) - parser.add_argument("--learning-rate", type=float, default=0.15) + parser.add_argument( + "--baseline-pi-floor", + type=float, + default=0.0, + help=( + "Floor on the inclusion probability the refit's Horvitz-Thompson " + "baseline divides each selected row's dense weight by (microcosm#355). " + "0 (default) is the untrimmed baseline. Requires --dataset-households." + ), + ) + parser.add_argument( + "--no-size-checkpoint", + action="store_true", + help=( + "Do not materialize the dense solve and the informed L0 search as " + "size_selection_checkpoint.{npz,json} in --out after a " + "--dataset-households run." + ), + ) + parser.add_argument( + "--epochs", type=int, help="Defaults to the role's doctrine solve length." + ) + parser.add_argument( + "--learning-rate", type=float, help="Defaults to the role's learning rate." + ) parser.add_argument( "--target-weight-rule", - choices=("uniform", "grain_equal"), - default=UK_LOCAL_SOLVE_DOCTRINE.target_weight_rule, + choices=("uniform", "grain_equal", "family_equal"), + help="Defaults to the role's doctrine; any other admitted rule is a receipted override.", ) parser.add_argument("--engine-blocks", type=int, default=1) parser.add_argument("--source-year", type=int) parser.add_argument("--calibration-year", type=int) parser.add_argument("--source-lineage-modulus", type=int) - parser.add_argument("--expected-constituency-vintage", default="2024_pcon") + parser.add_argument( + "--expected-constituency-vintage", + help="Dense role only: constituency vintage required from the ladder.", + ) parser.add_argument("--skip-holdout", action="store_true") parser.add_argument("--release-candidate", action="store_true") parser.add_argument("--review-date", type=date.fromisoformat, default=date.today()) @@ -174,6 +326,41 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: type=Path, help="Import an identity-verified historical size search, skipping its dense solve and search.", ) + # The national role's own knobs are declared so the dense refusal table + # can name them; the national role itself is served by the calibration + # seam until its graph dispatch lands. + parser.add_argument( + "--target-loss-cap", + type=float, + help="National role only: receipted override of the seam doctrine's per-target loss cap.", + ) + parser.add_argument( + "--allow-unpinned-feed", + action="store_true", + help="National role only: allow a Ledger artifact whose feed commit is not the committed Chronicle pin.", + ) + parser.add_argument( + "--incumbent-h5", + type=Path, + help="National role only: the incumbent dataset the finished candidate is evaluated against.", + ) + parser.add_argument( + "--incumbent-sha256", + help="National role only: the incumbent's SHA-256, verified before it is read.", + ) + parser.add_argument( + "--incumbent-label", + default="enhanced_frs_2024_25", + help="National role only: the incumbent's label in the score receipt.", + ) + parser.add_argument( + "--logbook-prev-row-digest", + type=sha256_argument, + help=( + "Optional current Logbook chain head. If omitted, " + "POPULACE_LOGBOOK_PREV_ROW_DIGEST is used, then genesis null." + ), + ) parser.add_argument( "--graph-store", type=Path, @@ -188,49 +375,32 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: action="store_true", help="Validate the request and print the compiled operation inventory without fitting or writing files.", ) + add_staging_arguments( + parser, + repository=UK_STAGING_REPOSITORY, + default_upload_interval_seconds=STAGING_UPLOAD_INTERVAL_SECONDS, + ) + add_staged_dataset_arguments(parser, repository=UK_STAGED_DATASET_REPOSITORY) args = parser.parse_args(argv) + validate_staging_arguments(parser, args) + validate_staged_dataset_arguments(parser, args) if args.input_h5 is None and any( (args.input_sidecar, args.input_spine_gates, args.input_sha256) ): parser.error("Input H5 sidecar/pin options require --input-h5.") - if args.dataset_households is None and ( - args.selection_seed is not None - or args.selection_pi_hi != 1.0 - or args.resume_size_checkpoint - ): - parser.error("Selection options require --dataset-households.") if args.matched_size_scorecard is not None and args.dataset_households is None: parser.error("A matched-size scorecard requires --dataset-households.") - if args.release_candidate and args.skip_holdout: - parser.error("A release candidate must evaluate applicable holdouts.") - if args.release_candidate: - if args.dataset_households is not None: - parser.error( - "Exact-count builds need separate matched-size evidence before release promotion." - ) - if ( - args.epochs != UK_LOCAL_SOLVE_EPOCHS - or args.n_clones != UK_LOCAL_CLONE_COUNT - or args.target_weight_rule != UK_LOCAL_SOLVE_DOCTRINE.target_weight_rule - or args.engine_blocks != 1 - or args.measure_exclusions is not None - ): - parser.error( - "A release candidate must use the maintained solve doctrine, pool count, single engine and reviewed exclusions." - ) - if args.ladder_sha256 is None or ( - args.input_h5 is not None and args.input_sha256 is None - ): - parser.error( - "A release candidate requires explicit input H5 and ladder digest pins." - ) if args.resume_size_checkpoint and args.input_h5 is None: parser.error( "Historical size checkpoints bind an input H5; raw builds resume using --graph-store." ) + resolve_role_arguments(args) return args +_parse_args = parse_args + + def _pin(path: Path, expected: str | None = None) -> dict: record = file_artifact(path) if expected is not None and expected != record["sha256"]: @@ -238,42 +408,13 @@ def _pin(path: Path, expected: str | None = None) -> dict: return {"sha256": record["sha256"], "size_bytes": record["size_bytes"]} -def _checkpoint_identity(args, config, pins) -> dict | None: - if args.resume_size_checkpoint is None: - return None - # Match the existing checkpoint schema exactly. The import kernel also - # verifies ordered targets, weights, household axis and recomputed losses. +def _input_record(path: Path, pin: dict, *, pinned: bool) -> dict: + """The rowwise manifest's input record: path, digest, bytes and the pin flag.""" return { - "dataset_pin": pins["dataset"], - "ladder_pin": pins["ladder"], - "ledger_facts_sha256": args.ledger_facts_sha256, - "ledger_manifest_sha256": args.ledger_manifest_sha256, - "seed": args.seed, - "selection_seed": args.seed - if args.selection_seed is None - else args.selection_seed, - "n_clones": args.n_clones, - "dataset_households": args.dataset_households, - "epochs": args.epochs, - "learning_rate": args.learning_rate, - "sample_fraction": args.sample_fraction, - "sample_seed": args.sample_seed, - "source_year": config.source_year, - "source_lineage_modulus": args.source_lineage_modulus, - "calibration_year": config.calibration_year, - "target_weight_rule": args.target_weight_rule, - "engine_blocks": args.engine_blocks, - "measure_exclusions": None - if args.measure_exclusions is None - else str(args.measure_exclusions), - "doctrine": { - "target_loss_cap": float(UK_LOCAL_TARGET_LOSS_CAP), - "max_weight_ratio": float(UK_LOCAL_MAX_WEIGHT_RATIO), - "scale_rule": UK_LOCAL_SOLVE_DOCTRINE.scale_rule, - "target_weight_rule": UK_LOCAL_SOLVE_DOCTRINE.target_weight_rule, - "solve_epochs": int(UK_LOCAL_SOLVE_EPOCHS), - "clone_count": int(UK_LOCAL_CLONE_COUNT), - }, + "path": str(path.resolve()), + "sha256": str(pin["sha256"]), + "bytes": int(pin["size_bytes"]), + "pin_verified": bool(pinned), } @@ -285,9 +426,37 @@ class PreparedUKFullBuild: bindings: dict spine_provenance: ArtifactInput | None = None comparison_sources: dict[str, Path] | None = None + pins: dict | None = None + inputs: dict | None = None + + +def _stderr_progress(line: str) -> None: + """Solver progress (epoch losses), as the rowwise tool prints them.""" + print(line, file=sys.stderr, flush=True) + + +def _solve_observer(args: argparse.Namespace, telemetry): + """Readable stderr lines plus the thinned staging rows of the dense solve.""" + from .solve_progress import uk_solve_progress_callback + + sinks = [uk_solve_progress_callback(_stderr_progress)] + if telemetry is not None: + sinks.append( + thinned_epochs( + telemetry.calibration_progress, every=staging_epoch_every(args) + ) + ) + + def observer(event: dict[str, object]) -> None: + for sink in sinks: + sink(event) + return observer -def prepare_full_build(args: argparse.Namespace) -> PreparedUKFullBuild: + +def prepare_full_build( + args: argparse.Namespace, *, telemetry=None, attempt: dict | None = None +) -> PreparedUKFullBuild: from .calibration_run import load_bound_spine_checkpoint from .graph import uk_spine_endpoint from .spine_build import ( @@ -297,12 +466,21 @@ def prepare_full_build(args: argparse.Namespace) -> PreparedUKFullBuild: prepare_uk_spine_execution, ) + posture = posture_of(args) release = load_uk_frs_release() pins = {"ladder": _pin(args.ladder, args.ladder_sha256)} + inputs = { + "ladder": _input_record( + args.ladder, pins["ladder"], pinned=args.ladder_sha256 is not None + ) + } sources = {"uk_ladder": args.ladder, "uk_ledger_facts": args.ledger_facts} provenance = None if args.input_h5 is not None: pins["dataset"] = _pin(args.input_h5, args.input_sha256) + inputs["dataset"] = _input_record( + args.input_h5, pins["dataset"], pinned=args.input_sha256 is not None + ) frame, _ = load_uk_national_frame(args.input_h5) sidecar_path = args.input_sidecar or args.input_h5.with_suffix(".build.json") gates_path = args.input_spine_gates or args.input_h5.with_suffix( @@ -356,6 +534,20 @@ def prepare_full_build(args: argparse.Namespace) -> PreparedUKFullBuild: source_fraction = raw.sample_fraction stages = prepared.stage_names engine, engine_identity = prepared.engine, prepared.engine_identity + inputs["dataset"] = { + "path": None, + "sha256": None, + "bytes": None, + "pin_verified": False, + "spine_request": str(args.spine_request.resolve()), + } + stage( + telemetry, + "input_pinning", + "completed", + dataset_sha256=inputs["dataset"]["sha256"], + ladder_sha256=pins["ladder"]["sha256"], + ) config = UKFullBuildConfig( calibration_year=args.calibration_year or release.calibration_year, time_period=time_period, @@ -363,6 +555,9 @@ def prepare_full_build(args: argparse.Namespace) -> PreparedUKFullBuild: if args.source_year is not None else int(time_period), geography_levels=args.target_geographies, + target_families=("census_households/constituency",) + if args.households_only + else None, n_clones=args.n_clones, sample_fraction=args.sample_fraction, source_sample_fraction=source_fraction, @@ -378,9 +573,20 @@ def prepare_full_build(args: argparse.Namespace) -> PreparedUKFullBuild: dataset_households=args.dataset_households, selection_seed=args.selection_seed, selection_pi_hi=args.selection_pi_hi, + baseline_pi_floor=args.baseline_pi_floor, target_weight_rule=args.target_weight_rule, ), ) + args._calibration_year = int(config.calibration_year) + if attempt is not None: + state = attempt["state"] + attempt["code_pin"] = git_code_pin(REPOSITORY) + state.input_pins_digest = role_pins_digest(pins) + append_phase(state, "configured") + append_phase(state, "inputs_pinned") + state.identity_digest = candidate_identity_digest( + pins=pins, args=args, source_year=config.source_year + ) if args.release_candidate and config.effective_sample_fraction != 1.0: raise ValueError( "Sampled builds cannot request release-candidate certification." @@ -411,6 +617,13 @@ def prepare_full_build(args: argparse.Namespace) -> PreparedUKFullBuild: SourceRef("uk_input_mass_reference", "raw-bytes-v1"), ), ) + checkpoint_identity = ( + None + if args.resume_size_checkpoint is None + else uk_size_checkpoint_identity( + args, pins=pins, source_year=config.source_year + ) + ) full = uk_full_graph( config, spine=spine, @@ -418,7 +631,7 @@ def prepare_full_build(args: argparse.Namespace) -> PreparedUKFullBuild: spine_weight_kind=weight_kind, optional_target_sources=tuple(optional), review_date=args.review_date.isoformat(), - checkpoint_identity=_checkpoint_identity(args, config, pins), + checkpoint_identity=checkpoint_identity, ) if args.resume_size_checkpoint: from .size_checkpoint import ( @@ -445,6 +658,7 @@ def prepare_full_build(args: argparse.Namespace) -> PreparedUKFullBuild: ) bindings = { "schema": "microcosm.uk.full-build-request.v1", + "release_role": posture.role, "configuration": asdict(config), "target_scope": "all" if config.geography_levels is None @@ -474,7 +688,10 @@ def prepare_full_build(args: argparse.Namespace) -> PreparedUKFullBuild: ), ), ) - register_uk_full_kernels(kernels) + register_uk_full_kernels( + kernels, + progress_callback=None if args.dry_run else _solve_observer(args, telemetry), + ) register_uk_full_gate_kernels( kernels, coverage_engine=engine, engine_identity=engine_identity ) @@ -491,7 +708,7 @@ def prepare_full_build(args: argparse.Namespace) -> PreparedUKFullBuild: if path is not None } return PreparedUKFullBuild( - full, kernels, sources, bindings, provenance, comparisons + full, kernels, sources, bindings, provenance, comparisons, pins, inputs ) @@ -556,10 +773,38 @@ def _persist_checkpoint(manifest, store, args, phase: str) -> None: ) -def _output_locations(prepared: PreparedUKFullBuild, args: argparse.Namespace): +def _argument_sources(args: argparse.Namespace) -> dict[str, Path]: + """The path-valued request arguments, for the output-location check + before (or without) a prepared build.""" + return { + name: path + for name, path in ( + ("input_h5", args.input_h5), + ("spine_request", args.spine_request), + ("input_sidecar", args.input_sidecar), + ("input_spine_gates", args.input_spine_gates), + ("ladder", args.ladder), + ("ledger_facts", args.ledger_facts), + ("measure_exclusions", args.measure_exclusions), + ("register_json", args.register_json), + ("input_mass_reference", args.input_mass_reference), + ("native_scorecard", args.native_scorecard), + ("matched_size_scorecard", args.matched_size_scorecard), + ("resume_size_checkpoint", args.resume_size_checkpoint), + ) + if path is not None + } + + +def _output_locations(prepared: PreparedUKFullBuild | None, args: argparse.Namespace): output = args.out.resolve() graph_store = (args.graph_store or output / ".graph-store").resolve() - for source in {**prepared.sources, **(prepared.comparison_sources or {})}.values(): + sources = ( + _argument_sources(args) + if prepared is None + else {**prepared.sources, **(prepared.comparison_sources or {})} + ) + for source in sources.values(): source = source.resolve() if source.is_relative_to(output): raise ValueError( @@ -574,8 +819,28 @@ def _output_locations(prepared: PreparedUKFullBuild, args: argparse.Namespace): return output, graph_store -def execute_full_build(prepared: PreparedUKFullBuild, args: argparse.Namespace) -> int: - """Stage complete files, then publish their completion marker last.""" +def _release_stem(posture) -> tuple[str, str]: + """The role's dataset stem and gate-report filename for the FRS vintage.""" + vintage = load_uk_frs_release().vintage + return ( + posture.dataset_filename(vintage).removesuffix(".h5"), + posture.gate_report_filename(vintage), + ) + + +def execute_full_build( + prepared: PreparedUKFullBuild, + args: argparse.Namespace, + *, + telemetry=None, + attempt: dict | None = None, +) -> int: + """Stage complete files, then publish their completion marker last. + + With an ``attempt`` (a non-dry run from :func:`main`) the published + bundle is then staged as a dataset, the staging telemetry is closed, the + rowwise manifest gains both receipts and the Logbook row is spooled. + """ if args.dry_run: return _execute_full_build(prepared, args) from microcosm.build.artifact_files import publish_staged_bundle @@ -584,13 +849,17 @@ def execute_full_build(prepared: PreparedUKFullBuild, args: argparse.Namespace) output.parent.mkdir(parents=True, exist_ok=True) # Operational attempt identity never enters a scientific node/cache key. args.attempt_evidence = graph_store / "uk-full-attempts" / uuid.uuid4().hex + record: dict = {} with tempfile.TemporaryDirectory( prefix=f".{output.name}.full-build-", dir=output.parent ) as temporary: staged_args = argparse.Namespace(**vars(args)) staged_args.out = Path(temporary) staged_args.graph_store = graph_store - status = _execute_full_build(prepared, staged_args) + staged_args.published_out = output + status = _execute_full_build( + prepared, staged_args, telemetry=telemetry, attempt=attempt, record=record + ) if not (staged_args.out / "build.json").exists(): materialize_bytes( canonical_json( @@ -611,19 +880,189 @@ def execute_full_build(prepared: PreparedUKFullBuild, args: argparse.Namespace) } destinations = {role: output / path.name for role, path in staged.items()} publish_staged_bundle(staged, destinations, completion_role="manifest") + stem, _ = _release_stem(posture_of(args)) if status == 0: print( - f"UK full build: {output / f'microcosm_uk_{prepared.full.config.calibration_year}.h5'}; " - f"target scope {prepared.bindings['target_scope']}." + f"UK full build: {output / f'{stem}.h5'}; " + f"target scope {prepared.bindings['target_scope']}.", + file=sys.stderr, + ) + if attempt is not None: + _close_attempt( + args, + attempt, + output=output, + stem=stem, + status=status, + record=record, + telemetry=telemetry, ) return status -def _execute_full_build(prepared: PreparedUKFullBuild, args: argparse.Namespace) -> int: +def _close_attempt( + args: argparse.Namespace, + attempt: dict, + *, + output: Path, + stem: str, + status: int, + record: dict, + telemetry, +) -> None: + """Stage the published bundle, close the telemetry, spool the Logbook row.""" + state: AttemptState = attempt["state"] + manifest = record.get("manifest") + blocked = bool(record.get("blocking_failures")) + if manifest is not None: + append_phase(state, "published") + args._gate_report = {"gates": record.get("gate_rows", {})} + staged_dataset = stage_dataset( + args, + manifest=manifest, + output_paths={"manifest": output / MANIFEST_FILENAME}, + run_id=state.build_id if telemetry is None else telemetry.run_id, + telemetry=telemetry, + ) + append_phase(state, STAGED_DATASET_PHASES[staged_dataset["status"]]) + try: + finalize_staging_telemetry(args, telemetry) + finally: + manifest["staging_delivery"] = staging_delivery(telemetry) + manifest["staged_dataset"] = staged_dataset + replace_manifest(output / MANIFEST_FILENAME, manifest) + if (output / SHA256SUMS_FILENAME).is_file(): + refresh_sha256sums_entry(output, MANIFEST_FILENAME) + state.artifact_location = local_artifact_reference( + output / f"{stem}.h5", repository_hint=REPOSITORY + ) + else: + finalize_staging_telemetry(args, telemetry) + spool_path = record_candidate_attempt( + state=state, + started_at=attempt["started_at"], + started_ts=attempt["started_ts"], + seed=args.seed, + code_pin=str(attempt["code_pin"]), + disposition="failed" if (blocked or status != 0) else "iterating", + predecessor=attempt["predecessor"], + spool_dir=output / "logbook-spool", + rung=UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], + ) + print(f"Wrote Logbook row: {spool_path}", file=sys.stderr) + if manifest is not None: + print(json_text(manifest), end="") + if blocked: + print( + "Gate battery blocked the artifact at f100; evidence bundle " + f"written, artifact unreleasable: {record['blocking_failures'][:5]}", + file=sys.stderr, + ) + + +def _apply_graph_gate_verdicts( + state: AttemptState, gate_rows: dict, report_path: Path +) -> None: + receipt = local_artifact_reference(report_path, repository_hint=REPOSITORY) + state.gate_verdicts = { + str(gate_id): { + "verdict": str(payload["status"]), + "receipt": f"{receipt}#/gates/{gate_id}", + } + for gate_id, payload in gate_rows.items() + } + + +def _materialize_size_checkpoint( + manifest, store, *, args: argparse.Namespace, pins: dict, source_year: int +) -> dict | None: + """Write ``size_selection_checkpoint.{npz,json}`` from the stored solve. + + The dense result and the informed L0 search are graph artifacts; the + checkpoint the rowwise tool writes before the exact-count draw is + materialized from them with the same identity a resume presents through + ``--resume-size-checkpoint``. A full-pool search (k equal to the pool) + carries no gate probabilities and writes no checkpoint. + """ + from microcosm.calibrate.artifacts import ( + decode_calibration_result, + decode_problem, + ) + from microcosm.frame import Frame + + from .dataset_size import UKSizeSelection + from .size_checkpoint import write_uk_size_checkpoint + + if "uk.full.size_search" not in manifest.nodes: + return None + selection_meta = json.loads( + _payload(manifest, store, "uk.full.size_search", "selection") + ) + if selection_meta.get("method") != "contribution_informed_l0": + return None + pool = manifest.population("uk.full.pool") + problem = decode_problem(_payload(manifest, store, "uk.full.problem", "problem")) + initial = Frame( + {entity: pool.table(entity) for entity in pool.entities}, + pool.schema, + {"household": problem.problem.initial_weights}, + pool.strata, + mass_log=pool.mass_log, + metadata=pool.metadata, + ) + dense = decode_calibration_result( + _payload(manifest, store, "uk.full.dense", "result"), + frame=initial, + problem=problem, + ) + search = decode_calibration_result( + _payload(manifest, store, "uk.full.size_search", "result"), + frame=initial, + problem=problem, + ) + selection = UKSizeSelection( + selection=search, + protected=np.asarray(selection_meta["protected"], dtype=bool), + households=int(selection_meta["households"]), + epochs=int(selection_meta["epochs"]), + learning_rate=float(selection_meta["learning_rate"]), + seed=int(selection_meta["seed"]), + search_pi_hi=float(selection_meta["pi_hi"]), + ) + return write_uk_size_checkpoint( + args.out, + frame=initial, + dense=dense, + selection=selection, + identity=uk_size_checkpoint_identity(args, pins=pins, source_year=source_year), + provenance={ + "graph_artifacts": { + "dense": manifest.nodes["uk.full.dense"].opaque_artifacts["result"], + "search": manifest.nodes["uk.full.size_search"].opaque_artifacts[ + "result" + ], + } + }, + ) + + +def _execute_full_build( + prepared: PreparedUKFullBuild, + args: argparse.Namespace, + *, + telemetry=None, + attempt: dict | None = None, + record: dict | None = None, +) -> int: full, kernels, sources = prepared.full, prepared.kernels, prepared.sources if args.dry_run: print(json.dumps(full.operation_inventory(), indent=2)) return 0 + posture = posture_of(args) + state: AttemptState | None = None if attempt is None else attempt["state"] + record = {} if record is None else record + stem, gate_report_name = _release_stem(posture) + published_root = Path(getattr(args, "published_out", args.out)) args.out.mkdir(parents=True, exist_ok=True) store = ContentStore(args.graph_store or args.out / ".graph-store") graph = full.graph @@ -655,6 +1094,7 @@ def _execute_full_build(prepared: PreparedUKFullBuild, args: argparse.Namespace) _persist_checkpoint(checkpoint, store, args, endpoint) resume = "require" if args.resume == "require" else "auto" # Persist preflight outcomes before any solver can reject them. + stage(telemetry, "target_compilation", "started") preflight_graph = _through(graph, "uk.full.gates.preflight") preflight = run_graph( compile_graph(preflight_graph), @@ -665,11 +1105,33 @@ def _execute_full_build(prepared: PreparedUKFullBuild, args: argparse.Namespace) ) _persist_checkpoint(preflight, store, args, "preflight") _materialize_evidence(preflight, store, args.out) + if "uk.full.target_selection" in preflight.nodes: + selection = json.loads( + _payload(preflight, store, "uk.full.target_selection", "selection") + ) + included = selection.get("receipt", {}).get("included", []) + stage( + telemetry, + "target_compilation", + "completed", + selected_target_count=len(included), + ) + if state is not None: + append_phase(state, "targets_bound") _, admission = decode_full_gate_report( _payload(preflight, store, "uk.full.gates.preflight", "gate_report") ) if not admission["artifact_permitted"]: return 1 + stage( + telemetry, + "calibration", + "started", + dataset_households=args.dataset_households, + epochs=int(args.epochs), + epoch_every=staging_epoch_every(args), + resumed_from_checkpoint=args.resume_size_checkpoint is not None, + ) manifest = run_graph( compile_graph(_through(graph, "uk.full.gates.calibrated")), sources=sources, @@ -680,16 +1142,72 @@ def _execute_full_build(prepared: PreparedUKFullBuild, args: argparse.Namespace) _persist_checkpoint(manifest, store, args, "numerical") _materialize_evidence(manifest, store, args.out) terminal_files = materialize_uk_terminal_artifacts( - manifest, - store, - directory=args.out, - stem=f"microcosm_uk_{full.config.calibration_year}", + manifest, store, directory=args.out, stem=stem ) - _, enforcement = decode_full_gate_report( - _payload(manifest, store, "uk.full.gates.calibrated", "gate_report") + gate_report_bytes = _payload( + manifest, store, "uk.full.gates.calibrated", "gate_report" + ) + gate_report_path = args.out / gate_report_name + terminal_files["full_gates"] = { + **materialize_bytes(gate_report_bytes, gate_report_path), + "graph_artifact_key": manifest.nodes[ + "uk.full.gates.calibrated" + ].opaque_artifacts["gate_report"], + } + if state is not None: + append_phase(state, "solved") + diagnostics = json.loads( + _payload(manifest, store, "uk.full.gates.calibrated", "calibration_diagnostics") + ) + stage( + telemetry, + "calibration", + "completed", + final_loss=diagnostics.get("final_loss"), + n_nonzero=diagnostics.get("n_nonzero"), + realized_households=diagnostics.get("n_records"), + ) + if ( + args.dataset_households is not None + and not args.no_size_checkpoint + and args.resume_size_checkpoint is None + ): + receipt = _materialize_size_checkpoint( + manifest, + store, + args=args, + pins=prepared.pins or {}, + source_year=full.config.source_year, + ) + if receipt is not None and state is not None: + append_phase(state, "size_selection_checkpointed") + _, enforcement = decode_full_gate_report(gate_report_bytes) + gate_document = json.loads(gate_report_bytes) + gate_rows = { + str(outcome["id"]): {k: v for k, v in outcome.items() if k != "id"} + for outcome in gate_document["report"]["outcomes"] + } + record["gate_rows"] = gate_rows + record["blocking_failures"] = list(enforcement["enforced_blocking"]) + if state is not None: + _apply_graph_gate_verdicts(state, gate_rows, published_root / gate_report_name) + append_phase( + state, + "candidate_blocked" + if enforcement["enforced_blocking"] + else "candidate_gated", + ) + stage( + telemetry, + "gate_battery", + "completed", + gate_statuses=gate_statuses({"gates": gate_rows}), + blocking_failure_count=len(enforcement["enforced_blocking"]), + diagnostic_failure_count=len(enforcement["diagnostic_failures"]), ) if not enforcement["artifact_permitted"]: return 1 + stage(telemetry, "output_bundle", "started") manifest = run_graph( compile_graph(graph), sources=sources, @@ -700,8 +1218,9 @@ def _execute_full_build(prepared: PreparedUKFullBuild, args: argparse.Namespace) descriptor = json.loads( _payload(manifest, store, "uk.full.export.prepare", "export_descriptor") ) - dataset = args.out / f"microcosm_uk_{full.config.calibration_year}.h5" - materialize_uk_export(manifest.population(full.population), descriptor, dataset) + dataset = args.out / f"{stem}.h5" + frame = manifest.population(full.population) + materialize_uk_export(frame, descriptor, dataset) graph = add_uk_export_continuation( graph, population=full.population, @@ -742,7 +1261,7 @@ def _execute_full_build(prepared: PreparedUKFullBuild, args: argparse.Namespace) ), ), evidence_files={ - "full_gates": "uk.full.gates.calibrated.gate_report.json", + "full_gates": gate_report_name, "diagnostics": terminal_files["calibration_diagnostics"]["filename"], "holdout": terminal_files["holdout"]["filename"], "target_diagnostics": terminal_files["target_diagnostics"]["filename"], @@ -751,8 +1270,7 @@ def _execute_full_build(prepared: PreparedUKFullBuild, args: argparse.Namespace) }, ) evidence_sources = { - "exported_evidence_full_gates": args.out - / "uk.full.gates.calibrated.gate_report.json", + "exported_evidence_full_gates": gate_report_path, "exported_evidence_diagnostics": args.out / terminal_files["calibration_diagnostics"]["filename"], "exported_evidence_holdout": args.out / terminal_files["holdout"]["filename"], @@ -795,10 +1313,67 @@ def _execute_full_build(prepared: PreparedUKFullBuild, args: argparse.Namespace) certification_file = materialize_bytes( certification_payload, args.out / "certification.json" ) + + def output_record(path: Path) -> dict: + artifact = file_artifact(path) + return { + "path": str(published_root / path.name), + "sha256": artifact["sha256"], + "bytes": int(artifact["size_bytes"]), + } + + outputs = { + "dataset": output_record(dataset), + "calibration_diagnostics": output_record( + args.out / terminal_files["calibration_diagnostics"]["filename"] + ), + "local_gate_report": output_record(gate_report_path), + "solve_diagnostics": output_record( + args.out / terminal_files["target_diagnostics"]["filename"] + ), + "area_support_summary": output_record( + args.out / terminal_files["area_support"]["filename"] + ), + "holdout": output_record(args.out / terminal_files["holdout"]["filename"]), + "target_registry": output_record( + args.out / terminal_files["target_registry"]["filename"] + ), + } + inputs = prepared.inputs or { + "dataset": {"path": None, "sha256": None, "bytes": None, "pin_verified": False}, + "ladder": {"path": None, "sha256": None, "bytes": None, "pin_verified": False}, + } + rowwise_manifest = rowwise_candidate_manifest_from_graph( + final, + store, + args=args, + posture=posture, + pins=prepared.pins or {}, + terminal_files=terminal_files, + frame=frame, + outputs=outputs, + source_year=full.config.source_year, + inputs=inputs, + ladder_provenance=json.loads( + _payload(final, store, "uk.full.target_compilation", "surface") + ).get("ladder_provenance", {}), + code={"git_commit": git_commit(), "git_dirty": git_dirty()}, + runtime=runtime_provenance(), + created_at=datetime.now(UTC).isoformat(), + ) + materialize_bytes( + json_text(rowwise_manifest).encode(), args.out / MANIFEST_FILENAME + ) + record["manifest"] = rowwise_manifest completion = { **package, "kind": "uk_full_build_completion", + "release_role": posture.role, "candidate_manifest": candidate_file, + "rowwise_candidate_manifest": { + **file_artifact(args.out / MANIFEST_FILENAME), + "note": "staging receipts are appended after publication", + }, "certification": { **certification_file, "graph_artifact_key": final.nodes["uk.full.certification"].opaque_artifacts[ @@ -807,26 +1382,99 @@ def _execute_full_build(prepared: PreparedUKFullBuild, args: argparse.Namespace) }, } materialize_bytes(canonical_json(completion), args.out / "build.json") + stage( + telemetry, + "output_bundle", + "completed", + output_bytes={key: int(entry["bytes"]) for key, entry in outputs.items()}, + ) return ( 0 if package["readback_passed"] and not enforcement["enforced_blocking"] else 1 ) +def _dry_run(args: argparse.Namespace) -> int: + """Plan without solving or writing; loop over candidate clone counts.""" + counts = args.candidate_clone_counts + if counts is None: + return execute_full_build(prepare_full_build(args), args) + inventories = [] + for count in counts: + planned = argparse.Namespace(**vars(args)) + planned.n_clones = int(count) + prepared = prepare_full_build(planned) + inventories.append( + {"n_clones": int(count), **prepared.full.operation_inventory()} + ) + print(json.dumps(inventories, indent=2)) + return 0 + + def main(argv: list[str] | None = None) -> int: args = parse_args(argv) + validate_cli_args(args) + posture = posture_of(args) + if posture.role == "national": + print( + "error: --release-role national is validated here but served by the " + "retained calibration seam (tools/build_uk_rowwise_candidate.py) " + "until the graph national dispatch lands in the next commit " + "(microcosm#901 phase 4).", + file=sys.stderr, + ) + raise SystemExit(2) + if args.candidate_clone_counts is not None and not args.dry_run: + raise ValueError("--candidate-clone-counts is valid only with --dry-run.") + if args.dry_run: + # Dry runs plan without solving or writing and record no Logbook + # row on any path, so they need no chain configuration. + return _dry_run(args) + # Argument refusals above cost nothing; the credential check reaches the + # Hub, so it runs last, still before any input is read. + preflight_staged_dataset(args) + started_at = time.perf_counter() + started_ts = datetime.now(UTC) + digest = preflight_digest(posture.pipeline) + state = AttemptState( + build_id=new_candidate_build_id( + seed=args.seed, + timestamp=started_ts, + rung=UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], + ), + identity_digest=digest, + input_pins_digest=digest, + phases_reached=["attempt_started"], + gate_verdicts={ + "pipeline": { + "verdict": "running", + "receipt": "pending-build-scoped-terminal-receipt", + } + }, + ) + # Logbook chain configuration is validated before any terminal work: a + # malformed or conflicting head refuses the run with no row and no side + # effects. + predecessor = resolve_predecessor(args.logbook_prev_row_digest) + telemetry = create_staging_telemetry(args, build_id=state.build_id) + attempt = { + "state": state, + "started_at": started_at, + "started_ts": started_ts, + "code_pin": "unresolved-local-git-code-pin", + "predecessor": predecessor, + } prepared = None try: - prepared = prepare_full_build(args) - return execute_full_build(prepared, args) + prepared = prepare_full_build(args, telemetry=telemetry, attempt=attempt) + return execute_full_build(prepared, args, telemetry=telemetry, attempt=attempt) except Exception as error: safe_output = False - if prepared is not None: - try: - _output_locations(prepared, args) - safe_output = True - except ValueError: - pass - if not args.dry_run and safe_output: + try: + _output_locations(prepared, args) + safe_output = True + except ValueError: + pass + if safe_output: materialize_bytes( canonical_json( { @@ -841,6 +1489,20 @@ def main(argv: list[str] | None = None) -> int: ), args.out / "failure.json", ) + if state.spool_path is None: + record_candidate_error( + error=error, + state=state, + started_at=started_at, + started_ts=started_ts, + seed=args.seed, + code_pin=str(attempt["code_pin"]), + predecessor=predecessor, + base_dir=args.out.resolve(), + spool_dir=args.out.resolve() / "logbook-spool", + rung=UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], + ) + fail_staging_telemetry(telemetry, error) print(f"UK full build failed: {error}", file=sys.stderr) return 1 diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py index 808582ae2..8fc0b3c0d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py @@ -65,6 +65,7 @@ class UKFullBuildConfig: time_period: str = field(default_factory=lambda: load_uk_frs_release().time_period) source_year: int = field(default_factory=lambda: load_uk_frs_release().survey_year) geography_levels: tuple[str, ...] | None = None + target_families: tuple[str, ...] | None = None n_clones: int = UK_LOCAL_CLONE_COUNT sample_fraction: float = 1.0 source_sample_fraction: float = 1.0 @@ -102,6 +103,14 @@ def __post_init__(self) -> None: ) if len(set(self.geography_levels)) != len(self.geography_levels): raise ValueError("Geographic target levels must not repeat.") + if self.target_families is not None and ( + not self.target_families + or any(not isinstance(f, str) or not f for f in self.target_families) + or len(set(self.target_families)) != len(self.target_families) + ): + raise ValueError( + "Use explicit non-repeating target families or omit the selector for all." + ) if self.seed != self.calibration.seed: raise ValueError( "Pool and dense solve share the existing build seed; selection_seed is separate." @@ -180,6 +189,7 @@ def uk_full_graph( calibration_year=config.calibration_year, time_period=config.time_period, geography_levels=config.geography_levels, + target_families=config.target_families, engine_blocks=config.engine_blocks, sample_fraction=config.effective_sample_fraction, target_weight_rule=config.calibration.target_weight_rule, @@ -330,8 +340,14 @@ def token(dtype): ) -def register_uk_full_kernels(registry: KernelRegistry) -> KernelRegistry: - """Extend the existing UK source/stage registry with the full build.""" +def register_uk_full_kernels( + registry: KernelRegistry, *, progress_callback=None +) -> KernelRegistry: + """Extend the existing UK source/stage registry with the full build. + + ``progress_callback`` is an operational observer of the dense solve's + epochs (staging telemetry, stderr progress); it never enters a node key. + """ # Source graph registries already contain these primitive kernels. for kernel in (UKBoundSpineKernel(), UKIdentityKernel(), UKClaimKernel()): @@ -341,5 +357,5 @@ def register_uk_full_kernels(registry: KernelRegistry) -> KernelRegistry: registry.register(kernel) register_uk_population_kernels(registry) register_uk_target_kernels(registry) - register_uk_calibration_kernels(registry) + register_uk_calibration_kernels(registry, progress_callback=progress_callback) return registry diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py index e0793606d..8d5b361d5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py @@ -332,6 +332,14 @@ class UKDenseSolveKernel(_CalibrationKernel): dependencies=_DEPENDENCIES, ) + def __init__(self, *, progress_callback=None): + # An operational observer of the solve's epochs (progress lines, + # staging telemetry rows). It is registered like the gate kernels' + # coverage engine: instance state, never part of the node key or + # the implementation hash, so an observed and an unobserved solve + # share one cache entry. + self.progress_callback = progress_callback + def run(self, context): from .graph_terminal import decode_full_gate_report @@ -384,6 +392,7 @@ def run(self, context): binding["target_loss_weights"], dtype=np.float64 ), target_loss_cap=binding["target_loss_cap"], + progress_callback=self.progress_callback, ) return KernelResult( artifacts={ @@ -848,10 +857,12 @@ def uk_calibration_nodes( ) -def register_uk_calibration_kernels(registry: KernelRegistry) -> KernelRegistry: +def register_uk_calibration_kernels( + registry: KernelRegistry, *, progress_callback=None +) -> KernelRegistry: + registry.register(UKDenseSolveKernel(progress_callback=progress_callback)) for kernel in ( UKSizeCheckpointImportKernel, - UKDenseSolveKernel, UKSizeSearchKernel, UKSizeDrawKernel, UKSizeRefitKernel, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py index c60106d58..965d08daf 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py @@ -55,7 +55,10 @@ population_columns, population_slices, ) -from .ladder_targets import ladder_vs_chronicle_household_dispersion +from .ladder_targets import ( + ladder_target_provenance, + ladder_vs_chronicle_household_dispersion, +) from .ledger_targets import uk_census_household_uprating, uk_ledger_households_total from .local_rowwise import UKRowwiseNationalRows, prepare_uk_full_solve @@ -210,6 +213,7 @@ def run(self, context: KernelContext) -> KernelResult: period=period, ) dispersion = ladder_vs_chronicle_household_dispersion(ladder, local.specs) + ladder_provenance = ladder_target_provenance(ladder) surface, reconciliation = uk_local_target_surface( full_problem._joint_surface_registry(local, national), bound_national_target_ids=full_problem._national_contract_target_ids( @@ -249,6 +253,7 @@ def run(self, context: KernelContext) -> KernelResult: "cross_geography": reconciliation, "census_household_uprating": reconciliation["census_household_uprating"], "household_dispersion": dispersion, + "ladder_provenance": ladder_provenance, "measure_exclusions": inputs["measure_exclusions"], "reviewed_unbound_higher_targets": inputs[ "reviewed_unbound_higher_targets" @@ -285,6 +290,18 @@ def run(self, context: KernelContext) -> KernelResult: full = json.loads(context.artifacts["surface"].payload) registry = registry_from_payload(full["registry"]) levels = context.params.get("geography_levels") + families = context.params.get("target_families") + if families is not None: + # An explicit family filter (``--households-only`` binds only + # ``census_households/constituency``) narrows the compiled + # registry before the geography selector; a token names either + # a family or a family at one grain. + registry = registry.select( + predicate=lambda spec: ( + spec.family in families + or f"{spec.family}/{target_geography(spec)}" in families + ) + ) selected = select_targets( registry, geography_levels=levels, geography_resolver=target_geography ) @@ -293,6 +310,7 @@ def run(self, context: KernelContext) -> KernelResult: payload = { "registry": registry_payload(selected.registry), "receipt": selected.receipt, + "target_families": None if families is None else list(families), } return KernelResult(artifacts={"selection": canonical_json(payload)}) @@ -541,6 +559,7 @@ def append_uk_target_nodes( calibration_year: int, time_period: str, geography_levels: tuple[str, ...] | None = None, + target_families: tuple[str, ...] | None = None, engine_blocks: int = 1, sample_fraction: float = 1.0, target_weight_rule: str = "uniform", @@ -551,6 +570,8 @@ def append_uk_target_nodes( if geography_levels is not None and not geography_levels: raise ValueError("An explicit geography selector must contain levels.") + if target_families is not None and not target_families: + raise ValueError("An explicit family selector must contain families.") cells = population_columns(graph, population) slices = population_slices(cells) compile_sources = ("uk_ladder", "uk_ledger_facts", *optional_sources) @@ -582,7 +603,10 @@ def append_uk_target_nodes( "uk.full.target_selection", UKFullTargetSelectionKernel.ref, population=population, - params={"geography_levels": geography_levels}, + params={ + "geography_levels": geography_levels, + "target_families": target_families, + }, artifact_inputs=(surface,), artifact_outputs=(ArtifactOutput("selection", TARGET_SELECTION_TYPE),), description="Select all geographies unless a target filter is explicitly requested.", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py index fc367dc51..a9e60bd54 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py @@ -9,6 +9,7 @@ from __future__ import annotations import hashlib +import io import json import sys from collections.abc import Mapping @@ -1115,3 +1116,471 @@ def materialize_uk_terminal_artifacts( "graph_artifact_key": key, } return inventory + + +# --------------------------------------------------------------------------- +# The rowwise candidate manifest projected from a finished graph +# --------------------------------------------------------------------------- +# +# ``rowwise_candidate_manifest.json`` is the dense line's evidence contract: +# the release pre-flight (``tools/preflight_uk_local_release_candidate.py``), +# the dense release assembler (``tools/assemble_uk_dense_release_dir.py``) and +# the staged-dataset lane read its schema-4 shape. The rowwise tool renders +# it from live objects (the solve, the clone, the problem); the graph keeps +# those as stored artifacts, so this projection reads the same facts back +# from the run manifest and the content store. It is new UK code by +# necessity: main's ``_manifest`` cannot run without the live objects. The +# projection adds one ``graph`` block naming the artifacts it stood on. +# +# Per-epoch ``calibration_progress`` rows: the dense solve node forwards its +# epochs through the observer registered on ``UKDenseSolveKernel``; the size +# search and refit nodes solve through ``dataset_size`` without an observer, +# so a size run stages no epoch rows for those two phases (recorded in the +# manifest's ``graph.epoch_rows`` field). + +_LADDER_TARGET_PREFIX = "ons.census.households@" +_NATIONAL_MATERIALIZATION = "uk_national_measure" + + +def _graph_payload(manifest, store, node: str, artifact: str) -> bytes: + return store.load_bytes(manifest.nodes[node].opaque_artifacts[artifact]) + + +def _optional_graph_json(manifest, store, node: str, artifact: str): + if node not in manifest.nodes: + return None + return json.loads(_graph_payload(manifest, store, node, artifact)) + + +def _gate_rows(document: Mapping) -> dict[str, dict]: + """The battery-shaped ``gates`` mapping of one graph gate report.""" + rows = {} + for outcome in document["report"]["outcomes"]: + entry = dict(outcome) + rows[str(entry.pop("id"))] = entry + return rows + + +def _fit_rows(target_rows: pd.DataFrame, materialization: Mapping[str, str]): + """Split the terminal target diagnostics into local and national rows.""" + kinds = target_rows["name"].map(materialization) + if kinds.isna().any(): + raise ValueError( + "Terminal target diagnostics name targets the ordered problem lacks." + ) + national = target_rows[kinds == _NATIONAL_MATERIALIZATION].reset_index(drop=True) + local = target_rows[kinds != _NATIONAL_MATERIALIZATION].reset_index(drop=True) + return local, national + + +def rowwise_candidate_manifest_from_graph( + final_manifest, + store, + *, + args, + posture, + pins: Mapping[str, Mapping[str, object]], + terminal_files: Mapping[str, Mapping[str, object]], + frame: Frame, + outputs: Mapping[str, Mapping[str, object]], + source_year: int, + inputs: Mapping[str, Mapping[str, object]], + ladder_provenance: Mapping[str, object], + code: Mapping[str, object], + runtime: Mapping[str, str], + created_at: str, +) -> dict: + """Project the schema-4 rowwise candidate manifest from stored artifacts. + + ``frame`` is the exported population, ``outputs`` the published file + records (``path``/``sha256``/``bytes``) keyed as the rowwise tool keys + them, ``inputs`` the ``dataset``/``ladder`` artifact records with their + ``pin_verified`` flags and ``code``/``runtime`` the git and package pins. + Every numerical fact comes from a graph artifact named in ``graph``. + """ + from microcosm.calibrate.artifacts import decode_problem + + from .calibration_run import UK_LOCAL_GATE_SCOPE + from .diagnostics import uk_fit_by_family, uk_support_limited_misses + from .graph_targets import registry_from_payload + from .national_sampling import UK_SAMPLE_RUNG_TOKENS + from .rowwise_cli import ( + gate_failures_by_criticality, + local_vintage_census, + release_verdict, + rowwise_parameters, + ) + + nodes = final_manifest.nodes + problem = decode_problem( + _graph_payload(final_manifest, store, "uk.full.problem", "problem") + ) + bindings = dict(problem.bindings) + surface = json.loads( + _graph_payload(final_manifest, store, "uk.full.target_compilation", "surface") + ) + gate_document = json.loads( + _graph_payload(final_manifest, store, "uk.full.gates.calibrated", "gate_report") + ) + diagnostics = json.loads( + _graph_payload( + final_manifest, store, "uk.full.gates.calibrated", "calibration_diagnostics" + ) + ) + target_rows = pd.read_csv( + io.BytesIO( + _graph_payload( + final_manifest, + store, + "uk.full.gates.calibrated", + "target_diagnostics_csv", + ) + ) + ) + support = pd.read_csv( + io.BytesIO( + _graph_payload( + final_manifest, store, "uk.full.gates.calibrated", "area_support_csv" + ) + ) + ) + holdout = json.loads( + _graph_payload(final_manifest, store, "uk.full.holdout", "holdout") + ) + geography_gate = json.loads( + _graph_payload(final_manifest, store, "uk.full.geography_gate", "gate") + ) + sampling = json.loads( + _graph_payload(final_manifest, store, "uk.full.sample", "sampling") + )["receipt"] + size_receipt = _optional_graph_json( + final_manifest, store, "uk.full.size_refit", "size" + ) + spine_provenance = ( + _optional_graph_json( + final_manifest, store, "uk.full.spine_checkpoint", "spine_provenance" + ) + or {} + ) + enforcement = dict(gate_document["enforcement"]) + gate_rows = _gate_rows(gate_document) + local_scope = { + gate_id: gate_rows[gate_id] + for gate_id in UK_LOCAL_GATE_SCOPE + if gate_id in gate_rows + } + blocking_lines, diagnostic_lines = gate_failures_by_criticality( + {"gates": gate_rows} + ) + enforced = set(enforcement["enforced_blocking"]) + unenforced = set(enforcement["unenforced_release_failures"]) + blocking_failures = [ + line for line in blocking_lines if line[1:].split("]")[0] in enforced + ] + unenforced_failures = [ + line for line in blocking_lines if line[1:].split("]")[0] in unenforced + ] + releasable, release_posture = release_verdict( + sample_fraction=args.sample_fraction, + engine_blocks=args.engine_blocks, + release_blocking_gates_passed=bool( + enforcement["release_blocking_gates_passed"] + ), + ) + materialization = { + str(problem.problem.targets[index].row_name): str(row.get("materialization")) + for index, row in enumerate(problem.target_metadata) + } + ladder_rows = sum( + 1 + for name, kind in materialization.items() + if kind != _NATIONAL_MATERIALIZATION and name.startswith(_LADDER_TARGET_PREFIX) + ) + national_rows = sum( + 1 for kind in materialization.values() if kind == _NATIONAL_MATERIALIZATION + ) + local_rows = len(materialization) - ladder_rows - national_rows + local_fit, national_fit = _fit_rows(target_rows, materialization) + abs_errors = target_rows["abs_relative_error"].to_numpy(dtype=np.float64) + support_by_grain = { + ("la" if grain == "local_authority" else str(grain)): rows.reset_index( + drop=True + ) + for grain, rows in support.groupby("geography_level", sort=True) + } + household_weights = np.asarray( + frame.weights_for("household").values, dtype=np.float64 + ) + design = pd.Series( + np.asarray(problem.problem.initial_weights.values, dtype=np.float64), + index=pd.Index(list(problem.entity_ids)), + ) + exported_ids = frame.table("household")["household_id"].tolist() + design_weights = design.reindex(exported_ids).to_numpy(dtype=np.float64) + if np.isnan(design_weights).any(): + raise ValueError("Exported households are not a subset of the ordered pool.") + with np.errstate(divide="ignore", invalid="ignore"): + ratio_vs_design = float(np.nanmax(np.divide(household_weights, design_weights))) + calibration_record = frame.mass_log[-1] + if "calibration" not in str(calibration_record.reason): + raise ValueError( + "exported frame's latest mass record is not the calibration record: " + f"{calibration_record.reason!r}." + ) + old_total = float(calibration_record.old_total) + new_total = float(calibration_record.new_total) + area_gate = gate_rows.get("uk_local_area_support", {}) + area_details = ( + area_gate.get("details", {}) if isinstance(area_gate, Mapping) else {} + ) + past_cap = diagnostics.get("past_cap_census") or {} + weight_kind = uk_household_weight_kind(frame).value + graph_keys = { + node_id: dict(receipt.opaque_artifacts) + for node_id, receipt in nodes.items() + if node_id + in { + "uk.full.problem", + "uk.full.target_compilation", + "uk.full.target_selection", + "uk.full.gates.calibrated", + "uk.full.holdout", + "uk.full.package", + "uk.full.sample", + "uk.full.geography_gate", + "uk.full.size_refit", + "uk.full.spine_checkpoint", + } + } + return { + "schema_version": 4, + "build_kind": "uk_rowwise_calibrated_candidate", + "release_role": posture.role, + "release_id": posture.release_id, + "candidate_scope": "adjudicated_partial", + "created_at": created_at, + "git_commit": code.get("git_commit"), + "git_dirty": code.get("git_dirty"), + "bound_target_families": list(bindings.get("bound_families", ())), + "binding_adjudications": dict(bindings.get("binding_adjudications", {})), + "cross_grain": dict(bindings.get("cross_geography", {})), + "ladder_assignment_provenance": dict(ladder_provenance), + "household_dispersion": dict(surface.get("household_dispersion", {})), + "parameters": rowwise_parameters(args, source_year=source_year), + "inputs": { + "dataset": dict(inputs["dataset"]), + "ladder": dict(inputs["ladder"]), + }, + "identity": { + "spine": { + **dict(inputs["dataset"]), + "spine_provenance": dict(spine_provenance), + }, + "ladder": { + **dict(inputs["ladder"]), + "layer_vintages": dict(ladder_provenance), + "matches_local_area_crosswalk_pin": True, + }, + "targets": dict(surface["source_validation"]["targets"]), + "code": dict(code), + "runtime": dict(runtime), + "sampling": dict(sampling), + "survey_year": int(source_year), + "calibration_year": int(surface["calibration_year"]), + }, + "sampling": dict(sampling), + "rung_surface": { + **dict(bindings.get("rung_surface", {})), + "rung": UK_SAMPLE_RUNG_TOKENS[float(args.sample_fraction)], + "fraction": float(args.sample_fraction), + "unreachable_check": "completed", + }, + "outputs": {key: dict(value) for key, value in outputs.items()}, + "geography": { + "constituencies_assigned": int( + support.loc[ + support["geography_level"] == "constituency", "area_code" + ].nunique() + ), + "local_authorities_assigned": int( + support.loc[ + support["geography_level"] == "local_authority", "area_code" + ].nunique() + ), + "missing_geography_rows": 0, + "ladder_gate": {**geography_gate, "phase": "post_calibration"}, + }, + "gate": {**geography_gate, "phase": "post_calibration"}, + "weights": { + "household_weight_kind": weight_kind, + "household_weight_kind_chain": [ + {"stage": "staging", "kind": "importance"}, + *( + [] + if float(args.sample_fraction) == 1.0 + else [{"stage": "sample", "kind": "importance"}] + ), + {"stage": "ladder_clone", "kind": "importance"}, + {"stage": "rowwise_calibration", "kind": weight_kind}, + ], + "mass_log_records_before_calibration": len(frame.mass_log) - 1, + "mass_log_records": len(frame.mass_log), + "calibration_mass_change": { + "entity": str(calibration_record.entity), + "old_total": old_total, + "new_total": new_total, + "relative_shift": (new_total - old_total) / old_total, + "declared_factor": calibration_record.declared_factor, + "reason": str(calibration_record.reason), + }, + "abs_delta": abs(new_total - old_total), + "declared_stretch_bound": float(posture.doctrine.max_weight_ratio), + "stretch_reference": "pool_design" + if size_receipt is None + else "normalized_horvitz_thompson_w_over_q", + "realized_max_weight_ratio_vs_stretch_reference": ( + ratio_vs_design + if size_receipt is None + else float(diagnostics["realized_max_weight_ratio"]) + ), + "realized_max_weight_ratio_vs_design": ratio_vs_design, + }, + "solve": { + "n_targets": int(len(materialization)), + "n_targets_by_kind": { + "local": int(local_rows), + "ladder": int(ladder_rows), + "national": int(national_rows), + }, + "n_households": int(frame.n("household")), + "pool_households": int(len(problem.entity_ids)), + "dataset_size": None if size_receipt is None else dict(size_receipt), + "initial_loss": diagnostics["initial_loss"], + "final_loss": diagnostics["final_loss"], + "max_abs_relative_error": float(abs_errors.max()) + if len(abs_errors) + else None, + "median_abs_relative_error": float(np.median(abs_errors)) + if len(abs_errors) + else None, + "n_nonzero": int(diagnostics["n_nonzero"]), + "past_cap": { + "n_targets": past_cap.get("n_targets"), + "past_at_init": past_cap.get("initial_past_cap"), + "past_at_final": past_cap.get("final_past_cap"), + "escaped": past_cap.get("escaped"), + "frozen": past_cap.get("frozen"), + "pushed_out": past_cap.get("pushed_out"), + }, + "loss_shape": "capped_relative_error", + "target_weight_rule": args.target_weight_rule, + "target_weight_rule_override": ( + {} + if args.target_weight_rule == posture.target_weight_rule + else { + "target_weight_rule": { + "default": posture.target_weight_rule, + "effective": args.target_weight_rule, + } + } + ), + "measure_resolution": dict(bindings.get("measure_resolution", {})), + "cross_grain": dict(bindings.get("cross_geography", {})), + "binding_adjudications": dict(bindings.get("binding_adjudications", {})), + "area_support_exclusions": { + "resource": "local_area_support_exclusions.json", + "entries_stood_on": sorted(area_details.get("reviewed_exclusions", {})), + "stale": list(area_details.get("stale_exclusions", [])), + "unknown": list(area_details.get("unknown_exclusions", [])), + }, + }, + "diagnostics": { + "schema_version": diagnostics["schema_version"], + "target_registry": diagnostics.get("target_registry"), + "weakest_families": diagnostics["uk_diagnostics"].get("weakest_families"), + "weakest_areas_by_fit": diagnostics["uk_diagnostics"].get( + "weakest_areas_by_fit" + ), + "rotated_holdout": diagnostics["uk_diagnostics"].get("rotated_holdout"), + }, + "support": { + "min_assigned_households": int(support["assigned_households"].min()), + "min_nonzero_households": int(support["nonzero_households"].min()), + "min_effective_sample_size": float(support["effective_sample_size"].min()), + "by_geography_level": { + str(level): { + "min_assigned_households": int(rows["assigned_households"].min()), + "min_nonzero_households": int(rows["nonzero_households"].min()), + "min_effective_sample_size": float( + rows["effective_sample_size"].min() + ), + "min_nonzero_source_households": int( + rows["nonzero_source_households"].min() + ), + } + for level, rows in support.groupby("geography_level", sort=True) + }, + }, + "fit": { + "local_by_family": uk_fit_by_family(local_fit), + "national_by_family": uk_fit_by_family(national_fit), + "weakest_families": sorted( + [*uk_fit_by_family(local_fit), *uk_fit_by_family(national_fit)], + key=lambda row: ( + -float(row["worst_abs_relative_error"]), + row["family"], + ), + )[:10], + "weakest_areas_by_fit": dict( + diagnostics["uk_diagnostics"].get("weakest_areas_by_fit") or {} + ), + "support_limited_misses": dict( + uk_support_limited_misses( + local_fit, support_by_grain, max_abs_relative_error=0.25 + ) + if len(local_fit) + else {} + ), + "rotated_holdout": dict(holdout), + }, + "vintages": local_vintage_census( + registry_from_payload(surface["local_registry"]) + ), + "failing_gate_ids": sorted( + gate_id + for gate_id, payload in gate_rows.items() + if not isinstance(payload, Mapping) or payload.get("status") != "passed" + ), + "releasable": bool(releasable and args.dataset_households is None), + "release_posture": { + **release_posture, + **( + {} + if args.dataset_households is None + else {"size_certification_present": False} + ), + }, + "census_household_uprating": dict( + surface.get("census_household_uprating") + or {"applied": False, "reason": "no cross-grain receipt"} + ), + "measure_exclusions": { + str(name): dict(record) + for name, record in sorted( + (surface.get("measure_exclusions") or {}).items() + ) + }, + "blocked_at_f100": bool(blocking_failures), + "blocking_failures": blocking_failures, + "diagnostic_failures": diagnostic_lines, + "release_gate_failures_not_enforced": unenforced_failures, + "local_gate_scope": sorted(local_scope), + "graph": { + "artifacts": graph_keys, + "terminal_files": { + key: dict(value) for key, value in terminal_files.items() + }, + "enforcement": enforcement, + "epoch_rows": "dense_solve_only", + }, + } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py new file mode 100644 index 000000000..2b02f41d8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py @@ -0,0 +1,796 @@ +"""The UK rowwise release roles' shared command surface (microcosm#823). + +Both UK dataset lines are built by one command whose ``--release-role`` +fixes every solve default and refuses the other role's flags. The helpers +here are the object-free part of that surface: the role-defaulted argument +resolution, the posture-aware validator and the two refusal tables, the +recorded run parameters, the role's output filenames and the dense role's +Logbook attempt helpers. They were moved from +``tools/build_uk_rowwise_candidate.py`` so the graph full-build driver +(:mod:`microcosm.build.uk_runtime.full_build_cli`) and the rowwise tool +parse, default, refuse and record identically; the tool imports them back. + +The private spellings (``_validate_cli_args`` and friends) are kept as +aliases so the drivers' tests can patch and call the names they always did. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import uuid +from collections.abc import Mapping +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import numpy as np + +from microcosm.build.logbook import canonical_json_bytes +from microcosm.build.logbook_adoption import ( + AttemptState, + apply_error_verdict, + error_receipt_path, + local_artifact_reference, + record_terminal_attempt, + write_error_receipt, +) +from microcosm.build.uk_runtime.ledger_targets import _spec_geography +from microcosm.build.uk_runtime.national_sampling import ( + UK_SAMPLE_RUNG_TOKENS, + UK_SAMPLE_SEED_DEFAULT, +) +from microcosm.build.uk_runtime.rowwise_posture import ( + UK_ROWWISE_DENSE_POSTURE, + UKRowwisePosture, + uk_rowwise_posture, +) +from microcosm.calibrate import TargetRegistry + +__all__ = [ + "AREA_SUPPORT_FILENAME", + "BUDGET_ITERS", + "BUILD_RECORD_FILENAME", + "CALIBRATION_DIAGNOSTICS_FILENAME", + "CONSERVE_MASS", + "DATASET_SIZE_SELECTION_FILENAME", + "DENSE_REFERENCE_DIAGNOSTICS_FILENAME", + "L0_LAMBDA", + "LOCAL_REGISTRY_FILENAME", + "MANIFEST_FILENAME", + "NATIONAL_CONTRACT_REGISTRY_FILENAME", + "NATIONAL_REGISTRY_FILENAME", + "PAST_CAP_FILENAME", + "REPOSITORY", + "ROLE_DEFAULTED_ARGUMENTS", + "SCORE_RECEIPT_FILENAME", + "SIZE_RUN_ONLY_OUTPUTS", + "SOLVE_DIAGNOSTICS_FILENAME", + "TARGET_RECORDS", + "UK_CANDIDATE_PIPELINE", + "candidate_clone_counts_argument", + "candidate_identity_digest", + "doctrine_bounds", + "git_commit", + "git_dirty", + "gate_failures_by_criticality", + "is_release_blocking", + "local_vintage_census", + "json_text", + "new_candidate_build_id", + "output_paths", + "posture_of", + "record_candidate_attempt", + "record_candidate_error", + "refuse_dense_role_arguments", + "refuse_national_role_arguments", + "release_verdict", + "resolve_role_arguments", + "rowwise_parameters", + "validate_cli_args", +] + +MANIFEST_FILENAME = "rowwise_candidate_manifest.json" +SOLVE_DIAGNOSTICS_FILENAME = "solve_diagnostics.csv" +CALIBRATION_DIAGNOSTICS_FILENAME = "calibration_diagnostics.json" +AREA_SUPPORT_FILENAME = "area_support_summary.csv" +PAST_CAP_FILENAME = "past_cap_census.json" +LOCAL_REGISTRY_FILENAME = "local_target_registry.json" +#: National-role outputs (the calibration seam's evidence shape). +BUILD_RECORD_FILENAME = "build_record.json" +NATIONAL_REGISTRY_FILENAME = "national_target_registry.json" +NATIONAL_CONTRACT_REGISTRY_FILENAME = "national_contract_registry.json" +SCORE_RECEIPT_FILENAME = "score_vs_incumbent.json" +DENSE_REFERENCE_DIAGNOSTICS_FILENAME = "dense_reference_diagnostics.csv" +DATASET_SIZE_SELECTION_FILENAME = "dataset_size_selection.csv" + +#: Outputs a run writes only when ``--dataset-households`` is set. +SIZE_RUN_ONLY_OUTPUTS = frozenset({"dense_reference", "selection"}) +_SIZE_RUN_ONLY_OUTPUTS = SIZE_RUN_ONLY_OUTPUTS + +#: The solve options every rowwise run records beside its doctrine. +CONSERVE_MASS = False +TARGET_RECORDS: int | None = None +L0_LAMBDA = 0.0 +BUDGET_ITERS = 10 +_CONSERVE_MASS = CONSERVE_MASS +_TARGET_RECORDS = TARGET_RECORDS +_L0_LAMBDA = L0_LAMBDA +_BUDGET_ITERS = BUDGET_ITERS + +# The dense role's Logbook pipeline, kept as a module name for the Logbook +# helpers and the contract-pin tests; the posture record +# (``rowwise_posture.py``) is the source of truth for both roles. +UK_CANDIDATE_PIPELINE = UK_ROWWISE_DENSE_POSTURE.pipeline +_UK_CANDIDATE_PIPELINE = UK_CANDIDATE_PIPELINE + +#: The checkout the Logbook references anchor to (the workspace root above +#: ``packages/``); the current directory when the package runs installed. +REPOSITORY = next( + ( + parent + for parent in Path(__file__).resolve().parents + if (parent / "pyproject.toml").is_file() and (parent / "packages").is_dir() + ), + Path.cwd(), +) +_REPOSITORY = REPOSITORY + + +def json_text(payload: Any) -> str: + """The evidence files' JSON rendering: sorted keys, two-space indent.""" + + return ( + json.dumps( + payload, + allow_nan=False, + indent=2, + sort_keys=True, + ) + + "\n" + ) + + +_json_text = json_text + + +def new_candidate_build_id( + *, seed: int, timestamp: datetime, rung: str = "f100" +) -> str: + """The dense role's attempt id; the national role mints the seam's.""" + + instant = timestamp.astimezone(UTC) + return ( + f"{UK_ROWWISE_DENSE_POSTURE.build_id_prefix}{rung}-s{seed}-" + f"{instant.strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:8]}" + ) + + +_new_candidate_build_id = new_candidate_build_id + + +def posture_of(args: argparse.Namespace) -> UKRowwisePosture: + """The release-role posture bound to parsed arguments.""" + + posture = getattr(args, "_posture", None) + if not isinstance(posture, UKRowwisePosture): + raise RuntimeError("arguments carry no release-role posture; parse them first.") + return posture + + +_posture_of = posture_of + + +def candidate_clone_counts_argument(value: str) -> tuple[int, ...]: + parts = value.split(",") + if not value.strip() or any(not part.strip() for part in parts): + raise argparse.ArgumentTypeError( + "candidate clone counts must be a non-empty comma list of positive integers" + ) + try: + counts = [int(part.strip()) for part in parts] + except ValueError as error: + raise argparse.ArgumentTypeError( + "candidate clone counts must be a comma list of positive integers" + ) from error + if any(count <= 0 for count in counts): + raise argparse.ArgumentTypeError( + "candidate clone counts must all be positive integers" + ) + return tuple(sorted(set(counts))) + + +_candidate_clone_counts_argument = candidate_clone_counts_argument + + +def doctrine_bounds(posture: UKRowwisePosture) -> dict[str, Any]: + return posture.doctrine_bounds() + + +_doctrine_bounds = doctrine_bounds + + +def rowwise_parameters(args: argparse.Namespace, *, source_year: int) -> dict[str, Any]: + """The run parameters a rowwise manifest, plan and identity digest record.""" + + posture = posture_of(args) + return { + "release_role": posture.role, + "n_clones": None if args.n_clones is None else int(args.n_clones), + "dataset_households": args.dataset_households, + "seed": int(args.seed), + "selection_seed": None + if args.dataset_households is None + else int(args.seed if args.selection_seed is None else args.selection_seed), + "selection_pi_hi": None + if args.dataset_households is None + else float(args.selection_pi_hi), + "baseline_pi_floor": None + if args.dataset_households is None + else float(args.baseline_pi_floor), + "size_checkpoint": bool( + args.dataset_households is not None + and not args.no_size_checkpoint + and args.resume_size_checkpoint is None + ), + "resume_size_checkpoint": None + if args.resume_size_checkpoint is None + else str(args.resume_size_checkpoint.expanduser().resolve()), + "source_year": source_year, + "source_lineage_modulus": args.source_lineage_modulus, + "sample_fraction": float(args.sample_fraction), + "sample_seed": int(args.sample_seed), + "engine_blocks": int(args.engine_blocks), + "target_weight_rule": args.target_weight_rule, + "release_candidate": bool(args.release_candidate), + "skip_holdout": bool(args.skip_holdout), + "epochs": int(args.epochs), + "learning_rate": float(args.learning_rate), + "expected_constituency_vintage": ( + None + if args.expected_constituency_vintage is None + else str(args.expected_constituency_vintage) + ), + "doctrine": doctrine_bounds(posture), + "solve_options": { + "conserve_mass": CONSERVE_MASS, + "target_records": TARGET_RECORDS, + "l0_lambda": L0_LAMBDA, + "budget_iters": BUDGET_ITERS, + }, + } + + +_parameters = rowwise_parameters + + +def candidate_identity_digest( + *, + pins: dict[str, dict[str, object]], + args: argparse.Namespace, + source_year: int, +) -> str: + payload = { + "build_kind": "uk_rowwise_calibrated_candidate", + "inputs": pins, + "parameters": rowwise_parameters(args, source_year=source_year), + "source_year": source_year, + } + return hashlib.sha256(canonical_json_bytes(payload)).hexdigest() + + +_candidate_identity_digest = candidate_identity_digest + + +def record_candidate_attempt( + *, + state: AttemptState, + started_at: float, + started_ts: datetime, + seed: int, + code_pin: str, + disposition: str, + predecessor: str | None, + spool_dir: Path, + rung: str = "f100", +) -> Path: + return record_terminal_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + pipeline=UK_CANDIDATE_PIPELINE, + rung=rung, + seed=seed, + code_pin=code_pin, + disposition=disposition, + predecessor=predecessor, + spool_dir=spool_dir, + ) + + +_record_candidate_attempt = record_candidate_attempt + + +def record_candidate_error( + *, + error: BaseException, + state: AttemptState, + started_at: float, + started_ts: datetime, + seed: int, + code_pin: str, + predecessor: str | None, + base_dir: Path, + spool_dir: Path, + rung: str = "f100", +) -> None: + error_path = write_error_receipt( + error_receipt_path(base_dir, build_id=state.build_id), + state=state, + pipeline=UK_CANDIDATE_PIPELINE, + error=error, + ) + apply_error_verdict( + state, + f"{local_artifact_reference(error_path, repository_hint=REPOSITORY)}#/error_type", + ) + record_candidate_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + seed=seed, + code_pin=code_pin, + disposition="failed", + predecessor=predecessor, + spool_dir=spool_dir, + rung=rung, + ) + + +_record_candidate_error = record_candidate_error + + +#: Solve arguments whose argparse default is ``None`` so an explicit value can +#: be told from the role's default: the other role's refusal table keys on +#: what was actually given. +ROLE_DEFAULTED_ARGUMENTS = ( + "n_clones", + "seed", + "sample_seed", + "epochs", + "learning_rate", + "target_weight_rule", + "expected_constituency_vintage", +) +_ROLE_DEFAULTED_ARGUMENTS = ROLE_DEFAULTED_ARGUMENTS + + +def resolve_role_arguments(args: argparse.Namespace) -> UKRowwisePosture: + """Bind the declared role's posture and fill its defaults into unset arguments.""" + + posture = uk_rowwise_posture(args.release_role) + args._explicit_arguments = frozenset( + name for name in ROLE_DEFAULTED_ARGUMENTS if getattr(args, name) is not None + ) + if args.n_clones is None: + args.n_clones = posture.clone_count + if args.seed is None: + args.seed = posture.seed + if args.sample_seed is None: + args.sample_seed = UK_SAMPLE_SEED_DEFAULT + if args.epochs is None: + args.epochs = posture.epochs + if args.learning_rate is None: + args.learning_rate = posture.learning_rate + if args.target_weight_rule is None: + args.target_weight_rule = posture.target_weight_rule + if args.expected_constituency_vintage is None: + args.expected_constituency_vintage = posture.expected_constituency_vintage + args._posture = posture + return posture + + +_resolve_role_arguments = resolve_role_arguments + + +def validate_cli_args(args: argparse.Namespace) -> None: + posture = posture_of(args) + # The declared role is checked against the parameters first: the other + # role's flags are refused by name before any value is range-checked. + if posture.role == "national": + refuse_dense_role_arguments(args, posture) + else: + refuse_national_role_arguments(args, posture) + if args.selection_seed is not None and args.dataset_households is None: + raise ValueError("--selection-seed requires --dataset-households.") + if not (0.0 < args.selection_pi_hi <= 1.0): + raise ValueError("--selection-pi-hi must be in (0, 1].") + if args.selection_pi_hi != 1.0 and args.dataset_households is None: + raise ValueError("--selection-pi-hi requires --dataset-households.") + if not (0.0 <= args.baseline_pi_floor <= 1.0): + raise ValueError("--baseline-pi-floor must be in [0, 1].") + if args.baseline_pi_floor != 0.0 and args.dataset_households is None: + raise ValueError("--baseline-pi-floor requires --dataset-households.") + if args.no_size_checkpoint and args.dataset_households is None: + raise ValueError("--no-size-checkpoint requires --dataset-households.") + if args.resume_size_checkpoint is not None: + if args.dataset_households is None: + raise ValueError("--resume-size-checkpoint requires --dataset-households.") + if args.no_size_checkpoint: + raise ValueError( + "--resume-size-checkpoint already implies no new checkpoint; " + "drop --no-size-checkpoint." + ) + if args.dataset_households is not None: + if args.dataset_households <= 0: + raise ValueError("--dataset-households must be positive.") + if args.release_candidate: + raise ValueError( + "--dataset-households is candidate-only: size-specific matched comparison and promotion scorecard are required before release." + ) + # Size-selection arguments are validated first so their refusals name + # the size flag at fault; the pinned Ledger inputs are then mandatory. + ledger_values = ( + args.ledger_facts, + args.ledger_facts_sha256, + args.ledger_manifest_sha256, + ) + if not all(value is not None for value in ledger_values): + raise ValueError( + "--ledger-facts, --ledger-facts-sha256, and " + "--ledger-manifest-sha256 are mandatory and must be supplied together." + ) + # The input pin is required only when an input H5 is given: a build + # from a spine request executes the spine stages in its own graph and + # has no checkpoint H5 to pin. + input_pin_required = args.input_h5 is not None and args.input_sha256 is None + if posture.ladder_required: + if args.ladder is None: + raise ValueError("--release-role dense requires --ladder.") + if input_pin_required or args.ladder_sha256 is None: + raise ValueError( + "the joint registry path requires --input-sha256 and --ladder-sha256." + ) + elif input_pin_required: + raise ValueError("--release-role national requires --input-sha256.") + if args.release_candidate: + required_release = { + "--ladder-sha256": args.ladder_sha256, + "--ledger-facts": args.ledger_facts, + "--ledger-facts-sha256": args.ledger_facts_sha256, + "--ledger-manifest-sha256": args.ledger_manifest_sha256, + } + if args.input_h5 is not None: + required_release = {"--input-sha256": args.input_sha256, **required_release} + missing_release = [ + name for name, value in required_release.items() if value is None + ] + if missing_release: + raise ValueError( + "--release-candidate requires pinned joint inputs: " + + ", ".join(missing_release) + ) + refused = [] + if args.target_weight_rule != posture.target_weight_rule: + refused.append("--target-weight-rule") + if args.epochs != posture.epochs: + refused.append(f"--epochs != doctrine {posture.epochs}") + if args.n_clones != posture.clone_count: + refused.append(f"--n-clones != doctrine {posture.clone_count}") + if args.measure_exclusions is not None: + refused.append("--measure-exclusions") + if args.skip_holdout: + refused.append("--skip-holdout") + if args.engine_blocks > 1: + refused.append("--engine-blocks > 1") + if args.sample_fraction != 1.0: + refused.append("--sample-fraction != 1.0") + if refused: + raise ValueError( + "--release-candidate refuses non-release settings: " + + ", ".join(refused) + ) + if args.n_clones is not None and args.n_clones <= 0: + raise ValueError("--n-clones must be positive.") + if args.seed < 0: + raise ValueError("--seed must be non-negative.") + if args.sample_fraction not in UK_SAMPLE_RUNG_TOKENS: + raise ValueError( + "--sample-fraction must be one of " + f"{sorted(UK_SAMPLE_RUNG_TOKENS)}, got {args.sample_fraction!r}." + ) + if args.sample_seed < 0: + raise ValueError("--sample-seed must be non-negative.") + if args.engine_blocks <= 0: + raise ValueError("--engine-blocks must be positive.") + if args.engine_blocks > 1 and args.engine_blocks != args.n_clones: + raise ValueError("--engine-blocks greater than one must equal --n-clones.") + if args.source_year is not None and args.source_year <= 0: + raise ValueError("--source-year must be positive.") + if args.epochs <= 0: + raise ValueError("--epochs must be positive.") + if not np.isfinite(args.learning_rate) or args.learning_rate <= 0: + raise ValueError("--learning-rate must be positive and finite.") + if args.target_loss_cap is not None and ( + not np.isfinite(args.target_loss_cap) or args.target_loss_cap <= 0 + ): + raise ValueError("--target-loss-cap must be positive and finite.") + if ( + args.expected_constituency_vintage is not None + and not str(args.expected_constituency_vintage).strip() + ): + raise ValueError("--expected-constituency-vintage must be non-empty.") + + +_validate_cli_args = validate_cli_args + + +def refuse_dense_role_arguments( + args: argparse.Namespace, posture: UKRowwisePosture +) -> None: + """The national role's refusal table: nothing of the clone surface may be given. + + ``--release-candidate`` is refused outright with the seam's own reason: the + calibration-seam battery covers six of the declared entries and must never + sign a shippability claim; a national cut's verdict comes only from the + release-cut certification producer (``tools/certify_uk_release_cut.py``). + """ + + if args.release_candidate: + raise ValueError( + "--release-candidate is refused on the national role: the " + "calibration seam's scoped battery cannot sign shippability; run " + "the release-cut certification producer " + "(tools/certify_uk_release_cut.py) on the finished build instead." + ) + explicit = args._explicit_arguments + refused: list[str] = [] + if args.ladder is not None: + refused.append("--ladder") + if args.ladder_sha256 is not None: + refused.append("--ladder-sha256") + if "expected_constituency_vintage" in explicit: + refused.append("--expected-constituency-vintage") + if args.source_year is not None: + refused.append("--source-year") + if args.source_lineage_modulus is not None: + refused.append("--source-lineage-modulus") + if "n_clones" in explicit: + refused.append("--n-clones") + if args.candidate_clone_counts is not None: + refused.append("--candidate-clone-counts") + if args.engine_blocks != 1: + refused.append("--engine-blocks") + if args.households_only: + refused.append("--households-only") + if args.skip_holdout: + refused.append("--skip-holdout") + if args.dataset_households is not None: + refused.append("--dataset-households") + if args.selection_seed is not None: + refused.append("--selection-seed") + if args.selection_pi_hi != 1.0: + refused.append("--selection-pi-hi") + if args.baseline_pi_floor != 0.0: + refused.append("--baseline-pi-floor") + if args.no_size_checkpoint: + refused.append("--no-size-checkpoint") + if args.resume_size_checkpoint is not None: + refused.append("--resume-size-checkpoint") + if args.sample_fraction != 1.0: + refused.append("--sample-fraction") + if "sample_seed" in explicit: + refused.append("--sample-seed") + if "seed" in explicit and args.seed != posture.seed: + # The seam doctrine's seed is a reviewed constant, not a knob. + refused.append(f"--seed != doctrine {posture.seed}") + if args.target_weight_rule not in posture.allowed_target_weight_rules: + refused.append(f"--target-weight-rule {args.target_weight_rule}") + if refused: + raise ValueError( + "--release-role national refuses the dense role's arguments: " + + ", ".join(refused) + ) + + +_refuse_dense_role_arguments = refuse_dense_role_arguments + + +def refuse_national_role_arguments( + args: argparse.Namespace, posture: UKRowwisePosture +) -> None: + """The dense role's refusal table: the seam's knobs are not its own.""" + + refused: list[str] = [] + if args.target_loss_cap is not None: + refused.append("--target-loss-cap") + if args.allow_unpinned_feed: + refused.append("--allow-unpinned-feed") + if args.incumbent_h5 is not None or args.incumbent_sha256 is not None: + refused.append("--incumbent-h5/--incumbent-sha256") + if args.target_weight_rule not in posture.allowed_target_weight_rules: + refused.append(f"--target-weight-rule {args.target_weight_rule}") + if refused: + raise ValueError( + "--release-role dense refuses the national role's arguments: " + + ", ".join(refused) + ) + + +_refuse_national_role_arguments = refuse_national_role_arguments + + +def output_paths( + out_dir: Path, + *, + posture: UKRowwisePosture, + vintage: str, +) -> dict[str, Path]: + """The role's output paths for one FRS release vintage (``2024_25``).""" + + dataset = out_dir / posture.dataset_filename(vintage) + if posture.role == "national": + return { + "dataset": dataset, + "manifest": out_dir / MANIFEST_FILENAME, + "calibration_diagnostics": out_dir / CALIBRATION_DIAGNOSTICS_FILENAME, + "build_record": out_dir / BUILD_RECORD_FILENAME, + "terminal_gates": out_dir / posture.gate_report_filename(vintage), + "national_registry": out_dir / NATIONAL_REGISTRY_FILENAME, + "contract_registry": out_dir / NATIONAL_CONTRACT_REGISTRY_FILENAME, + "score_receipt": out_dir / SCORE_RECEIPT_FILENAME, + } + return { + "dataset": dataset, + "manifest": out_dir / MANIFEST_FILENAME, + "diagnostics": out_dir / SOLVE_DIAGNOSTICS_FILENAME, + "support": out_dir / AREA_SUPPORT_FILENAME, + "past_cap": out_dir / PAST_CAP_FILENAME, + "calibration_diagnostics": out_dir / CALIBRATION_DIAGNOSTICS_FILENAME, + "local_gates": out_dir / posture.gate_report_filename(vintage), + "local_registry": out_dir / LOCAL_REGISTRY_FILENAME, + "dense_reference": out_dir / DENSE_REFERENCE_DIAGNOSTICS_FILENAME, + "selection": out_dir / DATASET_SIZE_SELECTION_FILENAME, + } + + +_output_paths = output_paths + + +def gate_failures_by_criticality( + gate_report: Mapping[str, Any], +) -> tuple[list[str], list[str]]: + """Split a persisted battery report's failure lines by criticality. + + Returns ``(release_blocking, diagnostic)``, each entry-prefixed like + :class:`GateBatteryBlockedError`'s lines. Only ``failed`` and + ``evidence_absent`` entries are failures; ``not_applicable`` and + ``unreached`` entries are not. + """ + + blocking: list[str] = [] + diagnostic: list[str] = [] + gates = gate_report.get("gates", {}) + if not isinstance(gates, Mapping): + return blocking, diagnostic + for gate_id, payload in gates.items(): + if not isinstance(payload, Mapping): + continue + status = payload.get("status") + if status not in {"failed", "evidence_absent"}: + continue + lines = [f"[{gate_id}] {line}" for line in payload.get("failures") or ()] + if not lines: + lines = [f"[{gate_id}] {payload.get('reason') or status}"] + bucket = blocking if is_release_blocking(payload) else diagnostic + bucket.extend(lines) + return blocking, diagnostic + + +_gate_failures_by_criticality = gate_failures_by_criticality + + +def is_release_blocking(payload: Mapping[str, Any]) -> bool: + """Fail-closed criticality read. + + Only an entry that explicitly declares ``criticality: diagnostic`` is + exempt from vetoing the release; a missing or unknown criticality is + treated as release-blocking, so partial schema drift on one persisted + entry cannot drop a failed gate out of both the blocking list and + ``all_gates_passed``. + """ + + return payload.get("criticality") != "diagnostic" + + +_is_release_blocking = is_release_blocking + + +def release_verdict( + *, + sample_fraction: float, + engine_blocks: int, + release_blocking_gates_passed: bool, +) -> tuple[bool, dict[str, bool]]: + """``releasable`` needs the full rung, a single-block engine resolution and + every release-blocking gate passed. + + Per-block engine resolution mis-measures population-normalised formulas + (each block reproduces a national aggregate: the ×K land-value artefact + behind the #736 erratum), so a run resolved in more than one block is + diagnostic-only whatever its gates say. The posture is written beside the + verdict so a reader sees which leg failed. + """ + + posture = { + "full_rung": float(sample_fraction) == 1.0, + "single_block_engine": int(engine_blocks) == 1, + "release_blocking_gates_passed": bool(release_blocking_gates_passed), + } + return all(posture.values()), posture + + +_release_verdict = release_verdict + + +def local_vintage_census(registry: TargetRegistry) -> list[dict[str, object]]: + counts: dict[tuple[str, str, str, str], int] = {} + for spec in registry.specs: + resolved = str(spec.metadata.get("ledger_fact_period", "")) + target = str(spec.period) + if not resolved or resolved == target: + continue + level, _ = _spec_geography(spec) + key = (spec.family, level, resolved, target) + counts[key] = counts.get(key, 0) + 1 + return [ + { + "family": family, + "geography_level": level, + "resolved_period": resolved, + "target_period": target, + "cells": cells, + } + for (family, level, resolved, target), cells in sorted(counts.items()) + ] + + +_local_vintage_census = local_vintage_census + + +def git_commit() -> str | None: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + return None + return result.stdout.strip() + + +_git_commit = git_commit + + +def git_dirty() -> bool | None: + """Measured, not asserted: tracked modifications in the working tree. + + ``None`` when git cannot answer (no repository), so a downstream + assembler records the pin as unmeasured rather than clean. + """ + + result = subprocess.run( + ["git", "status", "--porcelain", "--untracked-files=no"], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + return None + return bool(result.stdout.strip()) + + +_git_dirty = git_dirty diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py new file mode 100644 index 000000000..e2e07aa7b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py @@ -0,0 +1,717 @@ +"""Staging telemetry and staged-dataset delivery of the UK rowwise roles. + +Two destinations, one run id: reviewed aggregate telemetry goes to the +staging repository under ``runs//``, the finished bundle a manifest +vouches for to the private artifact repository under ``staged//``. +Both are best-effort evidence about the build, never a release, and the +build must never abort on its own progress report. + +These helpers were moved from ``tools/build_uk_rowwise_candidate.py`` so the +graph full-build driver and the rowwise tool stage identically; the tool +imports them back. ``_hub_api`` and ``_hub_token`` are the test seams the +drivers' tests patch on this module. +""" + +from __future__ import annotations + +import argparse +import sys +import tempfile +from collections.abc import Callable, Mapping +from importlib import metadata +from pathlib import Path +from typing import Any + +from microcosm.build.staging_dataset import ( + StagedDatasetBundle, + disabled_staged_dataset, + local_only_staged_dataset, + stage_bundle, + write_sidecars, +) +from microcosm.build.staging_storage import HuggingFaceDatasetStorage +from microcosm.build.staging_v2 import ( + StagingContractError, + StagingTelemetryV2, + disabled_staging_delivery, +) +from microcosm.build.uk_runtime.rowwise_cli import ( + BUDGET_ITERS, + MANIFEST_FILENAME, + SIZE_RUN_ONLY_OUTPUTS, + json_text, + posture_of, +) +from microcosm.build.uk_runtime.staging import UK_STAGED_DATASET_PREFIX + +__all__ = [ + "STAGED_DATASET_PHASES", + "STAGING_EPOCH_EVERY", + "STAGING_MAX_EPOCH_ROWS", + "STAGING_UPLOAD_INTERVAL_SECONDS", + "add_staging_artifact", + "create_staging_telemetry", + "fail_staging_telemetry", + "finalize_staging_telemetry", + "fit_summary", + "gate_statuses", + "preflight_staged_dataset", + "publish_staged_files", + "replace_manifest", + "stage", + "stage_dataset", + "staged_dataset_mode", + "staging_delivery", + "staging_epoch_every", + "thinned_epochs", +] + +# Best-effort telemetry upload cadence. The Hub allows about 128 commits per +# hour per repository and one cycle is up to eight single-file commits, so +# the shared 30-second default exhausts the budget on a multi-hour solve +# and loses uploads (the v20 national run did); five minutes keeps a +# 1,500-epoch run well inside it. +STAGING_UPLOAD_INTERVAL_SECONDS = 300.0 +# Staging telemetry keeps one row per forwarded epoch in +# calibration_progress.json and one event in events.ndjson, both under the +# contract's 5 MiB remote cap. A size run at 2,000 epochs solves the dense +# pool, up to ten full-length L0 probes and the refit: about 24,000 epochs, +# which would breach the cap mid-run. Forwarding every tenth epoch and the +# last epoch of each phase keeps the loss curve and stays near 0.8 MB. +STAGING_EPOCH_EVERY = 10 +# ...and never more than this many forwarded epochs per run, whatever --epochs +# says: the stride grows with the run so the cap holds by construction. +STAGING_MAX_EPOCH_ROWS = 2400 +STAGED_DATASET_PHASES = { + "uploaded": "dataset_staged", + "already_staged": "dataset_staged", + "failed": "dataset_stage_failed", + "skipped": "dataset_stage_skipped", +} +_STAGING_UPLOAD_INTERVAL_SECONDS = STAGING_UPLOAD_INTERVAL_SECONDS +_STAGING_EPOCH_EVERY = STAGING_EPOCH_EVERY +_STAGING_MAX_EPOCH_ROWS = STAGING_MAX_EPOCH_ROWS +_STAGED_DATASET_PHASES = STAGED_DATASET_PHASES + + +def _hub_api() -> Any: + """The Hub client used for telemetry and the staged dataset (test seam).""" + + from huggingface_hub import HfApi + + return HfApi() + + +def _hub_token() -> str | None: + """The ambient Hub credential, if any (test seam).""" + + from huggingface_hub import get_token + + return get_token() + + +def staged_dataset_mode(args: argparse.Namespace) -> str: + if args.no_staging or args.no_staged_dataset: + return "disabled" + if args.staging_local_only: + return "local_only" + return "local_and_remote" + + +_staged_dataset_mode = staged_dataset_mode + + +def preflight_staged_dataset(args: argparse.Namespace) -> None: + """Refuse a remote dataset stage the run could not complete. + + The bundle upload is the last step of a multi-hour run, so the credential + and the repository are checked before the spine is read. Telemetry stays + best-effort with no pre-flight, as on the national command. + """ + + if args.dry_run or staged_dataset_mode(args) != "local_and_remote": + return + repo_id = str(args.staged_dataset_repo_id).strip() + hint = "pass --staging-local-only or --no-staged-dataset to keep the bundle local" + if not _hub_token(): + raise ValueError( + f"remote dataset staging to {repo_id} needs a Hugging Face write " + f"credential (HF_TOKEN or `hf auth login`); {hint}." + ) + storage = HuggingFaceDatasetStorage(repo_id, api=_hub_api()) + try: + storage.head_revision() + except Exception as error: + # The transport's own message is not chained: it can carry request + # URLs and identifiers, and the type name is enough to act on. + raise ValueError( + f"remote dataset staging cannot reach {repo_id} " + f"({type(error).__name__}); {hint}." + ) from None + _require_write_credential(storage, hint=hint) + + +_preflight_staged_dataset = preflight_staged_dataset + + +def _require_write_credential(storage: HuggingFaceDatasetStorage, *, hint: str) -> None: + """Refuse a credential that can see the repository but cannot write it. + + A read token, or a fine-grained token scoped to another owner, passes the + reachability check and is refused by the Hub with 403 only when the upload + starts, hours later. The scope is read from the Hub's own description of + the token; when it cannot be read the upload itself is the proof. + """ + + try: + can_write = storage.credential_can_write() + except Exception: + can_write = None + if can_write is False: + raise ValueError( + f"remote dataset staging to {storage.repo_id} needs a write credential: " + "the ambient Hugging Face token is read-only or is not scoped to this " + "repository or its owner (a fine-grained token needs repo.write on " + f"{storage.repo_id} or on {storage.repo_id.split('/', 1)[0]}); {hint}." + ) + if can_write is None: + print( + "warning: the Hugging Face credential's write scope could not be read; " + "the upload at the end of the run will prove it.", + file=sys.stderr, + flush=True, + ) + + +def create_staging_telemetry( + args: argparse.Namespace, *, build_id: str +) -> StagingTelemetryV2 | None: + if args.no_staging: + return None + local_only = bool(args.staging_local_only) + out_dir = args.out.expanduser().resolve() + posture = posture_of(args) + return StagingTelemetryV2( + run_id=args.staging_run_id or build_id, + country_code="GB", + operation_id=posture.staging_operation_id, + pipeline_id=posture.pipeline, + pipeline_version=metadata.version("microcosm-build"), + candidate_id=args.staging_candidate_id or build_id, + local_dir=args.staging_dir or out_dir / "staging", + run_kind="calibration", + delivery_mode="local_only" if local_only else "local_and_remote", + repo_id=None if local_only else args.staging_repo_id, + upload_interval_seconds=args.staging_upload_interval_seconds, + api=None if local_only else _hub_api(), + ) + + +_create_staging_telemetry = create_staging_telemetry + + +def stage( + telemetry: StagingTelemetryV2 | None, + stage_id: str, + event_status: str = "started", + **details: Any, +) -> None: + """Forward one stage event to the best-effort telemetry. + + A contract or content refusal of the event is reported and the event + dropped; the build must never abort on its own progress report. Each + event is judged on its own details, so a refused event does not silence + the ones that follow. + """ + + if telemetry is None: + return + try: + telemetry.stage(stage_id, event_status=event_status, **details) + except StagingContractError as error: + print( + f"warning: staging telemetry refused the {stage_id!r} stage event " + f"({type(error).__name__}: {error}); the event is not staged, the " + "build continues.", + file=sys.stderr, + flush=True, + ) + + +_stage = stage + + +def staging_epoch_every(args: argparse.Namespace) -> int: + """The epoch stride that keeps the forwarded rows under the row budget. + + A dense run solves once; a size run solves the pool, up to ``budget_iters`` + full-length probes and the refit. The stride is at least + ``STAGING_EPOCH_EVERY`` and grows so at most ``STAGING_MAX_EPOCH_ROWS`` + epochs are forwarded, keeping ``calibration_progress.json`` and + ``events.ndjson`` under the contract's 5 MiB cap for any ``--epochs``. + """ + + solves = 1 if args.dataset_households is None else 2 + BUDGET_ITERS + total = int(args.epochs) * solves + return max(STAGING_EPOCH_EVERY, -(-total // STAGING_MAX_EPOCH_ROWS)) + + +_staging_epoch_every = staging_epoch_every + + +def thinned_epochs( + sink: Callable[[Mapping[str, Any]], None], *, every: int = STAGING_EPOCH_EVERY +) -> Callable[[dict[str, object]], None]: + """Forward every ``every``-th epoch and each phase's last epoch to ``sink``. + + The kernel flags probe epochs with ``budget_search: True``; the staging + contract records that field as an integer or null, so the flag becomes 1 + (the national command never runs a budget search and never met this). A + contract or content refusal from the telemetry is reported once and stops + the forwarding: the solve must never abort on its own progress report. + """ + + disabled = False + + def callback(event: dict[str, object]) -> None: + nonlocal disabled + if disabled or event.get("kind") != "calibration_epoch": + return + epoch = int(event["epoch"]) + epochs = int(event["epochs"]) + if epoch % every != 0 and epoch != epochs: + return + forwarded = dict(event) + budget_search = forwarded.get("budget_search") + if isinstance(budget_search, bool): + forwarded["budget_search"] = 1 if budget_search else None + try: + sink(forwarded) + except StagingContractError as error: + disabled = True + print( + "warning: staging telemetry refused a calibration progress row " + f"({type(error).__name__}); epoch progress is no longer forwarded, " + "the solve continues.", + file=sys.stderr, + flush=True, + ) + + return callback + + +_thinned_epochs = thinned_epochs + + +def gate_statuses(gate_report: Mapping[str, Any]) -> dict[str, str]: + return { + str(gate_id): str(entry.get("status")) + for gate_id, entry in gate_report.get("gates", {}).items() + if isinstance(entry, Mapping) + } + + +_gate_statuses = gate_statuses + + +def fail_staging_telemetry( + telemetry: StagingTelemetryV2 | None, error: BaseException +) -> None: + if telemetry is None or telemetry.status != "running": + return + try: + telemetry.fail(error) + telemetry.validate_local_bundle() + except Exception: + pass + + +_fail_staging_telemetry = fail_staging_telemetry + + +def finalize_staging_telemetry( + args: argparse.Namespace, telemetry: StagingTelemetryV2 | None +) -> None: + if telemetry is None: + return + try: + telemetry.complete(message="UK rowwise candidate staging run completed.") + except StagingContractError as error: + _warn_telemetry("could not close the staging run", error) + return + try: + if args.staging_read_back: + # Requested explicitly, so a failed read-back is the run's failure, + # as on the national command. + telemetry.verify_remote() + finally: + try: + telemetry.validate_local_bundle() + except StagingContractError as error: + _warn_telemetry("the local staging bundle does not validate", error) + + +_finalize_staging_telemetry = finalize_staging_telemetry + + +def _warn_telemetry(what: str, error: BaseException) -> None: + print( + f"warning: {what} ({type(error).__name__}: {error}); the build's own " + "evidence is unaffected.", + file=sys.stderr, + flush=True, + ) + + +def staging_delivery(telemetry: StagingTelemetryV2 | None) -> dict[str, Any]: + if telemetry is None: + return disabled_staging_delivery("--no-staging") + return telemetry.delivery_summary + + +_staging_delivery = staging_delivery + + +def stage_dataset( + args: argparse.Namespace, + *, + manifest: Mapping[str, Any], + output_paths: Mapping[str, Path], + run_id: str, + telemetry: StagingTelemetryV2 | None, +) -> dict[str, Any]: + """Stage the published bundle under ``staged//``; record, never raise. + + The bundle is every file the manifest registers as an output plus the + manifest and two sidecars, verified from disk against the manifest's own + digests. Nothing else in the run directory is eligible. + """ + + mode = staged_dataset_mode(args) + if mode == "disabled": + return disabled_staged_dataset( + "--no-staging" if args.no_staging else "--no-staged-dataset" + ) + repository = ( + None if mode == "local_only" else str(args.staged_dataset_repo_id).strip() + ) + stage(telemetry, "dataset_staging", "started", mode=mode, repository=repository) + statuses = gate_statuses(getattr(args, "_gate_report", {}) or {}) + bundle = StagedDatasetBundle.from_manifest( + output_paths["manifest"].parent, + run_id=run_id, + manifest_name=MANIFEST_FILENAME, + extra_summary={ + "dataset_households": manifest["parameters"]["dataset_households"], + "pool_rows": manifest["solve"]["pool_households"], + "realized_households": manifest["solve"]["n_households"], + "final_loss": manifest["solve"]["final_loss"], + "release_posture": manifest["release_posture"], + "gate_statuses": statuses, + }, + ) + telemetry_reference = ( + None + if telemetry is None + else { + "repository": telemetry.repo_id, + "prefix": telemetry.repo_run_prefix, + "mode": telemetry.delivery_mode, + } + ) + write_sidecars( + bundle, + repository=repository, + prefix=UK_STAGED_DATASET_PREFIX, + telemetry=telemetry_reference, + ) + if mode == "local_only": + delivery = local_only_staged_dataset(bundle, prefix=UK_STAGED_DATASET_PREFIX) + else: + print( + f"staging the dataset bundle to {repository} under " + f"{bundle.remote_prefix(UK_STAGED_DATASET_PREFIX)}...", + file=sys.stderr, + flush=True, + ) + storage = HuggingFaceDatasetStorage(repository, api=_hub_api()) + delivery = stage_bundle( + bundle, storage=storage, prefix=UK_STAGED_DATASET_PREFIX + ) + print(_staged_dataset_line(delivery), file=sys.stderr, flush=True) + stage( + telemetry, + "dataset_staging", + "completed", + status=delivery["status"], + repository=delivery["repository"], + revision=delivery["revision"], + error_code=delivery["error_code"], + file_count=len(delivery["files"]), + ) + add_staging_artifact( + telemetry, + "staged_dataset", + delivery, + artifact_kind="build_metadata", + classification="non_row_level", + ) + add_staging_artifact( + telemetry, + "fit_summary", + fit_summary( + manifest, + run_id=run_id, + gate_statuses=statuses, + staged_dataset=delivery, + ), + artifact_kind="aggregate_diagnostics", + classification="aggregate", + ) + return delivery + + +_stage_dataset = stage_dataset + + +def _staged_dataset_line(delivery: Mapping[str, Any]) -> str: + status = delivery["status"] + if status in ("uploaded", "already_staged"): + return ( + f"staged dataset: {status} at {delivery['repository']}/" + f"{delivery['prefix']} (revision {delivery['revision']})" + ) + if status == "failed": + return ( + f"staged dataset: failed ({delivery['error_code']}); the bundle and " + "its sidecars stay local and can be re-staged with " + "tools/stage_uk_rowwise_candidate.py" + ) + return ( + f"staged dataset: skipped ({delivery['mode']}); sidecars written beside " + "the bundle" + ) + + +def add_staging_artifact( + telemetry: StagingTelemetryV2 | None, + logical_name: str, + payload: Mapping[str, Any], + *, + artifact_kind: str, + classification: str, +) -> None: + """Attach a reviewed aggregate JSON artifact to the telemetry run. + + A content-policy refusal is reported and skipped: the telemetry is + best-effort and must never fail a finished build. + """ + + if telemetry is None: + return + with tempfile.TemporaryDirectory(prefix=".staging-artifact.") as scratch: + source = Path(scratch) / f"{logical_name}.json" + source.write_text(json_text(payload), encoding="utf-8") + try: + telemetry.add_artifact( + logical_name, + source, + artifact_kind=artifact_kind, + classification=classification, + ) + except StagingContractError as error: + print( + f"warning: staging artifact {logical_name} was refused by the " + f"content policy and is not staged: {error}", + file=sys.stderr, + flush=True, + ) + + +_add_staging_artifact = add_staging_artifact + + +def fit_summary( + manifest: Mapping[str, Any], + *, + run_id: str, + gate_statuses: Mapping[str, str], + staged_dataset: Mapping[str, Any], +) -> dict[str, Any]: + """Aggregate fit, gate and size evidence shaped for a reviewed artifact. + + The content policy admits JSON objects without arrays of objects, so the + per-family rows become mappings keyed by family and anything row-shaped + is dropped by :func:`_aggregate_only`. + """ + + solve = manifest["solve"] + fit = manifest.get("fit") or {} + size = solve.get("dataset_size") + parameters = manifest["parameters"] + return { + "schema_name": "microcosm.uk.rowwise-fit-summary", + "schema_version": 1, + "run_id": run_id, + "build_kind": manifest["build_kind"], + "releasable": manifest["releasable"], + "release_posture": _aggregate_only(manifest["release_posture"]), + "git_commit": manifest["git_commit"], + "git_dirty": manifest["git_dirty"], + "parameters": { + key: parameters.get(key) + for key in ( + "n_clones", + "dataset_households", + "seed", + "epochs", + "sample_fraction", + "release_candidate", + "skip_holdout", + "target_weight_rule", + ) + }, + "targets": { + "count": solve["n_targets"], + "by_kind": _aggregate_only(solve["n_targets_by_kind"]), + }, + "pool_rows": solve["pool_households"], + "realized_households": solve["n_households"], + "loss": { + "initial": solve["initial_loss"], + "final": solve["final_loss"], + "max_abs_relative_error": solve["max_abs_relative_error"], + "median_abs_relative_error": solve["median_abs_relative_error"], + }, + "fit_by_family": { + "local": _rows_by_key(fit.get("local_by_family"), key="family"), + "national": _rows_by_key(fit.get("national_by_family"), key="family"), + }, + "weakest_areas_by_fit": _aggregate_only(fit.get("weakest_areas_by_fit")), + "rotated_holdout": _aggregate_only(fit.get("rotated_holdout")), + "gates": dict(gate_statuses), + "failing_gate_ids": list(manifest.get("failing_gate_ids", [])), + "blocking_failure_count": len(manifest.get("blocking_failures", [])), + # The receipt's per-row arrays (pool_row_indices, inclusion + # probabilities) live in dataset_size_selection.csv and would push the + # artifact past the 5 MiB cap on a real run. + "dataset_size": None + if size is None + else _aggregate_only( + { + key: value + for key, value in size.items() + if key not in ("pool_row_indices", "inclusion_probabilities") + } + ), + "staged_dataset": { + key: staged_dataset[key] + for key in ("repository", "prefix", "revision", "status", "error_code") + }, + } + + +_fit_summary = fit_summary + + +def _rows_by_key(rows: Any, *, key: str) -> dict[str, Any]: + """Turn a list of row mappings into a mapping keyed by ``row[key]``.""" + + if not isinstance(rows, list): + return {} + keyed: dict[str, Any] = {} + for row in rows: + if not isinstance(row, Mapping) or key not in row: + continue + keyed[str(row[key])] = _aggregate_only( + {name: value for name, value in row.items() if name != key} + ) + return keyed + + +def _aggregate_only(value: Any) -> Any: + """Drop row-shaped data (lists holding mappings) recursively.""" + + if isinstance(value, Mapping): + kept = {} + for name, item in value.items(): + cleaned = _aggregate_only(item) + if cleaned is not _DROPPED: + kept[str(name)] = cleaned + return kept + if isinstance(value, (list, tuple)): + if any(isinstance(item, Mapping) for item in value): + return _DROPPED + return [ + item + for item in (_aggregate_only(entry) for entry in value) + if item is not _DROPPED + ] + return value + + +_DROPPED = object() + + +def replace_manifest(path: Path, manifest: Mapping[str, Any]) -> None: + """Rewrite the published manifest atomically with appended evidence.""" + + handle, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary_path = Path(temporary) + try: + with open(handle, "w", encoding="utf-8") as stream: + stream.write(json_text(manifest)) + temporary_path.replace(path) + except BaseException: + temporary_path.unlink(missing_ok=True) + raise + + +_replace_manifest = replace_manifest + + +def publish_staged_files( + staged: Mapping[str, Path], + output_paths: Mapping[str, Path], +) -> None: + out_dir = output_paths["manifest"].parent + created_out_dir = not out_dir.exists() + out_dir.mkdir(parents=True, exist_ok=True) + publish_order = ( + "dataset", + "diagnostics", + "support", + "past_cap", + "calibration_diagnostics", + "local_registry", + "dense_reference", + "selection", + "manifest", + ) + published: list[Path] = [] + succeeded = False + try: + for key in publish_order: + if key in SIZE_RUN_ONLY_OUTPUTS and not staged[key].exists(): + continue + destination = output_paths[key] + if destination.exists(): + raise FileExistsError( + "candidate output appeared during publication; refusing " + f"to overwrite {destination}." + ) + staged[key].replace(destination) + published.append(destination) + succeeded = True + finally: + if not succeeded: + for path in reversed(published): + path.unlink(missing_ok=True) + if created_out_dir: + try: + out_dir.rmdir() + except OSError: + pass + + +_publish_staged_files = publish_staged_files diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/size_checkpoint.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/size_checkpoint.py index f588be885..e6127b8bc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/size_checkpoint.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/size_checkpoint.py @@ -15,6 +15,7 @@ from __future__ import annotations +import argparse import hashlib import json import math @@ -27,6 +28,7 @@ import numpy as np from microcosm.build.uk_runtime.dataset_size import UKSizeSelection +from microcosm.build.uk_runtime.rowwise_cli import doctrine_bounds, posture_of from microcosm.calibrate import ( CalibrationProblem, CalibrationResult, @@ -55,6 +57,54 @@ ) +def uk_size_checkpoint_identity( + args: argparse.Namespace, + *, + pins: Mapping[str, Mapping[str, object]], + source_year: int, +) -> dict[str, object]: + """Everything a size checkpoint must share with the run that resumes it. + + The pool (spine, ladder, clones, seed, sampling), the target surface + (ledger digests, year, rule, engine blocks) and the solve settings the + checkpointed dense solve and search were made with. The draw threshold is + deliberately absent: re-drawing at another threshold is the point. Both + UK drivers write and resume with this one mapping (moved here from + ``tools/build_uk_rowwise_candidate.py``), so a checkpoint cut by either + resumes on the other. + """ + posture = posture_of(args) + return { + "release_role": posture.role, + "dataset_pin": dict(pins["dataset"]), + "ladder_pin": dict(pins["ladder"]), + "ledger_facts_sha256": args.ledger_facts_sha256, + "ledger_manifest_sha256": args.ledger_manifest_sha256, + "seed": int(args.seed), + "selection_seed": int( + args.seed if args.selection_seed is None else args.selection_seed + ), + "n_clones": None if args.n_clones is None else int(args.n_clones), + "dataset_households": args.dataset_households, + "epochs": int(args.epochs), + "learning_rate": float(args.learning_rate), + "sample_fraction": float(args.sample_fraction), + "sample_seed": int(args.sample_seed), + "source_year": int(source_year), + "source_lineage_modulus": args.source_lineage_modulus, + "calibration_year": getattr(args, "_calibration_year", None), + "target_weight_rule": args.target_weight_rule, + "engine_blocks": int(args.engine_blocks), + "measure_exclusions": ( + None if args.measure_exclusions is None else str(args.measure_exclusions) + ), + # The solve doctrine the dense solve and the search run under: a + # resume after a doctrine change must refuse, not run under the old + # bound while the manifest declares the new one. + "doctrine": doctrine_bounds(posture), + } + + @dataclass(frozen=True) class UKSizeCheckpointRestore: """A checkpoint's dense solve and selection, rebuilt on the re-derived pool.""" diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_dense_release_assembler.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_dense_release_assembler.py index 9f384585e..dd19d03d7 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_dense_release_assembler.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_dense_release_assembler.py @@ -850,3 +850,124 @@ def test_assembler_refuses_a_national_role_manifest( str(tmp_path / "releases"), ] ) + + +def _resigned_report(release_id: str) -> dict: + report = _signed_report() + report["release_id"] = release_id + report["attestation"]["release_id"] = release_id + report["attestation"]["signature"] = None + report["attestation"]["signature"] = hmac.new( + KEY_BYTES, _canonical_json_bytes(report), hashlib.sha256 + ).hexdigest() + return report + + +def test_assembler_stages_a_graph_built_dense_candidate( + tmp_path: Path, monkeypatch, capsys +) -> None: + """The graph driver's bundle assembles like a rowwise-tool bundle. + + ``main`` runs the synthetic graph with local-only staging, so the manifest + carries the staging receipt, the staged-dataset sidecars and the Logbook + row the assembler hash-joins. The signed local battery report and the + incumbent-surface companions are supplied as the certification and + scoring steps supply them. + """ + + pytest.importorskip("tables") + from microcosm.build.logbook import load_spool_rows + from microcosm.build.uk_runtime import full_build_cli as cli + from test_support.microcosm_build.uk_full_build_cli import ( + STEM, + graph_dense_bundle, + ) + + monkeypatch.setenv("MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY", KEY) + monkeypatch.setattr(cli, "git_commit", lambda: "b" * 40) + monkeypatch.setattr(cli, "git_dirty", lambda: False) + candidate = graph_dense_bundle( + tmp_path, + monkeypatch, + "--release-candidate", + staging="--staging-local-only", + ) + # The graph driver prints the finished manifest to stdout, as the tool does. + printed = json.loads(capsys.readouterr().out) + assert printed["release_role"] == "dense" + spine = tmp_path / "spine.h5" + row = load_spool_rows(candidate / "logbook-spool")[0] + manifest_path = candidate / "rowwise_candidate_manifest.json" + manifest = json.loads(manifest_path.read_text()) + assert manifest["staging_delivery"]["mode"] == "local_only" + assert manifest["staged_dataset"]["status"] == "skipped" + report_path = candidate / f"{STEM}.local_gates.json" + report_path.write_text(json.dumps(_resigned_report(row.build_id))) + manifest["outputs"]["local_gate_report"]["sha256"] = _sha(report_path) + manifest["outputs"]["local_gate_report"]["bytes"] = report_path.stat().st_size + manifest_path.write_text(json.dumps(manifest)) + diagnostics_path = Path(manifest["outputs"]["calibration_diagnostics"]["path"]) + (candidate / "score_vs_incumbent.json").write_text( + json.dumps( + { + "candidate_fitted_surface_loss": 0.0146, + "incumbent_fitted_surface_loss": 0.181, + "rows_compared": 2, + "incumbent_missing_areas": {}, + "artifacts": { + "candidate_diagnostics": {"sha256": _sha(diagnostics_path)}, + "incumbent_household_metrics": {"sha256": "6" * 64}, + "incumbent_wide_weights": {"sha256": "7" * 64}, + }, + } + ) + ) + incumbent_manifest = tmp_path / "incumbent_local_surface_manifest.json" + incumbent_manifest.write_text( + json.dumps( + { + "period": 2025, + "households": 52846, + "inputs": {"incumbent_h5": {"sha256": "5" * 64}}, + "outputs": { + "metrics": {"sha256": "6" * 64}, + "weights": {"sha256": "7" * 64}, + }, + } + ) + ) + chronicle = manifest["identity"]["targets"]["chronicle"] + evaluation = { + "schema_version": 2, + "kind": "uk_incumbent_surface_evaluation", + "measure_resolution": {"blocks": 1}, + **_surface_rows(), + "identity": { + "candidate_dataset_sha256": _sha(candidate / f"{STEM}.h5"), + "candidate_manifest_sha256": _sha(manifest_path), + "candidate_diagnostics_sha256": _sha(diagnostics_path), + "ledger_facts_sha256": chronicle["facts_sha256"], + "ledger_manifest_sha256": chronicle["manifest_sha256"], + "incumbent_manifest_sha256": _sha(incumbent_manifest), + "incumbent_metrics_sha256": "6" * 64, + "incumbent_weights_sha256": "7" * 64, + }, + } + evaluation["summary"] = dc.uk_incumbent_surface_assessment(evaluation) + (candidate / "incumbent_surface_evaluation.json").write_text(json.dumps(evaluation)) + assembler = _load("assemble_uk_dense_release_dir") + out = tmp_path / "releases" + assert ( + assembler.main(_assemble_args(candidate, spine, incumbent_manifest, out)) == 0 + ) + summary = json.loads(capsys.readouterr().out) + release_dir = out / UK_DENSE_RELEASE_ID + assert summary["release_id"] == UK_DENSE_RELEASE_ID + for name in dc._UK_DENSE_REQUIRED_RELEASE_FILES: + assert (release_dir / name).is_file(), name + build_manifest = json.loads((release_dir / "build_manifest.json").read_text()) + assert build_manifest["staging"]["mode"] == "local_only" + coverage = json.loads((release_dir / "uk_source_coverage.json").read_text()) + assert coverage["doctrine"]["clone_count"] == 15 + assert coverage["holdout"]["n_folds"] == 5 + dc.validate_release_dir(release_dir) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py index 2052c6858..67e655c27 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py @@ -1,69 +1,7 @@ """The canonical CLI restores declared files and preserves failure/scope semantics.""" -import hashlib -import json -from dataclasses import replace -from pathlib import Path - -import pytest - -from microcosm.build.gate_battery import ( - GateOutcome, - GatePhaseReport, - GateStatus, - gate_phase_report_payload, -) -from microcosm.build.gates import GateResult -from microcosm.build.uk_runtime import full_build_cli as cli -from microcosm.build.uk_runtime.full_certification import FULL_CERTIFICATION_TYPE -from microcosm.build.uk_runtime.full_gates import ( - classify_full_gate_outcomes, - uk_full_gate_manifest, -) -from microcosm.build.uk_runtime.graph_build import UKFullBuildConfig, UKFullGraph -from microcosm.build.uk_runtime.graph_calibration import UKCalibrationNodes -from microcosm.build.uk_runtime.graph_targets import TARGET_SELECTION_TYPE -from microcosm.build.uk_runtime.graph_terminal import ( - FULL_DIAGNOSTICS_CSV_TYPE, - FULL_DIAGNOSTICS_TYPE, - FULL_GATE_REPORT_TYPE, - FULL_HOLDOUT_TYPE, - FULL_SUPPORT_CSV_TYPE, - add_uk_export_preparation, - register_uk_terminal_kernels, -) -from microcosm.graph import ( - ArtifactInput, - ArtifactOutput, - Capabilities, - Determinism, - Graph, - KernelBase, - KernelRegistry, - KernelResult, - Node, - Owned, - SourceRef, - StructuralDelta, -) -from microcosm.graph.canonical import canonical_json -from test_support.microcosm_build.uk_graph_terminal import _frame - - -def arguments(tmp_path, *extra): - return cli.parse_args( - [ - "--input-h5", - str(tmp_path / "spine.h5"), - "--ladder", - str(tmp_path / "ladder.npz"), - "--ledger-facts", - str(tmp_path / "ledger"), - "--out", - str(tmp_path / "out"), - *extra, - ] - ) +# ruff: noqa: F403, F405 +from test_support.microcosm_build.uk_full_build_cli import * def test_every_scope_and_size_control_keeps_default_all(tmp_path): @@ -82,256 +20,165 @@ def test_every_scope_and_size_control_keeps_default_all(tmp_path): arguments(tmp_path, "--target-geographies", "national") -def test_source_sampling_cannot_be_reapplied_as_pool_sampling(): - config = UKFullBuildConfig(calibration_year=2025, source_sample_fraction=0.1) - assert config.sample_fraction == 1.0 - assert config.effective_sample_fraction == 0.1 - with pytest.raises(ValueError, match="second time"): - replace(config, sample_fraction=0.1) - - -def gate_payload(phase, failed=None): - selection = { - "schema": "microcosm.calibrate.target-selection.v1", - "selector": {"geography_levels": None, "explicit": False}, - "included": [ - {"name": "count", "period": 2025, "geography_level": "country"}, - {"name": "local", "period": 2025, "geography_level": "constituency"}, - ], - "excluded": [], - } - gates = uk_full_gate_manifest(selection) - report = GatePhaseReport( - phase, - tuple( - GateOutcome( - entry, - GateStatus.FAILED if entry.id == failed else GateStatus.PASSED, - GateResult( - name=entry.id, - passed=entry.id != failed, - details={}, - failures=("synthetic failure",) if entry.id == failed else (), - ), - ) - for entry in gates.gates - if entry.phase == phase - ), - ) - return canonical_json( - { - "schema_version": 1, - "kind": "uk_full_gate_report", - "selection_receipt": selection, - "sample_fraction": 1.0, - "release_candidate": False, - "report": gate_phase_report_payload(report, gates=gates), - "enforcement": classify_full_gate_outcomes( - report, sample_fraction=1.0, release_candidate=False - ), - } - ) - - -class Fixture(KernelBase): - ref = "uk.test.cli-frame@1" - capabilities = Capabilities( - Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE +def test_release_role_is_required_and_supplies_the_dense_posture(tmp_path): + with pytest.raises(SystemExit): + cli.parse_args(["--input-h5", "x.h5", "--out", str(tmp_path)]) + args = arguments(tmp_path) + posture = cli.UK_ROWWISE_DENSE_POSTURE + assert args._posture is posture + assert (args.n_clones, args.seed, args.epochs, args.learning_rate) == ( + posture.clone_count, + posture.seed, + posture.epochs, + posture.learning_rate, ) + assert (args.n_clones, args.epochs, args.learning_rate) == (15, 1500, 0.15) + assert args.target_weight_rule == "grain_equal" + assert args.expected_constituency_vintage == "2024_pcon" + assert args.sample_seed == 578 + assert args.staging_upload_interval_seconds == 300.0 + assert args._explicit_arguments == frozenset() + cli.validate_cli_args(args) + explicit = arguments(tmp_path, "--epochs", "8", "--n-clones", "2") + assert (explicit.epochs, explicit.n_clones) == (8, 2) + assert explicit._explicit_arguments == frozenset({"epochs", "n_clones"}) - def implementation_hash(self): - return hashlib.sha256(self.ref.encode()).hexdigest() - - def run(self, context): - return KernelResult(frame=_frame()) - - -class Evidence(KernelBase): - ref = "uk.test.cli-evidence@1" - capabilities = Capabilities(Determinism.DETERMINISTIC) - def implementation_hash(self): - return hashlib.sha256(self.ref.encode()).hexdigest() - - def run(self, context): - phase = context.params["phase"] - artifacts = {"gate_report": gate_payload(phase, context.params.get("failed"))} - if phase == "terminal": - artifacts.update( - calibration_diagnostics=b'{"fixture":true}', - target_diagnostics_csv=b"name,actual\ncount,100\n", - area_support_csv=b"area,households\nfixture,2\n", - ) - return KernelResult(artifacts=artifacts) - - -class Holdout(Evidence): - ref = "uk.test.cli-holdout@1" - - def run(self, context): - return KernelResult( - artifacts={"holdout": b'{"fixture":true}', "selection": b'{"fixture":true}'} +@pytest.mark.parametrize( + ("extra", "needle"), + [ + (["--target-loss-cap", "5"], "--target-loss-cap"), + (["--allow-unpinned-feed"], "--allow-unpinned-feed"), + (["--incumbent-h5", "incumbent.h5"], "--incumbent-h5"), + (["--target-weight-rule", "family_equal"], "--target-weight-rule family_equal"), + ], +) +def test_dense_role_refuses_the_national_knobs(tmp_path, extra, needle): + args = arguments(tmp_path, *extra) + with pytest.raises(ValueError, match="--release-role dense refuses") as excinfo: + cli.validate_cli_args(args) + assert needle in str(excinfo.value) + + +def test_dense_role_requires_the_ladder_and_the_pins(tmp_path): + argv = [ + "--release-role", + "dense", + "--input-h5", + str(tmp_path / "spine.h5"), + "--out", + str(tmp_path / "out"), + "--ledger-facts", + str(tmp_path / "ledger"), + "--ledger-facts-sha256", + PIN, + "--ledger-manifest-sha256", + PIN, + ] + with pytest.raises(ValueError, match="requires --ladder"): + cli.validate_cli_args(cli.parse_args(argv)) + with pytest.raises(ValueError, match="--input-sha256 and --ladder-sha256"): + cli.validate_cli_args( + cli.parse_args([*argv, "--ladder", str(tmp_path / "ladder.npz")]) ) - - -class Certification(Evidence): - ref = "uk.test.cli-certification@1" - - def run(self, context): - return KernelResult( - artifacts={ - "certification_readiness": b'{"fixture":true,"release_authorized":false}' - } + # A spine-request build has no input H5 to pin: only the ladder pin is due. + request = [ + "--release-role", + "dense", + "--spine-request", + str(tmp_path / "request.json"), + "--ladder", + str(tmp_path / "ladder.npz"), + "--ladder-sha256", + PIN, + "--out", + str(tmp_path / "out"), + "--ledger-facts", + str(tmp_path / "ledger"), + "--ledger-facts-sha256", + PIN, + "--ledger-manifest-sha256", + PIN, + ] + cli.validate_cli_args(cli.parse_args(request)) + + +def test_national_role_is_validated_then_refused_until_phase_four(tmp_path, capsys): + argv = [ + "--release-role", + "national", + "--input-h5", + str(tmp_path / "spine.h5"), + "--input-sha256", + PIN, + "--out", + str(tmp_path / "out"), + "--ledger-facts", + str(tmp_path / "ledger"), + "--ledger-facts-sha256", + PIN, + "--ledger-manifest-sha256", + PIN, + "--no-staging", + ] + national = cli.parse_args(argv) + assert national._posture is cli.uk_rowwise_posture("national") + assert national.n_clones is None + assert (national.seed, national.epochs, national.learning_rate) == (0, 1500, 0.02) + cli.validate_cli_args(national) + with pytest.raises(SystemExit) as exit_info: + cli.main(argv) + assert exit_info.value.code == 2 + assert "next commit" in capsys.readouterr().err + # The dense refusal table still applies before the national refusal. + with pytest.raises(ValueError, match="--release-role national refuses"): + cli.main([*argv, "--ladder", str(tmp_path / "ladder.npz")]) + + +def test_candidate_clone_counts_are_dry_run_only(tmp_path): + with pytest.raises(ValueError, match="only with --dry-run"): + cli.main( + [ + "--release-role", + "dense", + "--input-h5", + str(tmp_path / "missing.h5"), + "--input-sha256", + PIN, + "--ladder", + str(tmp_path / "missing.npz"), + "--ladder-sha256", + PIN, + "--ledger-facts", + str(tmp_path / "ledger"), + "--ledger-facts-sha256", + PIN, + "--ledger-manifest-sha256", + PIN, + "--out", + str(tmp_path / "out"), + "--candidate-clone-counts", + "1,2,4", + "--no-staging", + ] ) -@pytest.fixture(autouse=True) -def certification_service_fixture(monkeypatch): - # Scientific certification validation has its own graph-artifact tests. - # This suite tests the filesystem/execution service with synthetic evidence. - def append(graph, *, population, **kwargs): - return replace( - graph, - nodes=( - *graph.nodes, - Node( - "uk.full.certification", - Certification.ref, - population=population, - artifact_outputs=( - ArtifactOutput( - "certification_readiness", FULL_CERTIFICATION_TYPE - ), - ), - ), - ), - ) - - monkeypatch.setattr(cli, "append_uk_full_certification_node", append) +def test_source_sampling_cannot_be_reapplied_as_pool_sampling(): + config = UKFullBuildConfig(calibration_year=2025, source_sample_fraction=0.1) + assert config.sample_fraction == 1.0 + assert config.effective_sample_fraction == 0.1 + with pytest.raises(ValueError, match="second time"): + replace(config, sample_fraction=0.1) -def prepared(tmp_path, failed=None): - frame = _frame() - fixture = tmp_path / "fixture.txt" - fixture.write_text("constant source") - identifiers = { - "person_id", - "person_household_id", - "person_benunit_id", - "household_id", - "benunit_id", - } - root = Node( - "uk.full.calibrated", - Fixture.ref, - structural=StructuralDelta.CREATE, - sources=("fixture",), - outputs=tuple( - Owned( - e, - str(c), - "string" - if frame.table(e)[c].dtype.kind in "OUS" - else str(frame.table(e)[c].dtype), - ) - for e in frame.entities - for c in frame.table(e).columns - if c not in identifiers - ), - ) - nodes = [ - root, - Node( - "uk.full.gates.preflight", - Evidence.ref, - population=root.id, - params={"phase": "preflight", "failed": failed}, - artifact_outputs=(ArtifactOutput("gate_report", FULL_GATE_REPORT_TYPE),), - ), - Node( - "uk.full.gates.calibrated", - Evidence.ref, - population=root.id, - params={"phase": "terminal", "failed": failed}, - artifact_outputs=( - ArtifactOutput("gate_report", FULL_GATE_REPORT_TYPE), - ArtifactOutput("calibration_diagnostics", FULL_DIAGNOSTICS_TYPE), - ArtifactOutput("target_diagnostics_csv", FULL_DIAGNOSTICS_CSV_TYPE), - ArtifactOutput("area_support_csv", FULL_SUPPORT_CSV_TYPE), - ), - ), - Node( - "uk.full.holdout", - Holdout.ref, - population=root.id, - artifact_outputs=( - ArtifactOutput("holdout", FULL_HOLDOUT_TYPE), - ArtifactOutput("selection", TARGET_SELECTION_TYPE), - ), - ), - ] - # CLI materialization consumes the public target-selection endpoint. - nodes.append( - Node( - "uk.full.target_selection", - Holdout.ref, - population=root.id, - artifact_outputs=( - ArtifactOutput("holdout", FULL_HOLDOUT_TYPE), - ArtifactOutput("selection", TARGET_SELECTION_TYPE), - ), - ) - ) - nodes = [ - replace( - node, - artifact_inputs=( - ArtifactInput( - "preflight", - "uk.full.gates.preflight", - "gate_report", - FULL_GATE_REPORT_TYPE, - ), - ArtifactInput( - "holdout", "uk.full.holdout", "holdout", FULL_HOLDOUT_TYPE - ), - ArtifactInput( - "selection", - "uk.full.target_selection", - "selection", - TARGET_SELECTION_TYPE, - ), - ), - ) - if node.id == "uk.full.gates.calibrated" - else node - for node in nodes - ] - graph = Graph("uk", (SourceRef("fixture", "raw-bytes-v1"),), tuple(nodes)) - graph = add_uk_export_preparation( - graph, - population=root.id, - bindings={"target_scope": "all"}, - artifact_inputs=( - ArtifactInput( - "gates", - "uk.full.gates.calibrated", - "gate_report", - FULL_GATE_REPORT_TYPE, - ), - ), - ) - calibration = UKCalibrationNodes( - (), root.id, "unused", "unused", "unused", None, "unused" - ) - full = UKFullGraph(graph, calibration, UKFullBuildConfig(calibration_year=2025)) - kernels = KernelRegistry() - for kernel in (Fixture(), Evidence(), Holdout(), Certification()): - kernels.register(kernel) - register_uk_terminal_kernels(kernels) - return cli.PreparedUKFullBuild( - full, kernels, {"fixture": fixture}, {"target_scope": "all"} +def test_households_only_binds_the_census_family_on_the_selection_node(): + config = UKFullBuildConfig( + calibration_year=2025, target_families=("census_households/constituency",) ) + assert config.target_families == ("census_households/constituency",) + with pytest.raises(ValueError, match="target families"): + UKFullBuildConfig(calibration_year=2025, target_families=()) def test_cli_cold_and_required_replay_recreate_dataset_and_sidecars( @@ -345,7 +192,8 @@ def test_cli_cold_and_required_replay_recreate_dataset_and_sidecars( expected = json.loads((out / "build.json").read_text()) assert expected["readback_passed"] is True assert expected["release_authorized"] is False - assert (out / "microcosm_uk_2025.targets.csv").read_text().startswith("name,actual") + assert expected["release_role"] == "dense" + assert (out / f"{STEM}.targets.csv").read_text().startswith("name,target_name") for path in out.iterdir(): if path.is_file(): path.unlink() @@ -355,14 +203,119 @@ def forbidden(*args): "Required replay repeated a completed numerical/evidence node" ) - for kernel in (Fixture, Evidence, Holdout): + for kernel in (Fixture, Evidence, Holdout, Targets, Population): monkeypatch.setattr(kernel, "run", forbidden) args.resume = "require" assert cli.execute_full_build(prepared(tmp_path), args) == 0 actual = json.loads((out / "build.json").read_text()) assert actual["content_sha256"] == expected["content_sha256"] - assert (out / "microcosm_uk_2025.h5").is_file() - assert (out / "microcosm_uk_2025.holdout.json").is_file() + # The output names come from the dense posture and the FRS vintage. + assert (out / f"{STEM}.h5").is_file() + assert (out / f"{STEM}.holdout.json").is_file() + assert (out / f"{STEM}.local_gates.json").is_file() + assert not (out / "microcosm_uk_2025.h5").exists() + + +def test_dense_run_projects_the_rowwise_candidate_manifest(tmp_path): + pytest.importorskip("tables") + args = arguments(tmp_path, "--release-candidate") + assert cli.execute_full_build(prepared(tmp_path), args) == 0 + out = args.out + manifest = json.loads((out / "rowwise_candidate_manifest.json").read_text()) + assert manifest["schema_version"] == 4 + assert manifest["build_kind"] == "uk_rowwise_calibrated_candidate" + assert manifest["release_role"] == "dense" + assert manifest["release_id"] == UK_DENSE_RELEASE_ID + assert manifest["parameters"]["release_role"] == "dense" + assert manifest["parameters"]["release_candidate"] is True + assert ( + manifest["parameters"]["doctrine"] + == cli.UK_ROWWISE_DENSE_POSTURE.doctrine_bounds() + ) + assert (manifest["parameters"]["n_clones"], manifest["parameters"]["epochs"]) == ( + 15, + 1500, + ) + for key in ( + "solve", + "fit", + "weights", + "cross_grain", + "census_household_uprating", + "outputs", + "releasable", + "identity", + "bound_target_families", + "binding_adjudications", + "measure_exclusions", + "blocked_at_f100", + "blocking_failures", + "diagnostic_failures", + "release_gate_failures_not_enforced", + "failing_gate_ids", + "sampling", + "rung_surface", + "support", + "geography", + "vintages", + "graph", + ): + assert key in manifest, key + assert set(manifest["identity"]) >= { + "code", + "ladder", + "runtime", + "spine", + "targets", + } + assert manifest["identity"]["spine"]["pin_verified"] is True + assert manifest["identity"]["ladder"]["pin_verified"] is True + assert manifest["identity"]["targets"]["paired_ladder_sha256"] == PIN + assert manifest["solve"]["n_targets_by_kind"] == { + "local": 0, + "ladder": 1, + "national": 0, + } + assert manifest["solve"]["measure_resolution"] == {"blocks": 1} + assert manifest["solve"]["target_weight_rule_override"] == {} + assert manifest["solve"]["n_households"] == 2 + assert manifest["solve"]["pool_households"] == 2 + assert manifest["fit"]["rotated_holdout"]["n_folds"] == 5 + assert manifest["fit"]["local_by_family"][0]["family"] == "census_households" + assert manifest["weights"]["realized_max_weight_ratio_vs_design"] == 1.0 + assert manifest["weights"]["household_weight_kind"] == "calibrated" + assert manifest["census_household_uprating"]["applied"] is True + assert manifest["releasable"] is True and manifest["blocked_at_f100"] is False + assert manifest["release_posture"]["release_blocking_gates_passed"] is True + dataset = manifest["outputs"]["dataset"] + assert Path(dataset["path"]) == out / f"{STEM}.h5" + assert ( + dataset["sha256"] + == hashlib.sha256((out / f"{STEM}.h5").read_bytes()).hexdigest() + ) + assert Path(manifest["outputs"]["local_gate_report"]["path"]).name == ( + f"{STEM}.local_gates.json" + ) + assert manifest["graph"]["epoch_rows"] == "dense_solve_only" + assert "uk.full.problem" in manifest["graph"]["artifacts"] + # The staged bundle is every registered output plus the manifest; each + # registered file is on disk beside it with its recorded digest. + for entry in manifest["outputs"].values(): + path = Path(entry["path"]) + assert path.parent == out and path.is_file() + assert hashlib.sha256(path.read_bytes()).hexdigest() == entry["sha256"] + + +def test_blocked_gate_projects_an_unreleasable_manifest(tmp_path): + pytest.importorskip("tables") + args = arguments(tmp_path) + build = prepared(tmp_path, "uk_local_target_fit") + assert cli.execute_full_build(build, args) == 1 + manifest = json.loads((args.out / "rowwise_candidate_manifest.json").read_text()) + assert manifest["releasable"] is False + assert manifest["blocked_at_f100"] is True + assert manifest["blocking_failures"] == ["[uk_local_target_fit] synthetic failure"] + assert manifest["failing_gate_ids"] == ["uk_local_target_fit"] @pytest.mark.parametrize( @@ -379,7 +332,7 @@ def test_cli_retains_failed_evidence_and_correct_status(tmp_path, failure, expor build = prepared(tmp_path, failure) assert cli.execute_full_build(build, args) == 1 assert (args.out / "uk.full.gates.calibrated.gate_report.json").is_file() - assert (args.out / "microcosm_uk_2025.h5").exists() == exported + assert (args.out / f"{STEM}.h5").exists() == exported def test_dry_run_has_no_files_or_kernel_execution(tmp_path, monkeypatch, capsys): @@ -400,11 +353,62 @@ def test_rejected_output_inside_source_never_writes_failure_sidecar( build = prepared(tmp_path) build = replace(build, sources={"fixture": tmp_path}) monkeypatch.setattr(cli, "parse_args", lambda argv: args) - monkeypatch.setattr(cli, "prepare_full_build", lambda args: build) + monkeypatch.setattr(cli, "prepare_full_build", lambda args, **kwargs: build) assert cli.main([]) == 1 assert not args.out.exists() +def test_main_runs_the_logbook_envelope_around_a_dense_build(tmp_path, monkeypatch): + pytest.importorskip("tables") + from microcosm.build.logbook import load_spool_rows + + monkeypatch.delenv("POPULACE_LOGBOOK_PREV_ROW_DIGEST", raising=False) + args = arguments(tmp_path, "--seed", "7") + build = prepared(tmp_path) + monkeypatch.setattr(cli, "parse_args", lambda argv: args) + monkeypatch.setattr(cli, "prepare_full_build", lambda args, **kwargs: build) + assert cli.main([]) == 0 + rows = load_spool_rows(args.out / "logbook-spool") + assert len(rows) == 1 + row = rows[0] + assert row.pipeline == "uk-local-candidate" + assert row.build_id.startswith("uk-local-candidate-f100-s7-") + assert row.rung == "f100" and row.seed == 7 + assert row.disposition == "iterating" + assert row.artifact_location.endswith(f"{STEM}.h5") + assert "published" in row.phases_reached + assert row.gate_verdicts and all( + item["verdict"] == "passed" and ".local_gates.json#/gates/" in item["receipt"] + for item in row.gate_verdicts.values() + ) + manifest = json.loads((args.out / "rowwise_candidate_manifest.json").read_text()) + assert manifest["staging_delivery"]["enabled"] is False + assert manifest["staged_dataset"]["status"] == "skipped" + + +def test_main_records_a_failed_attempt_when_the_build_raises(tmp_path, monkeypatch): + from microcosm.build.logbook import load_spool_rows + + monkeypatch.delenv("POPULACE_LOGBOOK_PREV_ROW_DIGEST", raising=False) + args = arguments(tmp_path) + build = prepared(tmp_path) + monkeypatch.setattr(cli, "parse_args", lambda argv: args) + monkeypatch.setattr(cli, "prepare_full_build", lambda args, **kwargs: build) + monkeypatch.setattr( + cli, + "run_graph", + lambda *a, **k: (_ for _ in ()).throw(RuntimeError("synthetic refusal")), + ) + assert cli.main([]) == 1 + failure = json.loads((args.out / "failure.json").read_text()) + assert failure["message"] == "synthetic refusal" + rows = load_spool_rows(args.out / "logbook-spool") + assert len(rows) == 1 and rows[0].disposition == "failed" + receipts = list((args.out / "logbook-receipts").rglob("error.json")) + assert len(receipts) == 1 + assert json.loads(receipts[0].read_text())["error_type"].endswith("RuntimeError") + + @pytest.mark.parametrize("resume", ["auto", "require"]) def test_source_phase_evidence_survives_later_exception(tmp_path, monkeypatch, resume): args = arguments(tmp_path) @@ -447,7 +451,7 @@ def refuse_transferred(compiled, **kwargs): monkeypatch.setattr(cli, "run_graph", refuse_transferred) monkeypatch.setattr(cli, "parse_args", lambda argv: args) - monkeypatch.setattr(cli, "prepare_full_build", lambda args: build) + monkeypatch.setattr(cli, "prepare_full_build", lambda args, **kwargs: build) assert cli.main([]) == 1 assert (args.out / "build.json").read_bytes() == previous failure = json.loads((args.out / "failure.json").read_bytes()) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_preparation.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_preparation.py index 80e3766c3..58a8023d2 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_preparation.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_preparation.py @@ -37,6 +37,8 @@ def checkpoint_request(tmp_path, toy_ladder, monkeypatch): ) args = cli.parse_args( [ + "--release-role", + "dense", "--input-h5", str(path), "--input-sidecar", diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_local_release_preflight.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_local_release_preflight.py index cc57fa9e5..6dc4720ae 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_local_release_preflight.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_local_release_preflight.py @@ -358,3 +358,56 @@ def test_preflight_does_not_infer_tenure_success_from_ladder_uprating(tmp_path): "A17" in failure and "skipped attempted" in failure for failure in failures ) assert not any("A15" in failure for failure in failures) + + +def _resign(report: dict, *, release_id: str) -> dict: + """The fixture report re-signed for another attempt id.""" + + import hashlib + import hmac + + from microcosm.build.gate_battery import _canonical_json_bytes + + signed = json.loads(json.dumps(report)) + signed["release_id"] = release_id + signed["attestation"]["release_id"] = release_id + signed["attestation"]["signature"] = None + signed["attestation"]["signature"] = hmac.new( + base64.b64decode(KEY), _canonical_json_bytes(signed), hashlib.sha256 + ).hexdigest() + return signed + + +def test_graph_built_manifest_passes_the_candidate_preflight( + tmp_path: Path, monkeypatch +) -> None: + """The graph driver's projected manifest is the schema the pre-flight reads. + + The graph's own gate report is the full-build battery document, not the + signed local battery report; the signed report is supplied here as the + certification step will, under the graph's ``*.local_gates.json`` name. + """ + + pytest.importorskip("tables") + from microcosm.build.logbook import load_spool_rows + from test_support.microcosm_build.uk_full_build_cli import ( + STEM, + graph_dense_bundle, + ) + + out = graph_dense_bundle(tmp_path, monkeypatch, "--release-candidate") + module = _load() + build_id = load_spool_rows(out / "logbook-spool")[0].build_id + (out / f"{STEM}.local_gates.json").write_text( + json.dumps(_resign(_signed_report(), release_id=build_id)) + ) + manifest = json.loads((out / "rowwise_candidate_manifest.json").read_text()) + assert manifest["release_role"] == "dense" + assert manifest["parameters"]["release_candidate"] is True + assert module.check_candidate_dir(out, today=date(2026, 9, 4)) == [] + # The same projection is refused for what the pre-flight refuses: a size + # run, or a run whose manifest lost its uprating receipt. + manifest["census_household_uprating"]["applied"] = False + (out / "rowwise_candidate_manifest.json").write_text(json.dumps(manifest)) + failures = module.check_candidate_dir(out, today=date(2026, 9, 4)) + assert any("A15" in line for line in failures) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py index 2af56ec23..c04272cde 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py @@ -361,6 +361,102 @@ def forbidden(*_args, **_kwargs): assert not (output_dir / "logbook-spool").exists() +def test_graph_driver_dry_run_prints_the_operation_inventory( + monkeypatch, capsys, tmp_path +) -> None: + """The graph driver's dry run plans the same request without solving. + + The tool's dry run (above) prints the fenced clone/matrix plan it computes + in process; the graph driver prints the compiled operation inventory of + the same request. Both refuse to write. A bound spine checkpoint stands + in for the tool's sidecar-free households-only path, and the Ledger pins + are the committed feed's because the graph refuses any other. + """ + + pytest.importorskip("tables") + pytest.importorskip("h5py") + from microcosm.build.uk_runtime import full_build_cli as cli + from microcosm.build.uk_runtime import spine_build + from microcosm.build.uk_runtime.chronicle_feed import load_uk_chronicle_feed + from microcosm.build.uk_runtime.national_frame import load_uk_national_frame + from test_support.microcosm_build.uk_calibration_run import _bound_checkpoint + + input_h5 = tmp_path / "spine.h5" + ladder_path = tmp_path / "ladder.npz" + output_dir = tmp_path / "dry-run-output" + _write_staging_h5(input_h5) + _write_ladder(ladder_path) + frame, _ = load_uk_national_frame(input_h5) + sidecar_path, gates_path, sidecar = _bound_checkpoint(tmp_path, frame) + sidecar["stages"] = ["frs_spine"] + sidecar["sampling"] = {"fraction": 1.0, "seed": 578} + sidecar_path.write_text(json.dumps(sidecar)) + monkeypatch.setattr(spine_build, "_rules_engine", lambda: object()) + monkeypatch.setattr( + spine_build, "_rules_engine_provenance", lambda: {"version": "fixture"} + ) + monkeypatch.setattr( + cli, "run_graph", lambda *a, **k: pytest.fail("dry run executed graph") + ) + feed = load_uk_chronicle_feed() + argv = [ + "--release-role", + "dense", + "--input-h5", + str(input_h5), + "--input-sidecar", + str(sidecar_path), + "--input-spine-gates", + str(gates_path), + "--input-sha256", + hashlib.sha256(input_h5.read_bytes()).hexdigest(), + "--ladder", + str(ladder_path), + "--ladder-sha256", + hashlib.sha256(ladder_path.read_bytes()).hexdigest(), + "--ledger-facts", + str(tmp_path / "ledger"), + "--ledger-facts-sha256", + feed.facts_sha256, + "--ledger-manifest-sha256", + feed.manifest_sha256, + "--out", + str(output_dir), + "--n-clones", + "2", + "--seed", + "7", + "--dry-run", + "--no-staging", + ] + assert cli.main(argv) == 0 + plan = json.loads(capsys.readouterr().out) + assert plan["default_scope"] == "all_geographies" + assert plan["configuration"]["n_clones"] == 2 + assert plan["configuration"]["seed"] == 7 + assert plan["configuration"]["calibration"]["epochs"] == 1500 + assert plan["configuration"]["target_families"] is None + assert {node["id"] for node in plan["nodes"]} >= { + "uk.full.dense", + "uk.full.target_selection", + "uk.full.gates.calibrated", + } + assert not output_dir.exists() + assert not (output_dir / "logbook-spool").exists() + # --candidate-clone-counts plans one inventory per requested K. + assert cli.main([*argv, "--candidate-clone-counts", "1,2"]) == 0 + inventories = json.loads(capsys.readouterr().out) + assert [item["n_clones"] for item in inventories] == [1, 2] + assert [item["configuration"]["n_clones"] for item in inventories] == [1, 2] + # --households-only narrows the selection node to the census family. + assert cli.main([*argv, "--households-only"]) == 0 + plan = json.loads(capsys.readouterr().out) + assert plan["configuration"]["target_families"] == [ + "census_households/constituency" + ] + assert not output_dir.exists() + + def test_candidate_sampling_rung_receipt_and_engine_block_validation( monkeypatch, capsys, @@ -503,8 +599,9 @@ def forbidden(*_args, **_kwargs): } -def test_candidate_clone_count_planning_is_dry_run_only(tmp_path) -> None: - builder = _load_builder_module() +@_BOTH_DRIVERS +def test_candidate_clone_count_planning_is_dry_run_only(driver, tmp_path) -> None: + builder = _load_builder_module(driver) with pytest.raises(ValueError, match="only with --dry-run"): builder.main( [ @@ -1677,8 +1774,11 @@ def test_gate_criticality_reads_fail_closed() -> None: assert diagnostic == ["[uk_local_weight_ratio] ratio 578 > 100"] -def test_release_candidate_refuses_non_doctrine_solve_settings(tmp_path) -> None: - builder = _load_builder_module() +@_BOTH_DRIVERS +def test_release_candidate_refuses_non_doctrine_solve_settings( + driver, tmp_path +) -> None: + builder = _load_builder_module(driver) pin = "0" * 64 base = [ "--input-h5", @@ -1718,8 +1818,9 @@ def test_release_candidate_refuses_non_doctrine_solve_settings(tmp_path) -> None ) -def test_candidate_requires_pinned_ledger_inputs(tmp_path) -> None: - builder = _load_builder_module() +@_BOTH_DRIVERS +def test_candidate_requires_pinned_ledger_inputs(driver, tmp_path) -> None: + builder = _load_builder_module(driver) args = builder._parse_args( [ "--input-h5", @@ -1924,8 +2025,9 @@ def test_size_candidate_exports_compact_links_and_cannot_claim_dense_release( assert int(selection["certainty"].sum()) == size["protected_carriers"] -def test_selection_seed_requires_a_dataset_size(tmp_path): - builder = _load_builder_module() +@_BOTH_DRIVERS +def test_selection_seed_requires_a_dataset_size(driver, tmp_path): + builder = _load_builder_module(driver) args = builder._parse_args( [ "--input-h5", @@ -1944,6 +2046,7 @@ def test_selection_seed_requires_a_dataset_size(tmp_path): builder._validate_cli_args(args) +@_BOTH_DRIVERS @pytest.mark.parametrize( ("argv_tail", "message"), [ @@ -1955,8 +2058,10 @@ def test_selection_seed_requires_a_dataset_size(tmp_path): (["--dataset-households", "10", "--baseline-pi-floor", "1.5"], r"in \[0, 1\]"), ], ) -def test_selection_pi_hi_is_candidate_only_and_bounded(tmp_path, argv_tail, message): - builder = _load_builder_module() +def test_selection_pi_hi_is_candidate_only_and_bounded( + driver, tmp_path, argv_tail, message +): + builder = _load_builder_module(driver) args = builder._parse_args( [ "--input-h5", @@ -1985,8 +2090,9 @@ def test_dense_candidate_manifest_has_no_size_sidecars(tmp_path): assert builder._SIZE_RUN_ONLY_OUTPUTS == {"dense_reference", "selection"} -def test_size_cli_refuses_promotion_without_separate_certification(tmp_path): - builder = _load_builder_module() +@_BOTH_DRIVERS +def test_size_cli_refuses_promotion_without_separate_certification(driver, tmp_path): + builder = _load_builder_module(driver) args = builder._parse_args( [ "--input-h5", @@ -2413,7 +2519,7 @@ class Refusing(builder.StagingTelemetryV2): def calibration_progress(self, event): raise StagingContentError("Staging file exceeds the 5242880-byte limit.") - monkeypatch.setattr(builder, "StagingTelemetryV2", Refusing) + monkeypatch.setattr(rowwise_staging, "StagingTelemetryV2", Refusing) out = tmp_path / "refused-rows" status = builder.main(_build_args(input_h5, ladder_path, flags, out)) assert status == 0 @@ -2442,7 +2548,7 @@ class Invalid(builder.StagingTelemetryV2): def validate_local_bundle(self): raise StagingContractError("synthetic bundle defect") - monkeypatch.setattr(builder, "StagingTelemetryV2", Invalid) + monkeypatch.setattr(rowwise_staging, "StagingTelemetryV2", Invalid) out = tmp_path / "invalid-bundle" status = builder.main(_build_args(input_h5, ladder_path, flags, out)) assert status == 0 @@ -2494,8 +2600,8 @@ def test_remote_staging_uploads_telemetry_and_the_bundle_in_one_commit( builder, monkeypatch, tmp_path, remote=True ) hub = _FakeHub() - monkeypatch.setattr(builder, "_hub_api", lambda: hub) - monkeypatch.setattr(builder, "_hub_token", lambda: "hf_test_token") + monkeypatch.setattr(rowwise_staging, "_hub_api", lambda: hub) + monkeypatch.setattr(rowwise_staging, "_hub_token", lambda: "hf_test_token") out = tmp_path / "remote" status = builder.main( _build_args( @@ -2643,8 +2749,8 @@ def test_remote_staging_failure_is_recorded_and_the_build_still_succeeds( builder, monkeypatch, tmp_path, remote=True ) hub = _FakeHub(fail_commit=True) - monkeypatch.setattr(builder, "_hub_api", lambda: hub) - monkeypatch.setattr(builder, "_hub_token", lambda: "hf_test_token") + monkeypatch.setattr(rowwise_staging, "_hub_api", lambda: hub) + monkeypatch.setattr(rowwise_staging, "_hub_token", lambda: "hf_test_token") out = tmp_path / "failed-upload" status = builder.main(_build_args(input_h5, ladder_path, flags, out)) assert status == 0 @@ -2695,8 +2801,8 @@ def test_no_staged_dataset_keeps_telemetry_remote_and_the_bundle_local( builder, monkeypatch, tmp_path, remote=True ) hub = _FakeHub() - monkeypatch.setattr(builder, "_hub_api", lambda: hub) - monkeypatch.setattr(builder, "_hub_token", lambda: "hf_test_token") + monkeypatch.setattr(rowwise_staging, "_hub_api", lambda: hub) + monkeypatch.setattr(rowwise_staging, "_hub_token", lambda: "hf_test_token") out = tmp_path / "telemetry-only" status = builder.main( _build_args(input_h5, ladder_path, flags, out, "--no-staged-dataset") @@ -2712,15 +2818,16 @@ def test_no_staged_dataset_keeps_telemetry_remote_and_the_bundle_local( assert not (out / "sha256sums.txt").exists() +@_BOTH_DRIVERS def test_remote_dataset_staging_is_refused_up_front_without_credential_or_repo( - monkeypatch, tmp_path, capsys + driver, monkeypatch, tmp_path, capsys ): - builder = _load_builder_module() + builder = _load_builder_module(driver) input_h5, ladder_path, flags = _staging_run_setup( builder, monkeypatch, tmp_path, remote=True ) out = tmp_path / "refused" - monkeypatch.setattr(builder, "_hub_token", lambda: None) + monkeypatch.setattr(rowwise_staging, "_hub_token", lambda: None) with pytest.raises(ValueError, match="write credential"): builder.main(_build_args(input_h5, ladder_path, flags, out)) assert not out.exists() @@ -2729,8 +2836,8 @@ class Unreachable: def repo_info(self, **kwargs): raise RuntimeError("503 token=do-not-record") - monkeypatch.setattr(builder, "_hub_token", lambda: "hf_test_token") - monkeypatch.setattr(builder, "_hub_api", lambda: Unreachable()) + monkeypatch.setattr(rowwise_staging, "_hub_token", lambda: "hf_test_token") + monkeypatch.setattr(rowwise_staging, "_hub_api", lambda: Unreachable()) with pytest.raises(ValueError, match="cannot reach") as info: builder.main(_build_args(input_h5, ladder_path, flags, out)) assert "do-not-record" not in str(info.value) @@ -2738,7 +2845,7 @@ def repo_info(self, **kwargs): # A read token sees the private repository but cannot upload: refused # before the spine is read, not after the solve (the Hub answers 403). - monkeypatch.setattr(builder, "_hub_api", lambda: _FakeHub(role="read")) + monkeypatch.setattr(rowwise_staging, "_hub_api", lambda: _FakeHub(role="read")) with pytest.raises(ValueError, match="read-only"): builder.main(_build_args(input_h5, ladder_path, flags, out)) assert not out.exists() @@ -2749,7 +2856,9 @@ def repo_info(self, **kwargs): {"entity": {"type": "user", "name": "someone"}, "permissions": ["repo.write"]} ] monkeypatch.setattr( - builder, "_hub_api", lambda: _FakeHub(role="fineGrained", scopes=user_scoped) + rowwise_staging, + "_hub_api", + lambda: _FakeHub(role="fineGrained", scopes=user_scoped), ) with pytest.raises(ValueError, match="repo.write"): builder.main(_build_args(input_h5, ladder_path, flags, out)) @@ -2761,17 +2870,25 @@ def repo_info(self, **kwargs): } ] org_hub = _FakeHub(role="fineGrained", scopes=org_scoped) - monkeypatch.setattr(builder, "_hub_api", lambda: org_hub) - assert builder.main( + monkeypatch.setattr(rowwise_staging, "_hub_api", lambda: org_hub) + status = builder.main( _build_args(input_h5, ladder_path, flags, tmp_path / "org-scoped") - ) in (0, 1) - assert org_hub.commits and org_hub.commits[0]["repo_id"] == ( - "policyengine/populace-uk-private" ) + if driver == "graph": + # The pre-flight admits the org-scoped token; the synthetic spine + # carries no bound checkpoint sidecar, so the graph driver refuses + # later, on its inputs, never on the credential. + assert status == 1 + assert "sidecar absent" in capsys.readouterr().err + else: + assert status in (0, 1) + assert org_hub.commits and org_hub.commits[0]["repo_id"] == ( + "policyengine/populace-uk-private" + ) # Argument refusals cost nothing and come first: a missing credential is # never the reported reason when the arguments are wrong. - monkeypatch.setattr(builder, "_hub_token", lambda: None) + monkeypatch.setattr(rowwise_staging, "_hub_token", lambda: None) with pytest.raises(ValueError, match="only with --dry-run"): builder.main( _build_args( @@ -2779,12 +2896,18 @@ def repo_info(self, **kwargs): ) ) assert not out.exists() + if driver == "graph": + # The local build, the re-stage tool and the in-process dry-run plan + # below need the tool's synthetic households-only path; the graph + # driver's dry run is pinned by + # ``test_graph_driver_dry_run_prints_the_operation_inventory``. + return # The re-stage tool refuses the same credential the same way. stager = _load_tool("stage_uk_rowwise_candidate") monkeypatch.setattr(stager, "_hub_api", lambda: _FakeHub(role="read")) local_out = tmp_path / "local" - monkeypatch.setattr(builder, "_hub_token", lambda: None) + monkeypatch.setattr(rowwise_staging, "_hub_token", lambda: None) assert builder.main( _build_args(input_h5, ladder_path, flags, local_out, "--staging-local-only") ) in (0, 1) @@ -2793,7 +2916,7 @@ def repo_info(self, **kwargs): # A dry run plans without staging, so it needs neither credential nor repo. capsys.readouterr() - monkeypatch.setattr(builder, "_hub_token", lambda: None) + monkeypatch.setattr(rowwise_staging, "_hub_token", lambda: None) assert ( builder.main(_build_args(input_h5, ladder_path, flags, out, "--dry-run")) == 0 ) @@ -2802,8 +2925,9 @@ def repo_info(self, **kwargs): assert not out.exists() -def test_release_role_is_required(tmp_path) -> None: - builder = _load_builder_module() +@_BOTH_DRIVERS +def test_release_role_is_required(driver, tmp_path) -> None: + builder = _load_builder_module(driver) argv = _dense_argv(tmp_path) argv.remove("--release-role") argv.remove("dense") @@ -2813,8 +2937,9 @@ def test_release_role_is_required(tmp_path) -> None: builder._parse_args([*argv, "--release-role", "local"]) -def test_release_role_supplies_the_solve_defaults(tmp_path) -> None: - builder = _load_builder_module() +@_BOTH_DRIVERS +def test_release_role_supplies_the_solve_defaults(driver, tmp_path) -> None: + builder = _load_builder_module(driver) dense = builder._parse_args(_dense_argv(tmp_path)) posture = builder.UK_ROWWISE_DENSE_POSTURE assert (dense.n_clones, dense.seed, dense.epochs, dense.learning_rate) == ( @@ -2846,6 +2971,7 @@ def test_release_role_supplies_the_solve_defaults(tmp_path) -> None: ) +@_BOTH_DRIVERS @pytest.mark.parametrize( ("extra", "needle"), [ @@ -2854,21 +2980,23 @@ def test_release_role_supplies_the_solve_defaults(tmp_path) -> None: (["--target-weight-rule", "family_equal"], "--target-weight-rule family_equal"), ], ) -def test_dense_role_refusal_table(tmp_path, extra, needle) -> None: - builder = _load_builder_module() +def test_dense_role_refusal_table(driver, tmp_path, extra, needle) -> None: + builder = _load_builder_module(driver) args = builder._parse_args(_dense_argv(tmp_path, *extra)) with pytest.raises(ValueError, match="--release-role dense refuses") as excinfo: builder._validate_cli_args(args) assert needle in str(excinfo.value) -def test_dense_role_requires_the_ladder(tmp_path) -> None: - builder = _load_builder_module() +@_BOTH_DRIVERS +def test_dense_role_requires_the_ladder(driver, tmp_path) -> None: + builder = _load_builder_module(driver) argv = _role_argv(tmp_path, "dense", "--ladder-sha256", "3" * 64) with pytest.raises(ValueError, match="requires --ladder"): builder._validate_cli_args(builder._parse_args(argv)) +@_BOTH_DRIVERS @pytest.mark.parametrize( ("extra", "needle"), [ @@ -2894,16 +3022,17 @@ def test_dense_role_requires_the_ladder(tmp_path) -> None: (["--target-weight-rule", "grain_equal"], "--target-weight-rule grain_equal"), ], ) -def test_national_role_refusal_table(tmp_path, extra, needle) -> None: - builder = _load_builder_module() +def test_national_role_refusal_table(driver, tmp_path, extra, needle) -> None: + builder = _load_builder_module(driver) args = builder._parse_args(_role_argv(tmp_path, "national", *extra)) with pytest.raises(ValueError, match="--release-role national refuses") as excinfo: builder._validate_cli_args(args) assert needle in str(excinfo.value) -def test_national_role_refuses_release_candidate_with_the_seam_reason(tmp_path): - builder = _load_builder_module() +@_BOTH_DRIVERS +def test_national_role_refuses_release_candidate_with_the_seam_reason(driver, tmp_path): + builder = _load_builder_module(driver) args = builder._parse_args(_role_argv(tmp_path, "national", "--release-candidate")) with pytest.raises(ValueError, match="cannot sign shippability"): builder._validate_cli_args(args) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py index 1f0a16040..f724e2b77 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py @@ -19,7 +19,7 @@ from microcosm.build.logbook import load_spool_rows from microcosm.build.staging_v2 import validate_v2_bundle -from microcosm.build.uk_runtime import calibration_run +from microcosm.build.uk_runtime import calibration_run, rowwise_staging from microcosm.build.uk_runtime.calibration_run import UK_CALIBRATION_GATE_SCOPE from microcosm.build.uk_runtime.chronicle_feed import ( UKChronicleFeedPinError, @@ -395,8 +395,8 @@ def test_uk_national_role_publishes_telemetry_and_the_bundle_to_the_hub( builder, monkeypatch, tmp_path ) hub = candidate._FakeHub() - monkeypatch.setattr(builder, "_hub_api", lambda: hub) - monkeypatch.setattr(builder, "_hub_token", lambda: "hf_test_token") + monkeypatch.setattr(rowwise_staging, "_hub_api", lambda: hub) + monkeypatch.setattr(rowwise_staging, "_hub_token", lambda: "hf_test_token") out = tmp_path / "national" assert ( diff --git a/test_support/microcosm_build/uk_full_build_cli.py b/test_support/microcosm_build/uk_full_build_cli.py new file mode 100644 index 000000000..17b1f15d9 --- /dev/null +++ b/test_support/microcosm_build/uk_full_build_cli.py @@ -0,0 +1,712 @@ +"""Synthetic dense full-build fixtures for the canonical UK CLI: the +kernel stand-ins, payload builders, the prepared build and the +``graph_dense_bundle`` runner the release pre-flight and assembler tests +feed.""" + +# ruff: noqa: F401 + +import hashlib +import json +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pytest + +from microcosm.build.gate_battery import ( + GateOutcome, + GatePhaseReport, + GateStatus, + gate_phase_report_payload, +) +from microcosm.build.gates import GateResult +from microcosm.build.uk_runtime import full_build_cli as cli +from microcosm.build.uk_runtime.full_certification import FULL_CERTIFICATION_TYPE +from microcosm.build.uk_runtime.full_gates import ( + classify_full_gate_outcomes, + uk_full_gate_manifest, +) +from microcosm.build.uk_runtime.graph_build import UKFullBuildConfig, UKFullGraph +from microcosm.build.uk_runtime.graph_calibration import UKCalibrationNodes +from microcosm.build.uk_runtime.graph_population import ( + GEOGRAPHY_GATE_TYPE, + POPULATION_RECEIPT_TYPE, +) +from microcosm.build.uk_runtime.graph_targets import ( + TARGET_SELECTION_TYPE, + TARGET_SURFACE_TYPE, + registry_payload, +) +from microcosm.build.uk_runtime.graph_terminal import ( + FULL_DIAGNOSTICS_CSV_TYPE, + FULL_DIAGNOSTICS_TYPE, + FULL_GATE_REPORT_TYPE, + FULL_HOLDOUT_TYPE, + FULL_SUPPORT_CSV_TYPE, + add_uk_export_preparation, + register_uk_terminal_kernels, +) +from microcosm.build.uk_runtime.release_identity import UK_DENSE_RELEASE_ID +from microcosm.calibrate import ( + Target, + TargetRegistry, + TargetSet, + build_constraint_matrix, +) +from microcosm.calibrate.artifacts import PROBLEM_TYPE, encode_problem +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json +from test_support.microcosm_build.uk_graph_terminal import _frame + +PIN = "0" * 64 + +STEM = "microcosm_uk_2024_25_local" + + +def _placeholder(path: Path, payload: bytes) -> str: + if not path.exists(): + path.write_bytes(payload) + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def arguments(tmp_path, *extra, role="dense", staging="--no-staging"): + """A dense request over stand-in input files, with staging disabled. + + The pins are the stand-ins' real digests so the validator and a real + preparation would both accept them; the Ledger pins are synthetic + because these tests never compile targets. + """ + spine = tmp_path / "spine.h5" + ladder = tmp_path / "ladder.npz" + return cli.parse_args( + [ + "--release-role", + role, + "--input-h5", + str(spine), + "--input-sha256", + _placeholder(spine, b"spine stand-in"), + "--ladder", + str(ladder), + "--ladder-sha256", + _placeholder(ladder, b"ladder stand-in"), + "--ledger-facts", + str(tmp_path / "ledger"), + "--ledger-facts-sha256", + PIN, + "--ledger-manifest-sha256", + PIN, + "--out", + str(tmp_path / "out"), + staging, + *extra, + ] + ) + + +SELECTION = { + "schema": "microcosm.calibrate.target-selection.v1", + "selector": {"geography_levels": None, "explicit": False}, + "included": [ + {"name": "count", "period": 2025, "geography_level": "country"}, + {"name": "local", "period": 2025, "geography_level": "constituency"}, + ], + "excluded": [], +} + + +def gate_payload(phase, failed=None): + gates = uk_full_gate_manifest(SELECTION) + report = GatePhaseReport( + phase, + tuple( + GateOutcome( + entry, + GateStatus.FAILED if entry.id == failed else GateStatus.PASSED, + GateResult( + name=entry.id, + passed=entry.id != failed, + details={}, + failures=("synthetic failure",) if entry.id == failed else (), + ), + ) + for entry in gates.gates + if entry.phase == phase + ), + ) + return canonical_json( + { + "schema_version": 1, + "kind": "uk_full_gate_report", + "selection_receipt": SELECTION, + "sample_fraction": 1.0, + "release_candidate": False, + "report": gate_phase_report_payload(report, gates=gates), + "enforcement": classify_full_gate_outcomes( + report, sample_fraction=1.0, release_candidate=False + ), + } + ) + + +LADDER_TARGET = "ons.census.households@E14000001" + + +def problem_payload() -> bytes: + """One ordered local problem over the two fixture households.""" + frame = _frame() + targets = TargetSet( + [ + Target( + LADDER_TARGET, + "household", + lambda f: np.ones(f.n("household")), + 100.0, + period=2025, + ) + ] + ) + return encode_problem( + build_constraint_matrix(frame, targets, weight_entity="household"), + entity_ids=frame.table("household")["household_id"].tolist(), + target_metadata=[ + { + "materialization": "uk_local_surface", + "geography_level": "constituency", + "geography_id": "E14000001", + "family": "census_households", + "source": "fixture", + } + ], + bindings={ + "mass_reason": "fixture selected constraints", + "max_weight_ratio": 10.0, + "target_loss_weights": [1.0], + "target_loss_cap": 10.0, + "bound_families": ["census_households/constituency"], + "binding_adjudications": { + "stood_on": {"census_households/constituency": ["fixture"]} + }, + "rung_surface": {"dropped_cells": 0}, + "measure_resolution": {"blocks": 1}, + "cross_geography": { + "unbound_bridges": [], + "empty_legs_licensed": [], + "census_household_uprating": uprating_receipt(), + }, + "measure_exclusions": measure_exclusions(), + "calibration_year": 2025, + }, + ) + + +def uprating_receipt() -> dict: + def hold(name: str) -> dict: + return { + "target_name": name, + "geography_level": "constituency", + "from_period": 2021, + "to_period": 2025, + "attempted": True, + "eligible": True, + "applied": True, + "skipped": False, + "reason": "census_vintage_hold_uprated", + } + + cells = { + "applied": True, + "cells": 1, + "total_cells": 1, + "attempted_cells": 1, + "eligible_cells": 1, + "skipped_cells": 0, + } + return { + "applied": True, + "grains": { + "constituency": {"factor": 1.0335597414671631}, + "local_authority": {"factor": 1.0335595204737118}, + }, + "household_cells": {**cells, "holds": [hold(LADDER_TARGET)]}, + "tenure_cells": { + **cells, + "holds": [hold("ons.tenure.owned_outright@E14000001")], + }, + } + + +def measure_exclusions() -> dict: + return { + "obr.housing_benefit": { + "reason": "synthetic gap", + "tracking": "microcosm#869", + "approved_by": "synthetic_reviewer", + "adjudication": "synthetic decision", + "approved_on": "2026-09-03", + "expires_on": "2026-10-03", + } + } + + +def surface_payload() -> bytes: + return canonical_json( + { + "local_registry": registry_payload(TargetRegistry([], country="uk")), + "census_household_uprating": uprating_receipt(), + "household_dispersion": {"constituency": {"max_ratio": 1.0}}, + "ladder_provenance": {"constituency": "2024_pcon"}, + "measure_exclusions": measure_exclusions(), + "source_validation": { + "targets": { + "chronicle": { + "path_name": "chronicle-uk-artifact-fixture", + "facts_sha256": PIN, + "manifest_sha256": PIN, + "fact_row_count": 1, + "schema_version": "v1", + }, + "paired_ladder_sha256": PIN, + } + }, + "calibration_year": 2025, + } + ) + + +def diagnostics_payload() -> bytes: + return canonical_json( + { + "schema_version": 8, + "n_records": 2, + "n_nonzero": 2, + "initial_loss": 0.5, + "final_loss": 0.01, + "realized_max_weight_ratio": 1.0, + "fraction_within_10pct": 1.0, + "past_cap_census": { + "n_targets": 1, + "initial_past_cap": 0, + "final_past_cap": 0, + "escaped": 0, + "frozen": 0, + "pushed_out": 0, + }, + "targets": [ + { + "name": f"{LADDER_TARGET}@2025", + "target": 100.0, + "compiled_target": 100.0, + "initial_estimate": 90.0, + "final_estimate": 100.0, + } + ], + "target_registry": {"country": "uk", "specs": []}, + "uk_diagnostics": { + "weights": {"n_nonzero": 2}, + "weakest_families": [], + "weakest_areas_by_fit": {}, + "rotated_holdout": holdout_payload_dict(), + }, + } + ) + + +def holdout_payload_dict() -> dict: + return { + "report_only": True, + "method": "rotated_folds", + "n_folds": 5, + "mean_holdout_loss": 0.2, + "worst_holdout_loss": 0.3, + "folds": [], + } + + +TARGET_ROWS_CSV = ( + "name,target_name,target,estimate,relative_error,abs_relative_error,family," + "geography_level,area_type,area_code,metric\n" + f"{LADDER_TARGET}@2025,{LADDER_TARGET}@2025,100.0,100.0,0.0,0.0," + "census_households,constituency,constituency,E14000001,households\n" +).encode() + +SUPPORT_CSV = ( + b"area_code,assigned_households,nonzero_households,nonzero_source_households," + b"weight_sum,max_weight,effective_sample_size,geography_level\n" + b"E14000001,1,1,1,13.0,13.0,1.0,constituency\n" + b"E14000002,1,1,1,87.0,87.0,1.0,constituency\n" + b"E09000001,1,1,1,13.0,13.0,1.0,local_authority\n" + b"E07000008,1,1,1,87.0,87.0,1.0,local_authority\n" +) + + +class Fixture(KernelBase): + ref = "uk.test.cli-frame@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def implementation_hash(self): + return hashlib.sha256(self.ref.encode()).hexdigest() + + def run(self, context): + return KernelResult(frame=_frame()) + + +class Evidence(KernelBase): + ref = "uk.test.cli-evidence@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def implementation_hash(self): + return hashlib.sha256(self.ref.encode()).hexdigest() + + def run(self, context): + phase = context.params["phase"] + artifacts = {"gate_report": gate_payload(phase, context.params.get("failed"))} + if phase == "terminal": + artifacts.update( + calibration_diagnostics=diagnostics_payload(), + target_diagnostics_csv=TARGET_ROWS_CSV, + area_support_csv=SUPPORT_CSV, + ) + return KernelResult(artifacts=artifacts) + + +class Holdout(Evidence): + ref = "uk.test.cli-holdout@1" + + def run(self, context): + return KernelResult( + artifacts={ + "holdout": canonical_json(holdout_payload_dict()), + "selection": canonical_json( + {"registry": {"country": "uk", "specs": []}, "receipt": SELECTION} + ), + } + ) + + +class Targets(Evidence): + """The compiled surface and the ordered problem the projection reads.""" + + ref = "uk.test.cli-targets@1" + + def run(self, context): + return KernelResult( + artifacts={"surface": surface_payload(), "problem": problem_payload()} + ) + + +class Population(Evidence): + """The sampling receipt and the geography gate of the pool.""" + + ref = "uk.test.cli-population@1" + + def run(self, context): + return KernelResult( + artifacts={ + "sampling": canonical_json( + { + "receipt": { + "fraction": 1.0, + "seed": 578, + "sampled": False, + "pre_household_count": 2, + "post_household_count": 2, + "rung_token": "f100", + } + } + ), + "gate": canonical_json( + { + "name": "uk_local_geography_ladder", + "passed": True, + "failures": [], + "details": {}, + } + ), + } + ) + + +class Certification(Evidence): + ref = "uk.test.cli-certification@1" + + def run(self, context): + return KernelResult( + artifacts={ + "certification_readiness": b'{"fixture":true,"release_authorized":false}' + } + ) + + +def patch_certification(monkeypatch) -> None: + """Replace the scientific certification node with the synthetic one.""" + + def append(graph, *, population, **kwargs): + return replace( + graph, + nodes=( + *graph.nodes, + Node( + "uk.full.certification", + Certification.ref, + population=population, + artifact_outputs=( + ArtifactOutput( + "certification_readiness", FULL_CERTIFICATION_TYPE + ), + ), + ), + ), + ) + + monkeypatch.setattr(cli, "append_uk_full_certification_node", append) + + +@pytest.fixture(autouse=True) +def certification_service_fixture(monkeypatch): + # Scientific certification validation has its own graph-artifact tests. + # This suite tests the filesystem/execution service with synthetic evidence. + patch_certification(monkeypatch) + + +def graph_dense_bundle(tmp_path, monkeypatch, *extra, staging="--no-staging") -> Path: + """Run the synthetic dense build through ``main`` and return its bundle. + + The other UK test modules feed the resulting ``rowwise_candidate_manifest.json`` + to the release pre-flight and the dense assembler. + """ + monkeypatch.delenv("POPULACE_LOGBOOK_PREV_ROW_DIGEST", raising=False) + patch_certification(monkeypatch) + args = arguments(tmp_path, *extra, staging=staging) + build = prepared(tmp_path) + monkeypatch.setattr(cli, "parse_args", lambda argv: args) + monkeypatch.setattr(cli, "prepare_full_build", lambda args, **kwargs: build) + assert cli.main([]) == 0 + return args.out + + +def prepared(tmp_path, failed=None): + frame = _frame() + fixture = tmp_path / "fixture.txt" + fixture.write_text("constant source") + identifiers = { + "person_id", + "person_household_id", + "person_benunit_id", + "household_id", + "benunit_id", + } + root = Node( + "uk.full.calibrated", + Fixture.ref, + structural=StructuralDelta.CREATE, + sources=("fixture",), + outputs=tuple( + Owned( + e, + str(c), + "string" + if frame.table(e)[c].dtype.kind in "OUS" + else str(frame.table(e)[c].dtype), + ) + for e in frame.entities + for c in frame.table(e).columns + if c not in identifiers + ), + ) + nodes = [ + root, + Node( + "uk.full.gates.preflight", + Evidence.ref, + population=root.id, + params={"phase": "preflight", "failed": failed}, + artifact_outputs=(ArtifactOutput("gate_report", FULL_GATE_REPORT_TYPE),), + ), + Node( + "uk.full.gates.calibrated", + Evidence.ref, + population=root.id, + params={"phase": "terminal", "failed": failed}, + artifact_outputs=( + ArtifactOutput("gate_report", FULL_GATE_REPORT_TYPE), + ArtifactOutput("calibration_diagnostics", FULL_DIAGNOSTICS_TYPE), + ArtifactOutput("target_diagnostics_csv", FULL_DIAGNOSTICS_CSV_TYPE), + ArtifactOutput("area_support_csv", FULL_SUPPORT_CSV_TYPE), + ), + ), + Node( + "uk.full.holdout", + Holdout.ref, + population=root.id, + artifact_outputs=( + ArtifactOutput("holdout", FULL_HOLDOUT_TYPE), + ArtifactOutput("selection", TARGET_SELECTION_TYPE), + ), + ), + # CLI materialization consumes the public target-selection endpoint. + Node( + "uk.full.target_selection", + Holdout.ref, + population=root.id, + artifact_outputs=( + ArtifactOutput("holdout", FULL_HOLDOUT_TYPE), + ArtifactOutput("selection", TARGET_SELECTION_TYPE), + ), + ), + # The manifest projection reads the compiled surface, the ordered + # problem, the sampling receipt and the geography gate. + Node( + "uk.full.target_compilation", + Targets.ref, + population=root.id, + artifact_outputs=( + ArtifactOutput("surface", TARGET_SURFACE_TYPE), + ArtifactOutput("problem", PROBLEM_TYPE), + ), + ), + Node( + "uk.full.problem", + Targets.ref, + population=root.id, + artifact_outputs=( + ArtifactOutput("surface", TARGET_SURFACE_TYPE), + ArtifactOutput("problem", PROBLEM_TYPE), + ), + ), + Node( + "uk.full.sample", + Population.ref, + population=root.id, + artifact_outputs=( + ArtifactOutput("sampling", POPULATION_RECEIPT_TYPE), + ArtifactOutput("gate", GEOGRAPHY_GATE_TYPE), + ), + ), + Node( + "uk.full.geography_gate", + Population.ref, + population=root.id, + artifact_outputs=( + ArtifactOutput("sampling", POPULATION_RECEIPT_TYPE), + ArtifactOutput("gate", GEOGRAPHY_GATE_TYPE), + ), + ), + ] + nodes = [ + replace( + node, + artifact_inputs=( + ArtifactInput( + "preflight", + "uk.full.gates.preflight", + "gate_report", + FULL_GATE_REPORT_TYPE, + ), + ArtifactInput( + "holdout", "uk.full.holdout", "holdout", FULL_HOLDOUT_TYPE + ), + ArtifactInput( + "selection", + "uk.full.target_selection", + "selection", + TARGET_SELECTION_TYPE, + ), + ArtifactInput( + "surface", + "uk.full.target_compilation", + "surface", + TARGET_SURFACE_TYPE, + ), + ArtifactInput("problem", "uk.full.problem", "problem", PROBLEM_TYPE), + ArtifactInput( + "sampling", "uk.full.sample", "sampling", POPULATION_RECEIPT_TYPE + ), + ArtifactInput( + "geography_gate", + "uk.full.geography_gate", + "gate", + GEOGRAPHY_GATE_TYPE, + ), + ), + ) + if node.id == "uk.full.gates.calibrated" + else node + for node in nodes + ] + graph = Graph("uk", (SourceRef("fixture", "raw-bytes-v1"),), tuple(nodes)) + graph = add_uk_export_preparation( + graph, + population=root.id, + bindings={"target_scope": "all"}, + artifact_inputs=( + ArtifactInput( + "gates", + "uk.full.gates.calibrated", + "gate_report", + FULL_GATE_REPORT_TYPE, + ), + ), + ) + calibration = UKCalibrationNodes( + (), root.id, "unused", "unused", "unused", None, "unused" + ) + full = UKFullGraph(graph, calibration, UKFullBuildConfig(calibration_year=2025)) + kernels = KernelRegistry() + for kernel in ( + Fixture(), + Evidence(), + Holdout(), + Targets(), + Population(), + Certification(), + ): + kernels.register(kernel) + register_uk_terminal_kernels(kernels) + spine = tmp_path / "spine.h5" + ladder = tmp_path / "ladder.npz" + pins = { + "dataset": { + "sha256": _placeholder(spine, b"spine stand-in"), + "size_bytes": spine.stat().st_size, + }, + "ladder": { + "sha256": _placeholder(ladder, b"ladder stand-in"), + "size_bytes": ladder.stat().st_size, + }, + } + inputs = { + name: { + "path": str(path.resolve()), + "sha256": pins[name]["sha256"], + "bytes": pins[name]["size_bytes"], + "pin_verified": True, + } + for name, path in (("dataset", spine), ("ladder", ladder)) + } + return cli.PreparedUKFullBuild( + full, + kernels, + {"fixture": fixture}, + {"target_scope": "all"}, + pins=pins, + inputs=inputs, + ) + + +__all__ = [name for name in globals() if not name.startswith("__")] diff --git a/test_support/microcosm_build/uk_rowwise_candidate.py b/test_support/microcosm_build/uk_rowwise_candidate.py index fb3fd9188..c462a3692 100644 --- a/test_support/microcosm_build/uk_rowwise_candidate.py +++ b/test_support/microcosm_build/uk_rowwise_candidate.py @@ -23,6 +23,7 @@ ladder_target_provenance, load_uk_oa_ladder, read_uk_single_year_weight_metadata, + rowwise_staging, write_uk_national_frame, ) from microcosm.build.uk_runtime.national_frame import ( @@ -119,7 +120,18 @@ def _fixture_hierarchy( ) -def _load_builder_module(): +#: The two UK dense drivers whose command surface must agree: the rowwise +#: tool (``tools/build_uk_rowwise_candidate.py``) and the graph full build +#: (``microcosm.build.uk_runtime.full_build_cli``). Role and pure-CLI tests +#: run against both; the in-process build tests stay on the tool. +_BOTH_DRIVERS = pytest.mark.parametrize("driver", ["tool", "graph"]) + + +def _load_builder_module(driver: str = "tool"): + if driver == "graph": + from microcosm.build.uk_runtime import full_build_cli + + return full_build_cli root = _TEST_PATHS.repository path = root / "tools" / "build_uk_rowwise_candidate.py" spec = importlib.util.spec_from_file_location( @@ -408,8 +420,13 @@ def _configure_households_only_inputs( "measure_exclusions": {}, "reviewed_unbound_higher_targets": {}, } + # The graph driver compiles its targets in a graph node and has no + # in-process loader to patch; the tool's seam is patched when present. monkeypatch.setattr( - builder, "_load_joint_target_inputs", lambda _args: joint_inputs + builder, + "_load_joint_target_inputs", + lambda _args: joint_inputs, + raising=False, ) return [ *_mandatory_input_flags(input_h5, ladder_path), diff --git a/tools/build_uk_rowwise_candidate.py b/tools/build_uk_rowwise_candidate.py index 8d46c0b34..ce618ae6c 100644 --- a/tools/build_uk_rowwise_candidate.py +++ b/tools/build_uk_rowwise_candidate.py @@ -25,14 +25,11 @@ import importlib import json import shutil -import subprocess import sys import tempfile import time -import uuid -from collections.abc import Callable, Mapping +from collections.abc import Mapping from datetime import UTC, date, datetime -from importlib import metadata from pathlib import Path from typing import Any @@ -47,21 +44,16 @@ ) from microcosm.build.gates import GateResult from microcosm.build.ledger_artifact import load_ledger_consumer_artifact -from microcosm.build.logbook import canonical_json_bytes from microcosm.build.logbook_adoption import ( AttemptState, append_phase, - apply_error_verdict, atomic_write_json, - error_receipt_path, git_code_pin, local_artifact_reference, preflight_digest, - record_terminal_attempt, resolve_predecessor, role_pins_digest, sha256_argument, - write_error_receipt, ) from microcosm.build.staging_cli import ( add_staged_dataset_arguments, @@ -71,19 +63,11 @@ ) from microcosm.build.staging_dataset import ( SHA256SUMS_FILENAME, - StagedDatasetBundle, - disabled_staged_dataset, - local_only_staged_dataset, parse_sha256sums, refresh_sha256sums_entry, - stage_bundle, - write_sidecars, ) -from microcosm.build.staging_storage import HuggingFaceDatasetStorage from microcosm.build.staging_v2 import ( - StagingContractError, StagingTelemetryV2, - disabled_staging_delivery, ) from microcosm.build.target_materialization import resolve_target_measures from microcosm.build.uk_runtime import ( @@ -155,14 +139,70 @@ UK_SAMPLE_SEED_DEFAULT, sample_uk_spine_frame, ) +from microcosm.build.uk_runtime.rowwise_cli import ( + _BUDGET_ITERS, + _CONSERVE_MASS, + _L0_LAMBDA, + _REPOSITORY, + _SIZE_RUN_ONLY_OUTPUTS, # noqa: F401 (read by the driver tests) + _TARGET_RECORDS, + _UK_CANDIDATE_PIPELINE, # noqa: F401 (read by the driver tests) + AREA_SUPPORT_FILENAME, # noqa: F401 (read by the driver tests) + CALIBRATION_DIAGNOSTICS_FILENAME, # noqa: F401 (read by the driver tests) + DATASET_SIZE_SELECTION_FILENAME, # noqa: F401 (read by the driver tests) + DENSE_REFERENCE_DIAGNOSTICS_FILENAME, + LOCAL_REGISTRY_FILENAME, # noqa: F401 (read by the driver tests) + MANIFEST_FILENAME, # noqa: F401 (read by the driver tests) + PAST_CAP_FILENAME, # noqa: F401 (read by the driver tests) + SOLVE_DIAGNOSTICS_FILENAME, # noqa: F401 (read by the driver tests) + _candidate_clone_counts_argument, + _candidate_identity_digest, + _doctrine_bounds, # noqa: F401 (read by the driver tests) + _gate_failures_by_criticality, + _git_commit, + _git_dirty, + _is_release_blocking, + _json_text, + _local_vintage_census, + _new_candidate_build_id, + _output_paths, + _parameters, + _posture_of, + _record_candidate_attempt, + _record_candidate_error, + _refuse_national_role_arguments, # noqa: F401 (read by the driver tests) + _release_verdict, + _resolve_role_arguments, + _validate_cli_args, +) from microcosm.build.uk_runtime.rowwise_posture import ( UK_ROWWISE_DENSE_POSTURE, UK_ROWWISE_RELEASE_ROLES, UKRowwisePosture, - uk_rowwise_posture, + uk_rowwise_posture, # noqa: F401 (read by the driver tests) +) +from microcosm.build.uk_runtime.rowwise_staging import ( + _STAGED_DATASET_PHASES, + _STAGING_MAX_EPOCH_ROWS, # noqa: F401 (read by the driver tests) + _STAGING_UPLOAD_INTERVAL_SECONDS, + _add_staging_artifact, + _create_staging_telemetry, + _fail_staging_telemetry, + _finalize_staging_telemetry, + _gate_statuses, + _preflight_staged_dataset, + _publish_staged_files, + _replace_manifest, + _stage, + _stage_dataset, + _staging_delivery, + _staging_epoch_every, + _thinned_epochs, +) +from microcosm.build.uk_runtime.size_checkpoint import ( + uk_size_checkpoint_identity as _size_checkpoint_identity, ) from microcosm.build.uk_runtime.staging import ( - UK_STAGED_DATASET_PREFIX, UK_STAGED_DATASET_REPOSITORY, UK_STAGING_REPOSITORY, ) @@ -172,55 +212,11 @@ BOUND_TARGET_FAMILIES = ("census_households/constituency",) BOUND_NATIONAL_TARGETS: tuple[str, ...] = () -MANIFEST_FILENAME = "rowwise_candidate_manifest.json" -SOLVE_DIAGNOSTICS_FILENAME = "solve_diagnostics.csv" -CALIBRATION_DIAGNOSTICS_FILENAME = "calibration_diagnostics.json" -AREA_SUPPORT_FILENAME = "area_support_summary.csv" -PAST_CAP_FILENAME = "past_cap_census.json" -LOCAL_REGISTRY_FILENAME = "local_target_registry.json" -#: National-role outputs (the calibration seam's evidence shape). -BUILD_RECORD_FILENAME = "build_record.json" -NATIONAL_REGISTRY_FILENAME = "national_target_registry.json" -NATIONAL_CONTRACT_REGISTRY_FILENAME = "national_contract_registry.json" -SCORE_RECEIPT_FILENAME = "score_vs_incumbent.json" -DENSE_REFERENCE_DIAGNOSTICS_FILENAME = "dense_reference_diagnostics.csv" -DATASET_SIZE_SELECTION_FILENAME = "dataset_size_selection.csv" - -#: Outputs a run writes only when ``--dataset-households`` is set. -_SIZE_RUN_ONLY_OUTPUTS = frozenset({"dense_reference", "selection"}) - -_CONSERVE_MASS = False -_TARGET_RECORDS: int | None = None -_L0_LAMBDA = 0.0 -_BUDGET_ITERS = 10 -# The dense role's Logbook pipeline and gate-policy suffix, kept as module -# names for the Logbook helpers and the contract-pin tests; the posture -# record (``rowwise_posture.py``) is the source of truth for both roles. -_UK_CANDIDATE_PIPELINE = UK_ROWWISE_DENSE_POSTURE.pipeline +# The dense role's gate-policy suffix, kept as a module name for the +# contract-pin tests; the posture record (``rowwise_posture.py``) is the +# source of truth for both roles, and the shared CLI helpers now live in +# ``uk_runtime/rowwise_cli.py`` / ``rowwise_staging.py``. _LOCAL_GATE_POLICY_SUFFIX = UK_ROWWISE_DENSE_POSTURE.gate_policy_suffix -# Best-effort telemetry upload cadence. The Hub allows about 128 commits per -# hour per repository and one cycle is up to eight single-file commits, so -# the shared 30-second default exhausts the budget on a multi-hour solve -# and loses uploads (the v20 national run did); five minutes keeps a -# 1,500-epoch run well inside it. -_STAGING_UPLOAD_INTERVAL_SECONDS = 300.0 -# Staging telemetry keeps one row per forwarded epoch in -# calibration_progress.json and one event in events.ndjson, both under the -# contract's 5 MiB remote cap. A size run at 2,000 epochs solves the dense -# pool, up to ten full-length L0 probes and the refit: about 24,000 epochs, -# which would breach the cap mid-run. Forwarding every tenth epoch and the -# last epoch of each phase keeps the loss curve and stays near 0.8 MB. -_STAGING_EPOCH_EVERY = 10 -# ...and never more than this many forwarded epochs per run, whatever --epochs -# says: the stride grows with the run so the cap holds by construction. -_STAGING_MAX_EPOCH_ROWS = 2400 -_STAGED_DATASET_PHASES = { - "uploaded": "dataset_staged", - "already_staged": "dataset_staged", - "failed": "dataset_stage_failed", - "skipped": "dataset_stage_skipped", -} -_REPOSITORY = Path(__file__).resolve().parents[1] _PAST_CAP_COUNT_KEYS = ( "n_targets", "past_at_init", @@ -243,46 +239,6 @@ def __init__( self.ladder = ladder -def _new_candidate_build_id( - *, seed: int, timestamp: datetime, rung: str = "f100" -) -> str: - """The dense role's attempt id; the national role mints the seam's.""" - - instant = timestamp.astimezone(UTC) - return ( - f"{UK_ROWWISE_DENSE_POSTURE.build_id_prefix}{rung}-s{seed}-" - f"{instant.strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:8]}" - ) - - -def _posture_of(args: argparse.Namespace) -> UKRowwisePosture: - """The release-role posture bound to parsed arguments.""" - - posture = getattr(args, "_posture", None) - if not isinstance(posture, UKRowwisePosture): - raise RuntimeError("arguments carry no release-role posture; parse them first.") - return posture - - -def _candidate_clone_counts_argument(value: str) -> tuple[int, ...]: - parts = value.split(",") - if not value.strip() or any(not part.strip() for part in parts): - raise argparse.ArgumentTypeError( - "candidate clone counts must be a non-empty comma list of positive integers" - ) - try: - counts = [int(part.strip()) for part in parts] - except ValueError as error: - raise argparse.ArgumentTypeError( - "candidate clone counts must be a comma list of positive integers" - ) from error - if any(count <= 0 for count in counts): - raise argparse.ArgumentTypeError( - "candidate clone counts must all be positive integers" - ) - return tuple(sorted(set(counts))) - - def _sample_candidate_frame( frame, *, @@ -603,128 +559,6 @@ def _refuse_stale_size_checkpoint(args: argparse.Namespace, out_dir: Path) -> No ) -def _size_checkpoint_identity( - args: argparse.Namespace, - *, - pins: Mapping[str, Mapping[str, object]], - source_year: int, -) -> dict[str, object]: - """Everything a size checkpoint must share with the run that resumes it. - - The pool (spine, ladder, clones, seed, sampling), the target surface - (ledger digests, year, rule, engine blocks) and the solve settings the - checkpointed dense solve and search were made with. The draw threshold is - deliberately absent: re-drawing at another threshold is the point. - """ - posture = _posture_of(args) - return { - "release_role": posture.role, - "dataset_pin": dict(pins["dataset"]), - "ladder_pin": dict(pins["ladder"]), - "ledger_facts_sha256": args.ledger_facts_sha256, - "ledger_manifest_sha256": args.ledger_manifest_sha256, - "seed": int(args.seed), - "selection_seed": int( - args.seed if args.selection_seed is None else args.selection_seed - ), - "n_clones": None if args.n_clones is None else int(args.n_clones), - "dataset_households": args.dataset_households, - "epochs": int(args.epochs), - "learning_rate": float(args.learning_rate), - "sample_fraction": float(args.sample_fraction), - "sample_seed": int(args.sample_seed), - "source_year": int(source_year), - "source_lineage_modulus": args.source_lineage_modulus, - "calibration_year": getattr(args, "_calibration_year", None), - "target_weight_rule": args.target_weight_rule, - "engine_blocks": int(args.engine_blocks), - "measure_exclusions": ( - None if args.measure_exclusions is None else str(args.measure_exclusions) - ), - # The solve doctrine the dense solve and the search run under: a - # resume after a doctrine change must refuse, not run under the old - # bound while the manifest declares the new one. - "doctrine": _doctrine_bounds(posture), - } - - -def _candidate_identity_digest( - *, - pins: dict[str, dict[str, object]], - args: argparse.Namespace, - source_year: int, -) -> str: - payload = { - "build_kind": "uk_rowwise_calibrated_candidate", - "inputs": pins, - "parameters": _parameters(args, source_year=source_year), - "source_year": source_year, - } - return hashlib.sha256(canonical_json_bytes(payload)).hexdigest() - - -def _record_candidate_attempt( - *, - state: AttemptState, - started_at: float, - started_ts: datetime, - seed: int, - code_pin: str, - disposition: str, - predecessor: str | None, - spool_dir: Path, - rung: str = "f100", -) -> Path: - return record_terminal_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - pipeline=_UK_CANDIDATE_PIPELINE, - rung=rung, - seed=seed, - code_pin=code_pin, - disposition=disposition, - predecessor=predecessor, - spool_dir=spool_dir, - ) - - -def _record_candidate_error( - *, - error: BaseException, - state: AttemptState, - started_at: float, - started_ts: datetime, - seed: int, - code_pin: str, - predecessor: str | None, - base_dir: Path, - spool_dir: Path, - rung: str = "f100", -) -> None: - error_path = write_error_receipt( - error_receipt_path(base_dir, build_id=state.build_id), - state=state, - pipeline=_UK_CANDIDATE_PIPELINE, - error=error, - ) - apply_error_verdict( - state, - f"{local_artifact_reference(error_path, repository_hint=_REPOSITORY)}#/error_type", - ) - _record_candidate_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - seed=seed, - code_pin=code_pin, - disposition="failed", - predecessor=predecessor, - spool_dir=spool_dir, - rung=rung, - ) - - def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -961,45 +795,6 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: return args -#: Solve arguments whose argparse default is ``None`` so an explicit value can -#: be told from the role's default: the other role's refusal table keys on -#: what was actually given. -_ROLE_DEFAULTED_ARGUMENTS = ( - "n_clones", - "seed", - "sample_seed", - "epochs", - "learning_rate", - "target_weight_rule", - "expected_constituency_vintage", -) - - -def _resolve_role_arguments(args: argparse.Namespace) -> UKRowwisePosture: - """Bind the declared role's posture and fill its defaults into unset arguments.""" - - posture = uk_rowwise_posture(args.release_role) - args._explicit_arguments = frozenset( - name for name in _ROLE_DEFAULTED_ARGUMENTS if getattr(args, name) is not None - ) - if args.n_clones is None: - args.n_clones = posture.clone_count - if args.seed is None: - args.seed = posture.seed - if args.sample_seed is None: - args.sample_seed = UK_SAMPLE_SEED_DEFAULT - if args.epochs is None: - args.epochs = posture.epochs - if args.learning_rate is None: - args.learning_rate = posture.learning_rate - if args.target_weight_rule is None: - args.target_weight_rule = posture.target_weight_rule - if args.expected_constituency_vintage is None: - args.expected_constituency_vintage = posture.expected_constituency_vintage - args._posture = posture - return posture - - def main(argv: list[str] | None = None) -> int: """Run the rowwise candidate build.""" @@ -2255,197 +2050,6 @@ def _national_manifest( } -def _hub_api() -> Any: - """The Hub client used for telemetry and the staged dataset (test seam).""" - - from huggingface_hub import HfApi - - return HfApi() - - -def _hub_token() -> str | None: - """The ambient Hub credential, if any (test seam).""" - - from huggingface_hub import get_token - - return get_token() - - -def _staged_dataset_mode(args: argparse.Namespace) -> str: - if args.no_staging or args.no_staged_dataset: - return "disabled" - if args.staging_local_only: - return "local_only" - return "local_and_remote" - - -def _preflight_staged_dataset(args: argparse.Namespace) -> None: - """Refuse a remote dataset stage the run could not complete. - - The bundle upload is the last step of a multi-hour run, so the credential - and the repository are checked before the spine is read. Telemetry stays - best-effort with no pre-flight, as on the national command. - """ - - if args.dry_run or _staged_dataset_mode(args) != "local_and_remote": - return - repo_id = str(args.staged_dataset_repo_id).strip() - hint = "pass --staging-local-only or --no-staged-dataset to keep the bundle local" - if not _hub_token(): - raise ValueError( - f"remote dataset staging to {repo_id} needs a Hugging Face write " - f"credential (HF_TOKEN or `hf auth login`); {hint}." - ) - storage = HuggingFaceDatasetStorage(repo_id, api=_hub_api()) - try: - storage.head_revision() - except Exception as error: - # The transport's own message is not chained: it can carry request - # URLs and identifiers, and the type name is enough to act on. - raise ValueError( - f"remote dataset staging cannot reach {repo_id} " - f"({type(error).__name__}); {hint}." - ) from None - _require_write_credential(storage, hint=hint) - - -def _require_write_credential(storage: HuggingFaceDatasetStorage, *, hint: str) -> None: - """Refuse a credential that can see the repository but cannot write it. - - A read token, or a fine-grained token scoped to another owner, passes the - reachability check and is refused by the Hub with 403 only when the upload - starts, hours later. The scope is read from the Hub's own description of - the token; when it cannot be read the upload itself is the proof. - """ - - try: - can_write = storage.credential_can_write() - except Exception: - can_write = None - if can_write is False: - raise ValueError( - f"remote dataset staging to {storage.repo_id} needs a write credential: " - "the ambient Hugging Face token is read-only or is not scoped to this " - "repository or its owner (a fine-grained token needs repo.write on " - f"{storage.repo_id} or on {storage.repo_id.split('/', 1)[0]}); {hint}." - ) - if can_write is None: - print( - "warning: the Hugging Face credential's write scope could not be read; " - "the upload at the end of the run will prove it.", - file=sys.stderr, - flush=True, - ) - - -def _create_staging_telemetry( - args: argparse.Namespace, *, build_id: str -) -> StagingTelemetryV2 | None: - if args.no_staging: - return None - local_only = bool(args.staging_local_only) - out_dir = args.out.expanduser().resolve() - posture = _posture_of(args) - return StagingTelemetryV2( - run_id=args.staging_run_id or build_id, - country_code="GB", - operation_id=posture.staging_operation_id, - pipeline_id=posture.pipeline, - pipeline_version=metadata.version("microcosm-build"), - candidate_id=args.staging_candidate_id or build_id, - local_dir=args.staging_dir or out_dir / "staging", - run_kind="calibration", - delivery_mode="local_only" if local_only else "local_and_remote", - repo_id=None if local_only else args.staging_repo_id, - upload_interval_seconds=args.staging_upload_interval_seconds, - api=None if local_only else _hub_api(), - ) - - -def _stage( - telemetry: StagingTelemetryV2 | None, - stage_id: str, - event_status: str = "started", - **details: Any, -) -> None: - """Forward one stage event to the best-effort telemetry. - - A contract or content refusal of the event is reported and the event - dropped; the build must never abort on its own progress report. Each - event is judged on its own details, so a refused event does not silence - the ones that follow. - """ - - if telemetry is None: - return - try: - telemetry.stage(stage_id, event_status=event_status, **details) - except StagingContractError as error: - print( - f"warning: staging telemetry refused the {stage_id!r} stage event " - f"({type(error).__name__}: {error}); the event is not staged, the " - "build continues.", - file=sys.stderr, - flush=True, - ) - - -def _staging_epoch_every(args: argparse.Namespace) -> int: - """The epoch stride that keeps the forwarded rows under the row budget. - - A dense run solves once; a size run solves the pool, up to ``budget_iters`` - full-length probes and the refit. The stride is at least - ``_STAGING_EPOCH_EVERY`` and grows so at most ``_STAGING_MAX_EPOCH_ROWS`` - epochs are forwarded, keeping ``calibration_progress.json`` and - ``events.ndjson`` under the contract's 5 MiB cap for any ``--epochs``. - """ - - solves = 1 if args.dataset_households is None else 2 + _BUDGET_ITERS - total = int(args.epochs) * solves - return max(_STAGING_EPOCH_EVERY, -(-total // _STAGING_MAX_EPOCH_ROWS)) - - -def _thinned_epochs( - sink: Callable[[Mapping[str, Any]], None], *, every: int = _STAGING_EPOCH_EVERY -) -> Callable[[dict[str, object]], None]: - """Forward every ``every``-th epoch and each phase's last epoch to ``sink``. - - The kernel flags probe epochs with ``budget_search: True``; the staging - contract records that field as an integer or null, so the flag becomes 1 - (the national command never runs a budget search and never met this). A - contract or content refusal from the telemetry is reported once and stops - the forwarding: the solve must never abort on its own progress report. - """ - - disabled = False - - def callback(event: dict[str, object]) -> None: - nonlocal disabled - if disabled or event.get("kind") != "calibration_epoch": - return - epoch = int(event["epoch"]) - epochs = int(event["epochs"]) - if epoch % every != 0 and epoch != epochs: - return - forwarded = dict(event) - budget_search = forwarded.get("budget_search") - if isinstance(budget_search, bool): - forwarded["budget_search"] = 1 if budget_search else None - try: - sink(forwarded) - except StagingContractError as error: - disabled = True - print( - "warning: staging telemetry refused a calibration progress row " - f"({type(error).__name__}); epoch progress is no longer forwarded, " - "the solve continues.", - file=sys.stderr, - flush=True, - ) - - return callback - - def _size_checkpoint_state(solve: UKRowwiseDoctrineSolve) -> str | None: if solve.size_receipt is None or not solve.size_receipt.get("checkpoint"): return None @@ -2457,349 +2061,6 @@ def _size_checkpoint_state(solve: UKRowwiseDoctrineSolve) -> str | None: return None -def _gate_statuses(gate_report: Mapping[str, Any]) -> dict[str, str]: - return { - str(gate_id): str(entry.get("status")) - for gate_id, entry in gate_report.get("gates", {}).items() - if isinstance(entry, Mapping) - } - - -def _fail_staging_telemetry( - telemetry: StagingTelemetryV2 | None, error: BaseException -) -> None: - if telemetry is None or telemetry.status != "running": - return - try: - telemetry.fail(error) - telemetry.validate_local_bundle() - except Exception: - pass - - -def _finalize_staging_telemetry( - args: argparse.Namespace, telemetry: StagingTelemetryV2 | None -) -> None: - if telemetry is None: - return - try: - telemetry.complete(message="UK rowwise candidate staging run completed.") - except StagingContractError as error: - _warn_telemetry("could not close the staging run", error) - return - try: - if args.staging_read_back: - # Requested explicitly, so a failed read-back is the run's failure, - # as on the national command. - telemetry.verify_remote() - finally: - try: - telemetry.validate_local_bundle() - except StagingContractError as error: - _warn_telemetry("the local staging bundle does not validate", error) - - -def _warn_telemetry(what: str, error: BaseException) -> None: - print( - f"warning: {what} ({type(error).__name__}: {error}); the build's own " - "evidence is unaffected.", - file=sys.stderr, - flush=True, - ) - - -def _staging_delivery(telemetry: StagingTelemetryV2 | None) -> dict[str, Any]: - if telemetry is None: - return disabled_staging_delivery("--no-staging") - return telemetry.delivery_summary - - -def _stage_dataset( - args: argparse.Namespace, - *, - manifest: Mapping[str, Any], - output_paths: Mapping[str, Path], - run_id: str, - telemetry: StagingTelemetryV2 | None, -) -> dict[str, Any]: - """Stage the published bundle under ``staged//``; record, never raise. - - The bundle is every file the manifest registers as an output plus the - manifest and two sidecars, verified from disk against the manifest's own - digests. Nothing else in the run directory is eligible. - """ - - mode = _staged_dataset_mode(args) - if mode == "disabled": - return disabled_staged_dataset( - "--no-staging" if args.no_staging else "--no-staged-dataset" - ) - repository = ( - None if mode == "local_only" else str(args.staged_dataset_repo_id).strip() - ) - _stage(telemetry, "dataset_staging", "started", mode=mode, repository=repository) - gate_statuses = _gate_statuses(getattr(args, "_gate_report", {}) or {}) - bundle = StagedDatasetBundle.from_manifest( - output_paths["manifest"].parent, - run_id=run_id, - manifest_name=MANIFEST_FILENAME, - extra_summary={ - "dataset_households": manifest["parameters"]["dataset_households"], - "pool_rows": manifest["solve"]["pool_households"], - "realized_households": manifest["solve"]["n_households"], - "final_loss": manifest["solve"]["final_loss"], - "release_posture": manifest["release_posture"], - "gate_statuses": gate_statuses, - }, - ) - telemetry_reference = ( - None - if telemetry is None - else { - "repository": telemetry.repo_id, - "prefix": telemetry.repo_run_prefix, - "mode": telemetry.delivery_mode, - } - ) - write_sidecars( - bundle, - repository=repository, - prefix=UK_STAGED_DATASET_PREFIX, - telemetry=telemetry_reference, - ) - if mode == "local_only": - delivery = local_only_staged_dataset(bundle, prefix=UK_STAGED_DATASET_PREFIX) - else: - print( - f"staging the dataset bundle to {repository} under " - f"{bundle.remote_prefix(UK_STAGED_DATASET_PREFIX)}...", - file=sys.stderr, - flush=True, - ) - storage = HuggingFaceDatasetStorage(repository, api=_hub_api()) - delivery = stage_bundle( - bundle, storage=storage, prefix=UK_STAGED_DATASET_PREFIX - ) - print(_staged_dataset_line(delivery), file=sys.stderr, flush=True) - _stage( - telemetry, - "dataset_staging", - "completed", - status=delivery["status"], - repository=delivery["repository"], - revision=delivery["revision"], - error_code=delivery["error_code"], - file_count=len(delivery["files"]), - ) - _add_staging_artifact( - telemetry, - "staged_dataset", - delivery, - artifact_kind="build_metadata", - classification="non_row_level", - ) - _add_staging_artifact( - telemetry, - "fit_summary", - _fit_summary( - manifest, - run_id=run_id, - gate_statuses=gate_statuses, - staged_dataset=delivery, - ), - artifact_kind="aggregate_diagnostics", - classification="aggregate", - ) - return delivery - - -def _staged_dataset_line(delivery: Mapping[str, Any]) -> str: - status = delivery["status"] - if status in ("uploaded", "already_staged"): - return ( - f"staged dataset: {status} at {delivery['repository']}/" - f"{delivery['prefix']} (revision {delivery['revision']})" - ) - if status == "failed": - return ( - f"staged dataset: failed ({delivery['error_code']}); the bundle and " - "its sidecars stay local and can be re-staged with " - "tools/stage_uk_rowwise_candidate.py" - ) - return ( - f"staged dataset: skipped ({delivery['mode']}); sidecars written beside " - "the bundle" - ) - - -def _add_staging_artifact( - telemetry: StagingTelemetryV2 | None, - logical_name: str, - payload: Mapping[str, Any], - *, - artifact_kind: str, - classification: str, -) -> None: - """Attach a reviewed aggregate JSON artifact to the telemetry run. - - A content-policy refusal is reported and skipped: the telemetry is - best-effort and must never fail a finished build. - """ - - if telemetry is None: - return - with tempfile.TemporaryDirectory(prefix=".staging-artifact.") as scratch: - source = Path(scratch) / f"{logical_name}.json" - source.write_text(_json_text(payload), encoding="utf-8") - try: - telemetry.add_artifact( - logical_name, - source, - artifact_kind=artifact_kind, - classification=classification, - ) - except StagingContractError as error: - print( - f"warning: staging artifact {logical_name} was refused by the " - f"content policy and is not staged: {error}", - file=sys.stderr, - flush=True, - ) - - -def _fit_summary( - manifest: Mapping[str, Any], - *, - run_id: str, - gate_statuses: Mapping[str, str], - staged_dataset: Mapping[str, Any], -) -> dict[str, Any]: - """Aggregate fit, gate and size evidence shaped for a reviewed artifact. - - The content policy admits JSON objects without arrays of objects, so the - per-family rows become mappings keyed by family and anything row-shaped - is dropped by :func:`_aggregate_only`. - """ - - solve = manifest["solve"] - fit = manifest.get("fit") or {} - size = solve.get("dataset_size") - parameters = manifest["parameters"] - return { - "schema_name": "microcosm.uk.rowwise-fit-summary", - "schema_version": 1, - "run_id": run_id, - "build_kind": manifest["build_kind"], - "releasable": manifest["releasable"], - "release_posture": _aggregate_only(manifest["release_posture"]), - "git_commit": manifest["git_commit"], - "git_dirty": manifest["git_dirty"], - "parameters": { - key: parameters.get(key) - for key in ( - "n_clones", - "dataset_households", - "seed", - "epochs", - "sample_fraction", - "release_candidate", - "skip_holdout", - "target_weight_rule", - ) - }, - "targets": { - "count": solve["n_targets"], - "by_kind": _aggregate_only(solve["n_targets_by_kind"]), - }, - "pool_rows": solve["pool_households"], - "realized_households": solve["n_households"], - "loss": { - "initial": solve["initial_loss"], - "final": solve["final_loss"], - "max_abs_relative_error": solve["max_abs_relative_error"], - "median_abs_relative_error": solve["median_abs_relative_error"], - }, - "fit_by_family": { - "local": _rows_by_key(fit.get("local_by_family"), key="family"), - "national": _rows_by_key(fit.get("national_by_family"), key="family"), - }, - "weakest_areas_by_fit": _aggregate_only(fit.get("weakest_areas_by_fit")), - "rotated_holdout": _aggregate_only(fit.get("rotated_holdout")), - "gates": dict(gate_statuses), - "failing_gate_ids": list(manifest.get("failing_gate_ids", [])), - "blocking_failure_count": len(manifest.get("blocking_failures", [])), - # The receipt's per-row arrays (pool_row_indices, inclusion - # probabilities) live in dataset_size_selection.csv and would push the - # artifact past the 5 MiB cap on a real run. - "dataset_size": None - if size is None - else _aggregate_only( - { - key: value - for key, value in size.items() - if key not in ("pool_row_indices", "inclusion_probabilities") - } - ), - "staged_dataset": { - key: staged_dataset[key] - for key in ("repository", "prefix", "revision", "status", "error_code") - }, - } - - -def _rows_by_key(rows: Any, *, key: str) -> dict[str, Any]: - """Turn a list of row mappings into a mapping keyed by ``row[key]``.""" - - if not isinstance(rows, list): - return {} - keyed: dict[str, Any] = {} - for row in rows: - if not isinstance(row, Mapping) or key not in row: - continue - keyed[str(row[key])] = _aggregate_only( - {name: value for name, value in row.items() if name != key} - ) - return keyed - - -def _aggregate_only(value: Any) -> Any: - """Drop row-shaped data (lists holding mappings) recursively.""" - - if isinstance(value, Mapping): - kept = {} - for name, item in value.items(): - cleaned = _aggregate_only(item) - if cleaned is not _DROPPED: - kept[str(name)] = cleaned - return kept - if isinstance(value, (list, tuple)): - if any(isinstance(item, Mapping) for item in value): - return _DROPPED - return [ - item - for item in (_aggregate_only(entry) for entry in value) - if item is not _DROPPED - ] - return value - - -_DROPPED = object() - - -def _replace_manifest(path: Path, manifest: Mapping[str, Any]) -> None: - """Rewrite the published manifest atomically with appended evidence.""" - - handle, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) - temporary_path = Path(temporary) - try: - with open(handle, "w", encoding="utf-8") as stream: - stream.write(_json_text(manifest)) - temporary_path.replace(path) - except BaseException: - temporary_path.unlink(missing_ok=True) - raise - - def _clone_with_ladder_binding( dataset: Any, ladder: UkOaLadder, @@ -3207,28 +2468,6 @@ def _joint_dry_run_plan( } -def _local_vintage_census(registry: TargetRegistry) -> list[dict[str, object]]: - counts: dict[tuple[str, str, str, str], int] = {} - for spec in registry.specs: - resolved = str(spec.metadata.get("ledger_fact_period", "")) - target = str(spec.period) - if not resolved or resolved == target: - continue - level, _ = _spec_geography(spec) - key = (spec.family, level, resolved, target) - counts[key] = counts.get(key, 0) + 1 - return [ - { - "family": family, - "geography_level": level, - "resolved_period": resolved, - "target_period": target, - "cells": cells, - } - for (family, level, resolved, target), cells in sorted(counts.items()) - ] - - def _build_bound_problem( assignment: _LadderAssignment, *, @@ -3518,73 +2757,6 @@ def _run_local_gate_battery( return payload, ladder.result -def _gate_failures_by_criticality( - gate_report: Mapping[str, Any], -) -> tuple[list[str], list[str]]: - """Split a persisted battery report's failure lines by criticality. - - Returns ``(release_blocking, diagnostic)``, each entry-prefixed like - :class:`GateBatteryBlockedError`'s lines. Only ``failed`` and - ``evidence_absent`` entries are failures; ``not_applicable`` and - ``unreached`` entries are not. - """ - - blocking: list[str] = [] - diagnostic: list[str] = [] - gates = gate_report.get("gates", {}) - if not isinstance(gates, Mapping): - return blocking, diagnostic - for gate_id, payload in gates.items(): - if not isinstance(payload, Mapping): - continue - status = payload.get("status") - if status not in {"failed", "evidence_absent"}: - continue - lines = [f"[{gate_id}] {line}" for line in payload.get("failures") or ()] - if not lines: - lines = [f"[{gate_id}] {payload.get('reason') or status}"] - bucket = blocking if _is_release_blocking(payload) else diagnostic - bucket.extend(lines) - return blocking, diagnostic - - -def _is_release_blocking(payload: Mapping[str, Any]) -> bool: - """Fail-closed criticality read. - - Only an entry that explicitly declares ``criticality: diagnostic`` is - exempt from vetoing the release; a missing or unknown criticality is - treated as release-blocking, so partial schema drift on one persisted - entry cannot drop a failed gate out of both the blocking list and - ``all_gates_passed``. - """ - - return payload.get("criticality") != "diagnostic" - - -def _release_verdict( - *, - sample_fraction: float, - engine_blocks: int, - release_blocking_gates_passed: bool, -) -> tuple[bool, dict[str, bool]]: - """``releasable`` needs the full rung, a single-block engine resolution and - every release-blocking gate passed. - - Per-block engine resolution mis-measures population-normalised formulas - (each block reproduces a national aggregate: the ×K land-value artefact - behind the #736 erratum), so a run resolved in more than one block is - diagnostic-only whatever its gates say. The posture is written beside the - verdict so a reader sees which leg failed. - """ - - posture = { - "full_rung": float(sample_fraction) == 1.0, - "single_block_engine": int(engine_blocks) == 1, - "release_blocking_gates_passed": bool(release_blocking_gates_passed), - } - return all(posture.values()), posture - - def _apply_gate_verdicts( state: AttemptState, report: Mapping[str, object], @@ -4134,55 +3306,6 @@ def _manifest( } -def _parameters(args: argparse.Namespace, *, source_year: int) -> dict[str, Any]: - posture = _posture_of(args) - return { - "release_role": posture.role, - "n_clones": None if args.n_clones is None else int(args.n_clones), - "dataset_households": args.dataset_households, - "seed": int(args.seed), - "selection_seed": None - if args.dataset_households is None - else int(args.seed if args.selection_seed is None else args.selection_seed), - "selection_pi_hi": None - if args.dataset_households is None - else float(args.selection_pi_hi), - "baseline_pi_floor": None - if args.dataset_households is None - else float(args.baseline_pi_floor), - "size_checkpoint": bool( - args.dataset_households is not None - and not args.no_size_checkpoint - and args.resume_size_checkpoint is None - ), - "resume_size_checkpoint": None - if args.resume_size_checkpoint is None - else str(args.resume_size_checkpoint.expanduser().resolve()), - "source_year": source_year, - "source_lineage_modulus": args.source_lineage_modulus, - "sample_fraction": float(args.sample_fraction), - "sample_seed": int(args.sample_seed), - "engine_blocks": int(args.engine_blocks), - "target_weight_rule": args.target_weight_rule, - "release_candidate": bool(args.release_candidate), - "skip_holdout": bool(args.skip_holdout), - "epochs": int(args.epochs), - "learning_rate": float(args.learning_rate), - "expected_constituency_vintage": ( - None - if args.expected_constituency_vintage is None - else str(args.expected_constituency_vintage) - ), - "doctrine": _doctrine_bounds(posture), - "solve_options": { - "conserve_mass": _CONSERVE_MASS, - "target_records": _TARGET_RECORDS, - "l0_lambda": _L0_LAMBDA, - "budget_iters": _BUDGET_ITERS, - }, - } - - def _design_weights_for(solve: UKRowwiseDoctrineSolve) -> np.ndarray: """The pool design weights aligned to the solve's exported rows.""" if solve.selected_support is None or solve.dense_reference is None: @@ -4313,10 +3436,6 @@ def _local_output_registry( return TargetRegistry(specs, country="uk") -def _doctrine_bounds(posture: UKRowwisePosture) -> dict[str, Any]: - return posture.doctrine_bounds() - - def _gate_payload(gate: GateResult, *, phase: str) -> dict[str, Any]: return { "name": str(gate.name), @@ -4384,219 +3503,6 @@ def _validate_support_summary(support: pd.DataFrame) -> None: raise RuntimeError("area support summary contains invalid values.") -def _validate_cli_args(args: argparse.Namespace) -> None: - posture = _posture_of(args) - # The declared role is checked against the parameters first: the other - # role's flags are refused by name before any value is range-checked. - if posture.role == "national": - _refuse_dense_role_arguments(args, posture) - else: - _refuse_national_role_arguments(args, posture) - if args.selection_seed is not None and args.dataset_households is None: - raise ValueError("--selection-seed requires --dataset-households.") - if not (0.0 < args.selection_pi_hi <= 1.0): - raise ValueError("--selection-pi-hi must be in (0, 1].") - if args.selection_pi_hi != 1.0 and args.dataset_households is None: - raise ValueError("--selection-pi-hi requires --dataset-households.") - if not (0.0 <= args.baseline_pi_floor <= 1.0): - raise ValueError("--baseline-pi-floor must be in [0, 1].") - if args.baseline_pi_floor != 0.0 and args.dataset_households is None: - raise ValueError("--baseline-pi-floor requires --dataset-households.") - if args.no_size_checkpoint and args.dataset_households is None: - raise ValueError("--no-size-checkpoint requires --dataset-households.") - if args.resume_size_checkpoint is not None: - if args.dataset_households is None: - raise ValueError("--resume-size-checkpoint requires --dataset-households.") - if args.no_size_checkpoint: - raise ValueError( - "--resume-size-checkpoint already implies no new checkpoint; " - "drop --no-size-checkpoint." - ) - if args.dataset_households is not None: - if args.dataset_households <= 0: - raise ValueError("--dataset-households must be positive.") - if args.release_candidate: - raise ValueError( - "--dataset-households is candidate-only: size-specific matched comparison and promotion scorecard are required before release." - ) - # Size-selection arguments are validated first so their refusals name - # the size flag at fault; the pinned Ledger inputs are then mandatory. - ledger_values = ( - args.ledger_facts, - args.ledger_facts_sha256, - args.ledger_manifest_sha256, - ) - if not all(value is not None for value in ledger_values): - raise ValueError( - "--ledger-facts, --ledger-facts-sha256, and " - "--ledger-manifest-sha256 are mandatory and must be supplied together." - ) - if posture.ladder_required: - if args.ladder is None: - raise ValueError("--release-role dense requires --ladder.") - if args.input_sha256 is None or args.ladder_sha256 is None: - raise ValueError( - "the joint registry path requires --input-sha256 and --ladder-sha256." - ) - elif args.input_sha256 is None: - raise ValueError("--release-role national requires --input-sha256.") - if args.release_candidate: - required_release = { - "--input-sha256": args.input_sha256, - "--ladder-sha256": args.ladder_sha256, - "--ledger-facts": args.ledger_facts, - "--ledger-facts-sha256": args.ledger_facts_sha256, - "--ledger-manifest-sha256": args.ledger_manifest_sha256, - } - missing_release = [ - name for name, value in required_release.items() if value is None - ] - if missing_release: - raise ValueError( - "--release-candidate requires pinned joint inputs: " - + ", ".join(missing_release) - ) - refused = [] - if args.target_weight_rule != posture.target_weight_rule: - refused.append("--target-weight-rule") - if args.epochs != posture.epochs: - refused.append(f"--epochs != doctrine {posture.epochs}") - if args.n_clones != posture.clone_count: - refused.append(f"--n-clones != doctrine {posture.clone_count}") - if args.measure_exclusions is not None: - refused.append("--measure-exclusions") - if args.skip_holdout: - refused.append("--skip-holdout") - if args.engine_blocks > 1: - refused.append("--engine-blocks > 1") - if args.sample_fraction != 1.0: - refused.append("--sample-fraction != 1.0") - if refused: - raise ValueError( - "--release-candidate refuses non-release settings: " - + ", ".join(refused) - ) - if args.n_clones is not None and args.n_clones <= 0: - raise ValueError("--n-clones must be positive.") - if args.seed < 0: - raise ValueError("--seed must be non-negative.") - if args.sample_fraction not in UK_SAMPLE_RUNG_TOKENS: - raise ValueError( - "--sample-fraction must be one of " - f"{sorted(UK_SAMPLE_RUNG_TOKENS)}, got {args.sample_fraction!r}." - ) - if args.sample_seed < 0: - raise ValueError("--sample-seed must be non-negative.") - if args.engine_blocks <= 0: - raise ValueError("--engine-blocks must be positive.") - if args.engine_blocks > 1 and args.engine_blocks != args.n_clones: - raise ValueError("--engine-blocks greater than one must equal --n-clones.") - if args.source_year is not None and args.source_year <= 0: - raise ValueError("--source-year must be positive.") - if args.epochs <= 0: - raise ValueError("--epochs must be positive.") - if not np.isfinite(args.learning_rate) or args.learning_rate <= 0: - raise ValueError("--learning-rate must be positive and finite.") - if args.target_loss_cap is not None and ( - not np.isfinite(args.target_loss_cap) or args.target_loss_cap <= 0 - ): - raise ValueError("--target-loss-cap must be positive and finite.") - if ( - args.expected_constituency_vintage is not None - and not str(args.expected_constituency_vintage).strip() - ): - raise ValueError("--expected-constituency-vintage must be non-empty.") - - -def _refuse_dense_role_arguments( - args: argparse.Namespace, posture: UKRowwisePosture -) -> None: - """The national role's refusal table: nothing of the clone surface may be given. - - ``--release-candidate`` is refused outright with the seam's own reason: the - calibration-seam battery covers six of the declared entries and must never - sign a shippability claim; a national cut's verdict comes only from the - release-cut certification producer (``tools/certify_uk_release_cut.py``). - """ - - if args.release_candidate: - raise ValueError( - "--release-candidate is refused on the national role: the " - "calibration seam's scoped battery cannot sign shippability; run " - "the release-cut certification producer " - "(tools/certify_uk_release_cut.py) on the finished build instead." - ) - explicit = args._explicit_arguments - refused: list[str] = [] - if args.ladder is not None: - refused.append("--ladder") - if args.ladder_sha256 is not None: - refused.append("--ladder-sha256") - if "expected_constituency_vintage" in explicit: - refused.append("--expected-constituency-vintage") - if args.source_year is not None: - refused.append("--source-year") - if args.source_lineage_modulus is not None: - refused.append("--source-lineage-modulus") - if "n_clones" in explicit: - refused.append("--n-clones") - if args.candidate_clone_counts is not None: - refused.append("--candidate-clone-counts") - if args.engine_blocks != 1: - refused.append("--engine-blocks") - if args.households_only: - refused.append("--households-only") - if args.skip_holdout: - refused.append("--skip-holdout") - if args.dataset_households is not None: - refused.append("--dataset-households") - if args.selection_seed is not None: - refused.append("--selection-seed") - if args.selection_pi_hi != 1.0: - refused.append("--selection-pi-hi") - if args.baseline_pi_floor != 0.0: - refused.append("--baseline-pi-floor") - if args.no_size_checkpoint: - refused.append("--no-size-checkpoint") - if args.resume_size_checkpoint is not None: - refused.append("--resume-size-checkpoint") - if args.sample_fraction != 1.0: - refused.append("--sample-fraction") - if "sample_seed" in explicit: - refused.append("--sample-seed") - if "seed" in explicit and args.seed != posture.seed: - # The seam doctrine's seed is a reviewed constant, not a knob. - refused.append(f"--seed != doctrine {posture.seed}") - if args.target_weight_rule not in posture.allowed_target_weight_rules: - refused.append(f"--target-weight-rule {args.target_weight_rule}") - if refused: - raise ValueError( - "--release-role national refuses the dense role's arguments: " - + ", ".join(refused) - ) - - -def _refuse_national_role_arguments( - args: argparse.Namespace, posture: UKRowwisePosture -) -> None: - """The dense role's refusal table: the seam's knobs are not its own.""" - - refused: list[str] = [] - if args.target_loss_cap is not None: - refused.append("--target-loss-cap") - if args.allow_unpinned_feed: - refused.append("--allow-unpinned-feed") - if args.incumbent_h5 is not None or args.incumbent_sha256 is not None: - refused.append("--incumbent-h5/--incumbent-sha256") - if args.target_weight_rule not in posture.allowed_target_weight_rules: - refused.append(f"--target-weight-rule {args.target_weight_rule}") - if refused: - raise ValueError( - "--release-role dense refuses the national role's arguments: " - + ", ".join(refused) - ) - - def _load_candidate_evaluator(importer=importlib.import_module): """The common-surface scorer (microcosm#967), loaded when an incumbent is given. @@ -4770,40 +3676,6 @@ def _evaluate_against_incumbent( } -def _output_paths( - out_dir: Path, - *, - posture: UKRowwisePosture, - vintage: str, -) -> dict[str, Path]: - """The role's output paths for one FRS release vintage (``2024_25``).""" - - dataset = out_dir / posture.dataset_filename(vintage) - if posture.role == "national": - return { - "dataset": dataset, - "manifest": out_dir / MANIFEST_FILENAME, - "calibration_diagnostics": out_dir / CALIBRATION_DIAGNOSTICS_FILENAME, - "build_record": out_dir / BUILD_RECORD_FILENAME, - "terminal_gates": out_dir / posture.gate_report_filename(vintage), - "national_registry": out_dir / NATIONAL_REGISTRY_FILENAME, - "contract_registry": out_dir / NATIONAL_CONTRACT_REGISTRY_FILENAME, - "score_receipt": out_dir / SCORE_RECEIPT_FILENAME, - } - return { - "dataset": dataset, - "manifest": out_dir / MANIFEST_FILENAME, - "diagnostics": out_dir / SOLVE_DIAGNOSTICS_FILENAME, - "support": out_dir / AREA_SUPPORT_FILENAME, - "past_cap": out_dir / PAST_CAP_FILENAME, - "calibration_diagnostics": out_dir / CALIBRATION_DIAGNOSTICS_FILENAME, - "local_gates": out_dir / posture.gate_report_filename(vintage), - "local_registry": out_dir / LOCAL_REGISTRY_FILENAME, - "dense_reference": out_dir / DENSE_REFERENCE_DIAGNOSTICS_FILENAME, - "selection": out_dir / DATASET_SIZE_SELECTION_FILENAME, - } - - def _validate_output_paths( output_paths: Mapping[str, Path], *, @@ -4829,50 +3701,6 @@ def _validate_output_paths( ) -def _publish_staged_files( - staged: Mapping[str, Path], - output_paths: Mapping[str, Path], -) -> None: - out_dir = output_paths["manifest"].parent - created_out_dir = not out_dir.exists() - out_dir.mkdir(parents=True, exist_ok=True) - publish_order = ( - "dataset", - "diagnostics", - "support", - "past_cap", - "calibration_diagnostics", - "local_registry", - "dense_reference", - "selection", - "manifest", - ) - published: list[Path] = [] - succeeded = False - try: - for key in publish_order: - if key in _SIZE_RUN_ONLY_OUTPUTS and not staged[key].exists(): - continue - destination = output_paths[key] - if destination.exists(): - raise FileExistsError( - "candidate output appeared during publication; refusing " - f"to overwrite {destination}." - ) - staged[key].replace(destination) - published.append(destination) - succeeded = True - finally: - if not succeeded: - for path in reversed(published): - path.unlink(missing_ok=True) - if created_out_dir: - try: - out_dir.rmdir() - except OSError: - pass - - def _assert_artifacts_unchanged( *, input_h5: Path, @@ -4928,47 +3756,5 @@ def _artifact_info( } -def _git_commit() -> str | None: - result = subprocess.run( - ["git", "rev-parse", "HEAD"], - check=False, - capture_output=True, - text=True, - ) - if result.returncode != 0: - return None - return result.stdout.strip() - - -def _git_dirty() -> bool | None: - """Measured, not asserted: tracked modifications in the working tree. - - ``None`` when git cannot answer (no repository), so a downstream - assembler records the pin as unmeasured rather than clean. - """ - - result = subprocess.run( - ["git", "status", "--porcelain", "--untracked-files=no"], - check=False, - capture_output=True, - text=True, - ) - if result.returncode != 0: - return None - return bool(result.stdout.strip()) - - -def _json_text(payload: Any) -> str: - return ( - json.dumps( - payload, - allow_nan=False, - indent=2, - sort_keys=True, - ) - + "\n" - ) - - if __name__ == "__main__": raise SystemExit(main()) From 6c5283eddeafce997cefa2a393e4cb8413956dc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 25 Sep 2026 20:33:26 +0100 Subject: [PATCH 23/44] Dispatch the national release role to the retained seam engine and stub the candidate tool over the graph driver microcosm-build-uk --release-role national now serves main's national line by identity: the tool's national branch (target inputs with the feed-pin check, doctrine overrides from explicit flags, the seam run through run_uk_calibration, the schema-4 national manifest, the incumbent evaluation and score receipt, the sha256sums entry) moves into uk_runtime/national_role.py unchanged and the graph driver dispatches to run_national_role after validation and before any graph preparation; a national --spine-request is refused because the seam reads a pinned input H5. The seam engine, the national calibration stage, the release-cut battery and the #967 certifier stay exactly as main has them, so the national-role, posture, scoring-route and certify tests keep passing. tools/build_uk_rowwise_candidate.py is a stub over full_build_cli.main (both roles); tools/evaluate_uk_incumbent_surface.py, the CGT observation-period test and the gate-register pin test stop loading the tool by path and read the package (full_measure.resolve_uk_full_measures, the dense posture). tools/build_uk_rowwise_dataset.py stays main's: main's driver and ladder-clone tests load it by path and it still serves --candidate-clone-counts. Tests: test_uk_rowwise_candidate drops the 25 in-process tests whose subject was the retired tool body (listed in the PR body), keeps the 40 CLI and role cases on the graph driver, retargets five tests at rowwise_cli/rowwise_staging and pins the stub; test_uk_rowwise_national_role retargets its 20 seam-consumer patch sites at national_role; test_uk_full_build_cli gains the dispatch, dry-run and spine-request cases. Coverage gap to close in the docs phase: --staging-local-only and remote staging on the graph driver are no longer pinned by a test. Verified: whole uk group 2,571 passed / 20 skipped; primary 90 + secondary 140 + spot-check 145 passed; ci_test_groups --verify ok; ruff clean; both entry points' --help exit 0. Co-Authored-By: Claude Fable 5.1 --- .../microcosm/build/uk_runtime/__init__.py | 2 + .../build/uk_runtime/full_build_cli.py | 38 +- .../build/uk_runtime/national_role.py | 914 ++++ .../shared/test_gate_battery_contract_pins.py | 14 +- .../uk/test_uk_cgt_observation_period.py | 14 +- .../engine_free/uk/test_uk_full_build_cli.py | 90 +- .../uk/test_uk_rowwise_candidate.py | 2726 +----------- .../uk/test_uk_rowwise_national_role.py | 111 +- .../microcosm_build/uk_full_build_cli.py | 21 + .../microcosm_build/uk_rowwise_candidate.py | 94 +- tools/build_uk_rowwise_candidate.py | 3764 +---------------- tools/evaluate_uk_incumbent_surface.py | 16 +- 12 files changed, 1239 insertions(+), 6565 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/national_role.py diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index 8cd7bae7a..b6961763e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -403,6 +403,7 @@ validate_uk_national_frame, write_uk_national_frame, ) +from microcosm.build.uk_runtime.national_role import run_national_role from microcosm.build.uk_runtime.national_sampling import ( sample_uk_spine_frame, uk_spine_source_family_units, @@ -791,6 +792,7 @@ "UK_NATIONAL_TARGET_WEIGHT_RULE", "uk_doctrine_with_overrides", "uk_national_target_loss_weights", + "run_national_role", "UKNationalStage", "UKRowwiseDatasetResult", "UKReleaseInputColumn", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py index 6cdd5152c..2ba6bfc33 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py @@ -7,12 +7,14 @@ ``--release-role`` declares which UK dataset line the run builds and is required (microcosm#823): ``dense`` is the K-clone joint national + local surface under the local doctrine, built here through the graph; ``national`` -is parsed and validated by the same posture-aware validator but is served by -the retained calibration seam (``tools/build_uk_rowwise_candidate.py``) until -the graph dispatch lands in the next commit. The role supplies every unset -solve default and refuses the other role's flags through -:mod:`microcosm.build.uk_runtime.rowwise_cli`, so the graph driver and the -rowwise tool parse, default and refuse identically. +is parsed and validated by the same posture-aware validator and then +dispatched, before any graph preparation, to the retained calibration seam +through :mod:`microcosm.build.uk_runtime.national_role` (its Logbook row, +staging telemetry, manifest and staged bundle live there). The role supplies +every unset solve default and refuses the other role's flags through +:mod:`microcosm.build.uk_runtime.rowwise_cli`. ``tools/build_uk_rowwise_candidate.py`` +and ``tools/build_uk_full.py`` are stubs over :func:`main`, so every UK line +is built by this one command. A non-dry dense run is wrapped in the rowwise tool's operational envelope: the Logbook attempt (a spooled row under ``/logbook-spool`` on every @@ -71,6 +73,7 @@ validate_staging_arguments, ) from ..staging_dataset import SHA256SUMS_FILENAME, refresh_sha256sums_entry +from . import national_role from .calibration_run import runtime_provenance from .chronicle_feed import load_uk_chronicle_feed from .frs_release import load_uk_frs_release @@ -191,10 +194,9 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: help=( "Which UK dataset line this run builds: 'dense' (the K-clone joint " "national + local surface under the local doctrine, built through " - "the graph) or 'national' (validated here; served by the retained " - "calibration seam until the graph dispatch lands). The role " - "supplies every unset solve default and refuses the other role's " - "flags." + "the graph) or 'national' (the certified national line, dispatched " + "to the retained calibration seam). The role supplies every unset " + "solve default and refuses the other role's flags." ), ) population = parser.add_mutually_exclusive_group(required=True) @@ -326,9 +328,8 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: type=Path, help="Import an identity-verified historical size search, skipping its dense solve and search.", ) - # The national role's own knobs are declared so the dense refusal table - # can name them; the national role itself is served by the calibration - # seam until its graph dispatch lands. + # The national role's own knobs: the dense refusal table names them and + # the national dispatch (``national_role``) reads them. parser.add_argument( "--target-loss-cap", type=float, @@ -1415,14 +1416,9 @@ def main(argv: list[str] | None = None) -> int: validate_cli_args(args) posture = posture_of(args) if posture.role == "national": - print( - "error: --release-role national is validated here but served by the " - "retained calibration seam (tools/build_uk_rowwise_candidate.py) " - "until the graph national dispatch lands in the next commit " - "(microcosm#901 phase 4).", - file=sys.stderr, - ) - raise SystemExit(2) + # The national line is the retained calibration seam under the + # driver's posture (microcosm#823); no graph is prepared for it. + return national_role.run_national_role(args) if args.candidate_clone_counts is not None and not args.dry_run: raise ValueError("--candidate-clone-counts is valid only with --dry-run.") if args.dry_run: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_role.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_role.py new file mode 100644 index 000000000..7d49f7310 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_role.py @@ -0,0 +1,914 @@ +"""The UK rowwise driver's national release role (microcosm#823). + +``microcosm-build-uk --release-role national`` builds the certified national +line by dispatching to the retained calibration seam +(:func:`~microcosm.build.uk_runtime.calibration_run.run_uk_calibration`): no +cloning, national targets only, the seam doctrine with explicit flags as +receipted overrides, the six calibration-seam gates, the seam-shaped +``build_record.json`` that ``tools/certify_uk_release_cut.py`` certifies, a +frozen ``national_target_registry.json`` for the scorer, and the same +staging telemetry (run id = the attempt id) and staged bundle as the dense +role. The driver adds the pinned input, the rowwise manifest beside the +seam's evidence, and the optional end-of-build evaluation against the +incumbent (microcosm#578 rule 1 on the common surface). + +These functions were moved from ``tools/build_uk_rowwise_candidate.py`` +(microcosm#901 phase 4) and consume only the shared rowwise command surface +(:mod:`~microcosm.build.uk_runtime.rowwise_cli`, +:mod:`~microcosm.build.uk_runtime.rowwise_staging`) and package APIs; the +graph full-build driver (:mod:`~microcosm.build.uk_runtime.full_build_cli`) +dispatches here before any graph preparation. The private spellings are +kept as aliases so the drivers' tests can patch the names they always did. +The posture-driven graph national path (the next PR on this line) replaces +this dispatch. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib +import json +import sys +from collections.abc import Mapping +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.ledger_artifact import load_ledger_consumer_artifact +from microcosm.build.logbook_adoption import atomic_write_json +from microcosm.build.staging_dataset import ( + SHA256SUMS_FILENAME, + parse_sha256sums, + refresh_sha256sums_entry, +) +from microcosm.build.staging_v2 import StagingTelemetryV2 +from microcosm.build.uk_runtime.calibration_run import ( + UKCalibrationRunPaths, + new_uk_calibration_attempt_id, + run_uk_calibration, + runtime_provenance, +) +from microcosm.build.uk_runtime.chronicle_feed import ( + require_committed_uk_chronicle_feed_pin, +) +from microcosm.build.uk_runtime.diagnostics import uk_fit_by_family +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release +from microcosm.build.uk_runtime.ledger_targets import compile_uk_target_registry +from microcosm.build.uk_runtime.measure_simulation import ( + UKMeasureResolver, + apply_uk_calibration_measure_exclusions, + load_uk_calibration_measure_exclusions, +) +from microcosm.build.uk_runtime.national_doctrine import uk_doctrine_with_overrides +from microcosm.build.uk_runtime.rowwise_cli import ( + git_commit, + git_dirty, + json_text, + output_paths, + posture_of, + rowwise_parameters, +) +from microcosm.build.uk_runtime.rowwise_posture import UKRowwisePosture +from microcosm.build.uk_runtime.rowwise_staging import ( + add_staging_artifact, + create_staging_telemetry, + fail_staging_telemetry, + finalize_staging_telemetry, + gate_statuses, + preflight_staged_dataset, + replace_manifest, + stage, + stage_dataset, + staging_delivery, + staging_epoch_every, + thinned_epochs, +) +from microcosm.calibrate import TargetRegistry + +__all__ = ["national_dry_run", "run_national_role"] + + +def _read_json(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"{path} must hold a JSON object.") + return payload + + +def _ledger_facts_pin(artifact: Any) -> dict[str, object]: + facts_path = ( + artifact.path / "consumer_facts.jsonl" + if artifact.path.is_dir() + else artifact.path + ) + return {"sha256": artifact.facts_sha256, "size_bytes": facts_path.stat().st_size} + + +def _load_national_target_inputs(args: argparse.Namespace) -> dict[str, Any]: + """The national role's target surface: the pinned Ledger artifact, compiled. + + The artifact must be the committed Chronicle feed pin unless + ``--allow-unpinned-feed`` records a reviewed diagnostic run; the + compiled register, less the measure exclusions, is the solve surface and + the full compiled register keeps the band edges; ``--register-json`` + requires the re-derived register to be the frozen scoring surface. + """ + + artifact = load_ledger_consumer_artifact( + args.ledger_facts, + expected_facts_sha256=args.ledger_facts_sha256, + expected_manifest_sha256=args.ledger_manifest_sha256, + ) + pin = require_committed_uk_chronicle_feed_pin( + artifact.facts_sha256, + manifest_sha256=artifact.manifest_sha256, + allow_unpinned_feed=bool(args.allow_unpinned_feed), + ) + calibration_year = int(load_uk_frs_release().calibration_year) + compilation = compile_uk_target_registry( + artifact.facts, target_period=calibration_year + ) + if compilation.unsupported: + raise SystemExit( + f"{len(compilation.unsupported)} national target references " + "failed to compile" + ) + exclusions = load_uk_calibration_measure_exclusions(args.measure_exclusions) + registry, exclusion_receipt = apply_uk_calibration_measure_exclusions( + compilation.registry, exclusions + ) + if args.register_json is not None: + try: + frozen = TargetRegistry.from_json(args.register_json) + except ValueError as error: + raise SystemExit( + f"error: frozen scoring register is unusable: {error}" + ) from error + if frozen.version != registry.version: + raise SystemExit( + "re-derived register differs from the frozen scoring register: " + f"{registry.version} vs {frozen.version}" + ) + return { + "artifact": artifact, + "calibration_year": calibration_year, + "national_registry": registry, + "band_edge_registry": compilation.registry, + "measure_exclusions": exclusion_receipt, + "chronicle_feed_pin": pin.to_dict(), + } + + +def _national_doctrine_overrides(args: argparse.Namespace) -> dict[str, Any]: + """The receipted per-run overrides of the seam doctrine, explicit flags only.""" + + explicit = args._explicit_arguments + overrides: dict[str, Any] = {} + if "epochs" in explicit: + overrides["epochs"] = int(args.epochs) + if "learning_rate" in explicit: + overrides["learning_rate"] = float(args.learning_rate) + if "target_weight_rule" in explicit: + overrides["target_weight_rule"] = str(args.target_weight_rule) + if args.target_loss_cap is not None: + overrides["target_loss_cap"] = float(args.target_loss_cap) + return overrides + + +def _require_bound_input(args: argparse.Namespace) -> None: + """The national role builds from a bound spine checkpoint, never a request. + + The graph driver admits ``--spine-request`` as the dense build's + population source; the seam engine reads a pinned ``--input-h5``. + """ + + if args.input_h5 is None: + raise ValueError( + "--release-role national builds from a bound spine checkpoint: pass " + "--input-h5 with --input-sha256 (--spine-request executes the spine " + "stages only in the graph dense build)." + ) + + +def national_dry_run(args: argparse.Namespace) -> int: + """Compile the national target surface and print the plan; write nothing.""" + + posture = posture_of(args) + input_h5 = _require_file(args.input_h5, label="--input-h5") + input_artifact = _artifact_info(input_h5) + _verify_requested_pin("--input-h5", input_artifact, requested=args.input_sha256) + frs_release = load_uk_frs_release() + inputs = _load_national_target_inputs(args) + doctrine, doctrine_overrides = uk_doctrine_with_overrides( + **_national_doctrine_overrides(args) + ) + plan = { + "schema_version": 3, + "build_kind": "uk_national_calibrated_candidate_plan", + "release_role": posture.role, + "release_id": posture.release_id, + "dry_run": True, + "calibration_year": inputs["calibration_year"], + "inputs": {"dataset": dict(input_artifact)}, + "targets": { + "chronicle": inputs["artifact"].provenance(), + "compiled": len(inputs["band_edge_registry"].specs), + "active": len(inputs["national_registry"].specs), + "excluded": len(inputs["measure_exclusions"]), + "register_sha256": inputs["national_registry"].version, + }, + "doctrine": { + field: getattr(doctrine, field) + for field in ( + "epochs", + "learning_rate", + "max_weight_ratio", + "seed", + "target_loss_cap", + "scale_rule", + "target_weight_rule", + "mass_rule", + "l0_lambda", + ) + }, + "doctrine_overrides": dict(doctrine_overrides), + "parameters": rowwise_parameters( + args, + source_year=_source_year( + args.source_year, time_period=str(frs_release.time_period) + ), + ), + "engine": "not_run", + "incumbent": _incumbent_arguments(args), + "releasable": False, + } + print(json_text(plan)) + return 0 + + +_national_dry_run = national_dry_run + + +def run_national_role(args: argparse.Namespace) -> int: + """Serve ``--release-role national``: the dry-run plan or the seam build. + + The validated request arrives from the driver's ``main`` before any graph + preparation. A dry run plans without solving, writing or staging; a build + runs the staged-dataset pre-flight (argument refusals cost nothing; the + credential check reaches the Hub, so it runs last, still before any input + is read) and then the seam build under the driver's posture. + """ + + _require_bound_input(args) + if args.dry_run: + return national_dry_run(args) + preflight_staged_dataset(args) + return _run_national_role(args) + + +def _run_national_role(args: argparse.Namespace) -> int: + """Build the national line: the calibration seam under the driver's posture. + + The seam library (:func:`run_uk_calibration`) resolves the measures from + the input file, solves under the seam doctrine, runs the six + calibration-seam gates, writes the H5, the diagnostics, the signed gate + report, the build record and the Logbook row exactly as the retired + seam command did, so a national cut built here is bit-for-bit the seam's. + The driver adds what the dense role has: the pinned input, the role's + doctrine and overrides, staging telemetry under the attempt id, the + rowwise manifest beside the seam's evidence, and the staged bundle. + """ + + posture = posture_of(args) + out_dir = args.out.expanduser().resolve() + input_h5 = _require_file(args.input_h5, label="--input-h5") + if out_dir.exists() and not out_dir.is_dir(): + raise ValueError(f"--out must be a directory path, got {out_dir}.") + input_artifact = _artifact_info(input_h5) + _verify_requested_pin("--input-h5", input_artifact, requested=args.input_sha256) + incumbent = _incumbent_arguments(args) + # The attempt id is minted before telemetry opens so the staging run id + # and the Logbook row agree, as on the dense role. + build_id = new_uk_calibration_attempt_id(timestamp=datetime.now(UTC)) + telemetry = create_staging_telemetry(args, build_id=build_id) + try: + return _run_national_attempt( + args, + posture=posture, + out_dir=out_dir, + input_h5=input_h5, + input_artifact=input_artifact, + build_id=build_id, + telemetry=telemetry, + incumbent=incumbent, + ) + except BaseException as error: + fail_staging_telemetry(telemetry, error) + raise + + +def _run_national_attempt( + args: argparse.Namespace, + *, + posture: UKRowwisePosture, + out_dir: Path, + input_h5: Path, + input_artifact: Mapping[str, Any], + build_id: str, + telemetry: StagingTelemetryV2 | None, + incumbent: Mapping[str, Any] | None = None, +) -> int: + stage( + telemetry, + "input_pinning", + "completed", + dataset_sha256=input_artifact["sha256"], + ) + if telemetry is not None: + telemetry.set_sample({"mode": "full"}) + frs_release = load_uk_frs_release() + calibration_year = int(frs_release.calibration_year) + args._calibration_year = calibration_year + args._frs_vintage = str(frs_release.vintage) + source_year = _source_year( + args.source_year, time_period=str(frs_release.time_period) + ) + paths_by_role = output_paths(out_dir, posture=posture, vintage=args._frs_vintage) + _validate_output_paths(paths_by_role, input_h5=input_h5, ladder_path=None) + stage(telemetry, "target_compilation", "started") + inputs = _load_national_target_inputs(args) + stage( + telemetry, + "target_compilation", + "completed", + compiled_target_count=len(inputs["band_edge_registry"].specs), + active_target_count=len(inputs["national_registry"].specs), + ) + doctrine, doctrine_overrides = uk_doctrine_with_overrides( + **_national_doctrine_overrides(args) + ) + if doctrine.target_weight_rule != args.target_weight_rule: + raise RuntimeError( + "national doctrine override did not bind the requested rule." + ) + args._doctrine_override_receipt = doctrine_overrides + out_dir.mkdir(parents=True, exist_ok=True) + # The frozen register is the scorer's input: the same artifact this run + # solved against, by content hash. + inputs["national_registry"].to_json(paths_by_role["national_registry"]) + # The full compiled register beside it: the band edges a pruned scoring + # surface must never redraw (#803), for the end-of-build evaluation and + # for a re-score by hand (--band-edge-registry-json). + inputs["band_edge_registry"].to_json(paths_by_role["contract_registry"]) + resolver = UKMeasureResolver( + simulation_source=input_h5, + scratch_dir=out_dir, + year=calibration_year, + frame=None, + ) + paths = UKCalibrationRunPaths( + input_h5=input_h5, + staging_h5=paths_by_role["dataset"], + diagnostics_json=paths_by_role["calibration_diagnostics"], + build_record_json=paths_by_role["build_record"], + terminal_gate_json=paths_by_role["terminal_gates"], + ) + evidence: dict[str, Any] = {} + + def publish_manifest() -> None: + # The seam has written its evidence; the manifest describes it and the + # staged bundle (every manifest-registered output) follows, as on the + # dense role. The staged copy predates the evidence blocks appended + # below; staged_manifest.json describes the remote side. + args._gate_report = _read_json(paths.terminal_gate_json) + manifest = _national_manifest( + args, + posture=posture, + build_id=build_id, + build_record=_read_json(paths.build_record_json), + build_record_path=paths.build_record_json, + gate_report=args._gate_report, + diagnostics=_read_json(paths.diagnostics_json), + inputs=inputs, + doctrine_overrides=doctrine_overrides, + output_paths=paths_by_role, + input_artifact=input_artifact, + source_year=source_year, + ) + replace_manifest(paths_by_role["manifest"], manifest) + evidence["manifest"] = manifest + evidence["staged_dataset"] = stage_dataset( + args, + manifest=manifest, + output_paths=paths_by_role, + run_id=build_id if telemetry is None else telemetry.run_id, + telemetry=telemetry, + ) + + def evaluate() -> None: + # After the bundle is staged (the evaluation never blocks staging), + # before the telemetry completes (the receipt rides it as an artifact). + evidence["evaluation"] = _evaluate_against_incumbent( + args, + incumbent=incumbent, + inputs=inputs, + output_paths=paths_by_role, + telemetry=telemetry, + calibration_year=calibration_year, + out_dir=out_dir, + ) + + def finalize_staging() -> None: + publish_manifest() + evaluate() + finalize_staging_telemetry(args, telemetry) + + def event_callback(stage_id: str, status: str, details: Mapping[str, Any]) -> None: + stage(telemetry, stage_id, status, **dict(details)) + + result = run_uk_calibration( + paths=paths, + build_id=build_id, + input_sha256=str(args.input_sha256), + ledger_artifact=inputs["artifact"], + register_registry=inputs["national_registry"], + band_edge_registry=inputs["band_edge_registry"], + calibration_year=calibration_year, + exclusion_receipt=inputs["measure_exclusions"], + doctrine=doctrine, + doctrine_overrides=doctrine_overrides, + measure_resolver=resolver, + source_pins={ + "input_h5": { + "sha256": str(args.input_sha256), + "size_bytes": int(input_artifact["bytes"]), + }, + "ledger_facts": _ledger_facts_pin(inputs["artifact"]), + }, + run_config_extra={ + "release_role": posture.role, + "calibration_year": calibration_year, + "allow_unpinned_feed": bool(args.allow_unpinned_feed), + "chronicle_feed_pin": inputs["chronicle_feed_pin"], + "rowwise_driver_parameters": rowwise_parameters( + args, source_year=source_year + ), + }, + release_id=posture.release_id, + logbook_prev_row_digest=args.logbook_prev_row_digest, + progress_callback=( + None + if telemetry is None + else thinned_epochs( + telemetry.calibration_progress, every=staging_epoch_every(args) + ) + ), + event_callback=None if telemetry is None else event_callback, + staging_delivery=staging_delivery(telemetry), + staging_finalizer=None if telemetry is None else finalize_staging, + staging_delivery_provider=( + None if telemetry is None else (lambda: telemetry.delivery_summary) + ), + ) + if telemetry is None: + publish_manifest() + evaluate() + manifest = evidence["manifest"] + # The seam rewrote its record with the delivery summary after the + # finalizer; the manifest binds the record as it now is. + manifest["outputs"]["build_record"] = _artifact_info(paths.build_record_json) + manifest["build_record"] = { + "path": str(paths.build_record_json), + "sha256": result.build_record_sha256, + } + manifest["staging_delivery"] = staging_delivery(telemetry) + manifest["staged_dataset"] = evidence["staged_dataset"] + evaluation = evidence.get("evaluation", {"status": "not_requested"}) + manifest["evaluation"] = evaluation + if evaluation.get("status") == "completed": + manifest["outputs"]["score_receipt"] = evaluation["receipt"] + replace_manifest(paths_by_role["manifest"], manifest) + if (out_dir / SHA256SUMS_FILENAME).is_file(): + # The uploaded copies list the files as uploaded; the local sums + # list the record, the receipt and the manifest as they now are, + # evidence included. + refresh_sha256sums_entry(out_dir, paths.build_record_json.name) + if evaluation.get("status") == "completed": + _list_sha256sums_entry(out_dir, paths_by_role["score_receipt"].name) + refresh_sha256sums_entry(out_dir, paths_by_role["manifest"].name) + print(json_text(manifest)) + return 0 + + +def _national_fit_by_family( + diagnostics: Mapping[str, Any], registry: TargetRegistry +) -> list[dict[str, object]]: + families = {str(spec.name): str(spec.family or "") for spec in registry.specs} + rows = [] + for row in diagnostics.get("targets", []): + if not isinstance(row, Mapping): + continue + error = row.get("relative_error") + if error is None: + continue + # The diagnostics name a target by its register spec name and, on + # period-suffixed rows, by a materialized name; the family lives on + # the spec. + labels = [ + str(label) + for label in (row.get("target_name"), row.get("name")) + if label is not None + ] + family = next((families[label] for label in labels if label in families), "") + rows.append( + { + "target_name": labels[0] if labels else "", + "family": family, + "abs_relative_error": abs(float(error)), + } + ) + if not rows: + return [] + return uk_fit_by_family(pd.DataFrame(rows)) + + +def _national_manifest( + args: argparse.Namespace, + *, + posture: UKRowwisePosture, + build_id: str, + build_record: Mapping[str, Any], + build_record_path: Path, + gate_report: Mapping[str, Any], + diagnostics: Mapping[str, Any], + inputs: Mapping[str, Any], + doctrine_overrides: Mapping[str, Any], + output_paths: Mapping[str, Path], + input_artifact: Mapping[str, Any], + source_year: int, +) -> dict[str, Any]: + """The rowwise manifest of a national-role run, over the seam's evidence.""" + + calibration = build_record["calibration"] + solve = calibration["solve"] + statuses = gate_statuses(gate_report) + abs_errors = np.asarray( + [ + abs(float(row["relative_error"])) + for row in diagnostics.get("targets", []) + if isinstance(row, Mapping) and row.get("relative_error") is not None + ], + dtype=np.float64, + ) + fit_rows = _national_fit_by_family(diagnostics, inputs["national_registry"]) + outputs = { + "dataset": _artifact_info(output_paths["dataset"]), + "calibration_diagnostics": _artifact_info( + output_paths["calibration_diagnostics"] + ), + "build_record": _artifact_info(build_record_path), + "terminal_gate_report": _artifact_info(output_paths["terminal_gates"]), + "national_target_registry": _artifact_info(output_paths["national_registry"]), + "national_contract_registry": _artifact_info(output_paths["contract_registry"]), + } + return { + "schema_version": 4, + "build_kind": "uk_national_calibrated_candidate", + "release_role": posture.role, + "release_id": posture.release_id, + "build_id": build_id, + "candidate_scope": "national", + "created_at": datetime.now(UTC).isoformat(), + "git_commit": git_commit(), + "git_dirty": git_dirty(), + "parameters": rowwise_parameters(args, source_year=source_year), + "inputs": {"dataset": dict(input_artifact)}, + "identity": { + "spine": { + **dict(input_artifact), + "spine_provenance": dict(build_record["spine_provenance"]), + }, + "targets": {"chronicle": inputs["artifact"].provenance()}, + "code": {"git_commit": git_commit(), "git_dirty": git_dirty()}, + "runtime": runtime_provenance(), + "sampling": {"mode": "full"}, + "survey_year": source_year, + "calibration_year": int(inputs["calibration_year"]), + }, + "sampling": {"mode": "full"}, + "outputs": outputs, + "weights": dict(calibration["weights"]), + "solve": { + "n_targets": int(solve["n_targets"]), + "n_targets_by_kind": { + "national": int(solve["n_targets"]), + "local": 0, + "ladder": 0, + }, + "n_households": int(solve["n_households"]), + "pool_households": int(solve["n_households"]), + "initial_loss": float(solve["initial_loss"]), + "final_loss": float(solve["final_loss"]), + "n_nonzero": int(solve["n_nonzero"]), + "max_abs_relative_error": ( + float(abs_errors.max()) if abs_errors.size else None + ), + "median_abs_relative_error": ( + float(np.median(abs_errors)) if abs_errors.size else None + ), + "effective_sample_size": calibration.get("effective_sample_size"), + "max_weight_ratio": calibration.get("max_weight_ratio"), + "target_weight_rule": str(args.target_weight_rule), + "target_weight_rule_override": dict( + doctrine_overrides.get("target_weight_rule", {}) + ), + "doctrine_overrides": dict(doctrine_overrides), + "measure_resolution": calibration.get("measure_resolution"), + }, + "fit": { + "national_by_family": fit_rows, + "weakest_families": sorted( + fit_rows, + key=lambda row: ( + -float(row["worst_abs_relative_error"]), + row["family"], + ), + )[:10], + }, + "gate": { + "scope": list(posture.gate_scope), + "posture": posture.gate_posture, + "release_id": gate_report.get("release_id"), + "statuses": statuses, + }, + "failing_gate_ids": sorted( + gate_id for gate_id, status in statuses.items() if status != "passed" + ), + "releasable": False, + "release_posture": { + "release_candidate": False, + "shippable_by": "tools/certify_uk_release_cut.py", + "calibration_seam_gates_passed": bool(statuses) + and all(status == "passed" for status in statuses.values()), + }, + "measure_exclusions": dict(inputs["measure_exclusions"]), + "build_record": { + "path": str(build_record_path), + "sha256": outputs["build_record"]["sha256"], + }, + } + + +def _verify_requested_pin( + label: str, + artifact: Mapping[str, Any], + *, + requested: str | None, +) -> None: + if requested is None: + artifact["pin_verified"] = False + return + measured = str(artifact["sha256"]) + if measured != requested: + raise SystemExit( + f"error: {label} sha mismatch: measured {measured}, pinned {requested}" + ) + artifact["pin_verified"] = True + + +def _load_candidate_evaluator(importer=importlib.import_module): + """The common-surface scorer (microcosm#967), loaded when an incumbent is given. + + Lazy so the driver imports without it; a build asked to evaluate refuses + up front, before any solve, when the scorer is not in the tree. + """ + + try: + return importer("microcosm.build.uk_runtime.candidate_score") + except ImportError as error: + raise ValueError( + "--incumbent-h5 needs the common-surface scorer (microcosm#967): " + "microcosm.build.uk_runtime.candidate_score is not in this tree." + ) from error + + +def _incumbent_arguments(args: argparse.Namespace) -> dict[str, Any] | None: + """The national role's optional incumbent, pinned and verified up front.""" + + if args.incumbent_h5 is None and args.incumbent_sha256 is None: + return None + if args.incumbent_h5 is None or args.incumbent_sha256 is None: + raise ValueError( + "--incumbent-h5 and --incumbent-sha256 must be given together." + ) + _load_candidate_evaluator() + incumbent_h5 = _require_file(args.incumbent_h5, label="--incumbent-h5") + info = _artifact_info(incumbent_h5) + _verify_requested_pin("--incumbent-h5", info, requested=args.incumbent_sha256) + return { + "path": str(incumbent_h5), + "sha256": info["sha256"], + "bytes": info["bytes"], + "label": str(args.incumbent_label), + } + + +def _list_sha256sums_entry(out_dir: Path, name: str) -> None: + """List a file the build wrote after the sidecars in the local sums.""" + + sums_path = out_dir / SHA256SUMS_FILENAME + entries = parse_sha256sums(sums_path.read_text(encoding="utf-8")) + if name in {listed for _, listed in entries}: + refresh_sha256sums_entry(out_dir, name) + return + digest = _artifact_info(out_dir / name)["sha256"] + sums_path.write_text( + sums_path.read_text(encoding="utf-8") + f"{digest} {name}\n", + encoding="utf-8", + ) + + +#: Receipt blocks that are record arrays (lists of mappings): the reviewed +#: telemetry artifact policy refuses them, and the verdict, the pruned block +#: and the aggregates carry everything a reviewer reads. The full receipt stays +#: beside the outputs and in the staged bundle. +_SCORE_RECEIPT_TELEMETRY_EXCLUDED = ( + "target_drift", + "signed_asymmetries", + "measure_resolution", +) + + +def _score_receipt_telemetry_summary(score: Mapping[str, Any]) -> dict[str, Any]: + return { + key: value + for key, value in score.items() + if key not in _SCORE_RECEIPT_TELEMETRY_EXCLUDED + } + + +def _evaluate_against_incumbent( + args: argparse.Namespace, + *, + incumbent: Mapping[str, Any] | None, + inputs: Mapping[str, Any], + output_paths: Mapping[str, Path], + telemetry: StagingTelemetryV2 | None, + calibration_year: int, + out_dir: Path, +) -> dict[str, Any]: + """Score the finished candidate against the incumbent; never fail the build. + + Runs after the staged bundle is on the Hub and before the telemetry + completes, so the receipt rides the run as a reviewed artifact. Rows the + incumbent cannot materialize are pruned from both arms and warned about + by name; the verdict (microcosm#578 rule 1 on the common surface) is + what the release-cut certifier requires to be ``passed``. An error is + recorded and warned, never raised: staging and the exit code stand. + """ + + if incumbent is None: + return { + "status": "not_requested", + "note": ( + "no --incumbent-h5: the rule-1 score receipt the release-cut " + "certifier needs was not produced; score the candidate with " + "tools/score_uk_national_candidate.py before certification." + ), + } + stage( + telemetry, + "incumbent_evaluation", + "started", + incumbent_sha256=incumbent["sha256"], + ) + candidate = _artifact_info(output_paths["dataset"]) + try: + module = _load_candidate_evaluator() + score = module.evaluate_uk_candidate_against_incumbent( + candidate_h5=output_paths["dataset"], + incumbent_h5=Path(incumbent["path"]), + candidate_sha256=candidate["sha256"], + incumbent_sha256=incumbent["sha256"], + target_registry=inputs["national_registry"], + calibration_year=calibration_year, + measure_resolver_factory=module.uk_default_measure_resolver_factory( + out_dir, calibration_year + ), + candidate_label=output_paths["dataset"].stem, + incumbent_label=incumbent["label"], + band_edge_registry=inputs["band_edge_registry"], + ) + atomic_write_json(output_paths["score_receipt"], score) + except Exception as error: # noqa: BLE001 - the evaluation never fails a finished build + message = f"{type(error).__name__}: {error}"[:600] + print( + f"warning: the incumbent evaluation failed ({message}); the build's " + "evidence and staging are unaffected, and the candidate cannot be " + "certified until it is re-scored with " + "tools/score_uk_national_candidate.py.", + file=sys.stderr, + flush=True, + ) + stage(telemetry, "incumbent_evaluation", "failed", error=message) + return { + "status": "error", + "error": message, + "incumbent": dict(incumbent), + "receipt": None, + } + warning = module.pruned_warning(score) + if warning is not None: + print(warning, file=sys.stderr, flush=True) + evaluation = score["evaluation"] + pruned = score["incumbent_unresolvable_pruned"] + add_staging_artifact( + telemetry, + "score_vs_incumbent", + _score_receipt_telemetry_summary(score), + artifact_kind="aggregate_diagnostics", + classification="aggregate", + ) + stage( + telemetry, + "incumbent_evaluation", + "completed", + verdict=evaluation["verdict"], + n_scored=int(evaluation["scored_surface"]["n_scored"]), + n_pruned=int(evaluation["scored_surface"]["n_pruned"]), + ) + return { + "status": "completed", + "verdict": evaluation["verdict"], + "rule_1": dict(evaluation["rule_1"]), + "scored_surface": dict(evaluation["scored_surface"]), + "pruned_measures": list(pruned["measures"]), + "pruned_families": dict(pruned["families"]), + "receipt": _artifact_info(output_paths["score_receipt"]), + "incumbent": dict(incumbent), + } + + +def _validate_output_paths( + output_paths: Mapping[str, Path], + *, + input_h5: Path, + ladder_path: Path | None, +) -> None: + resolved = {name: path.resolve() for name, path in output_paths.items()} + if len(set(resolved.values())) != len(resolved): + raise ValueError("candidate output paths must be distinct.") + protected = {input_h5.resolve()} + if ladder_path is not None: + protected.add(ladder_path.resolve()) + collisions = sorted(str(path) for path in resolved.values() if path in protected) + if collisions: + raise ValueError( + "candidate outputs must differ from --input-h5 and --ladder; " + f"collision(s): {collisions}." + ) + existing = sorted(str(path) for path in resolved.values() if path.exists()) + if existing: + raise FileExistsError( + f"refusing to overwrite existing candidate artifact(s): {existing}." + ) + + +def _require_file(path: Path, *, label: str) -> Path: + resolved = path.expanduser().resolve() + if not resolved.is_file(): + raise FileNotFoundError(f"{label} artifact not found: {resolved}.") + return resolved + + +def _source_year(requested: int | None, *, time_period: str) -> int: + if requested is not None: + if requested <= 0: + raise ValueError("--source-year must be positive.") + return requested + prefix = str(time_period).strip()[:4] + if len(prefix) != 4 or not prefix.isdigit(): + raise ValueError( + "Could not infer source year from input H5 time_period; pass --source-year." + ) + return int(prefix) + + +def _artifact_info( + path: Path, + *, + reported_path: Path | None = None, +) -> dict[str, Any]: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return { + "path": str((reported_path or path).resolve()), + "sha256": digest.hexdigest(), + "bytes": int(path.stat().st_size), + } diff --git a/packages/microcosm-build/tests/engine_free/shared/test_gate_battery_contract_pins.py b/packages/microcosm-build/tests/engine_free/shared/test_gate_battery_contract_pins.py index 32a85be13..f6d9aa869 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_gate_battery_contract_pins.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_gate_battery_contract_pins.py @@ -423,22 +423,16 @@ def test_entry_ids_mirror_the_local_battery_scope(self) -> None: ) def test_scoped_digests_mirror_the_live_local_manifest(self) -> None: - import importlib.util - from microcosm.build.uk_runtime.calibration_run import UK_LOCAL_GATE_SCOPE from microcosm.build.uk_runtime.release_certification import _scoped_digests - - spec = importlib.util.spec_from_file_location( - "build_uk_rowwise_candidate", - _TEST_PATHS.repository / "tools" / "build_uk_rowwise_candidate.py", + from microcosm.build.uk_runtime.rowwise_posture import ( + UK_ROWWISE_DENSE_POSTURE, ) - builder = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(builder) + live = _scoped_digests( frozenset(UK_LOCAL_GATE_SCOPE), phases=tuple(data_contract._UK_DENSE_GATE_PHASES), - policy_suffix=str(builder._LOCAL_GATE_POLICY_SUFFIX), + policy_suffix=str(UK_ROWWISE_DENSE_POSTURE.gate_policy_suffix), ) for field, mirrored in data_contract._UK_DENSE_GATE_DIGESTS.items(): assert mirrored == live[field], field diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_observation_period.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_observation_period.py index eaa44550f..9b01fbc5b 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_observation_period.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_observation_period.py @@ -2,9 +2,6 @@ # ruff: noqa: F403, F405 from test_support.microcosm_build.uk_cgt_observation_period import * -from test_support.paths import paths_for - -_TEST_PATHS = paths_for("microcosm-build") @pytest.mark.parametrize("route", ["national", "local"]) @@ -44,26 +41,21 @@ def resolver_factory(**kwargs): "cgt_period_contract" ] else: + from microcosm.build.uk_runtime import full_measure from microcosm.build.uk_runtime.local_rowwise import ( build_uk_rowwise_local_matrix, solve_uk_rowwise_weights_under_doctrine, ) - spec = importlib.util.spec_from_file_location( - "cgt_rowwise_builder", - _TEST_PATHS.repository / "tools/build_uk_rowwise_candidate.py", - ) - builder = importlib.util.module_from_spec(spec) - spec.loader.exec_module(builder) monkeypatch.setattr( - builder, + full_measure, "compute_household_metrics", lambda _sim, _area, *, period, household_ids: pd.DataFrame( {"households": np.ones(len(household_ids))}, index=household_ids ), ) prepared, restore, national, metrics, engine_receipt = ( - builder._resolve_candidate_engine_surface( + full_measure.resolve_uk_full_measures( original, registry, period=2025, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py index 67e655c27..f67671cca 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py @@ -103,14 +103,81 @@ def test_dense_role_requires_the_ladder_and_the_pins(tmp_path): cli.validate_cli_args(cli.parse_args(request)) -def test_national_role_is_validated_then_refused_until_phase_four(tmp_path, capsys): +def test_national_role_is_validated_then_dispatched_to_the_seam(tmp_path, monkeypatch): + """The national line never prepares a graph (microcosm#901 phase 4). + + The validated request goes to ``national_role.run_national_role`` after + the role tables have run; the seam's own pre-flight, Logbook, staging + and manifest live behind that call. + """ + argv = _national_argv(tmp_path) + national = cli.parse_args(argv) + assert national._posture is cli.uk_rowwise_posture("national") + assert national.n_clones is None + assert (national.seed, national.epochs, national.learning_rate) == (0, 1500, 0.02) + cli.validate_cli_args(national) + served = [] + monkeypatch.setattr( + cli.national_role, "run_national_role", lambda args: served.append(args) or 7 + ) + monkeypatch.setattr( + cli, + "prepare_full_build", + lambda *a, **k: pytest.fail("the national role prepared a graph"), + ) + monkeypatch.setattr( + cli, + "preflight_staged_dataset", + lambda args: pytest.fail("the driver pre-flighted before dispatching"), + ) + assert cli.main(argv) == 7 + (args,) = served + assert args.release_role == "national" + assert args._posture is national._posture + assert not (tmp_path / "out").exists() + # The dense refusal table still applies before the dispatch. + with pytest.raises(ValueError, match="--release-role national refuses"): + cli.main([*argv, "--ladder", str(tmp_path / "ladder.npz")]) + assert served == [args] + + +def test_national_dry_run_dispatches_to_the_seam_plan(tmp_path, monkeypatch, capsys): + """``--dry-run`` on the national role plans through ``national_dry_run``: + no graph, no staged-dataset pre-flight, nothing written.""" + monkeypatch.setattr( + cli.national_role, + "national_dry_run", + lambda args: ( + print(json.dumps({"dry_run": args.dry_run, "role": args.release_role})) or 0 + ), + ) + monkeypatch.setattr( + cli, + "prepare_full_build", + lambda *a, **k: pytest.fail("the national dry run prepared a graph"), + ) + monkeypatch.setattr( + cli.national_role, + "preflight_staged_dataset", + lambda args: pytest.fail("a dry run reached the Hub pre-flight"), + ) + assert cli.main(_national_argv(tmp_path, "--dry-run")) == 0 + assert json.loads(capsys.readouterr().out) == {"dry_run": True, "role": "national"} + assert not (tmp_path / "out").exists() + + +def test_national_role_refuses_a_spine_request(tmp_path, monkeypatch): + """The seam engine reads a bound spine; ``--spine-request`` is dense-only.""" + monkeypatch.setattr( + cli.national_role, + "preflight_staged_dataset", + lambda args: pytest.fail("the refusal must precede the Hub pre-flight"), + ) argv = [ "--release-role", "national", - "--input-h5", - str(tmp_path / "spine.h5"), - "--input-sha256", - PIN, + "--spine-request", + str(tmp_path / "request.json"), "--out", str(tmp_path / "out"), "--ledger-facts", @@ -121,18 +188,9 @@ def test_national_role_is_validated_then_refused_until_phase_four(tmp_path, caps PIN, "--no-staging", ] - national = cli.parse_args(argv) - assert national._posture is cli.uk_rowwise_posture("national") - assert national.n_clones is None - assert (national.seed, national.epochs, national.learning_rate) == (0, 1500, 0.02) - cli.validate_cli_args(national) - with pytest.raises(SystemExit) as exit_info: + with pytest.raises(ValueError, match="bound spine checkpoint.*--input-h5"): cli.main(argv) - assert exit_info.value.code == 2 - assert "next commit" in capsys.readouterr().err - # The dense refusal table still applies before the national refusal. - with pytest.raises(ValueError, match="--release-role national refuses"): - cli.main([*argv, "--ladder", str(tmp_path / "ladder.npz")]) + assert not (tmp_path / "out").exists() def test_candidate_clone_counts_are_dry_run_only(tmp_path): diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py index c04272cde..f6d4098b7 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py @@ -1,364 +1,26 @@ -"""Tests using shared support from ``test_support.microcosm_build.uk_rowwise_candidate``.""" +"""The UK rowwise command surface on the graph full-build driver (microcosm#823). + +The dense role's synthetic end-to-end contract is pinned by +``test_uk_full_build_cli.py``; here are the release-role tables, the pure-CLI +refusals, the shared command helpers and the staged-dataset pre-flight, as +the retired rowwise tool pinned them, plus the tool's stub. +""" # ruff: noqa: F403, F405 from test_support.microcosm_build.uk_rowwise_candidate import * -def test_candidate_build_writes_calibrated_h5_and_evidence( - monkeypatch, tmp_path -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "candidate" - _write_staging_h5(input_h5, households_per_region=52) - ladder = _write_ladder(ladder_path) - household_flags = _configure_households_only_inputs( - builder, monkeypatch, input_h5=input_h5, ladder_path=ladder_path - ) - import microcosm.build.uk_runtime.battery_bindings as battery_bindings - - monkeypatch.setattr( - battery_bindings, - "_local_area_roster", - lambda _resource, levels: { - "constituency": tuple(sorted(set(ladder.constituency_code))), - "local_authority": tuple(sorted(set(ladder.local_authority_code))), - }, - ) - holdout = { - "report_only": True, - "method": "rotated_folds", - "target_loss_cap": 10.0, - "loss_weight_scale": "held_local_grains_only", - "target_weight_rule": "grain_equal", - "population": "held_out_local_targets", - "grains": ["constituency", "local_authority"], - "n_folds": 5, - "seed": 20260529, - "solve_seed": 7, - "mean_holdout_loss": 0.1, - "worst_holdout_loss": 0.2, - "fold_losses": [0.1, 0.1, 0.2, 0.05, 0.05], - "folds": [ - { - "fold": fold, - "n_train_targets": 3, - "n_holdout_targets": 1, - "holdout_target_indices": [fold % 4], - "training_national_rows": 0, - "holdout_loss": loss, - } - for fold, loss in enumerate([0.1, 0.1, 0.2, 0.05, 0.05]) - ], - } - monkeypatch.setattr( - builder, - "rotated_uk_local_holdout", - lambda *_args, **_kwargs: holdout, - ) - - assert ( - builder.main( - [ - "--input-h5", - str(input_h5), - "--release-role", - "dense", - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - "--n-clones", - "2", - "--seed", - "7", - "--epochs", - "2", - *household_flags, - ] - ) - == 0 - ) - - candidate_h5 = output_dir / "microcosm_uk_2024_25_local.h5" - expected_sidecars = { - builder.MANIFEST_FILENAME, - builder.SOLVE_DIAGNOSTICS_FILENAME, - builder.AREA_SUPPORT_FILENAME, - builder.PAST_CAP_FILENAME, - builder.CALIBRATION_DIAGNOSTICS_FILENAME, - builder.LOCAL_REGISTRY_FILENAME, - "microcosm_uk_2024_25_local.local_gates.json", - } - assert candidate_h5.exists() - assert expected_sidecars <= {path.name for path in output_dir.iterdir()} - - candidate_kind, candidate_mass_log = read_uk_single_year_weight_metadata( - candidate_h5 - ) - with pd.HDFStore(candidate_h5, mode="r") as store: - candidate_household = store["household"] - assert candidate_kind is WeightKind.CALIBRATED - assert candidate_household["source_year"].unique().tolist() == [2023] - assert len(set(candidate_household["source_household_key"])) == 200 - assert {"2023:1", "2023:206"} <= set(candidate_household["source_household_key"]) - assert len(candidate_mass_log) == 3 - calibration_records = [ - record - for record in candidate_mass_log - if "census_households/constituency" in record.reason - ] - assert calibration_records == [candidate_mass_log[-1]] - # The kernel-minted record declares the realized factor (the hand-minted - # predecessor left it None) — declared-vs-realized is validated by the - # kernel at with_weights time. - record = candidate_mass_log[-1] - assert record.declared_factor == pytest.approx(record.new_total / record.old_total) - - manifest = json.loads((output_dir / builder.MANIFEST_FILENAME).read_text()) - # The manifest declares the role it was built under (microcosm#823): the - # dense pre-flight and assembler refuse any other, and the parameters - # carry the role's doctrine block verbatim. - assert manifest["schema_version"] == 4 - assert manifest["release_role"] == "dense" - assert manifest["release_id"] == "microcosm-uk-2024-25-dense" - assert manifest["parameters"]["release_role"] == "dense" - assert manifest["parameters"]["n_clones"] == 2 - assert manifest["parameters"]["doctrine"] == ( - builder.UK_ROWWISE_DENSE_POSTURE.doctrine_bounds() - ) - assert manifest["outputs"]["dataset"]["path"].endswith( - "microcosm_uk_2024_25_local.h5" - ) - assert manifest["candidate_scope"] == "adjudicated_partial" - assert manifest["bound_target_families"] == ["census_households/constituency"] - adjudications = manifest["binding_adjudications"] - assert adjudications["register_resource"] == "local_binding_adjudications.json" - assert adjudications["bound_families"] == ["census_households/constituency"] - assert adjudications["evaluated_on"] - seed = adjudications["stood_on"]["census_households/constituency"][ - "census_disclosure_control_noise" - ] - assert seed["approved_by"] == "juaristi22" - assert seed["adjudication"] == "microcosm#802" - assert seed["approved_on"] == "2026-08-31" - assert seed["expires_on"] == "2026-11-30" - assert adjudications["dormant"] == [ - "full_frs_tei_band_unavailable", - "hmrc_spi_frame_model_proxy", - "population_universe_private_households", - "uc_unit_vs_household_grain", - "voa_dwellings_vs_household_frame", - ] - cross_grain = manifest["cross_grain"] - assert cross_grain["bound_national_targets"] == [] - assert cross_grain["bound_higher_targets"] == [] - assert cross_grain["inconsistencies_in_force"] == [] - assert cross_grain["groups"] == [] - assert cross_grain["empty_legs_licensed"] == [] - assert cross_grain["controls_without_lower_rows"] == [] - assert cross_grain["absence"] - assert manifest["ladder_assignment_provenance"] == ladder_target_provenance(ladder) - assert manifest["identity"]["targets"]["paired_ladder_sha256"] == ( - hashlib.sha256(ladder_path.read_bytes()).hexdigest() - ) - assert manifest["identity"]["targets"]["chronicle"]["artifact_id"] == ( - "synthetic-households-only-fixture" - ) - assert manifest["household_dispersion"]["countries"] - assert manifest["gate"]["passed"] is True - assert manifest["gate"]["phase"] == "post_calibration" - assert manifest["gate"]["details"] - assert ( - manifest["inputs"]["dataset"]["sha256"] - == hashlib.sha256(input_h5.read_bytes()).hexdigest() - ) - assert manifest["inputs"]["dataset"]["bytes"] == input_h5.stat().st_size - assert ( - manifest["inputs"]["ladder"]["sha256"] - == hashlib.sha256(ladder_path.read_bytes()).hexdigest() - ) - assert manifest["inputs"]["ladder"]["bytes"] == ladder_path.stat().st_size - assert manifest["parameters"]["n_clones"] == 2 - assert manifest["parameters"]["seed"] == 7 - assert manifest["parameters"]["source_year"] == 2023 - assert manifest["parameters"]["source_lineage_modulus"] is None - assert manifest["parameters"]["epochs"] == 2 - assert manifest["parameters"]["learning_rate"] == pytest.approx(0.15) - assert manifest["parameters"]["expected_constituency_vintage"] == "2024_pcon" - assert [ - row["kind"] for row in manifest["weights"]["household_weight_kind_chain"] - ] == ["importance", "importance", "calibrated"] - assert manifest["weights"]["mass_log_records_before_calibration"] == 2 - assert manifest["weights"]["mass_log_records"] == 3 - mass_change = manifest["weights"]["calibration_mass_change"] - assert mass_change["old_total"] == pytest.approx(33.0) - assert mass_change["new_total"] == pytest.approx( - candidate_household["household_weight"].sum() - ) - assert mass_change["relative_shift"] == pytest.approx( - (mass_change["new_total"] - 33.0) / 33.0 - ) - assert manifest["parameters"]["doctrine"] == { - "target_loss_cap": 10.0, - "max_weight_ratio": 10.0, - "scale_rule": "default_target_loss_scales", - "target_weight_rule": "grain_equal", - "solve_epochs": 1500, - "clone_count": 15, - } - assert manifest["census_household_uprating"]["applied"] is False - assert manifest["solve"]["n_targets"] == 4 - assert manifest["solve"]["n_households"] == 416 - assert np.isfinite(manifest["solve"]["initial_loss"]) - assert np.isfinite(manifest["solve"]["final_loss"]) - assert np.isfinite(manifest["solve"]["max_abs_relative_error"]) - assert np.isfinite(manifest["solve"]["median_abs_relative_error"]) - assert manifest["solve"]["past_cap"]["n_targets"] == 4 - assert manifest["support"]["min_assigned_households"] == 104 - assert manifest["support"]["min_nonzero_households"] == 104 - assert manifest["support"]["min_effective_sample_size"] == pytest.approx(104.0) - - diagnostics = pd.read_csv(output_dir / builder.SOLVE_DIAGNOSTICS_FILENAME) - support = pd.read_csv(output_dir / builder.AREA_SUPPORT_FILENAME) - past_cap = json.loads((output_dir / builder.PAST_CAP_FILENAME).read_text()) - calibration_diagnostics = json.loads( - (output_dir / builder.CALIBRATION_DIAGNOSTICS_FILENAME).read_text() - ) - assert len(diagnostics) == 4 - assert diagnostics["metric"].unique().tolist() == ["households"] - assert len(support) == 8 - assert past_cap["n_targets"] == 4 - assert calibration_diagnostics["schema_version"] == 8 - uk_diagnostics = calibration_diagnostics["uk_diagnostics"] - assert len(uk_diagnostics["weakest_families"]) == 1 - assert len(uk_diagnostics["weakest_areas_by_fit"]["bottom_by_fit"]) == 4 - assert uk_diagnostics["weakest_areas_by_fit"]["n_areas_scored"] == 4 - assert { - row["country"] for row in uk_diagnostics["weakest_areas_by_fit"]["countries"] - } == { - "England", - "Northern Ireland", - "Scotland", - "Wales", - } - assert ( - manifest["diagnostics"]["weakest_families"] - == uk_diagnostics["weakest_families"] - ) - assert uk_diagnostics["rotated_holdout"] == holdout - assert manifest["diagnostics"]["rotated_holdout"] == holdout - assert "calibration_diagnostics" in manifest["outputs"] - rows = _spool_rows(output_dir) - assert len(rows) == 1 - row = rows[0] - assert row.pipeline == "uk-local-candidate" - assert row.rung == "f100" - assert row.seed == 7 - assert row.disposition == "iterating" - assert row.artifact_location == _local_ref(candidate_h5) - assert set(row.gate_verdicts) == set(builder.UK_LOCAL_GATE_SCOPE) - assert {item["verdict"] for item in row.gate_verdicts.values()} == {"passed"} - assert all( - ".local_gates.json#/gates/" in item["receipt"] - for item in row.gate_verdicts.values() - ) - - -def test_candidate_dry_run_plans_without_solve_or_write( - monkeypatch, - capsys, - tmp_path, -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "dry-run-output" - _write_staging_h5(input_h5) - ladder = _write_ladder(ladder_path) - household_flags = _configure_households_only_inputs( - builder, monkeypatch, input_h5=input_h5, ladder_path=ladder_path - ) - - def forbidden(*_args, **_kwargs): - pytest.fail("dry run called a solve or dataset writer") - - monkeypatch.setattr( - builder, - "solve_uk_rowwise_weights_under_doctrine", - forbidden, - ) - monkeypatch.setattr(builder, "write_uk_rowwise_dataset", forbidden) +def test_rowwise_candidate_tool_is_a_stub_over_the_graph_driver() -> None: + """Both release roles are served by the graph driver's ``main``.""" - assert ( - builder.main( - [ - "--input-h5", - str(input_h5), - "--release-role", - "dense", - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - "--n-clones", - "2", - "--seed", - "7", - "--dry-run", - *household_flags, - ] - ) - == 0 - ) - captured = capsys.readouterr() - plan = json.loads(captured.out) - assert plan["dry_run"] is True - assert plan["sampling"] == { - "fraction": 1.0, - "seed": 578, - "rung_token": "f100", - "sampled": False, - "pre_household_count": 12, - "post_household_count": 12, - } - assert plan["bound_target_families"] == ["census_households/constituency"] - adjudications = plan["binding_adjudications"] - assert adjudications["register_resource"] == "local_binding_adjudications.json" - assert adjudications["bound_families"] == ["census_households/constituency"] - assert adjudications["evaluated_on"] - assert ( - "census_disclosure_control_noise" - in adjudications["stood_on"]["census_households/constituency"] + spec = importlib.util.spec_from_file_location( + "build_uk_rowwise_candidate", + _TEST_PATHS.repository / "tools" / "build_uk_rowwise_candidate.py", ) - assert adjudications["dormant"] == [ - "full_frs_tei_band_unavailable", - "hmrc_spi_frame_model_proxy", - "population_universe_private_households", - "uc_unit_vs_household_grain", - "voa_dwellings_vs_household_frame", - ] - cross_grain = plan["cross_grain"] - assert cross_grain["bound_national_targets"] == [] - assert cross_grain["bound_higher_targets"] == [] - assert cross_grain["inconsistencies_in_force"] == [] - assert cross_grain["groups"] == [] - assert cross_grain["empty_legs_licensed"] == [] - assert cross_grain["controls_without_lower_rows"] == [] - assert cross_grain["absence"] - assert plan["ladder_assignment_provenance"] == ladder_target_provenance(ladder) - assert plan["shapes"]["person"][0] == 24 - assert plan["shapes"]["benunit"][0] == 24 - assert plan["shapes"]["household"][0] == 24 - assert plan["shapes"]["local_matrix"] == [4, 24] - assert plan["target_count"] == 4 - assert not output_dir.exists() - assert not (output_dir / "logbook-spool").exists() + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + assert module.main is _load_builder_module().main def test_graph_driver_dry_run_prints_the_operation_inventory( @@ -457,151 +119,8 @@ def test_graph_driver_dry_run_prints_the_operation_inventory( assert not output_dir.exists() -def test_candidate_sampling_rung_receipt_and_engine_block_validation( - monkeypatch, - capsys, - tmp_path, -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "dry-run-output" - _write_staging_h5(input_h5) - _write_ladder(ladder_path) - household_flags = _configure_households_only_inputs( - builder, monkeypatch, input_h5=input_h5, ladder_path=ladder_path - ) - - def compact_sampler_forbidden(*_args, **_kwargs): - pytest.fail("rowwise spine path called the certified-compact sampler") - - monkeypatch.setattr( - builder, - "sample_uk_national_frame", - compact_sampler_forbidden, - raising=False, - ) - monkeypatch.setattr( - builder, - "sample_uk_spine_frame", - lambda frame, **_kwargs: ( - frame, - { - "fraction": 0.01, - "seed": 578, - "rung_token": "f001", - "pre_household_count": 12, - "post_household_count": 12, - "pre_family_count": 4, - "post_family_count": 4, - "normalization_factor": 1.0, - "strata_count": 4, - "receipt": {"synthetic_fixture": True}, - }, - ), - raising=False, - ) - - assert ( - builder._parse_args( - [ - "--input-h5", - str(input_h5), - "--release-role", - "dense", - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - ] - ).n_clones - == builder.UK_ROWWISE_DENSE_POSTURE.clone_count - ) - with pytest.raises(ValueError, match="must equal --n-clones"): - builder.main( - [ - "--input-h5", - str(input_h5), - "--release-role", - "dense", - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - "--n-clones", - "4", - "--engine-blocks", - "2", - "--dry-run", - *household_flags, - ] - ) - assert ( - builder.main( - [ - "--input-h5", - str(input_h5), - "--release-role", - "dense", - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - "--n-clones", - "2", - "--sample-fraction", - "0.01", - "--sample-seed", - "578", - "--dry-run", - *household_flags, - ] - ) - == 0 - ) - plan = json.loads(capsys.readouterr().out) - assert plan["sampling"]["fraction"] == 0.01 - assert plan["sampling"]["rung_token"] == "f001" - assert plan["sampling"]["pre_household_count"] == 12 - assert plan["sampling"]["post_household_count"] >= 1 - assert plan["sampling"]["normalization_factor"] > 0 - - -def test_candidate_f100_does_not_call_any_sampler(monkeypatch, tmp_path) -> None: - pytest.importorskip("tables") +def test_candidate_clone_count_planning_is_dry_run_only(tmp_path) -> None: builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - _write_staging_h5(input_h5) - frame, _ = builder.load_uk_national_frame(input_h5) - - def forbidden(*_args, **_kwargs): - pytest.fail("f100 called a sampler") - - monkeypatch.setattr(builder, "sample_uk_national_frame", forbidden, raising=False) - monkeypatch.setattr(builder, "sample_uk_spine_frame", forbidden, raising=False) - - sampled, receipt = builder._sample_candidate_frame( - frame, - fraction=1.0, - seed=578, - ) - - assert sampled is frame - assert receipt == { - "fraction": 1.0, - "seed": 578, - "rung_token": "f100", - "sampled": False, - "pre_household_count": 12, - "post_household_count": 12, - } - - -@_BOTH_DRIVERS -def test_candidate_clone_count_planning_is_dry_run_only(driver, tmp_path) -> None: - builder = _load_builder_module(driver) with pytest.raises(ValueError, match="only with --dry-run"): builder.main( [ @@ -629,1156 +148,95 @@ def test_candidate_clone_count_planning_is_dry_run_only(driver, tmp_path) -> Non ) -def test_candidate_engine_surface_reuses_one_resolver( +def test_candidate_publication_rolls_back_on_interrupt( monkeypatch, tmp_path, ) -> None: - pytest.importorskip("tables") builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - _write_staging_h5(input_h5) - frame, _ = builder.load_uk_national_frame(input_h5) - constructions = [] - cgt_period_contract = { - "version": "uk-cgt-measurement-v2", - "input_period": "2024", - "calibration_period": 2025, - "bound_measurements": { - "cgt_2024_gains": { - "model_variable": "capital_gains", - "measurement_period": 2024, - } - }, - } - - class StubResolver: - def __init__(self, **kwargs): - constructions.append(kwargs) - self.simulation = object() - self.contract_targets = {} - - def receipt(self): - return { - "mode": "stub", - "policyengine_uk_version": "test", - "cgt_period_contract": cgt_period_contract, - } - - monkeypatch.setattr( - builder, - "compute_household_metrics", - lambda _simulation, area_type, *, household_ids, **_kwargs: pd.DataFrame( - {f"{area_type}_metric": np.ones(len(household_ids))}, - index=household_ids, - ), - ) - registry = TargetRegistry([], country="uk") - prepared, restore, national, local_metrics, receipt = ( - builder._resolve_candidate_engine_surface( - frame, - registry, - period=2025, - scratch_dir=tmp_path / "scratch", - resolver_factory=StubResolver, - ) + staging_dir = tmp_path / "staging" + output_dir = tmp_path / "candidate" + staging_dir.mkdir() + output_paths = rowwise_cli.output_paths( + output_dir, + posture=builder.UK_ROWWISE_DENSE_POSTURE, + vintage="2024_25", ) + staged = {key: staging_dir / path.name for key, path in output_paths.items()} + for path in staged.values(): + path.write_text("complete staged artifact\n") - assert len(constructions) == 1 - assert receipt == { - "mode": "stub", - "engine_version": "test", - "households": 12, - "persons": 12, - "benunits": 12, - "national_inputs": 0, - "local_metrics": {"constituency": 1, "la": 1}, - "blocks": 1, - "cgt_period_contract": cgt_period_contract, - } - assert set(local_metrics) == {"constituency", "la"} - assert len(national.targets) == 0 - assert restore(prepared).table("household").equals(frame.table("household")) - - -@pytest.mark.parametrize("second_cgt_period", [2024, 2025, None]) -def test_candidate_engine_surface_resolves_real_per_clone_blocks( - monkeypatch, - tmp_path, - second_cgt_period, -) -> None: - pytest.importorskip("tables") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - _write_staging_h5(input_h5) - frame, _ = builder.load_uk_national_frame(input_h5) - ladder = _write_ladder(ladder_path) - clone = builder._clone_with_ladder_binding( - frame, - ladder, - n_clones=2, - seed=7, - source_year=2023, - expected_constituency_vintage="2024_pcon", - source_lineage_modulus=None, - ).result - constructions = [] - - class StubResolver: - def __init__(self, **kwargs): - constructions.append(kwargs) - self.simulation = object() - self.contract_targets = {} - self.cgt_period = 2024 if len(constructions) == 1 else second_cgt_period - - def receipt(self): - receipt = {"mode": "stub", "policyengine_uk_version": "test"} - if self.cgt_period is not None: - receipt["cgt_period_contract"] = { - "version": "uk-cgt-measurement-v2", - "input_period": "2024", - "calibration_period": 2025, - "bound_measurements": { - "cgt_2024_gains": { - "model_variable": "capital_gains", - "measurement_period": self.cgt_period, - } - }, - } - return receipt + original_replace = Path.replace - monkeypatch.setattr( - builder, - "compute_household_metrics", - lambda _simulation, area_type, *, household_ids, **_kwargs: pd.DataFrame( - {f"{area_type}_metric": np.arange(len(household_ids), dtype=float)}, - index=household_ids, - ), - ) + def interrupt_support(self, target): + if Path(target) == output_paths["support"]: + raise KeyboardInterrupt + return original_replace(self, target) - def resolve(): - return builder._resolve_candidate_engine_surface( - clone.frame, - TargetRegistry([], country="uk"), - period=2025, - scratch_dir=tmp_path / "block-scratch", - resolver_factory=StubResolver, - blocks=2, - ) + monkeypatch.setattr(Path, "replace", interrupt_support) + with pytest.raises(KeyboardInterrupt): + rowwise_staging.publish_staged_files(staged, output_paths) - if second_cgt_period != 2024: - with pytest.raises(RuntimeError, match="CGT period contract is inconsistent"): - resolve() - return + assert not output_dir.exists() - prepared, restore, _, metrics, receipt = resolve() - assert len(constructions) == 2 - assert [len(call["frame"].table("household")) for call in constructions] == [ - 12, - 12, - ] - assert receipt["blocks"] == 2 - assert receipt["cgt_period_contract"]["bound_measurements"] == { - "cgt_2024_gains": { - "model_variable": "capital_gains", - "measurement_period": 2024, - } - } - assert receipt["deviation"] == "per_clone_block_engine_resolution" - sensitivity = receipt["block_sensitivity"] - assert ( - "ons/corporate_land_value" - in sensitivity["known_population_normalised_measures"] +def test_release_verdict_requires_single_block_engine() -> None: + builder = rowwise_cli + releasable, posture = builder._release_verdict( + sample_fraction=1.0, engine_blocks=1, release_blocking_gates_passed=True ) - assert set(sensitivity["present_in_this_run"]) <= set( - sensitivity["known_population_normalised_measures"] + assert releasable is True and all(posture.values()) + # A per-block engine resolution never writes a releasable artifact, even + # with every release-blocking gate passed on the full rung (#736 erratum). + releasable, posture = builder._release_verdict( + sample_fraction=1.0, engine_blocks=15, release_blocking_gates_passed=True ) - assert "not evidence for adjudication" in sensitivity["caveat"] + assert releasable is False + assert posture == { + "full_rung": True, + "single_block_engine": False, + "release_blocking_gates_passed": True, + } assert ( - metrics["constituency"].index.tolist() - == clone.frame.table("household")["household_id"].tolist() - ) - assert restore(prepared).table("household").equals(clone.frame.table("household")) - - -def test_joint_candidate_f100_and_f001_end_to_end( - monkeypatch, - tmp_path, - capsys, -) -> None: - """The driver solves one local/ladder/national matrix at both rung postures.""" - - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - _write_staging_h5( - input_h5, - households_per_region=200, - region_masses=(4.0, 10.0, 10.0, 9.0), - ) - ladder_frame = _ladder_frame() - ladder_frame.loc[0, "households"] = 1.0 - english = ladder_frame.iloc[0].copy() - extra_english = [] - for suffix in (2, 3): - row = english.copy() - row["oa_code"] = f"E0000000{suffix}" - row["lsoa_code"] = row["oa_code"] - row["msoa_code"] = row["oa_code"] - row["constituency_code"] = f"E1400000{suffix}" - row["local_authority_code"] = f"E0900000{suffix}" - row["households"] = 1.0 - extra_english.append(row) - ladder_frame = pd.concat( - [ladder_frame, pd.DataFrame(extra_english)], ignore_index=True + builder._release_verdict( + sample_fraction=0.1, engine_blocks=1, release_blocking_gates_passed=True + )[0] + is False ) - payload = assemble_uk_oa_ladder(ladder_frame, _ladder_metadata()) - np.savez_compressed(ladder_path, **payload) - ladder = load_uk_oa_ladder(ladder_path) - from microcosm.build.uk_runtime.ledger_targets import UK_CROSS_GRAIN_BRIDGES - household_bridge = UK_CROSS_GRAIN_BRIDGES[0] - reviewed_missing = { - "ons.household_composition.unrelated_adult_households", - "ons.household_composition.lone_parent_non_dependent_children_households", - "ons.household_composition.multi_family_households", - } - selected_composition = tuple( - target_id - for target_id in household_bridge.higher_target_ids - if target_id not in reviewed_missing - ) - fanout_target_id = "dwp.uc.payment_distribution_single" - fanout_names = tuple(f"payment-band-{index}" for index in range(3)) - national_registry = TargetRegistry( - [ - *[ - TargetSpec( - name=target_id, - entity="household", - measure=f"national/composition_{index}", - value=33.0, - period=2025, - source="synthetic national fixture", - family="ons", - metadata={ - "contract_target_id": target_id, - "ledger_geography_level": "country", - "ledger_geography_id": "K02000001", - }, - hierarchy=_fixture_hierarchy( - target_id, - provider_id="ons", - provider_label="Office for National Statistics", - category_id="ons.household_composition", - category_label="Household composition", - geography_id="K02000001", - geography_label="United Kingdom", - geography_level="country", - target_label="Household composition", - ), - ) - for index, target_id in enumerate(selected_composition) - ], - *[ - TargetSpec( - name=name, - entity="household", - measure=f"national/payment_band_{index}", - value=33.0, - period=2025, - source="synthetic fan-out fixture", - family="dwp_uc", - metadata={ - "contract_target_id": fanout_target_id, - "ledger_geography_level": "country", - "ledger_geography_id": "K03000001", - }, - hierarchy=_fixture_hierarchy( - name, - provider_id="dwp", - provider_label="Department for Work and Pensions", - category_id="dwp.universal_credit", - category_label="Universal Credit", - geography_id="K03000001", - geography_label="Great Britain", - geography_level="country", - target_label="Universal Credit payment distribution", - ), - ) - for index, name in enumerate(fanout_names) - ], - ], - country="uk", - ) - local_registry = TargetRegistry( - [ - *_household_specs_for_ladder(ladder), - TargetSpec( - name="ons.tenure.owned_outright@E09000001", - entity="household", - measure="tenure/owned_outright", - value=1.0, - period=2025, - source="synthetic local fact fixture", - family="ons", - metadata={ - "contract_target_id": "ons.tenure.owned_outright", - "geography_level": "local_authority", - "geography_id": "E09000001", - "ledger_fact_period": "2023", +def test_gate_criticality_reads_fail_closed() -> None: + builder = rowwise_cli + assert builder._is_release_blocking({"criticality": "release_blocking"}) is True + assert builder._is_release_blocking({"criticality": "diagnostic"}) is False + # Missing or unknown criticality vetoes: schema drift on one entry cannot + # drop a failed gate out of both the blocking list and all_gates_passed. + assert builder._is_release_blocking({}) is True + assert builder._is_release_blocking({"criticality": "advisory"}) is True + blocking, diagnostic = builder._gate_failures_by_criticality( + { + "gates": { + "uk_local_area_support": { + "status": "failed", + "failures": ["ESS 42.3 < 50"], + }, + "uk_local_weight_ratio": { + "status": "failed", + "criticality": "diagnostic", + "failures": ["ratio 578 > 100"], + }, + "uk_local_target_fit": { + "status": "passed", + "criticality": "diagnostic", }, - hierarchy=_fixture_hierarchy( - "ons.tenure.owned_outright@E09000001", - provider_id="ons", - provider_label="Office for National Statistics", - category_id="ons.housing", - category_label="Housing", - geography_id="E09000001", - geography_label="City of London", - geography_level="local_authority", - target_label="Owned outright", - ), - ), - ], - country="uk", - ) - artifact = SimpleNamespace( - facts=None, - provenance=lambda: { - "facts_sha256": "1" * 64, - "manifest_sha256": "2" * 64, - "artifact_id": "synthetic-joint-fixture", - }, - ) - joint_inputs = { - "artifact": artifact, - "calibration_year": 2025, - "national_registry": national_registry, - "band_edge_registry": national_registry, - "local_registry": local_registry, - "measure_exclusions": { - f"compiled::{target_id}": { - "tracking": "microcosm#791", - "reason": "relationship-to-head is unavailable", - } - for target_id in reviewed_missing - }, - "reviewed_unbound_higher_targets": { - target_id: { - "tracking": "microcosm#791", - "reason": "relationship-to-head is unavailable", } - for target_id in reviewed_missing - }, - } - monkeypatch.setattr( - builder, "_load_joint_target_inputs", lambda _args: joint_inputs - ) - monkeypatch.setattr(builder, "load_bound_spine_sidecar", lambda *_args: {}) - monkeypatch.setattr( - builder, "spine_provenance_from_sidecar", lambda *_args: {"synthetic": True} - ) - joint_flags = _mandatory_input_flags(input_h5, ladder_path) - - constructions = [] - - class StubResolver: - def __init__(self, **kwargs): - constructions.append(kwargs) - self.frame = kwargs["frame"] - # A live resolver writes the frame to a scratch H5 through the - # national-frame writer, which validates the mass chain; a block - # must therefore carry a record whose total equals its weights. - validate_uk_national_frame(self.frame) - self.simulation = object() - self.contract_targets = {} - - def receipt(self): - return {"mode": "stub", "policyengine_uk_version": "test"} - - monkeypatch.setattr( - builder, - "resolve_target_measures", - # A live resolver injects ENGINE INPUTS (scratch columns the - # materialization reads), some of which also exist on another entity - # (region, esa_* on the spine); the driver must drop them before the - # prepared frame or the flattening rule refuses the duplicate column. - lambda _factory, _registry, provider, **_kwargs: SimpleNamespace( - measure_inputs={ - ("household", "stub_engine_input"): np.ones( - len(provider.frame.table("household")), dtype=float - ), - ("person", "region"): np.zeros( - len(provider.frame.table("person")), dtype=float - ), - } - ), - ) - - def _stub_materialize(adapter, registry, *, period, band_edge_registry=None): - # Materialization is what mints the prepared measure columns the - # national rows compile against; the stub writes them from the - # injected input so the lifecycle matches the real stage. - for spec in registry.specs: - table = adapter.tables[spec.entity] - table[spec.measure] = np.ones(len(table), dtype=float) - return SimpleNamespace(skipped=()) - - monkeypatch.setattr(builder, "materialize_uk_ledger_targets", _stub_materialize) - monkeypatch.setattr( - builder, - "compute_household_metrics", - lambda _simulation, area_type, *, household_ids, **_kwargs: pd.DataFrame( - { - "households": np.ones(len(household_ids), dtype=float), - **( - {"tenure/owned_outright": np.ones(len(household_ids), dtype=float)} - if area_type == "la" - else {} - ), - }, - index=household_ids, - ), - ) - real_resolve = builder._resolve_candidate_engine_surface - monkeypatch.setattr( - builder, - "_resolve_candidate_engine_surface", - lambda *args, **kwargs: real_resolve( - *args, resolver_factory=StubResolver, **kwargs - ), - ) - monkeypatch.setattr( - builder, - "rotated_uk_local_holdout", - lambda *_args, **_kwargs: {"report_only": True, "folds": []}, - ) - import microcosm.build.uk_runtime.battery_bindings as battery_bindings - - monkeypatch.setattr( - battery_bindings, - "_local_area_roster", - lambda _resource, levels: { - "constituency": tuple(sorted(set(ladder.constituency_code))), - "local_authority": tuple(sorted(set(ladder.local_authority_code))), - }, - ) - real_support_summary = builder.uk_ladder_area_support_summary - - def support_summary(household, ladder_arg): - if "household_weight" in household: - return real_support_summary(household, ladder_arg) - support = pd.DataFrame( - { - "nonzero_households": [len(household)], - "effective_sample_size": [float(len(household))], - "nonzero_source_households": [ - household["source_household_id"].nunique() - ], - } - ) - return {"constituency": support, "la": support} - - monkeypatch.setattr(builder, "uk_ladder_area_support_summary", support_summary) - - dry_out = tmp_path / "joint-dry" - assert ( - builder.main( - [ - "--input-h5", - str(input_h5), - "--release-role", - "dense", - "--ladder", - str(ladder_path), - "--out", - str(dry_out), - "--n-clones", - "2", - "--dry-run", - *joint_flags, - ] - ) - == 0 - ) - dry_plan = json.loads(capsys.readouterr().out) - dry_unbound = dry_plan["cross_grain"]["unbound_bridges"] - assert dry_plan["cross_grain"]["empty_legs_licensed"] == [] - assert dry_plan["cross_grain"]["controls_without_lower_rows"] == [] - assert [entry["bridge_id"] for entry in dry_unbound] == [household_bridge.bridge_id] - assert dry_unbound[0]["missing"] == sorted(reviewed_missing) - dry_fanout = dry_plan["cross_grain"]["fanout_targets_not_controls"] - assert dry_fanout == [ - { - "target_id": fanout_target_id, - "geography_id": "K03000001", - "cells": 3, - "cell_names": list(fanout_names), - "activated_sum": 99.0, - "reason": ( - "The activated cells are a band subset, so this distribution " - "is not a cross-grain control." - ), - } - ] - assert "fanout_controls_summed" not in dry_plan["cross_grain"] - assert not dry_out.exists() - - f100_out = tmp_path / "joint-f100" - assert ( - builder.main( - [ - "--input-h5", - str(input_h5), - "--release-role", - "dense", - "--ladder", - str(ladder_path), - "--out", - str(f100_out), - "--n-clones", - "2", - "--epochs", - "2", - "--skip-holdout", - *joint_flags, - ] - ) - == 0 - ) - f100 = json.loads((f100_out / builder.MANIFEST_FILENAME).read_text()) - assert f100["schema_version"] == 4 - # The written rowwise artifact carries the shared ``clone_index`` name on - # every table: the compact national loader must refuse it (flattening - # rule) and the rowwise reader must undo the export rename. - from microcosm.build.uk_runtime.rowwise_dataset import ( - ladder_clone_index_column, - load_uk_rowwise_dataset, - ) - - dataset_path = Path(f100["outputs"]["dataset"]["path"]) - with pytest.raises(ValueError, match="globally unique"): - builder.load_uk_national_frame(dataset_path) - reloaded, provenance = load_uk_rowwise_dataset(dataset_path) - assert provenance.source_h5 == dataset_path.resolve() or str( - provenance.source_h5 - ).endswith(dataset_path.name) - for entity in ("person", "benunit", "household"): - assert ladder_clone_index_column(entity) in reloaded.table(entity).columns - assert "clone_index" not in reloaded.table(entity).columns - assert len(reloaded.table("household")) == f100["solve"]["n_households"] - assert reloaded.weights_for("household").total == pytest.approx( - f100["weights"]["calibration_mass_change"]["new_total"] - ) - # Exact: the reader undoes the export rename and nothing else, so every - # table equals the written one with clone_index renamed back. - for entity in ("person", "benunit", "household"): - written = pd.read_hdf(dataset_path, entity) - expected = written.rename( - columns={"clone_index": ladder_clone_index_column(entity)} - ) - if entity == "household": - # The frame carries the weight as its typed vector, not a column. - np.testing.assert_array_equal( - reloaded.weights_for("household").values, - expected["household_weight"].to_numpy(dtype="float64"), - ) - expected = expected.drop(columns=["household_weight"]) - got = reloaded.table(entity) - assert sorted(got.columns) == sorted(expected.columns) - pd.testing.assert_frame_equal( - got[sorted(got.columns)].reset_index(drop=True), - expected[sorted(expected.columns)].reset_index(drop=True), - check_dtype=True, - ) - assert f100["solve"]["n_targets_by_kind"] == { - "local": 1, - "ladder": 12, - "national": len(national_registry.specs), - } - assert f100["solve"]["n_targets"] == 13 + len(national_registry.specs) - assert f100["solve"]["measure_resolution"]["mode"] == "stub" - assert f100["cross_grain"]["unbound_bridges"] == dry_unbound - assert f100["solve"]["cross_grain"]["unbound_bridges"] == dry_unbound - assert f100["cross_grain"]["empty_legs_licensed"] == [] - assert f100["solve"]["cross_grain"]["empty_legs_licensed"] == [] - assert f100["cross_grain"]["controls_without_lower_rows"] == [] - assert f100["solve"]["cross_grain"]["controls_without_lower_rows"] == [] - assert f100["cross_grain"]["fanout_targets_not_controls"] == dry_fanout - assert f100["solve"]["cross_grain"]["fanout_targets_not_controls"] == dry_fanout - assert "fanout_controls_summed" not in f100["cross_grain"] - assert "fanout_controls_summed" not in f100["solve"]["cross_grain"] - assert f100["releasable"] is True - assert f100["measure_exclusions"] == joint_inputs["measure_exclusions"] - assert f100["census_household_uprating"]["applied"] is False - assert _spool_rows(f100_out)[0].rung == "f100" - - f001_out = tmp_path / "joint-f001" - assert ( - builder.main( - [ - "--input-h5", - str(input_h5), - "--release-role", - "dense", - "--ladder", - str(ladder_path), - "--out", - str(f001_out), - "--n-clones", - "1", - "--sample-fraction", - "0.01", - "--epochs", - "2", - "--skip-holdout", - *joint_flags, - ] - ) - == 0 - ) - f001 = json.loads((f001_out / builder.MANIFEST_FILENAME).read_text()) - assert f001["rung_surface"]["dropped_cells"] > 0 - assert f001["rung_surface"]["dropped_unreachable_cells"] >= 0 - assert isinstance(f001["rung_surface"]["dropped_unreachable_by_grain"], dict) - assert f001["rung_surface"]["dropped_by_grain"]["constituency"] >= 1 - assert f001["rung_surface"]["dropped_by_grain"]["la"] >= 1 - assert "uk_local_area_support" in f001["failing_gate_ids"] - assert f001["releasable"] is False - assert _spool_rows(f001_out)[0].rung == "f001" - assert len(constructions) == 2 - - -def test_candidate_refusal_records_receipt_and_reraises( - monkeypatch, - tmp_path, -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "candidate" - _write_staging_h5(input_h5) - _write_ladder(ladder_path) - household_flags = _configure_households_only_inputs( - builder, monkeypatch, input_h5=input_h5, ladder_path=ladder_path - ) - - def failing_gate(*_args, **_kwargs): - return builder.GateResult( - name="spine_agreement", - passed=False, - failures=("post-calibration coverage failed",), - details={"minimum": 0}, - ) - - original = builder.UK_GATE_REGISTRY["spine_agreement"] - monkeypatch.setattr( - builder, - "UK_GATE_REGISTRY", - { - **builder.UK_GATE_REGISTRY, - "spine_agreement": replace(original, evaluator=failing_gate), - }, - ) - - with pytest.raises( - builder.GateBatteryBlockedError, match="post-calibration coverage failed" - ): - builder.main( - [ - "--input-h5", - str(input_h5), - "--release-role", - "dense", - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - "--n-clones", - "2", - "--seed", - "7", - "--epochs", - "2", - *household_flags, - ] - ) - - rows = _spool_rows(output_dir) - assert len(rows) == 1 - row = rows[0] - assert row.disposition == "failed" - gate_report_path = output_dir / "microcosm_uk_2024_25_local.local_gates.json" - assert gate_report_path.exists() - assert row.gate_verdicts["uk_local_geography_ladder_post_calibration"] == { - "verdict": "failed", - "receipt": ( - f"{_local_ref(gate_report_path)}" - "#/gates/uk_local_geography_ladder_post_calibration" - ), - } - assert row.gate_verdicts["pipeline_error"]["verdict"] == "error" - assert row.gate_verdicts["pipeline_error"]["receipt"].endswith("#/error_type") - - -def test_candidate_binding_adjudication_failure_records_failed_row( - monkeypatch, - tmp_path, -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "candidate" - _write_staging_h5(input_h5) - _write_ladder(ladder_path) - household_flags = _configure_households_only_inputs( - builder, monkeypatch, input_h5=input_h5, ladder_path=ladder_path - ) - - import microcosm.build.uk_runtime.local_rowwise as local_rowwise - - monkeypatch.setattr( - local_rowwise, - "load_uk_reviewed_exclusion_register", - lambda *_args, **_kwargs: {}, - ) - - with pytest.raises(ValueError, match="census_disclosure_control_noise"): - builder.main( - [ - "--input-h5", - str(input_h5), - "--release-role", - "dense", - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - "--n-clones", - "2", - "--seed", - "7", - "--epochs", - "2", - *household_flags, - ] - ) - - rows = _spool_rows(output_dir) - assert len(rows) == 1 - row = rows[0] - assert row.disposition == "failed" - assert "targets_bound" in row.phases_reached - assert "solved" not in row.phases_reached - assert row.gate_verdicts["pipeline_error"]["verdict"] == "error" - assert row.gate_verdicts["pipeline_error"]["receipt"].endswith("#/error_type") - - -def test_candidate_setup_failure_records_failed_row(monkeypatch, tmp_path) -> None: - """A pre-solve setup failure (ladder load) still spools a failed row. - - Adversarial-review finding on #666: input verification, frame/ladder - loading, cloning, and target binding used to run before the recording - envelope opened, so their failures escaped with no Logbook row. - """ - - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "candidate" - _write_staging_h5(input_h5) - _write_ladder(ladder_path) - household_flags = _configure_households_only_inputs( - builder, monkeypatch, input_h5=input_h5, ladder_path=ladder_path - ) - - def failing_ladder_load(_path): - raise RuntimeError("ladder artifact refused to parse") - - monkeypatch.setattr(builder, "load_uk_oa_ladder", failing_ladder_load) - - with pytest.raises(RuntimeError, match="ladder artifact refused to parse"): - builder.main( - [ - "--input-h5", - str(input_h5), - "--release-role", - "dense", - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - "--n-clones", - "2", - "--seed", - "7", - "--epochs", - "2", - *household_flags, - ] - ) - - rows = _spool_rows(output_dir) - assert len(rows) == 1 - row = rows[0] - assert row.disposition == "failed" - assert row.gate_verdicts["pipeline_error"]["verdict"] == "error" - assert row.gate_verdicts["pipeline_error"]["receipt"].endswith("#/error_type") - assert "inputs_pinned" in row.phases_reached - assert "cloned" not in row.phases_reached - # Real input pins were promoted before the failure; the preflight - # placeholder digest must not survive into the row. - assert row.input_pins_digest != builder.preflight_digest( - builder._UK_CANDIDATE_PIPELINE - ) - - -def test_households_only_targets_come_from_compiled_chronicle_registry( - tmp_path, -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - first_path = tmp_path / "assignment_ladder.npz" - second_path = tmp_path / "target_ladder.npz" - _write_staging_h5(input_h5) - assignment_ladder = _write_ladder(first_path) - target_ladder = _write_ladder( - second_path, - household_counts=(4.0, 9.0, 10.0, 10.0), - ) - assignment = builder._clone_with_ladder_binding( - input_h5, - assignment_ladder, - n_clones=2, - seed=7, - source_year=2023, - expected_constituency_vintage="2024_pcon", - source_lineage_modulus=None, - ) - - registry = TargetRegistry(_household_specs_for_ladder(target_ladder), country="uk") - _, problem, cross_grain = builder._build_bound_problem( - assignment, - local_registry=registry, - period=2025, - ) - - expected = sorted( - float(spec.value) - for spec in registry.specs - if spec.metadata["geography_level"] == "constituency" - ) - assert sorted(problem.targets.tolist()) == expected - assert cross_grain["census_household_uprating"]["applied"] is False - assert cross_grain["bound_national_targets"] == [] - - # The households-only scope applies the same per-grain A15 factor as the - # joint scope and carries its receipt into the manifest (Max's review). - uprating = { - "applied": True, - "period": 2025, - "grains": { - "constituency": { - "cells": len(expected), - "census_households_total": sum(expected), - "census_years": [2021, 2022], - "factor": 1.1, - } - }, - } - _, uprated_problem, uprated_cross_grain = builder._build_bound_problem( - assignment, - local_registry=registry, - period=2025, - census_household_uprating=uprating, - ) - assert sorted(uprated_problem.targets.tolist()) == pytest.approx( - [value * 1.1 for value in expected] - ) - receipt = uprated_cross_grain["census_household_uprating"] - assert receipt["applied"] is True - assert receipt["household_cells"]["cells"] == len(expected) - assert receipt["household_cells"]["skipped_cells"] == 0 - assert ( - problem.target_frame["target_name"] - .str.startswith("ons.census.households@") - .all() - ) - assert dict( - zip( - problem.target_frame["area_code"], - problem.target_frame["target_name"], - strict=True, - ) - ) == { - str(spec.metadata["geography_id"]): spec.name - for spec in registry.specs - if spec.metadata["geography_level"] == "constituency" - } - - -def test_candidate_dry_run_refuses_ladder_sidecar_collision( - tmp_path, -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - output_dir = tmp_path / "candidate" - temporary_ladder = tmp_path / "ladder.npz" - ladder_path = output_dir / builder.MANIFEST_FILENAME - _write_staging_h5(input_h5) - _write_ladder(temporary_ladder) - output_dir.mkdir() - temporary_ladder.replace(ladder_path) - ladder_bytes = ladder_path.read_bytes() - household_flags = [ - *_mandatory_input_flags(input_h5, ladder_path), - "--households-only", - ] - - with pytest.raises(ValueError, match="differ"): - builder.main( - [ - "--input-h5", - str(input_h5), - "--release-role", - "dense", - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - "--dry-run", - *household_flags, - ] - ) - - assert ladder_path.read_bytes() == ladder_bytes - assert list(output_dir.iterdir()) == [ladder_path] - - -def test_candidate_publication_rolls_back_on_interrupt( - monkeypatch, - tmp_path, -) -> None: - builder = _load_builder_module() - staging_dir = tmp_path / "staging" - output_dir = tmp_path / "candidate" - staging_dir.mkdir() - output_paths = builder._output_paths( - output_dir, - posture=builder.UK_ROWWISE_DENSE_POSTURE, - vintage="2024_25", - ) - staged = {key: staging_dir / path.name for key, path in output_paths.items()} - for path in staged.values(): - path.write_text("complete staged artifact\n") - - original_replace = Path.replace - - def interrupt_support(self, target): - if Path(target) == output_paths["support"]: - raise KeyboardInterrupt - return original_replace(self, target) - - monkeypatch.setattr(Path, "replace", interrupt_support) - with pytest.raises(KeyboardInterrupt): - builder._publish_staged_files(staged, output_paths) - - assert not output_dir.exists() - - -def test_candidate_weight_ratio_failure_is_reported_and_blocks( - monkeypatch, tmp_path, capsys -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "candidate" - _write_staging_h5( - input_h5, households_per_region=200, region_masses=(4.0, 10.0, 10.0, 9.0) - ) - _write_ladder(ladder_path) - ladder = load_uk_oa_ladder(ladder_path) - _configure_households_only_inputs( - builder, monkeypatch, input_h5=input_h5, ladder_path=ladder_path - ) - import microcosm.build.uk_runtime.battery_bindings as battery_bindings - - monkeypatch.setattr( - battery_bindings, - "_local_area_roster", - lambda _resource, levels: { - "constituency": tuple(sorted(set(ladder.constituency_code))), - "local_authority": tuple(sorted(set(ladder.local_authority_code))), - }, - ) - ratio = builder.UK_GATE_REGISTRY["weight_ratio"] - monkeypatch.setattr( - builder, - "UK_GATE_REGISTRY", - { - **builder.UK_GATE_REGISTRY, - "weight_ratio": replace( - ratio, - evaluator=_failing_gate_evaluator( - builder, "weight_ratio", "ratio 104.6 > 100" - ), - ), - }, - ) - - assert builder.main(_joint_f100_args(input_h5, ladder_path, output_dir)) == 1 - - capsys.readouterr() - manifest = json.loads((output_dir / builder.MANIFEST_FILENAME).read_text()) - assert manifest["failing_gate_ids"] == ["uk_local_weight_ratio"] - assert manifest["blocked_at_f100"] is True - assert manifest["diagnostic_failures"] == [] - assert manifest["blocking_failures"] == [ - "[uk_local_weight_ratio] ratio 104.6 > 100" - ] - assert manifest["releasable"] is False - report = json.loads( - Path(manifest["outputs"]["local_gate_report"]["path"]).read_text() - ) - assert report["gates"]["uk_local_weight_ratio"]["criticality"] == "release_blocking" - assert report["gates"]["uk_local_weight_ratio"]["status"] == "failed" - assert _spool_rows(output_dir)[0].disposition == "failed" - - -def test_candidate_block_partitions_failures_by_criticality( - monkeypatch, tmp_path, capsys -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "candidate" - _write_staging_h5( - input_h5, households_per_region=200, region_masses=(4.0, 10.0, 10.0, 9.0) - ) - _write_ladder(ladder_path) - _configure_households_only_inputs( - builder, monkeypatch, input_h5=input_h5, ladder_path=ladder_path - ) - registry = builder.UK_GATE_REGISTRY - monkeypatch.setattr( - builder, - "UK_GATE_REGISTRY", - { - **registry, - "area_support": replace( - registry["area_support"], - evaluator=_failing_gate_evaluator( - builder, "area_support", "ESS 42.3 < 50" - ), - ), - "weight_ratio": replace( - registry["weight_ratio"], - evaluator=_failing_gate_evaluator( - builder, "weight_ratio", "ratio 578 > 100" - ), - ), - }, - ) - - assert builder.main(_joint_f100_args(input_h5, ladder_path, output_dir)) == 1 - - captured = capsys.readouterr() - assert "artifact unreleasable" in captured.err - manifest = json.loads((output_dir / builder.MANIFEST_FILENAME).read_text()) - assert manifest["failing_gate_ids"] == [ - "uk_local_area_support", - "uk_local_weight_ratio", - ] - assert manifest["blocked_at_f100"] is True - assert manifest["blocking_failures"] == [ - "[uk_local_area_support] ESS 42.3 < 50", - "[uk_local_weight_ratio] ratio 578 > 100", - ] - assert manifest["diagnostic_failures"] == [] - assert manifest["releasable"] is False - assert _spool_rows(output_dir)[0].disposition == "failed" - - -def test_release_verdict_requires_single_block_engine() -> None: - builder = _load_builder_module() - releasable, posture = builder._release_verdict( - sample_fraction=1.0, engine_blocks=1, release_blocking_gates_passed=True - ) - assert releasable is True and all(posture.values()) - # A per-block engine resolution never writes a releasable artifact, even - # with every release-blocking gate passed on the full rung (#736 erratum). - releasable, posture = builder._release_verdict( - sample_fraction=1.0, engine_blocks=15, release_blocking_gates_passed=True - ) - assert releasable is False - assert posture == { - "full_rung": True, - "single_block_engine": False, - "release_blocking_gates_passed": True, - } - assert ( - builder._release_verdict( - sample_fraction=0.1, engine_blocks=1, release_blocking_gates_passed=True - )[0] - is False - ) - - -def test_gate_criticality_reads_fail_closed() -> None: - builder = _load_builder_module() - assert builder._is_release_blocking({"criticality": "release_blocking"}) is True - assert builder._is_release_blocking({"criticality": "diagnostic"}) is False - # Missing or unknown criticality vetoes: schema drift on one entry cannot - # drop a failed gate out of both the blocking list and all_gates_passed. - assert builder._is_release_blocking({}) is True - assert builder._is_release_blocking({"criticality": "advisory"}) is True - blocking, diagnostic = builder._gate_failures_by_criticality( - { - "gates": { - "uk_local_area_support": { - "status": "failed", - "failures": ["ESS 42.3 < 50"], - }, - "uk_local_weight_ratio": { - "status": "failed", - "criticality": "diagnostic", - "failures": ["ratio 578 > 100"], - }, - "uk_local_target_fit": { - "status": "passed", - "criticality": "diagnostic", - }, - } - } + } ) assert blocking == ["[uk_local_area_support] ESS 42.3 < 50"] assert diagnostic == ["[uk_local_weight_ratio] ratio 578 > 100"] -@_BOTH_DRIVERS -def test_release_candidate_refuses_non_doctrine_solve_settings( - driver, tmp_path -) -> None: - builder = _load_builder_module(driver) +def test_release_candidate_refuses_non_doctrine_solve_settings(tmp_path) -> None: + builder = _load_builder_module() pin = "0" * 64 base = [ "--input-h5", @@ -1799,235 +257,49 @@ def test_release_candidate_refuses_non_doctrine_solve_settings( pin, "--out", str(tmp_path / "out"), - "--release-candidate", - ] - # The doctrine defaults are the release posture: nothing to refuse. - args = builder._parse_args(base) - builder._validate_cli_args(args) - assert args.n_clones == builder.UK_ROWWISE_DENSE_POSTURE.clone_count == 15 - assert args.epochs == builder.UK_ROWWISE_DENSE_POSTURE.epochs == 1500 - assert args.target_weight_rule == "grain_equal" - - with pytest.raises(ValueError, match=r"--epochs != doctrine 1500"): - builder._validate_cli_args(builder._parse_args([*base, "--epochs", "512"])) - with pytest.raises(ValueError, match=r"--n-clones != doctrine 15"): - builder._validate_cli_args(builder._parse_args([*base, "--n-clones", "10"])) - with pytest.raises(ValueError, match=r"--target-weight-rule"): - builder._validate_cli_args( - builder._parse_args([*base, "--target-weight-rule", "uniform"]) - ) - - -@_BOTH_DRIVERS -def test_candidate_requires_pinned_ledger_inputs(driver, tmp_path) -> None: - builder = _load_builder_module(driver) - args = builder._parse_args( - [ - "--input-h5", - str(tmp_path / "spine.h5"), - "--release-role", - "dense", - "--input-sha256", - "0" * 64, - "--ladder", - str(tmp_path / "ladder.npz"), - "--ladder-sha256", - "0" * 64, - "--out", - str(tmp_path / "out"), - ] - ) - with pytest.raises(ValueError, match="mandatory"): - builder._validate_cli_args(args) - - -def test_candidate_multi_block_engine_run_is_never_releasable( - monkeypatch, tmp_path, capsys -) -> None: - """End to end: ``--engine-blocks K`` on f100 writes ``releasable: false``. - - Every release-blocking gate passes here; the posture alone withholds the - verdict, and the manifest names the leg (``single_block_engine``). This is - the assertion that catches a future caller bypassing ``_release_verdict``. - """ - - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "candidate" - _write_staging_h5( - input_h5, households_per_region=200, region_masses=(4.0, 10.0, 10.0, 9.0) - ) - _write_ladder(ladder_path) - ladder = load_uk_oa_ladder(ladder_path) - _configure_households_only_inputs( - builder, monkeypatch, input_h5=input_h5, ladder_path=ladder_path - ) - import microcosm.build.uk_runtime.battery_bindings as battery_bindings - - monkeypatch.setattr( - battery_bindings, - "_local_area_roster", - lambda _resource, levels: { - "constituency": tuple(sorted(set(ladder.constituency_code))), - "local_authority": tuple(sorted(set(ladder.local_authority_code))), - }, - ) - - args = [ - *_joint_f100_args(input_h5, ladder_path, output_dir), - "--engine-blocks", - "2", + "--release-candidate", ] - assert builder.main(args) == 0 + # The doctrine defaults are the release posture: nothing to refuse. + args = builder._parse_args(base) + builder._validate_cli_args(args) + assert args.n_clones == builder.UK_ROWWISE_DENSE_POSTURE.clone_count == 15 + assert args.epochs == builder.UK_ROWWISE_DENSE_POSTURE.epochs == 1500 + assert args.target_weight_rule == "grain_equal" - capsys.readouterr() - manifest = json.loads((output_dir / builder.MANIFEST_FILENAME).read_text()) - assert manifest["parameters"]["engine_blocks"] == 2 - assert manifest["blocking_failures"] == [] - assert manifest["releasable"] is False - assert manifest["release_posture"] == { - "full_rung": True, - "single_block_engine": False, - "release_blocking_gates_passed": True, - } + with pytest.raises(ValueError, match=r"--epochs != doctrine 1500"): + builder._validate_cli_args(builder._parse_args([*base, "--epochs", "512"])) + with pytest.raises(ValueError, match=r"--n-clones != doctrine 15"): + builder._validate_cli_args(builder._parse_args([*base, "--n-clones", "10"])) + with pytest.raises(ValueError, match=r"--target-weight-rule"): + builder._validate_cli_args( + builder._parse_args([*base, "--target-weight-rule", "uniform"]) + ) -def test_size_candidate_exports_compact_links_and_cannot_claim_dense_release( - monkeypatch, tmp_path -): - pytest.importorskip("tables") - pytest.importorskip("h5py") +def test_candidate_requires_pinned_ledger_inputs(tmp_path) -> None: builder = _load_builder_module() - input_h5 = tmp_path / "spine.h5" - ladder_path = tmp_path / "ladder.npz" - out = tmp_path / "k300" - _write_staging_h5(input_h5, households_per_region=52) - ladder = _write_ladder(ladder_path) - import microcosm.build.uk_runtime.battery_bindings as bindings - - monkeypatch.setattr( - bindings, - "_local_area_roster", - lambda _resource, levels: { - "constituency": tuple(sorted(set(ladder.constituency_code))), - "local_authority": tuple(sorted(set(ladder.local_authority_code))), - }, - ) - flags = _configure_households_only_inputs( - builder, monkeypatch, input_h5=input_h5, ladder_path=ladder_path - ) - status = builder.main( + args = builder._parse_args( [ "--input-h5", - str(input_h5), + str(tmp_path / "spine.h5"), "--release-role", "dense", + "--input-sha256", + "0" * 64, "--ladder", - str(ladder_path), - *flags, + str(tmp_path / "ladder.npz"), + "--ladder-sha256", + "0" * 64, "--out", - str(out), - "--n-clones", - "2", - "--dataset-households", - "300", - "--selection-seed", - "11", - "--epochs", - "2", - "--skip-holdout", - "--seed", - "7", + str(tmp_path / "out"), ] ) - assert status in (0, 1) # Gate failures remain reportable candidates. - manifest = json.loads((out / builder.MANIFEST_FILENAME).read_text()) - assert manifest["releasable"] is False - assert manifest["release_posture"]["size_certification_present"] is False - size = manifest["solve"]["dataset_size"] - assert size["requested_households"] == size["realized_households"] == 300 - assert size["pool_households"] == 416 - assert manifest["parameters"]["n_clones"] == 2 - assert ( - manifest["weights"]["stretch_reference"] - == "normalized_horvitz_thompson_w_over_q" - ) - path = out / "microcosm_uk_2024_25_local.h5" - with pd.HDFStore(path, "r") as store: - households = store["household"] - persons = store["person"] - benunits = store["benunit"] - assert len(households) == 300 - assert set(persons.person_household_id) == set(households.household_id) - assert set(persons.person_benunit_id) == set(benunits.benunit_id) - assert len(_spool_rows(out)) == 1 - - # The selection seed moves the draw only; the manifest records both seeds. - assert manifest["parameters"]["seed"] == 7 - assert manifest["parameters"]["selection_seed"] == 11 - assert size["seed"] == 11 - assert manifest["parameters"]["selection_pi_hi"] == 1.0 - assert size["selection_pi_hi"] == 1.0 - assert manifest["parameters"]["baseline_pi_floor"] == 0.0 - assert size["baseline_pi_floor"] == 0.0 - assert size["baseline_floored_rows"] == 0 - assert size["refit_baseline"] == "normalized_horvitz_thompson_w_over_q" - assert size["selection_receipt"]["pi_hi"] == 1.0 - assert size["selection_feasibility"]["requested_pi_hi"] == 1.0 - assert size["selection_feasibility"]["feasible_at_requested_pi_hi"] is True - - # The dense solve the selection was cut from ships as evidence. - dense = size["dense_reference"] - assert dense["final_loss"] == size["dense_loss"] - assert dense["n_households"] == 416 - assert dense["weights"]["n_records"] == 416 - assert {"effective_sample_size", "max_to_median_positive_weight"} <= set( - dense["weights"] - ) - assert dense["diagnostics_file"] == builder.DENSE_REFERENCE_DIAGNOSTICS_FILENAME - outputs = manifest["outputs"] - dense_csv = out / builder.DENSE_REFERENCE_DIAGNOSTICS_FILENAME - selection_csv = out / builder.DATASET_SIZE_SELECTION_FILENAME - assert Path(outputs["dense_reference_diagnostics"]["path"]) == dense_csv.resolve() - assert Path(outputs["dataset_size_selection"]["path"]) == selection_csv.resolve() - assert ( - outputs["dataset_size_selection"]["sha256"] - == hashlib.sha256(selection_csv.read_bytes()).hexdigest() - ) - dense_rows = pd.read_csv(dense_csv) - assert len(dense_rows) == manifest["solve"]["n_targets"] - assert dense_rows.columns[0] == "grain" - assert {"target", "final_estimate", "abs_relative_error"} <= set(dense_rows.columns) - selection = pd.read_csv(selection_csv) - assert list(selection.columns) == [ - "pool_row_index", - "household_id", - "clone_index", - "design_weight", - "inclusion_probability", - "certainty", - "ht_baseline_weight", - "refit_weight", - ] - assert len(selection) == 300 - assert selection["pool_row_index"].is_unique - assert selection["pool_row_index"].max() < 416 - assert set(selection["household_id"]) == set(households.household_id) - assert (selection["refit_weight"] > 0).all() - assert (selection["design_weight"] > 0).all() - assert ( - int(selection["certainty"].sum()) - == size["selection_receipt"]["certainty_count"] - ) - assert int(selection["certainty"].sum()) == size["protected_carriers"] + with pytest.raises(ValueError, match="mandatory"): + builder._validate_cli_args(args) -@_BOTH_DRIVERS -def test_selection_seed_requires_a_dataset_size(driver, tmp_path): - builder = _load_builder_module(driver) +def test_selection_seed_requires_a_dataset_size(tmp_path): + builder = _load_builder_module() args = builder._parse_args( [ "--input-h5", @@ -2046,7 +318,6 @@ def test_selection_seed_requires_a_dataset_size(driver, tmp_path): builder._validate_cli_args(args) -@_BOTH_DRIVERS @pytest.mark.parametrize( ("argv_tail", "message"), [ @@ -2058,10 +329,8 @@ def test_selection_seed_requires_a_dataset_size(driver, tmp_path): (["--dataset-households", "10", "--baseline-pi-floor", "1.5"], r"in \[0, 1\]"), ], ) -def test_selection_pi_hi_is_candidate_only_and_bounded( - driver, tmp_path, argv_tail, message -): - builder = _load_builder_module(driver) +def test_selection_pi_hi_is_candidate_only_and_bounded(tmp_path, argv_tail, message): + builder = _load_builder_module() args = builder._parse_args( [ "--input-h5", @@ -2080,7 +349,7 @@ def test_selection_pi_hi_is_candidate_only_and_bounded( def test_dense_candidate_manifest_has_no_size_sidecars(tmp_path): - builder = _load_builder_module() + builder = rowwise_cli paths = builder._output_paths( tmp_path, posture=builder.UK_ROWWISE_DENSE_POSTURE, vintage="2024_25" ) @@ -2090,9 +359,8 @@ def test_dense_candidate_manifest_has_no_size_sidecars(tmp_path): assert builder._SIZE_RUN_ONLY_OUTPUTS == {"dense_reference", "selection"} -@_BOTH_DRIVERS -def test_size_cli_refuses_promotion_without_separate_certification(driver, tmp_path): - builder = _load_builder_module(driver) +def test_size_cli_refuses_promotion_without_separate_certification(tmp_path): + builder = _load_builder_module() args = builder._parse_args( [ "--input-h5", @@ -2112,381 +380,8 @@ def test_size_cli_refuses_promotion_without_separate_certification(driver, tmp_p builder._validate_cli_args(args) -def test_size_candidate_checkpoints_before_the_draw_and_resumes_from_it( - monkeypatch, tmp_path, capsys -): - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - from microcosm.build.uk_runtime.size_checkpoint import ( - SIZE_CHECKPOINT_ARRAYS_FILENAME, - SIZE_CHECKPOINT_MANIFEST_FILENAME, - ) - - input_h5 = tmp_path / "spine.h5" - ladder_path = tmp_path / "ladder.npz" - _write_staging_h5(input_h5, households_per_region=52) - ladder = _write_ladder(ladder_path) - import microcosm.build.uk_runtime.battery_bindings as bindings - - monkeypatch.setattr( - bindings, - "_local_area_roster", - lambda _resource, levels: { - "constituency": tuple(sorted(set(ladder.constituency_code))), - "local_authority": tuple(sorted(set(ladder.local_authority_code))), - }, - ) - common = [ - "--input-h5", - str(input_h5), - "--release-role", - "dense", - "--ladder", - str(ladder_path), - *_configure_households_only_inputs( - builder, monkeypatch, input_h5=input_h5, ladder_path=ladder_path - ), - "--n-clones", - "2", - "--dataset-households", - "300", - "--epochs", - "2", - "--skip-holdout", - "--seed", - "7", - ] - first = tmp_path / "first" - status = builder.main([*common, "--out", str(first), "--selection-pi-hi", "0.5"]) - assert status in (0, 1) - # The solve is no longer silent: probe verdicts and the search stop reach - # stderr as they happen, beside the phase lines. - err = capsys.readouterr().err - assert "probe 1/10 done:" in err and "search stopped:" in err - assert "dense solve: epoch 2/2" in err and "refit: epoch 2/2" in err - assert "size selection checkpoint written to" in err - assert (first / SIZE_CHECKPOINT_ARRAYS_FILENAME).is_file() - checkpoint = json.loads((first / SIZE_CHECKPOINT_MANIFEST_FILENAME).read_text()) - assert checkpoint["selection"]["households"] == 300 - assert checkpoint["selection"]["search_pi_hi"] == 0.5 - assert checkpoint["identity"]["dataset_households"] == 300 - assert checkpoint["identity"]["epochs"] == 2 - # The identity carries the solve doctrine; the provenance names the - # writing run (reported on resume, not compared). - assert checkpoint["identity"]["doctrine"] == builder._doctrine_bounds( - builder.UK_ROWWISE_DENSE_POSTURE - ) - assert checkpoint["identity"]["release_role"] == "dense" - assert set(checkpoint["provenance"]) == {"code_pin", "build_id"} - manifest = json.loads((first / builder.MANIFEST_FILENAME).read_text()) - written = manifest["solve"]["dataset_size"]["checkpoint"]["written"] - assert "written_at" not in written and "directory" not in written - size_first = manifest["solve"]["dataset_size"] - assert 0 < size_first["certainty_share"] <= 1 - assert ( - size_first["boundary_draws"] - == 300 - (size_first["selection_receipt"]["certainty_count"]) - ) - assert size_first["zero_target_rows"] >= 0 - weights_block = manifest["weights"] - assert weights_block["stretch_reference"] == "normalized_horvitz_thompson_w_over_q" - assert weights_block["realized_max_weight_ratio_vs_stretch_reference"] > 0 - assert weights_block["realized_max_weight_ratio_vs_design"] > 0 - assert manifest["parameters"]["size_checkpoint"] is True - assert manifest["parameters"]["resume_size_checkpoint"] is None - written = manifest["solve"]["dataset_size"]["checkpoint"]["written"] - assert written["arrays_sha256"] == checkpoint["arrays_sha256"] - assert manifest["solve"]["dataset_size"]["selection_reused"] is False - rows = _spool_rows(first) - assert len(rows) == 1 - assert "size_selection_checkpointed" in rows[0].phases_reached - - # Resume on the same inputs: no dense solve, no search, same draw and refit. - second = tmp_path / "second" - status = builder.main( - [ - *common, - "--out", - str(second), - "--selection-pi-hi", - "0.5", - "--resume-size-checkpoint", - str(first), - ] - ) - assert status in (0, 1) - assert not (second / SIZE_CHECKPOINT_ARRAYS_FILENAME).exists() - resumed = json.loads((second / builder.MANIFEST_FILENAME).read_text()) - assert resumed["parameters"]["size_checkpoint"] is False - assert resumed["parameters"]["resume_size_checkpoint"] == str(first.resolve()) - size = resumed["solve"]["dataset_size"] - assert size["selection_reused"] is True - assert size["selection_search_pi_hi"] == 0.5 - assert ( - size["checkpoint"]["resumed_from"]["arrays_sha256"] - == (checkpoint["arrays_sha256"]) - ) - assert size["dense_loss"] == manifest["solve"]["dataset_size"]["dense_loss"] - assert ( - size["selection_l0_lambda"] - == (manifest["solve"]["dataset_size"]["selection_l0_lambda"]) - ) - first_selection = pd.read_csv(first / builder.DATASET_SIZE_SELECTION_FILENAME) - second_selection = pd.read_csv(second / builder.DATASET_SIZE_SELECTION_FILENAME) - pd.testing.assert_frame_equal(first_selection, second_selection) - assert "size_selection_resumed" in _spool_rows(second)[0].phases_reached - - # Another threshold re-draws from the same checkpoint and records both. - third = tmp_path / "third" - status = builder.main( - [ - *common, - "--out", - str(third), - "--selection-pi-hi", - "1.0", - "--resume-size-checkpoint", - str(first), - ] - ) - assert status in (0, 1) - redrawn = json.loads((third / builder.MANIFEST_FILENAME).read_text()) - assert redrawn["solve"]["dataset_size"]["selection_pi_hi"] == 1.0 - assert redrawn["solve"]["dataset_size"]["selection_search_pi_hi"] == 0.5 - - # A resume whose inputs differ refuses by name, before any solve. - different_epochs = list(common) - different_epochs[different_epochs.index("--epochs") + 1] = "3" - with pytest.raises(ValueError, match="epochs: checkpoint 2 != run 3"): - builder.main( - [ - *different_epochs, - "--out", - str(tmp_path / "fourth"), - "--resume-size-checkpoint", - str(first), - ] - ) - with pytest.raises(ValueError, match="requires --dataset-households"): - builder.main( - [ - "--input-h5", - str(input_h5), - "--release-role", - "dense", - "--ladder", - str(ladder_path), - "--out", - str(tmp_path / "fifth"), - "--resume-size-checkpoint", - str(first), - ] - ) - # An --out that already holds a checkpoint refuses before any solve - # (Vahid's should-fix 2): the checkpoint writer's own refusal came hours - # too late. - stale = tmp_path / "stale" - stale.mkdir() - (stale / SIZE_CHECKPOINT_ARRAYS_FILENAME).write_bytes(b"stale") - (stale / SIZE_CHECKPOINT_MANIFEST_FILENAME).write_text("{}") - with pytest.raises(FileExistsError, match="already holds a size checkpoint"): - builder.main([*common, "--out", str(stale)]) - assert not (stale / builder.MANIFEST_FILENAME).exists() - assert manifest["solve"]["dataset_size"]["checkpoint"]["written"]["stage"] == ( - "before_exact_count_draw" - ) - resumed_receipt = resumed["solve"]["dataset_size"]["checkpoint"]["resumed_from"] - assert ( - resumed_receipt["provenance"]["build_id"] - == checkpoint["provenance"]["build_id"] - ) - assert "written_at" not in resumed_receipt - - # --------------------------------------------------------------------------- - # Staging: telemetry to runs// and the staged dataset bundle. - - # A checkpoint written before the release role existed (no release_role - # in its identity) refuses to resume: the identity is the run's - # contract, and a pre-role checkpoint is rebuilt, never grandfathered. - legacy = tmp_path / "legacy" - shutil.copytree(first, legacy) - legacy_manifest = json.loads( - (legacy / SIZE_CHECKPOINT_MANIFEST_FILENAME).read_text() - ) - del legacy_manifest["identity"]["release_role"] - (legacy / SIZE_CHECKPOINT_MANIFEST_FILENAME).write_text(json.dumps(legacy_manifest)) - with pytest.raises(ValueError, match="release_role: absent in checkpoint"): - builder.main( - [ - *common, - "--out", - str(tmp_path / "from-legacy"), - "--resume-size-checkpoint", - str(legacy), - ] - ) - assert not (tmp_path / "from-legacy" / builder.MANIFEST_FILENAME).exists() - - -def test_candidate_build_stages_telemetry_locally_and_inventories_the_bundle( - monkeypatch, tmp_path, capsys -): - from microcosm.build.staging_v2 import validate_v2_bundle - - builder = _load_builder_module() - input_h5, ladder_path, flags = _staging_run_setup(builder, monkeypatch, tmp_path) - out = tmp_path / "candidate" - status = builder.main(_build_args(input_h5, ladder_path, flags, out)) - assert status == 0 - captured = capsys.readouterr() - manifest = json.loads((out / builder.MANIFEST_FILENAME).read_text()) - # The stdout manifest is the on-disk manifest, evidence blocks included. - assert json.loads(captured.out)["staged_dataset"] == manifest["staged_dataset"] - assert "staged dataset: skipped (local_only)" in captured.err - - run_id = _single_run_id(out) - rows = load_spool_rows(out / "logbook-spool") - assert rows[0].build_id == run_id - bundle = validate_v2_bundle(out / "staging", run_id) - run_manifest = bundle["run_manifest"] - assert run_manifest["status"] == "completed" - assert run_manifest["run_kind"] == "calibration" - assert run_manifest["operation_id"] == "uk_rowwise_candidate" - assert run_manifest["pipeline"]["id"] == "uk-local-candidate" - assert run_manifest["non_release"] is True - assert run_manifest["sample"] == {"mode": "full"} - assert run_manifest["delivery"]["mode"] == "local_only" - assert run_manifest["delivery"]["upload_attempts"] == 0 - assert {a["logical_name"] for a in run_manifest["artifacts"]} == { - "fit_summary", - "staged_dataset", - } - run_dir = out / "staging" / "runs" / run_id - fit_summary = json.loads((run_dir / "artifacts" / "fit_summary.json").read_text()) - assert fit_summary["run_id"] == run_id - assert set(fit_summary["gates"]) == set(builder.UK_LOCAL_GATE_SCOPE) - assert fit_summary["loss"]["final"] == manifest["solve"]["final_loss"] - assert set(fit_summary["fit_by_family"]["local"]) == {"census_households"} - staged_artifact = json.loads( - (run_dir / "artifacts" / "staged_dataset.json").read_text() - ) - assert staged_artifact == manifest["staged_dataset"] - - # Every phase reports started then completed, in build order; the - # calibration progress keeps the last epoch only (thinning at 2 epochs). - events = bundle["events"] - completed = [e["stage_id"] for e in events if e["status"] == "completed"] - assert completed == [ - "input_pinning", - "target_compilation", - "cloning", - "surface_resolution", - "calibration", - "gate_battery", - "holdout", - "output_bundle", - "dataset_staging", - "complete", - ] - for stage in ("target_compilation", "cloning", "calibration", "gate_battery"): - transitions = [e["status"] for e in events if e["stage_id"] == stage] - assert transitions == ["started", "completed"], stage - calibration_rows = json.loads((run_dir / "calibration_progress.json").read_text())[ - "events" - ] - assert [ - (row["epoch"], row["epochs"], row["phase"]) for row in calibration_rows - ] == [(2, 2, None)] - calibration_done = next( - e - for e in events - if e["stage_id"] == "calibration" and e["status"] == "completed" - ) - assert calibration_done["details"]["final_loss"] == manifest["solve"]["final_loss"] - assert calibration_done["details"]["size_checkpoint"] is None - - # The manifest carries both receipts; the bundle inventory is the outputs. - assert manifest["staging_delivery"]["mode"] == "local_only" - assert manifest["staging_delivery"]["run_id"] == run_id - staged = manifest["staged_dataset"] - assert staged["mode"] == "local_only" and staged["status"] == "skipped" - assert staged["prefix"] == f"staged/{run_id}" - assert set(staged["files"]) == { - Path(entry["path"]).name for entry in manifest["outputs"].values() - } - for entry in manifest["outputs"].values(): - assert staged["files"][Path(entry["path"]).name]["sha256"] == entry["sha256"] - # The local sums verify the directory as it is, evidence blocks included. - for line in (out / "sha256sums.txt").read_text().splitlines(): - digest, name = line.split(" ") - assert hashlib.sha256((out / name).read_bytes()).hexdigest() == digest, name - inventory = json.loads((out / "staged_manifest.json").read_text()) - assert inventory["run_id"] == run_id and inventory["files"] == staged["files"] - assert inventory["summary"]["releasable"] is True - assert inventory["telemetry"] == { - "repository": None, - "prefix": f"runs/{run_id}", - "mode": "local_only", - } - # The sidecars are evidence about the outputs, never outputs themselves. - assert "sha256sums" not in manifest["outputs"] - assert "staged_manifest" not in manifest["outputs"] - - -def test_size_candidate_stages_the_search_and_refit_phases(monkeypatch, tmp_path): - from microcosm.build.staging_v2 import validate_v2_bundle - - builder = _load_builder_module() - input_h5, ladder_path, flags = _staging_run_setup(builder, monkeypatch, tmp_path) - out = tmp_path / "k300" - status = builder.main( - _build_args( - input_h5, - ladder_path, - flags, - out, - "--dataset-households", - "300", - "--selection-seed", - "11", - ) - ) - assert status in (0, 1) - run_id = _single_run_id(out) - bundle = validate_v2_bundle(out / "staging", run_id) - assert bundle["run_manifest"]["status"] == "completed" - rows = json.loads( - (out / "staging" / "runs" / run_id / "calibration_progress.json").read_text() - )["events"] - phases = [row["phase"] for row in rows] - assert phases[0] is None and "size_search" in phases and phases[-1] == "size_refit" - probe_rows = [row for row in rows if row["budget_search"] == 1] - assert probe_rows and all(row["l0_lambda"] is not None for row in probe_rows) - assert all(row["epoch"] == 2 for row in rows) - calibration_done = next( - e - for e in bundle["events"] - if e["stage_id"] == "calibration" and e["status"] == "completed" - ) - assert calibration_done["details"]["size_checkpoint"] == "written" - assert calibration_done["details"]["realized_households"] == 300 - manifest = json.loads((out / builder.MANIFEST_FILENAME).read_text()) - assert manifest["releasable"] is False - assert manifest["staged_dataset"]["status"] == "skipped" - assert "dataset_size_selection.csv" in manifest["staged_dataset"]["files"] - fit_summary = json.loads( - ( - out / "staging" / "runs" / run_id / "artifacts" / "fit_summary.json" - ).read_text() - ) - assert fit_summary["dataset_size"]["requested_households"] == 300 - assert "pool_row_indices" not in fit_summary["dataset_size"] - assert fit_summary["releasable"] is False - - def test_staging_epoch_stride_keeps_the_forwarded_rows_bounded(): - builder = _load_builder_module() + builder = rowwise_staging def stride(epochs, households): return builder._staging_epoch_every( @@ -2500,332 +395,20 @@ def stride(epochs, households): assert stride(10000, 55000) == 50 assert stride(100000, None) == 42 for epochs, households in ((2000, 55000), (10000, 55000), (100000, None)): - solves = 1 if households is None else 2 + builder._BUDGET_ITERS + solves = 1 if households is None else 2 + rowwise_cli.BUDGET_ITERS assert ( epochs * solves / stride(epochs, households) <= builder._STAGING_MAX_EPOCH_ROWS ) -def test_telemetry_content_refusal_never_aborts_the_solve( - monkeypatch, tmp_path, capsys -): - from microcosm.build.staging_v2 import StagingContentError, validate_v2_bundle - - builder = _load_builder_module() - input_h5, ladder_path, flags = _staging_run_setup(builder, monkeypatch, tmp_path) - - class Refusing(builder.StagingTelemetryV2): - def calibration_progress(self, event): - raise StagingContentError("Staging file exceeds the 5242880-byte limit.") - - monkeypatch.setattr(rowwise_staging, "StagingTelemetryV2", Refusing) - out = tmp_path / "refused-rows" - status = builder.main(_build_args(input_h5, ladder_path, flags, out)) - assert status == 0 - err = capsys.readouterr().err - assert err.count("no longer forwarded") == 1 - run_id = _single_run_id(out) - bundle = validate_v2_bundle(out / "staging", run_id) - assert bundle["run_manifest"]["status"] == "completed" - assert not ( - out / "staging" / "runs" / run_id / "calibration_progress.json" - ).exists() - manifest = json.loads((out / builder.MANIFEST_FILENAME).read_text()) - assert manifest["staging_delivery"]["mode"] == "local_only" - assert load_spool_rows(out / "logbook-spool")[0].disposition == "iterating" - - -def test_invalid_local_telemetry_bundle_is_a_warning_not_the_runs_failure( - monkeypatch, tmp_path, capsys -): - from microcosm.build.staging_v2 import StagingContractError - - builder = _load_builder_module() - input_h5, ladder_path, flags = _staging_run_setup(builder, monkeypatch, tmp_path) - - class Invalid(builder.StagingTelemetryV2): - def validate_local_bundle(self): - raise StagingContractError("synthetic bundle defect") - - monkeypatch.setattr(rowwise_staging, "StagingTelemetryV2", Invalid) - out = tmp_path / "invalid-bundle" - status = builder.main(_build_args(input_h5, ladder_path, flags, out)) - assert status == 0 - err = capsys.readouterr().err - assert "does not validate" in err and "synthetic bundle defect" in err - manifest = json.loads((out / builder.MANIFEST_FILENAME).read_text()) - assert manifest["staging_delivery"]["mode"] == "local_only" - assert manifest["staged_dataset"]["status"] == "skipped" - rows = load_spool_rows(out / "logbook-spool") - assert rows and rows[0].disposition == "iterating" - - -def test_no_staging_records_both_opt_outs(monkeypatch, tmp_path): - builder = _load_builder_module() - input_h5, ladder_path, flags = _staging_run_setup( - builder, monkeypatch, tmp_path, remote=True - ) - out = tmp_path / "quiet" - status = builder.main( - _build_args(input_h5, ladder_path, flags, out, "--no-staging") - ) - assert status == 0 - assert not (out / "staging").exists() - assert not (out / "sha256sums.txt").exists() - manifest = json.loads((out / builder.MANIFEST_FILENAME).read_text()) - assert manifest["staging_delivery"]["mode"] == "disabled" - assert manifest["staging_delivery"]["opt_out_reason"] == "--no-staging" - assert manifest["staged_dataset"] == { - "contract_version": 1, - "mode": "disabled", - "repository": None, - "prefix": None, - "run_id": None, - "revision": None, - "status": "skipped", - "error_code": None, - "opt_out_reason": "--no-staging", - "files": {}, - } - rows = load_spool_rows(out / "logbook-spool") - assert "dataset_stage_skipped" in rows[0].phases_reached - - -def test_remote_staging_uploads_telemetry_and_the_bundle_in_one_commit( - monkeypatch, tmp_path, capsys -): - builder = _load_builder_module() - input_h5, ladder_path, flags = _staging_run_setup( - builder, monkeypatch, tmp_path, remote=True - ) - hub = _FakeHub() - monkeypatch.setattr(rowwise_staging, "_hub_api", lambda: hub) - monkeypatch.setattr(rowwise_staging, "_hub_token", lambda: "hf_test_token") - out = tmp_path / "remote" - status = builder.main( - _build_args( - input_h5, ladder_path, flags, out, "--staging-upload-interval-seconds", "0" - ) - ) - assert status == 0 - err = capsys.readouterr().err - run_id = _single_run_id(out) - manifest = json.loads((out / builder.MANIFEST_FILENAME).read_text()) - - # Telemetry went to runs// of the staging repository, artifacts - # included, under the fixed prefix and nothing else. - telemetry_paths = hub.paths("policyengine/populace-uk-staging") - assert telemetry_paths == sorted( - f"runs/{run_id}/{name}" - for name in ( - "run_manifest.json", - "progress.json", - "events.ndjson", - "calibration_progress.json", - "artifacts/fit_summary.json", - "artifacts/staged_dataset.json", - ) - ) - delivery = manifest["staging_delivery"] - assert delivery["mode"] == "local_and_remote" - assert delivery["configured_repository"] == "policyengine/populace-uk-staging" - assert delivery["upload_successes"] == delivery["upload_attempts"] > 0 - remote_progress = json.loads( - hub.files[("policyengine/populace-uk-staging", f"runs/{run_id}/progress.json")] - ) - assert remote_progress["status"] == "completed" - - # The bundle went to staged// of the private repository in one - # commit: every output, the manifest as built, and the two sidecars. - assert len(hub.commits) == 1 - commit = hub.commits[0] - assert commit["repo_id"] == "policyengine/populace-uk-private" - expected = {Path(e["path"]).name for e in manifest["outputs"].values()} | { - builder.MANIFEST_FILENAME, - "staged_manifest.json", - "sha256sums.txt", - } - assert commit["paths"] == sorted(f"staged/{run_id}/{name}" for name in expected) - assert hub.paths("policyengine/populace-uk-private") == commit["paths"] - staged = manifest["staged_dataset"] - assert staged["status"] == "uploaded" - assert staged["repository"] == "policyengine/populace-uk-private" - assert staged["prefix"] == f"staged/{run_id}" - assert staged["revision"] == hub.sha - assert ( - f"staged dataset: uploaded at policyengine/populace-uk-private/staged/{run_id}" - in err - ) - remote_h5 = hub.files[ - ( - "policyengine/populace-uk-private", - f"staged/{run_id}/{Path(manifest['outputs']['dataset']['path']).name}", - ) - ] - assert ( - remote_h5 - == (out / Path(manifest["outputs"]["dataset"]["path"]).name).read_bytes() - ) - # The uploaded manifest is the one the bundle was built from; the local - # copy gained the two evidence blocks afterwards. - remote_manifest = json.loads( - hub.files[ - ( - "policyengine/populace-uk-private", - f"staged/{run_id}/{builder.MANIFEST_FILENAME}", - ) - ] - ) - assert ( - "staged_dataset" not in remote_manifest - and "staging_delivery" not in remote_manifest - ) - assert remote_manifest["outputs"] == manifest["outputs"] - rows = load_spool_rows(out / "logbook-spool") - assert "dataset_staged" in rows[0].phases_reached - assert rows[0].disposition == "iterating" - - # Re-staging a directory whose record already says these outputs are - # uploaded touches nothing: the driver's record and revision stand, the - # sidecars keep their bytes, and no commit is made. - stager = _load_tool("stage_uk_rowwise_candidate") - monkeypatch.setattr(stager, "_hub_api", lambda: hub) - sidecar_bytes = (out / "staged_manifest.json").read_bytes() - capsys.readouterr() - assert stager.main(["--run-dir", str(out)]) == 0 - assert "nothing to do" in capsys.readouterr().err - restaged = json.loads((out / builder.MANIFEST_FILENAME).read_text())[ - "staged_dataset" - ] - assert restaged == staged - assert (out / "staged_manifest.json").read_bytes() == sidecar_bytes - for line in (out / "sha256sums.txt").read_text().splitlines(): - digest, name = line.split(" ") - assert hashlib.sha256((out / name).read_bytes()).hexdigest() == digest, name - assert len(hub.commits) == 1 - - # A record that says the upload failed while the Hub already holds these - # outputs: the re-stage finds the bundle and records its own commit, not - # the repository head, which has moved on since. - manifest_path = out / builder.MANIFEST_FILENAME - manifest = json.loads(manifest_path.read_text()) - manifest["staged_dataset"] = { - **staged, - "status": "failed", - "revision": None, - "error_code": "UPLOAD_FAILED", - } - manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True)) - hub.sha = "e" * 40 - assert stager.main(["--run-dir", str(out)]) == 0 - recovered = json.loads(manifest_path.read_text())["staged_dataset"] - assert recovered["status"] == "already_staged" - assert recovered["revision"] == staged["revision"] != hub.sha - assert len(hub.commits) == 1 - for line in (out / "sha256sums.txt").read_text().splitlines(): - digest, name = line.split(" ") - assert hashlib.sha256((out / name).read_bytes()).hexdigest() == digest, name - - # Consumers fetch by run id and get digest-verified local files. - fetcher = _load_tool("fetch_uk_staged_dataset") - monkeypatch.setattr(fetcher, "_hub_api", lambda: hub) - dest = tmp_path / "fetched" - capsys.readouterr() - assert fetcher.main(["--run-id", run_id, "--dest", str(dest)]) == 0 - listed = capsys.readouterr().out.splitlines() - assert str(dest / Path(manifest["outputs"]["dataset"]["path"]).name) in listed - assert (dest / "sha256sums.txt").is_file() - assert ( - dest / Path(manifest["outputs"]["dataset"]["path"]).name - ).read_bytes() == remote_h5 - - -def test_remote_staging_failure_is_recorded_and_the_build_still_succeeds( +def test_remote_dataset_staging_is_refused_up_front_without_credential_or_repo( monkeypatch, tmp_path, capsys ): builder = _load_builder_module() input_h5, ladder_path, flags = _staging_run_setup( builder, monkeypatch, tmp_path, remote=True ) - hub = _FakeHub(fail_commit=True) - monkeypatch.setattr(rowwise_staging, "_hub_api", lambda: hub) - monkeypatch.setattr(rowwise_staging, "_hub_token", lambda: "hf_test_token") - out = tmp_path / "failed-upload" - status = builder.main(_build_args(input_h5, ladder_path, flags, out)) - assert status == 0 - err = capsys.readouterr().err - assert "staged dataset upload failed" in err and "do-not-record" not in err - manifest = json.loads((out / builder.MANIFEST_FILENAME).read_text()) - staged = manifest["staged_dataset"] - assert staged["status"] == "failed" and staged["error_code"] == "UPLOAD_FAILED" - assert staged["revision"] is None and staged["files"] - assert "do-not-record" not in json.dumps(manifest) - assert hub.paths("policyengine/populace-uk-private") == [] - # Telemetry still completed and recorded the outcome. - run_id = _single_run_id(out) - progress = json.loads( - hub.files[("policyengine/populace-uk-staging", f"runs/{run_id}/progress.json")] - ) - assert progress["status"] == "completed" - events = [ - json.loads(line) - for line in hub.files[ - ("policyengine/populace-uk-staging", f"runs/{run_id}/events.ndjson") - ] - .decode() - .splitlines() - if line - ] - done = next( - e - for e in events - if e["stage_id"] == "dataset_staging" and e["status"] == "completed" - ) - assert done["details"]["status"] == "failed" - assert done["details"]["error_code"] == "UPLOAD_FAILED" - rows = load_spool_rows(out / "logbook-spool") - assert rows[0].disposition == "iterating" - assert "dataset_stage_failed" in rows[0].phases_reached - # The sidecars are in place for a later re-stage. - assert (out / "sha256sums.txt").is_file() and ( - out / "staged_manifest.json" - ).is_file() - - -def test_no_staged_dataset_keeps_telemetry_remote_and_the_bundle_local( - monkeypatch, tmp_path -): - builder = _load_builder_module() - input_h5, ladder_path, flags = _staging_run_setup( - builder, monkeypatch, tmp_path, remote=True - ) - hub = _FakeHub() - monkeypatch.setattr(rowwise_staging, "_hub_api", lambda: hub) - monkeypatch.setattr(rowwise_staging, "_hub_token", lambda: "hf_test_token") - out = tmp_path / "telemetry-only" - status = builder.main( - _build_args(input_h5, ladder_path, flags, out, "--no-staged-dataset") - ) - assert status == 0 - assert hub.paths("policyengine/populace-uk-private") == [] - assert hub.commits == [] - assert hub.paths("policyengine/populace-uk-staging") - manifest = json.loads((out / builder.MANIFEST_FILENAME).read_text()) - assert manifest["staging_delivery"]["mode"] == "local_and_remote" - assert manifest["staged_dataset"]["mode"] == "disabled" - assert manifest["staged_dataset"]["opt_out_reason"] == "--no-staged-dataset" - assert not (out / "sha256sums.txt").exists() - - -@_BOTH_DRIVERS -def test_remote_dataset_staging_is_refused_up_front_without_credential_or_repo( - driver, monkeypatch, tmp_path, capsys -): - builder = _load_builder_module(driver) - input_h5, ladder_path, flags = _staging_run_setup( - builder, monkeypatch, tmp_path, remote=True - ) out = tmp_path / "refused" monkeypatch.setattr(rowwise_staging, "_hub_token", lambda: None) with pytest.raises(ValueError, match="write credential"): @@ -2874,17 +457,11 @@ def repo_info(self, **kwargs): status = builder.main( _build_args(input_h5, ladder_path, flags, tmp_path / "org-scoped") ) - if driver == "graph": - # The pre-flight admits the org-scoped token; the synthetic spine - # carries no bound checkpoint sidecar, so the graph driver refuses - # later, on its inputs, never on the credential. - assert status == 1 - assert "sidecar absent" in capsys.readouterr().err - else: - assert status in (0, 1) - assert org_hub.commits and org_hub.commits[0]["repo_id"] == ( - "policyengine/populace-uk-private" - ) + # The pre-flight admits the org-scoped token; the synthetic spine + # carries no bound checkpoint sidecar, so the graph driver refuses + # later, on its inputs, never on the credential. + assert status == 1 + assert "sidecar absent" in capsys.readouterr().err # Argument refusals cost nothing and come first: a missing credential is # never the reported reason when the arguments are wrong. @@ -2896,38 +473,10 @@ def repo_info(self, **kwargs): ) ) assert not out.exists() - if driver == "graph": - # The local build, the re-stage tool and the in-process dry-run plan - # below need the tool's synthetic households-only path; the graph - # driver's dry run is pinned by - # ``test_graph_driver_dry_run_prints_the_operation_inventory``. - return - - # The re-stage tool refuses the same credential the same way. - stager = _load_tool("stage_uk_rowwise_candidate") - monkeypatch.setattr(stager, "_hub_api", lambda: _FakeHub(role="read")) - local_out = tmp_path / "local" - monkeypatch.setattr(rowwise_staging, "_hub_token", lambda: None) - assert builder.main( - _build_args(input_h5, ladder_path, flags, local_out, "--staging-local-only") - ) in (0, 1) - with pytest.raises(SystemExit, match="read-only"): - stager.main(["--run-dir", str(local_out)]) - - # A dry run plans without staging, so it needs neither credential nor repo. - capsys.readouterr() - monkeypatch.setattr(rowwise_staging, "_hub_token", lambda: None) - assert ( - builder.main(_build_args(input_h5, ladder_path, flags, out, "--dry-run")) == 0 - ) - plan = json.loads(capsys.readouterr().out) - assert plan["parameters"]["dataset_households"] is None - assert not out.exists() -@_BOTH_DRIVERS -def test_release_role_is_required(driver, tmp_path) -> None: - builder = _load_builder_module(driver) +def test_release_role_is_required(tmp_path) -> None: + builder = _load_builder_module() argv = _dense_argv(tmp_path) argv.remove("--release-role") argv.remove("dense") @@ -2937,9 +486,8 @@ def test_release_role_is_required(driver, tmp_path) -> None: builder._parse_args([*argv, "--release-role", "local"]) -@_BOTH_DRIVERS -def test_release_role_supplies_the_solve_defaults(driver, tmp_path) -> None: - builder = _load_builder_module(driver) +def test_release_role_supplies_the_solve_defaults(tmp_path) -> None: + builder = _load_builder_module() dense = builder._parse_args(_dense_argv(tmp_path)) posture = builder.UK_ROWWISE_DENSE_POSTURE assert (dense.n_clones, dense.seed, dense.epochs, dense.learning_rate) == ( @@ -2971,7 +519,6 @@ def test_release_role_supplies_the_solve_defaults(driver, tmp_path) -> None: ) -@_BOTH_DRIVERS @pytest.mark.parametrize( ("extra", "needle"), [ @@ -2980,23 +527,21 @@ def test_release_role_supplies_the_solve_defaults(driver, tmp_path) -> None: (["--target-weight-rule", "family_equal"], "--target-weight-rule family_equal"), ], ) -def test_dense_role_refusal_table(driver, tmp_path, extra, needle) -> None: - builder = _load_builder_module(driver) +def test_dense_role_refusal_table(tmp_path, extra, needle) -> None: + builder = _load_builder_module() args = builder._parse_args(_dense_argv(tmp_path, *extra)) with pytest.raises(ValueError, match="--release-role dense refuses") as excinfo: builder._validate_cli_args(args) assert needle in str(excinfo.value) -@_BOTH_DRIVERS -def test_dense_role_requires_the_ladder(driver, tmp_path) -> None: - builder = _load_builder_module(driver) +def test_dense_role_requires_the_ladder(tmp_path) -> None: + builder = _load_builder_module() argv = _role_argv(tmp_path, "dense", "--ladder-sha256", "3" * 64) with pytest.raises(ValueError, match="requires --ladder"): builder._validate_cli_args(builder._parse_args(argv)) -@_BOTH_DRIVERS @pytest.mark.parametrize( ("extra", "needle"), [ @@ -3022,17 +567,16 @@ def test_dense_role_requires_the_ladder(driver, tmp_path) -> None: (["--target-weight-rule", "grain_equal"], "--target-weight-rule grain_equal"), ], ) -def test_national_role_refusal_table(driver, tmp_path, extra, needle) -> None: - builder = _load_builder_module(driver) +def test_national_role_refusal_table(tmp_path, extra, needle) -> None: + builder = _load_builder_module() args = builder._parse_args(_role_argv(tmp_path, "national", *extra)) with pytest.raises(ValueError, match="--release-role national refuses") as excinfo: builder._validate_cli_args(args) assert needle in str(excinfo.value) -@_BOTH_DRIVERS -def test_national_role_refuses_release_candidate_with_the_seam_reason(driver, tmp_path): - builder = _load_builder_module(driver) +def test_national_role_refuses_release_candidate_with_the_seam_reason(tmp_path): + builder = _load_builder_module() args = builder._parse_args(_role_argv(tmp_path, "national", "--release-candidate")) with pytest.raises(ValueError, match="cannot sign shippability"): builder._validate_cli_args(args) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py index f724e2b77..76797eb57 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py @@ -1,9 +1,11 @@ """The UK rowwise driver's national release role (microcosm#823). -The national role delegates the build to the calibration seam library, so -these tests stand on the seam run suite's synthetic frame, sidecar and -register (loaded from that module) and on the candidate suite's driver -loader and fake Hub. +The national role delegates the build to the calibration seam library +through ``uk_runtime.national_role`` (the graph driver dispatches there +before any graph preparation), so these tests stand on the seam run suite's +synthetic frame, sidecar and register (loaded from that module) and on the +candidate suite's driver loader and fake Hub; the seam's consumers are +patched on ``national_role``. """ from __future__ import annotations @@ -19,7 +21,7 @@ from microcosm.build.logbook import load_spool_rows from microcosm.build.staging_v2 import validate_v2_bundle -from microcosm.build.uk_runtime import calibration_run, rowwise_staging +from microcosm.build.uk_runtime import calibration_run, national_role, rowwise_staging from microcosm.build.uk_runtime.calibration_run import UK_CALIBRATION_GATE_SCOPE from microcosm.build.uk_runtime.chronicle_feed import ( UKChronicleFeedPinError, @@ -45,7 +47,7 @@ def __init__(self, **kwargs): self.kwargs = kwargs -def _national_inputs(builder, monkeypatch, tmp_path: Path, *, resolver=None): +def _national_inputs(monkeypatch, tmp_path: Path, *, resolver=None): frame = seam._frame() input_h5 = tmp_path / "spine.h5" write_uk_national_frame(frame, input_h5) @@ -69,26 +71,28 @@ def _national_inputs(builder, monkeypatch, tmp_path: Path, *, resolver=None): to_dict=lambda: {"facts_sha256": "1" * 64, "manifest_sha256": "2" * 64} ) monkeypatch.setattr( - builder, "load_ledger_consumer_artifact", lambda path, **kwargs: artifact + national_role, "load_ledger_consumer_artifact", lambda path, **kwargs: artifact ) monkeypatch.setattr( - builder, + national_role, "require_committed_uk_chronicle_feed_pin", lambda facts_sha256, **kwargs: pin, ) monkeypatch.setattr( - builder, + national_role, "compile_uk_target_registry", lambda facts, target_period: SimpleNamespace(registry=registry, unsupported=()), ) - monkeypatch.setattr(builder, "load_uk_calibration_measure_exclusions", lambda p: ()) monkeypatch.setattr( - builder, + national_role, "load_uk_calibration_measure_exclusions", lambda p: () + ) + monkeypatch.setattr( + national_role, "apply_uk_calibration_measure_exclusions", lambda reg, exclusions: (reg, {}), ) monkeypatch.setattr( - builder, + national_role, "UKMeasureResolver", (lambda **kwargs: None) if resolver is None else resolver, ) @@ -178,10 +182,10 @@ def test_uk_national_role_delegates_to_the_seam_library(monkeypatch, tmp_path): pytest.importorskip("tables") builder = candidate._load_builder_module() input_h5, registry, artifact, pin = _national_inputs( - builder, monkeypatch, tmp_path, resolver=_FakeResolver + monkeypatch, tmp_path, resolver=_FakeResolver ) calls: list[dict] = [] - monkeypatch.setattr(builder, "run_uk_calibration", _fake_seam_run(calls)) + monkeypatch.setattr(national_role, "run_uk_calibration", _fake_seam_run(calls)) out = tmp_path / "national" assert builder.main(_argv(input_h5, out, "--no-staging")) == 0 @@ -314,9 +318,7 @@ def test_uk_national_role_builds_the_seam_evidence_and_stages_locally( ): pytest.importorskip("tables") builder = candidate._load_builder_module() - input_h5, registry, _artifact, _pin = _national_inputs( - builder, monkeypatch, tmp_path - ) + input_h5, registry, _artifact, _pin = _national_inputs(monkeypatch, tmp_path) out = tmp_path / "national" assert ( @@ -391,9 +393,7 @@ def test_uk_national_role_publishes_telemetry_and_the_bundle_to_the_hub( ): pytest.importorskip("tables") builder = candidate._load_builder_module() - input_h5, _registry, _artifact, _pin = _national_inputs( - builder, monkeypatch, tmp_path - ) + input_h5, _registry, _artifact, _pin = _national_inputs(monkeypatch, tmp_path) hub = candidate._FakeHub() monkeypatch.setattr(rowwise_staging, "_hub_api", lambda: hub) monkeypatch.setattr(rowwise_staging, "_hub_token", lambda: "hf_test_token") @@ -465,9 +465,7 @@ def test_uk_national_dry_run_prints_the_plan_and_writes_nothing( ): pytest.importorskip("tables") builder = candidate._load_builder_module() - input_h5, registry, _artifact, _pin = _national_inputs( - builder, monkeypatch, tmp_path - ) + input_h5, registry, _artifact, _pin = _national_inputs(monkeypatch, tmp_path) out = tmp_path / "national" assert builder.main(_argv(input_h5, out, "--dry-run")) == 0 @@ -493,11 +491,9 @@ def test_uk_national_role_marks_the_staging_run_failed_on_a_refusal( ): pytest.importorskip("tables") builder = candidate._load_builder_module() - input_h5, _registry, _artifact, _pin = _national_inputs( - builder, monkeypatch, tmp_path - ) + input_h5, _registry, _artifact, _pin = _national_inputs(monkeypatch, tmp_path) monkeypatch.setattr( - builder, + national_role, "compile_uk_target_registry", lambda facts, target_period: SimpleNamespace( registry=None, unsupported=("dwp.uc.households",) @@ -584,12 +580,10 @@ def test_uk_national_role_refuses_a_feed_outside_the_committed_pin( pytest.importorskip("tables") builder = candidate._load_builder_module() - input_h5, _registry, _artifact, _pin = _national_inputs( - builder, monkeypatch, tmp_path - ) + input_h5, _registry, _artifact, _pin = _national_inputs(monkeypatch, tmp_path) # The fixture stubs the check; this test wants the real one. monkeypatch.setattr( - builder, + national_role, "require_committed_uk_chronicle_feed_pin", require_committed_uk_chronicle_feed_pin, ) @@ -600,13 +594,15 @@ def test_uk_national_role_refuses_a_feed_outside_the_committed_pin( manifest_sha256=manifest_sha256, ) monkeypatch.setattr( - builder, "load_ledger_consumer_artifact", lambda path, **kwargs: artifact + national_role, "load_ledger_consumer_artifact", lambda path, **kwargs: artifact ) def compile_must_not_run(facts, target_period): raise AssertionError("the register compiled before the feed pin was checked") - monkeypatch.setattr(builder, "compile_uk_target_registry", compile_must_not_run) + monkeypatch.setattr( + national_role, "compile_uk_target_registry", compile_must_not_run + ) out = tmp_path / "national" with pytest.raises(UKChronicleFeedPinError, match="manifest: loaded"): builder.main(_argv(input_h5, out, "--staging-local-only")) @@ -623,10 +619,10 @@ def test_uk_national_role_records_an_unpinned_feed_override( pytest.importorskip("tables") builder = candidate._load_builder_module() input_h5, _registry, _artifact, _pin = _national_inputs( - builder, monkeypatch, tmp_path, resolver=_FakeResolver + monkeypatch, tmp_path, resolver=_FakeResolver ) monkeypatch.setattr( - builder, + national_role, "require_committed_uk_chronicle_feed_pin", require_committed_uk_chronicle_feed_pin, ) @@ -634,10 +630,10 @@ def test_uk_national_role_records_an_unpinned_feed_override( tmp_path / "ledger", facts_sha256="0" * 64, manifest_sha256="c" * 64 ) monkeypatch.setattr( - builder, "load_ledger_consumer_artifact", lambda path, **kwargs: artifact + national_role, "load_ledger_consumer_artifact", lambda path, **kwargs: artifact ) calls: list[dict] = [] - monkeypatch.setattr(builder, "run_uk_calibration", _fake_seam_run(calls)) + monkeypatch.setattr(national_role, "run_uk_calibration", _fake_seam_run(calls)) out = tmp_path / "national" assert ( builder.main(_argv(input_h5, out, "--no-staging", "--allow-unpinned-feed")) == 0 @@ -765,9 +761,7 @@ def test_uk_dense_role_refuses_the_incumbent_flags(tmp_path) -> None: def test_uk_national_role_requires_the_incumbent_pair(monkeypatch, tmp_path): pytest.importorskip("tables") builder = candidate._load_builder_module() - input_h5, _registry, _artifact, _pin = _national_inputs( - builder, monkeypatch, tmp_path - ) + input_h5, _registry, _artifact, _pin = _national_inputs(monkeypatch, tmp_path) with pytest.raises(ValueError, match="must be given together"): builder.main( _argv( @@ -781,13 +775,12 @@ def test_uk_national_role_requires_the_incumbent_pair(monkeypatch, tmp_path): def test_uk_national_role_refuses_the_incumbent_without_the_scorer() -> None: - builder = candidate._load_builder_module() def missing(name: str): raise ImportError(name) with pytest.raises(ValueError, match="microcosm#967"): - builder._load_candidate_evaluator(importer=missing) + national_role._load_candidate_evaluator(importer=missing) def test_uk_national_role_evaluates_against_the_incumbent( @@ -795,10 +788,8 @@ def test_uk_national_role_evaluates_against_the_incumbent( ): pytest.importorskip("tables") builder = candidate._load_builder_module() - input_h5, registry, _artifact, _pin = _national_inputs( - builder, monkeypatch, tmp_path - ) - monkeypatch.setattr(builder, "run_uk_calibration", _fake_seam_run([])) + input_h5, registry, _artifact, _pin = _national_inputs(monkeypatch, tmp_path) + monkeypatch.setattr(national_role, "run_uk_calibration", _fake_seam_run([])) calls: list[dict] = [] module = _fake_evaluator( lambda kwargs: _receipt( @@ -807,7 +798,7 @@ def test_uk_national_role_evaluates_against_the_incumbent( calls, ) monkeypatch.setattr( - builder, "_load_candidate_evaluator", lambda importer=None: module + national_role, "_load_candidate_evaluator", lambda importer=None: module ) out = tmp_path / "national" @@ -855,17 +846,15 @@ def test_uk_national_role_records_an_evaluation_error_without_failing( ): pytest.importorskip("tables") builder = candidate._load_builder_module() - input_h5, _registry, _artifact, _pin = _national_inputs( - builder, monkeypatch, tmp_path - ) - monkeypatch.setattr(builder, "run_uk_calibration", _fake_seam_run([])) + input_h5, _registry, _artifact, _pin = _national_inputs(monkeypatch, tmp_path) + monkeypatch.setattr(national_role, "run_uk_calibration", _fake_seam_run([])) def explode(kwargs): raise RuntimeError("the incumbent frame refused to load") module = _fake_evaluator(explode, []) monkeypatch.setattr( - builder, "_load_candidate_evaluator", lambda importer=None: module + national_role, "_load_candidate_evaluator", lambda importer=None: module ) out = tmp_path / "national" @@ -890,13 +879,11 @@ def test_uk_national_role_evaluates_after_staging_the_bundle( ): pytest.importorskip("tables") builder = candidate._load_builder_module() - input_h5, _registry, _artifact, _pin = _national_inputs( - builder, monkeypatch, tmp_path - ) + input_h5, _registry, _artifact, _pin = _national_inputs(monkeypatch, tmp_path) calls: list[dict] = [] module = _fake_evaluator(lambda kwargs: _receipt(kwargs), calls) monkeypatch.setattr( - builder, "_load_candidate_evaluator", lambda importer=None: module + national_role, "_load_candidate_evaluator", lambda importer=None: module ) out = tmp_path / "national" @@ -960,12 +947,10 @@ def test_uk_national_role_evaluates_after_staging_the_bundle( def test_uk_national_dry_run_records_the_incumbent(monkeypatch, tmp_path, capsys): pytest.importorskip("tables") builder = candidate._load_builder_module() - input_h5, _registry, _artifact, _pin = _national_inputs( - builder, monkeypatch, tmp_path - ) + input_h5, _registry, _artifact, _pin = _national_inputs(monkeypatch, tmp_path) module = _fake_evaluator(lambda kwargs: _receipt(kwargs), []) monkeypatch.setattr( - builder, "_load_candidate_evaluator", lambda importer=None: module + national_role, "_load_candidate_evaluator", lambda importer=None: module ) assert ( @@ -991,8 +976,7 @@ def test_uk_national_dry_run_records_the_incumbent(monkeypatch, tmp_path, capsys def test_uk_national_role_loads_the_real_scorer_by_its_public_name() -> None: """The driver calls the scorer's public factory name; loading the real module (no stand-in) proves the name exists (Vahid's #965 note 1).""" - builder = candidate._load_builder_module() - module = builder._load_candidate_evaluator() + module = national_role._load_candidate_evaluator() assert module.__name__ == "microcosm.build.uk_runtime.candidate_score" assert callable(module.uk_default_measure_resolver_factory) assert callable(module.evaluate_uk_candidate_against_incumbent) @@ -1002,7 +986,6 @@ def test_uk_score_receipt_telemetry_summary_keeps_the_pruned_block() -> None: """The staged copy drops the three per-target arrays and keeps the pruned block whole, including a populated pruned_targets mapping (the block a real run fills with 120 rows; Vahid's #965 note 2).""" - builder = candidate._load_builder_module() pruned_targets = { f"dwp/uc/family_{i}": { "name": f"dwp/uc/family_{i}", @@ -1030,7 +1013,7 @@ def test_uk_score_receipt_telemetry_summary_keeps_the_pruned_block() -> None: }, "evaluation": {"verdict": "passed"}, } - summary = builder._score_receipt_telemetry_summary(score) + summary = national_role._score_receipt_telemetry_summary(score) assert set(summary) == { "artifacts", "incumbent_unresolvable_pruned", diff --git a/test_support/microcosm_build/uk_full_build_cli.py b/test_support/microcosm_build/uk_full_build_cli.py index 17b1f15d9..b407b2328 100644 --- a/test_support/microcosm_build/uk_full_build_cli.py +++ b/test_support/microcosm_build/uk_full_build_cli.py @@ -117,6 +117,27 @@ def arguments(tmp_path, *extra, role="dense", staging="--no-staging"): ) +def _national_argv(tmp_path, *extra): + return [ + "--release-role", + "national", + "--input-h5", + str(tmp_path / "spine.h5"), + "--input-sha256", + PIN, + "--out", + str(tmp_path / "out"), + "--ledger-facts", + str(tmp_path / "ledger"), + "--ledger-facts-sha256", + PIN, + "--ledger-manifest-sha256", + PIN, + "--no-staging", + *extra, + ] + + SELECTION = { "schema": "microcosm.calibrate.target-selection.v1", "selector": {"geography_levels": None, "explicit": False}, diff --git a/test_support/microcosm_build/uk_rowwise_candidate.py b/test_support/microcosm_build/uk_rowwise_candidate.py index c462a3692..400d1b32e 100644 --- a/test_support/microcosm_build/uk_rowwise_candidate.py +++ b/test_support/microcosm_build/uk_rowwise_candidate.py @@ -1,4 +1,6 @@ -"""Synthetic end-to-end contract for the first UK rowwise candidate.""" +"""Shared support for the UK rowwise command-surface tests: the synthetic +ladder and staging inputs, the fixture hierarchy, the driver loader over +both dense drivers, the fake Hub and the role argument builders.""" # ruff: noqa: F401 @@ -8,8 +10,6 @@ import hashlib import importlib.util import json -import shutil -from dataclasses import replace from pathlib import Path from types import SimpleNamespace @@ -17,18 +17,15 @@ import pandas as pd import pytest -from microcosm.build.logbook import LOGBOOK_ROW_FIELDS, load_spool_rows from microcosm.build.uk_runtime import ( assemble_uk_oa_ladder, - ladder_target_provenance, load_uk_oa_ladder, - read_uk_single_year_weight_metadata, + rowwise_cli, rowwise_staging, write_uk_national_frame, ) from microcosm.build.uk_runtime.national_frame import ( uk_national_frame, - validate_uk_national_frame, ) from microcosm.calibrate import ( CalibrationHierarchy, @@ -84,17 +81,6 @@ def _spool_only_by_default(monkeypatch: pytest.MonkeyPatch) -> None: ) -def _spool_rows(output_dir: Path): - rows = load_spool_rows(output_dir / "logbook-spool") - for row in rows: - assert frozenset(row.to_mapping()) == LOGBOOK_ROW_FIELDS - return rows - - -def _local_ref(path: Path) -> str: - return f"local://{path.resolve().as_posix().lstrip('/')}" - - def _fixture_hierarchy( name: str, *, @@ -120,28 +106,16 @@ def _fixture_hierarchy( ) -#: The two UK dense drivers whose command surface must agree: the rowwise -#: tool (``tools/build_uk_rowwise_candidate.py``) and the graph full build -#: (``microcosm.build.uk_runtime.full_build_cli``). Role and pure-CLI tests -#: run against both; the in-process build tests stay on the tool. -_BOTH_DRIVERS = pytest.mark.parametrize("driver", ["tool", "graph"]) +def _load_builder_module(): + """The one UK rowwise driver: the graph full build (microcosm#901 phase 4). + ``tools/build_uk_rowwise_candidate.py`` is a stub over its ``main``, so + the role and pure-CLI tests run against the package module. + """ -def _load_builder_module(driver: str = "tool"): - if driver == "graph": - from microcosm.build.uk_runtime import full_build_cli + from microcosm.build.uk_runtime import full_build_cli - return full_build_cli - root = _TEST_PATHS.repository - path = root / "tools" / "build_uk_rowwise_candidate.py" - spec = importlib.util.spec_from_file_location( - "build_uk_rowwise_candidate", - path, - ) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return module + return full_build_cli def _ladder_metadata() -> dict[str, object]: @@ -434,46 +408,6 @@ def _configure_households_only_inputs( ] -def _failing_gate_evaluator(builder, name: str, message: str): - def evaluator(*_args, **_kwargs): - return builder.GateResult( - name=name, passed=False, failures=(message,), details={"minimum": 0} - ) - - return evaluator - - -def _joint_f100_args(input_h5: Path, ladder_path: Path, output_dir: Path) -> list[str]: - return [ - "--input-h5", - str(input_h5), - "--release-role", - "dense", - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - "--n-clones", - "2", - "--seed", - "7", - "--epochs", - "2", - "--skip-holdout", - *_mandatory_input_flags(input_h5, ladder_path), - "--households-only", - ] - - -def _load_tool(name: str): - root = _TEST_PATHS.repository - spec = importlib.util.spec_from_file_location(name, root / "tools" / f"{name}.py") - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return module - - class _FakeHub: """One fake Hub serving the telemetry repo and the private dataset repo.""" @@ -613,12 +547,6 @@ def _build_args(input_h5, ladder_path, flags, out, *extra): ] -def _single_run_id(out: Path) -> str: - runs = sorted(path.name for path in (out / "staging" / "runs").iterdir()) - assert len(runs) == 1, runs - return runs[0] - - def _role_argv(tmp_path: Path, role: str, *extra: str) -> list[str]: return [ "--input-h5", diff --git a/tools/build_uk_rowwise_candidate.py b/tools/build_uk_rowwise_candidate.py index ce618ae6c..2da217134 100644 --- a/tools/build_uk_rowwise_candidate.py +++ b/tools/build_uk_rowwise_candidate.py @@ -1,3760 +1,14 @@ -"""Build a joint local, ladder, and national UK rowwise candidate. - -Pinned Ledger facts supply the local and national registries. The command -samples before cloning, resolves both local grains and national measures on the -cloned frame, and calibrates every row in one doctrine solve. A dry run compiles -the registries and reports analytical matrix/support evidence without running -the policy engine, solving, or writing output files. - -The pinned Ledger arguments are mandatory; ``--households-only`` binds only -the Chronicle census-household constituency targets from the same registry. - -``--release-role`` declares which UK dataset line the run builds and is -required: ``dense`` is the joint K-clone surface described above under the -local doctrine; ``national`` builds the certified national line without -cloning, on national targets only, under the calibration-seam doctrine -(microcosm#823). The role fixes every solve default and refuses the other -role's flags, so the declared role is checked against the parameters. +"""Build a UK rowwise candidate in either release role through the graph driver. + +Both roles are served by ``microcosm-build-uk`` +(:mod:`microcosm.build.uk_runtime.full_build_cli`): ``--release-role dense`` +builds the K-clone joint national + local surface through the graph, and +``--release-role national`` dispatches to the retained calibration seam +(:mod:`microcosm.build.uk_runtime.national_role`). This historical entry point +re-exports the driver's ``main`` (microcosm#901 phase 4). """ -from __future__ import annotations - -import argparse -import dataclasses -import hashlib -import importlib -import json -import shutil -import sys -import tempfile -import time -from collections.abc import Mapping -from datetime import UTC, date, datetime -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd - -from microcosm.build.gate_battery import ( - BlockingMode, - EvidenceContext, - GateBatteryBlockedError, - GateBatteryRun, -) -from microcosm.build.gates import GateResult -from microcosm.build.ledger_artifact import load_ledger_consumer_artifact -from microcosm.build.logbook_adoption import ( - AttemptState, - append_phase, - atomic_write_json, - git_code_pin, - local_artifact_reference, - preflight_digest, - resolve_predecessor, - role_pins_digest, - sha256_argument, -) -from microcosm.build.staging_cli import ( - add_staged_dataset_arguments, - add_staging_arguments, - validate_staged_dataset_arguments, - validate_staging_arguments, -) -from microcosm.build.staging_dataset import ( - SHA256SUMS_FILENAME, - parse_sha256sums, - refresh_sha256sums_entry, -) -from microcosm.build.staging_v2 import ( - StagingTelemetryV2, -) -from microcosm.build.target_materialization import resolve_target_measures -from microcosm.build.uk_runtime import ( - UK_GATE_REGISTRY, - CalibrationFrameAdapter, - UKLadderRowwiseDatasetResult, - UkOaLadder, - UKRowwiseDoctrineSolve, - UKRowwiseLocalMatrix, - UKRowwiseNationalRows, - build_uk_rowwise_local_matrix, - build_uk_rowwise_local_surface_matrix, - clone_uk_dataset_with_ladder_geography, - compile_uk_local_target_registry, - compile_uk_target_registry, - compute_household_metrics, - drop_injected_measure_inputs, - inject_measure_inputs, - ladder_clone_index_column, - ladder_target_provenance, - ladder_vs_chronicle_household_dispersion, - load_bound_spine_sidecar, - load_uk_local_area_crosswalk, - load_uk_national_frame, - load_uk_oa_ladder, - local_target_census, - materialize_uk_ledger_targets, - require_adjudicated_uk_local_binding, - rotated_uk_local_holdout, - runtime_provenance, - solve_uk_rowwise_weights_under_doctrine, - spine_provenance_from_sidecar, - uk_area_region_codes, - uk_census_household_uprating, - uk_fit_by_family, - uk_household_weight_kind, - uk_ladder_area_support_summary, - uk_ledger_households_total, - uk_local_doctrine_with_overrides, - uk_local_target_surface, - uk_support_limited_misses, - uk_time_period, - uk_weight_summary, - write_uk_calibration_diagnostics, - write_uk_rowwise_dataset, -) -from microcosm.build.uk_runtime.calibration_run import ( - UK_LOCAL_GATE_SCOPE, - UKCalibrationRunPaths, - finalize_uk_scoped_gate_report, - new_uk_calibration_attempt_id, - run_uk_calibration, - uk_local_gate_scope_exclusions, - uk_scoped_gate_manifest, -) -from microcosm.build.uk_runtime.chronicle_feed import ( - require_committed_uk_chronicle_feed_pin, -) -from microcosm.build.uk_runtime.frs_release import load_uk_frs_release -from microcosm.build.uk_runtime.ledger_targets import _spec_geography -from microcosm.build.uk_runtime.measure_simulation import ( - UKMeasureResolver, - apply_uk_calibration_measure_exclusions, - load_uk_calibration_measure_exclusions, -) -from microcosm.build.uk_runtime.national_doctrine import uk_doctrine_with_overrides -from microcosm.build.uk_runtime.national_sampling import ( - UK_SAMPLE_RUNG_TOKENS, - UK_SAMPLE_SEED_DEFAULT, - sample_uk_spine_frame, -) -from microcosm.build.uk_runtime.rowwise_cli import ( - _BUDGET_ITERS, - _CONSERVE_MASS, - _L0_LAMBDA, - _REPOSITORY, - _SIZE_RUN_ONLY_OUTPUTS, # noqa: F401 (read by the driver tests) - _TARGET_RECORDS, - _UK_CANDIDATE_PIPELINE, # noqa: F401 (read by the driver tests) - AREA_SUPPORT_FILENAME, # noqa: F401 (read by the driver tests) - CALIBRATION_DIAGNOSTICS_FILENAME, # noqa: F401 (read by the driver tests) - DATASET_SIZE_SELECTION_FILENAME, # noqa: F401 (read by the driver tests) - DENSE_REFERENCE_DIAGNOSTICS_FILENAME, - LOCAL_REGISTRY_FILENAME, # noqa: F401 (read by the driver tests) - MANIFEST_FILENAME, # noqa: F401 (read by the driver tests) - PAST_CAP_FILENAME, # noqa: F401 (read by the driver tests) - SOLVE_DIAGNOSTICS_FILENAME, # noqa: F401 (read by the driver tests) - _candidate_clone_counts_argument, - _candidate_identity_digest, - _doctrine_bounds, # noqa: F401 (read by the driver tests) - _gate_failures_by_criticality, - _git_commit, - _git_dirty, - _is_release_blocking, - _json_text, - _local_vintage_census, - _new_candidate_build_id, - _output_paths, - _parameters, - _posture_of, - _record_candidate_attempt, - _record_candidate_error, - _refuse_national_role_arguments, # noqa: F401 (read by the driver tests) - _release_verdict, - _resolve_role_arguments, - _validate_cli_args, -) -from microcosm.build.uk_runtime.rowwise_posture import ( - UK_ROWWISE_DENSE_POSTURE, - UK_ROWWISE_RELEASE_ROLES, - UKRowwisePosture, - uk_rowwise_posture, # noqa: F401 (read by the driver tests) -) -from microcosm.build.uk_runtime.rowwise_staging import ( - _STAGED_DATASET_PHASES, - _STAGING_MAX_EPOCH_ROWS, # noqa: F401 (read by the driver tests) - _STAGING_UPLOAD_INTERVAL_SECONDS, - _add_staging_artifact, - _create_staging_telemetry, - _fail_staging_telemetry, - _finalize_staging_telemetry, - _gate_statuses, - _preflight_staged_dataset, - _publish_staged_files, - _replace_manifest, - _stage, - _stage_dataset, - _staging_delivery, - _staging_epoch_every, - _thinned_epochs, -) -from microcosm.build.uk_runtime.size_checkpoint import ( - uk_size_checkpoint_identity as _size_checkpoint_identity, -) -from microcosm.build.uk_runtime.staging import ( - UK_STAGED_DATASET_REPOSITORY, - UK_STAGING_REPOSITORY, -) -from microcosm.calibrate import TargetRegistry, TargetSpec -from microcosm.diagnostics import DiagnosticsWriteFailure -from microcosm.frame import Frame, MassChangeRecord - -BOUND_TARGET_FAMILIES = ("census_households/constituency",) -BOUND_NATIONAL_TARGETS: tuple[str, ...] = () -# The dense role's gate-policy suffix, kept as a module name for the -# contract-pin tests; the posture record (``rowwise_posture.py``) is the -# source of truth for both roles, and the shared CLI helpers now live in -# ``uk_runtime/rowwise_cli.py`` / ``rowwise_staging.py``. -_LOCAL_GATE_POLICY_SUFFIX = UK_ROWWISE_DENSE_POSTURE.gate_policy_suffix -_PAST_CAP_COUNT_KEYS = ( - "n_targets", - "past_at_init", - "past_at_final", - "escaped", - "frozen", - "pushed_out", -) - - -class _LadderAssignment: - """A clone paired in memory with the exact ladder object that produced it.""" - - def __init__( - self, - result: UKLadderRowwiseDatasetResult, - ladder: UkOaLadder, - ) -> None: - self.result = result - self.ladder = ladder - - -def _sample_candidate_frame( - frame, - *, - fraction: float, - seed: int, -) -> tuple[Any, dict[str, Any]]: - """Sample spine families below f100; keep the full rung untouched.""" - - pre_count = len(frame.table("household")) - if fraction == 1.0: - return frame, { - "fraction": 1.0, - "seed": int(seed), - "rung_token": UK_SAMPLE_RUNG_TOKENS[fraction], - "sampled": False, - "pre_household_count": int(pre_count), - "post_household_count": int(pre_count), - } - - sampled, receipt = sample_uk_spine_frame( - frame, - fraction=fraction, - seed=seed, - ) - return sampled, {"sampled": True, **receipt} - - -def _resolve_candidate_engine_surface( - frame, - national_registry, - *, - period: int, - scratch_dir: Path, - band_edge_registry=None, - resolver_factory=UKMeasureResolver, - blocks: int = 1, -) -> tuple[Any, Any, UKRowwiseNationalRows, dict[str, pd.DataFrame], dict[str, Any]]: - """Resolve national inputs and local metrics on the cloned frame. - - ``blocks=1`` uses one scratch-mode engine for the whole clone. The - reviewed escape hatch ``blocks=K`` resolves each clone index separately, - then rejoins every entity-level prepared column by its stable entity id so - the full-frame target materialization and single solve retain frame order. - """ - - household = frame.table("household") - if blocks < 1: - raise ValueError("engine resolution blocks must be positive.") - if blocks == 1: - block_frames = [(None, frame)] - else: - clone_column = ladder_clone_index_column("household") - if clone_column not in household.columns: - raise ValueError(f"per-clone engine resolution requires {clone_column}.") - clone_indices = tuple(sorted(household[clone_column].unique().tolist())) - if len(clone_indices) != blocks: - raise ValueError( - "engine resolution blocks must match the realized clone indices: " - f"requested {blocks}, found {clone_indices}." - ) - person = frame.table("person") - block_frames = [] - for clone_index in clone_indices: - household_ids = set( - household.loc[ - household[clone_column] == clone_index, - "household_id", - ].tolist() - ) - person_mask = person["person_household_id"].isin(household_ids) - block = frame.select(person_mask) - # The block carries a K-th of the cloned mass while its log still - # ends on the full-clone record, and the scratch export validates - # the chain. Declare the subset explicitly: old = the cloned - # total, new = the block total, reason naming the block. The block - # frame is engine scratch and is discarded after resolution. - block_weights = block.weights_for("household") - full_total = float(frame.weights_for("household").total) - block_total = float(block_weights.total) - subset_record = MassChangeRecord( - entity="household", - old_total=full_total, - new_total=block_total, - declared_factor=block_total / full_total, - reason=( - f"engine resolution block {clone_index} of {blocks}: " - "scratch subset of the cloned frame for measure " - "resolution only, discarded after resolution" - ), - ) - block = Frame( - { - **{name: block.table(name) for name in block.entities}, - **{name: block.link(name) for name in block.links}, - }, - block.schema, - { - entity: block.weights_for(entity) - for entity in block.weighted_entities - }, - block.strata, - mass_log=(*block.mass_log, subset_record), - metadata=block.metadata, - ) - block_frames.append((clone_index, block)) - - measure_parts: dict[tuple[str, str], list[pd.Series]] = {} - metric_parts: dict[str, list[pd.DataFrame]] = { - "constituency": [], - "la": [], - } - resolver_receipts: list[Mapping[str, Any]] = [] - national_input_keys: set[tuple[str, str]] | None = None - for clone_index, block_frame in block_frames: - block_scratch = ( - scratch_dir if clone_index is None else scratch_dir / f"clone-{clone_index}" - ) - resolver = resolver_factory( - simulation_source=None, - scratch_dir=block_scratch, - year=period, - frame=block_frame, - ) - resolution = resolve_target_measures( - lambda block_frame=block_frame: CalibrationFrameAdapter(block_frame), - national_registry, - resolver, - period=period, - ) - keys = set(resolution.measure_inputs) - if national_input_keys is None: - national_input_keys = keys - elif keys != national_input_keys: - raise RuntimeError( - "per-clone engine resolution returned inconsistent national inputs." - ) - for (entity, variable), values in resolution.measure_inputs.items(): - entity_table = block_frame.table(entity) - entity_id = f"{entity}_id" - measure_parts.setdefault((entity, variable), []).append( - pd.Series( - np.asarray(values), - index=entity_table[entity_id].tolist(), - ) - ) - block_household_ids = block_frame.table("household")["household_id"].tolist() - for area_type in metric_parts: - metric_parts[area_type].append( - compute_household_metrics( - resolver.simulation, - area_type, - period=period, - household_ids=block_household_ids, - ) - ) - resolver_receipts.append(resolver.receipt()) - del resolver - simulation_input = block_scratch / "simulation-input.h5" - simulation_input.unlink(missing_ok=True) - try: - block_scratch.rmdir() - except OSError: - pass - - measure_inputs: dict[tuple[str, str], np.ndarray] = {} - for (entity, variable), parts in measure_parts.items(): - combined = pd.concat(parts) - if combined.index.has_duplicates: - raise RuntimeError( - f"per-clone engine resolution duplicated {entity} ids for {variable}." - ) - ordered_ids = frame.table(entity)[f"{entity}_id"] - ordered = combined.reindex(ordered_ids.tolist()) - if ordered.isna().any(): - raise RuntimeError( - f"per-clone engine resolution missed {entity} rows for {variable}." - ) - measure_inputs[(entity, variable)] = ordered.to_numpy() - - full_household_ids = household["household_id"].tolist() - local_metrics = {} - for area_type, parts in metric_parts.items(): - combined = pd.concat(parts) - if combined.index.has_duplicates: - raise RuntimeError( - f"per-clone engine resolution duplicated {area_type} household ids." - ) - ordered = combined.reindex(full_household_ids) - if ordered.isna().any().any(): - raise RuntimeError( - f"per-clone engine resolution missed {area_type} household rows." - ) - local_metrics[area_type] = ordered - - adapter = CalibrationFrameAdapter(frame) - # Injected engine inputs are scratch state for materialization only: - # they must be dropped before the prepared frame is assembled, or the - # flattening rule refuses columns that now exist on two entities - # (region, esa_* on the live spine). Same lifecycle as the national stage. - original_columns = { - entity: set(table.columns) for entity, table in adapter.tables.items() - } - inject_measure_inputs(adapter, measure_inputs) - materialized = materialize_uk_ledger_targets( - adapter, - national_registry, - period=period, - band_edge_registry=( - national_registry if band_edge_registry is None else band_edge_registry - ), - ) - if materialized.skipped: - raise RuntimeError( - "candidate national target materialization skipped row(s): " - f"{[skip.__dict__ for skip in materialized.skipped]}." - ) - modes = {receipt.get("mode") for receipt in resolver_receipts} - versions = {receipt.get("policyengine_uk_version") for receipt in resolver_receipts} - if len(modes) != 1 or len(versions) != 1: - raise RuntimeError("per-clone engine resolver provenance is inconsistent.") - cgt_period_contract = resolver_receipts[0].get("cgt_period_contract") - if any( - block_receipt.get("cgt_period_contract") != cgt_period_contract - for block_receipt in resolver_receipts[1:] - ): - raise RuntimeError("per-clone CGT period contract is inconsistent.") - receipt = { - "mode": next(iter(modes)), - "engine_version": next(iter(versions)), - "households": len(frame.table("household")), - "persons": len(frame.table("person")), - "benunits": len(frame.table("benunit")), - "national_inputs": len(measure_inputs), - "local_metrics": { - area_type: len(metrics.columns) - for area_type, metrics in local_metrics.items() - }, - "blocks": blocks, - } - if cgt_period_contract is not None: - receipt["cgt_period_contract"] = cgt_period_contract - if blocks > 1: - receipt["deviation"] = "per_clone_block_engine_resolution" - present = sorted( - column - for column in UK_BLOCK_SENSITIVE_MEASURE_COLUMNS - if column in measure_inputs - ) - receipt["block_sensitivity"] = { - "known_population_normalised_measures": list( - UK_BLOCK_SENSITIVE_MEASURE_COLUMNS - ), - "present_in_this_run": present, - "caveat": ( - "per-block engine resolution mis-measures population-normalised " - "formulas (each block reproduces a national aggregate); rows " - "on these measures are not evidence for adjudication from this " - "run. Resolve in a single block before ruling on them." - ), - } - try: - scratch_dir.rmdir() - except OSError: - pass - drop_injected_measure_inputs(adapter, measure_inputs, original_columns) - national_rows = UKRowwiseNationalRows( - targets=national_registry.to_target_set(), - registry=national_registry, - families=tuple(sorted({spec.family for spec in national_registry.specs})), - ) - return ( - adapter.prepared_frame(), - adapter.restore, - national_rows, - local_metrics, - receipt, - ) - - -def _pin_from_artifact(info: Mapping[str, Any]) -> dict[str, object]: - return { - "sha256": str(info["sha256"]), - "size_bytes": int(info["bytes"]), - } - - -def _stderr_progress(line: str) -> None: - """Solver progress (epoch losses, budget probes, the search verdict).""" - print(line, file=sys.stderr, flush=True) - - -def _refuse_stale_size_checkpoint(args: argparse.Namespace, out_dir: Path) -> None: - """Refuse an --out holding a checkpoint before the solve, not after it. - - The checkpoint writer refuses to overwrite, but it runs after the dense - solve and the search; a stale checkpoint in --out must fail here, before - the hours are spent. - """ - if args.dataset_households is None or args.no_size_checkpoint: - return - if args.resume_size_checkpoint is not None: - return - from microcosm.build.uk_runtime.size_checkpoint import ( - SIZE_CHECKPOINT_ARRAYS_FILENAME, - SIZE_CHECKPOINT_MANIFEST_FILENAME, - ) - - existing = sorted( - str(out_dir / name) - for name in (SIZE_CHECKPOINT_ARRAYS_FILENAME, SIZE_CHECKPOINT_MANIFEST_FILENAME) - if (out_dir / name).exists() - ) - if existing: - raise FileExistsError( - "refusing to run into an --out that already holds a size checkpoint: " - f"{existing}. Resume from it with --resume-size-checkpoint, or choose " - "another --out." - ) - - -def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--release-role", - choices=UK_ROWWISE_RELEASE_ROLES, - required=True, - help=( - "Which UK dataset line this run builds: 'national' (no cloning, " - "national targets only, the calibration-seam doctrine) or 'dense' " - "(the K-clone joint national + local surface under the local " - "doctrine). The role supplies every unset solve default and " - "refuses the other role's flags." - ), - ) - parser.add_argument( - "--input-h5", - type=Path, - required=True, - help="National Microcosm UK staging H5.", - ) - parser.add_argument( - "--input-sha256", - type=sha256_argument, - help="Pinned SHA-256 of --input-h5 (required for the joint registry path).", - ) - parser.add_argument( - "--ladder", - type=Path, - help="Full-UK OA geography ladder NPZ (required by the dense role).", - ) - parser.add_argument( - "--ladder-sha256", - type=sha256_argument, - help="Pinned SHA-256 of --ladder (required for the joint registry path).", - ) - parser.add_argument("--ledger-facts", type=Path) - parser.add_argument("--ledger-facts-sha256", type=sha256_argument) - parser.add_argument("--ledger-manifest-sha256", type=sha256_argument) - parser.add_argument("--measure-exclusions", type=Path) - parser.add_argument("--register-json", type=Path) - parser.add_argument( - "--target-weight-rule", - choices=("uniform", "grain_equal", "family_equal"), - help=( - "Target-weighting rule; defaults to the role's doctrine " - "(dense: grain_equal, national: family_equal). Any other admitted " - "rule is a receipted override." - ), - ) - parser.add_argument( - "--target-loss-cap", - type=float, - help=( - "National role only: receipted override of the seam doctrine's " - "per-target loss cap." - ), - ) - parser.add_argument( - "--allow-unpinned-feed", - action="store_true", - help=( - "National role only: allow a Ledger artifact whose feed commit is " - "not the committed Chronicle pin (development runs)." - ), - ) - parser.add_argument( - "--incumbent-h5", - type=Path, - help=( - "National role only: the incumbent dataset the finished candidate " - "is evaluated against (microcosm#578 rule 1 on the surface both " - "can materialize). The evaluation runs after the bundle is staged " - "and never blocks the build; its receipt is what the release-cut " - "certifier reads." - ), - ) - parser.add_argument( - "--incumbent-sha256", - help="National role only: the incumbent's SHA-256, verified before it is read.", - ) - parser.add_argument( - "--incumbent-label", - default="enhanced_frs_2024_25", - help="National role only: the incumbent's label in the score receipt.", - ) - parser.add_argument("--release-candidate", action="store_true") - parser.add_argument( - "--households-only", - action="store_true", - help="Bind only Chronicle census-household constituency targets.", - ) - parser.add_argument("--skip-holdout", action="store_true") - parser.add_argument( - "--out", - type=Path, - required=True, - help="Output directory for the candidate H5 and evidence sidecars.", - ) - parser.add_argument( - "--dataset-households", - type=int, - help="Exact output household count after informed L0 and refit; pool clone K is unchanged. Candidate-only until size certification.", - ) - parser.add_argument( - "--n-clones", - type=int, - help="Dense role only; defaults to the doctrine clone count.", - ) - parser.add_argument( - "--candidate-clone-counts", - type=_candidate_clone_counts_argument, - help="Dry-run only comma-separated candidate clone counts.", - ) - parser.add_argument("--seed", type=int, help="Defaults to the role's seed.") - parser.add_argument( - "--selection-seed", - type=int, - help=( - "Seed for the size selection only (informed L0 search, exact-count " - "draw, refit); defaults to --seed. The pool, ladder assignment and " - "dense reference stay on --seed, so two selections compare on one " - "pool. Requires --dataset-households." - ), - ) - parser.add_argument( - "--selection-pi-hi", - type=float, - default=1.0, - help=( - "Certainty threshold of the exact-count draw: gates whose learned open " - "probability reaches it are taken with certainty. 1.0 (default) keeps " - "only the protected carriers certain; a lower value promotes learned " - "near-certain gates (the US exact-k ladder runs 0.95). Candidate-only; " - "recorded in the size receipt. Requires --dataset-households." - ), - ) - parser.add_argument( - "--baseline-pi-floor", - type=float, - default=0.0, - help=( - "Floor on the inclusion probability the refit's Horvitz-Thompson " - "baseline divides each selected row's dense weight by: a boundary " - "row drawn at a few in a million otherwise starts at millions of " - "households and starves every other row under the stretch bound " - "(microcosm#355, Q50 2026-09-10). 0 (default) is the untrimmed " - "baseline. Candidate-only; recorded in the size receipt with the " - "rows it trimmed. Requires --dataset-households; a resumed " - "checkpoint may use a different floor." - ), - ) - parser.add_argument( - "--no-size-checkpoint", - action="store_true", - help=( - "Do not persist the dense solve and the informed L0 search before the " - "exact-count draw. By default a --dataset-households run writes " - "size_selection_checkpoint.{npz,json} into --out so a draw refusal " - "costs a re-draw, not the pool solve (microcosm#355)." - ), - ) - parser.add_argument( - "--resume-size-checkpoint", - type=Path, - help=( - "Directory holding a size_selection_checkpoint written by an earlier " - "--dataset-households run on the same inputs: the pool and the target " - "surface are re-derived and verified, the dense solve and the search " - "are restored, and the run continues at the exact-count draw " - "(--selection-pi-hi may differ; both thresholds are recorded). " - "Requires --dataset-households and the same seeds, epochs and pins." - ), - ) - parser.add_argument( - "--sample-fraction", - type=float, - default=1.0, - help="Spine sampling rung: 0.01, 0.10, or 1.0.", - ) - parser.add_argument( - "--sample-seed", - type=int, - help=f"Dense role only; defaults to {UK_SAMPLE_SEED_DEFAULT}.", - ) - parser.add_argument( - "--engine-blocks", - type=int, - default=1, - help="Resolve one engine or one block per clone (must equal --n-clones).", - ) - parser.add_argument( - "--source-year", - type=int, - help="Survey year recorded for lineage (calibration uses the FRS release year).", - ) - parser.add_argument("--source-lineage-modulus", type=int) - parser.add_argument( - "--epochs", type=int, help="Defaults to the role's doctrine solve length." - ) - parser.add_argument( - "--learning-rate", - type=float, - help="Defaults to the role's learning rate (dense 0.15, national 0.02).", - ) - parser.add_argument( - "--expected-constituency-vintage", - help="Dense role only: constituency vintage required from the ladder.", - ) - parser.add_argument( - "--dry-run", - action="store_true", - help=( - "Print the fenced clone/matrix plan without solving or writing any file." - ), - ) - parser.add_argument( - "--logbook-prev-row-digest", - type=sha256_argument, - help=( - "Optional current Logbook chain head. If omitted, " - "POPULACE_LOGBOOK_PREV_ROW_DIGEST is used, then genesis null." - ), - ) - add_staging_arguments( - parser, - repository=UK_STAGING_REPOSITORY, - default_upload_interval_seconds=_STAGING_UPLOAD_INTERVAL_SECONDS, - ) - add_staged_dataset_arguments(parser, repository=UK_STAGED_DATASET_REPOSITORY) - args = parser.parse_args(argv) - validate_staging_arguments(parser, args) - validate_staged_dataset_arguments(parser, args) - _resolve_role_arguments(args) - return args - - -def main(argv: list[str] | None = None) -> int: - """Run the rowwise candidate build.""" - - args = _parse_args(argv) - _validate_cli_args(args) - posture = _posture_of(args) - if posture.role == "national": - if args.dry_run: - return _national_dry_run(args) - # Argument refusals above cost nothing; the credential check reaches - # the Hub, so it runs last, still before any input is read. - _preflight_staged_dataset(args) - return _run_national_role(args) - if args.candidate_clone_counts is not None and not args.dry_run: - raise ValueError("--candidate-clone-counts is valid only with --dry-run.") - if _CONSERVE_MASS: - raise NotImplementedError( - "the candidate manifest's calibration_mass_change block reads " - "the kernel's free-mass record; a conserve-mass doctrine run " - "appends no record and needs its own reviewed manifest shape " - "before this constant may flip." - ) - if args.dry_run: - # Dry runs plan without solving or writing and record no Logbook - # row on any path, so they need no chain configuration. - return _run_candidate(args, attempt=None) - # Argument refusals above cost nothing; the credential check reaches the - # Hub, so it runs last, still before any input is read. - _preflight_staged_dataset(args) - started_at = time.perf_counter() - started_ts = datetime.now(UTC) - digest = preflight_digest(posture.pipeline) - state = AttemptState( - build_id=_new_candidate_build_id( - seed=args.seed, - timestamp=started_ts, - rung=UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], - ), - identity_digest=digest, - input_pins_digest=digest, - phases_reached=["attempt_started"], - gate_verdicts={ - "pipeline": { - "verdict": "running", - "receipt": "pending-build-scoped-terminal-receipt", - } - }, - ) - # Logbook chain configuration is validated before any terminal work: a - # malformed or conflicting head refuses the run with no row and no side - # effects (#666 adversarial-review finding). - predecessor = resolve_predecessor(args.logbook_prev_row_digest) - # Staging telemetry opens with the attempt, as in the spine builder, so an - # early refusal still leaves a failed run under runs//. - telemetry = _create_staging_telemetry(args, build_id=state.build_id) - try: - return _run_candidate( - args, - attempt={ - "state": state, - "started_at": started_at, - "started_ts": started_ts, - "code_pin": "unresolved-local-git-code-pin", - "predecessor": predecessor, - }, - telemetry=telemetry, - ) - except BaseException as error: - _fail_staging_telemetry(telemetry, error) - raise - - -def _run_candidate( - args: argparse.Namespace, - *, - attempt: dict[str, object] | None, - telemetry: StagingTelemetryV2 | None = None, -) -> int: - """Build the candidate, recording every non-dry terminal outcome. - - The recording envelope opens before input verification so that setup - failures — unreadable inputs, frame or ladder load errors, clone and - target-binding refusals — still spool a failed row (#666 - adversarial-review finding). Dry runs pass ``attempt=None`` and record - nothing. - """ - - out_dir = args.out.expanduser().resolve() - try: - input_h5 = _require_file(args.input_h5, label="--input-h5") - ladder_path = _require_file(args.ladder, label="--ladder") - if out_dir.exists() and not out_dir.is_dir(): - raise ValueError(f"--out must be a directory path, got {out_dir}.") - - input_artifact = _artifact_info(input_h5) - ladder_artifact = _artifact_info(ladder_path) - _verify_requested_pin("--input-h5", input_artifact, requested=args.input_sha256) - _verify_requested_pin("--ladder", ladder_artifact, requested=args.ladder_sha256) - pins = { - "dataset": _pin_from_artifact(input_artifact), - "ladder": _pin_from_artifact(ladder_artifact), - } - _stage( - telemetry, - "input_pinning", - "completed", - dataset_sha256=input_artifact["sha256"], - ladder_sha256=ladder_artifact["sha256"], - ) - state: AttemptState | None = None - if attempt is not None: - unpacked_state = attempt["state"] - assert isinstance(unpacked_state, AttemptState) - state = unpacked_state - attempt["code_pin"] = git_code_pin(_REPOSITORY) - state.input_pins_digest = role_pins_digest(pins) - append_phase(state, "configured") - append_phase(state, "inputs_pinned") - national_frame, _national_provenance = load_uk_national_frame(input_h5) - frs_release = load_uk_frs_release() - calibration_year = int(frs_release.calibration_year) - args._calibration_year = calibration_year - args._frs_vintage = str(frs_release.vintage) - if args.households_only: - args._spine_provenance = {} - else: - spine_sidecar_path = input_h5.with_suffix(".build.json") - spine_sidecar = load_bound_spine_sidecar( - spine_sidecar_path, - national_frame, - ) - args._spine_provenance = spine_provenance_from_sidecar( - spine_sidecar_path, - spine_sidecar, - ) - national_frame, sampling = _sample_candidate_frame( - national_frame, - fraction=args.sample_fraction, - seed=args.sample_seed, - ) - args._sampling_receipt = sampling - if telemetry is not None and args.sample_fraction == 1.0: - # The contract's only sampling statement is "full"; a rung below - # f100 stages a null sample, as the spine builder does. - telemetry.set_sample({"mode": "full"}) - source_year = _source_year( - args.source_year, - time_period=uk_time_period(national_frame), - ) - if state is not None: - # The identity digest waits on the frame-derived source year; - # earlier failures record with the preflight placeholder. - state.identity_digest = _candidate_identity_digest( - pins=pins, - args=args, - source_year=source_year, - ) - output_paths = _output_paths( - out_dir, - posture=_posture_of(args), - vintage=args._frs_vintage, - ) - _validate_output_paths( - output_paths, - input_h5=input_h5, - ladder_path=ladder_path, - ) - _refuse_stale_size_checkpoint(args, out_dir) - ladder = load_uk_oa_ladder(ladder_path) - target_provenance = ladder_target_provenance(ladder) - _stage(telemetry, "target_compilation", "started") - joint_inputs = _load_joint_target_inputs(args) - facts = getattr(joint_inputs.get("artifact"), "facts", None) - if facts is None: - joint_inputs["census_household_uprating"] = { - "applied": False, - "reason": "the joint target inputs carry no Ledger facts.", - } - else: - joint_inputs["census_household_uprating"] = uk_census_household_uprating( - joint_inputs["local_registry"], - uk_ledger_households_total( - facts, period=joint_inputs["calibration_year"] - ), - period=joint_inputs["calibration_year"], - ) - joint_inputs["household_dispersion"] = ladder_vs_chronicle_household_dispersion( - ladder, joint_inputs["local_registry"].specs - ) - _stage( - telemetry, - "target_compilation", - "completed", - local_target_count=len(joint_inputs["local_registry"].specs), - national_target_count=len(joint_inputs["national_registry"].specs), - census_household_uprating_applied=bool( - joint_inputs["census_household_uprating"].get("applied") - ), - ) - if args.release_candidate and not joint_inputs["census_household_uprating"].get( - "applied" - ): - raise SystemExit( - "error: --release-candidate requires the A15 census household " - "uprating: " - + str(joint_inputs["census_household_uprating"].get("reason")) - ) - posture = _posture_of(args) - doctrine, doctrine_override = uk_local_doctrine_with_overrides( - posture.doctrine, - ( - {} - if args.target_weight_rule == posture.target_weight_rule - else {"target_weight_rule": args.target_weight_rule} - ), - ) - args._doctrine_override_receipt = doctrine_override - if doctrine.target_weight_rule != args.target_weight_rule: - raise RuntimeError( - "local doctrine override did not bind the requested rule." - ) - - print("cloning through the ladder route...", file=sys.stderr, flush=True) - _stage(telemetry, "cloning", "started", clone_count=int(args.n_clones)) - assignment = _clone_with_ladder_binding( - national_frame, - ladder, - n_clones=args.n_clones, - seed=args.seed, - source_year=source_year, - expected_constituency_vintage=args.expected_constituency_vintage, - source_lineage_modulus=args.source_lineage_modulus, - ) - clone = assignment.result - if ( - args.dataset_households is not None - and args.dataset_households > clone.frame.n("household") - ): - raise ValueError( - "--dataset-households exceeds the cloned pool; selection never clamps the request." - ) - _stage( - telemetry, - "cloning", - "completed", - clone_count=int(args.n_clones), - pool_rows=int(clone.frame.n("household")), - ) - if state is not None: - append_phase(state, "cloned") - - if not args.households_only and args.dry_run: - plan = _joint_dry_run_plan( - args, - clone=clone, - sampled_spine=national_frame, - ladder=ladder, - joint_inputs=joint_inputs, - source_year=source_year, - input_artifact=input_artifact, - ladder_artifact=ladder_artifact, - target_provenance=target_provenance, - ) - _assert_artifacts_unchanged( - input_h5=input_h5, - input_artifact=input_artifact, - ladder_path=ladder_path, - ladder_artifact=ladder_artifact, - ) - print(_json_text(plan), end="") - return 0 - - _stage(telemetry, "surface_resolution", "started") - if args.households_only: - print( - "binding Chronicle census household targets...", - file=sys.stderr, - flush=True, - ) - household, problem, cross_grain = _build_bound_problem( - assignment, - local_registry=joint_inputs["local_registry"], - period=joint_inputs["calibration_year"], - census_household_uprating=joint_inputs.get("census_household_uprating"), - ) - solve_frame = clone.frame - restore = None - national_rows = None - bound_families = BOUND_TARGET_FAMILIES - measure_resolution: Mapping[str, Any] = {} - args._rung_surface = { - "fraction": float(args.sample_fraction), - "dropped_cells": 0, - "dropped_by_grain": {}, - "dropped_by_family": {}, - } - else: - print("resolving joint local and national surface...", file=sys.stderr) - ( - solve_frame, - restore, - national_rows, - local_metrics, - measure_resolution, - ) = _resolve_candidate_engine_surface( - clone.frame, - joint_inputs["national_registry"], - period=joint_inputs["calibration_year"], - scratch_dir=out_dir.parent - / f".{out_dir.name}.candidate-engine-scratch", - band_edge_registry=joint_inputs["band_edge_registry"], - blocks=args.engine_blocks, - ) - ( - household, - problem, - cross_grain, - bound_families, - rung_surface, - ) = _build_joint_problem( - assignment, - local_registry=joint_inputs["local_registry"], - national_registry=joint_inputs["national_registry"], - local_metrics=local_metrics, - period=joint_inputs["calibration_year"], - sample_fraction=args.sample_fraction, - reviewed_unbound_higher_targets=joint_inputs[ - "reviewed_unbound_higher_targets" - ], - census_household_uprating=joint_inputs.get("census_household_uprating"), - ) - args._rung_surface = rung_surface - args._bound_families = tuple(bound_families) - args._joint_inputs_receipt = joint_inputs - args._measure_resolution = dict(measure_resolution) - _stage( - telemetry, - "surface_resolution", - "completed", - target_count=int(problem.matrix.shape[0]), - bound_family_count=len(bound_families), - engine_blocks=int(args.engine_blocks), - ) - if state is not None: - append_phase(state, "targets_bound") - - if args.dry_run: - binding_adjudications = require_adjudicated_uk_local_binding( - bound_families, - problem.target_frame, - ) - _assert_artifacts_unchanged( - input_h5=input_h5, - input_artifact=input_artifact, - ladder_path=ladder_path, - ladder_artifact=ladder_artifact, - ) - plan = _dry_run_plan( - args, - clone=clone, - problem=problem, - source_year=source_year, - input_artifact=input_artifact, - ladder_artifact=ladder_artifact, - target_provenance=target_provenance, - binding_adjudications=binding_adjudications, - cross_grain=cross_grain, - ) - print(_json_text(plan), end="") - return 0 - - assert attempt is not None - assert state is not None - started_at = attempt["started_at"] - started_ts = attempt["started_ts"] - assert isinstance(started_at, float) - assert isinstance(started_ts, datetime) - predecessor = attempt["predecessor"] - assert predecessor is None or isinstance(predecessor, str) - code_pin = str(attempt["code_pin"]) - - print( - f"solving {problem.matrix.shape[0]} targets x " - f"{problem.matrix.shape[1]} households under the doctrine...", - file=sys.stderr, - flush=True, - ) - checkpoint_identity = _size_checkpoint_identity( - args, pins=pins, source_year=source_year - ) - resume_checkpoint = ( - None - if args.resume_size_checkpoint is None - else args.resume_size_checkpoint.expanduser().resolve() - ) - write_checkpoint = ( - args.dataset_households is not None - and not args.no_size_checkpoint - and resume_checkpoint is None - ) - if resume_checkpoint is not None: - print( - f"resuming the size selection from {resume_checkpoint}...", - file=sys.stderr, - flush=True, - ) - _stage( - telemetry, - "calibration", - "started", - target_count=int(problem.matrix.shape[0]), - pool_rows=int(problem.matrix.shape[1]), - dataset_households=args.dataset_households, - epochs=int(args.epochs), - epoch_every=_staging_epoch_every(args), - resumed_from_checkpoint=resume_checkpoint is not None, - ) - solve = solve_uk_rowwise_weights_under_doctrine( - solve_frame, - problem, - bound_families=bound_families, - national_rows=national_rows, - target_weight_rule=args.target_weight_rule, - restore=restore, - epochs=args.epochs, - learning_rate=args.learning_rate, - conserve_mass=_CONSERVE_MASS, - target_records=_TARGET_RECORDS, - dataset_households=args.dataset_households, - l0_lambda=_L0_LAMBDA, - budget_iters=_BUDGET_ITERS, - seed=args.seed, - selection_seed=args.selection_seed, - selection_pi_hi=args.selection_pi_hi, - baseline_pi_floor=args.baseline_pi_floor, - size_checkpoint_dir=out_dir if write_checkpoint else None, - resume_size_checkpoint=resume_checkpoint, - checkpoint_identity=checkpoint_identity, - checkpoint_provenance={"code_pin": code_pin, "build_id": state.build_id}, - progress=_stderr_progress, - progress_events=( - None - if telemetry is None - else _thinned_epochs( - telemetry.calibration_progress, every=_staging_epoch_every(args) - ) - ), - ) - _validate_solve_result(solve, problem=problem) - if solve.size_receipt is not None and solve.size_receipt.get("checkpoint"): - checkpoint = solve.size_receipt["checkpoint"] - if "written" in checkpoint: - append_phase(state, "size_selection_checkpointed") - print( - f"size selection checkpoint written to {out_dir}", - file=sys.stderr, - flush=True, - ) - elif "resumed_from" in checkpoint: - append_phase(state, "size_selection_resumed") - append_phase(state, "solved") - _stage( - telemetry, - "calibration", - "completed", - final_loss=float(solve.final_loss), - n_nonzero=int(solve.n_nonzero), - realized_households=int(solve.frame.n("household")), - size_checkpoint=_size_checkpoint_state(solve), - ) - - # The kernel minted the calibration mass record inside calibrate() (the - # CALIBRATED kind transition is enforced there too); the record names - # the bound families via the doctrine's mass reason. - calibration_record = solve.frame.mass_log[-1] - if "calibration" not in calibration_record.reason: - raise ValueError( - "calibrated frame's latest mass record is not the calibration " - f"record: {calibration_record.reason!r}." - ) - support = _candidate_area_support( - solve.frame.table("household"), - ladder, - weights=solve.weights, - ) - _validate_support_summary(support) - local_diagnostics = _local_gate_diagnostics(solve.diagnostics) - target_registry, target_geography_levels = _local_diagnostics_registry( - solve, - problem, - national_registry=( - None if args.households_only else joint_inputs["national_registry"] - ), - ) - _stage(telemetry, "gate_battery", "started") - try: - gate_report, candidate_gate = _run_local_gate_battery( - frame=solve.frame, - support=support, - diagnostics=local_diagnostics, - report_path=output_paths["local_gates"], - release_id=state.build_id, - evaluated_on=started_ts.date(), - enforce_only=( - None - if args.sample_fraction == 1.0 - else ("uk_local_geography_ladder_post_calibration",) - ), - # The battery attests the posture it ran under: a release - # candidate blocks on absent evidence and can be shippable. - release_candidate=bool(args.release_candidate), - ) - except GateBatteryBlockedError: - # Write-then-block, extended to the whole evidence bundle: a - # release-blocking failure at f100 still writes the diagnostics, - # manifest and artifact (marked unreleasable) so the block can be - # reviewed; only a non-passing ladder verdict is structural and - # re-raises. The Logbook row records the attempt as failed. - gate_report = json.loads( - output_paths["local_gates"].read_text(encoding="utf-8") - ) - _apply_gate_verdicts(state, gate_report, output_paths["local_gates"]) - ladder_entry = gate_report["gates"].get( - "uk_local_geography_ladder_post_calibration", {} - ) - if ladder_entry.get("status") != "passed": - raise - candidate_gate = clone.gate - blocked_failures, diagnostic_failures = _gate_failures_by_criticality( - gate_report - ) - if not blocked_failures: - # The battery blocked, yet the persisted report names no - # failed release-blocking entry: the report and the error - # disagree, which is structural. - raise - unenforced_failures = [] - else: - _apply_gate_verdicts(state, gate_report, output_paths["local_gates"]) - # Nothing blocked. Release-blocking entries can still hold a - # failure here: below f100 only the ladder gate is enforced, and - # a dev build tolerates absent evidence. Those lines are reported - # as not enforced, never as a block. - unenforced_failures, diagnostic_failures = _gate_failures_by_criticality( - gate_report - ) - blocked_failures = [] - args._gate_report = gate_report - args._blocked_failures = blocked_failures - args._diagnostic_failures = diagnostic_failures - args._unenforced_release_failures = ( - [] if blocked_failures else unenforced_failures - ) - append_phase( - state, "candidate_gated" if not blocked_failures else "candidate_blocked" - ) - _stage( - telemetry, - "gate_battery", - "completed", - gate_statuses=_gate_statuses(gate_report), - blocking_failure_count=len(blocked_failures), - diagnostic_failure_count=len(diagnostic_failures), - ) - - _stage(telemetry, "holdout", "started", skipped=bool(args.skip_holdout)) - if args.skip_holdout: - rotated_holdout = {"skipped": True} - else: - rotated_holdout = rotated_uk_local_holdout( - solve_frame, - problem, - bound_families=bound_families, - national_rows=national_rows, - target_weight_rule=args.target_weight_rule, - restore=restore, - epochs=args.epochs, - learning_rate=args.learning_rate, - conserve_mass=_CONSERVE_MASS, - target_records=_TARGET_RECORDS, - dataset_households=args.dataset_households, - l0_lambda=_L0_LAMBDA, - budget_iters=_BUDGET_ITERS, - solve_seed=args.seed, - selection_seed=args.selection_seed, - selection_pi_hi=args.selection_pi_hi, - baseline_pi_floor=args.baseline_pi_floor, - ) - args._rotated_holdout = rotated_holdout - _stage(telemetry, "holdout", "completed", skipped=bool(args.skip_holdout)) - - candidate = dataclasses.replace( - clone, - frame=solve.frame, - gate=candidate_gate, - output_path=None, - ) - support_by_grain = { - ("la" if grain == "local_authority" else str(grain)): rows.reset_index( - drop=True - ) - for grain, rows in support.groupby("geography_level", sort=True) - } - args._support_limited_misses = uk_support_limited_misses( - solve.diagnostics, - support_by_grain, - max_abs_relative_error=0.25, - ) - _assert_artifacts_unchanged( - input_h5=input_h5, - input_artifact=input_artifact, - ladder_path=ladder_path, - ladder_artifact=ladder_artifact, - ) - - _stage(telemetry, "output_bundle", "started") - manifest = _write_output_bundle( - args, - candidate=candidate, - clone=clone, - problem=problem, - solve=solve, - local_diagnostics=local_diagnostics, - target_registry=target_registry, - target_geography_levels=target_geography_levels, - rotated_holdout=rotated_holdout, - support=support, - calibration_record=calibration_record, - source_year=source_year, - output_paths=output_paths, - input_artifact=input_artifact, - ladder_artifact=ladder_artifact, - target_provenance=target_provenance, - cross_grain=cross_grain, - ) - _stage( - telemetry, - "output_bundle", - "completed", - output_bytes={ - key: int(entry["bytes"]) for key, entry in manifest["outputs"].items() - }, - ) - append_phase(state, "published") - # The staged dataset and the telemetry receipt are evidence about the - # published bundle, so they are appended to the manifest after it is - # on disk (the national build record is rewritten the same way); the - # copy inside the staged bundle predates them and staged_manifest.json - # describes the remote side. - staged_dataset = _stage_dataset( - args, - manifest=manifest, - output_paths=output_paths, - run_id=state.build_id if telemetry is None else telemetry.run_id, - telemetry=telemetry, - ) - append_phase(state, _STAGED_DATASET_PHASES[staged_dataset["status"]]) - try: - _finalize_staging_telemetry(args, telemetry) - finally: - manifest["staging_delivery"] = _staging_delivery(telemetry) - manifest["staged_dataset"] = staged_dataset - _replace_manifest(output_paths["manifest"], manifest) - if (output_paths["manifest"].parent / SHA256SUMS_FILENAME).is_file(): - # The uploaded copy lists the manifest as uploaded; the local - # copy lists the manifest as it now is, evidence included. - refresh_sha256sums_entry( - output_paths["manifest"].parent, output_paths["manifest"].name - ) - state.artifact_location = local_artifact_reference( - output_paths["dataset"], - repository_hint=_REPOSITORY, - ) - spool_path = _record_candidate_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - seed=args.seed, - code_pin=code_pin, - disposition="failed" if blocked_failures else "iterating", - predecessor=predecessor, - spool_dir=out_dir / "logbook-spool", - rung=UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], - ) - print(f"Wrote Logbook row: {spool_path}", file=sys.stderr) - print(_json_text(manifest), end="") - if blocked_failures: - print( - "Gate battery blocked the artifact at f100; evidence bundle " - f"written, artifact unreleasable: {blocked_failures[:5]}", - file=sys.stderr, - ) - return 1 - return 0 - except Exception as error: - if attempt is None: - # Dry runs record no row on any path, including failures. - raise - failed_state = attempt["state"] - assert isinstance(failed_state, AttemptState) - failed_started_at = attempt["started_at"] - failed_started_ts = attempt["started_ts"] - assert isinstance(failed_started_at, float) - assert isinstance(failed_started_ts, datetime) - failed_predecessor = attempt["predecessor"] - assert failed_predecessor is None or isinstance(failed_predecessor, str) - _record_candidate_error( - error=error, - state=failed_state, - started_at=failed_started_at, - started_ts=failed_started_ts, - seed=args.seed, - code_pin=str(attempt["code_pin"]), - predecessor=failed_predecessor, - base_dir=out_dir, - spool_dir=out_dir / "logbook-spool", - rung=UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], - ) - raise - - -def _read_json(path: Path) -> dict[str, Any]: - payload = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(payload, dict): - raise ValueError(f"{path} must hold a JSON object.") - return payload - - -def _ledger_facts_pin(artifact: Any) -> dict[str, object]: - facts_path = ( - artifact.path / "consumer_facts.jsonl" - if artifact.path.is_dir() - else artifact.path - ) - return {"sha256": artifact.facts_sha256, "size_bytes": facts_path.stat().st_size} - - -def _load_national_target_inputs(args: argparse.Namespace) -> dict[str, Any]: - """The national role's target surface: the pinned Ledger artifact, compiled. - - The artifact must be the committed Chronicle feed pin unless - ``--allow-unpinned-feed`` records a reviewed diagnostic run; the - compiled register, less the measure exclusions, is the solve surface and - the full compiled register keeps the band edges; ``--register-json`` - requires the re-derived register to be the frozen scoring surface. - """ - - artifact = load_ledger_consumer_artifact( - args.ledger_facts, - expected_facts_sha256=args.ledger_facts_sha256, - expected_manifest_sha256=args.ledger_manifest_sha256, - ) - pin = require_committed_uk_chronicle_feed_pin( - artifact.facts_sha256, - manifest_sha256=artifact.manifest_sha256, - allow_unpinned_feed=bool(args.allow_unpinned_feed), - ) - calibration_year = int(load_uk_frs_release().calibration_year) - compilation = compile_uk_target_registry( - artifact.facts, target_period=calibration_year - ) - if compilation.unsupported: - raise SystemExit( - f"{len(compilation.unsupported)} national target references " - "failed to compile" - ) - exclusions = load_uk_calibration_measure_exclusions(args.measure_exclusions) - registry, exclusion_receipt = apply_uk_calibration_measure_exclusions( - compilation.registry, exclusions - ) - if args.register_json is not None: - try: - frozen = TargetRegistry.from_json(args.register_json) - except ValueError as error: - raise SystemExit( - f"error: frozen scoring register is unusable: {error}" - ) from error - if frozen.version != registry.version: - raise SystemExit( - "re-derived register differs from the frozen scoring register: " - f"{registry.version} vs {frozen.version}" - ) - return { - "artifact": artifact, - "calibration_year": calibration_year, - "national_registry": registry, - "band_edge_registry": compilation.registry, - "measure_exclusions": exclusion_receipt, - "chronicle_feed_pin": pin.to_dict(), - } - - -def _national_doctrine_overrides(args: argparse.Namespace) -> dict[str, Any]: - """The receipted per-run overrides of the seam doctrine, explicit flags only.""" - - explicit = args._explicit_arguments - overrides: dict[str, Any] = {} - if "epochs" in explicit: - overrides["epochs"] = int(args.epochs) - if "learning_rate" in explicit: - overrides["learning_rate"] = float(args.learning_rate) - if "target_weight_rule" in explicit: - overrides["target_weight_rule"] = str(args.target_weight_rule) - if args.target_loss_cap is not None: - overrides["target_loss_cap"] = float(args.target_loss_cap) - return overrides - - -def _national_dry_run(args: argparse.Namespace) -> int: - """Compile the national target surface and print the plan; write nothing.""" - - posture = _posture_of(args) - input_h5 = _require_file(args.input_h5, label="--input-h5") - input_artifact = _artifact_info(input_h5) - _verify_requested_pin("--input-h5", input_artifact, requested=args.input_sha256) - frs_release = load_uk_frs_release() - inputs = _load_national_target_inputs(args) - doctrine, doctrine_overrides = uk_doctrine_with_overrides( - **_national_doctrine_overrides(args) - ) - plan = { - "schema_version": 3, - "build_kind": "uk_national_calibrated_candidate_plan", - "release_role": posture.role, - "release_id": posture.release_id, - "dry_run": True, - "calibration_year": inputs["calibration_year"], - "inputs": {"dataset": dict(input_artifact)}, - "targets": { - "chronicle": inputs["artifact"].provenance(), - "compiled": len(inputs["band_edge_registry"].specs), - "active": len(inputs["national_registry"].specs), - "excluded": len(inputs["measure_exclusions"]), - "register_sha256": inputs["national_registry"].version, - }, - "doctrine": { - field: getattr(doctrine, field) - for field in ( - "epochs", - "learning_rate", - "max_weight_ratio", - "seed", - "target_loss_cap", - "scale_rule", - "target_weight_rule", - "mass_rule", - "l0_lambda", - ) - }, - "doctrine_overrides": dict(doctrine_overrides), - "parameters": _parameters( - args, - source_year=_source_year( - args.source_year, time_period=str(frs_release.time_period) - ), - ), - "engine": "not_run", - "incumbent": _incumbent_arguments(args), - "releasable": False, - } - print(_json_text(plan)) - return 0 - - -def _run_national_role(args: argparse.Namespace) -> int: - """Build the national line: the calibration seam under the driver's posture. - - The seam library (:func:`run_uk_calibration`) resolves the measures from - the input file, solves under the seam doctrine, runs the six - calibration-seam gates, writes the H5, the diagnostics, the signed gate - report, the build record and the Logbook row exactly as the retiring - seam command did, so a national cut built here is bit-for-bit the seam's. - The driver adds what the dense role has: the pinned input, the role's - doctrine and overrides, staging telemetry under the attempt id, the - rowwise manifest beside the seam's evidence, and the staged bundle. - """ - - posture = _posture_of(args) - out_dir = args.out.expanduser().resolve() - input_h5 = _require_file(args.input_h5, label="--input-h5") - if out_dir.exists() and not out_dir.is_dir(): - raise ValueError(f"--out must be a directory path, got {out_dir}.") - input_artifact = _artifact_info(input_h5) - _verify_requested_pin("--input-h5", input_artifact, requested=args.input_sha256) - incumbent = _incumbent_arguments(args) - # The attempt id is minted before telemetry opens so the staging run id - # and the Logbook row agree, as on the dense role. - build_id = new_uk_calibration_attempt_id(timestamp=datetime.now(UTC)) - telemetry = _create_staging_telemetry(args, build_id=build_id) - try: - return _run_national_attempt( - args, - posture=posture, - out_dir=out_dir, - input_h5=input_h5, - input_artifact=input_artifact, - build_id=build_id, - telemetry=telemetry, - incumbent=incumbent, - ) - except BaseException as error: - _fail_staging_telemetry(telemetry, error) - raise - - -def _run_national_attempt( - args: argparse.Namespace, - *, - posture: UKRowwisePosture, - out_dir: Path, - input_h5: Path, - input_artifact: Mapping[str, Any], - build_id: str, - telemetry: StagingTelemetryV2 | None, - incumbent: Mapping[str, Any] | None = None, -) -> int: - _stage( - telemetry, - "input_pinning", - "completed", - dataset_sha256=input_artifact["sha256"], - ) - if telemetry is not None: - telemetry.set_sample({"mode": "full"}) - frs_release = load_uk_frs_release() - calibration_year = int(frs_release.calibration_year) - args._calibration_year = calibration_year - args._frs_vintage = str(frs_release.vintage) - source_year = _source_year( - args.source_year, time_period=str(frs_release.time_period) - ) - output_paths = _output_paths(out_dir, posture=posture, vintage=args._frs_vintage) - _validate_output_paths(output_paths, input_h5=input_h5, ladder_path=None) - _stage(telemetry, "target_compilation", "started") - inputs = _load_national_target_inputs(args) - _stage( - telemetry, - "target_compilation", - "completed", - compiled_target_count=len(inputs["band_edge_registry"].specs), - active_target_count=len(inputs["national_registry"].specs), - ) - doctrine, doctrine_overrides = uk_doctrine_with_overrides( - **_national_doctrine_overrides(args) - ) - if doctrine.target_weight_rule != args.target_weight_rule: - raise RuntimeError( - "national doctrine override did not bind the requested rule." - ) - args._doctrine_override_receipt = doctrine_overrides - out_dir.mkdir(parents=True, exist_ok=True) - # The frozen register is the scorer's input: the same artifact this run - # solved against, by content hash. - inputs["national_registry"].to_json(output_paths["national_registry"]) - # The full compiled register beside it: the band edges a pruned scoring - # surface must never redraw (#803), for the end-of-build evaluation and - # for a re-score by hand (--band-edge-registry-json). - inputs["band_edge_registry"].to_json(output_paths["contract_registry"]) - resolver = UKMeasureResolver( - simulation_source=input_h5, - scratch_dir=out_dir, - year=calibration_year, - frame=None, - ) - paths = UKCalibrationRunPaths( - input_h5=input_h5, - staging_h5=output_paths["dataset"], - diagnostics_json=output_paths["calibration_diagnostics"], - build_record_json=output_paths["build_record"], - terminal_gate_json=output_paths["terminal_gates"], - ) - evidence: dict[str, Any] = {} - - def publish_manifest() -> None: - # The seam has written its evidence; the manifest describes it and the - # staged bundle (every manifest-registered output) follows, as on the - # dense role. The staged copy predates the evidence blocks appended - # below; staged_manifest.json describes the remote side. - args._gate_report = _read_json(paths.terminal_gate_json) - manifest = _national_manifest( - args, - posture=posture, - build_id=build_id, - build_record=_read_json(paths.build_record_json), - build_record_path=paths.build_record_json, - gate_report=args._gate_report, - diagnostics=_read_json(paths.diagnostics_json), - inputs=inputs, - doctrine_overrides=doctrine_overrides, - output_paths=output_paths, - input_artifact=input_artifact, - source_year=source_year, - ) - _replace_manifest(output_paths["manifest"], manifest) - evidence["manifest"] = manifest - evidence["staged_dataset"] = _stage_dataset( - args, - manifest=manifest, - output_paths=output_paths, - run_id=build_id if telemetry is None else telemetry.run_id, - telemetry=telemetry, - ) - - def evaluate() -> None: - # After the bundle is staged (the evaluation never blocks staging), - # before the telemetry completes (the receipt rides it as an artifact). - evidence["evaluation"] = _evaluate_against_incumbent( - args, - incumbent=incumbent, - inputs=inputs, - output_paths=output_paths, - telemetry=telemetry, - calibration_year=calibration_year, - out_dir=out_dir, - ) - - def finalize_staging() -> None: - publish_manifest() - evaluate() - _finalize_staging_telemetry(args, telemetry) - - def event_callback(stage_id: str, status: str, details: Mapping[str, Any]) -> None: - _stage(telemetry, stage_id, status, **dict(details)) - - result = run_uk_calibration( - paths=paths, - build_id=build_id, - input_sha256=str(args.input_sha256), - ledger_artifact=inputs["artifact"], - register_registry=inputs["national_registry"], - band_edge_registry=inputs["band_edge_registry"], - calibration_year=calibration_year, - exclusion_receipt=inputs["measure_exclusions"], - doctrine=doctrine, - doctrine_overrides=doctrine_overrides, - measure_resolver=resolver, - source_pins={ - "input_h5": { - "sha256": str(args.input_sha256), - "size_bytes": int(input_artifact["bytes"]), - }, - "ledger_facts": _ledger_facts_pin(inputs["artifact"]), - }, - run_config_extra={ - "release_role": posture.role, - "calibration_year": calibration_year, - "allow_unpinned_feed": bool(args.allow_unpinned_feed), - "chronicle_feed_pin": inputs["chronicle_feed_pin"], - "rowwise_driver_parameters": _parameters(args, source_year=source_year), - }, - release_id=posture.release_id, - logbook_prev_row_digest=args.logbook_prev_row_digest, - progress_callback=( - None - if telemetry is None - else _thinned_epochs( - telemetry.calibration_progress, every=_staging_epoch_every(args) - ) - ), - event_callback=None if telemetry is None else event_callback, - staging_delivery=_staging_delivery(telemetry), - staging_finalizer=None if telemetry is None else finalize_staging, - staging_delivery_provider=( - None if telemetry is None else (lambda: telemetry.delivery_summary) - ), - ) - if telemetry is None: - publish_manifest() - evaluate() - manifest = evidence["manifest"] - # The seam rewrote its record with the delivery summary after the - # finalizer; the manifest binds the record as it now is. - manifest["outputs"]["build_record"] = _artifact_info(paths.build_record_json) - manifest["build_record"] = { - "path": str(paths.build_record_json), - "sha256": result.build_record_sha256, - } - manifest["staging_delivery"] = _staging_delivery(telemetry) - manifest["staged_dataset"] = evidence["staged_dataset"] - evaluation = evidence.get("evaluation", {"status": "not_requested"}) - manifest["evaluation"] = evaluation - if evaluation.get("status") == "completed": - manifest["outputs"]["score_receipt"] = evaluation["receipt"] - _replace_manifest(output_paths["manifest"], manifest) - if (out_dir / SHA256SUMS_FILENAME).is_file(): - # The uploaded copies list the files as uploaded; the local sums - # list the record, the receipt and the manifest as they now are, - # evidence included. - refresh_sha256sums_entry(out_dir, paths.build_record_json.name) - if evaluation.get("status") == "completed": - _list_sha256sums_entry(out_dir, output_paths["score_receipt"].name) - refresh_sha256sums_entry(out_dir, output_paths["manifest"].name) - print(_json_text(manifest)) - return 0 - - -def _national_fit_by_family( - diagnostics: Mapping[str, Any], registry: TargetRegistry -) -> list[dict[str, object]]: - families = {str(spec.name): str(spec.family or "") for spec in registry.specs} - rows = [] - for row in diagnostics.get("targets", []): - if not isinstance(row, Mapping): - continue - error = row.get("relative_error") - if error is None: - continue - # The diagnostics name a target by its register spec name and, on - # period-suffixed rows, by a materialized name; the family lives on - # the spec. - labels = [ - str(label) - for label in (row.get("target_name"), row.get("name")) - if label is not None - ] - family = next((families[label] for label in labels if label in families), "") - rows.append( - { - "target_name": labels[0] if labels else "", - "family": family, - "abs_relative_error": abs(float(error)), - } - ) - if not rows: - return [] - return uk_fit_by_family(pd.DataFrame(rows)) - - -def _national_manifest( - args: argparse.Namespace, - *, - posture: UKRowwisePosture, - build_id: str, - build_record: Mapping[str, Any], - build_record_path: Path, - gate_report: Mapping[str, Any], - diagnostics: Mapping[str, Any], - inputs: Mapping[str, Any], - doctrine_overrides: Mapping[str, Any], - output_paths: Mapping[str, Path], - input_artifact: Mapping[str, Any], - source_year: int, -) -> dict[str, Any]: - """The rowwise manifest of a national-role run, over the seam's evidence.""" - - calibration = build_record["calibration"] - solve = calibration["solve"] - statuses = _gate_statuses(gate_report) - abs_errors = np.asarray( - [ - abs(float(row["relative_error"])) - for row in diagnostics.get("targets", []) - if isinstance(row, Mapping) and row.get("relative_error") is not None - ], - dtype=np.float64, - ) - fit_rows = _national_fit_by_family(diagnostics, inputs["national_registry"]) - outputs = { - "dataset": _artifact_info(output_paths["dataset"]), - "calibration_diagnostics": _artifact_info( - output_paths["calibration_diagnostics"] - ), - "build_record": _artifact_info(build_record_path), - "terminal_gate_report": _artifact_info(output_paths["terminal_gates"]), - "national_target_registry": _artifact_info(output_paths["national_registry"]), - "national_contract_registry": _artifact_info(output_paths["contract_registry"]), - } - return { - "schema_version": 4, - "build_kind": "uk_national_calibrated_candidate", - "release_role": posture.role, - "release_id": posture.release_id, - "build_id": build_id, - "candidate_scope": "national", - "created_at": datetime.now(UTC).isoformat(), - "git_commit": _git_commit(), - "git_dirty": _git_dirty(), - "parameters": _parameters(args, source_year=source_year), - "inputs": {"dataset": dict(input_artifact)}, - "identity": { - "spine": { - **dict(input_artifact), - "spine_provenance": dict(build_record["spine_provenance"]), - }, - "targets": {"chronicle": inputs["artifact"].provenance()}, - "code": {"git_commit": _git_commit(), "git_dirty": _git_dirty()}, - "runtime": runtime_provenance(), - "sampling": {"mode": "full"}, - "survey_year": source_year, - "calibration_year": int(inputs["calibration_year"]), - }, - "sampling": {"mode": "full"}, - "outputs": outputs, - "weights": dict(calibration["weights"]), - "solve": { - "n_targets": int(solve["n_targets"]), - "n_targets_by_kind": { - "national": int(solve["n_targets"]), - "local": 0, - "ladder": 0, - }, - "n_households": int(solve["n_households"]), - "pool_households": int(solve["n_households"]), - "initial_loss": float(solve["initial_loss"]), - "final_loss": float(solve["final_loss"]), - "n_nonzero": int(solve["n_nonzero"]), - "max_abs_relative_error": ( - float(abs_errors.max()) if abs_errors.size else None - ), - "median_abs_relative_error": ( - float(np.median(abs_errors)) if abs_errors.size else None - ), - "effective_sample_size": calibration.get("effective_sample_size"), - "max_weight_ratio": calibration.get("max_weight_ratio"), - "target_weight_rule": str(args.target_weight_rule), - "target_weight_rule_override": dict( - doctrine_overrides.get("target_weight_rule", {}) - ), - "doctrine_overrides": dict(doctrine_overrides), - "measure_resolution": calibration.get("measure_resolution"), - }, - "fit": { - "national_by_family": fit_rows, - "weakest_families": sorted( - fit_rows, - key=lambda row: ( - -float(row["worst_abs_relative_error"]), - row["family"], - ), - )[:10], - }, - "gate": { - "scope": list(posture.gate_scope), - "posture": posture.gate_posture, - "release_id": gate_report.get("release_id"), - "statuses": statuses, - }, - "failing_gate_ids": sorted( - gate_id for gate_id, status in statuses.items() if status != "passed" - ), - "releasable": False, - "release_posture": { - "release_candidate": False, - "shippable_by": "tools/certify_uk_release_cut.py", - "calibration_seam_gates_passed": bool(statuses) - and all(status == "passed" for status in statuses.values()), - }, - "measure_exclusions": dict(inputs["measure_exclusions"]), - "build_record": { - "path": str(build_record_path), - "sha256": outputs["build_record"]["sha256"], - }, - } - - -def _size_checkpoint_state(solve: UKRowwiseDoctrineSolve) -> str | None: - if solve.size_receipt is None or not solve.size_receipt.get("checkpoint"): - return None - checkpoint = solve.size_receipt["checkpoint"] - if "written" in checkpoint: - return "written" - if "resumed_from" in checkpoint: - return "resumed" - return None - - -def _clone_with_ladder_binding( - dataset: Any, - ladder: UkOaLadder, - *, - n_clones: int, - seed: int, - source_year: int, - expected_constituency_vintage: str | None, - source_lineage_modulus: int | None, -) -> _LadderAssignment: - clone = clone_uk_dataset_with_ladder_geography( - dataset, - ladder, - n_clones=n_clones, - seed=seed, - source_year=source_year, - expected_constituency_vintage=expected_constituency_vintage, - source_lineage_modulus=source_lineage_modulus, - ) - return _LadderAssignment(clone, ladder) - - -def _verify_requested_pin( - label: str, - artifact: Mapping[str, Any], - *, - requested: str | None, -) -> None: - if requested is None: - artifact["pin_verified"] = False - return - measured = str(artifact["sha256"]) - if measured != requested: - raise SystemExit( - f"error: {label} sha mismatch: measured {measured}, pinned {requested}" - ) - artifact["pin_verified"] = True - - -def _load_joint_target_inputs(args: argparse.Namespace) -> dict[str, Any]: - artifact = load_ledger_consumer_artifact( - args.ledger_facts, - expected_facts_sha256=args.ledger_facts_sha256, - expected_manifest_sha256=args.ledger_manifest_sha256, - ) - calibration_year = int(load_uk_frs_release().calibration_year) - national_compilation = compile_uk_target_registry( - artifact.facts, target_period=calibration_year - ) - if national_compilation.unsupported: - raise SystemExit( - f"{len(national_compilation.unsupported)} national target references " - "failed to compile" - ) - local_compilation = compile_uk_local_target_registry( - artifact.facts, - target_period=calibration_year, - crosswalk=load_uk_local_area_crosswalk(), - ) - if local_compilation.unsupported: - raise SystemExit( - f"{len(local_compilation.unsupported)} local target references " - "failed to compile" - ) - exclusions = load_uk_calibration_measure_exclusions(args.measure_exclusions) - national_registry, exclusion_receipt = apply_uk_calibration_measure_exclusions( - national_compilation.registry, exclusions - ) - national_specs_by_name = { - spec.name: spec for spec in national_compilation.registry.specs - } - reviewed_unbound_higher_targets = { - str( - national_specs_by_name[name].metadata.get( - "contract_target_id", national_specs_by_name[name].name - ) - ): record - for name, record in exclusion_receipt.items() - } - if args.register_json is not None: - try: - frozen = TargetRegistry.from_json(args.register_json) - except ValueError as error: - raise SystemExit( - f"error: frozen scoring register is unusable: {error}" - ) from error - if frozen.version != national_registry.version: - raise SystemExit( - "re-derived register differs from the frozen scoring register: " - f"{national_registry.version} vs {frozen.version}" - ) - return { - "artifact": artifact, - "calibration_year": calibration_year, - "national_registry": national_registry, - "band_edge_registry": national_compilation.registry, - "local_registry": local_compilation.registry, - "measure_exclusions": exclusion_receipt, - "reviewed_unbound_higher_targets": reviewed_unbound_higher_targets, - } - - -def _national_contract_target_ids(registry: TargetRegistry) -> tuple[str, ...]: - return tuple( - sorted( - { - str(spec.metadata.get("contract_target_id", spec.name)) - for spec in registry.specs - } - ) - ) - - -def _joint_surface_registry( - local_registry: TargetRegistry, - national_registry: TargetRegistry, -) -> TargetRegistry: - """Put national controls beside local cells for cross-grain reconciliation.""" - - return TargetRegistry( - [*local_registry.specs, *national_registry.specs], - country="uk", - ) - - -def _build_joint_problem( - assignment: _LadderAssignment, - *, - local_registry: TargetRegistry, - national_registry: TargetRegistry, - local_metrics: Mapping[str, pd.DataFrame], - period: int, - sample_fraction: float, - reviewed_unbound_higher_targets: Mapping[str, Mapping[str, object]], - census_household_uprating: Mapping[str, Any] | None = None, -) -> tuple[ - pd.DataFrame, - UKRowwiseLocalMatrix, - dict[str, Any], - tuple[str, ...], - dict[str, Any], -]: - household = assignment.result.frame.table("household").reset_index(drop=True) - household_index = pd.Index(household["household_id"], name="household_id") - metrics = { - grain: frame.set_axis(household_index, axis="index") - for grain, frame in local_metrics.items() - } - assigned = { - "constituency": pd.Series( - household["constituency_code"].astype(str).to_numpy(), - index=household_index, - ), - "la": pd.Series( - household["local_authority_code"].astype(str).to_numpy(), - index=household_index, - ), - } - national_ids = _national_contract_target_ids(national_registry) - surface, cross_grain = uk_local_target_surface( - _joint_surface_registry(local_registry, national_registry), - bound_national_target_ids=national_ids, - period=period, - reviewed_unbound_higher_targets=reviewed_unbound_higher_targets, - census_household_uprating=census_household_uprating, - area_region_codes=uk_area_region_codes(assignment.ladder), - ) - covered = { - grain: set(values.astype(str).tolist()) for grain, values in assigned.items() - } - covered_mask = pd.Series( - [ - str(row.area_code) in covered[str(row.area_type)] - for row in surface.itertuples(index=False) - ], - index=surface.index, - dtype=bool, - ) - dropped = surface.loc[~covered_mask] - if sample_fraction < 1.0: - surface = surface.loc[covered_mask].reset_index(drop=True) - # Below f100 a covered area can still carry a nonzero cell with no metric - # support in the sample (no self-employed household among three drawn - # rows). The builder refuses such a cell at every rung; at development - # rungs the cell is dropped here and receipted instead. f100 stays strict. - unreachable = surface.iloc[0:0] - if sample_fraction < 1.0 and len(surface): - nonzero_by_grain = { - grain: (metrics[grain] != 0).groupby(assigned[grain]).sum() - for grain in metrics - } - unreachable_mask = pd.Series( - [ - float(row.value) != 0.0 - and str(row.metric) in nonzero_by_grain[str(row.area_type)].columns - and str(row.area_code) in nonzero_by_grain[str(row.area_type)].index - and int( - nonzero_by_grain[str(row.area_type)].loc[ - str(row.area_code), str(row.metric) - ] - ) - == 0 - for row in surface.itertuples(index=False) - ], - index=surface.index, - dtype=bool, - ) - unreachable = surface.loc[unreachable_mask] - surface = surface.loc[~unreachable_mask].reset_index(drop=True) - rung_surface = { - "dropped_unreachable_cells": int(len(unreachable)), - "dropped_unreachable_by_grain": { - str(key): int(value) - for key, value in unreachable.groupby("area_type").size().items() - }, - "dropped_unreachable_by_family": { - str(key): int(value) - for key, value in unreachable.groupby("family").size().items() - }, - "fraction": float(sample_fraction), - "dropped_cells": int(len(dropped) if sample_fraction < 1.0 else 0), - "dropped_by_grain": ( - { - str(key): int(value) - for key, value in dropped.groupby("area_type").size().items() - } - if sample_fraction < 1.0 - else {} - ), - "dropped_by_family": ( - { - str(key): int(value) - for key, value in dropped.groupby("family").size().items() - } - if sample_fraction < 1.0 - else {} - ), - } - rosters = { - "constituency": tuple(map(str, np.unique(assignment.ladder.constituency_code))), - "la": tuple(map(str, np.unique(assignment.ladder.local_authority_code))), - } - problem = build_uk_rowwise_local_surface_matrix( - metrics, - assigned, - surface, - area_codes_by_grain=rosters, - require_every_assigned_area_covered=(sample_fraction == 1.0), - ) - local_bound = tuple( - sorted( - { - f"{row.family}/{row.area_type}" - for row in surface[["family", "area_type"]] - .drop_duplicates() - .itertuples(index=False) - } - ) - ) - national_bound = tuple( - f"national/{family}" - for family in sorted({spec.family for spec in national_registry.specs}) - ) - return ( - household, - problem, - cross_grain, - (*local_bound, *national_bound), - rung_surface, - ) - - -def _joint_dry_run_plan( - args: argparse.Namespace, - *, - clone: UKLadderRowwiseDatasetResult, - sampled_spine: Any, - ladder: UkOaLadder, - joint_inputs: Mapping[str, Any], - source_year: int, - input_artifact: Mapping[str, Any], - ladder_artifact: Mapping[str, Any], - target_provenance: Mapping[str, Any], -) -> dict[str, Any]: - national_registry = joint_inputs["national_registry"] - surface, cross_grain = uk_local_target_surface( - _joint_surface_registry( - joint_inputs["local_registry"], - national_registry, - ), - bound_national_target_ids=_national_contract_target_ids(national_registry), - period=joint_inputs["calibration_year"], - reviewed_unbound_higher_targets=joint_inputs["reviewed_unbound_higher_targets"], - area_region_codes=uk_area_region_codes(ladder), - census_household_uprating=joint_inputs.get("census_household_uprating"), - ) - household = clone.frame.table("household") - covered = { - "constituency": set(household["constituency_code"].astype(str)), - "la": set(household["local_authority_code"].astype(str)), - } - covered_mask = pd.Series( - [ - str(row.area_code) in covered[str(row.area_type)] - for row in surface.itertuples(index=False) - ], - index=surface.index, - dtype=bool, - ) - dropped = surface.loc[~covered_mask] - active_surface = ( - surface.loc[covered_mask].reset_index(drop=True) - if args.sample_fraction < 1.0 - else surface - ) - household_count = len(clone.frame.table("household")) - clone_support: dict[str, object] = {} - for clone_count in args.candidate_clone_counts or (args.n_clones,): - candidate = ( - clone - if clone_count == args.n_clones - else _clone_with_ladder_binding( - sampled_spine, - ladder, - n_clones=clone_count, - seed=args.seed, - source_year=source_year, - expected_constituency_vintage=args.expected_constituency_vintage, - source_lineage_modulus=args.source_lineage_modulus, - ).result - ) - # The typed frame weights are the authority; the persisted - # household_weight column is an export artefact the loaded spine - # does not carry, so attach them the way the real support path does. - candidate_household = candidate.frame.table("household").copy() - candidate_household["household_weight"] = np.asarray( - candidate.frame.weights_for("household").values, dtype=np.float64 - ) - summaries = uk_ladder_area_support_summary(candidate_household, ladder) - clone_support[str(clone_count)] = { - grain: { - "minimum_rows": int(rows["nonzero_households"].min()), - "minimum_effective_sample_size": float( - rows["effective_sample_size"].min() - ), - "minimum_distinct_sources": int( - rows["nonzero_source_households"].min() - ), - } - for grain, rows in summaries.items() - } - return { - "schema_version": 3, - "build_kind": "uk_rowwise_calibrated_candidate_plan", - "release_role": _posture_of(args).role, - "dry_run": True, - "survey_year": source_year, - "calibration_year": joint_inputs["calibration_year"], - "identity": { - "spine": dict(input_artifact), - "ladder": dict(ladder_artifact), - "targets": { - "chronicle": joint_inputs["artifact"].provenance(), - "paired_ladder_sha256": str(ladder_artifact["sha256"]), - }, - }, - "sampling": dict(args._sampling_receipt), - "rung_surface": { - "rung": UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], - "fraction": args.sample_fraction, - "dropped_cells": int(len(dropped) if args.sample_fraction < 1.0 else 0), - "dropped_by_grain": ( - { - str(key): int(value) - for key, value in dropped.groupby("area_type").size().items() - } - if args.sample_fraction < 1.0 - else {} - ), - "dropped_by_family": ( - { - str(key): int(value) - for key, value in dropped.groupby("family").size().items() - } - if args.sample_fraction < 1.0 - else {} - ), - "unreachable_check": "deferred_to_build", - }, - "vintages": _local_vintage_census(joint_inputs["local_registry"]), - "cross_grain": cross_grain, - "matrix": { - "rows": int(len(active_surface) + len(national_registry.specs)), - "columns": household_count, - "local_rows": len(active_surface), - "national_rows": len(national_registry.specs), - }, - "candidate_clone_counts": list(args.candidate_clone_counts or (args.n_clones,)), - "candidate_clone_support": clone_support, - "parameters": _parameters(args, source_year=source_year), - "releasable": False, - "engine": "not_run", - "ladder_assignment_provenance": dict(target_provenance), - "household_dispersion": dict(joint_inputs["household_dispersion"]), - } - - -def _build_bound_problem( - assignment: _LadderAssignment, - *, - local_registry: TargetRegistry, - period: int | str, - census_household_uprating: Mapping[str, Any] | None = None, -) -> tuple[pd.DataFrame, UKRowwiseLocalMatrix, dict[str, Any]]: - """Bind Chronicle census household targets at constituency grain only. - - The constituency cells go through the same ``uk_local_target_surface`` - pass as the joint scope, so the per-grain A15 factor applies here too and - its receipt reaches the manifest; the scope has no national controls. - """ - clone = assignment.result - household = clone.frame.table("household").reset_index(drop=True) - household_index = pd.Index( - household["household_id"], - name="household_id", - ) - metrics = pd.DataFrame( - {"households": np.ones(len(household), dtype=np.float64)}, - index=household_index, - ) - assigned = pd.Series( - household["constituency_code"].astype(str).to_numpy(), - index=household_index, - name="constituency_code", - ) - household_specs = sorted( - ( - spec - for spec in local_registry.specs - if spec.name.startswith("ons.census.households@") - and _spec_geography(spec)[0] == "constituency" - ), - key=lambda spec: _spec_geography(spec)[1], - ) - if not household_specs: - raise ValueError( - "households-only binding requires Chronicle constituency household specs." - ) - surface, cross_grain = uk_local_target_surface( - TargetRegistry(household_specs, country="uk"), - bound_national_target_ids=BOUND_NATIONAL_TARGETS, - period=period, - census_household_uprating=census_household_uprating, - area_region_codes=uk_area_region_codes(assignment.ladder), - ) - surface = surface.sort_values("area_code", kind="mergesort").reset_index(drop=True) - targets = pd.DataFrame( - { - "code": surface["area_code"].astype(str).to_numpy(), - "households": surface["value"].to_numpy(dtype=np.float64), - } - ) - problem = build_uk_rowwise_local_matrix( - metrics, - assigned, - targets, - area_type="constituency", - code_column="code", - ) - target_identity = surface.set_index(surface["area_code"].astype(str))[ - ["target_name", "contract_target_id", "hierarchy"] - ] - joined_identity = problem.target_frame[["area_code"]].join( - target_identity, - on="area_code", - validate="many_to_one", - ) - if ( - joined_identity[["target_name", "contract_target_id", "hierarchy"]] - .isna() - .any() - .any() - ): - raise ValueError( - "households-only target identities do not cover every matrix area code." - ) - problem.target_frame["target_name"] = joined_identity["target_name"].to_numpy() - problem.target_frame["contract_target_id"] = joined_identity[ - "contract_target_id" - ].to_numpy() - problem.target_frame["hierarchy"] = joined_identity["hierarchy"].to_numpy() - return ( - household, - problem, - {"bound_national_targets": list(BOUND_NATIONAL_TARGETS), **cross_grain}, - ) - - -def _candidate_area_support( - household: pd.DataFrame, - ladder: UkOaLadder, - *, - weights: np.ndarray, -) -> pd.DataFrame: - weighted_household = household.copy() - weighted_household["household_weight"] = np.asarray(weights, dtype=np.float64) - summaries = uk_ladder_area_support_summary(weighted_household, ladder) - return pd.concat( - ( - summaries["constituency"].assign(geography_level="constituency"), - summaries["la"].assign(geography_level="local_authority"), - ), - ignore_index=True, - )[ - [ - "geography_level", - "area_code", - "assigned_households", - "nonzero_households", - "nonzero_source_households", - "weight_sum", - "max_weight", - "effective_sample_size", - ] - ] - - -def _local_gate_diagnostics(diagnostics: pd.DataFrame) -> pd.DataFrame: - result = diagnostics.copy() - required = {"family", "area_type", "area_code", "metric"} - missing = sorted(required - set(result.columns)) - if missing: - raise ValueError(f"local diagnostics are missing binding columns {missing}.") - if result[list(required)].isna().any().any(): - raise ValueError("local diagnostics contain unclassified binding rows.") - return result - - -def _local_diagnostics_registry( - solve: UKRowwiseDoctrineSolve, - problem: UKRowwiseLocalMatrix, - *, - national_registry: TargetRegistry | None = None, -) -> tuple[TargetRegistry, dict[str, str]]: - targets = tuple(solve.calibration_result.problem.targets) - expected = len(problem.target_frame) + ( - 0 if national_registry is None else len(national_registry.specs) - ) - if len(targets) != expected: - raise RuntimeError( - "candidate diagnostics registry is not aligned to the solve." - ) - specs: list[TargetSpec] = [] - geography: dict[str, str] = {} - for target, row in zip( - targets[: len(problem.target_frame)], - problem.target_frame.itertuples(index=False), - strict=True, - ): - metric = str(row.metric) - family = ( - str(row.family) - if "family" in problem.target_frame.columns - else local_target_census.family_for_metric(metric) - ) - spec = TargetSpec( - name=str(target.name), - entity=str(target.entity), - value=float(target.value), - measure=f"rowwise_metric:{metric}", - filter=f"rowwise_area:{row.area_code}", - period=target.period, - source=str(target.source), - family=family, - metadata={key: str(value) for key, value in target.metadata.items()}, - hierarchy=target.hierarchy, - ) - specs.append(spec) - geography[spec.to_target().row_name] = str(row.area_type) - if national_registry is not None: - specs.extend(national_registry.specs) - for spec in national_registry.specs: - level, _ = _spec_geography(spec) - geography[spec.to_target().row_name] = level - return TargetRegistry(specs, country="uk"), geography - - -#: Measure columns whose policyengine-uk formulas normalise by a population -#: total (a fixed national aggregate allocated by each household's share of -#: total weighted corporate wealth, or a term scaled by a weight sum). Under -#: ``--engine-blocks K`` the engine sees one clone block at a time, so each -#: block reproduces the whole aggregate and the column comes out K× (receipt -#: R15 in ``experiments/762-uk-rowwise-candidate-receipts.md``: corporate -#: land value ×15.000 at K=15). Evidence from a per-block run must not -#: adjudicate these rows; the release posture is single-block. -UK_BLOCK_SENSITIVE_MEASURE_COLUMNS = ( - "ons/corporate_land_value", - "ons/land_value", - "slc/student_loan_repayment/england", -) - - -def _run_local_gate_battery( - *, - frame: Any, - support: pd.DataFrame, - diagnostics: pd.DataFrame, - report_path: Path, - release_id: str, - evaluated_on: date, - enforce_only: tuple[str, ...] | None = None, - release_candidate: bool = False, -) -> tuple[dict[str, object], GateResult]: - manifest = uk_scoped_gate_manifest( - UK_LOCAL_GATE_SCOPE, - phases=("terminal",), - policy_suffix=_LOCAL_GATE_POLICY_SUFFIX, - ) - battery = GateBatteryRun( - manifest, - release_id=release_id, - report_path=report_path, - release_candidate=release_candidate, - registry=UK_GATE_REGISTRY, - ) - phase = battery.run_phase( - "terminal", - EvidenceContext( - frame=frame, - artifacts={ - "uk_area_support_summary": support, - "local_target_diagnostics": diagnostics, - # The run's own clock, not the wall clock: the same artifact - # must reproduce the same exclusion verdicts. - "exclusions_evaluated_on": evaluated_on, - }, - ), - ) - if enforce_only is None: - try: - battery.enforce("terminal", mode=BlockingMode.BLOCKS_ARTIFACT) - except GateBatteryBlockedError: - payload = battery.report_payload() - finalize_uk_scoped_gate_report( - payload, - posture="local_candidate", - scope_exclusions=uk_local_gate_scope_exclusions(), - aggregate_admin_measurement=None, - ) - atomic_write_json(report_path, payload) - raise - else: - unknown = sorted(set(enforce_only) - set(UK_LOCAL_GATE_SCOPE)) - if unknown: - raise ValueError(f"enforce_only names unknown local gates: {unknown}.") - selected_blocking = [ - outcome - for outcome in phase.blocking_outcomes(release_candidate=release_candidate) - if outcome.entry.id in enforce_only - ] - if selected_blocking: - payload = battery.report_payload() - finalize_uk_scoped_gate_report( - payload, - posture="local_candidate", - scope_exclusions=uk_local_gate_scope_exclusions(), - aggregate_admin_measurement=None, - ) - atomic_write_json(report_path, payload) - failures = [ - failure - for outcome in selected_blocking - if outcome.result is not None - for failure in outcome.result.failures - ] - raise GateBatteryBlockedError("terminal", failures, report_path) - payload = battery.report_payload() - finalize_uk_scoped_gate_report( - payload, - posture="local_candidate", - scope_exclusions=uk_local_gate_scope_exclusions(), - aggregate_admin_measurement=None, - ) - atomic_write_json(report_path, payload) - ladder = next( - outcome - for outcome in phase.outcomes - if outcome.entry.id == "uk_local_geography_ladder_post_calibration" - ) - if ladder.result is None or not ladder.result.passed: - raise RuntimeError( - "a non-passing local geography-ladder result escaped battery enforcement." - ) - return payload, ladder.result - - -def _apply_gate_verdicts( - state: AttemptState, - report: Mapping[str, object], - report_path: Path, -) -> None: - gates = report.get("gates") - if not isinstance(gates, Mapping) or set(gates) != set(UK_LOCAL_GATE_SCOPE): - raise RuntimeError("local gate report does not cover the declared scope.") - receipt = local_artifact_reference(report_path, repository_hint=_REPOSITORY) - state.gate_verdicts = { - gate_id: { - "verdict": str(payload["status"]), - "receipt": f"{receipt}#/gates/{gate_id}", - } - for gate_id, payload in gates.items() - if isinstance(payload, Mapping) - } - if set(state.gate_verdicts) != set(UK_LOCAL_GATE_SCOPE): - raise RuntimeError("local gate verdicts are malformed.") - - -def _dry_run_plan( - args: argparse.Namespace, - *, - clone: UKLadderRowwiseDatasetResult, - problem: UKRowwiseLocalMatrix, - source_year: int, - input_artifact: Mapping[str, Any], - ladder_artifact: Mapping[str, Any], - target_provenance: Mapping[str, Any], - binding_adjudications: Mapping[str, Any], - cross_grain: Mapping[str, Any], -) -> dict[str, Any]: - return { - "schema_version": 3, - "build_kind": "uk_rowwise_calibrated_candidate_plan", - "release_role": _posture_of(args).role, - "dry_run": True, - "candidate_scope": "adjudicated_partial", - "bound_target_families": list(args._bound_families), - "binding_adjudications": dict(binding_adjudications), - "cross_grain": dict(cross_grain), - "ladder_assignment_provenance": dict(target_provenance), - "household_dispersion": dict( - args._joint_inputs_receipt["household_dispersion"] - ), - "identity": { - "targets": { - "chronicle": args._joint_inputs_receipt["artifact"].provenance(), - "paired_ladder_sha256": str(ladder_artifact["sha256"]), - } - }, - "inputs": { - "dataset": dict(input_artifact), - "ladder": dict(ladder_artifact), - }, - "sampling": dict(args._sampling_receipt), - "survey_year": source_year, - "calibration_year": ( - args._joint_inputs_receipt["calibration_year"] - if args._joint_inputs_receipt is not None - else source_year - ), - "rung_surface": { - "rung": UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], - "fraction": args.sample_fraction, - "unreachable_check": "completed", - }, - "releasable": args.sample_fraction == 1.0 - and args.engine_blocks == 1 - and args.dataset_households is None, - "parameters": _parameters(args, source_year=source_year), - "shapes": { - "person": list(clone.frame.table("person").shape), - "benunit": list(clone.frame.table("benunit").shape), - "household": list(clone.frame.table("household").shape), - "local_matrix": list(problem.matrix.shape), - }, - "target_count": int(len(problem.targets)), - "gate": _gate_payload(clone.gate, phase="post_clone"), - } - - -def _write_output_bundle( - args: argparse.Namespace, - *, - candidate: UKLadderRowwiseDatasetResult, - clone: UKLadderRowwiseDatasetResult, - problem: UKRowwiseLocalMatrix, - solve: UKRowwiseDoctrineSolve, - local_diagnostics: pd.DataFrame, - target_registry: TargetRegistry, - target_geography_levels: Mapping[str, str], - rotated_holdout: Mapping[str, object], - support: pd.DataFrame, - calibration_record: MassChangeRecord, - source_year: int, - output_paths: Mapping[str, Path], - input_artifact: Mapping[str, Any], - ladder_artifact: Mapping[str, Any], - target_provenance: Mapping[str, Any], - cross_grain: Mapping[str, Any], -) -> dict[str, Any]: - """Stage the complete bundle, then publish atomically per file.""" - - out_dir = output_paths["manifest"].parent - out_dir.parent.mkdir(parents=True, exist_ok=True) - staging_dir = Path( - tempfile.mkdtemp( - prefix=f".{out_dir.name}.rowwise-candidate.", - dir=out_dir.parent, - ) - ) - try: - staged = {key: staging_dir / path.name for key, path in output_paths.items()} - print( - f"staging candidate for {output_paths['dataset']}...", - file=sys.stderr, - flush=True, - ) - write_uk_rowwise_dataset(candidate, staged["dataset"]) - solve.diagnostics.to_csv(staged["diagnostics"], index=False) - if solve.dense_reference is not None: - _dense_reference_diagnostics_frame(solve).to_csv( - staged["dense_reference"], index=False - ) - _dataset_size_selection_frame(solve, problem=problem, clone=clone).to_csv( - staged["selection"], index=False - ) - support = support.copy() - support["support_below_floor"] = ( - (support["assigned_households"] < 50) - | (support["effective_sample_size"] < 50.0) - | (support["nonzero_source_households"] < 50) - ) - support.to_csv(staged["support"], index=False) - staged["past_cap"].write_text(_json_text(dict(solve.past_cap_census or {}))) - local_registry = _local_output_registry( - solve, - problem, - period=( - args._joint_inputs_receipt["calibration_year"] - if args._joint_inputs_receipt is not None - else source_year - ), - ) - local_registry.to_json(staged["local_registry"]) - diagnostics_outcome = write_uk_calibration_diagnostics( - solve.calibration_result, - staged["calibration_diagnostics"], - solve.frame, - target_geography_levels=target_geography_levels, - target_registry=target_registry, - local_area_support=support, - rotated_holdout=rotated_holdout, - build={ - "build_kind": "uk_rowwise_calibrated_candidate", - "candidate_scope": "adjudicated_partial", - }, - ) - if isinstance(diagnostics_outcome, DiagnosticsWriteFailure): - raise RuntimeError( - "UK candidate assembly requires calibration diagnostics, but their " - f"canonical write failed [{diagnostics_outcome.error_code}]: " - f"{diagnostics_outcome.message}" - ) - calibration_diagnostics = json.loads( - diagnostics_outcome.path.read_text(encoding="utf-8") - ) - args._weakest_areas_by_fit = calibration_diagnostics["uk_diagnostics"][ - "weakest_areas_by_fit" - ] - - outputs = { - "dataset": _artifact_info( - staged["dataset"], - reported_path=output_paths["dataset"], - ), - "solve_diagnostics": _artifact_info( - staged["diagnostics"], - reported_path=output_paths["diagnostics"], - ), - "area_support_summary": _artifact_info( - staged["support"], - reported_path=output_paths["support"], - ), - "past_cap_census": _artifact_info( - staged["past_cap"], - reported_path=output_paths["past_cap"], - ), - "calibration_diagnostics": _artifact_info( - staged["calibration_diagnostics"], - reported_path=output_paths["calibration_diagnostics"], - ), - "local_gate_report": _artifact_info(output_paths["local_gates"]), - "local_target_registry": _artifact_info( - staged["local_registry"], - reported_path=output_paths["local_registry"], - ), - } - if solve.dense_reference is not None: - outputs["dense_reference_diagnostics"] = _artifact_info( - staged["dense_reference"], - reported_path=output_paths["dense_reference"], - ) - outputs["dataset_size_selection"] = _artifact_info( - staged["selection"], - reported_path=output_paths["selection"], - ) - manifest = _manifest( - args, - candidate=candidate, - clone=clone, - problem=problem, - solve=solve, - support=support, - calibration_record=calibration_record, - source_year=source_year, - input_artifact=input_artifact, - ladder_artifact=ladder_artifact, - target_provenance=target_provenance, - cross_grain=cross_grain, - calibration_diagnostics=calibration_diagnostics, - outputs=outputs, - ) - staged["manifest"].write_text(_json_text(manifest)) - _publish_staged_files(staged, output_paths) - return manifest - finally: - shutil.rmtree(staging_dir) - - -def _manifest( - args: argparse.Namespace, - *, - candidate: UKLadderRowwiseDatasetResult, - clone: UKLadderRowwiseDatasetResult, - problem: UKRowwiseLocalMatrix, - solve: UKRowwiseDoctrineSolve, - support: pd.DataFrame, - calibration_record: MassChangeRecord, - source_year: int, - input_artifact: Mapping[str, Any], - ladder_artifact: Mapping[str, Any], - target_provenance: Mapping[str, Any], - cross_grain: Mapping[str, Any], - calibration_diagnostics: Mapping[str, Any], - outputs: Mapping[str, Any], -) -> dict[str, Any]: - abs_errors = solve.diagnostics["abs_relative_error"].to_numpy(dtype=np.float64) - old_total = float(calibration_record.old_total) - new_total = float(calibration_record.new_total) - past_cap = dict(solve.past_cap_census or {}) - gate_rows = args._gate_report.get("gates", {}) - # ``releasable`` follows the battery's own doctrine: release-blocking - # entries decide, diagnostic entries (target_fit, weight_ratio, ...) are - # reported but never veto. ``failing_gate_ids`` still lists every - # non-passing entry of either criticality. - release_gate_rows = { - gate_id: payload - for gate_id, payload in gate_rows.items() - if isinstance(payload, Mapping) and _is_release_blocking(payload) - } - all_gates_passed = bool(release_gate_rows) and all( - payload.get("status") == "passed" for payload in release_gate_rows.values() - ) - releasable, release_posture = _release_verdict( - sample_fraction=args.sample_fraction, - engine_blocks=args.engine_blocks, - release_blocking_gates_passed=all_gates_passed, - ) - area_gate = gate_rows.get("uk_local_area_support", {}) - area_exclusion_details = ( - area_gate.get("details", {}) if isinstance(area_gate, Mapping) else {} - ) - ladder_rows = int( - problem.target_frame["target_name"] - .astype(str) - .str.startswith("ons.census.households@") - .sum() - ) - local_rows = int(len(problem.target_frame) - ladder_rows) - sample_stage = ( - [] - if args.sample_fraction == 1.0 - else [ - { - "stage": "sample", - "kind": uk_household_weight_kind(clone.frame).value, - } - ] - ) - posture = _posture_of(args) - return { - "schema_version": 4, - "build_kind": "uk_rowwise_calibrated_candidate", - "release_role": posture.role, - "release_id": posture.release_id, - "candidate_scope": "adjudicated_partial", - "created_at": datetime.now(UTC).isoformat(), - "git_commit": _git_commit(), - "git_dirty": _git_dirty(), - "bound_target_families": list(args._bound_families), - "binding_adjudications": dict(solve.binding_adjudications), - "cross_grain": dict(cross_grain), - "ladder_assignment_provenance": dict(target_provenance), - "household_dispersion": dict( - args._joint_inputs_receipt["household_dispersion"] - ), - "parameters": _parameters(args, source_year=source_year), - "inputs": { - "dataset": dict(input_artifact), - "ladder": dict(ladder_artifact), - }, - "identity": { - "spine": { - **dict(input_artifact), - "spine_provenance": dict(args._spine_provenance), - }, - "ladder": { - **dict(ladder_artifact), - "layer_vintages": dict(target_provenance), - "matches_local_area_crosswalk_pin": True, - }, - "targets": { - "chronicle": args._joint_inputs_receipt["artifact"].provenance(), - "paired_ladder_sha256": str(ladder_artifact["sha256"]), - }, - "code": {"git_commit": _git_commit(), "git_dirty": _git_dirty()}, - "runtime": runtime_provenance(), - "sampling": dict(args._sampling_receipt), - "survey_year": source_year, - "calibration_year": ( - args._joint_inputs_receipt["calibration_year"] - if args._joint_inputs_receipt is not None - else source_year - ), - }, - "sampling": dict(args._sampling_receipt), - "rung_surface": { - **dict(args._rung_surface), - "rung": UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], - "fraction": args.sample_fraction, - "unreachable_check": "completed", - }, - "outputs": dict(outputs), - "geography": { - "constituencies_assigned": int( - support.loc[ - support["geography_level"] == "constituency", - "area_code", - ].nunique() - ), - "local_authorities_assigned": int( - support.loc[ - support["geography_level"] == "local_authority", - "area_code", - ].nunique() - ), - "missing_geography_rows": 0, - "ladder_gate": _gate_payload(candidate.gate, phase="post_calibration"), - }, - "gate": _gate_payload(candidate.gate, phase="post_calibration"), - "weights": { - "household_weight_kind": uk_household_weight_kind(candidate.frame).value, - "household_weight_kind_chain": [ - { - "stage": "staging", - "kind": uk_household_weight_kind(clone.frame).value, - }, - *sample_stage, - { - "stage": "ladder_clone", - "kind": uk_household_weight_kind(clone.frame).value, - }, - { - "stage": "rowwise_calibration", - "kind": uk_household_weight_kind(candidate.frame).value, - }, - ], - "mass_log_records_before_calibration": len(clone.frame.mass_log), - "mass_log_records": len(candidate.frame.mass_log), - "calibration_mass_change": { - "entity": str(calibration_record.entity), - "old_total": old_total, - "new_total": new_total, - "relative_shift": (new_total - old_total) / old_total, - "declared_factor": calibration_record.declared_factor, - "reason": str(calibration_record.reason), - }, - "abs_delta": abs(new_total - old_total), - "declared_stretch_bound": float(posture.doctrine.max_weight_ratio), - "stretch_reference": "pool_design" - if solve.size_receipt is None - else "normalized_horvitz_thompson_w_over_q", - # Against the frame the refit started from (the pool design on a - # dense run, the Horvitz-Thompson baseline on a size run)... - "realized_max_weight_ratio_vs_stretch_reference": float( - np.max( - np.divide( - np.asarray(solve.weights, dtype=np.float64), - np.asarray(solve.initial_weights), - ) - ) - ), - # ...and always against the pool design weights themselves. - "realized_max_weight_ratio_vs_design": float( - np.max( - np.divide( - np.asarray(solve.weights, dtype=np.float64), - np.asarray(_design_weights_for(solve), dtype=np.float64), - ) - ) - ), - }, - "solve": { - "n_targets": int(len(problem.targets) + len(solve.national_diagnostics)), - "n_targets_by_kind": { - "local": local_rows, - "ladder": ladder_rows, - "national": int(len(solve.national_diagnostics)), - }, - "n_households": int(solve.frame.n("household")), - "pool_households": int(problem.n_households), - "dataset_size": None - if solve.size_receipt is None - else { - **dict(solve.size_receipt), - "dense_reference": _dense_reference_summary(solve), - }, - "initial_loss": float(solve.initial_loss), - "final_loss": float(solve.final_loss), - "max_abs_relative_error": float(abs_errors.max()), - "median_abs_relative_error": float(np.median(abs_errors)), - "n_nonzero": int(solve.n_nonzero), - "past_cap": {key: int(past_cap[key]) for key in _PAST_CAP_COUNT_KEYS}, - "loss_shape": "capped_relative_error", - "target_weight_rule": args.target_weight_rule, - "target_weight_rule_override": dict(args._doctrine_override_receipt), - "measure_resolution": dict(args._measure_resolution), - "cross_grain": dict(cross_grain), - "binding_adjudications": dict(solve.binding_adjudications), - "area_support_exclusions": { - "resource": "local_area_support_exclusions.json", - "entries_stood_on": sorted( - area_exclusion_details.get("reviewed_exclusions", {}) - ), - "stale": list(area_exclusion_details.get("stale_exclusions", [])), - "unknown": list(area_exclusion_details.get("unknown_exclusions", [])), - }, - "past_cap_by_kind": { - "local": dict(solve.past_cap_census or {}), - "national": dict(solve.national_past_cap_census or {}), - "all": dict(solve.all_past_cap_census or {}), - }, - }, - "diagnostics": { - "schema_version": calibration_diagnostics["schema_version"], - "target_registry": calibration_diagnostics["target_registry"], - "weakest_families": calibration_diagnostics["uk_diagnostics"][ - "weakest_families" - ], - "weakest_areas_by_fit": calibration_diagnostics["uk_diagnostics"][ - "weakest_areas_by_fit" - ], - "rotated_holdout": calibration_diagnostics["uk_diagnostics"][ - "rotated_holdout" - ], - }, - "support": { - "min_assigned_households": int(support["assigned_households"].min()), - "min_nonzero_households": int(support["nonzero_households"].min()), - "min_effective_sample_size": float(support["effective_sample_size"].min()), - "by_geography_level": { - str(level): { - "min_assigned_households": int(rows["assigned_households"].min()), - "min_nonzero_households": int(rows["nonzero_households"].min()), - "min_effective_sample_size": float( - rows["effective_sample_size"].min() - ), - "min_nonzero_source_households": int( - rows["nonzero_source_households"].min() - ), - } - for level, rows in support.groupby("geography_level", sort=True) - }, - }, - "fit": { - "local_by_family": uk_fit_by_family(solve.diagnostics), - "national_by_family": uk_fit_by_family(solve.national_diagnostics), - "weakest_families": sorted( - [ - *uk_fit_by_family(solve.diagnostics), - *uk_fit_by_family(solve.national_diagnostics), - ], - key=lambda row: ( - -float(row["worst_abs_relative_error"]), - row["family"], - ), - )[:10], - "weakest_areas_by_fit": dict(args._weakest_areas_by_fit), - "support_limited_misses": dict(args._support_limited_misses), - "rotated_holdout": dict(args._rotated_holdout), - }, - "vintages": ( - _local_vintage_census(args._joint_inputs_receipt["local_registry"]) - if args._joint_inputs_receipt is not None - else [] - ), - "failing_gate_ids": sorted( - gate_id - for gate_id, payload in gate_rows.items() - if not isinstance(payload, Mapping) or payload.get("status") != "passed" - ), - "releasable": releasable and args.dataset_households is None, - "release_posture": { - **release_posture, - **( - {} - if args.dataset_households is None - else {"size_certification_present": False} - ), - }, - "census_household_uprating": dict( - cross_grain.get("census_household_uprating") - or {"applied": False, "reason": "no cross-grain receipt"} - ), - # The reviewed measure exclusions the national compile stood on - # (name -> register record), so the narrowing is in the evidence. - "measure_exclusions": { - str(name): dict(record) - for name, record in sorted( - ( - (getattr(args, "_joint_inputs_receipt", None) or {}).get( - "measure_exclusions" - ) - or {} - ).items() - ) - }, - "blocked_at_f100": bool(getattr(args, "_blocked_failures", [])), - "blocking_failures": list(getattr(args, "_blocked_failures", [])), - "diagnostic_failures": list(getattr(args, "_diagnostic_failures", [])), - "release_gate_failures_not_enforced": list( - getattr(args, "_unenforced_release_failures", []) - ), - } - - -def _design_weights_for(solve: UKRowwiseDoctrineSolve) -> np.ndarray: - """The pool design weights aligned to the solve's exported rows.""" - if solve.selected_support is None or solve.dense_reference is None: - return np.asarray(solve.initial_weights, dtype=np.float64) - return np.asarray(solve.dense_reference.initial_weights, dtype=np.float64)[ - np.asarray(solve.selected_support, dtype=np.int64) - ] - - -def _dense_reference_summary(solve: UKRowwiseDoctrineSolve) -> dict[str, Any] | None: - """Manifest-sized evidence of the dense solve a size run was cut from.""" - - dense = solve.dense_reference - if dense is None: - return None - local_errors = dense.diagnostics["abs_relative_error"].to_numpy(dtype=np.float64) - national_errors = dense.national_diagnostics["abs_relative_error"].to_numpy( - dtype=np.float64 - ) - past_cap = dict(dense.past_cap_census or {}) - return { - "initial_loss": float(dense.initial_loss), - "final_loss": float(dense.final_loss), - "n_nonzero": int(dense.n_nonzero), - "n_households": int(dense.weights.size), - "max_abs_relative_error": float(local_errors.max()) - if local_errors.size - else None, - "median_abs_relative_error": float(np.median(local_errors)) - if local_errors.size - else None, - "national_max_abs_relative_error": float(national_errors.max()) - if national_errors.size - else None, - "past_cap": {key: int(past_cap[key]) for key in _PAST_CAP_COUNT_KEYS}, - "weights": uk_weight_summary(dense.weights), - "local_by_family": uk_fit_by_family(dense.diagnostics), - "national_by_family": uk_fit_by_family(dense.national_diagnostics), - "diagnostics_file": DENSE_REFERENCE_DIAGNOSTICS_FILENAME, - } - - -def _dense_reference_diagnostics_frame(solve: UKRowwiseDoctrineSolve) -> pd.DataFrame: - """Every target's dense-reference estimate, local rows then national rows.""" - - dense = solve.dense_reference - assert dense is not None - local = dense.diagnostics.copy() - local.insert(0, "grain", local["area_type"].astype(str)) - national = dense.national_diagnostics.copy() - national.insert(0, "grain", "national") - return pd.concat([local, national], ignore_index=True, sort=False) - - -def _dataset_size_selection_frame( - solve: UKRowwiseDoctrineSolve, - *, - problem: UKRowwiseLocalMatrix, - clone: UKLadderRowwiseDatasetResult, -) -> pd.DataFrame: - """One row per selected pool household: identity, design, draw and refit.""" - - dense = solve.dense_reference - receipt = solve.size_receipt - assert dense is not None and receipt is not None - support = np.asarray(solve.selected_support, dtype=np.int64) - household = clone.frame.table("household") - clone_column = ladder_clone_index_column("household") - ids = household["household_id"].to_numpy()[support] - expected = np.asarray([problem.household_ids[i] for i in support]) - if not np.array_equal(ids, expected): - raise RuntimeError( - "the cloned pool's household order does not match the solve's " - "matrix columns; the selection sidecar would misattribute rows." - ) - inclusion = np.asarray(receipt["inclusion_probabilities"], dtype=np.float64) - if inclusion.shape != support.shape: - raise RuntimeError("selection receipt inclusion probabilities are misaligned.") - return pd.DataFrame( - { - "pool_row_index": support, - "household_id": ids, - "clone_index": household[clone_column].to_numpy()[support] - if clone_column in household.columns - else np.zeros(support.size, dtype=np.int64), - "design_weight": dense.initial_weights[support], - "inclusion_probability": inclusion, - "certainty": inclusion >= 1.0, - "ht_baseline_weight": np.asarray(solve.initial_weights, dtype=np.float64), - "refit_weight": np.asarray(solve.weights, dtype=np.float64), - } - ) - - -def _local_output_registry( - solve: UKRowwiseDoctrineSolve, - problem: UKRowwiseLocalMatrix, - *, - period: int, -) -> TargetRegistry: - specs = [] - targets = tuple(solve.calibration_result.problem.targets) - if len(targets) < len(problem.target_frame): - raise RuntimeError("candidate target set is shorter than its local surface.") - for target, row in zip( - targets[: len(problem.target_frame)], - problem.target_frame.itertuples(index=False), - strict=True, - ): - payload = row._asdict() - specs.append( - TargetSpec( - name=str(payload["target_name"]), - entity="household", - value=float(payload["value"]), - measure=str(payload["metric"]), - period=int(payload.get("period", period)), - source=str(payload.get("source", "uk_rowwise_local_surface")), - family=str(payload["family"]), - metadata={ - "area_type": str(payload["area_type"]), - "area_code": str(payload["area_code"]), - "metric": str(payload["metric"]), - }, - hierarchy=target.hierarchy, - ) - ) - return TargetRegistry(specs, country="uk") - - -def _gate_payload(gate: GateResult, *, phase: str) -> dict[str, Any]: - return { - "name": str(gate.name), - "passed": bool(gate.passed), - "failures": list(gate.failures), - "details": dict(gate.details), - "phase": phase, - } - - -def _validate_solve_result( - solve: UKRowwiseDoctrineSolve, - *, - problem: UKRowwiseLocalMatrix, -) -> None: - if solve.past_cap_census is None: - raise RuntimeError( - "doctrine solve returned no past-cap census; refusing candidate." - ) - expected_count = ( - problem.n_households - if solve.selected_support is None - else len(solve.selected_support) - ) - if len(solve.weights) != expected_count: - raise RuntimeError( - "doctrine solve returned a weight vector with the wrong length." - ) - weights = np.asarray(solve.weights, dtype=np.float64) - if not np.isfinite(weights).all() or (weights < 0).any(): - raise RuntimeError( - "doctrine solve returned non-finite or negative household weights." - ) - if not np.isfinite([solve.initial_loss, solve.final_loss]).all(): - raise RuntimeError("doctrine solve returned a non-finite loss.") - errors = solve.diagnostics["abs_relative_error"].to_numpy(dtype=np.float64) - if len(errors) != len(problem.targets) or not np.isfinite(errors).all(): - raise RuntimeError( - "doctrine solve returned incomplete or non-finite diagnostics." - ) - missing_counts = sorted(set(_PAST_CAP_COUNT_KEYS) - set(solve.past_cap_census)) - if missing_counts: - raise RuntimeError( - f"past-cap census is missing count field(s): {missing_counts}." - ) - - -def _validate_support_summary(support: pd.DataFrame) -> None: - required = { - "geography_level", - "area_code", - "assigned_households", - "nonzero_households", - "nonzero_source_households", - "effective_sample_size", - } - missing = sorted(required - set(support.columns)) - if missing or support.empty: - raise RuntimeError( - f"area support summary is empty or missing required columns: {missing}." - ) - numeric = sorted(required - {"geography_level", "area_code"}) - values = support[numeric].to_numpy(dtype=np.float64) - if not np.isfinite(values).all() or (values < 0).any(): - raise RuntimeError("area support summary contains invalid values.") - - -def _load_candidate_evaluator(importer=importlib.import_module): - """The common-surface scorer (microcosm#967), loaded when an incumbent is given. - - Lazy so the driver imports without it; a build asked to evaluate refuses - up front, before any solve, when the scorer is not in the tree. - """ - - try: - return importer("microcosm.build.uk_runtime.candidate_score") - except ImportError as error: - raise ValueError( - "--incumbent-h5 needs the common-surface scorer (microcosm#967): " - "microcosm.build.uk_runtime.candidate_score is not in this tree." - ) from error - - -def _incumbent_arguments(args: argparse.Namespace) -> dict[str, Any] | None: - """The national role's optional incumbent, pinned and verified up front.""" - - if args.incumbent_h5 is None and args.incumbent_sha256 is None: - return None - if args.incumbent_h5 is None or args.incumbent_sha256 is None: - raise ValueError( - "--incumbent-h5 and --incumbent-sha256 must be given together." - ) - _load_candidate_evaluator() - incumbent_h5 = _require_file(args.incumbent_h5, label="--incumbent-h5") - info = _artifact_info(incumbent_h5) - _verify_requested_pin("--incumbent-h5", info, requested=args.incumbent_sha256) - return { - "path": str(incumbent_h5), - "sha256": info["sha256"], - "bytes": info["bytes"], - "label": str(args.incumbent_label), - } - - -def _list_sha256sums_entry(out_dir: Path, name: str) -> None: - """List a file the build wrote after the sidecars in the local sums.""" - - sums_path = out_dir / SHA256SUMS_FILENAME - entries = parse_sha256sums(sums_path.read_text(encoding="utf-8")) - if name in {listed for _, listed in entries}: - refresh_sha256sums_entry(out_dir, name) - return - digest = _artifact_info(out_dir / name)["sha256"] - sums_path.write_text( - sums_path.read_text(encoding="utf-8") + f"{digest} {name}\n", - encoding="utf-8", - ) - - -#: Receipt blocks that are record arrays (lists of mappings): the reviewed -#: telemetry artifact policy refuses them, and the verdict, the pruned block -#: and the aggregates carry everything a reviewer reads. The full receipt stays -#: beside the outputs and in the staged bundle. -_SCORE_RECEIPT_TELEMETRY_EXCLUDED = ( - "target_drift", - "signed_asymmetries", - "measure_resolution", -) - - -def _score_receipt_telemetry_summary(score: Mapping[str, Any]) -> dict[str, Any]: - return { - key: value - for key, value in score.items() - if key not in _SCORE_RECEIPT_TELEMETRY_EXCLUDED - } - - -def _evaluate_against_incumbent( - args: argparse.Namespace, - *, - incumbent: Mapping[str, Any] | None, - inputs: Mapping[str, Any], - output_paths: Mapping[str, Path], - telemetry: StagingTelemetryV2 | None, - calibration_year: int, - out_dir: Path, -) -> dict[str, Any]: - """Score the finished candidate against the incumbent; never fail the build. - - Runs after the staged bundle is on the Hub and before the telemetry - completes, so the receipt rides the run as a reviewed artifact. Rows the - incumbent cannot materialize are pruned from both arms and warned about - by name; the verdict (microcosm#578 rule 1 on the common surface) is - what the release-cut certifier requires to be ``passed``. An error is - recorded and warned, never raised: staging and the exit code stand. - """ - - if incumbent is None: - return { - "status": "not_requested", - "note": ( - "no --incumbent-h5: the rule-1 score receipt the release-cut " - "certifier needs was not produced; score the candidate with " - "tools/score_uk_national_candidate.py before certification." - ), - } - _stage( - telemetry, - "incumbent_evaluation", - "started", - incumbent_sha256=incumbent["sha256"], - ) - candidate = _artifact_info(output_paths["dataset"]) - try: - module = _load_candidate_evaluator() - score = module.evaluate_uk_candidate_against_incumbent( - candidate_h5=output_paths["dataset"], - incumbent_h5=Path(incumbent["path"]), - candidate_sha256=candidate["sha256"], - incumbent_sha256=incumbent["sha256"], - target_registry=inputs["national_registry"], - calibration_year=calibration_year, - measure_resolver_factory=module.uk_default_measure_resolver_factory( - out_dir, calibration_year - ), - candidate_label=output_paths["dataset"].stem, - incumbent_label=incumbent["label"], - band_edge_registry=inputs["band_edge_registry"], - ) - atomic_write_json(output_paths["score_receipt"], score) - except Exception as error: # noqa: BLE001 - the evaluation never fails a finished build - message = f"{type(error).__name__}: {error}"[:600] - print( - f"warning: the incumbent evaluation failed ({message}); the build's " - "evidence and staging are unaffected, and the candidate cannot be " - "certified until it is re-scored with " - "tools/score_uk_national_candidate.py.", - file=sys.stderr, - flush=True, - ) - _stage(telemetry, "incumbent_evaluation", "failed", error=message) - return { - "status": "error", - "error": message, - "incumbent": dict(incumbent), - "receipt": None, - } - warning = module.pruned_warning(score) - if warning is not None: - print(warning, file=sys.stderr, flush=True) - evaluation = score["evaluation"] - pruned = score["incumbent_unresolvable_pruned"] - _add_staging_artifact( - telemetry, - "score_vs_incumbent", - _score_receipt_telemetry_summary(score), - artifact_kind="aggregate_diagnostics", - classification="aggregate", - ) - _stage( - telemetry, - "incumbent_evaluation", - "completed", - verdict=evaluation["verdict"], - n_scored=int(evaluation["scored_surface"]["n_scored"]), - n_pruned=int(evaluation["scored_surface"]["n_pruned"]), - ) - return { - "status": "completed", - "verdict": evaluation["verdict"], - "rule_1": dict(evaluation["rule_1"]), - "scored_surface": dict(evaluation["scored_surface"]), - "pruned_measures": list(pruned["measures"]), - "pruned_families": dict(pruned["families"]), - "receipt": _artifact_info(output_paths["score_receipt"]), - "incumbent": dict(incumbent), - } - - -def _validate_output_paths( - output_paths: Mapping[str, Path], - *, - input_h5: Path, - ladder_path: Path | None, -) -> None: - resolved = {name: path.resolve() for name, path in output_paths.items()} - if len(set(resolved.values())) != len(resolved): - raise ValueError("candidate output paths must be distinct.") - protected = {input_h5.resolve()} - if ladder_path is not None: - protected.add(ladder_path.resolve()) - collisions = sorted(str(path) for path in resolved.values() if path in protected) - if collisions: - raise ValueError( - "candidate outputs must differ from --input-h5 and --ladder; " - f"collision(s): {collisions}." - ) - existing = sorted(str(path) for path in resolved.values() if path.exists()) - if existing: - raise FileExistsError( - f"refusing to overwrite existing candidate artifact(s): {existing}." - ) - - -def _assert_artifacts_unchanged( - *, - input_h5: Path, - input_artifact: Mapping[str, Any], - ladder_path: Path, - ladder_artifact: Mapping[str, Any], -) -> None: - for label, path, before in ( - ("input H5", input_h5, input_artifact), - ("ladder", ladder_path, ladder_artifact), - ): - after = _artifact_info(path) - if after["sha256"] != before["sha256"] or after["bytes"] != before["bytes"]: - raise RuntimeError( - f"{label} changed during the candidate build; refusing to " - "bind mixed source bytes." - ) - - -def _require_file(path: Path, *, label: str) -> Path: - resolved = path.expanduser().resolve() - if not resolved.is_file(): - raise FileNotFoundError(f"{label} artifact not found: {resolved}.") - return resolved - - -def _source_year(requested: int | None, *, time_period: str) -> int: - if requested is not None: - if requested <= 0: - raise ValueError("--source-year must be positive.") - return requested - prefix = str(time_period).strip()[:4] - if len(prefix) != 4 or not prefix.isdigit(): - raise ValueError( - "Could not infer source year from input H5 time_period; pass --source-year." - ) - return int(prefix) - - -def _artifact_info( - path: Path, - *, - reported_path: Path | None = None, -) -> dict[str, Any]: - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return { - "path": str((reported_path or path).resolve()), - "sha256": digest.hexdigest(), - "bytes": int(path.stat().st_size), - } - +from microcosm.build.uk_runtime.full_build_cli import main if __name__ == "__main__": raise SystemExit(main()) diff --git a/tools/evaluate_uk_incumbent_surface.py b/tools/evaluate_uk_incumbent_surface.py index ba6ce5bc6..f217a6646 100644 --- a/tools/evaluate_uk_incumbent_surface.py +++ b/tools/evaluate_uk_incumbent_surface.py @@ -17,7 +17,6 @@ import argparse import collections import hashlib -import importlib.util import json import sys import tempfile @@ -29,6 +28,7 @@ from microcosm.build.ledger_artifact import load_ledger_consumer_artifact from microcosm.build.uk_runtime.frs_release import load_uk_frs_release +from microcosm.build.uk_runtime.full_measure import resolve_uk_full_measures from microcosm.build.uk_runtime.incumbent_surface_evaluation import ( GSS_REGION_CODES, classify_local_rows, @@ -56,17 +56,6 @@ from microcosm.data.contract import uk_incumbent_surface_assessment -def _driver(): - spec = importlib.util.spec_from_file_location( - "build_uk_rowwise_candidate", - Path(__file__).resolve().with_name("build_uk_rowwise_candidate.py"), - ) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return module - - def _parse_args(argv): p = argparse.ArgumentParser(description=__doc__) p.add_argument("--candidate-h5", required=True, type=Path) @@ -103,7 +92,6 @@ def _check_candidate_chronicle_identity( def main(argv=None) -> int: args = _parse_args(argv) - driver = _driver() artifact = load_ledger_consumer_artifact( args.ledger_facts, expected_facts_sha256=args.ledger_facts_sha256, @@ -206,7 +194,7 @@ def digest(path): print("resolving the engine over the frame ...", file=sys.stderr, flush=True) with tempfile.TemporaryDirectory(prefix="uk-incumbent-eval-") as scratch: prepared_frame, _restore, national_rows, local_metrics, resolution = ( - driver._resolve_candidate_engine_surface( + resolve_uk_full_measures( frame, resolver_registry, period=period, From bef59dda1a2e75351b0343239c06c893e0d4ab79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 25 Sep 2026 21:01:37 +0100 Subject: [PATCH 24/44] Retire the frozen HMRC tail stages and the spine exclusion list; regenerate the derived surfaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two frozen tail stages frs_hmrc_retained_leaves and hmrc_spi_income leave the manifest (35 -> 33 stages) together with their transform module, their source-stage resource and the UK_SPINE_EXCLUSIONS constant that hid them from every consumer. The FRS HMRC leaf columns now come from uk_runtime/frs_hmrc_source; the HMRC source contract, the SPI spine and income stages, the source runtime and the graph kernels read it; release_input_coverage refuses superseded_by and parses required_predecessor_stages, and the coverage-manifest tool loses its HMRC path. Because #1006 placed spi_income_band_donors between the support channel and the income spine, the audited HMRC family names it as a predecessor and the contract admits its two operation kinds; that is the one departure from #901's retirement. Regenerated, not pasted: the release-input coverage manifest (--check current, 145 required inputs; source_stages.json 73c3a14f…), the H2 spine parity fixture (byte-identical to the committed one, oracle edb1659b…); the gate-register pins did not move. The country-spec resource lists and stage counts follow; the eager package init re-points the six FRS_HMRC_* constants and drops the five retired names, pinned by a new eager export test; the graph driver gains a --staging-local-only case. Verified: retirement files 458 passed / 2 skipped; CLI, graph, spine, staging, spec, pins and coverage 255 passed; microcosm-data contract 254; H2 3 passed / 1 skipped; whole uk group 2,549 passed / 20 skipped (the one failure was the new CLI case before its fix, green since); shared-spec 2,100 passed / 48 skipped with one pre-existing failure fixed in the next commit; spot-check 337 passed / 1 skipped; ci_test_groups --verify ok; ruff clean. Co-Authored-By: Claude Fable 5.1 --- .../microcosm/build/uk/country_package.json | 5 - .../build/uk/efrs_parity_known_gaps.json | 2 + .../build/uk/hmrc_income_source_stages.json | 735 -------------- .../uk/release_input_coverage_manifest.json | 48 +- .../src/microcosm/build/uk/source_stages.json | 737 --------------- .../src/microcosm/build/uk/spec/sources.yaml | 597 ------------ .../microcosm/build/uk_runtime/__init__.py | 13 +- .../build/uk_runtime/country_adapter.py | 13 +- .../build/uk_runtime/frs_hmrc_leaves.py | 893 ------------------ .../build/uk_runtime/frs_hmrc_source.py | 311 ++++++ .../src/microcosm/build/uk_runtime/graph.py | 14 +- .../build/uk_runtime/graph_kernels.py | 5 +- .../build/uk_runtime/hmrc_source_contract.py | 279 ++---- .../uk_runtime/release_input_coverage.py | 93 +- .../build/uk_runtime/source_runtime.py | 19 +- .../microcosm/build/uk_runtime/spi_income.py | 2 +- .../microcosm/build/uk_runtime/spi_spine.py | 6 +- .../microcosm/build/uk_runtime/spine_build.py | 7 +- .../tests/engine/uk/test_uk_graph.py | 6 +- .../uk/test_uk_release_input_coverage.py | 64 +- .../engine_free/shared/test_country_spec.py | 16 +- .../uk/test_uk_battery_bindings.py | 18 +- .../engine_free/uk/test_uk_frs_hmrc_leaves.py | 556 ----------- .../engine_free/uk/test_uk_frs_hmrc_source.py | 149 +++ .../engine_free/uk/test_uk_full_build_cli.py | 45 + .../tests/engine_free/uk/test_uk_graph.py | 29 +- .../uk/test_uk_hmrc_income_source_manifest.py | 773 +++------------ .../uk/test_uk_hmrc_replay_artifacts.py | 27 +- .../uk/test_uk_national_sampling.py | 27 - ...test_uk_release_input_coverage_manifest.py | 83 +- .../engine_free/uk/test_uk_runtime_exports.py | 43 + .../engine_free/uk/test_uk_source_runtime.py | 17 +- .../engine_free/uk/test_uk_source_stages.py | 141 +-- .../uk/test_uk_spine_acceptance_receipt.py | 11 +- .../tests/engine_free/us/test_us_plan.py | 1 - .../uk/test_uk_staging_integration.py | 7 +- test_support/microcosm_build/uk_graph.py | 1 - .../uk_release_input_coverage.py | 2 +- test_support/microcosm_build/uk_spi_income.py | 2 +- .../microcosm_graph/acceptance_h_parity.py | 2 +- ...uild_uk_release_input_coverage_manifest.py | 136 +-- tools/graph_uk_spine_fixture.py | 4 +- 42 files changed, 1033 insertions(+), 4906 deletions(-) delete mode 100644 packages/microcosm-build/src/microcosm/build/uk/hmrc_income_source_stages.json delete mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_leaves.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_source.py delete mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_frs_hmrc_leaves.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_frs_hmrc_source.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_runtime_exports.py diff --git a/packages/microcosm-build/src/microcosm/build/uk/country_package.json b/packages/microcosm-build/src/microcosm/build/uk/country_package.json index b7be7d3be..5c616c405 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/uk/country_package.json @@ -127,11 +127,6 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, - { - "path": "hmrc_income_source_stages.json", - "kind": "legacy_json", - "schema_id": "legacy_json" - }, { "path": "ofgem_region_crosswalk.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_known_gaps.json b/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_known_gaps.json index 9fe5e1d56..e8577cb51 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_known_gaps.json +++ b/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_known_gaps.json @@ -638,6 +638,7 @@ "known_gaps": {}, "restored_required_columns": { "charitable_investment_gifts": { + "current_producer_stage": "hmrc_spi_income_spine", "effective_signal_mass_share": 0.00028055329260683216, "minimum_nondefault_mass_share": 1e-06, "positive_mass_signal_rows": 294, @@ -647,6 +648,7 @@ "support_channel": "spi" }, "gift_aid": { + "current_producer_stage": "hmrc_spi_income_spine", "effective_signal_mass_share": 0.01330315665904484, "minimum_nondefault_mass_share": 1e-06, "positive_mass_signal_rows": 12894, diff --git a/packages/microcosm-build/src/microcosm/build/uk/hmrc_income_source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/hmrc_income_source_stages.json deleted file mode 100644 index 8e8635ae4..000000000 --- a/packages/microcosm-build/src/microcosm/build/uk/hmrc_income_source_stages.json +++ /dev/null @@ -1,735 +0,0 @@ -{ - "version": 1, - "country": "uk", - "policy": "The UK HMRC/SPI income family is source-manifest-defined. Private donor data must be supplied locally, every artifact must be SHA-256 verified at runtime, retained FRS constituents and published bands fail closed, and the current replay keeps importance-kind weights because all 208 banded facts require an unavailable full FRS total-income measure.", - "stages": [ - { - "stage": "hmrc_spi_income", - "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24", - "source": "https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods", - "grain": "person", - "base_candidate": { - "filename": "populace_uk_2023.h5", - "tier": "frs", - "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", - "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", - "size_bytes": 1315880118, - "runtime_sha256_required": true - }, - "artifacts": [ - { - "role": "qrf_donor", - "kind": "private_microdata", - "format": "tab_delimited", - "survey": "Survey of Personal Incomes Public Use Tape 2022-23", - "vintage": "2022-23", - "tax_year_start": 2022, - "ukds_study_number": "SN 9422", - "doi": "10.5255/UKDA-SN-9422-1", - "filename": "put2223uk.tab", - "sha256": "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66", - "size_bytes": 141323762, - "reviewed_source": "PolicyEngine licensed copy from policyengine/policyengine-uk-data-private on Hugging Face, spi_2022_23.zip", - "access": "private_local_input", - "locator": "caller-supplied local input", - "runtime_sha256_required": true - }, - { - "role": "published_fact_surface", - "kind": "administrative_table", - "format": "ods", - "survey": "HMRC Personal Incomes Tables 3.6 and 3.7", - "publication": "https://www.gov.uk/government/statistics/personal-incomes-statistics-for-the-tax-year-2023-to-2024", - "vintage": "2023-24", - "tax_year_start": 2023, - "locator": "https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods", - "sha256": "ad063b06b2bdeef8600dbbb09d48153337a4966f8c7eea50df7a2e0304ebd73e", - "size_bytes": 166693, - "mime_type": "application/vnd.oasis.opendocument.spreadsheet", - "sheets": [ - "Table_3_6", - "Table_3_7" - ], - "mapped_build_period": 2023, - "period_mapping": "tax_year_start", - "runtime_sha256_required": true - } - ], - "operations": [ - { - "kind": "verify_certified_candidate", - "artifact": "base_candidate", - "runtime_sha256_required": true, - "fail_on_mismatch": true - }, - { - "kind": "retain_adjudicated_frs_hmrc_leaves", - "population": "certified_microcosm_uk_candidate_base_channel", - "source_vintage": "2023-24", - "mapped_build_period": 2023, - "annualization": "weekly raw FRS amounts * (365.25 / 7)", - "status": "adjudicated_partial_replay", - "retained_full_constituents": { - "hmrc_spi_pay": { - "spi_concept": "PAY", - "scope": "full", - "raw_sources": [ - "ADULT.INEARNS" - ], - "formula": "max(0, ADULT.INEARNS) * (365.25 / 7)" - }, - "hmrc_spi_unemployment_benefit_income": { - "spi_concept": "UBISJA", - "scope": "full", - "raw_sources": [ - "BENEFITS.BENEFIT=14:BENAMT", - "BENEFITS.BENEFIT=19:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT in {14, 19}) * (365.25 / 7)" - }, - "hmrc_spi_incapacity_benefit_income": { - "spi_concept": "INCPBEN", - "scope": "full", - "raw_sources": [ - "BENEFITS.BENEFIT=17:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 17) * (365.25 / 7)", - "observed_support": "structural zero in the audited 2023-24 FRS; retained so future vintages flow" - } - }, - "retained_named_subsets": { - "ossben_identifiable_subset": { - "spi_concept": "OSSBEN", - "raw_sources": [ - "BENEFITS.BENEFIT=13:BENAMT", - "BENEFITS.BENEFIT=16,VAR2 in {1,3}:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 13 or (BENEFIT == 16 and VAR2 in {1, 3})) * (365.25 / 7)", - "scope": "identifiable_subset" - }, - "srp_regular_code5": { - "spi_concept": "SRP", - "raw_sources": [ - "BENEFITS.BENEFIT=5:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 5) * (365.25 / 7)", - "scope": "regular_code5_subset" - } - }, - "source_absent_full_constituents": [ - "EPB", - "EXPS", - "TAXTERM", - "MOTHINC", - "OTHERINC" - ], - "full_concepts_forbidden_on_frs": [ - "hmrc_spi_employment_benefits", - "hmrc_spi_employment_expenses", - "hmrc_spi_taxable_termination_pay", - "hmrc_spi_miscellaneous_employment_income", - "hmrc_spi_other_income", - "hmrc_spi_other_social_security_income", - "hmrc_spi_state_pension_income" - ], - "forbid_proxy_substitution": [ - "employment_income", - "miscellaneous_income" - ], - "fail_on_missing_retained_constituent": true, - "fail_on_full_concept_alias": true - }, - { - "kind": "verify_pinned_hmrc_source_pair", - "artifact_roles": [ - "qrf_donor", - "published_fact_surface" - ], - "require_before_source_read": true, - "runtime_sha256_required": true, - "fail_on_mismatch": true - }, - { - "kind": "replace_zero_weight_spi_support", - "existing_channel": "spi", - "require_existing_weight": 0, - "replacement_strata": [ - "clone_index", - "household_is_capital_gains_clone", - "region" - ], - "spi_prior_national_household_mass_share": 0.5, - "output_weight_kind": "importance", - "preserve_total_household_mass": true, - "require_mass_change_record": true, - "mass_change_reason": "Allocate 50% of certified UK national household prior mass to the rebuilt 2022-23 SPI support channel; total national mass is conserved.", - "fail_on_live_existing_spi_mass": true - }, - { - "kind": "strict_read_private_table", - "artifact_role": "qrf_donor", - "filename": "put2223uk.tab", - "delimiter": "\t", - "weight": "FACT", - "required_columns": [ - "AGERANGE", - "GORCODE", - "SEX", - "FACT", - "PAY", - "EPB", - "EXPS", - "TAXTERM", - "INCPBEN", - "OSSBEN", - "UBISJA", - "MOTHINC", - "OTHERINC", - "PROFITS", - "CAPALL", - "LOSSBF", - "SRP", - "INCBBS", - "DIVIDENDS", - "PENSION", - "INCPROP", - "OTHERINV", - "GIFTAID", - "GIFTINV", - "TEI", - "TII", - "TI" - ], - "runtime_sha256_required": true, - "fail_on_missing_file": true, - "fail_on_missing_columns": true, - "fail_on_invalid_weight": true - }, - { - "kind": "fit_weighted_qrf_stage1", - "training_artifact_role": "qrf_donor", - "predictors": [ - "age", - "gender", - "region" - ], - "categorical_predictors": [ - "gender", - "region" - ], - "source_sampling_weight": "FACT", - "sample_size": 100000, - "sample_with_replacement": true, - "post_sample_fit_weight": "uniform", - "fit_weight_kind": "design", - "double_apply_source_weight": false, - "source_columns": { - "self_employment_income": [ - "PROFITS", - "CAPALL", - "LOSSBF" - ], - "savings_interest_income": [ - "INCBBS" - ], - "dividend_income": [ - "DIVIDENDS" - ], - "private_pension_income": [ - "PENSION" - ], - "property_income": [ - "INCPROP" - ], - "other_investment_income": [ - "OTHERINV" - ], - "gift_aid": [ - "GIFTAID" - ], - "charitable_investment_gifts": [ - "GIFTINV" - ], - "hmrc_spi_pay": [ - "PAY" - ], - "hmrc_spi_employment_benefits": [ - "EPB" - ], - "hmrc_spi_employment_expenses": [ - "EXPS" - ], - "hmrc_spi_incapacity_benefit_income": [ - "INCPBEN" - ], - "hmrc_spi_other_social_security_income": [ - "OSSBEN" - ], - "hmrc_spi_taxable_termination_pay": [ - "TAXTERM" - ], - "hmrc_spi_unemployment_benefit_income": [ - "UBISJA" - ], - "hmrc_spi_miscellaneous_employment_income": [ - "MOTHINC" - ], - "hmrc_spi_other_income": [ - "OTHERINC" - ], - "hmrc_spi_state_pension_income": [ - "SRP" - ] - }, - "derived_policyengine_outputs": { - "employment_income": { - "source_columns": [ - "PAY", - "EPB", - "TAXTERM" - ], - "formula": "hmrc_spi_pay + hmrc_spi_employment_benefits + hmrc_spi_taxable_termination_pay", - "derive_after_draw": true - } - }, - "outputs": [ - "self_employment_income", - "savings_interest_income", - "dividend_income", - "private_pension_income", - "property_income", - "other_investment_income", - "gift_aid", - "charitable_investment_gifts", - "hmrc_spi_pay", - "hmrc_spi_employment_benefits", - "hmrc_spi_employment_expenses", - "hmrc_spi_incapacity_benefit_income", - "hmrc_spi_other_social_security_income", - "hmrc_spi_taxable_termination_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_miscellaneous_employment_income", - "hmrc_spi_other_income", - "hmrc_spi_state_pension_income" - ], - "joint_draw": true, - "savings_interest_source_semantics": "INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine gross input", - "employment_income_source_semantics": "PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; it is not the Table 3.6 measure", - "hmrc_employed_income_source_semantics": "Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels", - "self_employment_income_source_semantics": "max(0, PROFITS - CAPALL - LOSSBF)", - "assessable_income_source_semantics": "QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates and TI equals TEI + TII exactly", - "source_ti_identity_fields": [ - "TI", - "TEI", - "TII" - ], - "source_leaf_reconciliation": { - "documentation_url": "https://doc.ukdataservice.ac.uk/doc/9422/mrdoc/pdf/9422_put_2223_full_documentation.pdf", - "composite_indicator": "AGERANGE == -1", - "formulas": { - "TEI": "max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, PROFITS - CAPALL - LOSSBF)", - "TII": "OTHERINV + DIVIDENDS + INCPROP + INCBBS", - "TI": "TEI + TII" - }, - "maximum_absolute_difference_gbp": { - "ordinary": { - "TEI": 15, - "TII": 10, - "TI": 20 - }, - "composite": { - "TEI": 180, - "TII": 10, - "TI": 180 - } - }, - "rationale": "The official PUT rounds source fields, averages documented composite records, then rounds remaining income fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain exact." - }, - "ti_identity_absolute_tolerance_gbp": 5, - "stochastic_aggregates_forbidden": [ - "hmrc_spi_employed_income", - "hmrc_spi_total_earned_income", - "hmrc_spi_total_investment_income", - "hmrc_spi_assessable_income" - ], - "require_all_predictors": true, - "require_all_outputs": true - }, - { - "kind": "fit_weighted_qrf_stage2", - "training_population": "certified_microcosm_uk_candidate_base_channel", - "target_population": "rebuilt_spi_support_channel", - "predictors": [ - "age", - "gender", - "region", - "employment_income", - "self_employment_income", - "savings_interest_income", - "dividend_income", - "private_pension_income", - "property_income" - ], - "reviewed_absent_predictors": { - "other_investment_income": "This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only stage-2 predictor: policyengine-uk-data frs_only.py defines exactly six income predictors and the certified Microcosm UK base candidate has no other_investment_income column." - }, - "categorical_predictors": [ - "gender", - "region" - ], - "weight": "household_weight", - "weight_mapping": "household_to_person", - "outputs": [ - "employee_pension_contributions", - "employer_pension_contributions", - "personal_pension_contributions", - "pension_contributions_via_salary_sacrifice", - "tax_free_savings_income", - "universal_credit_reported", - "pension_credit_reported", - "child_benefit_reported", - "housing_benefit_reported", - "income_support_reported", - "working_tax_credit_reported", - "child_tax_credit_reported", - "attendance_allowance_reported", - "state_pension_reported", - "dla_sc_reported", - "dla_m_reported", - "pip_m_reported", - "pip_dl_reported", - "sda_reported", - "carers_allowance_reported", - "iidb_reported", - "afcs_reported", - "bsp_reported", - "winter_fuel_allowance_reported", - "council_tax_benefit_reported", - "jsa_contrib_reported", - "jsa_income_reported", - "esa_contrib_reported", - "esa_income_reported" - ], - "reviewed_absent_outputs": { - "incapacity_benefit_reported": "Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not a populated loader layer.", - "maternity_allowance_reported": "Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training source can be materialized for this stage." - }, - "postprocess": { - "gross_savings_interest_income": "stage1 INCBBS draw + stage2 tax_free_savings_income", - "refresh_disability_categories": [ - "aa_category", - "dla_sc_category", - "dla_m_category", - "pip_m_category", - "pip_dl_category" - ], - "refresh_disability_flags": [ - "is_disabled_for_benefits", - "is_enhanced_disabled_for_benefits", - "is_severely_disabled_for_benefits" - ] - }, - "joint_draw": true, - "require_all_predictors": true, - "require_all_materializable_outputs": true, - "require_all_outputs": false - }, - { - "kind": "materialize_hmrc_income_bands_fail_closed", - "artifact_role": "published_fact_surface", - "mapped_build_period": 2023, - "period_mapping": "tax_year_start", - "column_index_base": 0, - "data_row_start_index": 5, - "stop_label": "All ranges", - "count_unit_multiplier": 1000, - "amount_unit_multiplier": 1000000, - "component_columns": { - "employment_income": { - "sheet": "Table_3_6", - "count_column_index": 4, - "amount_column_index": 5 - }, - "self_employment_income": { - "sheet": "Table_3_6", - "count_column_index": 1, - "amount_column_index": 2 - }, - "state_pension": { - "sheet": "Table_3_6", - "count_column_index": 7, - "amount_column_index": 8 - }, - "private_pension_income": { - "sheet": "Table_3_6", - "count_column_index": 10, - "amount_column_index": 11 - }, - "property_income": { - "sheet": "Table_3_7", - "count_column_index": 1, - "amount_column_index": 2 - }, - "savings_interest_income": { - "sheet": "Table_3_7", - "count_column_index": 4, - "amount_column_index": 5 - }, - "dividend_income": { - "sheet": "Table_3_7", - "count_column_index": 7, - "amount_column_index": 8 - }, - "other_investment_income": { - "sheet": "Table_3_7", - "count_column_index": 10, - "amount_column_index": 11 - } - }, - "required_band_lower_bounds_gbp": [ - 12570, - 15000, - 20000, - 30000, - 40000, - 50000, - 70000, - 100000, - 150000, - 200000, - 300000, - 500000, - 1000000 - ], - "required_measures": [ - "count", - "amount" - ], - "fail_on_missing_sheet": true, - "fail_on_missing_component": true, - "fail_on_missing_band": true, - "fail_on_non_numeric_value": true - }, - { - "kind": "classify_hmrc_income_facts_with_reviewed_fences", - "target_operation": "materialize_hmrc_income_bands_fail_closed", - "components": [ - "employment_income", - "self_employment_income", - "state_pension", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "other_investment_income" - ], - "breakdown_dependency": "hmrc_spi_assessable_income", - "frs_breakdown_status": "unavailable_full_measure", - "input_weight_kind": "importance", - "output_weight_kind": "importance", - "calibration_permitted": false, - "required_fact_count": 208, - "outcome_counts": { - "exact_pass": 0, - "exact_fail": 0, - "directional_pass": 0, - "directional_fail": 0, - "excluded_with_fence": 208 - }, - "classification_rationale": "Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound is valid.", - "reviewed_fences": [ - { - "fence_id": "frs_epb_source_absent", - "constituents": [ - "EPB" - ], - "raw_sources_searched": [ - "JOB.EXPBEN01-EXPBEN13", - "JOB.CARVAL", - "JOB.CARAMT", - "JOB.FUELAMT", - "JOB.VCHAMT", - "JOB.CHVAMT" - ], - "finding": "Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce complete taxable expenses payments and benefits.", - "mass_implication": "12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this is not monetary support.", - "rationale": "Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an imputation or proxy.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_exps_source_absent", - "constituents": [ - "EXPS" - ], - "raw_sources_searched": [ - "JOB.EXPBEN04/EXPBEN05", - "JOB.MILEAMT/JOB.MOTAMT", - "JOB.UMILEAMT/JOB.UMOTAMT", - "JOB.DEDUC1-DEDUC9", - "JOB.UDEDUC1-UDEDUC9" - ], - "finding": "Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense amount required by SPI.", - "mass_implication": "5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the true EXPS mass is not estimable.", - "rationale": "The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity with a negative sign.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_taxterm_source_absent", - "constituents": [ - "TAXTERM" - ], - "raw_sources_searched": [ - "ADULT.REDAMT", - "ADULT and JOB taxable-termination split search" - ], - "finding": "Missing. REDAMT is gross redundancy pay and has neither the taxable amount nor non-redundancy termination pay.", - "mass_implication": "0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable mass is unknown.", - "rationale": "Gross redundancy pay cannot be relabeled as taxable termination pay.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_mothinc_source_absent", - "constituents": [ - "MOTHINC" - ], - "raw_sources_searched": [ - "ODDJOB.OJAMT/ODDJOB.OJNOW", - "ADULT.ALLPAY2", - "ADULT.ROYYR2-ROYYR4", - "JOB.OWNOTHER" - ], - "finding": "Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI miscellaneous employment income would be a proxy.", - "mass_implication": "Odd-job-only effective person mass is 0.1724207%; the broader unresolved miscellaneous pool is 1.4650566%.", - "rationale": "The FRS instrument cannot separate the SPI miscellaneous-employment concept source-faithfully.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_otherinc_source_absent", - "constituents": [ - "OTHERINC" - ], - "raw_sources_searched": [ - "ADULT, ODDJOB, and JOB miscellaneous fields", - "PENSION", - "ACCOUNTS", - "ASSETS", - "BENEFITS" - ], - "finding": "Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be split between MOTHINC and OTHERINC from source evidence.", - "mass_implication": "No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate FRS effective person mass.", - "rationale": "A union of heterogeneous residual fields would be a new proxy, not a retained source constituent.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_ossben_identifiable_subset", - "constituents": [ - "OSSBEN", - "ossben_identifiable_subset" - ], - "raw_sources_searched": [ - "BENEFITS.BENAMT", - "BENEFITS.BENEFIT", - "BENEFITS.VAR2", - "BENEFITS codes 13, 16, 6, and 30" - ], - "finding": "Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted.", - "mass_implication": "1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; it is not full OSSBEN support.", - "rationale": "The retained column must remain explicitly named as a subset and cannot satisfy the full SPI concept.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_srp_regular_code5_subset", - "constituents": [ - "SRP", - "srp_regular_code5" - ], - "raw_sources_searched": [ - "BENEFITS.BENAMT where BENEFIT == 5", - "BENEFITS codes 6 and 9" - ], - "finding": "Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension.", - "mass_implication": "18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; it is not complete SRP support.", - "rationale": "The retained column must remain explicitly named as a subset and cannot be reported as the full published state-pension measure.", - "dependent_fence_ids": [] - }, - { - "fence_id": "full_frs_tei_band_unavailable", - "constituents": [ - "EPB", - "EXPS", - "TAXTERM", - "MOTHINC", - "OTHERINC", - "OSSBEN", - "SRP" - ], - "raw_sources_searched": [], - "finding": "The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income band assignment is unavailable on the FRS channel.", - "mass_implication": "Every one of the 208 published facts is banded by total income and therefore depends on this unavailable like-for-like measure.", - "rationale": "A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.", - "dependent_fence_ids": [ - "frs_epb_source_absent", - "frs_exps_source_absent", - "frs_taxterm_source_absent", - "frs_mothinc_source_absent", - "frs_otherinc_source_absent", - "frs_ossben_identifiable_subset", - "frs_srp_regular_code5_subset" - ] - } - ], - "fact_fence_id": "full_frs_tei_band_unavailable", - "blocked_dependency": "hmrc_spi_assessable_income", - "fail_on_unfenced_exclusion": true, - "fail_on_fact_count_mismatch": true, - "forbid_biased_estimate_or_delta": true - }, - { - "kind": "gate_distributional_effective_mass", - "columns": [ - "gift_aid", - "charitable_investment_gifts" - ], - "weight": "household_weight", - "weight_mapping": "household_to_person", - "support_channel_column": "person_support_channel", - "required_support_channel": "spi", - "mass_share_denominator": "all_person_effective_mass", - "minimum_nondefault_mass_share": 0.000001, - "fail_below_floor": true - } - ], - "official_table_components": [ - "employment_income", - "self_employment_income", - "state_pension", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "other_investment_income" - ], - "donor_relief_outputs": [ - "gift_aid", - "charitable_investment_gifts" - ], - "outputs": [ - "employment_income", - "self_employment_income", - "state_pension", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "other_investment_income", - "gift_aid", - "charitable_investment_gifts", - "hmrc_spi_employed_income", - "hmrc_spi_total_earned_income", - "hmrc_spi_total_investment_income", - "hmrc_spi_assessable_income" - ], - "notes": "Current-source adjudicated replay contract: the private 2022-23 SPI donor and public 2023-24 HMRC ODS are pinned by reviewed SHA-256 and size and verified together before either is opened. The QRF draws source leaves; HMRC employed income, TEI, TII, and TI are deterministic post-draw aggregates on the SPI channel, with TI exactly equal to TEI + TII. PolicyEngine employment_income remains the narrow PAY + EPB + TAXTERM input on SPI rows. Stage 2 mirrors policyengine-uk-data frs_only.py exactly: its income predictors are employment, self-employment, savings interest, dividends, private pension, and property income. Other investment income remains a stage-1 SPI draw and official HMRC fact component, but is excluded from stage 2 because the certified FRS candidate does not carry it. The FRS channel retains source-faithful full PAY, UBISJA, and INCPBEN plus explicitly named ossben_identifiable_subset and srp_regular_code5; EPB, EXPS, TAXTERM, MOTHINC, OTHERINC, full OSSBEN, and full SRP remain forbidden. Because the missing legs prevent a complete FRS TEI measure, none of the 208 non-overlapping total-income-band facts is exact or directional. Every fact is an excluded-with-fence record, no calibration is performed, and weights remain importance-kind. Gift Aid restoration still requires the rebuilt positive-mass SPI channel to clear the reviewed 1ppm effective-mass floor." - } - ] -} diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 5349441b9..115f05bdf 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -473,7 +473,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "3082ac56e23f2bb383513c324dc03a5f77ef3737fa3a839ef29a662b4f1e37f4", + "source_manifest_sha256": "89a9a3be58f29d49b371b5f395b290ee168085923f28a4cba1481dd57935b76e", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2026 release, Table 2.1a for 2024-25, via the vendored conditioning facts", "survey": "HMRC Capital Gains Tax statistics Table 2.1a (2024-25 individuals) and Advani-Summers capital-gains incidence" @@ -491,7 +491,7 @@ "required_mass_change_reason": "Capital-gains incidence anchor moves the mass of non-liable clone households back to their paired originals until the sub-exempt and loss-making clone mass match the Advani-Summers reporter composition at the redrawn liable mass; every pair's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "3082ac56e23f2bb383513c324dc03a5f77ef3737fa3a839ef29a662b4f1e37f4", + "source_manifest_sha256": "89a9a3be58f29d49b371b5f395b290ee168085923f28a4cba1481dd57935b76e", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, the HMRC Table 3 redrawn gains on the spine and Advani-Summers capital-gains incidence" @@ -514,7 +514,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "3082ac56e23f2bb383513c324dc03a5f77ef3737fa3a839ef29a662b4f1e37f4", + "source_manifest_sha256": "89a9a3be58f29d49b371b5f395b290ee168085923f28a4cba1481dd57935b76e", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -543,7 +543,7 @@ "required_mass_change_reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "3082ac56e23f2bb383513c324dc03a5f77ef3737fa3a839ef29a662b4f1e37f4", + "source_manifest_sha256": "89a9a3be58f29d49b371b5f395b290ee168085923f28a4cba1481dd57935b76e", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -563,7 +563,7 @@ "required_mass_change_reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "3082ac56e23f2bb383513c324dc03a5f77ef3737fa3a839ef29a662b4f1e37f4", + "source_manifest_sha256": "89a9a3be58f29d49b371b5f395b290ee168085923f28a4cba1481dd57935b76e", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -584,7 +584,7 @@ "required_mass_change_reason": "CGT asset-type assignment on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "3082ac56e23f2bb383513c324dc03a5f77ef3737fa3a839ef29a662b4f1e37f4", + "source_manifest_sha256": "89a9a3be58f29d49b371b5f395b290ee168085923f28a4cba1481dd57935b76e", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed" @@ -608,7 +608,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "3082ac56e23f2bb383513c324dc03a5f77ef3737fa3a839ef29a662b4f1e37f4", + "source_manifest_sha256": "89a9a3be58f29d49b371b5f395b290ee168085923f28a4cba1481dd57935b76e", "source_vintages": { "conditioning_resource": "hmrc_cgt_conditioning_facts.json", "conditioning_resource_sha256": "d4c2d8044030a9c9e197298281262d6bc12d2883cc75f25044b25b3dd9c33a2e", @@ -621,11 +621,8 @@ }, "hmrc_spi_income": { "band_measure": "hmrc_spi_assessable_income", - "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "calibration_permitted": false, - "canonical_source_manifest": "source_stages.json", - "canonical_source_manifest_sha256": "3082ac56e23f2bb383513c324dc03a5f77ef3737fa3a839ef29a662b4f1e37f4", "effective_mass_requirements": { "charitable_investment_gifts": { "mass_share_denominator": "all_person_effective_mass", @@ -664,6 +661,11 @@ "other_investment_income" ], "required_mass_change_reason": "Allocate 50% of certified UK national household prior mass to the rebuilt 2022-23 SPI support channel; total national mass is conserved.", + "required_predecessor_stages": [ + "frs_hmrc_spine_leaves", + "spi_support_channel", + "spi_income_band_donors" + ], "required_target_count": 208, "restoration_status": "adjudicated_partial_replay", "retained_frs_constituents": { @@ -694,8 +696,8 @@ "frs_srp_regular_code5_subset", "full_frs_tei_band_unavailable" ], - "source_manifest": "hmrc_income_source_stages.json", - "source_manifest_sha256": "c0341af7166ae3a85a3c1164e7d9e880c4b4aec122f1a8fa90c73b46c596e1ea", + "source_manifest": "source_stages.json", + "source_manifest_sha256": "89a9a3be58f29d49b371b5f395b290ee168085923f28a4cba1481dd57935b76e", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -703,14 +705,8 @@ "spi_donor": "2022-23" }, "spi_prior_national_household_mass_share": 0.5, - "stage": "hmrc_spi_income", - "status": "required_at_build", - "superseded_by": { - "reason": "The FRS spine build executes hmrc_spi_income_spine, which supersedes the June retained-leaves/hmrc_spi_income pair inside source_stages.json.", - "source_manifest": "source_stages.json", - "source_manifest_sha256": "3082ac56e23f2bb383513c324dc03a5f77ef3737fa3a839ef29a662b4f1e37f4", - "stage": "hmrc_spi_income_spine" - } + "stage": "hmrc_spi_income_spine", + "status": "required_at_build" }, "lcfs_consumption": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", @@ -742,7 +738,7 @@ "required_mass_change_reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "3082ac56e23f2bb383513c324dc03a5f77ef3737fa3a839ef29a662b4f1e37f4", + "source_manifest_sha256": "89a9a3be58f29d49b371b5f395b290ee168085923f28a4cba1481dd57935b76e", "source_vintages": { "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs and the vendored Chronicle facts for NEED 2023 mean kWh, the FY2024-25 Ofgem cap levels, road-fuel outturn (DESNZ, HMRC, ONS), the DfT VEH1103 licensed-car stock, NTS bus use and the DfT and devolved bus finance tables.", "survey": "Living Costs and Food Survey 2023-24" @@ -768,7 +764,7 @@ "required_mass_change_reason": "NTS bus-travel imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "3082ac56e23f2bb383513c324dc03a5f77ef3737fa3a839ef29a662b4f1e37f4", + "source_manifest_sha256": "89a9a3be58f29d49b371b5f395b290ee168085923f28a4cba1481dd57935b76e", "source_vintages": { "source": "Department for Transport National Travel Survey, UK Data Service SN 5340 (End User Licence, 19th edition, November 2025), DOI 10.5255/UKDA-SN-5340-19; local licensed tab files (household, individual, trip); England residents only from 2013.", "survey": "National Travel Survey 2002-2024" @@ -789,7 +785,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "3082ac56e23f2bb383513c324dc03a5f77ef3737fa3a839ef29a662b4f1e37f4", + "source_manifest_sha256": "89a9a3be58f29d49b371b5f395b290ee168085923f28a4cba1481dd57935b76e", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -813,7 +809,7 @@ "employee_pension_contributions" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "3082ac56e23f2bb383513c324dc03a5f77ef3737fa3a839ef29a662b4f1e37f4", + "source_manifest_sha256": "89a9a3be58f29d49b371b5f395b290ee168085923f28a4cba1481dd57935b76e", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -835,7 +831,7 @@ "student_loan_plan" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "3082ac56e23f2bb383513c324dc03a5f77ef3737fa3a839ef29a662b4f1e37f4", + "source_manifest_sha256": "89a9a3be58f29d49b371b5f395b290ee168085923f28a4cba1481dd57935b76e", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -870,7 +866,7 @@ "required_mass_change_reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "3082ac56e23f2bb383513c324dc03a5f77ef3737fa3a839ef29a662b4f1e37f4", + "source_manifest_sha256": "89a9a3be58f29d49b371b5f395b290ee168085923f28a4cba1481dd57935b76e", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index e3bf00965..d8868927d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -4898,743 +4898,6 @@ "student_loan_plan" ], "notes": "Reported PAYE repayers are classified without a country gate. England tertiary cohorts are then topped up PLAN_5 first and PLAN_2 second to the pinned liable stocks at the FRS release calibration year; PLAN_4 is never imputed." - }, - { - "stage": "frs_hmrc_retained_leaves", - "survey": "Family Resources Survey 2024-25", - "source": "Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input", - "grain": "person", - "artifacts": [], - "operations": [ - { - "kind": "verify_certified_candidate", - "artifact": "base_candidate", - "runtime_sha256_required": true, - "fail_on_mismatch": true - }, - { - "kind": "retain_adjudicated_frs_hmrc_leaves", - "population": "certified_microcosm_uk_candidate_base_channel", - "source_vintage": "2024-25", - "mapped_build_period": 2024, - "annualization": "weekly raw FRS amounts * (365.25 / 7)", - "status": "adjudicated_partial_replay", - "retained_full_constituents": { - "hmrc_spi_pay": { - "spi_concept": "PAY", - "scope": "full", - "raw_sources": [ - "ADULT.INEARNS" - ], - "formula": "max(0, ADULT.INEARNS) * (365.25 / 7)" - }, - "hmrc_spi_unemployment_benefit_income": { - "spi_concept": "UBISJA", - "scope": "full", - "raw_sources": [ - "BENEFITS.BENEFIT=14:BENAMT", - "BENEFITS.BENEFIT=19:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT in {14, 19}) * (365.25 / 7)" - }, - "hmrc_spi_incapacity_benefit_income": { - "spi_concept": "INCPBEN", - "scope": "full", - "raw_sources": [ - "BENEFITS.BENEFIT=17:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 17) * (365.25 / 7)", - "observed_support": "structural zero in the audited 2023-24 FRS; retained so future vintages flow" - } - }, - "retained_named_subsets": { - "ossben_identifiable_subset": { - "spi_concept": "OSSBEN", - "raw_sources": [ - "BENEFITS.BENEFIT=13:BENAMT", - "BENEFITS.BENEFIT=16,VAR2 in {1,3}:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 13 or (BENEFIT == 16 and VAR2 in {1, 3})) * (365.25 / 7)", - "scope": "identifiable_subset" - }, - "srp_regular_code5": { - "spi_concept": "SRP", - "raw_sources": [ - "BENEFITS.BENEFIT=5:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 5) * (365.25 / 7)", - "scope": "regular_code5_subset" - } - }, - "source_absent_full_constituents": [ - "EPB", - "EXPS", - "TAXTERM", - "MOTHINC", - "OTHERINC" - ], - "full_concepts_forbidden_on_frs": [ - "hmrc_spi_employment_benefits", - "hmrc_spi_employment_expenses", - "hmrc_spi_taxable_termination_pay", - "hmrc_spi_miscellaneous_employment_income", - "hmrc_spi_other_income", - "hmrc_spi_other_social_security_income", - "hmrc_spi_state_pension_income" - ], - "forbid_proxy_substitution": [ - "employment_income", - "miscellaneous_income" - ], - "fail_on_missing_retained_constituent": true, - "fail_on_full_concept_alias": true - } - ], - "outputs": [ - "hmrc_spi_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_incapacity_benefit_income", - "ossben_identifiable_subset", - "srp_regular_code5" - ], - "notes": "Retains the adjudicated source-faithful FRS HMRC leaf columns before the SPI income rebuild: full PAY, UBISJA, and INCPBEN, plus explicitly named OSSBEN and SRP subsets. The runtime verifies the certified candidate before retaining these leaves." - }, - { - "stage": "hmrc_spi_income", - "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24", - "source": "https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods", - "grain": "person", - "artifacts": [ - { - "role": "qrf_donor", - "kind": "private_microdata", - "format": "tab_delimited", - "survey": "Survey of Personal Incomes Public Use Tape 2022-23", - "vintage": "2022-23", - "tax_year_start": 2022, - "ukds_study_number": "SN 9422", - "doi": "10.5255/UKDA-SN-9422-1", - "filename": "put2223uk.tab", - "sha256": "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66", - "size_bytes": 141323762, - "reviewed_source": "PolicyEngine licensed UKDS mirror (private Hugging Face repository), spi_2022_23.zip", - "access": "private_local_input", - "locator": "caller-supplied local input", - "runtime_sha256_required": true - }, - { - "role": "published_fact_surface", - "kind": "administrative_table", - "format": "ods", - "survey": "HMRC Personal Incomes Tables 3.6 and 3.7", - "publication": "https://www.gov.uk/government/statistics/personal-incomes-statistics-for-the-tax-year-2023-to-2024", - "vintage": "2023-24", - "tax_year_start": 2023, - "locator": "https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods", - "sha256": "ad063b06b2bdeef8600dbbb09d48153337a4966f8c7eea50df7a2e0304ebd73e", - "size_bytes": 166693, - "mime_type": "application/vnd.oasis.opendocument.spreadsheet", - "sheets": [ - "Table_3_6", - "Table_3_7" - ], - "mapped_build_period": 2024, - "period_mapping": "latest_published_tax_year", - "runtime_sha256_required": true - } - ], - "operations": [ - { - "kind": "verify_pinned_hmrc_source_pair", - "artifact_roles": [ - "qrf_donor", - "published_fact_surface" - ], - "require_before_source_read": true, - "runtime_sha256_required": true, - "fail_on_mismatch": true - }, - { - "kind": "replace_zero_weight_spi_support", - "existing_channel": "spi", - "require_existing_weight": 0, - "replacement_strata": [ - "clone_index", - "household_is_capital_gains_clone", - "region" - ], - "spi_prior_national_household_mass_share": 0.5, - "output_weight_kind": "importance", - "preserve_total_household_mass": true, - "require_mass_change_record": true, - "mass_change_reason": "Allocate 50% of certified UK national household prior mass to the rebuilt 2022-23 SPI support channel; total national mass is conserved.", - "fail_on_live_existing_spi_mass": true - }, - { - "kind": "strict_read_private_table", - "artifact_role": "qrf_donor", - "filename": "put2223uk.tab", - "delimiter": "\t", - "weight": "FACT", - "required_columns": [ - "AGERANGE", - "GORCODE", - "SEX", - "FACT", - "PAY", - "EPB", - "EXPS", - "TAXTERM", - "INCPBEN", - "OSSBEN", - "UBISJA", - "MOTHINC", - "OTHERINC", - "PROFITS", - "CAPALL", - "LOSSBF", - "SRP", - "INCBBS", - "DIVIDENDS", - "PENSION", - "INCPROP", - "OTHERINV", - "GIFTAID", - "GIFTINV", - "TEI", - "TII", - "TI" - ], - "runtime_sha256_required": true, - "fail_on_missing_file": true, - "fail_on_missing_columns": true, - "fail_on_invalid_weight": true - }, - { - "kind": "fit_weighted_qrf_stage1", - "training_artifact_role": "qrf_donor", - "predictors": [ - "age", - "gender", - "region" - ], - "categorical_predictors": [ - "gender", - "region" - ], - "source_sampling_weight": "FACT", - "sample_size": 100000, - "sample_with_replacement": true, - "post_sample_fit_weight": "uniform", - "fit_weight_kind": "design", - "double_apply_source_weight": false, - "source_columns": { - "self_employment_income": [ - "PROFITS", - "CAPALL", - "LOSSBF" - ], - "savings_interest_income": [ - "INCBBS" - ], - "dividend_income": [ - "DIVIDENDS" - ], - "private_pension_income": [ - "PENSION" - ], - "property_income": [ - "INCPROP" - ], - "other_investment_income": [ - "OTHERINV" - ], - "gift_aid": [ - "GIFTAID" - ], - "charitable_investment_gifts": [ - "GIFTINV" - ], - "hmrc_spi_pay": [ - "PAY" - ], - "hmrc_spi_employment_benefits": [ - "EPB" - ], - "hmrc_spi_employment_expenses": [ - "EXPS" - ], - "hmrc_spi_incapacity_benefit_income": [ - "INCPBEN" - ], - "hmrc_spi_other_social_security_income": [ - "OSSBEN" - ], - "hmrc_spi_taxable_termination_pay": [ - "TAXTERM" - ], - "hmrc_spi_unemployment_benefit_income": [ - "UBISJA" - ], - "hmrc_spi_miscellaneous_employment_income": [ - "MOTHINC" - ], - "hmrc_spi_other_income": [ - "OTHERINC" - ], - "hmrc_spi_state_pension_income": [ - "SRP" - ] - }, - "derived_policyengine_outputs": { - "employment_income": { - "source_columns": [ - "PAY", - "EPB", - "TAXTERM" - ], - "formula": "hmrc_spi_pay + hmrc_spi_employment_benefits + hmrc_spi_taxable_termination_pay", - "derive_after_draw": true - } - }, - "outputs": [ - "self_employment_income", - "savings_interest_income", - "dividend_income", - "private_pension_income", - "property_income", - "other_investment_income", - "gift_aid", - "charitable_investment_gifts", - "hmrc_spi_pay", - "hmrc_spi_employment_benefits", - "hmrc_spi_employment_expenses", - "hmrc_spi_incapacity_benefit_income", - "hmrc_spi_other_social_security_income", - "hmrc_spi_taxable_termination_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_miscellaneous_employment_income", - "hmrc_spi_other_income", - "hmrc_spi_state_pension_income" - ], - "joint_draw": true, - "savings_interest_source_semantics": "INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine gross input", - "employment_income_source_semantics": "PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; it is not the Table 3.6 measure", - "hmrc_employed_income_source_semantics": "Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels", - "self_employment_income_source_semantics": "max(0, PROFITS - CAPALL - LOSSBF)", - "assessable_income_source_semantics": "QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates and TI equals TEI + TII exactly", - "source_ti_identity_fields": [ - "TI", - "TEI", - "TII" - ], - "source_leaf_reconciliation": { - "documentation_url": "https://doc.ukdataservice.ac.uk/doc/9422/mrdoc/pdf/9422_put_2223_full_documentation.pdf", - "composite_indicator": "AGERANGE == -1", - "formulas": { - "TEI": "max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, PROFITS - CAPALL - LOSSBF)", - "TII": "OTHERINV + DIVIDENDS + INCPROP + INCBBS", - "TI": "TEI + TII" - }, - "maximum_absolute_difference_gbp": { - "ordinary": { - "TEI": 15, - "TII": 10, - "TI": 20 - }, - "composite": { - "TEI": 180, - "TII": 10, - "TI": 180 - } - }, - "rationale": "The official PUT rounds source fields, averages documented composite records, then rounds remaining income fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain exact." - }, - "ti_identity_absolute_tolerance_gbp": 5, - "stochastic_aggregates_forbidden": [ - "hmrc_spi_employed_income", - "hmrc_spi_total_earned_income", - "hmrc_spi_total_investment_income", - "hmrc_spi_assessable_income" - ], - "require_all_predictors": true, - "require_all_outputs": true - }, - { - "kind": "fit_weighted_qrf_stage2", - "training_population": "certified_microcosm_uk_candidate_base_channel", - "target_population": "rebuilt_spi_support_channel", - "predictors": [ - "age", - "gender", - "region", - "employment_income", - "self_employment_income", - "savings_interest_income", - "dividend_income", - "private_pension_income", - "property_income" - ], - "reviewed_absent_predictors": { - "other_investment_income": "This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only stage-2 predictor: the incumbent UK data build's frs_only.py defines exactly six income predictors and the certified Microcosm UK base candidate has no other_investment_income column." - }, - "categorical_predictors": [ - "gender", - "region" - ], - "weight": "household_weight", - "weight_mapping": "household_to_person", - "outputs": [ - "employee_pension_contributions", - "employer_pension_contributions", - "personal_pension_contributions", - "pension_contributions_via_salary_sacrifice", - "tax_free_savings_income", - "universal_credit_reported", - "pension_credit_reported", - "child_benefit_reported", - "housing_benefit_reported", - "income_support_reported", - "working_tax_credit_reported", - "child_tax_credit_reported", - "attendance_allowance_reported", - "state_pension_reported", - "dla_sc_reported", - "dla_m_reported", - "pip_m_reported", - "pip_dl_reported", - "sda_reported", - "carers_allowance_reported", - "iidb_reported", - "afcs_reported", - "bsp_reported", - "winter_fuel_allowance_reported", - "council_tax_benefit_reported", - "jsa_contrib_reported", - "jsa_income_reported", - "esa_contrib_reported", - "esa_income_reported" - ], - "reviewed_absent_outputs": { - "incapacity_benefit_reported": "Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not a populated loader layer.", - "maternity_allowance_reported": "Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training source can be materialized for this stage." - }, - "postprocess": { - "gross_savings_interest_income": "stage1 INCBBS draw + stage2 tax_free_savings_income", - "refresh_disability_categories": [ - "aa_category", - "dla_sc_category", - "dla_m_category", - "pip_m_category", - "pip_dl_category" - ], - "refresh_disability_flags": [ - "is_disabled_for_benefits", - "is_enhanced_disabled_for_benefits", - "is_severely_disabled_for_benefits" - ] - }, - "joint_draw": true, - "require_all_predictors": true, - "require_all_materializable_outputs": true, - "require_all_outputs": false - }, - { - "kind": "materialize_hmrc_income_bands_fail_closed", - "artifact_role": "published_fact_surface", - "mapped_build_period": 2024, - "period_mapping": "latest_published_tax_year", - "column_index_base": 0, - "data_row_start_index": 5, - "stop_label": "All ranges", - "count_unit_multiplier": 1000, - "amount_unit_multiplier": 1000000, - "component_columns": { - "employment_income": { - "sheet": "Table_3_6", - "count_column_index": 4, - "amount_column_index": 5 - }, - "self_employment_income": { - "sheet": "Table_3_6", - "count_column_index": 1, - "amount_column_index": 2 - }, - "state_pension": { - "sheet": "Table_3_6", - "count_column_index": 7, - "amount_column_index": 8 - }, - "private_pension_income": { - "sheet": "Table_3_6", - "count_column_index": 10, - "amount_column_index": 11 - }, - "property_income": { - "sheet": "Table_3_7", - "count_column_index": 1, - "amount_column_index": 2 - }, - "savings_interest_income": { - "sheet": "Table_3_7", - "count_column_index": 4, - "amount_column_index": 5 - }, - "dividend_income": { - "sheet": "Table_3_7", - "count_column_index": 7, - "amount_column_index": 8 - }, - "other_investment_income": { - "sheet": "Table_3_7", - "count_column_index": 10, - "amount_column_index": 11 - } - }, - "required_band_lower_bounds_gbp": [ - 12570, - 15000, - 20000, - 30000, - 40000, - 50000, - 70000, - 100000, - 150000, - 200000, - 300000, - 500000, - 1000000 - ], - "required_measures": [ - "count", - "amount" - ], - "fail_on_missing_sheet": true, - "fail_on_missing_component": true, - "fail_on_missing_band": true, - "fail_on_non_numeric_value": true - }, - { - "kind": "classify_hmrc_income_facts_with_reviewed_fences", - "target_operation": "materialize_hmrc_income_bands_fail_closed", - "components": [ - "employment_income", - "self_employment_income", - "state_pension", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "other_investment_income" - ], - "breakdown_dependency": "hmrc_spi_assessable_income", - "frs_breakdown_status": "unavailable_full_measure", - "input_weight_kind": "importance", - "output_weight_kind": "importance", - "calibration_permitted": false, - "required_fact_count": 208, - "outcome_counts": { - "exact_pass": 0, - "exact_fail": 0, - "directional_pass": 0, - "directional_fail": 0, - "excluded_with_fence": 208 - }, - "classification_rationale": "Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound is valid.", - "reviewed_fences": [ - { - "fence_id": "frs_epb_source_absent", - "constituents": [ - "EPB" - ], - "raw_sources_searched": [ - "JOB.EXPBEN01-EXPBEN13", - "JOB.CARVAL", - "JOB.CARAMT", - "JOB.FUELAMT", - "JOB.VCHAMT", - "JOB.CHVAMT" - ], - "finding": "Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce complete taxable expenses payments and benefits.", - "mass_implication": "12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this is not monetary support.", - "rationale": "Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an imputation or proxy.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_exps_source_absent", - "constituents": [ - "EXPS" - ], - "raw_sources_searched": [ - "JOB.EXPBEN04/EXPBEN05", - "JOB.MILEAMT/JOB.MOTAMT", - "JOB.UMILEAMT/JOB.UMOTAMT", - "JOB.DEDUC1-DEDUC9", - "JOB.UDEDUC1-UDEDUC9" - ], - "finding": "Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense amount required by SPI.", - "mass_implication": "5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the true EXPS mass is not estimable.", - "rationale": "The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity with a negative sign.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_taxterm_source_absent", - "constituents": [ - "TAXTERM" - ], - "raw_sources_searched": [ - "ADULT.REDAMT", - "ADULT and JOB taxable-termination split search" - ], - "finding": "Missing. REDAMT is gross redundancy pay and has neither the taxable amount nor non-redundancy termination pay.", - "mass_implication": "0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable mass is unknown.", - "rationale": "Gross redundancy pay cannot be relabeled as taxable termination pay.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_mothinc_source_absent", - "constituents": [ - "MOTHINC" - ], - "raw_sources_searched": [ - "ODDJOB.OJAMT/ODDJOB.OJNOW", - "ADULT.ALLPAY2", - "ADULT.ROYYR2-ROYYR4", - "JOB.OWNOTHER" - ], - "finding": "Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI miscellaneous employment income would be a proxy.", - "mass_implication": "Odd-job-only effective person mass is 0.1724207%; the broader unresolved miscellaneous pool is 1.4650566%.", - "rationale": "The FRS instrument cannot separate the SPI miscellaneous-employment concept source-faithfully.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_otherinc_source_absent", - "constituents": [ - "OTHERINC" - ], - "raw_sources_searched": [ - "ADULT, ODDJOB, and JOB miscellaneous fields", - "PENSION", - "ACCOUNTS", - "ASSETS", - "BENEFITS" - ], - "finding": "Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be split between MOTHINC and OTHERINC from source evidence.", - "mass_implication": "No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate FRS effective person mass.", - "rationale": "A union of heterogeneous residual fields would be a new proxy, not a retained source constituent.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_ossben_identifiable_subset", - "constituents": [ - "OSSBEN", - "ossben_identifiable_subset" - ], - "raw_sources_searched": [ - "BENEFITS.BENAMT", - "BENEFITS.BENEFIT", - "BENEFITS.VAR2", - "BENEFITS codes 13, 16, 6, and 30" - ], - "finding": "Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted.", - "mass_implication": "1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; it is not full OSSBEN support.", - "rationale": "The retained column must remain explicitly named as a subset and cannot satisfy the full SPI concept.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_srp_regular_code5_subset", - "constituents": [ - "SRP", - "srp_regular_code5" - ], - "raw_sources_searched": [ - "BENEFITS.BENAMT where BENEFIT == 5", - "BENEFITS codes 6 and 9" - ], - "finding": "Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension.", - "mass_implication": "18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; it is not complete SRP support.", - "rationale": "The retained column must remain explicitly named as a subset and cannot be reported as the full published state-pension measure.", - "dependent_fence_ids": [] - }, - { - "fence_id": "full_frs_tei_band_unavailable", - "constituents": [ - "EPB", - "EXPS", - "TAXTERM", - "MOTHINC", - "OTHERINC", - "OSSBEN", - "SRP" - ], - "raw_sources_searched": [], - "finding": "The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income band assignment is unavailable on the FRS channel.", - "mass_implication": "Every one of the 208 published facts is banded by total income and therefore depends on this unavailable like-for-like measure.", - "rationale": "A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.", - "dependent_fence_ids": [ - "frs_epb_source_absent", - "frs_exps_source_absent", - "frs_taxterm_source_absent", - "frs_mothinc_source_absent", - "frs_otherinc_source_absent", - "frs_ossben_identifiable_subset", - "frs_srp_regular_code5_subset" - ] - } - ], - "fact_fence_id": "full_frs_tei_band_unavailable", - "blocked_dependency": "hmrc_spi_assessable_income", - "fail_on_unfenced_exclusion": true, - "fail_on_fact_count_mismatch": true, - "forbid_biased_estimate_or_delta": true - }, - { - "kind": "gate_distributional_effective_mass", - "columns": [ - "gift_aid", - "charitable_investment_gifts" - ], - "weight": "household_weight", - "weight_mapping": "household_to_person", - "support_channel_column": "person_support_channel", - "required_support_channel": "spi", - "mass_share_denominator": "all_person_effective_mass", - "minimum_nondefault_mass_share": 1e-06, - "fail_below_floor": true - } - ], - "official_table_components": [ - "employment_income", - "self_employment_income", - "state_pension", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "other_investment_income" - ], - "donor_relief_outputs": [ - "gift_aid", - "charitable_investment_gifts" - ], - "outputs": [ - "employment_income", - "self_employment_income", - "hmrc_spi_state_pension_income", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "other_investment_income", - "gift_aid", - "charitable_investment_gifts", - "hmrc_spi_employed_income", - "hmrc_spi_total_earned_income", - "hmrc_spi_total_investment_income", - "hmrc_spi_assessable_income" - ], - "notes": "Current-source adjudicated replay contract: the private 2022-23 SPI donor and public 2023-24 HMRC ODS are pinned by reviewed SHA-256 and size and verified together before either is opened. The QRF draws source leaves; HMRC employed income, TEI, TII, and TI are deterministic post-draw aggregates on the SPI channel, with TI exactly equal to TEI + TII. PolicyEngine employment_income remains the narrow PAY + EPB + TAXTERM input on SPI rows. Stage 2 mirrors the incumbent UK data build's frs_only.py exactly: its income predictors are employment, self-employment, savings interest, dividends, private pension, and property income. Other investment income remains a stage-1 SPI draw and official HMRC fact component, but is excluded from stage 2 because the certified FRS candidate does not carry it. The FRS channel retains source-faithful full PAY, UBISJA, and INCPBEN plus explicitly named ossben_identifiable_subset and srp_regular_code5; EPB, EXPS, TAXTERM, MOTHINC, OTHERINC, full OSSBEN, and full SRP remain forbidden. Because the missing legs prevent a complete FRS TEI measure, none of the 208 non-overlapping total-income-band facts is exact or directional. Every fact is an excluded-with-fence record, no calibration is performed, and weights remain importance-kind. Gift Aid restoration still requires the rebuilt positive-mass SPI channel to clear the reviewed 1ppm effective-mass floor." } ] } diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 807ceb643..1c9cc939e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -3913,600 +3913,3 @@ stages: rewrites: - student_loan_plan notes: Reported PAYE repayers are classified without a country gate. England tertiary cohorts are then topped up PLAN_5 first and PLAN_2 second to the pinned liable stocks at the FRS release calibration year; PLAN_4 is never imputed. -- stage: frs_hmrc_retained_leaves - survey: Family Resources Survey 2024-25 - source: Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input - grain: person - artifacts: [] - operations: - - kind: verify_certified_candidate - artifact: base_candidate - runtime_sha256_required: true - fail_on_mismatch: true - - kind: retain_adjudicated_frs_hmrc_leaves - population: certified_microcosm_uk_candidate_base_channel - source_vintage: 2024-25 - mapped_build_period: 2024 - annualization: weekly raw FRS amounts * (365.25 / 7) - status: adjudicated_partial_replay - retained_full_constituents: - hmrc_spi_pay: - spi_concept: PAY - scope: full - raw_sources: - - ADULT.INEARNS - formula: max(0, ADULT.INEARNS) * (365.25 / 7) - hmrc_spi_unemployment_benefit_income: - spi_concept: UBISJA - scope: full - raw_sources: - - BENEFITS.BENEFIT=14:BENAMT - - BENEFITS.BENEFIT=19:BENAMT - formula: sum(BENAMT where BENEFIT in {14, 19}) * (365.25 / 7) - hmrc_spi_incapacity_benefit_income: - spi_concept: INCPBEN - scope: full - raw_sources: - - BENEFITS.BENEFIT=17:BENAMT - formula: sum(BENAMT where BENEFIT == 17) * (365.25 / 7) - observed_support: structural zero in the audited 2023-24 FRS; retained so future vintages flow - retained_named_subsets: - ossben_identifiable_subset: - spi_concept: OSSBEN - raw_sources: - - BENEFITS.BENEFIT=13:BENAMT - - BENEFITS.BENEFIT=16,VAR2 in {1,3}:BENAMT - formula: sum(BENAMT where BENEFIT == 13 or (BENEFIT == 16 and VAR2 in {1, 3})) * (365.25 / 7) - scope: identifiable_subset - srp_regular_code5: - spi_concept: SRP - raw_sources: - - BENEFITS.BENEFIT=5:BENAMT - formula: sum(BENAMT where BENEFIT == 5) * (365.25 / 7) - scope: regular_code5_subset - source_absent_full_constituents: - - EPB - - EXPS - - TAXTERM - - MOTHINC - - OTHERINC - full_concepts_forbidden_on_frs: - - hmrc_spi_employment_benefits - - hmrc_spi_employment_expenses - - hmrc_spi_taxable_termination_pay - - hmrc_spi_miscellaneous_employment_income - - hmrc_spi_other_income - - hmrc_spi_other_social_security_income - - hmrc_spi_state_pension_income - forbid_proxy_substitution: - - employment_income - - miscellaneous_income - fail_on_missing_retained_constituent: true - fail_on_full_concept_alias: true - outputs: - - hmrc_spi_pay - - hmrc_spi_unemployment_benefit_income - - hmrc_spi_incapacity_benefit_income - - ossben_identifiable_subset - - srp_regular_code5 - notes: 'Retains the adjudicated source-faithful FRS HMRC leaf columns before the SPI income rebuild: full PAY, UBISJA, and INCPBEN, plus explicitly named OSSBEN and SRP subsets. The runtime verifies the certified candidate before retaining these leaves.' -- stage: hmrc_spi_income - survey: Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24 - source: https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods - grain: person - artifacts: - - role: qrf_donor - kind: private_microdata - format: tab_delimited - survey: Survey of Personal Incomes Public Use Tape 2022-23 - vintage: 2022-23 - tax_year_start: 2022 - ukds_study_number: SN 9422 - doi: 10.5255/UKDA-SN-9422-1 - filename: put2223uk.tab - sha256: 5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66 - size_bytes: 141323762 - reviewed_source: PolicyEngine licensed UKDS mirror (private Hugging Face repository), spi_2022_23.zip - access: private_local_input - locator: caller-supplied local input - runtime_sha256_required: true - - role: published_fact_surface - kind: administrative_table - format: ods - survey: HMRC Personal Incomes Tables 3.6 and 3.7 - publication: https://www.gov.uk/government/statistics/personal-incomes-statistics-for-the-tax-year-2023-to-2024 - vintage: 2023-24 - tax_year_start: 2023 - locator: https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods - sha256: ad063b06b2bdeef8600dbbb09d48153337a4966f8c7eea50df7a2e0304ebd73e - size_bytes: 166693 - mime_type: application/vnd.oasis.opendocument.spreadsheet - sheets: - - Table_3_6 - - Table_3_7 - mapped_build_period: 2024 - period_mapping: latest_published_tax_year - runtime_sha256_required: true - operations: - - kind: verify_pinned_hmrc_source_pair - artifact_roles: - - qrf_donor - - published_fact_surface - require_before_source_read: true - runtime_sha256_required: true - fail_on_mismatch: true - - kind: replace_zero_weight_spi_support - existing_channel: spi - require_existing_weight: 0 - replacement_strata: - - clone_index - - household_is_capital_gains_clone - - region - spi_prior_national_household_mass_share: 0.5 - output_weight_kind: importance - preserve_total_household_mass: true - require_mass_change_record: true - mass_change_reason: Allocate 50% of certified UK national household prior mass to the rebuilt 2022-23 SPI support channel; total national mass is conserved. - fail_on_live_existing_spi_mass: true - - kind: strict_read_private_table - artifact_role: qrf_donor - filename: put2223uk.tab - delimiter: "\t" - weight: FACT - required_columns: - - AGERANGE - - GORCODE - - SEX - - FACT - - PAY - - EPB - - EXPS - - TAXTERM - - INCPBEN - - OSSBEN - - UBISJA - - MOTHINC - - OTHERINC - - PROFITS - - CAPALL - - LOSSBF - - SRP - - INCBBS - - DIVIDENDS - - PENSION - - INCPROP - - OTHERINV - - GIFTAID - - GIFTINV - - TEI - - TII - - TI - runtime_sha256_required: true - fail_on_missing_file: true - fail_on_missing_columns: true - fail_on_invalid_weight: true - - kind: fit_weighted_qrf_stage1 - training_artifact_role: qrf_donor - predictors: - - age - - gender - - region - categorical_predictors: - - gender - - region - source_sampling_weight: FACT - sample_size: 100000 - sample_with_replacement: true - post_sample_fit_weight: uniform - fit_weight_kind: design - double_apply_source_weight: false - source_columns: - self_employment_income: - - PROFITS - - CAPALL - - LOSSBF - savings_interest_income: - - INCBBS - dividend_income: - - DIVIDENDS - private_pension_income: - - PENSION - property_income: - - INCPROP - other_investment_income: - - OTHERINV - gift_aid: - - GIFTAID - charitable_investment_gifts: - - GIFTINV - hmrc_spi_pay: - - PAY - hmrc_spi_employment_benefits: - - EPB - hmrc_spi_employment_expenses: - - EXPS - hmrc_spi_incapacity_benefit_income: - - INCPBEN - hmrc_spi_other_social_security_income: - - OSSBEN - hmrc_spi_taxable_termination_pay: - - TAXTERM - hmrc_spi_unemployment_benefit_income: - - UBISJA - hmrc_spi_miscellaneous_employment_income: - - MOTHINC - hmrc_spi_other_income: - - OTHERINC - hmrc_spi_state_pension_income: - - SRP - derived_policyengine_outputs: - employment_income: - source_columns: - - PAY - - EPB - - TAXTERM - formula: hmrc_spi_pay + hmrc_spi_employment_benefits + hmrc_spi_taxable_termination_pay - derive_after_draw: true - outputs: - - self_employment_income - - savings_interest_income - - dividend_income - - private_pension_income - - property_income - - other_investment_income - - gift_aid - - charitable_investment_gifts - - hmrc_spi_pay - - hmrc_spi_employment_benefits - - hmrc_spi_employment_expenses - - hmrc_spi_incapacity_benefit_income - - hmrc_spi_other_social_security_income - - hmrc_spi_taxable_termination_pay - - hmrc_spi_unemployment_benefit_income - - hmrc_spi_miscellaneous_employment_income - - hmrc_spi_other_income - - hmrc_spi_state_pension_income - joint_draw: true - savings_interest_source_semantics: INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine gross input - employment_income_source_semantics: PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; it is not the Table 3.6 measure - hmrc_employed_income_source_semantics: Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels - self_employment_income_source_semantics: max(0, PROFITS - CAPALL - LOSSBF) - assessable_income_source_semantics: QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates and TI equals TEI + TII exactly - source_ti_identity_fields: - - TI - - TEI - - TII - source_leaf_reconciliation: - documentation_url: https://doc.ukdataservice.ac.uk/doc/9422/mrdoc/pdf/9422_put_2223_full_documentation.pdf - composite_indicator: AGERANGE == -1 - formulas: - TEI: max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, PROFITS - CAPALL - LOSSBF) - TII: OTHERINV + DIVIDENDS + INCPROP + INCBBS - TI: TEI + TII - maximum_absolute_difference_gbp: - ordinary: - TEI: 15 - TII: 10 - TI: 20 - composite: - TEI: 180 - TII: 10 - TI: 180 - rationale: The official PUT rounds source fields, averages documented composite records, then rounds remaining income fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain exact. - ti_identity_absolute_tolerance_gbp: 5 - stochastic_aggregates_forbidden: - - hmrc_spi_employed_income - - hmrc_spi_total_earned_income - - hmrc_spi_total_investment_income - - hmrc_spi_assessable_income - require_all_predictors: true - require_all_outputs: true - - kind: fit_weighted_qrf_stage2 - training_population: certified_microcosm_uk_candidate_base_channel - target_population: rebuilt_spi_support_channel - predictors: - - age - - gender - - region - - employment_income - - self_employment_income - - savings_interest_income - - dividend_income - - private_pension_income - - property_income - reviewed_absent_predictors: - other_investment_income: 'This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only stage-2 predictor: the incumbent UK data build''s frs_only.py defines exactly six income predictors and the certified Microcosm UK base candidate has no other_investment_income column.' - categorical_predictors: - - gender - - region - weight: household_weight - weight_mapping: household_to_person - outputs: - - employee_pension_contributions - - employer_pension_contributions - - personal_pension_contributions - - pension_contributions_via_salary_sacrifice - - tax_free_savings_income - - universal_credit_reported - - pension_credit_reported - - child_benefit_reported - - housing_benefit_reported - - income_support_reported - - working_tax_credit_reported - - child_tax_credit_reported - - attendance_allowance_reported - - state_pension_reported - - dla_sc_reported - - dla_m_reported - - pip_m_reported - - pip_dl_reported - - sda_reported - - carers_allowance_reported - - iidb_reported - - afcs_reported - - bsp_reported - - winter_fuel_allowance_reported - - council_tax_benefit_reported - - jsa_contrib_reported - - jsa_income_reported - - esa_contrib_reported - - esa_income_reported - reviewed_absent_outputs: - incapacity_benefit_reported: Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not a populated loader layer. - maternity_allowance_reported: Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training source can be materialized for this stage. - postprocess: - gross_savings_interest_income: stage1 INCBBS draw + stage2 tax_free_savings_income - refresh_disability_categories: - - aa_category - - dla_sc_category - - dla_m_category - - pip_m_category - - pip_dl_category - refresh_disability_flags: - - is_disabled_for_benefits - - is_enhanced_disabled_for_benefits - - is_severely_disabled_for_benefits - joint_draw: true - require_all_predictors: true - require_all_materializable_outputs: true - require_all_outputs: false - - kind: materialize_hmrc_income_bands_fail_closed - artifact_role: published_fact_surface - mapped_build_period: 2024 - period_mapping: latest_published_tax_year - column_index_base: 0 - data_row_start_index: 5 - stop_label: All ranges - count_unit_multiplier: 1000 - amount_unit_multiplier: 1000000 - component_columns: - employment_income: - sheet: Table_3_6 - count_column_index: 4 - amount_column_index: 5 - self_employment_income: - sheet: Table_3_6 - count_column_index: 1 - amount_column_index: 2 - state_pension: - sheet: Table_3_6 - count_column_index: 7 - amount_column_index: 8 - private_pension_income: - sheet: Table_3_6 - count_column_index: 10 - amount_column_index: 11 - property_income: - sheet: Table_3_7 - count_column_index: 1 - amount_column_index: 2 - savings_interest_income: - sheet: Table_3_7 - count_column_index: 4 - amount_column_index: 5 - dividend_income: - sheet: Table_3_7 - count_column_index: 7 - amount_column_index: 8 - other_investment_income: - sheet: Table_3_7 - count_column_index: 10 - amount_column_index: 11 - required_band_lower_bounds_gbp: - - 12570 - - 15000 - - 20000 - - 30000 - - 40000 - - 50000 - - 70000 - - 100000 - - 150000 - - 200000 - - 300000 - - 500000 - - 1000000 - required_measures: - - count - - amount - fail_on_missing_sheet: true - fail_on_missing_component: true - fail_on_missing_band: true - fail_on_non_numeric_value: true - - kind: classify_hmrc_income_facts_with_reviewed_fences - target_operation: materialize_hmrc_income_bands_fail_closed - components: - - employment_income - - self_employment_income - - state_pension - - private_pension_income - - property_income - - savings_interest_income - - dividend_income - - other_investment_income - breakdown_dependency: hmrc_spi_assessable_income - frs_breakdown_status: unavailable_full_measure - input_weight_kind: importance - output_weight_kind: importance - calibration_permitted: false - required_fact_count: 208 - outcome_counts: - exact_pass: 0 - exact_fail: 0 - directional_pass: 0 - directional_fail: 0 - excluded_with_fence: 208 - classification_rationale: Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound is valid. - reviewed_fences: - - fence_id: frs_epb_source_absent - constituents: - - EPB - raw_sources_searched: - - JOB.EXPBEN01-EXPBEN13 - - JOB.CARVAL - - JOB.CARAMT - - JOB.FUELAMT - - JOB.VCHAMT - - JOB.CHVAMT - finding: Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce complete taxable expenses payments and benefits. - mass_implication: 12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this is not monetary support. - rationale: Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an imputation or proxy. - dependent_fence_ids: [] - - fence_id: frs_exps_source_absent - constituents: - - EXPS - raw_sources_searched: - - JOB.EXPBEN04/EXPBEN05 - - JOB.MILEAMT/JOB.MOTAMT - - JOB.UMILEAMT/JOB.UMOTAMT - - JOB.DEDUC1-DEDUC9 - - JOB.UDEDUC1-UDEDUC9 - finding: Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense amount required by SPI. - mass_implication: 5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the true EXPS mass is not estimable. - rationale: The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity with a negative sign. - dependent_fence_ids: [] - - fence_id: frs_taxterm_source_absent - constituents: - - TAXTERM - raw_sources_searched: - - ADULT.REDAMT - - ADULT and JOB taxable-termination split search - finding: Missing. REDAMT is gross redundancy pay and has neither the taxable amount nor non-redundancy termination pay. - mass_implication: 0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable mass is unknown. - rationale: Gross redundancy pay cannot be relabeled as taxable termination pay. - dependent_fence_ids: [] - - fence_id: frs_mothinc_source_absent - constituents: - - MOTHINC - raw_sources_searched: - - ODDJOB.OJAMT/ODDJOB.OJNOW - - ADULT.ALLPAY2 - - ADULT.ROYYR2-ROYYR4 - - JOB.OWNOTHER - finding: Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI miscellaneous employment income would be a proxy. - mass_implication: Odd-job-only effective person mass is 0.1724207%; the broader unresolved miscellaneous pool is 1.4650566%. - rationale: The FRS instrument cannot separate the SPI miscellaneous-employment concept source-faithfully. - dependent_fence_ids: [] - - fence_id: frs_otherinc_source_absent - constituents: - - OTHERINC - raw_sources_searched: - - ADULT, ODDJOB, and JOB miscellaneous fields - - PENSION - - ACCOUNTS - - ASSETS - - BENEFITS - finding: Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be split between MOTHINC and OTHERINC from source evidence. - mass_implication: No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate FRS effective person mass. - rationale: A union of heterogeneous residual fields would be a new proxy, not a retained source constituent. - dependent_fence_ids: [] - - fence_id: frs_ossben_identifiable_subset - constituents: - - OSSBEN - - ossben_identifiable_subset - raw_sources_searched: - - BENEFITS.BENAMT - - BENEFITS.BENEFIT - - BENEFITS.VAR2 - - BENEFITS codes 13, 16, 6, and 30 - finding: Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted. - mass_implication: 1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; it is not full OSSBEN support. - rationale: The retained column must remain explicitly named as a subset and cannot satisfy the full SPI concept. - dependent_fence_ids: [] - - fence_id: frs_srp_regular_code5_subset - constituents: - - SRP - - srp_regular_code5 - raw_sources_searched: - - BENEFITS.BENAMT where BENEFIT == 5 - - BENEFITS codes 6 and 9 - finding: Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension. - mass_implication: 18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; it is not complete SRP support. - rationale: The retained column must remain explicitly named as a subset and cannot be reported as the full published state-pension measure. - dependent_fence_ids: [] - - fence_id: full_frs_tei_band_unavailable - constituents: - - EPB - - EXPS - - TAXTERM - - MOTHINC - - OTHERINC - - OSSBEN - - SRP - raw_sources_searched: [] - finding: The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income band assignment is unavailable on the FRS channel. - mass_implication: Every one of the 208 published facts is banded by total income and therefore depends on this unavailable like-for-like measure. - rationale: 'A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.' - dependent_fence_ids: - - frs_epb_source_absent - - frs_exps_source_absent - - frs_taxterm_source_absent - - frs_mothinc_source_absent - - frs_otherinc_source_absent - - frs_ossben_identifiable_subset - - frs_srp_regular_code5_subset - fact_fence_id: full_frs_tei_band_unavailable - blocked_dependency: hmrc_spi_assessable_income - fail_on_unfenced_exclusion: true - fail_on_fact_count_mismatch: true - forbid_biased_estimate_or_delta: true - - kind: gate_distributional_effective_mass - columns: - - gift_aid - - charitable_investment_gifts - weight: household_weight - weight_mapping: household_to_person - support_channel_column: person_support_channel - required_support_channel: spi - mass_share_denominator: all_person_effective_mass - minimum_nondefault_mass_share: 1.0e-06 - fail_below_floor: true - official_table_components: - - employment_income - - self_employment_income - - state_pension - - private_pension_income - - property_income - - savings_interest_income - - dividend_income - - other_investment_income - donor_relief_outputs: - - gift_aid - - charitable_investment_gifts - outputs: - - employment_income - - self_employment_income - - hmrc_spi_state_pension_income - - private_pension_income - - property_income - - savings_interest_income - - dividend_income - - other_investment_income - - gift_aid - - charitable_investment_gifts - - hmrc_spi_employed_income - - hmrc_spi_total_earned_income - - hmrc_spi_total_investment_income - - hmrc_spi_assessable_income - notes: 'Current-source adjudicated replay contract: the private 2022-23 SPI donor and public 2023-24 HMRC ODS are pinned by reviewed SHA-256 and size and verified together before either is opened. The QRF draws source leaves; HMRC employed income, TEI, TII, and TI are deterministic post-draw aggregates on the SPI channel, with TI exactly equal to TEI + TII. PolicyEngine employment_income remains the narrow PAY + EPB + TAXTERM input on SPI rows. Stage 2 mirrors the incumbent UK data build''s frs_only.py exactly: its income predictors are employment, self-employment, savings interest, dividends, private pension, and property income. Other investment income remains a stage-1 SPI draw and official HMRC fact component, but is excluded from stage 2 because the certified FRS candidate does not carry it. The FRS channel retains source-faithful full PAY, UBISJA, and INCPBEN plus explicitly named ossben_identifiable_subset and srp_regular_code5; EPB, EXPS, TAXTERM, MOTHINC, OTHERINC, full OSSBEN, - and full SRP remain forbidden. Because the missing legs prevent a complete FRS TEI measure, none of the 208 non-overlapping total-income-band facts is exact or directional. Every fact is an excluded-with-fence record, no calibration is performed, and weights remain importance-kind. Gift Aid restoration still requires the rebuilt positive-mass SPI channel to clear the reviewed 1ppm effective-mass floor.' diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index b6961763e..4ff9386d7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -125,17 +125,13 @@ add_frs_employment, derive_frs_employment, ) -from microcosm.build.uk_runtime.frs_hmrc_leaves import ( +from microcosm.build.uk_runtime.frs_hmrc_source import ( FRS_HMRC_INCPBEN_COLUMN, FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, FRS_HMRC_PAY_COLUMN, FRS_HMRC_RETAINED_LEAF_COLUMNS, - FRS_HMRC_RETAINED_LEAVES_STAGE_NAME, FRS_HMRC_SRP_REGULAR_CODE5_COLUMN, FRS_HMRC_UBISJA_COLUMN, - UKFRSHMRCRetainedLeavesResult, - UKFRSHMRCRetainedLeavesStageTransform, - retain_uk_frs_hmrc_leaves, ) from microcosm.build.uk_runtime.frs_legacy_proxies import ( FRS_LEGACY_PROXY_OUTPUT_COLUMNS, @@ -269,7 +265,6 @@ ) from microcosm.build.uk_runtime.hmrc_source_contract import ( HMRC_DISTRIBUTIONAL_INPUTS, - UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE, assert_uk_hmrc_income_source_contract_current, ) from microcosm.build.uk_runtime.ladder_targets import ( @@ -682,7 +677,6 @@ "FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN", "FRS_HMRC_PAY_COLUMN", "FRS_HMRC_RETAINED_LEAF_COLUMNS", - "FRS_HMRC_RETAINED_LEAVES_STAGE_NAME", "FRS_HMRC_SRP_REGULAR_CODE5_COLUMN", "FRS_HMRC_UBISJA_COLUMN", "FRS_REGION_TO_COUNTRY", @@ -750,7 +744,6 @@ "SPI_SYNTHETIC_SUPPORT_CHANNEL", "UK_ENGLAND_WALES_REGION_CODES", "UK_GEOGRAPHY_LADDER_COLUMNS", - "UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE", "UK_LONDON_REGION_CODE", "UK_LOADER_INPUT_ALIASES", "UK_OA_LADDER_DERIVED_LAYERS", @@ -769,8 +762,6 @@ "UKFirmTargetLayout", "UKFirmVATRuleEvaluator", "UKFirmValidationReport", - "UKFRSHMRCRetainedLeavesResult", - "UKFRSHMRCRetainedLeavesStageTransform", "UKLadderRowwiseDatasetResult", "UKLocalSolveDoctrine", "UKRowwiseLocalMatrix", @@ -855,7 +846,6 @@ "classify_hmrc_replay_targets", "compute_household_metrics", "constituency_household_targets", - "ladder_vs_chronicle_household_dispersion", "compile_uk_local_target_registry", "load_uk_local_area_crosswalk", "create_uk_spi_support_tables", @@ -874,7 +864,6 @@ "frozen_vs_recomputed", "impute_uk_spi_income_support", "replace_uk_spi_support_tables", - "retain_uk_frs_hmrc_leaves", "add_frs_council_tax", "add_frs_disability", "add_frs_education", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py index 9a41de5e1..cc801db76 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py @@ -89,7 +89,7 @@ def build_uk_country_graph( from microcosm.graph import compile_graph from microcosm.graph.canonical import canonical_json - from .graph import UK_SPINE_EXCLUSIONS, uk_spine_graph + from .graph import uk_spine_graph from .graph_build import UKFullBuildConfig, uk_full_graph from .graph_evidence import add_uk_spine_gate_nodes from .graph_terminal import append_uk_full_gate_nodes @@ -123,15 +123,8 @@ def build_uk_country_graph( graph = append_uk_full_gate_nodes( full.graph, calibration=full.calibration, - # The same roster uk_spine_graph builds: main still declares the - # retired HMRC pair in the manifest and keeps it out of the spine - # through UK_SPINE_EXCLUSIONS, so the preflight binds evidence only - # from stages the spine graph actually runs. - spine_stage_names=tuple( - stage.stage - for stage in spec.sources.stages - if stage.stage not in UK_SPINE_EXCLUSIONS - ), + # The same roster uk_spine_graph builds: every manifest stage. + spine_stage_names=tuple(stage.stage for stage in spec.sources.stages), engine_identity=engine_identity, review_date=review_date, sample_fraction=config.effective_sample_fraction, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_leaves.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_leaves.py deleted file mode 100644 index 2154f2848..000000000 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_leaves.py +++ /dev/null @@ -1,893 +0,0 @@ -"""Retain adjudicated raw-FRS leaves for the UK HMRC income surface. - -The certified UK candidate was built from the 2023-24 FRS, but it does not -retain the raw constituents needed to compare its income measure with the -published HMRC tables. This stage reopens only ``adult.tab`` and -``benefits.tab`` and carries the source-faithful, adjudicated constituents -through the candidate's SPI, capital-gains, and geography-clone descendants. - -The two partial concepts deliberately keep subset names. They must never be -mistaken for the full SPI ``OSSBEN`` or ``SRP`` concepts. -""" - -from __future__ import annotations - -import hashlib -from collections.abc import Mapping -from dataclasses import dataclass, field -from pathlib import Path - -import numpy as np -import pandas as pd - -from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity -from microcosm.build.uk_runtime.national_frame import ( - uk_household_weight_kind, - uk_national_frame, - uk_time_period, - validate_uk_national_frame, -) -from microcosm.build.uk_runtime.spi_support import ( - HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - SPI_HMRC_INCAPACITY_BENEFIT_INCOME_COLUMN, - SPI_HMRC_PAY_COLUMN, - SPI_HMRC_UNEMPLOYMENT_BENEFIT_INCOME_COLUMN, -) -from microcosm.frame import Frame - -__all__ = [ - "FRS_HMRC_INCPBEN_COLUMN", - "FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN", - "FRS_HMRC_PAY_COLUMN", - "FRS_HMRC_RETAINED_LEAF_COLUMNS", - "FRS_HMRC_RETAINED_LEAF_SOURCE_EVIDENCE", - "FRS_HMRC_SRP_REGULAR_CODE5_COLUMN", - "FRS_HMRC_UBISJA_COLUMN", - "FRS_HMRC_RETAINED_LEAVES_STAGE_NAME", - "UKFRSHMRCRetainedLeavesResult", - "UKFRSHMRCRetainedLeavesStageTransform", - "UKFRSRawTableIdentity", - "retain_uk_frs_hmrc_leaves", -] - -FRS_SOURCE_VINTAGE = "2023-24" -FRS_SOURCE_BUILD_PERIOD = "2023" -FRS_WEEKS_IN_YEAR = 365.25 / 7 -FRS_HMRC_RETAINED_LEAVES_STAGE_NAME = "frs_hmrc_retained_leaves" - -# Full concepts use the normalized columns already consumed by the SPI/HMRC -# stage. Partial concepts are fenced under their adjudicated subset names. -FRS_HMRC_PAY_COLUMN = SPI_HMRC_PAY_COLUMN -FRS_HMRC_UBISJA_COLUMN = SPI_HMRC_UNEMPLOYMENT_BENEFIT_INCOME_COLUMN -FRS_HMRC_INCPBEN_COLUMN = SPI_HMRC_INCAPACITY_BENEFIT_INCOME_COLUMN -FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN = "ossben_identifiable_subset" -FRS_HMRC_SRP_REGULAR_CODE5_COLUMN = "srp_regular_code5" -FRS_HMRC_RETAINED_LEAF_COLUMNS = ( - FRS_HMRC_PAY_COLUMN, - FRS_HMRC_UBISJA_COLUMN, - FRS_HMRC_INCPBEN_COLUMN, - FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, - FRS_HMRC_SRP_REGULAR_CODE5_COLUMN, -) - -FRS_HMRC_RETAINED_LEAF_SOURCE_EVIDENCE: dict[str, dict[str, object]] = { - FRS_HMRC_PAY_COLUMN: { - "spi_concept": "PAY", - "scope": "full", - "raw_sources": ["ADULT.INEARNS"], - "formula": "max(0, ADULT.INEARNS) * (365.25 / 7)", - }, - FRS_HMRC_UBISJA_COLUMN: { - "spi_concept": "UBISJA", - "scope": "full", - "raw_sources": [ - "BENEFITS.BENEFIT=14:BENAMT", - "BENEFITS.BENEFIT=19:BENAMT", - ], - "formula": "sum(BENAMT where BENEFIT in {14, 19}) * (365.25 / 7)", - }, - FRS_HMRC_INCPBEN_COLUMN: { - "spi_concept": "INCPBEN", - "scope": "full", - "raw_sources": ["BENEFITS.BENEFIT=17:BENAMT"], - "formula": "sum(BENAMT where BENEFIT == 17) * (365.25 / 7)", - }, - FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN: { - "spi_concept": "OSSBEN", - "scope": "identifiable_subset", - "raw_sources": [ - "BENEFITS.BENEFIT=13:BENAMT", - "BENEFITS.BENEFIT=16,VAR2 in {1,3}:BENAMT", - ], - "formula": ( - "sum(BENAMT where BENEFIT == 13 or " - "(BENEFIT == 16 and VAR2 in {1, 3})) * (365.25 / 7)" - ), - }, - FRS_HMRC_SRP_REGULAR_CODE5_COLUMN: { - "spi_concept": "SRP", - "scope": "regular_code5_subset", - "raw_sources": ["BENEFITS.BENEFIT=5:BENAMT"], - "formula": "sum(BENAMT where BENEFIT == 5) * (365.25 / 7)", - }, -} - -_ADULT_REQUIRED_COLUMNS = ("sernum", "person", "inearns") -_BENEFITS_REQUIRED_COLUMNS = ( - "sernum", - "person", - "benefit", - "benamt", - "var2", -) -_CAPITAL_GAINS_FLAG = "household_is_capital_gains_clone" - - -@dataclass(frozen=True) -class UKFRSRawTableIdentity: - """Stable identity and extraction surface for one raw FRS table.""" - - path: Path - filename: str - source_vintage: str - sha256: str - size_bytes: int - rows: int - extracted_columns: tuple[str, ...] - - def evidence(self) -> dict[str, object]: - """Return JSON-safe source evidence.""" - - return { - "path": str(self.path), - "filename": self.filename, - "source_vintage": self.source_vintage, - "sha256": self.sha256, - "size_bytes": self.size_bytes, - "rows": self.rows, - "extracted_columns": list(self.extracted_columns), - } - - -@dataclass(frozen=True) -class UKFRSHMRCRetainedLeavesResult: - """National frame plus raw-source and lineage evidence. - - ``input_content_identity`` and ``output_content_identity`` are the - content identities (:func:`uk_frame_content_identity`) of the frame this - stage consumed and the frame it produced, derived inside the attesting - run. The SPI stage's descent fence compares against them, so the - guarantee survives a process boundary: a checkpoint-rehydrated frame is - content-identical to the one that was checkpointed, while a substituted - or tampered frame is not. - """ - - frame: Frame - adult_source: UKFRSRawTableIdentity - benefits_source: UKFRSRawTableIdentity - clone_id_multiplier: int - spi_person_id_offset: int - capital_gains_person_id_offset: int - raw_source_people: int - candidate_people: int - source_signal_rows: dict[str, int] - structural_zero_columns: tuple[str, ...] - input_content_identity: str - output_content_identity: str - #: Raw-survey people outside the candidate base. Zero on a full-scale - #: build (the completeness fence raises otherwise); on a #627 rung - #: sample it receipts how much of the raw surface the rung dropped. - source_people_outside_candidate: int = 0 - - def evidence(self) -> dict[str, object]: - """Return aggregate, JSON-safe evidence for a national build driver.""" - - return { - "stage": FRS_HMRC_RETAINED_LEAVES_STAGE_NAME, - "source_vintage": FRS_SOURCE_VINTAGE, - "mapped_build_period": uk_time_period(self.frame), - "sources": { - "adult": self.adult_source.evidence(), - "benefits": self.benefits_source.evidence(), - }, - "annualization": { - "days_per_year": 365.25, - "days_per_week": 7, - "weeks_per_year": FRS_WEEKS_IN_YEAR, - }, - "lineage": { - "clone_id_multiplier": self.clone_id_multiplier, - "spi_person_id_offset": self.spi_person_id_offset, - "capital_gains_person_id_offset": (self.capital_gains_person_id_offset), - "raw_source_people": self.raw_source_people, - "candidate_people": self.candidate_people, - "source_people_outside_candidate": ( - self.source_people_outside_candidate - ), - }, - "retained_leaves": { - column: { - **FRS_HMRC_RETAINED_LEAF_SOURCE_EVIDENCE[column], - "source_signal_rows": self.source_signal_rows[column], - "structural_zero": column in self.structural_zero_columns, - } - for column in FRS_HMRC_RETAINED_LEAF_COLUMNS - }, - } - - -@dataclass -class UKFRSHMRCRetainedLeavesStageTransform: - """Callable national-stage adapter retaining the last run's evidence.""" - - adult_tab_path: Path - benefits_tab_path: Path - #: Declared #627 rung build: relaxes the raw-surface completeness fence - #: into a receipted count. Never set on a release build. - sampled_rung: bool = False - last_result: UKFRSHMRCRetainedLeavesResult | None = field( - default=None, - init=False, - ) - - @classmethod - def from_raw_frs_directory( - cls, - raw_frs_directory: str | Path, - *, - sampled_rung: bool = False, - ) -> UKFRSHMRCRetainedLeavesStageTransform: - """Resolve the two permitted tables from a CLI-supplied directory.""" - - directory = Path(raw_frs_directory).expanduser() - return cls( - adult_tab_path=directory / "adult.tab", - benefits_tab_path=directory / "benefits.tab", - sampled_rung=sampled_rung, - ) - - def __call__(self, frame: Frame) -> Frame: - # The result records the content identities of the frame this stage - # consumed and produced, so the SPI stage's fence can assert descent - # from the frame the driver loaded and bound — including across a - # process boundary, where object identity cannot travel. - self.last_result = retain_uk_frs_hmrc_leaves( - frame, - adult_tab_path=self.adult_tab_path, - benefits_tab_path=self.benefits_tab_path, - sampled_rung=self.sampled_rung, - ) - return self.last_result.frame - - def checkpoint_metadata(self) -> dict[str, object]: - """JSON-safe evidence the stage checkpoint carries for a resume. - - The SPI stage consumes the retained-leaves evidence and the descent - identities; persisting them on the completed stage's run-context - record is what lets a later process resume past this stage without - re-running it. - """ - - if self.last_result is None: - raise RuntimeError( - "checkpoint metadata requires a completed retained-leaves run." - ) - return { - "evidence": self.last_result.evidence(), - "input_content_identity": self.last_result.input_content_identity, - "output_content_identity": self.last_result.output_content_identity, - } - - def resume_from_checkpoint( - self, - metadata: Mapping[str, object], - frame: Frame, - ) -> None: - """Rehydrate a completed run's evidence from its checkpoint record. - - ``frame`` is the stage's checkpointed output; the rehydrated result - exposes exactly the surface the SPI stage's descent fence reads. The - recorded output identity must match the loaded frame's content — a - mismatch means the record and the checkpoint have drifted apart, and - the resume fails closed. - """ - - evidence = metadata.get("evidence") - input_identity = metadata.get("input_content_identity") - output_identity = metadata.get("output_content_identity") - if ( - not isinstance(evidence, Mapping) - or not isinstance(input_identity, str) - or not isinstance(output_identity, str) - ): - raise RuntimeError( - "retained-leaves resume requires the checkpoint record to " - "carry the run's evidence and content identities; a record " - "without them cannot prove descent." - ) - if uk_frame_content_identity(frame) != output_identity: - raise RuntimeError( - "retained-leaves checkpoint content does not match its " - "recorded output identity; refusing to resume from a " - "drifted record." - ) - self.last_result = _ResumedRetainedLeaves( - frame=frame, - evidence_payload=dict(evidence), - input_content_identity=input_identity, - output_content_identity=output_identity, - ) - - -@dataclass(frozen=True) -class _ResumedRetainedLeaves: - """A completed retained-leaves run rehydrated from its checkpoint. - - Carries exactly the surface the SPI stage consumes: the output frame, - the JSON-safe evidence, and the descent content identities. - """ - - frame: Frame - evidence_payload: dict[str, object] - input_content_identity: str - output_content_identity: str - - def evidence(self) -> dict[str, object]: - return dict(self.evidence_payload) - - -@dataclass(frozen=True) -class _FileFingerprint: - device: int - inode: int - size_bytes: int - modified_ns: int - changed_ns: int - - -@dataclass(frozen=True) -class _CandidateLineage: - source_person_ids: np.ndarray - clone_id_multiplier: int - spi_person_id_offset: int - capital_gains_person_id_offset: int - canonical_raw_person_ids: frozenset[int] - - -def retain_uk_frs_hmrc_leaves( - frame: Frame, - *, - adult_tab_path: str | Path, - benefits_tab_path: str | Path, - sampled_rung: bool = False, -) -> UKFRSHMRCRetainedLeavesResult: - """Read two raw FRS tables and retain the adjudicated HMRC constituents. - - ``sampled_rung`` declares a #627 scale-ladder build: the candidate base - deliberately carries only a sampled subset of source families, so the - completeness fence (every raw-survey person present in the base) cannot - hold. The raw surface is restricted to surviving canonicals and the - dropped count is receipted instead — never silently. Full-scale builds - keep the strict fence. - """ - - validate_uk_national_frame(frame) - input_content_identity = uk_frame_content_identity(frame) - time_period = uk_time_period(frame) - if time_period not in {FRS_SOURCE_BUILD_PERIOD, FRS_SOURCE_VINTAGE}: - raise ValueError( - f"Raw FRS {FRS_SOURCE_VINTAGE} leaves may only map to build period " - f"{FRS_SOURCE_BUILD_PERIOD!r}; got {time_period!r}." - ) - - adult, adult_source = _read_raw_frs_table( - adult_tab_path, - expected_filename="adult.tab", - required_columns=_ADULT_REQUIRED_COLUMNS, - ) - benefits, benefits_source = _read_raw_frs_table( - benefits_tab_path, - expected_filename="benefits.tab", - required_columns=_BENEFITS_REQUIRED_COLUMNS, - ) - source_leaves = _materialize_source_leaves(adult, benefits) - lineage = _resolve_candidate_lineage(frame) - unknown_source_ids = sorted( - set(source_leaves.index) - lineage.canonical_raw_person_ids - ) - if unknown_source_ids and not sampled_rung: - raise ValueError( - "Raw FRS retained leaves contain person identity value(s) absent " - f"from the certified candidate base: {unknown_source_ids[:5]}." - ) - source_people_outside_candidate = len(unknown_source_ids) - # Signal-row evidence stays a fact about the SOURCE at every rung: - # structural_zero must never be asserted from a sampled-away surface - # (adversarial-review finding). The rung also cannot distinguish a - # compact genuinely missing raw people from sampling loss — that check - # remains the full-scale fence's, which stays strict. - full_source_leaves = source_leaves - if unknown_source_ids: - # A rung sample deliberately drops most source families; restrict the - # raw surface to the surviving canonicals and receipt the count. - source_leaves = source_leaves.loc[ - source_leaves.index.isin(list(lineage.canonical_raw_person_ids)) - ] - - person = frame.table("person").copy() - aligned = source_leaves.reindex(lineage.source_person_ids, fill_value=0.0) - if aligned.isna().any().any(): # pragma: no cover - defensive - raise RuntimeError("Raw FRS retained-leaf alignment produced missing values.") - values = aligned.to_numpy(dtype=float) - if not np.isfinite(values).all() or (values < 0.0).any(): - raise RuntimeError( - "Raw FRS retained-leaf alignment produced non-finite or negative values." - ) - for column in FRS_HMRC_RETAINED_LEAF_COLUMNS: - person[column] = aligned[column].to_numpy(dtype=float) - - # Person-only replacement: mass is untouched, so the kind and mass log - # carry through unchanged; Frame construction re-runs linkage validation. - result_frame = uk_national_frame( - person=person, - benunit=frame.table("benunit"), - household=frame.table("household"), - time_period=time_period, - weight_kind=uk_household_weight_kind(frame), - household_weights=frame.weights_for("household").values, - mass_log=frame.mass_log, - ) - validate_uk_national_frame(result_frame) - _validate_retained_leaf_propagation( - result_frame.table("person"), - source_person_ids=lineage.source_person_ids, - source_leaves=source_leaves, - ) - source_signal_rows = { - column: int((full_source_leaves[column] > 0.0).sum()) - for column in FRS_HMRC_RETAINED_LEAF_COLUMNS - } - structural_zero_columns = tuple( - column - for column in FRS_HMRC_RETAINED_LEAF_COLUMNS - if source_signal_rows[column] == 0 - ) - return UKFRSHMRCRetainedLeavesResult( - frame=result_frame, - adult_source=adult_source, - benefits_source=benefits_source, - clone_id_multiplier=lineage.clone_id_multiplier, - spi_person_id_offset=lineage.spi_person_id_offset, - capital_gains_person_id_offset=lineage.capital_gains_person_id_offset, - raw_source_people=len(source_leaves), - candidate_people=len(person), - source_people_outside_candidate=source_people_outside_candidate, - source_signal_rows=source_signal_rows, - structural_zero_columns=structural_zero_columns, - input_content_identity=input_content_identity, - output_content_identity=uk_frame_content_identity(result_frame), - ) - - -def _read_raw_frs_table( - path: str | Path, - *, - expected_filename: str, - required_columns: tuple[str, ...], -) -> tuple[pd.DataFrame, UKFRSRawTableIdentity]: - source_path = Path(path).expanduser().resolve() - if source_path.name.lower() != expected_filename: - raise ValueError( - f"Expected raw FRS table {expected_filename!r}, got {source_path.name!r}." - ) - if not source_path.is_file(): - raise FileNotFoundError(f"Raw FRS table not found: {source_path}.") - before = _file_fingerprint(source_path) - digest = _sha256(source_path) - after_hash = _file_fingerprint(source_path) - if after_hash != before: - raise RuntimeError(f"Raw FRS table changed while hashing: {source_path}.") - required = set(required_columns) - frame = pd.read_csv( - source_path, - sep="\t", - usecols=lambda column: str(column).strip().lower() in required, - ) - after_read = _file_fingerprint(source_path) - if after_read != before: - raise RuntimeError(f"Raw FRS table changed while reading: {source_path}.") - frame.columns = frame.columns.astype(str).str.strip().str.lower() - if frame.columns.duplicated().any(): - duplicates = frame.columns[frame.columns.duplicated()].tolist() - raise ValueError( - f"Raw FRS {expected_filename} has duplicate normalized columns: " - f"{duplicates}." - ) - missing = sorted(required - set(frame.columns)) - if missing: - raise ValueError( - f"Raw FRS {expected_filename} is missing required column(s): {missing}." - ) - frame = frame.loc[:, list(required_columns)] - identity = UKFRSRawTableIdentity( - path=source_path, - filename=expected_filename, - source_vintage=FRS_SOURCE_VINTAGE, - sha256=digest, - size_bytes=before.size_bytes, - rows=len(frame), - extracted_columns=required_columns, - ) - return frame, identity - - -def _materialize_source_leaves( - adult: pd.DataFrame, - benefits: pd.DataFrame, -) -> pd.DataFrame: - adult_ids = _raw_source_person_ids(adult, label="ADULT") - if pd.Index(adult_ids).duplicated().any(): - duplicates = pd.Index(adult_ids)[pd.Index(adult_ids).duplicated()].unique() - raise ValueError( - "Raw FRS ADULT person identities must be unique; duplicate " - f"value(s): {duplicates[:5].tolist()}." - ) - earnings = _finite_numeric(adult["inearns"], label="ADULT.INEARNS") - pay = np.maximum(earnings, 0.0) * FRS_WEEKS_IN_YEAR - adult_leaf = pd.DataFrame( - {FRS_HMRC_PAY_COLUMN: pay}, - index=pd.Index(adult_ids, name="source_person_id"), - ) - - benefit_ids = _raw_source_person_ids(benefits, label="BENEFITS") - benefit_codes = _strict_integer_values( - benefits["benefit"], - label="BENEFITS.BENEFIT", - minimum=0, - ) - relevant = np.isin(benefit_codes, (5, 13, 14, 16, 17, 19)) - amounts = np.zeros(len(benefits), dtype=float) - if relevant.any(): - relevant_amounts = _finite_numeric( - benefits.loc[relevant, "benamt"], - label="relevant BENEFITS.BENAMT", - ) - if (relevant_amounts < 0.0).any(): - raise ValueError("Relevant BENEFITS.BENAMT values must be non-negative.") - amounts[relevant] = relevant_amounts - - code16 = benefit_codes == 16 - contribution_based_esa = np.zeros(len(benefits), dtype=bool) - if code16.any(): - var2 = _strict_integer_values( - benefits.loc[code16, "var2"], - label="BENEFITS.VAR2 for BENEFIT=16", - ) - contribution_based_esa[code16] = np.isin(var2, (1, 3)) - - benefit_leaf = pd.DataFrame( - { - FRS_HMRC_UBISJA_COLUMN: amounts * np.isin(benefit_codes, (14, 19)), - FRS_HMRC_INCPBEN_COLUMN: amounts * (benefit_codes == 17), - FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN: amounts - * ((benefit_codes == 13) | contribution_based_esa), - FRS_HMRC_SRP_REGULAR_CODE5_COLUMN: amounts * (benefit_codes == 5), - }, - index=pd.Index(benefit_ids, name="source_person_id"), - ) - benefit_leaf = benefit_leaf.groupby(level=0, sort=False).sum() - benefit_leaf *= FRS_WEEKS_IN_YEAR - - source_ids = adult_leaf.index.union(benefit_leaf.index, sort=False) - result = pd.DataFrame( - 0.0, - index=source_ids, - columns=FRS_HMRC_RETAINED_LEAF_COLUMNS, - ) - result.loc[adult_leaf.index, FRS_HMRC_PAY_COLUMN] = adult_leaf[FRS_HMRC_PAY_COLUMN] - for column in benefit_leaf.columns: - result.loc[benefit_leaf.index, column] = benefit_leaf[column] - numeric = result.to_numpy(dtype=float) - if not np.isfinite(numeric).all() or (numeric < 0.0).any(): - raise RuntimeError("Raw FRS source-leaf materialization is invalid.") - return result - - -def _resolve_candidate_lineage(frame: Frame) -> _CandidateLineage: - person = frame.table("person") - household = frame.table("household") - _require_columns( - person, - ("person_id", "person_household_id"), - label="candidate person", - ) - _require_columns( - household, - ( - "household_id", - "clone_index", - HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - _CAPITAL_GAINS_FLAG, - ), - label="candidate household", - ) - person_ids = _strict_integer_values( - person["person_id"], label="candidate person_id", minimum=1 - ) - person_household_ids = _strict_integer_values( - person["person_household_id"], - label="candidate person_household_id", - minimum=1, - ) - household_ids = _strict_integer_values( - household["household_id"], label="candidate household_id", minimum=1 - ) - clone_index = _strict_integer_values( - household["clone_index"], label="candidate clone_index", minimum=0 - ) - spi = _strict_bool_values( - household[HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN], - label=HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - ) - capital_gains = _strict_bool_values( - household[_CAPITAL_GAINS_FLAG], label=_CAPITAL_GAINS_FLAG - ) - household_metadata = pd.DataFrame( - { - "household_id": household_ids, - "clone_index": clone_index, - "spi": spi, - "capital_gains": capital_gains, - } - ).set_index("household_id") - mapped = household_metadata.reindex(person_household_ids) - if mapped.isna().any().any(): - raise ValueError( - "Candidate person_household_id cannot map every person to lineage metadata." - ) - person_clone_index = mapped["clone_index"].to_numpy(dtype=np.int64) - person_spi = mapped["spi"].to_numpy(dtype=bool) - person_capital_gains = mapped["capital_gains"].to_numpy(dtype=bool) - - canonical_households = clone_index == 0 - canonical_people = person_clone_index == 0 - if not canonical_households.any() or not canonical_people.any(): - raise ValueError("Candidate lineage requires clone_index=0 rows.") - canonical_max = max( - int(household_ids[canonical_households].max()), - int(person_ids[canonical_people].max()), - ) - clone_multiplier = 10 ** max(1, len(str(canonical_max))) - clone_reversed_household_ids = household_ids - clone_index * clone_multiplier - clone_reversed_person_ids = person_ids - person_clone_index * clone_multiplier - clone_reversed_person_households = ( - person_household_ids - person_clone_index * clone_multiplier - ) - if ( - (clone_reversed_household_ids <= 0).any() - or (clone_reversed_person_ids <= 0).any() - or (clone_reversed_person_households <= 0).any() - ): - raise ValueError("Candidate clone reversal produced non-positive IDs.") - - canonical_household_metadata = pd.DataFrame( - { - "clone_household_id": household_ids[canonical_households], - "spi": spi[canonical_households], - "capital_gains": capital_gains[canonical_households], - } - ).set_index("clone_household_id") - expected_household_metadata = canonical_household_metadata.reindex( - clone_reversed_household_ids - ) - if expected_household_metadata.isna().any().any(): - raise ValueError( - "Candidate geography-clone household IDs do not reverse to the " - "clone_index=0 surface." - ) - if not np.array_equal( - expected_household_metadata["spi"].to_numpy(dtype=bool), spi - ) or not np.array_equal( - expected_household_metadata["capital_gains"].to_numpy(dtype=bool), - capital_gains, - ): - raise ValueError( - "Candidate geography clones disagree with canonical household flags." - ) - - canonical_person = pd.DataFrame( - { - "clone_person_id": person_ids[canonical_people], - "clone_household_id": person_household_ids[canonical_people], - "spi": person_spi[canonical_people], - "capital_gains": person_capital_gains[canonical_people], - } - ).set_index("clone_person_id") - expected_people = canonical_person.reindex(clone_reversed_person_ids) - if expected_people.isna().any().any(): - raise ValueError( - "Candidate geography-clone person IDs do not reverse to the " - "clone_index=0 surface." - ) - if not np.array_equal( - expected_people["clone_household_id"].to_numpy(dtype=np.int64), - clone_reversed_person_households, - ): - raise ValueError( - "Candidate geography-clone person/household memberships are inconsistent." - ) - descriptors = pd.DataFrame( - { - "clone_person_id": clone_reversed_person_ids, - "clone_index": person_clone_index, - } - ) - if descriptors.duplicated().any(): - raise ValueError( - "Candidate geography-clone person lineage contains duplicate descendants." - ) - - canonical_person_ids = person_ids[canonical_people] - canonical_person_spi = person_spi[canonical_people] - canonical_person_capital_gains = person_capital_gains[canonical_people] - raw_person = ~canonical_person_spi & ~canonical_person_capital_gains - pre_capital_gains = ~canonical_person_capital_gains - if not raw_person.any(): - raise ValueError("Candidate lineage has no canonical raw FRS people.") - spi_offset = int(canonical_person_ids[raw_person].max()) + 1 - capital_gains_offset = int(canonical_person_ids[pre_capital_gains].max()) + 1 - source_person_ids = ( - clone_reversed_person_ids - - person_spi.astype(np.int64) * spi_offset - - person_capital_gains.astype(np.int64) * capital_gains_offset - ) - canonical_raw_ids = frozenset( - int(value) for value in canonical_person_ids[raw_person] - ) - if (source_person_ids <= 0).any() or not set(source_person_ids).issubset( - canonical_raw_ids - ): - bad = sorted(set(source_person_ids) - canonical_raw_ids) - raise ValueError( - "Candidate SPI/capital-gains person IDs do not reverse to the raw " - f"FRS surface: {bad[:5]}." - ) - - raw_household = ~spi[canonical_households] & ~capital_gains[canonical_households] - pre_capital_household = ~capital_gains[canonical_households] - canonical_household_ids = household_ids[canonical_households] - if not raw_household.any(): - raise ValueError("Candidate lineage has no canonical raw FRS households.") - spi_household_offset = int(canonical_household_ids[raw_household].max()) + 1 - capital_household_offset = ( - int(canonical_household_ids[pre_capital_household].max()) + 1 - ) - source_household_ids = ( - clone_reversed_person_households - - person_spi.astype(np.int64) * spi_household_offset - - person_capital_gains.astype(np.int64) * capital_household_offset - ) - if not np.array_equal(source_person_ids // 1000, source_household_ids): - raise ValueError( - "Candidate reversed person IDs disagree with reversed household IDs." - ) - lineage_descriptors = pd.DataFrame( - { - "source_person_id": source_person_ids, - "clone_index": person_clone_index, - "spi": person_spi, - "capital_gains": person_capital_gains, - } - ) - if lineage_descriptors.duplicated().any(): - raise ValueError( - "Candidate person lineage contains duplicate stack identities." - ) - return _CandidateLineage( - source_person_ids=source_person_ids, - clone_id_multiplier=clone_multiplier, - spi_person_id_offset=spi_offset, - capital_gains_person_id_offset=capital_gains_offset, - canonical_raw_person_ids=canonical_raw_ids, - ) - - -def _validate_retained_leaf_propagation( - person: pd.DataFrame, - *, - source_person_ids: np.ndarray, - source_leaves: pd.DataFrame, -) -> None: - expected = source_leaves.reindex(source_person_ids, fill_value=0.0) - actual = person.loc[:, list(FRS_HMRC_RETAINED_LEAF_COLUMNS)].apply( - pd.to_numeric, errors="coerce" - ) - actual_values = actual.to_numpy(dtype=float) - if not np.isfinite(actual_values).all() or (actual_values < 0.0).any(): - raise RuntimeError("Retained FRS HMRC leaves must be finite and non-negative.") - if not np.array_equal(actual_values, expected.to_numpy(dtype=float)): - raise RuntimeError("Retained FRS HMRC leaves lost source-person alignment.") - - -def _raw_source_person_ids(frame: pd.DataFrame, *, label: str) -> np.ndarray: - households = _strict_integer_values( - frame["sernum"], label=f"{label}.SERNUM", minimum=1 - ) - people = _strict_integer_values(frame["person"], label=f"{label}.PERSON", minimum=1) - if (people >= 1000).any(): - raise ValueError(f"{label}.PERSON must be less than 1000.") - maximum_household = (np.iinfo(np.int64).max - people) // 1000 - if (households > maximum_household).any(): - raise ValueError(f"{label} source person identity exceeds int64 range.") - return households * 1000 + people - - -def _strict_integer_values( - values: pd.Series, - *, - label: str, - minimum: int | None = None, -) -> np.ndarray: - numeric = pd.to_numeric(values, errors="coerce").to_numpy( - dtype=float, na_value=np.nan - ) - if not np.isfinite(numeric).all(): - raise ValueError(f"{label} must contain finite numeric values.") - if not np.equal(numeric, np.floor(numeric)).all(): - raise ValueError(f"{label} must contain integer values.") - if (np.abs(numeric) > np.iinfo(np.int64).max).any(): - raise ValueError(f"{label} exceeds int64 range.") - result = numeric.astype(np.int64) - if minimum is not None and (result < minimum).any(): - raise ValueError(f"{label} must be at least {minimum}.") - return result - - -def _strict_bool_values(values: pd.Series, *, label: str) -> np.ndarray: - if pd.api.types.is_bool_dtype(values.dtype): - if values.isna().any(): - raise ValueError(f"{label} must not contain missing values.") - return values.to_numpy(dtype=bool) - numeric = _strict_integer_values(values, label=label) - if not np.isin(numeric, (0, 1)).all(): - raise ValueError(f"{label} must contain only boolean or 0/1 values.") - return numeric.astype(bool) - - -def _finite_numeric(values: pd.Series, *, label: str) -> np.ndarray: - numeric = pd.to_numeric(values, errors="coerce").to_numpy( - dtype=float, na_value=np.nan - ) - if not np.isfinite(numeric).all(): - raise ValueError(f"{label} must contain finite numeric values.") - return numeric - - -def _require_columns( - frame: pd.DataFrame, - columns: tuple[str, ...], - *, - label: str, -) -> None: - missing = sorted(set(columns) - set(frame.columns)) - if missing: - raise ValueError(f"{label} is missing required column(s): {missing}.") - - -def _file_fingerprint(path: Path) -> _FileFingerprint: - stat = path.stat() - return _FileFingerprint( - device=stat.st_dev, - inode=stat.st_ino, - size_bytes=stat.st_size, - modified_ns=stat.st_mtime_ns, - changed_ns=stat.st_ctime_ns, - ) - - -def _sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_source.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_source.py new file mode 100644 index 000000000..d0dd4c5f9 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_source.py @@ -0,0 +1,311 @@ +"""Source-faithful raw FRS extraction for the canonical HMRC spine stages. + +This module reads raw survey tables and preserves named partial income concepts. +It does not restore or accept a pre-existing candidate population. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.uk_runtime.spi_support import ( + SPI_HMRC_INCAPACITY_BENEFIT_INCOME_COLUMN, + SPI_HMRC_PAY_COLUMN, + SPI_HMRC_UNEMPLOYMENT_BENEFIT_INCOME_COLUMN, +) + +FRS_WEEKS_IN_YEAR = 365.25 / 7 + + +FRS_HMRC_PAY_COLUMN = SPI_HMRC_PAY_COLUMN + + +FRS_HMRC_UBISJA_COLUMN = SPI_HMRC_UNEMPLOYMENT_BENEFIT_INCOME_COLUMN + + +FRS_HMRC_INCPBEN_COLUMN = SPI_HMRC_INCAPACITY_BENEFIT_INCOME_COLUMN + + +FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN = "ossben_identifiable_subset" + + +FRS_HMRC_SRP_REGULAR_CODE5_COLUMN = "srp_regular_code5" + + +FRS_HMRC_RETAINED_LEAF_COLUMNS = ( + FRS_HMRC_PAY_COLUMN, + FRS_HMRC_UBISJA_COLUMN, + FRS_HMRC_INCPBEN_COLUMN, + FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, + FRS_HMRC_SRP_REGULAR_CODE5_COLUMN, +) + + +FRS_HMRC_RETAINED_LEAF_SOURCE_EVIDENCE: dict[str, dict[str, object]] = { + FRS_HMRC_PAY_COLUMN: { + "spi_concept": "PAY", + "scope": "full", + "raw_sources": ["ADULT.INEARNS"], + "formula": "max(0, ADULT.INEARNS) * (365.25 / 7)", + }, + FRS_HMRC_UBISJA_COLUMN: { + "spi_concept": "UBISJA", + "scope": "full", + "raw_sources": [ + "BENEFITS.BENEFIT=14:BENAMT", + "BENEFITS.BENEFIT=19:BENAMT", + ], + "formula": "sum(BENAMT where BENEFIT in {14, 19}) * (365.25 / 7)", + }, + FRS_HMRC_INCPBEN_COLUMN: { + "spi_concept": "INCPBEN", + "scope": "full", + "raw_sources": ["BENEFITS.BENEFIT=17:BENAMT"], + "formula": "sum(BENAMT where BENEFIT == 17) * (365.25 / 7)", + }, + FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN: { + "spi_concept": "OSSBEN", + "scope": "identifiable_subset", + "raw_sources": [ + "BENEFITS.BENEFIT=13:BENAMT", + "BENEFITS.BENEFIT=16,VAR2 in {1,3}:BENAMT", + ], + "formula": ( + "sum(BENAMT where BENEFIT == 13 or " + "(BENEFIT == 16 and VAR2 in {1, 3})) * (365.25 / 7)" + ), + }, + FRS_HMRC_SRP_REGULAR_CODE5_COLUMN: { + "spi_concept": "SRP", + "scope": "regular_code5_subset", + "raw_sources": ["BENEFITS.BENEFIT=5:BENAMT"], + "formula": "sum(BENAMT where BENEFIT == 5) * (365.25 / 7)", + }, +} + + +@dataclass(frozen=True) +class UKFRSRawTableIdentity: + """Stable identity and extraction surface for one raw FRS table.""" + + path: Path + filename: str + source_vintage: str + sha256: str + size_bytes: int + rows: int + extracted_columns: tuple[str, ...] + + def evidence(self) -> dict[str, object]: + """Return JSON-safe source evidence.""" + + return { + "path": str(self.path), + "filename": self.filename, + "source_vintage": self.source_vintage, + "sha256": self.sha256, + "size_bytes": self.size_bytes, + "rows": self.rows, + "extracted_columns": list(self.extracted_columns), + } + + +@dataclass(frozen=True) +class _FileFingerprint: + device: int + inode: int + size_bytes: int + modified_ns: int + changed_ns: int + + +def _read_raw_frs_table( + path: str | Path, + *, + expected_filename: str, + required_columns: tuple[str, ...], + source_vintage: str = "unspecified", +) -> tuple[pd.DataFrame, UKFRSRawTableIdentity]: + source_path = Path(path).expanduser().resolve() + if source_path.name.lower() != expected_filename: + raise ValueError( + f"Expected raw FRS table {expected_filename!r}, got {source_path.name!r}." + ) + if not source_path.is_file(): + raise FileNotFoundError(f"Raw FRS table not found: {source_path}.") + before = _file_fingerprint(source_path) + digest = _sha256(source_path) + after_hash = _file_fingerprint(source_path) + if after_hash != before: + raise RuntimeError(f"Raw FRS table changed while hashing: {source_path}.") + required = set(required_columns) + frame = pd.read_csv( + source_path, + sep="\t", + usecols=lambda column: str(column).strip().lower() in required, + ) + after_read = _file_fingerprint(source_path) + if after_read != before: + raise RuntimeError(f"Raw FRS table changed while reading: {source_path}.") + frame.columns = frame.columns.astype(str).str.strip().str.lower() + if frame.columns.duplicated().any(): + duplicates = frame.columns[frame.columns.duplicated()].tolist() + raise ValueError( + f"Raw FRS {expected_filename} has duplicate normalized columns: " + f"{duplicates}." + ) + missing = sorted(required - set(frame.columns)) + if missing: + raise ValueError( + f"Raw FRS {expected_filename} is missing required column(s): {missing}." + ) + frame = frame.loc[:, list(required_columns)] + identity = UKFRSRawTableIdentity( + path=source_path, + filename=expected_filename, + source_vintage=source_vintage, + sha256=digest, + size_bytes=before.size_bytes, + rows=len(frame), + extracted_columns=required_columns, + ) + return frame, identity + + +def _materialize_source_leaves( + adult: pd.DataFrame, + benefits: pd.DataFrame, +) -> pd.DataFrame: + adult_ids = _raw_source_person_ids(adult, label="ADULT") + if pd.Index(adult_ids).duplicated().any(): + duplicates = pd.Index(adult_ids)[pd.Index(adult_ids).duplicated()].unique() + raise ValueError( + "Raw FRS ADULT person identities must be unique; duplicate " + f"value(s): {duplicates[:5].tolist()}." + ) + earnings = _finite_numeric(adult["inearns"], label="ADULT.INEARNS") + pay = np.maximum(earnings, 0.0) * FRS_WEEKS_IN_YEAR + adult_leaf = pd.DataFrame( + {FRS_HMRC_PAY_COLUMN: pay}, + index=pd.Index(adult_ids, name="source_person_id"), + ) + + benefit_ids = _raw_source_person_ids(benefits, label="BENEFITS") + benefit_codes = _strict_integer_values( + benefits["benefit"], + label="BENEFITS.BENEFIT", + minimum=0, + ) + relevant = np.isin(benefit_codes, (5, 13, 14, 16, 17, 19)) + amounts = np.zeros(len(benefits), dtype=float) + if relevant.any(): + relevant_amounts = _finite_numeric( + benefits.loc[relevant, "benamt"], + label="relevant BENEFITS.BENAMT", + ) + if (relevant_amounts < 0.0).any(): + raise ValueError("Relevant BENEFITS.BENAMT values must be non-negative.") + amounts[relevant] = relevant_amounts + + code16 = benefit_codes == 16 + contribution_based_esa = np.zeros(len(benefits), dtype=bool) + if code16.any(): + var2 = _strict_integer_values( + benefits.loc[code16, "var2"], + label="BENEFITS.VAR2 for BENEFIT=16", + ) + contribution_based_esa[code16] = np.isin(var2, (1, 3)) + + benefit_leaf = pd.DataFrame( + { + FRS_HMRC_UBISJA_COLUMN: amounts * np.isin(benefit_codes, (14, 19)), + FRS_HMRC_INCPBEN_COLUMN: amounts * (benefit_codes == 17), + FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN: amounts + * ((benefit_codes == 13) | contribution_based_esa), + FRS_HMRC_SRP_REGULAR_CODE5_COLUMN: amounts * (benefit_codes == 5), + }, + index=pd.Index(benefit_ids, name="source_person_id"), + ) + benefit_leaf = benefit_leaf.groupby(level=0, sort=False).sum() + benefit_leaf *= FRS_WEEKS_IN_YEAR + + source_ids = adult_leaf.index.union(benefit_leaf.index, sort=False) + result = pd.DataFrame( + 0.0, + index=source_ids, + columns=FRS_HMRC_RETAINED_LEAF_COLUMNS, + ) + result.loc[adult_leaf.index, FRS_HMRC_PAY_COLUMN] = adult_leaf[FRS_HMRC_PAY_COLUMN] + for column in benefit_leaf.columns: + result.loc[benefit_leaf.index, column] = benefit_leaf[column] + numeric = result.to_numpy(dtype=float) + if not np.isfinite(numeric).all() or (numeric < 0.0).any(): + raise RuntimeError("Raw FRS source-leaf materialization is invalid.") + return result + + +def _raw_source_person_ids(frame: pd.DataFrame, *, label: str) -> np.ndarray: + households = _strict_integer_values( + frame["sernum"], label=f"{label}.SERNUM", minimum=1 + ) + people = _strict_integer_values(frame["person"], label=f"{label}.PERSON", minimum=1) + if (people >= 1000).any(): + raise ValueError(f"{label}.PERSON must be less than 1000.") + maximum_household = (np.iinfo(np.int64).max - people) // 1000 + if (households > maximum_household).any(): + raise ValueError(f"{label} source person identity exceeds int64 range.") + return households * 1000 + people + + +def _strict_integer_values( + values: pd.Series, + *, + label: str, + minimum: int | None = None, +) -> np.ndarray: + numeric = pd.to_numeric(values, errors="coerce").to_numpy( + dtype=float, na_value=np.nan + ) + if not np.isfinite(numeric).all(): + raise ValueError(f"{label} must contain finite numeric values.") + if not np.equal(numeric, np.floor(numeric)).all(): + raise ValueError(f"{label} must contain integer values.") + if (np.abs(numeric) > np.iinfo(np.int64).max).any(): + raise ValueError(f"{label} exceeds int64 range.") + result = numeric.astype(np.int64) + if minimum is not None and (result < minimum).any(): + raise ValueError(f"{label} must be at least {minimum}.") + return result + + +def _finite_numeric(values: pd.Series, *, label: str) -> np.ndarray: + numeric = pd.to_numeric(values, errors="coerce").to_numpy( + dtype=float, na_value=np.nan + ) + if not np.isfinite(numeric).all(): + raise ValueError(f"{label} must contain finite numeric values.") + return numeric + + +def _file_fingerprint(path: Path) -> _FileFingerprint: + stat = path.stat() + return _FileFingerprint( + device=stat.st_dev, + inode=stat.st_ino, + size_bytes=stat.st_size, + modified_ns=stat.st_mtime_ns, + changed_ns=stat.st_ctime_ns, + ) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py index ee1e3e316..044537f8c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py @@ -37,7 +37,6 @@ from .national_sampling import UK_SAMPLE_SEED_DEFAULT __all__ = [ - "UK_SPINE_EXCLUSIONS", "UK_SPINE_STRUCTURAL_STAGES", "uk_registry", "uk_spine_endpoint", @@ -46,15 +45,6 @@ ] -UK_SPINE_EXCLUSIONS = frozenset( - { - # These are the certified-candidate/H5 alternatives to the raw-FRS - # spine stages named below, not additional steps in this pipeline. - "frs_hmrc_retained_leaves", - "hmrc_spi_income", - } -) - UK_SPINE_STRUCTURAL_STAGES = frozenset( { "spi_support_channel", @@ -889,9 +879,7 @@ def _deduplicate(cells: Iterable[_Cell]) -> tuple[_Cell, ...]: def _manifest_stages(spec: CountrySpec) -> tuple[object, ...]: if spec.sources is None: raise ValueError("The UK graph requires a source-stage manifest.") - selected = tuple( - stage for stage in spec.sources.stages if stage.stage not in UK_SPINE_EXCLUSIONS - ) + selected = tuple(spec.sources.stages) if not selected or selected[0].stage != "frs_spine": raise ValueError("The UK FRS spine manifest must begin with 'frs_spine'.") unknown = [stage.stage for stage in selected[1:] if stage.stage not in _STAGE_CELLS] diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index e34993b1d..c71814c33 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -45,7 +45,7 @@ from microcosm.graph.population import dtype_for_token from .. import stage_evidence -from . import bus_use_incidence, uc_relationships +from . import bus_use_incidence, frs_hmrc_source, uc_relationships from .national_frame import UK_NATIONAL_SCHEMA from .rowwise_geography import id_multiplier_for_values @@ -82,7 +82,7 @@ "lcfs_consumption": "lcfs_consumption", "etb_vat": "etb_vat", "etb_services": "etb_services", - "frs_hmrc_spine_leaves": "frs_hmrc_leaves", + "frs_hmrc_spine_leaves": "spi_spine", "spi_support_channel": "spi_spine", "spi_income_band_donors": "spi_band_donors", "hmrc_spi_income_spine": "spi_spine", @@ -103,6 +103,7 @@ # Imported modules are not traversed by ``source_hash``. Bind relationship # helpers and the adapter's input-retention checks into every consuming stage. _STAGE_HELPER_MODULES = { + "frs_hmrc_spine_leaves": (frs_hmrc_source,), "frs_spine": (uc_relationships,), "frs_legacy_proxies": (uk_engine_adapter,), "frs_education_grant_split": (uk_engine_adapter,), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py index 946dc08cc..608175693 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py @@ -2,14 +2,13 @@ from __future__ import annotations -import copy import json from collections.abc import Mapping, Sequence from importlib.resources import files from pathlib import Path from typing import Any -from microcosm.build.uk_runtime.frs_hmrc_leaves import ( +from microcosm.build.uk_runtime.frs_hmrc_source import ( FRS_HMRC_RETAINED_LEAF_COLUMNS, FRS_HMRC_RETAINED_LEAF_SOURCE_EVIDENCE, ) @@ -31,7 +30,6 @@ CANONICAL_HMRC_FACT_FENCES, FULL_FRS_TI_BAND_FENCE_ID, ) -from microcosm.build.uk_runtime.release_identity import UK_RELEASE_TIER_FRS from microcosm.build.uk_runtime.spi_income import ( DEFAULT_SPI_DONOR_SAMPLE_SIZE, SPI_DERIVED_POLICYENGINE_SOURCE_COLUMNS, @@ -59,35 +57,19 @@ FRS_ONLY_SPI_FILL_PREDICTOR_COLUMNS, SPI_HMRC_DERIVED_AUXILIARY_COLUMNS, SPI_INCOME_QRF_OUTPUT_COLUMNS, - SPI_PRIOR_MASS_CHANGE_REASON, - SPI_REPLACEMENT_STRATA_COLUMNS, ) __all__ = [ - "CERTIFIED_UK_CANDIDATE_FILENAME", - "CERTIFIED_UK_CANDIDATE_REVISION", - "CERTIFIED_UK_CANDIDATE_SHA256", - "CERTIFIED_UK_CANDIDATE_SIZE_BYTES", - "CERTIFIED_UK_CANDIDATE_TIER", "HMRC_DISTRIBUTIONAL_INPUTS", - "UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE", "assert_uk_hmrc_income_source_contract_current", "uk_hmrc_weighted_qrf_output_columns", ] -UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE = "hmrc_income_source_stages.json" UK_CANONICAL_SOURCE_STAGES_RESOURCE = "source_stages.json" HMRC_DISTRIBUTIONAL_INPUTS = ( "gift_aid", "charitable_investment_gifts", ) -CERTIFIED_UK_CANDIDATE_FILENAME = "populace_uk_2023.h5" -CERTIFIED_UK_CANDIDATE_REVISION = "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z" -CERTIFIED_UK_CANDIDATE_TIER = UK_RELEASE_TIER_FRS -CERTIFIED_UK_CANDIDATE_SHA256 = ( - "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833" -) -CERTIFIED_UK_CANDIDATE_SIZE_BYTES = 1_315_880_118 _STAGE2_SOURCE_FAITHFUL_INCOME_PREDICTORS = ( "employment_income", "self_employment_income", @@ -113,13 +95,18 @@ } _EXPECTED_OPERATION_KINDS = ( - "verify_certified_candidate", "retain_adjudicated_frs_hmrc_leaves", + "derive", + "stack_zero_weight_donors", + "gate_zero_weight_strata", + "allocate_zero_weight_prior_mass", + "stack_income_band_donor_households", "verify_pinned_hmrc_source_pair", - "replace_zero_weight_spi_support", "strict_read_private_table", "fit_weighted_qrf_stage1", + "resample_band_donor_leaves", "fit_weighted_qrf_stage2", + "redraw_columns_from_fitted_qrf", "materialize_hmrc_income_bands_fail_closed", "classify_hmrc_income_facts_with_reviewed_fences", "gate_distributional_effective_mass", @@ -141,47 +128,9 @@ def assert_uk_hmrc_income_source_contract_current( failures.append(f"stages: expected exactly one stage, got {len(stages)}") _raise_failures(failures) stage = stages[0] - _expect(failures, "stage.stage", stage.get("stage"), "hmrc_spi_income") + _expect(failures, "stage.stage", stage.get("stage"), "hmrc_spi_income_spine") _expect(failures, "stage.grain", stage.get("grain"), "person") - base = _mapping(stage.get("base_candidate"), "base_candidate", failures) - _expect( - failures, - "base_candidate.filename", - base.get("filename"), - CERTIFIED_UK_CANDIDATE_FILENAME, - ) - _expect( - failures, - "base_candidate.tier", - base.get("tier"), - CERTIFIED_UK_CANDIDATE_TIER, - ) - _expect( - failures, - "base_candidate.revision", - base.get("revision"), - CERTIFIED_UK_CANDIDATE_REVISION, - ) - _expect( - failures, - "base_candidate.sha256", - base.get("sha256"), - CERTIFIED_UK_CANDIDATE_SHA256, - ) - _expect( - failures, - "base_candidate.size_bytes", - base.get("size_bytes"), - CERTIFIED_UK_CANDIDATE_SIZE_BYTES, - ) - _expect( - failures, - "base_candidate.runtime_sha256_required", - base.get("runtime_sha256_required"), - True, - ) - artifacts = _keyed_items( stage.get("artifacts"), key="role", @@ -303,10 +252,6 @@ def assert_uk_hmrc_income_source_contract_current( _EXPECTED_OPERATION_KINDS, ) - verify = operations.get("verify_certified_candidate", {}) - _expect(failures, "verify.artifact", verify.get("artifact"), "base_candidate") - _expect(failures, "verify.fail_on_mismatch", verify.get("fail_on_mismatch"), True) - frs_leaves = operations.get("retain_adjudicated_frs_hmrc_leaves", {}) _expect( failures, @@ -416,55 +361,20 @@ def assert_uk_hmrc_income_source_contract_current( ): _expect(failures, f"source_pair.{flag}", source_pair.get(flag), True) - prior = operations.get("replace_zero_weight_spi_support", {}) - _expect(failures, "prior.existing_channel", prior.get("existing_channel"), "spi") - _expect( - failures, - "prior.require_existing_weight", - prior.get("require_existing_weight"), - 0, - ) - _expect( - failures, - "prior.replacement_strata", - tuple(prior.get("replacement_strata", ())), - SPI_REPLACEMENT_STRATA_COLUMNS, - ) + prior = operations.get("allocate_zero_weight_prior_mass", {}) _expect( - failures, - "prior.mass_share", - prior.get("spi_prior_national_household_mass_share"), - DEFAULT_SPI_PRIOR_MASS_SHARE, + failures, "prior.mass_share", prior.get("share"), DEFAULT_SPI_PRIOR_MASS_SHARE ) + _expect(failures, "prior.strata", tuple(prior.get("strata", ())), ("region",)) _expect( - failures, - "prior.output_weight_kind", - prior.get("output_weight_kind"), - "importance", + failures, "prior.output_weight_kind", prior.get("weight_kind_out"), "importance" ) + _expect(failures, "prior.conservation", prior.get("conservation"), "exact_total") _expect( failures, - "prior.preserve_total_household_mass", - prior.get("preserve_total_household_mass"), - True, - ) - _expect( - failures, - "prior.require_mass_change_record", - prior.get("require_mass_change_record"), - True, - ) - _expect( - failures, - "prior.mass_change_reason", - prior.get("mass_change_reason"), - SPI_PRIOR_MASS_CHANGE_REASON, - ) - _expect( - failures, - "prior.fail_on_live_existing_spi_mass", - prior.get("fail_on_live_existing_spi_mass"), - True, + "frs_leaves.population", + frs_leaves.get("population"), + "uk_frs_raw_spine", ) strict = operations.get("strict_read_private_table", {}) @@ -629,7 +539,7 @@ def assert_uk_hmrc_income_source_contract_current( failures, "stage2.predictors", tuple(stage2.get("predictors", ())), - _STAGE2_SOURCE_FAITHFUL_PREDICTORS, + (*_STAGE2_SOURCE_FAITHFUL_PREDICTORS, "state_pension_receipt"), ) _expect( failures, @@ -900,32 +810,35 @@ def assert_uk_hmrc_income_source_contract_current( failures, "effective.fail_below_floor", effective.get("fail_below_floor"), True ) - _expect( - failures, - "stage.official_table_components", - tuple(stage.get("official_table_components", ())), - HMRC_SPI_INCOME_COMPONENTS, - ) - _expect( - failures, - "stage.donor_relief_outputs", - tuple(stage.get("donor_relief_outputs", ())), - HMRC_DISTRIBUTIONAL_INPUTS, + # The current spine declares new columns separately from rewrites; the + # legacy candidate stage's flat output list is not an ownership contract. + from microcosm.build.source_manifest import SourceStageSpec + from microcosm.build.uk_runtime.spi_spine import ( + UK_SPI_INCOME_SPINE_OUTPUT_COLUMNS, + UK_SPI_INCOME_SPINE_REWRITE_COLUMNS, + _assert_income_stage_parameters, + _support_stage_parameters, ) + + declared = set(stage.get("outputs", ())) | set(stage.get("rewrites", ())) _expect( failures, "stage.outputs", - tuple(stage.get("outputs", ())), - ( - *( - "hmrc_spi_state_pension_income" - if component == "state_pension" - else component - for component in HMRC_SPI_INCOME_COMPONENTS - ), - *HMRC_DISTRIBUTIONAL_INPUTS, - *SPI_HMRC_DERIVED_AUXILIARY_COLUMNS, - ), + declared, + set(UK_SPI_INCOME_SPINE_OUTPUT_COLUMNS) + | set(UK_SPI_INCOME_SPINE_REWRITE_COLUMNS), + ) + _raise_failures(failures) + declared_stages = payload["source_stages"] + _support_stage_parameters( + SourceStageSpec.from_mapping(declared_stages["spi_support_channel"]), + seed=42, + ) + _assert_income_stage_parameters( + SourceStageSpec.from_mapping(declared_stages["hmrc_spi_income_spine"]), + seed=42, + qrf_estimators=100, + donor_sample_size=DEFAULT_SPI_DONOR_SAMPLE_SIZE, ) _raise_failures(failures) @@ -988,71 +901,51 @@ def uk_hmrc_weighted_qrf_output_columns( def _load_payload(resource: Any | None) -> Mapping[str, Any]: - if resource is None: - frozen_payload = json.loads( - files("microcosm.build.uk") - .joinpath(UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE) - .read_text(encoding="utf-8") - ) - if not isinstance(frozen_payload, Mapping): - raise ValueError("UK HMRC source manifest root must be a JSON object.") - frozen_stages = frozen_payload.get("stages") - if not isinstance(frozen_stages, Sequence) or isinstance( - frozen_stages, (str, bytes) - ): - raise ValueError("UK HMRC source manifest stages must be a list.") - if len(frozen_stages) != 1 or not isinstance(frozen_stages[0], Mapping): - raise ValueError( - "UK HMRC source manifest must contain exactly one source stage." - ) - payload = json.loads( - files("microcosm.build.uk") - .joinpath(UK_CANONICAL_SOURCE_STAGES_RESOURCE) - .read_text(encoding="utf-8") - ) - if not isinstance(payload, Mapping): - raise ValueError("UK source manifest root must be a JSON object.") - stages = payload.get("stages") - if not isinstance(stages, Sequence) or isinstance(stages, (str, bytes)): - raise ValueError("UK source manifest stages must be a list.") - retained = [ - stage - for stage in stages - if isinstance(stage, Mapping) - and stage.get("stage") == "frs_hmrc_retained_leaves" - ] - hmrc = [ - stage - for stage in stages - if isinstance(stage, Mapping) and stage.get("stage") == "hmrc_spi_income" - ] - if len(retained) != 1 or len(hmrc) != 1: - raise ValueError( - "UK source manifest must contain exactly one " - "frs_hmrc_retained_leaves stage and one hmrc_spi_income stage." - ) - stage = copy.deepcopy(dict(hmrc[0])) - stage["base_candidate"] = copy.deepcopy( - dict(frozen_stages[0].get("base_candidate", {})) - ) - stage["operations"] = [ - *copy.deepcopy(list(retained[0].get("operations", ()))), - *copy.deepcopy(list(hmrc[0].get("operations", ()))), - ] - return { - "country": payload.get("country"), - "version": payload.get("version"), - "stages": [stage], - } - target = resource - if hasattr(target, "read_text"): - raw = target.read_text(encoding="utf-8") - else: - raw = Path(target).read_text(encoding="utf-8") + target = ( + files("microcosm.build.uk").joinpath(UK_CANONICAL_SOURCE_STAGES_RESOURCE) + if resource is None + else resource + ) + raw = ( + target.read_text(encoding="utf-8") + if hasattr(target, "read_text") + else Path(target).read_text(encoding="utf-8") + ) payload = json.loads(raw) if not isinstance(payload, Mapping): - raise ValueError("UK HMRC source manifest root must be a JSON object.") - return payload + raise ValueError("UK source manifest root must be a JSON object.") + stages = payload.get("stages") + if not isinstance(stages, list) or not all( + isinstance(stage, Mapping) for stage in stages + ): + raise ValueError("UK source manifest stages must be a list of objects.") + names = ( + "frs_hmrc_spine_leaves", + "spi_support_channel", + "spi_income_band_donors", + "hmrc_spi_income_spine", + ) + selected = {} + for name in names: + matches = [stage for stage in stages if stage.get("stage") == name] + if len(matches) != 1: + raise ValueError( + f"UK source manifest must contain exactly one {name} stage." + ) + selected[name] = matches[0] + # Audit the connected income family without inventing an executable stage. + income = dict(selected["hmrc_spi_income_spine"]) + income["operations"] = [ + operation + for name in names + for operation in selected[name].get("operations", ()) + ] + return { + "country": payload.get("country"), + "version": payload.get("version"), + "stages": [income], + "source_stages": selected, + } def _mapping(value: object, label: str, failures: list[str]) -> Mapping[str, Any]: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py index d8648137e..90c70456c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py @@ -185,28 +185,26 @@ def reviewed_exclusions(self) -> dict[str, str]: @property def required_build_stages(self) -> frozenset[str]: - """National stages that the checked-in family contract makes mandatory.""" + """Canonical producers and predecessors required by the family contract.""" return frozenset( - str(family["stage"]) - for family in self.family_coverage.values() - if family.get("status") == _REQUIRED_AT_BUILD_STATUS + stage + for stages in self.required_build_stage_options.values() + for stage in stages ) @property def required_build_stage_options(self) -> Mapping[str, tuple[str, ...]]: - """Per-family executable stage alternatives declared by the manifest.""" + """All mandatory stages per family; alternative producers are unsupported.""" - options: dict[str, tuple[str, ...]] = {} - for name, family in self.family_coverage.items(): - if family.get("status") != _REQUIRED_AT_BUILD_STATUS: - continue - stages = [str(family["stage"])] - superseded_by = family.get("superseded_by") - if isinstance(superseded_by, Mapping): - stages.append(str(superseded_by["stage"])) - options[str(name)] = tuple(dict.fromkeys(stages)) - return options + return { + str(name): ( + *tuple(family.get("required_predecessor_stages", ())), + str(family["stage"]), + ) + for name, family in self.family_coverage.items() + if family.get("status") == _REQUIRED_AT_BUILD_STATUS + } def _resource_text(resource: str) -> str: @@ -304,46 +302,24 @@ def _parse_family_coverage( f"{resource}: family {name!r} needs a lowercase SHA-256 " "for source_manifest_sha256." ) - superseded_by = raw_family.get("superseded_by") - parsed_superseded_by: dict[str, Any] | None = None - if superseded_by is not None: - if not isinstance(superseded_by, Mapping): - raise ValueError( - f"{resource}: family {name!r} superseded_by must be an object." - ) - superseding_stage = str(superseded_by.get("stage", "")).strip() - superseding_manifest = str(superseded_by.get("source_manifest", "")).strip() - superseding_sha = str( - superseded_by.get("source_manifest_sha256", "") - ).strip() - supersession_reason = str(superseded_by.get("reason", "")).strip() - if not superseding_stage: - raise ValueError( - f"{resource}: family {name!r} superseded_by needs a stage." - ) - if not superseding_manifest: - raise ValueError( - f"{resource}: family {name!r} superseded_by needs a " - "source_manifest." - ) - if len(superseding_sha) != 64 or any( - character not in "0123456789abcdef" for character in superseding_sha - ): - raise ValueError( - f"{resource}: family {name!r} superseded_by needs a " - "lowercase SHA-256 for source_manifest_sha256." - ) - if not supersession_reason: - raise ValueError( - f"{resource}: family {name!r} superseded_by needs a reason." - ) - parsed_superseded_by = { - **dict(superseded_by), - "stage": superseding_stage, - "source_manifest": superseding_manifest, - "source_manifest_sha256": superseding_sha, - "reason": supersession_reason, - } + if "superseded_by" in raw_family: + raise ValueError( + f"{resource}: family {name!r} must name its canonical producer; " + "superseded_by alternatives are no longer supported." + ) + predecessors = raw_family.get("required_predecessor_stages", []) + if ( + not isinstance(predecessors, list) + or any( + not isinstance(value, str) or not value.strip() + for value in predecessors + ) + or len(set(predecessors)) != len(predecessors) + or stage in predecessors + ): + raise ValueError( + f"{resource}: family {name!r} has invalid required_predecessor_stages." + ) try: base_candidate_tier = validate_uk_release_tier( raw_family.get("base_candidate_tier") @@ -436,11 +412,6 @@ def _parse_family_coverage( "stage": stage, "source_manifest": source_manifest, "source_manifest_sha256": source_manifest_sha256, - **( - {"superseded_by": parsed_superseded_by} - if parsed_superseded_by is not None - else {} - ), "base_candidate_tier": base_candidate_tier, "output_weight_kind": output_weight_kind, "required_mass_change_reason": required_mass_change_reason, @@ -1466,7 +1437,7 @@ def assert_uk_release_input_coverage_build_stages( missing = sorted( family for family, options in manifest.required_build_stage_options.items() - if actual.isdisjoint(options) + if not set(options).issubset(actual) ) if missing: raise ValueError( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py index e62b4fbb1..a3361b20a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py @@ -49,8 +49,6 @@ def _uk_nonnegative_outputs_by_stage() -> dict[str, tuple[str, ...]]: def uk_stage_implementations( *, - retained_leaves_transform: Callable[[Frame], Frame], - hmrc_income_transform: Callable[[Frame], Frame], frs_spine_transform: Callable[[Frame], Frame] | None = None, frs_relationships_transform: Callable[[Frame], Frame] | None = None, frs_employment_transform: Callable[[Frame], Frame] | None = None, @@ -84,10 +82,6 @@ def uk_stage_implementations( """Return the whole-stage implementation map for the UK source plan.""" implementations = { - "frs_hmrc_retained_leaves": retained_leaves_transform, - "hmrc_spi_income": hmrc_income_transform, - } - optional = { "frs_spine": frs_spine_transform, "frs_relationships": frs_relationships_transform, "frs_employment": frs_employment_transform, @@ -118,14 +112,11 @@ def uk_stage_implementations( "salary_sacrifice": salary_sacrifice_transform, "student_loans": student_loans_transform, } - implementations.update( - { - name: transform - for name, transform in optional.items() - if transform is not None - } - ) - return implementations + return { + name: transform + for name, transform in implementations.items() + if transform is not None + } def uk_source_operation_handlers() -> Mapping[str, SourceOperationHandler]: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py index afd8c0ece..71cea6dc6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py @@ -18,7 +18,7 @@ UKDWPDisabilityCategoryRates, UKDWPDisabilityFlagRates, ) -from microcosm.build.uk_runtime.frs_hmrc_leaves import ( +from microcosm.build.uk_runtime.frs_hmrc_source import ( FRS_HMRC_INCPBEN_COLUMN, FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, FRS_HMRC_PAY_COLUMN, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py index 65f8d2066..3fcefa253 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py @@ -12,7 +12,7 @@ from microcosm.build.gates import FitWeightRecord from microcosm.build.source_manifest import SourceStageSpec -from microcosm.build.uk_runtime.frs_hmrc_leaves import ( +from microcosm.build.uk_runtime.frs_hmrc_source import ( FRS_HMRC_INCPBEN_COLUMN, FRS_HMRC_PAY_COLUMN, FRS_HMRC_RETAINED_LEAF_COLUMNS, @@ -298,11 +298,13 @@ def __call__(self, frame: Frame) -> Frame: adult, adult_identity = _read_raw_frs_table( self.frs_raw_dir / str(artifacts["adult"]["locator"]), expected_filename="adult.tab", + source_vintage=str(artifacts["adult"].get("vintage", "unspecified")), required_columns=("sernum", "person", "inearns"), ) benefits, benefits_identity = _read_raw_frs_table( self.frs_raw_dir / str(artifacts["benefits"]["locator"]), expected_filename="benefits.tab", + source_vintage=str(artifacts["benefits"].get("vintage", "unspecified")), required_columns=("sernum", "person", "benefit", "benamt", "var2"), ) _assert_identity_matches_artifact(adult_identity.evidence(), artifacts["adult"]) @@ -320,7 +322,7 @@ def __call__(self, frame: Frame) -> Frame: if missing_ids: # A rung sample deliberately drops most source people; restrict # the raw surface to the survivors (the full-scale fence above - # stays strict — mirrors frs_hmrc_leaves' sampled_rung posture). + # stays strict). source_leaves = source_leaves.loc[ source_leaves.index.isin(person["person_id"].to_numpy()) ] diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py index fbd1158fe..0f9325f41 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py @@ -96,7 +96,6 @@ ) from microcosm.build.uk_runtime.frs_take_up import UKFRSTakeUpStageTransform from microcosm.build.uk_runtime.graph import ( - UK_SPINE_EXCLUSIONS, uk_registry, uk_spine_graph, uk_spine_operation_inventory, @@ -175,11 +174,7 @@ def _uk_spine_stage_names(spec) -> tuple[str, ...]: if spec.sources is None: raise ValueError("UK country spec has no source stages.") - declared = { - stage.stage - for stage in spec.sources.stages - if stage.stage not in UK_SPINE_EXCLUSIONS - } + declared = {stage.stage for stage in spec.sources.stages} compiled = compile_graph(uk_spine_graph(spec)) ordered = tuple(node_id for node_id in compiled.order if node_id in declared) if set(ordered) != declared: diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine/uk/test_uk_graph.py index e0146b07a..e3cb52046 100644 --- a/packages/microcosm-build/tests/engine/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine/uk/test_uk_graph.py @@ -59,11 +59,7 @@ def test_driver_projects_a_stage_record_for_every_graph_stage_on_the_fixture( from microcosm.build.uk_runtime import spine_build as driver country = load_country_spec("uk") - stages = [ - stage - for stage in country.sources.stages - if stage.stage not in UK_SPINE_EXCLUSIONS - ] + stages = list(country.sources.stages) _, implementations = fixture_stage_plan_inputs(fixture / "sources") graph = uk_spine_graph() compiled = compile_graph(graph) diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_release_input_coverage.py b/packages/microcosm-build/tests/engine/uk/test_uk_release_input_coverage.py index 5339e008d..b8049590b 100644 --- a/packages/microcosm-build/tests/engine/uk/test_uk_release_input_coverage.py +++ b/packages/microcosm-build/tests/engine/uk/test_uk_release_input_coverage.py @@ -444,7 +444,10 @@ def test_shipped_manifest_is_current(self) -> None: assert load_efrs_parity_known_gaps() == () assert manifest.required_build_stages == frozenset( { - "hmrc_spi_income", + "frs_hmrc_spine_leaves", + "spi_support_channel", + "spi_income_band_donors", + "hmrc_spi_income_spine", "cgt_incidence_clone", "cgt_band_donors", "hmrc_cgt_gains_spine", @@ -569,43 +572,48 @@ def test_required_family_stage_cannot_be_omitted(self) -> None: assert_uk_release_input_coverage_build_stages((), manifest=manifest) result = assert_uk_release_input_coverage_build_stages( - ("hmrc_spi_income",), + ("hmrc_spi_income_spine",), manifest=manifest, ) assert result is None - def test_spine_posture_satisfies_superseded_required_families(self) -> None: + def test_canonical_producers_and_predecessors_satisfy_required_families( + self, + ) -> None: manifest = load_uk_release_input_coverage_manifest() - spine_stages = tuple( - stage - for stage in manifest.required_build_stages - if stage != "hmrc_spi_income" - ) - result = assert_uk_release_input_coverage_build_stages( - (*spine_stages, "hmrc_spi_income_spine"), - manifest=manifest, - ) - assert result is None - assert ( - manifest.family_coverage["hmrc_spi_income"]["superseded_by"]["stage"] - == "hmrc_spi_income_spine" + assert_uk_release_input_coverage_build_stages( + tuple(manifest.required_build_stages), manifest=manifest + ) + assert { + "frs_hmrc_spine_leaves", + "spi_support_channel", + "spi_income_band_donors", + "hmrc_spi_income_spine", + } <= manifest.required_build_stages + assert {"hmrc_spi_income", "hmrc_cgt_gains"}.isdisjoint( + manifest.required_build_stages ) assert "hmrc_cgt_gains" not in manifest.family_coverage + assert all( + "superseded_by" not in family + for family in manifest.family_coverage.values() + ) - def test_supersession_does_not_hide_a_genuinely_missing_family(self) -> None: + @pytest.mark.parametrize( + "missing_stage", + [ + "frs_hmrc_spine_leaves", + "spi_support_channel", + "spi_income_band_donors", + "hmrc_spi_income_spine", + "student_loans", + ], + ) + def test_each_canonical_family_dependency_is_required(self, missing_stage) -> None: manifest = load_uk_release_input_coverage_manifest() - spine_stages = tuple( - stage - for stage in manifest.required_build_stages - if stage - not in { - "hmrc_spi_income", - "student_loans", - } - ) - with pytest.raises(ValueError, match="student_loans"): + with pytest.raises(ValueError, match="hmrc_spi_income|student_loans"): assert_uk_release_input_coverage_build_stages( - (*spine_stages, "hmrc_spi_income_spine"), + tuple(manifest.required_build_stages - {missing_stage}), manifest=manifest, ) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py b/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py index 2b88ca2df..213b2b857 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py @@ -938,7 +938,6 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "cgt_band_donor_support_bounds.json", "hmrc_income_release_gate_report.json", "hmrc_income_replay_report.json", - "hmrc_income_source_stages.json", "ofgem_region_crosswalk.json", "etb_policy_anchors.json", "etb_services_anchors.json", @@ -999,16 +998,20 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "nts_car_availability.json", ) - def test_uk_source_manifest_loads_thirty_stages(self) -> None: + def test_uk_source_manifest_contains_only_canonical_spine_stages(self) -> None: spec = load_country_spec("uk") assert spec.sources is not None - # 33 spine stages (uc_reporter_redraw #832, uc_deduction_attributes + # 34 spine stages (uc_reporter_redraw #832, uc_deduction_attributes # #685, frs_relationships #791, hmrc_cgt_asset_type_spine #725, # cgt_incidence_anchor #970, nts_bus_travel #930, then - # spi_income_band_donors (PolicyEngine/chronicle#280 lane) as the newest) plus the two - # certified-pair stages the June path still uses. - assert len(spec.sources.stages) == 36 + # spi_income_band_donors (PolicyEngine/chronicle#280 lane) and the SPI + # housing shell (#1012) as the newest); the frs_hmrc_retained_leaves / + # hmrc_spi_income pair is retired (#901). + assert len(spec.sources.stages) == 34 + assert not {"frs_hmrc_retained_leaves", "hmrc_spi_income"}.intersection( + stage.stage for stage in spec.sources.stages + ) class TestExistingPackagesGeneralize: @@ -1053,7 +1056,6 @@ def test_uk_package_loads(self) -> None: "cgt_band_donor_support_bounds.json", "hmrc_income_release_gate_report.json", "hmrc_income_replay_report.json", - "hmrc_income_source_stages.json", "ofgem_region_crosswalk.json", "etb_policy_anchors.json", "etb_services_anchors.json", diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_battery_bindings.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_battery_bindings.py index 34c8b9b60..713924fa9 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_battery_bindings.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_battery_bindings.py @@ -250,9 +250,9 @@ def _run_battery( artifacts: dict[str, object] = { "coverage_engine": object(), "exclusions_evaluated_on": clock, - # The staging pipeline's two scheduled stages declare no nonnegative - # outputs, so the nonnegative gate passes with zero required columns. - "build_stage_names": ("frs_hmrc_retained_leaves", "hmrc_spi_income"), + # This binding fixture schedules a stage with no declared nonnegative + # outputs; canonical family completeness has dedicated tests. + "build_stage_names": ("frs_household_draws",), } if fit_records is not None: artifacts["fit_weight_records"] = fit_records @@ -484,18 +484,12 @@ def test_nonnegative_binding_passes_clean_scheduled_columns(self) -> None: assert result.passed is True def test_nonnegative_binding_does_not_demand_unscheduled_stages(self) -> None: - # The national staging build schedules only the two HMRC stages, - # which declare no nonnegative outputs — the gate passes honestly - # with zero required columns rather than by silent pre-filtering. + # This isolated stage declares no nonnegative outputs. Outputs of + # unscheduled employment and income stages must not be demanded. binding = UK_GATE_REGISTRY["nonnegative_columns"] context = EvidenceContext( frame=self._nonnegative_frame(sic=None), - artifacts={ - "build_stage_names": ( - "frs_hmrc_retained_leaves", - "hmrc_spi_income", - ) - }, + artifacts={"build_stage_names": ("frs_household_draws",)}, ) result = binding.evaluate(context, {}) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_hmrc_leaves.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_hmrc_leaves.py deleted file mode 100644 index 80d81d5ee..000000000 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_hmrc_leaves.py +++ /dev/null @@ -1,556 +0,0 @@ -from __future__ import annotations - -import hashlib -from pathlib import Path - -import numpy as np -import pandas as pd -import pytest - -from microcosm.build.uk_runtime.frs_hmrc_leaves import ( - FRS_HMRC_INCPBEN_COLUMN, - FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, - FRS_HMRC_PAY_COLUMN, - FRS_HMRC_RETAINED_LEAF_COLUMNS, - FRS_HMRC_RETAINED_LEAF_SOURCE_EVIDENCE, - FRS_HMRC_RETAINED_LEAVES_STAGE_NAME, - FRS_HMRC_SRP_REGULAR_CODE5_COLUMN, - FRS_HMRC_UBISJA_COLUMN, - FRS_WEEKS_IN_YEAR, - UKFRSHMRCRetainedLeavesStageTransform, - retain_uk_frs_hmrc_leaves, -) -from microcosm.build.uk_runtime.national_frame import ( - UKNationalStage, - uk_national_frame, -) -from microcosm.build.uk_runtime.spi_support import ( - SPI_HMRC_EMPLOYMENT_BENEFITS_COLUMN, - SPI_HMRC_EMPLOYMENT_EXPENSES_COLUMN, - SPI_HMRC_MISCELLANEOUS_EMPLOYMENT_INCOME_COLUMN, - SPI_HMRC_OTHER_INCOME_COLUMN, - SPI_HMRC_OTHER_SOCIAL_SECURITY_INCOME_COLUMN, - SPI_HMRC_STATE_PENSION_INCOME_COLUMN, - SPI_HMRC_TAXABLE_TERMINATION_PAY_COLUMN, -) -from microcosm.frame import Frame - - -def _candidate() -> tuple[Frame, np.ndarray]: - raw_households = { - 1: (1001, 1002), - 2: (2001,), - } - raw_person_ids = tuple( - person_id for people in raw_households.values() for person_id in people - ) - spi_person_offset = max(raw_person_ids) + 1 - spi_household_offset = max(raw_households) + 1 - - pre_capital_stacks = [ - (household_id, False, people) for household_id, people in raw_households.items() - ] - pre_capital_stacks.append( - ( - 1 + spi_household_offset, - True, - tuple(person_id + spi_person_offset for person_id in raw_households[1]), - ) - ) - capital_person_offset = ( - max( - person_id - for _household_id, _spi, people in pre_capital_stacks - for person_id in people - ) - + 1 - ) - capital_household_offset = ( - max(household_id for household_id, _spi, _people in pre_capital_stacks) + 1 - ) - clone_zero_stacks = [ - (household_id, spi, False, people) - for household_id, spi, people in pre_capital_stacks - ] + [ - ( - household_id + capital_household_offset, - spi, - True, - tuple(person_id + capital_person_offset for person_id in people), - ) - for household_id, spi, people in pre_capital_stacks - ] - clone_multiplier = 10 ** len( - str( - max( - person_id - for _household_id, _spi, _capital, people in clone_zero_stacks - for person_id in people - ) - ) - ) - - household_rows: list[dict[str, object]] = [] - person_rows: list[dict[str, object]] = [] - for clone_index in range(2): - clone_offset = clone_index * clone_multiplier - for household_id, spi, capital_gains, people in clone_zero_stacks: - descendant_household_id = household_id + clone_offset - household_rows.append( - { - "household_id": descendant_household_id, - "household_weight": 0.0 if spi else 1.0, - "clone_index": clone_index, - "household_is_spi_synthetic": spi, - "household_is_capital_gains_clone": capital_gains, - } - ) - for person_id in people: - descendant_person_id = person_id + clone_offset - source_person_id = ( - person_id - - int(spi) * spi_person_offset - - int(capital_gains) * capital_person_offset - ) - person_rows.append( - { - "person_id": descendant_person_id, - "person_household_id": descendant_household_id, - "person_benunit_id": descendant_person_id, - "expected_source_person_id": source_person_id, - } - ) - # Frame requires group ids sorted ascending (it raises, never reorders), - # so the group tables sort; the person table stays SHUFFLED, which keeps - # this fixture's teeth: person row i never corresponds positionally to - # household row i, so lineage resolution must stay id-keyed — the - # 2024-25 FRS bug class this candidate exists to catch. - household = pd.DataFrame(household_rows).sort_values( - "household_id", ignore_index=True - ) - person = pd.DataFrame(person_rows).sample( - frac=1.0, random_state=7, ignore_index=True - ) - source_person_ids = person.pop("expected_source_person_id").to_numpy(dtype=int) - benunit = pd.DataFrame( - {"benunit_id": person["person_benunit_id"].copy()} - ).sort_values("benunit_id", ignore_index=True) - return ( - uk_national_frame( - person=person, - benunit=benunit, - household=household, - time_period="2023", - ), - source_person_ids, - ) - - -def _write_raw_tables( - directory: Path, - *, - include_incapacity: bool = True, -) -> tuple[Path, Path]: - adult_path = directory / "adult.tab" - benefits_path = directory / "benefits.tab" - pd.DataFrame( - { - "SERNUM": [1, 1, 2], - "PERSON": [1, 2, 1], - "INEARNS": [100.0, -1.0, 50.0], - "UNUSED": ["not", "extracted", "ever"], - } - ).to_csv(adult_path, sep="\t", index=False) - benefit_rows = [ - {"SERNUM": 1, "PERSON": 1, "BENEFIT": 14, "BENAMT": 10.0, "VAR2": 0}, - {"SERNUM": 1, "PERSON": 1, "BENEFIT": 19, "BENAMT": 2.0, "VAR2": 0}, - {"SERNUM": 1, "PERSON": 2, "BENEFIT": 13, "BENAMT": 4.0, "VAR2": 0}, - {"SERNUM": 1, "PERSON": 2, "BENEFIT": 16, "BENAMT": 5.0, "VAR2": 1}, - {"SERNUM": 1, "PERSON": 2, "BENEFIT": 16, "BENAMT": 99.0, "VAR2": 2}, - {"SERNUM": 1, "PERSON": 2, "BENEFIT": 16, "BENAMT": 6.0, "VAR2": 3}, - {"SERNUM": 2, "PERSON": 1, "BENEFIT": 5, "BENAMT": 6.0, "VAR2": 0}, - {"SERNUM": 2, "PERSON": 1, "BENEFIT": 999, "BENAMT": -1.0, "VAR2": 0}, - ] - if include_incapacity: - benefit_rows.append( - { - "SERNUM": 2, - "PERSON": 1, - "BENEFIT": 17, - "BENAMT": 3.0, - "VAR2": 0, - } - ) - pd.DataFrame(benefit_rows).to_csv(benefits_path, sep="\t", index=False) - return adult_path, benefits_path - - -def _expected_leaves(source_person_ids: np.ndarray) -> pd.DataFrame: - weekly = { - 1001: { - FRS_HMRC_PAY_COLUMN: 100.0, - FRS_HMRC_UBISJA_COLUMN: 12.0, - }, - 1002: { - FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN: 15.0, - }, - 2001: { - FRS_HMRC_PAY_COLUMN: 50.0, - FRS_HMRC_INCPBEN_COLUMN: 3.0, - FRS_HMRC_SRP_REGULAR_CODE5_COLUMN: 6.0, - }, - } - expected = pd.DataFrame( - 0.0, - index=np.arange(len(source_person_ids)), - columns=FRS_HMRC_RETAINED_LEAF_COLUMNS, - ) - for index, source_person_id in enumerate(source_person_ids): - for column, value in weekly[int(source_person_id)].items(): - expected.loc[index, column] = value * FRS_WEEKS_IN_YEAR - return expected - - -def test_retains_source_faithful_leaves_across_all_candidate_descendants( - tmp_path: Path, -) -> None: - dataset, source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path) - - result = retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - actual = result.frame.person.loc[:, list(FRS_HMRC_RETAINED_LEAF_COLUMNS)] - assert np.array_equal( - actual.to_numpy(), _expected_leaves(source_person_ids).to_numpy() - ) - assert result.clone_id_multiplier == 10_000 - assert result.spi_person_id_offset == 2_002 - assert result.capital_gains_person_id_offset == 3_005 - assert result.raw_source_people == 3 - assert result.candidate_people == len(dataset.person) - assert result.structural_zero_columns == () - assert result.source_signal_rows == { - FRS_HMRC_PAY_COLUMN: 2, - FRS_HMRC_UBISJA_COLUMN: 1, - FRS_HMRC_INCPBEN_COLUMN: 1, - FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN: 1, - FRS_HMRC_SRP_REGULAR_CODE5_COLUMN: 1, - } - assert ( - result.adult_source.sha256 - == hashlib.sha256(adult_path.read_bytes()).hexdigest() - ) - assert ( - result.benefits_source.sha256 - == hashlib.sha256(benefits_path.read_bytes()).hexdigest() - ) - assert result.adult_source.extracted_columns == ("sernum", "person", "inearns") - evidence = result.evidence() - assert evidence["stage"] == FRS_HMRC_RETAINED_LEAVES_STAGE_NAME - assert evidence["retained_leaves"][FRS_HMRC_PAY_COLUMN]["spi_concept"] == "PAY" - assert ( - evidence["retained_leaves"][FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN]["scope"] - == "identifiable_subset" - ) - - -def test_partial_leaves_never_populate_full_ossben_or_srp_columns( - tmp_path: Path, -) -> None: - dataset, _source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path) - - result = retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - forbidden = { - SPI_HMRC_EMPLOYMENT_BENEFITS_COLUMN, - SPI_HMRC_EMPLOYMENT_EXPENSES_COLUMN, - SPI_HMRC_OTHER_SOCIAL_SECURITY_INCOME_COLUMN, - SPI_HMRC_TAXABLE_TERMINATION_PAY_COLUMN, - SPI_HMRC_MISCELLANEOUS_EMPLOYMENT_INCOME_COLUMN, - SPI_HMRC_OTHER_INCOME_COLUMN, - SPI_HMRC_STATE_PENSION_INCOME_COLUMN, - } - assert forbidden.isdisjoint(result.frame.person.columns) - assert ( - FRS_HMRC_RETAINED_LEAF_SOURCE_EVIDENCE[ - FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN - ]["scope"] - == "identifiable_subset" - ) - - -def test_incpben_is_an_honest_structural_zero_when_code17_is_unobserved( - tmp_path: Path, -) -> None: - dataset, _source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path, include_incapacity=False) - - result = retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - assert (result.frame.person[FRS_HMRC_INCPBEN_COLUMN] == 0.0).all() - assert FRS_HMRC_INCPBEN_COLUMN in result.structural_zero_columns - assert result.evidence()["retained_leaves"][FRS_HMRC_INCPBEN_COLUMN][ - "structural_zero" - ] - - -def test_future_code17_observation_flows_without_a_schema_change( - tmp_path: Path, -) -> None: - dataset, source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path, include_incapacity=True) - - result = retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - expected = np.where(source_person_ids == 2001, 3.0 * FRS_WEEKS_IN_YEAR, 0.0) - assert np.array_equal( - result.frame.person[FRS_HMRC_INCPBEN_COLUMN].to_numpy(), expected - ) - assert FRS_HMRC_INCPBEN_COLUMN not in result.structural_zero_columns - - -def test_transform_is_national_stage_compatible_and_retains_evidence( - tmp_path: Path, -) -> None: - dataset, _source_person_ids = _candidate() - _write_raw_tables(tmp_path) - transform = UKFRSHMRCRetainedLeavesStageTransform.from_raw_frs_directory(tmp_path) - stage = UKNationalStage( - name=FRS_HMRC_RETAINED_LEAVES_STAGE_NAME, - transform=transform, - ) - - staged = stage.run(dataset) - - assert set(FRS_HMRC_RETAINED_LEAF_COLUMNS).issubset(staged.person.columns) - assert transform.last_result is not None - assert transform.last_result.frame is staged - - -def test_raw_source_identity_must_exist_on_candidate_base(tmp_path: Path) -> None: - dataset, _source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path) - adult = pd.read_csv(adult_path, sep="\t") - adult.loc[len(adult)] = {"SERNUM": 9, "PERSON": 1, "INEARNS": 1, "UNUSED": "x"} - adult.to_csv(adult_path, sep="\t", index=False) - - with pytest.raises(ValueError, match="absent from the certified candidate base"): - retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - -def test_sampled_rung_receipts_the_dropped_raw_surface(tmp_path: Path) -> None: - """A #627 rung build restricts the raw surface and receipts the drop. - - The completeness fence (every raw-survey person present in the base) - cannot hold when the base deliberately carries a sampled subset of - source families; declaring ``sampled_rung`` converts the raise into a - receipted count while the surviving surface stays source-faithful. - """ - - dataset, _source_person_ids = _candidate() - (tmp_path / "clean").mkdir() - (tmp_path / "extra").mkdir() - clean_adult_path, clean_benefits_path = _write_raw_tables(tmp_path / "clean") - strict = retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=clean_adult_path, - benefits_tab_path=clean_benefits_path, - ) - adult_path, benefits_path = _write_raw_tables(tmp_path / "extra") - adult = pd.read_csv(adult_path, sep="\t") - adult.loc[len(adult)] = {"SERNUM": 9, "PERSON": 1, "INEARNS": 1, "UNUSED": "x"} - adult.to_csv(adult_path, sep="\t", index=False) - - result = retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - sampled_rung=True, - ) - - assert result.source_people_outside_candidate == 1 - assert result.evidence()["lineage"]["source_people_outside_candidate"] == 1 - assert strict.source_people_outside_candidate == 0 - # The surviving surface attaches exactly what the strict run attaches. - pd.testing.assert_frame_equal( - result.frame.table("person"), strict.frame.table("person") - ) - # Signal-row evidence remains a fact about the SOURCE: the extra raw - # person's pay carrier is counted even though the rung dropped the row, - # so structural_zero can never be asserted from a sampled-away surface. - assert ( - result.source_signal_rows[FRS_HMRC_PAY_COLUMN] - == strict.source_signal_rows[FRS_HMRC_PAY_COLUMN] + 1 - ) - - -def test_candidate_clone_identity_mismatch_fails_closed(tmp_path: Path) -> None: - dataset, _source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path) - person = dataset.person.copy() - household = dataset.table("household") - clone_households = set(household.loc[household["clone_index"] == 1, "household_id"]) - tampered_row = person["person_household_id"].isin(clone_households).idxmax() - person.loc[tampered_row, "person_id"] += 500 - tampered = uk_national_frame( - person=person, - benunit=dataset.table("benunit"), - household=household, - time_period="2023", - household_weights=dataset.weights_for("household").values, - ) - - with pytest.raises(ValueError, match="person IDs do not reverse"): - retain_uk_frs_hmrc_leaves( - tampered, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - -@pytest.mark.parametrize( - ("column", "message"), - [ - ("INEARNS", "missing required column"), - ("PERSON", "missing required column"), - ], -) -def test_missing_required_raw_column_fails_closed( - tmp_path: Path, - column: str, - message: str, -) -> None: - dataset, _source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path) - adult = pd.read_csv(adult_path, sep="\t").drop(columns=[column]) - adult.to_csv(adult_path, sep="\t", index=False) - - with pytest.raises(ValueError, match=message): - retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - -def test_negative_relevant_benefit_amount_fails_closed(tmp_path: Path) -> None: - dataset, _source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path) - benefits = pd.read_csv(benefits_path, sep="\t") - benefits.loc[benefits["BENEFIT"] == 14, "BENAMT"] = -1.0 - benefits.to_csv(benefits_path, sep="\t", index=False) - - with pytest.raises(ValueError, match="must be non-negative"): - retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - -@pytest.mark.parametrize( - ("table", "column", "message"), - [ - ("adult", "INEARNS", "ADULT.INEARNS"), - ("benefits", "BENAMT", "relevant BENEFITS.BENAMT"), - ], -) -def test_nonfinite_source_amount_fails_closed( - tmp_path: Path, - table: str, - column: str, - message: str, -) -> None: - dataset, _source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path) - path = adult_path if table == "adult" else benefits_path - frame = pd.read_csv(path, sep="\t") - frame.loc[0, column] = np.inf - frame.to_csv(path, sep="\t", index=False) - - with pytest.raises(ValueError, match=message): - retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - -def test_missing_code16_var2_fails_closed(tmp_path: Path) -> None: - dataset, _source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path) - benefits = pd.read_csv(benefits_path, sep="\t") - benefits.loc[benefits["BENEFIT"] == 16, "VAR2"] = np.nan - benefits.to_csv(benefits_path, sep="\t", index=False) - - with pytest.raises(ValueError, match="VAR2 for BENEFIT=16"): - retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - -def test_checkpoint_metadata_round_trips_the_descent_evidence(tmp_path) -> None: - """A fresh process resumes the retained stage from its record alone. - - The rehydrated result exposes exactly the surface the SPI stage's - descent fence reads — evidence and both content identities — and a - checkpoint whose content no longer matches its recorded output identity - is refused as drifted. - """ - - from microcosm.build.uk_runtime.content_identity import ( - uk_frame_content_identity, - ) - - dataset, _source_person_ids = _candidate() - _write_raw_tables(tmp_path) - transform = UKFRSHMRCRetainedLeavesStageTransform.from_raw_frs_directory(tmp_path) - staged = transform(dataset) - metadata = transform.checkpoint_metadata() - - resumed = UKFRSHMRCRetainedLeavesStageTransform.from_raw_frs_directory(tmp_path) - resumed.resume_from_checkpoint(metadata, staged) - assert resumed.last_result is not None - assert resumed.last_result.frame is staged - assert resumed.last_result.evidence() == transform.last_result.evidence() - assert resumed.last_result.input_content_identity == uk_frame_content_identity( - dataset - ) - assert resumed.last_result.output_content_identity == uk_frame_content_identity( - staged - ) - - drifted = UKFRSHMRCRetainedLeavesStageTransform.from_raw_frs_directory(tmp_path) - with pytest.raises(RuntimeError, match="drifted record"): - drifted.resume_from_checkpoint(metadata, dataset) - - empty = UKFRSHMRCRetainedLeavesStageTransform.from_raw_frs_directory(tmp_path) - with pytest.raises(RuntimeError, match="cannot prove descent"): - empty.resume_from_checkpoint({}, staged) - - unrun = UKFRSHMRCRetainedLeavesStageTransform.from_raw_frs_directory(tmp_path) - with pytest.raises(RuntimeError, match="completed retained-leaves run"): - unrun.checkpoint_metadata() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_hmrc_source.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_hmrc_source.py new file mode 100644 index 000000000..03c657362 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_hmrc_source.py @@ -0,0 +1,149 @@ +"""Raw FRS extraction shared by the canonical spine income stages.""" + +from __future__ import annotations + +import hashlib +import importlib.util +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime.frs_hmrc_source import ( + FRS_HMRC_INCPBEN_COLUMN, + FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, + FRS_HMRC_PAY_COLUMN, + FRS_HMRC_RETAINED_LEAF_COLUMNS, + FRS_HMRC_SRP_REGULAR_CODE5_COLUMN, + FRS_HMRC_UBISJA_COLUMN, + FRS_WEEKS_IN_YEAR, + _materialize_source_leaves, + _read_raw_frs_table, +) + + +def _write_raw_tables( + directory: Path, + *, + include_incapacity: bool = True, +) -> tuple[Path, Path]: + adult_path = directory / "adult.tab" + benefits_path = directory / "benefits.tab" + pd.DataFrame( + { + "SERNUM": [1, 1, 2], + "PERSON": [1, 2, 1], + "INEARNS": [100.0, -1.0, 50.0], + "UNUSED": ["not", "extracted", "ever"], + } + ).to_csv(adult_path, sep="\t", index=False) + benefit_rows = [ + {"SERNUM": 1, "PERSON": 1, "BENEFIT": 14, "BENAMT": 10.0, "VAR2": 0}, + {"SERNUM": 1, "PERSON": 1, "BENEFIT": 19, "BENAMT": 2.0, "VAR2": 0}, + {"SERNUM": 1, "PERSON": 2, "BENEFIT": 13, "BENAMT": 4.0, "VAR2": 0}, + {"SERNUM": 1, "PERSON": 2, "BENEFIT": 16, "BENAMT": 5.0, "VAR2": 1}, + {"SERNUM": 1, "PERSON": 2, "BENEFIT": 16, "BENAMT": 99.0, "VAR2": 2}, + {"SERNUM": 1, "PERSON": 2, "BENEFIT": 16, "BENAMT": 6.0, "VAR2": 3}, + {"SERNUM": 2, "PERSON": 1, "BENEFIT": 5, "BENAMT": 6.0, "VAR2": 0}, + {"SERNUM": 2, "PERSON": 1, "BENEFIT": 999, "BENAMT": -1.0, "VAR2": 0}, + ] + if include_incapacity: + benefit_rows.append( + { + "SERNUM": 2, + "PERSON": 1, + "BENEFIT": 17, + "BENAMT": 3.0, + "VAR2": 0, + } + ) + pd.DataFrame(benefit_rows).to_csv(benefits_path, sep="\t", index=False) + return adult_path, benefits_path + + +def _expected_leaves(source_person_ids: np.ndarray) -> pd.DataFrame: + weekly = { + 1001: { + FRS_HMRC_PAY_COLUMN: 100.0, + FRS_HMRC_UBISJA_COLUMN: 12.0, + }, + 1002: { + FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN: 15.0, + }, + 2001: { + FRS_HMRC_PAY_COLUMN: 50.0, + FRS_HMRC_INCPBEN_COLUMN: 3.0, + FRS_HMRC_SRP_REGULAR_CODE5_COLUMN: 6.0, + }, + } + expected = pd.DataFrame( + 0.0, + index=np.arange(len(source_person_ids)), + columns=FRS_HMRC_RETAINED_LEAF_COLUMNS, + ) + for index, source_person_id in enumerate(source_person_ids): + for column, value in weekly[int(source_person_id)].items(): + expected.loc[index, column] = value * FRS_WEEKS_IN_YEAR + return expected + + +def _read_sources(directory): + adult, identity = _read_raw_frs_table( + directory / "adult.tab", + expected_filename="adult.tab", + source_vintage="2024-25", + required_columns=("sernum", "person", "inearns"), + ) + benefits, _ = _read_raw_frs_table( + directory / "benefits.tab", + expected_filename="benefits.tab", + required_columns=("sernum", "person", "benefit", "benamt", "var2"), + ) + return adult, benefits, identity + + +def test_raw_source_extraction_preserves_income_concepts(tmp_path): + adult_path, _ = _write_raw_tables(tmp_path) + adult, benefits, identity = _read_sources(tmp_path) + actual = _materialize_source_leaves(adult, benefits) + expected = _expected_leaves(actual.index.to_numpy()) + pd.testing.assert_frame_equal(actual.reset_index(drop=True), expected) + assert list(adult) == ["sernum", "person", "inearns"] + assert identity.sha256 == hashlib.sha256(adult_path.read_bytes()).hexdigest() + assert identity.rows == 3 + + +def test_absent_incapacity_stays_zero(tmp_path): + _write_raw_tables(tmp_path, include_incapacity=False) + adult, benefits, _ = _read_sources(tmp_path) + assert ( + _materialize_source_leaves(adult, benefits)[FRS_HMRC_INCPBEN_COLUMN] == 0 + ).all() + + +@pytest.mark.parametrize( + "mutation,match", + [ + ("duplicate", "unique"), + ("negative_benefit", "non-negative"), + ("nan_earnings", "finite"), + ], +) +def test_invalid_raw_source_values_are_refused(tmp_path, mutation, match): + _write_raw_tables(tmp_path) + adult, benefits, _ = _read_sources(tmp_path) + if mutation == "duplicate": + adult = pd.concat([adult, adult.iloc[:1]], ignore_index=True) + elif mutation == "negative_benefit": + benefits.loc[0, "benamt"] = -1 + else: + adult.loc[0, "inearns"] = np.nan + with pytest.raises(ValueError, match=match): + _materialize_source_leaves(adult, benefits) + + +def test_candidate_restoration_module_is_removed(): + assert ( + importlib.util.find_spec("microcosm.build.uk_runtime.frs_hmrc_leaves") is None + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py index f67671cca..f1e1efb29 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py @@ -523,3 +523,48 @@ def refuse_transferred(compiled, **kwargs): index["artifacts"]["spine.gates.assembled/gate_report"]["sha256"] == hashlib.sha256(gate_bytes).hexdigest() ) + + +def test_main_stages_the_bundle_locally_with_staging_local_only(tmp_path, monkeypatch): + """``--staging-local-only`` writes and validates the v2 bundle, uploads nothing.""" + + pytest.importorskip("tables") + from microcosm.build.staging_dataset import ( + SHA256SUMS_FILENAME, + STAGED_MANIFEST_FILENAME, + ) + from microcosm.build.staging_v2 import validate_v2_bundle + + staging_dir = tmp_path / "staging-bundle" + out = graph_dense_bundle( + tmp_path, + monkeypatch, + "--staging-dir", + str(staging_dir), + staging="--staging-local-only", + ) + manifest = json.loads((out / "rowwise_candidate_manifest.json").read_text()) + assert manifest["staging_delivery"]["mode"] == "local_only" + assert manifest["staging_delivery"]["upload_attempts"] == 0 + assert manifest["staged_dataset"]["mode"] == "local_only" + assert manifest["staged_dataset"]["status"] == "skipped" + assert manifest["staged_dataset"]["repository"] is None + # The published bundle carries its own inventory beside the manifest. + assert (out / STAGED_MANIFEST_FILENAME).is_file() + assert (out / SHA256SUMS_FILENAME).is_file() + + runs = sorted(path.name for path in (staging_dir / "runs").iterdir()) + assert len(runs) == 1 + bundle = validate_v2_bundle(staging_dir, runs[0]) + assert bundle["progress"]["status"] == "completed" + assert bundle["run_manifest"]["run_kind"] == "calibration" + transitions = [ + event["status"] + for event in bundle["events"] + if event["stage_id"] == "dataset_staging" + ] + assert transitions == ["started", "completed"] + artifacts = staging_dir / "runs" / runs[0] / "artifacts" + staged = json.loads((artifacts / "staged_dataset.json").read_text()) + assert staged["mode"] == "local_only" and staged["status"] == "skipped" + assert (artifacts / "fit_summary.json").is_file() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index ffca85fde..34d38ea9c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -33,29 +33,24 @@ def test_uk_expand_contract_rejects_unknown_source_ids() -> None: patch(_expand_population(), _expand_node(), _expand_result(bad_source=True)) -def test_uk_spine_graph_contains_manifest_stages_and_named_exclusions() -> None: +def test_uk_spine_graph_contains_every_manifest_stage() -> None: spec = load_country_spec("uk") assert spec.sources is not None - expected = tuple( - stage.stage - for stage in spec.sources.stages - if stage.stage not in UK_SPINE_EXCLUSIONS - ) + expected = tuple(stage.stage for stage in spec.sources.stages) graph = uk_spine_graph(spec) ids = {node.id for node in graph.nodes} - # 32 with the #832 uc_reporter_redraw, #685 uc_deduction_attributes, + # 34 with the #832 uc_reporter_redraw, #685 uc_deduction_attributes, # #791 frs_relationships, #725 hmrc_cgt_asset_type_spine, #970 # cgt_incidence_anchor, #930 nts_bus_travel and the income-anchor lane's - # spi_income_band_donors stages, and the SPI housing shell; the two named - # exclusions are the certified-pair alternatives, not steps of this pipeline. + # (PolicyEngine/chronicle#280) spi_income_band_donors stages and the #1012 + # spi_housing_shell; the frs_hmrc_retained_leaves / hmrc_spi_income + # certified-pair alternatives are retired (#901), so the manifest roster is + # the graph roster. assert len(expected) == 34 - assert UK_SPINE_EXCLUSIONS == { - "frs_hmrc_retained_leaves", - "hmrc_spi_income", - } + assert {"frs_hmrc_retained_leaves", "hmrc_spi_income"}.isdisjoint(expected) assert set(expected) <= ids - assert not (UK_SPINE_EXCLUSIONS & ids) + assert {"frs_hmrc_retained_leaves", "hmrc_spi_income"}.isdisjoint(ids) root_dtypes = { (owned.entity, owned.column): owned.dtype for owned in graph.node("create_uk_frs").outputs @@ -69,11 +64,7 @@ def test_uk_spine_graph_contains_manifest_stages_and_named_exclusions() -> None: def test_uk_spine_compile_order_is_derived_from_declared_inputs() -> None: spec = load_country_spec("uk") assert spec.sources is not None - expected = tuple( - stage.stage - for stage in spec.sources.stages - if stage.stage not in UK_SPINE_EXCLUSIONS - ) + expected = tuple(stage.stage for stage in spec.sources.stages) compiled = compile_graph(uk_spine_graph(spec)) stage_order = tuple(node_id for node_id in compiled.order if node_id in expected) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_hmrc_income_source_manifest.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_hmrc_income_source_manifest.py index 1aa794475..4661ad508 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_hmrc_income_source_manifest.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_hmrc_income_source_manifest.py @@ -1,4 +1,4 @@ -"""Contract tests for the raw UK HMRC/SPI income source manifest.""" +"""Canonical raw-spine HMRC contracts bind current producers and official sources.""" from __future__ import annotations @@ -8,689 +8,198 @@ from microcosm.build.uk_runtime.hmrc_source_contract import ( assert_uk_hmrc_income_source_contract_current, + uk_hmrc_weighted_qrf_output_columns, ) from test_support.paths import paths_for -_TEST_PATHS = paths_for("microcosm-build") - -_MANIFEST_PATH = ( - _TEST_PATHS.package - / "src" - / "microcosm" - / "build" - / "uk" - / "hmrc_income_source_stages.json" -) -_CANONICAL_SOURCE_STAGES_PATH = ( - _TEST_PATHS.package / "src" / "microcosm" / "build" / "uk" / "source_stages.json" -) -_COLLATED_ODS_URL = ( - "https://assets.publishing.service.gov.uk/media/" - "69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods" +MANIFEST = ( + paths_for("microcosm-build").package / "src/microcosm/build/uk/source_stages.json" ) -_COLLATED_ODS_SHA256 = ( - "ad063b06b2bdeef8600dbbb09d48153337a4966f8c7eea50df7a2e0304ebd73e" -) -_COLLATED_ODS_SIZE_BYTES = 166_693 -_SPI_DONOR_SHA256 = "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66" -_SPI_DONOR_SIZE_BYTES = 141_323_762 -_OFFICIAL_COMPONENTS = [ - "employment_income", - "self_employment_income", - "state_pension", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "other_investment_income", -] -_STAGE1_OUTPUTS = [ - "self_employment_income", - "savings_interest_income", - "dividend_income", - "private_pension_income", - "property_income", - "other_investment_income", - "gift_aid", - "charitable_investment_gifts", - "hmrc_spi_pay", - "hmrc_spi_employment_benefits", - "hmrc_spi_employment_expenses", - "hmrc_spi_incapacity_benefit_income", - "hmrc_spi_other_social_security_income", - "hmrc_spi_taxable_termination_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_miscellaneous_employment_income", - "hmrc_spi_other_income", - "hmrc_spi_state_pension_income", -] -_STAGE2_INCOME_PREDICTORS = [ - "employment_income", - "self_employment_income", - "savings_interest_income", - "dividend_income", - "private_pension_income", - "property_income", -] -_FRS_HMRC_FULL_CONSTITUENTS = [ - "hmrc_spi_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_incapacity_benefit_income", -] -_FRS_HMRC_NAMED_SUBSETS = [ - "ossben_identifiable_subset", - "srp_regular_code5", -] -_FRS_HMRC_SOURCE_ABSENT = [ - "EPB", - "EXPS", - "TAXTERM", - "MOTHINC", - "OTHERINC", -] -_DERIVED_AUXILIARIES = [ - "hmrc_spi_employed_income", - "hmrc_spi_total_earned_income", - "hmrc_spi_total_investment_income", - "hmrc_spi_assessable_income", -] -_COMPONENT_COLUMNS = { - "employment_income": ("Table_3_6", 4, 5), - "self_employment_income": ("Table_3_6", 1, 2), - "state_pension": ("Table_3_6", 7, 8), - "private_pension_income": ("Table_3_6", 10, 11), - "property_income": ("Table_3_7", 1, 2), - "savings_interest_income": ("Table_3_7", 4, 5), - "dividend_income": ("Table_3_7", 7, 8), - "other_investment_income": ("Table_3_7", 10, 11), -} - - -def _manifest() -> dict[str, object]: - return json.loads(_MANIFEST_PATH.read_text(encoding="utf-8")) - - -def _runtime_manifest() -> dict[str, object]: - payload = json.loads(_CANONICAL_SOURCE_STAGES_PATH.read_text(encoding="utf-8")) - retained = [ - stage - for stage in payload["stages"] - if stage["stage"] == "frs_hmrc_retained_leaves" - ] - hmrc = [stage for stage in payload["stages"] if stage["stage"] == "hmrc_spi_income"] - assert len(retained) == 1 - assert len(hmrc) == 1 - stage = dict(hmrc[0]) - stage["base_candidate"] = dict(_manifest()["stages"][0]["base_candidate"]) - stage["operations"] = [ - *retained[0]["operations"], - *hmrc[0]["operations"], - ] - return { - "country": payload["country"], - "version": payload["version"], - "stages": [stage], - } - - -def _stage(payload: dict[str, object]) -> dict[str, object]: - stages = payload["stages"] - assert isinstance(stages, list) - assert len(stages) == 1 - stage = stages[0] - assert isinstance(stage, dict) - return stage +INCOME = "hmrc_spi_income_spine" -def _by_role(stage: dict[str, object]) -> dict[str, dict[str, object]]: - artifacts = stage["artifacts"] - assert isinstance(artifacts, list) - assert all(isinstance(artifact, dict) for artifact in artifacts) - return {artifact["role"]: artifact for artifact in artifacts} +def _payload(): + return json.loads(MANIFEST.read_text()) -def _by_kind(stage: dict[str, object]) -> dict[str, dict[str, object]]: - operations = stage["operations"] - assert isinstance(operations, list) - assert all(isinstance(operation, dict) for operation in operations) - return {operation["kind"]: operation for operation in operations} +def _stage(payload, name=INCOME): + return next(stage for stage in payload["stages"] if stage["stage"] == name) -def test__given_uk_hmrc_manifest__then_one_scoped_stage_is_declared() -> None: - payload = _manifest() - stage = _stage(payload) - - assert payload["version"] == 1 - assert payload["country"] == "uk" - assert stage["stage"] == "hmrc_spi_income" - assert stage["grain"] == "person" - assert stage["official_table_components"] == _OFFICIAL_COMPONENTS - assert stage["donor_relief_outputs"] == [ - "gift_aid", - "charitable_investment_gifts", - ] - assert stage["outputs"] == [ - *_OFFICIAL_COMPONENTS, - "gift_aid", - "charitable_investment_gifts", - *_DERIVED_AUXILIARIES, - ] - assert [operation["kind"] for operation in stage["operations"]] == [ - "verify_certified_candidate", - "retain_adjudicated_frs_hmrc_leaves", - "verify_pinned_hmrc_source_pair", - "replace_zero_weight_spi_support", - "strict_read_private_table", - "fit_weighted_qrf_stage1", - "fit_weighted_qrf_stage2", - "materialize_hmrc_income_bands_fail_closed", - "classify_hmrc_income_facts_with_reviewed_fences", - "gate_distributional_effective_mass", - ] - - -def test__given_hmrc_source_artifacts__then_vintages_and_runtime_hashes_are_exact() -> ( - None -): - stage = _stage(_manifest()) - artifacts = _by_role(stage) - - assert len(stage["artifacts"]) == 2 - assert set(artifacts) == {"qrf_donor", "published_fact_surface"} - donor = artifacts["qrf_donor"] - assert donor["survey"] == "Survey of Personal Incomes Public Use Tape 2022-23" - assert donor["vintage"] == "2022-23" - assert donor["ukds_study_number"] == "SN 9422" - assert donor["doi"] == "10.5255/UKDA-SN-9422-1" - assert donor["filename"] == "put2223uk.tab" - assert donor["sha256"] == _SPI_DONOR_SHA256 - assert donor["size_bytes"] == _SPI_DONOR_SIZE_BYTES - assert donor["access"] == "private_local_input" - assert donor["locator"] == "caller-supplied local input" - assert donor["runtime_sha256_required"] is True - - surface = artifacts["published_fact_surface"] - assert surface["vintage"] == "2023-24" - assert surface["locator"] == _COLLATED_ODS_URL - assert surface["sha256"] == _COLLATED_ODS_SHA256 - assert surface["size_bytes"] == _COLLATED_ODS_SIZE_BYTES - assert surface["mime_type"] == ("application/vnd.oasis.opendocument.spreadsheet") - assert surface["sheets"] == ["Table_3_6", "Table_3_7"] - assert surface["mapped_build_period"] == 2023 - assert surface["period_mapping"] == "tax_year_start" - assert surface["runtime_sha256_required"] is True - assert "tax-year-2023-to-2024" in surface["publication"] - - base = stage["base_candidate"] - assert base["tier"] == "frs" - assert base["sha256"] == ( - "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833" +def _operation(payload, name, kind): + return next( + operation + for operation in _stage(payload, name)["operations"] + if operation["kind"] == kind ) - assert base["size_bytes"] == 1_315_880_118 - assert base["runtime_sha256_required"] is True - -def test__given_frs_channel__then_adjudicated_leaves_and_subsets_are_explicit() -> None: - operations = _by_kind(_stage(_manifest())) - leaves = operations["retain_adjudicated_frs_hmrc_leaves"] - assert list(leaves["retained_full_constituents"]) == _FRS_HMRC_FULL_CONSTITUENTS - assert list(leaves["retained_named_subsets"]) == _FRS_HMRC_NAMED_SUBSETS - assert leaves["source_absent_full_constituents"] == _FRS_HMRC_SOURCE_ABSENT - assert leaves["status"] == "adjudicated_partial_replay" - assert leaves["source_vintage"] == "2023-24" - assert leaves["mapped_build_period"] == 2023 - assert leaves["retained_full_constituents"]["hmrc_spi_pay"] == { - "spi_concept": "PAY", - "scope": "full", - "raw_sources": ["ADULT.INEARNS"], - "formula": "max(0, ADULT.INEARNS) * (365.25 / 7)", - } - assert leaves["retained_full_constituents"]["hmrc_spi_incapacity_benefit_income"][ - "observed_support" - ].startswith("structural zero") - assert ( - leaves["retained_named_subsets"]["ossben_identifiable_subset"]["scope"] - == "identifiable_subset" - ) - assert leaves["retained_named_subsets"]["srp_regular_code5"]["scope"] == ( - "regular_code5_subset" - ) - assert leaves["forbid_proxy_substitution"] == [ - "employment_income", - "miscellaneous_income", - ] - assert leaves["fail_on_missing_retained_constituent"] is True - assert leaves["fail_on_full_concept_alias"] is True - - source_pair = operations["verify_pinned_hmrc_source_pair"] - assert source_pair == { - "kind": "verify_pinned_hmrc_source_pair", - "artifact_roles": ["qrf_donor", "published_fact_surface"], - "require_before_source_read": True, - "runtime_sha256_required": True, - "fail_on_mismatch": True, - } - - -def test__given_spi_donor__then_both_qrf_stages_are_weighted_and_strict() -> None: - operations = _by_kind(_stage(_manifest())) - - strict_read = operations["strict_read_private_table"] - assert strict_read["artifact_role"] == "qrf_donor" - assert strict_read["filename"] == "put2223uk.tab" - assert strict_read["weight"] == "FACT" - assert strict_read["runtime_sha256_required"] is True - assert strict_read["fail_on_missing_file"] is True - assert strict_read["fail_on_missing_columns"] is True - assert strict_read["fail_on_invalid_weight"] is True +def test_canonical_hmrc_family_matches_runtime_and_has_no_candidate_dependency(): + assert_uk_hmrc_income_source_contract_current() + payload = _payload() + names = {stage["stage"] for stage in payload["stages"]} assert { - "EXPS", - "INCPBEN", - "OSSBEN", - "UBISJA", - "MOTHINC", - "OTHERINC", - "CAPALL", - "LOSSBF", - "SRP", - "TI", - } <= set(strict_read["required_columns"]) - - stage1 = operations["fit_weighted_qrf_stage1"] - assert stage1["source_sampling_weight"] == "FACT" - assert stage1["sample_size"] == 100_000 - assert stage1["sample_with_replacement"] is True - assert stage1["post_sample_fit_weight"] == "uniform" - assert stage1["fit_weight_kind"] == "design" - assert stage1["double_apply_source_weight"] is False - assert stage1["outputs"] == _STAGE1_OUTPUTS - employment_derivation = stage1["derived_policyengine_outputs"]["employment_income"] - assert employment_derivation["source_columns"] == [ - "PAY", - "EPB", - "TAXTERM", - ] - assert employment_derivation["formula"] == ( - "hmrc_spi_pay + hmrc_spi_employment_benefits + hmrc_spi_taxable_termination_pay" - ) - assert employment_derivation["derive_after_draw"] is True - assert "employment_income" not in stage1["source_columns"] - assert "employment_income" not in stage1["outputs"] - assert stage1["source_columns"]["other_investment_income"] == ["OTHERINV"] - assert stage1["source_columns"]["hmrc_spi_other_income"] == ["OTHERINC"] - assert stage1["source_columns"]["hmrc_spi_state_pension_income"] == ["SRP"] - assert stage1["ti_identity_absolute_tolerance_gbp"] == 5 - assert stage1["source_ti_identity_fields"] == ["TI", "TEI", "TII"] - reconciliation = stage1["source_leaf_reconciliation"] - assert reconciliation["composite_indicator"] == "AGERANGE == -1" - assert reconciliation["formulas"]["TEI"].startswith("max(0, PAY + EPB - EXPS)") - assert reconciliation["formulas"]["TII"] == ( - "OTHERINV + DIVIDENDS + INCPROP + INCBBS" - ) - assert reconciliation["formulas"]["TI"] == "TEI + TII" - assert reconciliation["maximum_absolute_difference_gbp"] == { - "ordinary": {"TEI": 15, "TII": 10, "TI": 20}, - "composite": {"TEI": 180, "TII": 10, "TI": 180}, - } - assert stage1["stochastic_aggregates_forbidden"] == _DERIVED_AUXILIARIES - assert all( - column not in stage1["source_columns"] for column in _DERIVED_AUXILIARIES - ) - assert all(column not in stage1["outputs"] for column in _DERIVED_AUXILIARIES) - assert "TEI + TII" in stage1["assessable_income_source_semantics"] - assert "deterministic post-draw" in stage1["assessable_income_source_semantics"] - assert stage1["source_columns"]["gift_aid"] == ["GIFTAID"] - assert stage1["source_columns"]["charitable_investment_gifts"] == ["GIFTINV"] - assert "state_pension" not in stage1["outputs"] - assert stage1["joint_draw"] is True - assert stage1["require_all_predictors"] is True - assert stage1["require_all_outputs"] is True - - stage2 = operations["fit_weighted_qrf_stage2"] - assert stage2["weight"] == "household_weight" - assert stage2["weight_mapping"] == "household_to_person" - assert stage2["predictors"] == [ - "age", - "gender", - "region", - *_STAGE2_INCOME_PREDICTORS, - ] - assert "other_investment_income" not in stage2["predictors"] - assert set(stage2["reviewed_absent_predictors"]) == {"other_investment_income"} + "frs_hmrc_spine_leaves", + "spi_support_channel", + "spi_income_band_donors", + INCOME, + } <= names + assert not ({"frs_hmrc_retained_leaves", "hmrc_spi_income"} & names) + for name in ("frs_hmrc_spine_leaves", "spi_support_channel", INCOME): + stage = _stage(payload, name) + assert "base_candidate" not in stage + assert "verify_certified_candidate" not in { + o["kind"] for o in stage["operations"] + } + assert not MANIFEST.with_name("hmrc_income_source_stages.json").exists() + + +def test_official_hmrc_sources_and_sampling_weights_remain_bound(): + stage = _stage(_payload()) + artifacts = {artifact["role"]: artifact for artifact in stage["artifacts"]} assert ( - "stage-1 SPI draw" - in stage2["reviewed_absent_predictors"]["other_investment_income"] + artifacts["qrf_donor"]["sha256"] + == "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66" ) assert ( - "exactly six income predictors" - in stage2["reviewed_absent_predictors"]["other_investment_income"] - ) - assert ( - "no other_investment_income column" - in stage2["reviewed_absent_predictors"]["other_investment_income"] - ) - assert "state_pension_reported" in stage2["outputs"] - assert "universal_credit_reported" in stage2["outputs"] - assert "employee_pension_contributions" in stage2["outputs"] - assert "incapacity_benefit_reported" not in stage2["outputs"] - assert "maternity_allowance_reported" not in stage2["outputs"] - assert set(stage2["reviewed_absent_outputs"]) == { - "incapacity_benefit_reported", - "maternity_allowance_reported", - } - assert stage2["require_all_predictors"] is True - assert stage2["require_all_materializable_outputs"] is True - assert stage2["require_all_outputs"] is False - assert stage2["postprocess"]["gross_savings_interest_income"] == ( - "stage1 INCBBS draw + stage2 tax_free_savings_income" + artifacts["published_fact_surface"]["sha256"] + == "ad063b06b2bdeef8600dbbb09d48153337a4966f8c7eea50df7a2e0304ebd73e" ) - assert "pip_dl_category" in stage2["postprocess"]["refresh_disability_categories"] - assert ( - "is_disabled_for_benefits" in stage2["postprocess"]["refresh_disability_flags"] - ) - - -def test__given_official_tables__then_all_eight_components_fail_closed() -> None: - operations = _by_kind(_stage(_manifest())) - materializer = operations["materialize_hmrc_income_bands_fail_closed"] - - actual_columns = { - component: ( - spec["sheet"], - spec["count_column_index"], - spec["amount_column_index"], - ) - for component, spec in materializer["component_columns"].items() - } - assert actual_columns == _COMPONENT_COLUMNS - assert materializer["mapped_build_period"] == 2023 - assert materializer["period_mapping"] == "tax_year_start" - assert materializer["required_measures"] == ["count", "amount"] - assert materializer["required_band_lower_bounds_gbp"] == [ - 12_570, - 15_000, - 20_000, - 30_000, - 40_000, - 50_000, - 70_000, - 100_000, - 150_000, - 200_000, - 300_000, - 500_000, - 1_000_000, - ] - assert materializer["fail_on_missing_sheet"] is True - assert materializer["fail_on_missing_component"] is True - assert materializer["fail_on_missing_band"] is True - assert materializer["fail_on_non_numeric_value"] is True - - -def test__given_materialized_hmrc_targets__then_all_facts_are_fenced_without_calibration() -> ( - None -): - operations = _by_kind(_stage(_manifest())) - - classification = operations["classify_hmrc_income_facts_with_reviewed_fences"] - assert classification["components"] == _OFFICIAL_COMPONENTS - assert classification["breakdown_dependency"] == "hmrc_spi_assessable_income" - assert classification["frs_breakdown_status"] == "unavailable_full_measure" - assert classification["input_weight_kind"] == "importance" - assert classification["output_weight_kind"] == "importance" - assert classification["calibration_permitted"] is False - assert classification["required_fact_count"] == 208 - assert classification["outcome_counts"] == { - "exact_pass": 0, - "exact_fail": 0, - "directional_pass": 0, - "directional_fail": 0, - "excluded_with_fence": 208, - } - fences = {fence["fence_id"]: fence for fence in classification["reviewed_fences"]} - assert set(fences) == { - "frs_epb_source_absent", - "frs_exps_source_absent", - "frs_taxterm_source_absent", - "frs_mothinc_source_absent", - "frs_otherinc_source_absent", - "frs_ossben_identifiable_subset", - "frs_srp_regular_code5_subset", - "full_frs_tei_band_unavailable", - } - for fence in fences.values(): - assert fence["constituents"] - assert fence["finding"].strip() - assert fence["mass_implication"].strip() - assert fence["rationale"].strip() - assert fences["frs_epb_source_absent"]["raw_sources_searched"] == [ - "JOB.EXPBEN01-EXPBEN13", - "JOB.CARVAL", - "JOB.CARAMT", - "JOB.FUELAMT", - "JOB.VCHAMT", - "JOB.CHVAMT", - ] - assert fences["frs_ossben_identifiable_subset"]["constituents"] == [ - "OSSBEN", - "ossben_identifiable_subset", - ] - assert fences["frs_srp_regular_code5_subset"]["constituents"] == [ - "SRP", - "srp_regular_code5", - ] - full_ti = fences["full_frs_tei_band_unavailable"] - assert len(full_ti["dependent_fence_ids"]) == 7 - assert "non-overlapping" in full_ti["rationale"] - assert classification["fact_fence_id"] == "full_frs_tei_band_unavailable" - assert classification["fail_on_unfenced_exclusion"] is True - assert classification["fail_on_fact_count_mismatch"] is True - assert classification["forbid_biased_estimate_or_delta"] is True - - prior = operations["replace_zero_weight_spi_support"] - assert prior["require_existing_weight"] == 0 - assert prior["spi_prior_national_household_mass_share"] == 0.5 - assert prior["output_weight_kind"] == "importance" - assert prior["preserve_total_household_mass"] is True - assert prior["require_mass_change_record"] is True - - effective = operations["gate_distributional_effective_mass"] - assert effective["columns"] == [ - "gift_aid", - "charitable_investment_gifts", - ] - assert effective["minimum_nondefault_mass_share"] == 0.000001 - assert effective["fail_below_floor"] is True - - -def test__given_standalone_contract__then_sources_are_explicit_artifacts() -> None: - artifacts = _by_role(_stage(_manifest())) - - assert artifacts["qrf_donor"]["access"] == "private_local_input" - assert artifacts["qrf_donor"]["locator"] == "caller-supplied local input" - assert artifacts["published_fact_surface"]["locator"] == _COLLATED_ODS_URL - assert all(artifact["runtime_sha256_required"] for artifact in artifacts.values()) - - -def test_runtime_source_contract_matches_committed_manifest() -> None: - assert_uk_hmrc_income_source_contract_current() + assert artifacts["published_fact_surface"]["mapped_build_period"] == 2024 + assert artifacts["published_fact_surface"]["vintage"] == "2023-24" + payload = _payload() + first = _operation(payload, INCOME, "fit_weighted_qrf_stage1") + second = _operation(payload, INCOME, "fit_weighted_qrf_stage2") + assert first["source_sampling_weight"] == "FACT" + assert first["post_sample_fit_weight"] == "uniform" + assert first["double_apply_source_weight"] is False + assert first["seed"] == 42 and second["seed"] == 43 + assert second["weight_mapping"] == "household_to_person" @pytest.mark.parametrize( - ("path", "replacement", "match"), + "stage_name,kind,field,replacement,match", [ ( - ("stages", 0, "base_candidate", "tier"), - "public", - "base_candidate.tier", - ), - ( - ("stages", 0, "base_candidate", "sha256"), - "0" * 64, - "base_candidate.sha256", - ), - ( - ("stages", 0, "artifacts", 1, "vintage"), - "2022-23", - "published_fact_surface.vintage", - ), - ( - ("stages", 0, "artifacts", 0, "sha256"), - "0" * 64, - "qrf_donor.sha256", + "frs_hmrc_spine_leaves", + "retain_adjudicated_frs_hmrc_leaves", + "population", + "candidate_h5", + "frs_leaves.population", ), ( - ("stages", 0, "artifacts", 1, "size_bytes"), - 1, - "published_fact_surface.size_bytes", + "frs_hmrc_spine_leaves", + "retain_adjudicated_frs_hmrc_leaves", + "source_vintage", + "2023-24", + "source_vintage", ), ( - ("stages", 0, "operations", 1, "status"), - "ready", - "frs_leaves.status", - ), - ( - ( - "stages", - 0, - "operations", - 3, - "spi_prior_national_household_mass_share", - ), - 0.25, + "spi_support_channel", + "allocate_zero_weight_prior_mass", + "share", + 0.1, "prior.mass_share", ), ( - ("stages", 0, "operations", 5, "sample_size"), - 50_000, - "stage1.sample_size", - ), - ( - ("stages", 0, "operations", 5, "outputs"), - ["employment_income"], - "stage1.outputs", - ), - ( - ("stages", 0, "operations", 5, "source_ti_identity_fields"), - ["TI"], - "stage1.source_ti_identity_fields", - ), - ( - ( - "stages", - 0, - "operations", - 5, - "derived_policyengine_outputs", - "employment_income", - "formula", - ), - "hmrc_spi_pay", - "stage1.derived_policyengine_outputs.employment_income.formula", - ), - ( - ( - "stages", - 0, - "operations", - 5, - "source_leaf_reconciliation", - "maximum_absolute_difference_gbp", - "ordinary", - "TEI", - ), - 1_000, - "stage1.source_leaf_reconciliation.maximum_absolute_difference_gbp", - ), - ( - ("stages", 0, "operations", 6, "reviewed_absent_outputs"), - {"maternity_allowance_reported": "changed"}, - "stage2.reviewed_absent_outputs", + "spi_support_channel", + "allocate_zero_weight_prior_mass", + "strata", + [], + "prior.strata", ), ( - ("stages", 0, "operations", 6, "reviewed_absent_predictors"), - {"other_investment_income": "changed"}, - "stage2.reviewed_absent_predictors", + "spi_support_channel", + "stack_zero_weight_donors", + "count", + 0, + "count drifted", ), + (INCOME, "strict_read_private_table", "weight", "uniform", "strict.weight"), ( - ( - "stages", - 0, - "operations", - 7, - "component_columns", - "savings_interest_income", - "amount_column_index", - ), - 6, - "materialize.component_columns", + INCOME, + "fit_weighted_qrf_stage1", + "post_sample_fit_weight", + "FACT", + "post_sample_fit_weight", ), + (INCOME, "fit_weighted_qrf_stage1", "source_columns", {}, "source_columns"), + (INCOME, "fit_weighted_qrf_stage1", "seed", 43, "seed drifted"), + (INCOME, "fit_weighted_qrf_stage2", "predictors", ["age"], "stage2.predictors"), + (INCOME, "redraw_columns_from_fitted_qrf", "rows", "all", "base redraw"), ( - ("stages", 0, "operations", 8, "output_weight_kind"), - "calibrated", - "classification.output_weight_kind", + INCOME, + "materialize_hmrc_income_bands_fail_closed", + "component_columns", + {}, + "component_columns", ), ( - ("stages", 0, "operations", 8, "required_fact_count"), + INCOME, + "classify_hmrc_income_facts_with_reviewed_fences", + "required_fact_count", 207, - "classification.required_fact_count", + "required_fact_count", ), ( - ( - "stages", - 0, - "operations", - 8, - "outcome_counts", - "excluded_with_fence", - ), - 207, - "classification.outcome_counts", - ), - ( - ("stages", 0, "operations", 9, "required_support_channel"), + INCOME, + "gate_distributional_effective_mass", + "required_support_channel", "frs", - "effective.required_support_channel", + "required_support_channel", ), ( - ("stages", 0, "operations", 9, "minimum_nondefault_mass_share"), + INCOME, + "gate_distributional_effective_mass", + "minimum_nondefault_mass_share", 0.01, - "effective.minimum_nondefault_mass_share", + "minimum_nondefault_mass_share", ), ], ) -def test_runtime_source_contract_rejects_manifest_drift( - tmp_path, - path, - replacement, - match, -) -> None: - payload = _runtime_manifest() - cursor = payload - for segment in path[:-1]: - cursor = cursor[segment] - cursor[path[-1]] = replacement - tampered = tmp_path / "hmrc_income_source_stages.json" - tampered.write_text(json.dumps(payload), encoding="utf-8") - +def test_source_contract_rejects_drift( + tmp_path, stage_name, kind, field, replacement, match +): + payload = _payload() + _operation(payload, stage_name, kind)[field] = replacement + path = tmp_path / "sources.json" + path.write_text(json.dumps(payload)) with pytest.raises(ValueError, match=match): - assert_uk_hmrc_income_source_contract_current(tampered) + assert_uk_hmrc_income_source_contract_current(path) @pytest.mark.parametrize( - ("collection", "key"), [("artifacts", "role"), ("operations", "kind")] + "collection,key", [("artifacts", "role"), ("operations", "kind")] ) -def test_runtime_source_contract_rejects_duplicate_keys( - tmp_path, - collection, - key, -) -> None: - payload = _runtime_manifest() - values = payload["stages"][0][collection] +def test_duplicate_source_declarations_are_refused(tmp_path, collection, key): + payload = _payload() + values = _stage(payload)[collection] values.append(dict(values[0])) - tampered = tmp_path / f"duplicate_{key}.json" - tampered.write_text(json.dumps(payload), encoding="utf-8") - + path = tmp_path / "sources.json" + path.write_text(json.dumps(payload)) with pytest.raises(ValueError, match=f"duplicate {key}"): - assert_uk_hmrc_income_source_contract_current(tampered) + assert_uk_hmrc_income_source_contract_current(path) + + +def test_missing_canonical_stage_is_refused(tmp_path): + payload = _payload() + payload["stages"] = [ + s for s in payload["stages"] if s["stage"] != "spi_support_channel" + ] + path = tmp_path / "sources.json" + path.write_text(json.dumps(payload)) + with pytest.raises(ValueError, match="exactly one spi_support_channel"): + assert_uk_hmrc_income_source_contract_current(path) + + +def test_tail_concentration_surface_contains_both_canonical_model_stages(): + payload = _payload() + expected = tuple( + dict.fromkeys( + output + for kind in ("fit_weighted_qrf_stage1", "fit_weighted_qrf_stage2") + for output in _operation(payload, INCOME, kind)["outputs"] + ) + ) + assert uk_hmrc_weighted_qrf_output_columns() == expected + assert {"gift_aid", "self_employment_income"} <= set(expected) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_hmrc_replay_artifacts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_hmrc_replay_artifacts.py index cf2c149c9..48c9af1ad 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_hmrc_replay_artifacts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_hmrc_replay_artifacts.py @@ -7,6 +7,8 @@ from importlib.resources import files from typing import Any +import pytest + from microcosm.build.uk_runtime.hmrc_income import ( HMRC_SPI_COLLATED_ODS_SHA256, HMRC_SPI_INCOME_BAND_LOWER_BOUNDS, @@ -16,6 +18,7 @@ from microcosm.build.uk_runtime.hmrc_replay import FULL_FRS_TI_BAND_FENCE_ID from microcosm.build.uk_runtime.release_input_coverage import ( DEFAULT_MINIMUM_NONDEFAULT_MASS_SHARE, + assert_uk_release_input_coverage_build_stages, ) from microcosm.build.uk_runtime.spi_income import ( SPI_DONOR_SHA256, @@ -121,22 +124,8 @@ def test_real_replay_binds_sources_identity_and_positive_mass() -> None: } assert sources["hmrc_surface"]["sha256"] == HMRC_SPI_COLLATED_ODS_SHA256 assert sources["hmrc_surface"]["mapped_build_period"] == "2023" - # June-freeze partition, made self-describing (adversarial-review - # disposition, microcosm#723): this report is evidence for the - # grandfathered June release and binds to the FROZEN manifest's period - # mapping - it deliberately does NOT follow the live build period, which - # moved to "2024" with the #723 signed re-map. It retires with the frozen - # manifest after #686 (#687's disposition), never regenerates against a - # different vintage. - frozen_stage = _resource("hmrc_income_source_stages.json")["stages"][0] - frozen_surface = next( - artifact - for artifact in frozen_stage["artifacts"] - if artifact.get("role") == "published_fact_surface" - ) - assert sources["hmrc_surface"]["mapped_build_period"] == str( - frozen_surface["mapped_build_period"] - ) + # This immutable June replay remains historical evidence. Current source + # contracts use the canonical spine stages and the 2024 build period. assert qrf["fits"] == { "uk_frs_only_spi_fill": {"weight_kind": "importance"}, "uk_spi_2022_23_income": {"weight_kind": "design"}, @@ -256,3 +245,9 @@ def test_committed_replay_artifacts_contain_no_row_level_payloads_or_local_paths serialized = json.dumps(payload, allow_nan=False, sort_keys=True) assert "/Users/" not in serialized assert "put2223uk.tab" not in serialized + + +def test_historical_candidate_stages_cannot_satisfy_current_build_contract(): + record = _resource(_BUILD_RECORD_RESOURCE) + with pytest.raises(ValueError, match="hmrc_spi_income"): + assert_uk_release_input_coverage_build_stages(record["stages"]) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_national_sampling.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_national_sampling.py index 000cea2ce..0c7be82a9 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_national_sampling.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_national_sampling.py @@ -420,33 +420,6 @@ def test_full_fraction_is_a_structural_no_op() -> None: assert receipt["uk_policy"]["spi_replacement_quota_checked"] is True -def test_sampled_frames_pass_the_real_stage_fence() -> None: - """The sampler's arithmetic is proven against the fence itself. - - The first credentialed rung run died because the sampler and - ``_resolve_candidate_lineage`` disagreed; this test closes the coverage - hole the adversarial review found by running the REAL fence over sampled - frames: every draw must resolve with the full frame's exact multiplier - and person-level SPI/CG offsets. - """ - - from microcosm.build.uk_runtime.frs_hmrc_leaves import ( - _resolve_candidate_lineage, - ) - - frame = _source_family_frame() - full = _resolve_candidate_lineage(frame) - for seed in (0, 3, 11, 42): - sampled, _receipt = sample_uk_national_frame(frame, fraction=0.5, seed=seed) - lineage = _resolve_candidate_lineage(sampled) - assert lineage.clone_id_multiplier == full.clone_id_multiplier - assert lineage.spi_person_id_offset == full.spi_person_id_offset - assert ( - lineage.capital_gains_person_id_offset - == full.capital_gains_person_id_offset - ) - - def test_spine_source_units_use_raw_family_regions() -> None: frame = _spine_family_frame() units, strata = uk_spine_source_family_units(frame) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_release_input_coverage_manifest.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_release_input_coverage_manifest.py index 7e47720f8..3491f7748 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_release_input_coverage_manifest.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_release_input_coverage_manifest.py @@ -62,6 +62,7 @@ def test_known_gap_register_records_post_candidate_restoration_separately() -> N assert gaps["exclusion_policy"]["tracking_note"].strip() for name, evidence in gaps["restored_required_columns"].items(): assert evidence["stage"] == "hmrc_spi_income" + assert evidence["current_producer_stage"] == "hmrc_spi_income_spine" assert evidence["support_channel"] == "spi" assert ( evidence["effective_signal_mass_share"] @@ -81,52 +82,31 @@ def test_frozen_candidate_retains_its_original_engine_provenance() -> None: def test_hmrc_family_period_fields_come_from_the_bytes_their_hash_names() -> None: - # Adversarial-review finding (2026-08-20): the family block renders the - # #723 re-mapped period fields from the CANONICAL manifest while the - # frozen mirror keeps its June bytes; each field set must bind to the - # sha256 of the file it actually came from. import hashlib - manifest = _resource("release_input_coverage_manifest.json") - family = manifest["family_coverage"]["hmrc_spi_income"] - - frozen_bytes = ( - files(_UK_PACKAGE).joinpath("hmrc_income_source_stages.json").read_bytes() - ) + family = _resource("release_input_coverage_manifest.json")["family_coverage"][ + "hmrc_spi_income" + ] canonical_bytes = files(_UK_PACKAGE).joinpath("source_stages.json").read_bytes() - assert family["source_manifest_sha256"] == hashlib.sha256(frozen_bytes).hexdigest() + assert family["source_manifest"] == "source_stages.json" assert ( - family["canonical_source_manifest_sha256"] - == hashlib.sha256(canonical_bytes).hexdigest() - ) - - frozen_stage = json.loads(frozen_bytes)["stages"][0] - frozen_surface = next( - artifact - for artifact in frozen_stage["artifacts"] - if artifact.get("role") == "published_fact_surface" + family["source_manifest_sha256"] == hashlib.sha256(canonical_bytes).hexdigest() ) - canonical_stage = next( + stage = next( stage for stage in json.loads(canonical_bytes)["stages"] - if stage.get("stage") == "hmrc_spi_income" + if stage["stage"] == "hmrc_spi_income_spine" ) - canonical_surface = next( + surface = next( artifact - for artifact in canonical_stage["artifacts"] - if artifact.get("role") == "published_fact_surface" + for artifact in stage["artifacts"] + if artifact["role"] == "published_fact_surface" ) - # The re-mapped fields equal the canonical declaration; the frozen mirror - # still declares the June mapping (its bytes are pinned elsewhere). assert family["source_vintages"]["mapped_build_period"] == str( - canonical_surface["mapped_build_period"] + surface["mapped_build_period"] ) - assert ( - family["source_vintages"]["period_mapping"] - == canonical_surface["period_mapping"] - ) - assert str(frozen_surface["mapped_build_period"]) == "2023" - assert frozen_surface["period_mapping"] == "tax_year_start" + assert family["source_vintages"]["period_mapping"] == surface["period_mapping"] + assert "canonical_source_manifest" not in family def test_promoted_manifest_requires_the_full_reference_surface() -> None: @@ -165,7 +145,7 @@ def test_hmrc_stage_is_required_while_the_208_fact_replay_remains_fenced() -> No assert family["status"] == "required_at_build" assert family["restoration_status"] == "adjudicated_partial_replay" - assert family["source_manifest"] == "hmrc_income_source_stages.json" + assert family["source_manifest"] == "source_stages.json" assert len(family["source_manifest_sha256"]) == 64 assert family["base_candidate_tier"] == "frs" assert family["source_vintages"] == { @@ -175,8 +155,11 @@ def test_hmrc_stage_is_required_while_the_208_fact_replay_remains_fenced() -> No "period_mapping": "latest_published_tax_year", } assert family["spi_prior_national_household_mass_share"] == 0.5 - assert family["canonical_source_manifest"] == "source_stages.json" - assert len(family["canonical_source_manifest_sha256"]) == 64 + assert family["required_predecessor_stages"] == [ + "frs_hmrc_spine_leaves", + "spi_support_channel", + "spi_income_band_donors", + ] assert family["required_mass_change_reason"] == ( "Allocate 50% of certified UK national household prior mass to the " "rebuilt 2022-23 SPI support channel; total national mass is conserved." @@ -273,18 +256,26 @@ def test_manifest_generation_rejects_candidate_tier_drift() -> None: generator.build_manifest(reference=reference, known_gaps_payload=gaps) -def test_hmrc_family_rejects_source_stage_tier_mismatch( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, +def test_hmrc_family_rejects_canonical_source_contract_drift( + monkeypatch, tmp_path ) -> None: generator = _load_generator() - source_stages = _resource("hmrc_income_source_stages.json") - source_stages["stages"][0]["base_candidate"]["tier"] = "cps-transfer" - drifted = tmp_path / "hmrc_income_source_stages.json" + source_stages = _resource("source_stages.json") + stage = next( + stage + for stage in source_stages["stages"] + if stage["stage"] == "spi_support_channel" + ) + operation = next( + operation + for operation in stage["operations"] + if operation["kind"] == "allocate_zero_weight_prior_mass" + ) + operation["share"] = 0.1 + drifted = tmp_path / "source_stages.json" drifted.write_text(json.dumps(source_stages), encoding="utf-8") - monkeypatch.setattr(generator, "HMRC_SOURCE_STAGES_PATH", drifted) - - with pytest.raises(ValueError, match="disagrees with the certified candidate"): + monkeypatch.setattr(generator, "SOURCE_STAGES_PATH", drifted) + with pytest.raises(ValueError, match="prior.mass_share"): generator._hmrc_family_coverage_contract(candidate_source={"tier": "frs"}) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_runtime_exports.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_runtime_exports.py new file mode 100644 index 000000000..36af6227f --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_runtime_exports.py @@ -0,0 +1,43 @@ +"""The eager ``microcosm.build.uk_runtime`` namespace after the HMRC tail retirement.""" + +from __future__ import annotations + +import pytest + +from microcosm.build import uk_runtime +from microcosm.build.uk_runtime import frs_hmrc_source + +RETIRED_NAMES = ( + "FRS_HMRC_RETAINED_LEAVES_STAGE_NAME", + "UKFRSHMRCRetainedLeavesResult", + "UKFRSHMRCRetainedLeavesStageTransform", + "retain_uk_frs_hmrc_leaves", + "UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE", +) + +FRS_HMRC_CONSTANTS = ( + "FRS_HMRC_INCPBEN_COLUMN", + "FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN", + "FRS_HMRC_PAY_COLUMN", + "FRS_HMRC_RETAINED_LEAF_COLUMNS", + "FRS_HMRC_SRP_REGULAR_CODE5_COLUMN", + "FRS_HMRC_UBISJA_COLUMN", +) + + +@pytest.mark.parametrize("name", RETIRED_NAMES) +def test_retired_candidate_names_are_gone(name: str) -> None: + assert name not in uk_runtime.__all__ + assert not hasattr(uk_runtime, name) + + +@pytest.mark.parametrize("name", FRS_HMRC_CONSTANTS) +def test_frs_hmrc_constants_come_from_the_source_module(name: str) -> None: + assert name in uk_runtime.__all__ + assert getattr(uk_runtime, name) is getattr(frs_hmrc_source, name) + + +def test_public_names_are_unique_and_resolvable() -> None: + assert len(uk_runtime.__all__) == len(set(uk_runtime.__all__)) + for name in uk_runtime.__all__: + assert hasattr(uk_runtime, name), name diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_source_runtime.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_source_runtime.py index 628cf2d7f..bb6558a9c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_source_runtime.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_source_runtime.py @@ -112,8 +112,6 @@ def hmrc(frame: Frame) -> Frame: return frame assert uk_stage_implementations( - retained_leaves_transform=retained, - hmrc_income_transform=hmrc, was_wealth_transform=retained, uc_deduction_attributes_transform=hmrc, regional_property_uprating_transform=hmrc, @@ -127,8 +125,6 @@ def hmrc(frame: Frame) -> Frame: salary_sacrifice_transform=hmrc, student_loans_transform=retained, ) == { - "frs_hmrc_retained_leaves": retained, - "hmrc_spi_income": hmrc, "was_wealth": retained, "uc_deduction_attributes": hmrc, "regional_property_uprating": hmrc, @@ -233,3 +229,16 @@ def test_materialize_rules_engine_predictors_refuses_an_unknown_year_rule() -> N with pytest.raises(SourceRuntimeError, match="Unknown UK year_rule"): handler(None, operation, _context(engine=_PeriodRecordingEngine())) + + +def test_canonical_source_roster_has_no_candidate_migration_stages() -> None: + from microcosm.build.country_spec import load_country_spec + + stages = load_country_spec("uk").sources.stage_map() + assert "frs_hmrc_retained_leaves" not in stages + assert "hmrc_spi_income" not in stages + assert { + "frs_hmrc_spine_leaves", + "spi_support_channel", + "hmrc_spi_income_spine", + } <= stages.keys() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py index 6fa3ccdd4..df3696524 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py @@ -1,7 +1,5 @@ from __future__ import annotations -import copy -import hashlib import json from pathlib import Path @@ -12,7 +10,7 @@ FORBIDDEN_SOURCE_DEPENDENCIES, SourceManifest, ) -from microcosm.build.uk_runtime.graph import UK_SPINE_EXCLUSIONS, uk_spine_graph +from microcosm.build.uk_runtime.graph import uk_spine_graph from microcosm.frame import Frame from microcosm.graph import compile_graph from test_support.paths import paths_for @@ -21,7 +19,6 @@ ROOT = _TEST_PATHS.repository UK_PACKAGE = ROOT / "packages/microcosm-build/src/microcosm/build/uk" -FROZEN_SOURCE_STAGES = UK_PACKAGE / "hmrc_income_source_stages.json" CANONICAL_SOURCE_STAGES = UK_PACKAGE / "source_stages.json" E3_STAGE_NAMES = [ "frs_employment", @@ -94,12 +91,7 @@ *UC_COHERENCE_STAGE_NAMES, *E9_STAGE_NAMES, *E8_STAGE_NAMES, - "frs_hmrc_retained_leaves", - "hmrc_spi_income", ] -FROZEN_SOURCE_STAGES_SHA256 = ( - "c0341af7166ae3a85a3c1164e7d9e880c4b4aec122f1a8fa90c73b46c596e1ea" -) def _load_json(path: Path) -> dict: @@ -111,11 +103,7 @@ def _identity(frame: Frame) -> Frame: def _uk_graph_stage_names(spec) -> list[str]: - manifest_stages = { - stage.stage - for stage in spec.sources.stages - if stage.stage not in UK_SPINE_EXCLUSIONS - } + manifest_stages = {stage.stage for stage in spec.sources.stages} return [ node_id for node_id in compile_graph(uk_spine_graph(spec)).order @@ -129,20 +117,6 @@ def _assert_no_forbidden_dependency(value: object) -> None: assert dependency not in text -def _expected_reviewed_source() -> str: - return ( - "PolicyEngine licensed UKDS mirror (private Hugging Face repository), " - "spi_2022_23.zip" - ) - - -def _rephrase_stage2_predictor_note(value: str) -> str: - return value.replace( - "policyengine-" + "uk-data frs_only.py", - "the incumbent UK data build's frs_only.py", - ) - - class TestUKSourceStagesManifest: def test_source_stages_json_loads_as_shared_manifest(self) -> None: manifest = SourceManifest.from_mapping(_load_json(CANONICAL_SOURCE_STAGES)) @@ -189,7 +163,7 @@ def test_e5_and_e6_follow_e7_and_precede_the_uc_rewrites(self) -> None: def test_age_tail_runs_immediately_after_frs_spine(self) -> None: canonical = _load_json(CANONICAL_SOURCE_STAGES) - spine = [stage["stage"] for stage in canonical["stages"][:-2]] + spine = [stage["stage"] for stage in canonical["stages"]] assert spine[1] == "age_tail" @@ -204,97 +178,13 @@ def test_age_tail_position_owns_the_only_later_age_rewrite_guard(self) -> None: assert "age" not in stage.outputs, stage.stage assert "age" not in stage.rewrites, stage.stage - def test_e8_block_is_final_and_the_certified_pair_stays_last(self) -> None: - # The E8 stages stay contiguous at the end of the spine, while the - # certified pair stays at [-2:] (the frozen-copy lockstep test reads - # them from there). age_tail is now the post-frs_spine block, before - # every stage that conditions on age. + def test_e8_block_is_final_and_all_stages_are_canonical(self) -> None: canonical = _load_json(CANONICAL_SOURCE_STAGES) names = [stage["stage"] for stage in canonical["stages"]] - - assert names[-2:] == ["frs_hmrc_retained_leaves", "hmrc_spi_income"] - spine = names[:-2] - start = spine.index(E8_STAGE_NAMES[0]) - assert spine[start : start + len(E8_STAGE_NAMES)] == E8_STAGE_NAMES - assert spine[start + len(E8_STAGE_NAMES) :] == [] - - def test_copy_is_lockstep_with_frozen_original_except_citation_rewrites( - self, - ) -> None: - frozen = _load_json(FROZEN_SOURCE_STAGES) - canonical = _load_json(CANONICAL_SOURCE_STAGES) - frozen_stage = frozen["stages"][0] - stage1, stage2 = canonical["stages"][-2:] - - expected_operations = copy.deepcopy(frozen_stage["operations"]) - predictor_note = expected_operations[6]["reviewed_absent_predictors"][ - "other_investment_income" - ] - expected_operations[6]["reviewed_absent_predictors"][ - "other_investment_income" - ] = _rephrase_stage2_predictor_note(predictor_note) - # FRS retained leaves now come from the FRS 2024-25 spine while the - # frozen HMRC fact surface stays byte-pinned. - expected_operations[1]["source_vintage"] = "2024-25" - expected_operations[1]["mapped_build_period"] = 2024 - # Signed period re-map (#723) for materialized HMRC SPI facts. - expected_operations[7]["mapped_build_period"] = 2024 - expected_operations[7]["period_mapping"] = "latest_published_tax_year" - - assert stage1["operations"] + stage2["operations"] == expected_operations - _assert_no_forbidden_dependency( - stage2["operations"][4]["reviewed_absent_predictors"][ - "other_investment_income" - ] - ) - - expected_artifacts = copy.deepcopy(frozen_stage["artifacts"]) - expected_artifacts[0]["reviewed_source"] = _expected_reviewed_source() - # Signed period re-map (#723): the ODS source surface remains the - # frozen 2023-24 file, but the canonical manifest declares that it is - # replayed against build period 2024. - expected_artifacts[1]["mapped_build_period"] = 2024 - expected_artifacts[1]["period_mapping"] = "latest_published_tax_year" - # Declared output-name correction (licensed-data acceptance finding): - # the frozen original listed the SPI concept "state_pension", but the - # stage writes the auxiliary column SPI_HMRC_STATE_PENSION_INCOME_COLUMN - # ("hmrc_spi_state_pension_income") — the model input state_pension is - # formula-owned and never a frame column here. Outputs became - # load-bearing when country_stage_plan compiled them into - # StagePlan.produces, so the copy declares the persisted truth. The - # operation payloads keep the concept name unchanged. - expected_outputs = [ - "hmrc_spi_state_pension_income" if name == "state_pension" else name - for name in frozen_stage["outputs"] - ] - assert stage2["outputs"] == expected_outputs - assert stage2["grain"] == frozen_stage["grain"] - assert stage2["artifacts"] == expected_artifacts - _assert_no_forbidden_dependency(stage2["artifacts"]) - _assert_no_forbidden_dependency(stage2["notes"]) - - def test_frozen_original_bytes_are_pinned(self) -> None: - digest = hashlib.sha256(FROZEN_SOURCE_STAGES.read_bytes()).hexdigest() - - assert digest == FROZEN_SOURCE_STAGES_SHA256 - - def test_country_stage_plan_assembles_two_certified_uk_national_stages( - self, - ) -> None: - spec = load_country_spec("uk") - plan = country_stage_plan( - spec, - { - "frs_hmrc_retained_leaves": _identity, - "hmrc_spi_income": _identity, - }, - stage_names=("frs_hmrc_retained_leaves", "hmrc_spi_income"), - ) - - assert [stage.name for stage in plan.stages] == [ - "frs_hmrc_retained_leaves", - "hmrc_spi_income", - ] + start = names.index(E8_STAGE_NAMES[0]) + assert names[start:] == E8_STAGE_NAMES + assert "frs_hmrc_retained_leaves" not in names + assert "hmrc_spi_income" not in names def test_country_stage_plan_assembles_spine_plan(self) -> None: spec = load_country_spec("uk") @@ -311,7 +201,7 @@ def test_country_stage_plan_assembles_spine_plan(self) -> None: @pytest.mark.parametrize( "implementations, match", [ - ({"frs_hmrc_retained_leaves": _identity}, "missing"), + ({"frs_spine": _identity}, "missing"), ( { "frs_spine": _identity, @@ -348,8 +238,6 @@ def test_country_stage_plan_assembles_spine_plan(self) -> None: "student_loans": _identity, "age_tail": _identity, "frs_relationships": _identity, - "frs_hmrc_retained_leaves": _identity, - "hmrc_spi_income": _identity, "hmrc_spi_income_fallback": _identity, }, "Unknown stage implementation", @@ -380,14 +268,17 @@ class TestDeclaredOutputsAreWrittenColumns: """ def test_stage1_outputs_are_exactly_the_retained_leaf_columns(self) -> None: - from microcosm.build.uk_runtime.frs_hmrc_leaves import ( + from microcosm.build.uk_runtime.frs_hmrc_source import ( FRS_HMRC_RETAINED_LEAF_COLUMNS, ) spec = load_country_spec("uk") stages = {stage.stage: stage for stage in spec.sources.stages} - stage1 = stages["frs_hmrc_retained_leaves"] - assert stage1.outputs == tuple(FRS_HMRC_RETAINED_LEAF_COLUMNS) + stage1 = stages["frs_hmrc_spine_leaves"] + assert stage1.outputs == ( + *FRS_HMRC_RETAINED_LEAF_COLUMNS, + "employer_pension_contributions", + ) def test_e3_outputs_are_backed_by_runtime_written_columns(self) -> None: from microcosm.build.uk_runtime.etb_services import ( @@ -1075,7 +966,7 @@ def test_stage2_outputs_are_backed_by_runtime_written_columns(self) -> None: spec = load_country_spec("uk") stages = {stage.stage: stage for stage in spec.sources.stages} - stage2 = stages["hmrc_spi_income"] + stage2 = stages["hmrc_spi_income_spine"] written = ( set(SPI_INCOME_IMPUTATION_COLUMNS) | set(SPI_HMRC_QRF_AUXILIARY_COLUMNS) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spine_acceptance_receipt.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spine_acceptance_receipt.py index 2bb7f2830..828b44156 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spine_acceptance_receipt.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spine_acceptance_receipt.py @@ -14,10 +14,7 @@ from importlib.resources import files from microcosm.build.country_spec import load_country_spec -from microcosm.build.uk_runtime.graph import ( - UK_SPINE_EXCLUSIONS, - uk_spine_graph, -) +from microcosm.build.uk_runtime.graph import uk_spine_graph from microcosm.graph import compile_graph @@ -32,11 +29,7 @@ def _receipt() -> dict: def _production_graph_stage_names() -> tuple[str, ...]: spec = load_country_spec("uk") assert spec.sources is not None - declared = { - stage.stage - for stage in spec.sources.stages - if stage.stage not in UK_SPINE_EXCLUSIONS - } + declared = {stage.stage for stage in spec.sources.stages} compiled = compile_graph(uk_spine_graph(spec)) return tuple(node_id for node_id in compiled.order if node_id in declared) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_plan.py b/packages/microcosm-build/tests/engine_free/us/test_us_plan.py index 83241ed62..6bbccdc29 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_plan.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_plan.py @@ -986,7 +986,6 @@ def test_no_incumbent_data_package_references_in_live_tree(self) -> None: # never import or execute the retired data package. "packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json", "packages/microcosm-build/src/microcosm/build/uk/frs_release.json", - "packages/microcosm-build/src/microcosm/build/uk/hmrc_income_source_stages.json", # The UK population contract's registry-parity accounting names the # retired data package by necessity: 651 rows at pinned ref ebf733c # = 609 mapped + 42 signed exclusions + 3 unmapped declarations. diff --git a/packages/microcosm-build/tests/integration/uk/test_uk_staging_integration.py b/packages/microcosm-build/tests/integration/uk/test_uk_staging_integration.py index ef92c176c..499bb96e4 100644 --- a/packages/microcosm-build/tests/integration/uk/test_uk_staging_integration.py +++ b/packages/microcosm-build/tests/integration/uk/test_uk_staging_integration.py @@ -12,7 +12,6 @@ from microcosm.build.country_spec import load_country_spec from microcosm.build.staging_v2 import validate_v2_bundle -from microcosm.build.uk_runtime.graph import UK_SPINE_EXCLUSIONS from test_support.paths import paths_for _TEST_PATHS = paths_for("microcosm-build") @@ -94,11 +93,7 @@ def test_uk_staging_smoke_command_runs_every_spine_stage(tmp_path: Path) -> None assert manifest["delivery"]["upload_attempts"] == 0 spec = load_country_spec("uk") - expected = [ - stage.stage - for stage in spec.sources.stages - if stage.stage not in UK_SPINE_EXCLUSIONS - ] + expected = [stage.stage for stage in spec.sources.stages] events = bundle["events"] for stage in expected: transitions = [ diff --git a/test_support/microcosm_build/uk_graph.py b/test_support/microcosm_build/uk_graph.py index 70e958cb2..7ecdd6236 100644 --- a/test_support/microcosm_build/uk_graph.py +++ b/test_support/microcosm_build/uk_graph.py @@ -12,7 +12,6 @@ from microcosm.build.country_spec import load_country_spec from microcosm.build.uk_runtime.graph import ( - UK_SPINE_EXCLUSIONS, UK_SPINE_STRUCTURAL_STAGES, uk_registry, uk_spine_graph, diff --git a/test_support/microcosm_build/uk_release_input_coverage.py b/test_support/microcosm_build/uk_release_input_coverage.py index 9f0cc8aa0..f47740126 100644 --- a/test_support/microcosm_build/uk_release_input_coverage.py +++ b/test_support/microcosm_build/uk_release_input_coverage.py @@ -156,7 +156,7 @@ def _hmrc_family_coverage() -> dict[str, dict[str, object]]: return { "hmrc_spi_income": { "status": "required_at_build", - "stage": "hmrc_spi_income", + "stage": "hmrc_spi_income_spine", "effective_mass_requirements": { "gift_aid": { "status": "distributional_required", diff --git a/test_support/microcosm_build/uk_spi_income.py b/test_support/microcosm_build/uk_spi_income.py index 26bd5dd11..970239fa8 100644 --- a/test_support/microcosm_build/uk_spi_income.py +++ b/test_support/microcosm_build/uk_spi_income.py @@ -11,7 +11,7 @@ import pytest from microcosm.build.uk_runtime import frs_disability, spi_income -from microcosm.build.uk_runtime.frs_hmrc_leaves import ( +from microcosm.build.uk_runtime.frs_hmrc_source import ( FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, FRS_HMRC_RETAINED_LEAF_COLUMNS, FRS_HMRC_SRP_REGULAR_CODE5_COLUMN, diff --git a/test_support/microcosm_graph/acceptance_h_parity.py b/test_support/microcosm_graph/acceptance_h_parity.py index 490057101..e3c4ed1d8 100644 --- a/test_support/microcosm_graph/acceptance_h_parity.py +++ b/test_support/microcosm_graph/acceptance_h_parity.py @@ -48,7 +48,7 @@ #: was taken under). KERNEL_PARITY = PARITY / "kernels" -#: H2: ``uk_spine.json`` — the 33-stage FRS spine expressed as a graph — plus +#: H2: ``uk_spine.json`` — the 34-stage FRS spine expressed as a graph — plus #: ``sources/``, the data-only bundle both the graph and the legacy oracle #: rebuild their transforms from. The root transform's weights differ at the #: last bit between machines, so both sides recompute the root from the raw diff --git a/tools/build_uk_release_input_coverage_manifest.py b/tools/build_uk_release_input_coverage_manifest.py index 6bb6da1c9..49fd0ae73 100644 --- a/tools/build_uk_release_input_coverage_manifest.py +++ b/tools/build_uk_release_input_coverage_manifest.py @@ -43,7 +43,7 @@ REFERENCE_PATH = UK_PACKAGE_DIR / "efrs_parity_reference.json" KNOWN_GAPS_PATH = UK_PACKAGE_DIR / "efrs_parity_known_gaps.json" MANIFEST_PATH = UK_PACKAGE_DIR / "release_input_coverage_manifest.json" -HMRC_SOURCE_STAGES_PATH = UK_PACKAGE_DIR / "hmrc_income_source_stages.json" +CGT_SOURCE_STAGES_PATH = UK_PACKAGE_DIR / "cgt_source_stages.json" SOURCE_STAGES_PATH = UK_PACKAGE_DIR / "source_stages.json" CANDIDATE_REPO_ID = "policyengine/populace-uk-private" @@ -95,6 +95,7 @@ "promotion_basis": "weighted release gate stale-exclusion remediation", "reviewed_on": "2026-07-13", "stage": "hmrc_spi_income", + "current_producer_stage": "hmrc_spi_income_spine", "support_channel": "spi", }, "gift_aid": { @@ -104,6 +105,7 @@ "promotion_basis": "weighted release gate stale-exclusion remediation", "reviewed_on": "2026-07-13", "stage": "hmrc_spi_income", + "current_producer_stage": "hmrc_spi_income_spine", "support_channel": "spi", }, } @@ -1036,77 +1038,28 @@ def _hmrc_family_coverage_contract( *, candidate_source: dict[str, Any], ) -> dict[str, Any]: - payload = _load(HMRC_SOURCE_STAGES_PATH) - stages = payload.get("stages") - if not isinstance(stages, list) or len(stages) != 1: - raise ValueError( - f"{HMRC_SOURCE_STAGES_PATH}: expected exactly one source stage." - ) - stage = stages[0] - if not isinstance(stage, dict) or stage.get("stage") != "hmrc_spi_income": - raise ValueError(f"{HMRC_SOURCE_STAGES_PATH}: expected hmrc_spi_income stage.") - canonical_payload = _load(SOURCE_STAGES_PATH) - canonical_stages = canonical_payload.get("stages") - if not isinstance(canonical_stages, list): - raise ValueError(f"{SOURCE_STAGES_PATH}: expected source stages list.") - canonical_matches = [ - candidate - for candidate in canonical_stages - if isinstance(candidate, dict) and candidate.get("stage") == "hmrc_spi_income" - ] - if len(canonical_matches) != 1: - raise ValueError( - f"{SOURCE_STAGES_PATH}: expected exactly one hmrc_spi_income stage." - ) - canonical_stage = canonical_matches[0] - base_candidate = stage.get("base_candidate") - if not isinstance(base_candidate, dict): - raise ValueError( - f"{HMRC_SOURCE_STAGES_PATH}: base_candidate must be an object." - ) - source_tier = validate_uk_release_tier(candidate_source.get("tier")) - base_candidate_tier = validate_uk_release_tier(base_candidate.get("tier")) - if base_candidate_tier != source_tier: - raise ValueError( - "HMRC source-stage base candidate tier disagrees with the certified " - f"candidate evidence: {base_candidate_tier!r} != {source_tier!r}." - ) - artifacts = { - artifact["role"]: artifact - for artifact in stage.get("artifacts", []) - if isinstance(artifact, dict) and isinstance(artifact.get("role"), str) - } - canonical_artifacts = { - artifact["role"]: artifact - for artifact in canonical_stage.get("artifacts", []) - if isinstance(artifact, dict) and isinstance(artifact.get("role"), str) - } - operations = { - operation["kind"]: operation - for operation in stage.get("operations", []) - if isinstance(operation, dict) and isinstance(operation.get("kind"), str) - } - required_artifacts = {"qrf_donor", "published_fact_surface"} - missing_artifacts = sorted(required_artifacts - set(artifacts)) - missing_canonical_artifacts = sorted(required_artifacts - set(canonical_artifacts)) - required_operations = { - "retain_adjudicated_frs_hmrc_leaves", - "verify_pinned_hmrc_source_pair", - "replace_zero_weight_spi_support", - "classify_hmrc_income_facts_with_reviewed_fences", - "gate_distributional_effective_mass", - } - missing_operations = sorted(required_operations - set(operations)) - if missing_artifacts or missing_canonical_artifacts or missing_operations: - raise ValueError( - f"{HMRC_SOURCE_STAGES_PATH}: incomplete HMRC family contract; " - f"missing_artifacts={missing_artifacts}, " - f"missing_canonical_artifacts={missing_canonical_artifacts}, " - f"missing_operations={missing_operations}." - ) + from microcosm.build.uk_runtime.hmrc_source_contract import ( + assert_uk_hmrc_income_source_contract_current, + ) + from microcosm.build.uk_runtime.spi_support import SPI_PRIOR_MASS_CHANGE_REASON + + assert_uk_hmrc_income_source_contract_current(SOURCE_STAGES_PATH) + payload = _load(SOURCE_STAGES_PATH) + stages = {stage["stage"]: stage for stage in payload["stages"]} + stage = stages["hmrc_spi_income_spine"] + artifacts = {artifact["role"]: artifact for artifact in stage["artifacts"]} + operations = {operation["kind"]: operation for operation in stage["operations"]} + frs_leaves = next( + operation + for operation in stages["frs_hmrc_spine_leaves"]["operations"] + if operation["kind"] == "retain_adjudicated_frs_hmrc_leaves" + ) + prior = next( + operation + for operation in stages["spi_support_channel"]["operations"] + if operation["kind"] == "allocate_zero_weight_prior_mass" + ) classification = operations["classify_hmrc_income_facts_with_reviewed_fences"] - frs_leaves = operations["retain_adjudicated_frs_hmrc_leaves"] - prior = operations["replace_zero_weight_spi_support"] effective = operations["gate_distributional_effective_mass"] floor = float(effective["minimum_nondefault_mass_share"]) if floor != EFFECTIVE_MASS_COVERAGE["minimum_nondefault_mass_share"]: @@ -1145,42 +1098,27 @@ def _hmrc_family_coverage_contract( # truthfully retains the 208-fact adjudicated-partial-replay verdict. "status": "required_at_build", "restoration_status": str(frs_leaves["status"]), - "stage": "hmrc_spi_income", - "source_manifest": HMRC_SOURCE_STAGES_PATH.name, - "source_manifest_sha256": _sha256(HMRC_SOURCE_STAGES_PATH), - # The two re-mapped period fields below come from the CANONICAL - # manifest (the #723 signed re-map lives there; the frozen mirror - # keeps its June bytes), so the bytes they derive from are pinned - # separately - evidence fields and their hash must name the same - # source (adversarial-review finding, 2026-08-20). - "canonical_source_manifest": SOURCE_STAGES_PATH.name, - "canonical_source_manifest_sha256": _sha256(SOURCE_STAGES_PATH), - "superseded_by": { - "stage": "hmrc_spi_income_spine", - "source_manifest": SOURCE_STAGES_PATH.name, - "source_manifest_sha256": _sha256(SOURCE_STAGES_PATH), - "reason": ( - "The FRS spine build executes hmrc_spi_income_spine, which " - "supersedes the June retained-leaves/hmrc_spi_income pair " - "inside source_stages.json." - ), - }, - "base_candidate_sha256": str(base_candidate["sha256"]), - "base_candidate_tier": base_candidate_tier, + "stage": "hmrc_spi_income_spine", + "source_manifest": SOURCE_STAGES_PATH.name, + "source_manifest_sha256": _sha256(SOURCE_STAGES_PATH), + "base_candidate_tier": validate_uk_release_tier(candidate_source["tier"]), + "required_predecessor_stages": [ + "frs_hmrc_spine_leaves", + "spi_support_channel", + "spi_income_band_donors", + ], "source_vintages": { "spi_donor": str(artifacts["qrf_donor"]["vintage"]), "hmrc_surface": str(artifacts["published_fact_surface"]["vintage"]), "mapped_build_period": str( - canonical_artifacts["published_fact_surface"]["mapped_build_period"] + artifacts["published_fact_surface"]["mapped_build_period"] ), "period_mapping": str( - canonical_artifacts["published_fact_surface"]["period_mapping"] + artifacts["published_fact_surface"]["period_mapping"] ), }, - "spi_prior_national_household_mass_share": float( - prior["spi_prior_national_household_mass_share"] - ), - "required_mass_change_reason": str(prior["mass_change_reason"]), + "spi_prior_national_household_mass_share": float(prior["share"]), + "required_mass_change_reason": SPI_PRIOR_MASS_CHANGE_REASON, "input_weight_kind": str(classification["input_weight_kind"]), "output_weight_kind": str(classification["output_weight_kind"]), "calibration_permitted": bool(classification["calibration_permitted"]), diff --git a/tools/graph_uk_spine_fixture.py b/tools/graph_uk_spine_fixture.py index cd5d2823f..de0401352 100644 --- a/tools/graph_uk_spine_fixture.py +++ b/tools/graph_uk_spine_fixture.py @@ -71,7 +71,7 @@ uk_frs_spine_seed_frame, ) from microcosm.build.uk_runtime.frs_take_up import UKFRSTakeUpStageTransform -from microcosm.build.uk_runtime.graph import UK_SPINE_EXCLUSIONS, uk_spine_graph +from microcosm.build.uk_runtime.graph import uk_spine_graph from microcosm.build.uk_runtime.hmrc_capital_gains import ( HMRC_CGT_GAIN_BAND_LOWER_BOUNDS, HMRC_CGT_INCOME_BAND_LOWER_BOUNDS, @@ -1118,8 +1118,6 @@ def _fixture_stages( assert spec.sources is not None stages: list[SourceStageSpec] = [] for committed in spec.sources.stages: - if committed.stage in UK_SPINE_EXCLUSIONS: - continue artifacts = [ dict(frs_artifacts[str(artifact["table"])]) if artifact.get("table") in frs_artifacts From 79f95fdbd27ca335a0b1573d193d63c29d8c4044 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 25 Sep 2026 21:02:00 +0100 Subject: [PATCH 25/44] Re-point the HDF write-site registry at spine_build for the non-release smoke marker The uk_smoke_non_release_root_attrs exclusion named tools/build_uk_frs_spine.py::_mark_non_release_h5; that function moved into uk_runtime/spine_build.py with the spine consolidation, so the registry's classification of every writable production HDF site was refusing. Path only. Verified: test_frame_serializer_registry passes; ruff clean. Co-Authored-By: Claude Fable 5.1 --- .../src/microcosm/build/frame_serializer_registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/microcosm-build/src/microcosm/build/frame_serializer_registry.py b/packages/microcosm-build/src/microcosm/build/frame_serializer_registry.py index 00ddd7612..d54a425bc 100644 --- a/packages/microcosm-build/src/microcosm/build/frame_serializer_registry.py +++ b/packages/microcosm-build/src/microcosm/build/frame_serializer_registry.py @@ -222,7 +222,7 @@ class HdfWriteExclusion: HdfWriteExclusion( exclusion_id="uk_smoke_non_release_root_attrs", writer=HdfWriteSite( - "tools/build_uk_frs_spine.py", + "packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py", "_mark_non_release_h5", ), reason="Adds non-release smoke metadata to existing root attributes only.", From 428e8f927fd41a0dc6448c691898f3d83cd3beff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 25 Sep 2026 21:16:57 +0100 Subject: [PATCH 26/44] Document the re-based UK full-build graph: driver, roles, spine module, changelog and receipts docs/uk-full-build-graph.md describes the one-graph build on main: the driver and its release roles (dense on the graph, national dispatched to the retained seam), every flag with the hashes main's validator requires, the 33-stage spine module with its gate nodes and stored evidence, the composed nodes and the driver-appended ones, the posture output names and manifest, the Logbook and staging envelope, the strict checkpoint gate, the known behaviour changes and the parity still owed. The four runbooks, the staging operations note, the Chronicle re-pin note, README and CLAUDE.md name the driver and the stubs and drop the retired June builder. Changelog fragments cover the graph build and driver, the shared additions, the HMRC tail retirement and the amendment renumbering. experiments/901-uk-main-rebase-receipts.md records the port set, the Phase-1 licensed A/B, the per-phase test counts, the behaviour changes and the parity blocker. Verified: tools/graph_acceptance_burndown.py --verify ok; every documented flag resolves against the drivers' --help. Co-Authored-By: Claude Fable 5.1 --- CLAUDE.md | 6 +- README.md | 11 +- ...raph-interface-amendments-26-27.changed.md | 1 + .../901-uk-hmrc-tail-retired.removed.md | 1 + changelog.d/uk-full-build.changed.md | 1 + changelog.d/uk-full-graph-contracts.added.md | 1 + docs/uk-chronicle-feed-repin.md | 9 +- docs/uk-dataset-size-plan-355.md | 6 +- docs/uk-dense-release-assembly-runbook-762.md | 15 ++- docs/uk-full-build-graph.md | 113 ++++++++++++++++++ docs/uk-national-calibration-runbook-623.md | 30 +++-- ...k-national-release-assembly-runbook-806.md | 8 +- docs/uk-staging-operations.md | 14 ++- experiments/901-uk-main-rebase-receipts.md | 85 +++++++++++++ 14 files changed, 264 insertions(+), 37 deletions(-) create mode 100644 changelog.d/901-uk-graph-interface-amendments-26-27.changed.md create mode 100644 changelog.d/901-uk-hmrc-tail-retired.removed.md create mode 100644 changelog.d/uk-full-build.changed.md create mode 100644 changelog.d/uk-full-graph-contracts.added.md create mode 100644 docs/uk-full-build-graph.md create mode 100644 experiments/901-uk-main-rebase-receipts.md diff --git a/CLAUDE.md b/CLAUDE.md index ceffd14e0..683e41a44 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -129,7 +129,8 @@ effect of another task. A UK rowwise run's **staged** bundle is inspection evidence, not a release: it never moves `releases/` or `latest.json` and is not loadable through the certified loader. The build's default is to upload that bundle (hundreds of megabytes of licensed microdata) -to the private repository; when you run `tools/build_uk_rowwise_candidate.py` +to the private repository; when you run `microcosm-build-uk` +(`tools/build_uk_full.py`, or its stub `tools/build_uk_rowwise_candidate.py`) yourself, pass `--staging-local-only` unless the operator asked for a staged upload. @@ -238,7 +239,8 @@ Update this guide in the same PR whenever the workspace layout, test commands, or release flow change. If you find it contradicting the repo, trust the repo and fix this file. -UK size experiments use `tools/build_uk_rowwise_candidate.py --release-role dense --dataset-households` +UK size experiments use `microcosm-build-uk --release-role dense --dataset-households` +(`tools/build_uk_full.py`; `tools/build_uk_rowwise_candidate.py` is a stub over it) with the same pool inputs as the dense candidate. The flag changes exported support, not clone K. Sizes remain candidate-only until their matched comparison and promotion scorecard are adjudicated; see diff --git a/README.md b/README.md index b3251ab4d..2971f9f1d 100644 --- a/README.md +++ b/README.md @@ -71,11 +71,12 @@ This writes `progress.json`, `events.ndjson`, `calibration_progress.json`, and final candidate diagnostics under `runs//` without updating production `latest.json`. -The UK commands (`tools/build_uk_frs_spine.py` and -`tools/build_uk_rowwise_candidate.py`, whose `--release-role` builds either -the national or the dense line) stage version 2 telemetry to -`policyengine/populace-uk-staging` under the same switch. The rowwise -candidate command also **stages the finished dataset bundle** it built, +The UK commands (`tools/build_uk_frs_spine.py`, a shim over the package's +`uk_runtime.spine_build`, and `microcosm-build-uk` / `tools/build_uk_full.py`, +whose `--release-role` builds either the national or the dense line; +`tools/build_uk_rowwise_candidate.py` is a stub over the same driver) stage +version 2 telemetry to `policyengine/populace-uk-staging` under the same +switch. The build command also **stages the finished dataset bundle** it built, national, dense or exact-count, under `staged//` in the private `policyengine/populace-uk-private` repository so the team can inspect it without publishing it: `releases/` and `latest.json` are untouched, the diff --git a/changelog.d/901-uk-graph-interface-amendments-26-27.changed.md b/changelog.d/901-uk-graph-interface-amendments-26-27.changed.md new file mode 100644 index 000000000..de26e0750 --- /dev/null +++ b/changelog.d/901-uk-graph-interface-amendments-26-27.changed.md @@ -0,0 +1 @@ +The two shared graph-contract amendments of microcosm#918, which that proposal recorded as 25 (a same-kind `WeightUpdate` is declarable) and 26 (the kernel context carries the version's metadata, mass log and column order), are numbered 26 and 27 on main, because main recorded the live-population observer opt-in (microcosm#950 and #951) as amendment 25 while #918 was open. Every code comment, docstring, test docstring, changelog fragment, acceptance heading and receipt moves in lockstep, and `docs/graph-interface.lock` is re-recorded because the renumbered comments live in `decl.py` and `kernel.py` (the owner sign-off rule of the acceptance record applies). The `_project_context` docstring now points at `_execute_graph`, where microcosm#938 moved the boundary selection, and amendment 27 states that the executor's boundary mass logs are live references under amendment 25's opt-in. The three root-level review artefacts of the shared-contract lane are dropped; the receipts under `experiments/` stay, with a note on the renumbering. diff --git a/changelog.d/901-uk-hmrc-tail-retired.removed.md b/changelog.d/901-uk-hmrc-tail-retired.removed.md new file mode 100644 index 000000000..3d81babcf --- /dev/null +++ b/changelog.d/901-uk-hmrc-tail-retired.removed.md @@ -0,0 +1 @@ +Removed the two frozen HMRC tail stages `frs_hmrc_retained_leaves` and `hmrc_spi_income` from the UK spine manifest (35 to 33 stages) together with their transform module `uk_runtime/frs_hmrc_leaves.py`, their source-stage resource `uk/hmrc_income_source_stages.json` (and their entries in `uk/source_stages.json` and `uk/spec/sources.yaml`) and the `UK_SPINE_EXCLUSIONS` constant that hid them from every consumer, so the manifest roster is the graph roster. The FRS HMRC leaf columns now come from `uk_runtime/frs_hmrc_source`, which the HMRC source contract, the SPI spine and income stages, the source runtime and the graph kernels read; `release_input_coverage` refuses `superseded_by` and parses `required_predecessor_stages`, and the coverage-manifest tool loses its HMRC path. Because microcosm#1006 placed `spi_income_band_donors` between the support channel and the income spine, the audited HMRC family names it as a predecessor and the contract admits its two operation kinds. The release-input coverage manifest and the H2 spine parity fixture were regenerated with their tools; the gate-register digests did not move. diff --git a/changelog.d/uk-full-build.changed.md b/changelog.d/uk-full-build.changed.md new file mode 100644 index 000000000..f4f462554 --- /dev/null +++ b/changelog.d/uk-full-build.changed.md @@ -0,0 +1 @@ +The UK full build is one executable graph served by one driver, `microcosm-build-uk` (`tools/build_uk_full.py`), which carries the release roles of microcosm#823. `--release-role dense` builds the K-clone joint national and local surface through the graph: source spine, geographic cloning, target compilation and selection, calibration, exact-count sizing, gates, diagnostics and a checked export, with all applicable geographies calibrated by default and `--target-geographies country` as an explicit filter in the same build. `--release-role national` is parsed and validated by the same posture-aware validator and then dispatched, before any graph is prepared, to the retained calibration seam through `uk_runtime.national_role`, so the national line is built by the same engine as before. The dense role keeps the posture's solve defaults and refusal tables, the `microcosm_uk_2024_25_local.*` output names, a schema-4 `rowwise_candidate_manifest.json` projected from the stored graph artifacts (which the dense release pre-flight and assembler accept), the Logbook row, staging telemetry around each graph phase and the staged bundle, and gains `--baseline-pi-floor`, `--no-size-checkpoint`, `--candidate-clone-counts` and `--households-only`. `tools/build_uk_rowwise_candidate.py` is now a stub over the driver and `tools/build_uk_frs_spine.py` a shim over `uk_runtime.spine_build`, into which the spine tool moved with its gate batteries as graph nodes and its stage evidence read back from the content store; every runbook command keeps working through them. Bound stage, gate and diagnostic artifacts are restored on replay, exported bytes are checked against their declared artifacts, and the graph's terminal node reports unsigned certification readiness. diff --git a/changelog.d/uk-full-graph-contracts.added.md b/changelog.d/uk-full-graph-contracts.added.md new file mode 100644 index 000000000..58ab43922 --- /dev/null +++ b/changelog.d/uk-full-graph-contracts.added.md @@ -0,0 +1 @@ +Added the country-agnostic pieces the UK full-build graph binds: `microcosm.build.artifact_files` (file artifacts, byte materialisation and staged-bundle publication), `microcosm.build.stage_evidence` (the typed stage-evidence artifact and its codec, which walks the observation and graph-source proxies the way the UK driver's collector did), the gate-battery phase-report payload codec with `GateBatteryRun.record_phase`, `microcosm.calibrate.artifacts` (ordered problem, solution and calibration-result artifacts), `microcosm.calibrate.target_selection` (ordered target-selection receipts) and the `TargetSpec` `to_dict`/`from_dict` codec on the registry. On the UK side, dense calibration, the informed size search, the exact-count draw and the refit are separate resumable graph nodes over the original pool. The same-kind weight update and the frame-context kernel fields land through the shared graph amendments 26 and 27. diff --git a/docs/uk-chronicle-feed-repin.md b/docs/uk-chronicle-feed-repin.md index cc4b68c59..8a830013d 100644 --- a/docs/uk-chronicle-feed-repin.md +++ b/docs/uk-chronicle-feed-repin.md @@ -52,9 +52,12 @@ refuses. The cross-grain legs of English constituencies and authorities come from `region_code_by_area` in `local_area_crosswalk.json`, regenerated from the sha-pinned ladder with `tools/generate_uk_local_area_crosswalk.py`. -The national calibration runner refuses a feed whose facts or manifest digest -differs from the committed pin. `--allow-unpinned-feed` is an explicit -diagnostic override recorded in the run manifest; it is not a re-pin procedure. +Both release roles of `microcosm-build-uk` refuse a feed whose facts or +manifest digest differs from the committed pin. On the national role +`--allow-unpinned-feed` is an explicit diagnostic override recorded in the run +manifest; it is not a re-pin procedure. The dense role refuses that flag: the +graph's target compilation checks the supplied hashes and the artifact against +the committed pin and has no override. History: the `ec7169b` re-pin (#887/#900) moved census household targets onto the same Chronicle compile path as every other bound UK local family; the diff --git a/docs/uk-dataset-size-plan-355.md b/docs/uk-dataset-size-plan-355.md index df667b8f8..926d8d56d 100644 --- a/docs/uk-dataset-size-plan-355.md +++ b/docs/uk-dataset-size-plan-355.md @@ -73,10 +73,12 @@ on [#870](https://github.com/PolicyEngine/microcosm/pull/870)'s branch. The PR i Use the inputs and environment from the existing [UK dense assembly runbook](uk-dense-release-assembly-runbook-762.md). -Pass the same pinned source arguments to the existing driver and add: +Pass the same pinned source arguments to the graph driver +(`microcosm-build-uk`, `tools/build_uk_full.py`; +`tools/build_uk_rowwise_candidate.py` is a stub over it) and add: ```bash -uv run python tools/build_uk_rowwise_candidate.py --release-role dense \ +uv run python tools/build_uk_full.py --release-role dense \ --input-h5 "$UK_SPINE_H5" --input-sha256 "$UK_SPINE_SHA256" \ --ladder "$UK_LADDER_NPZ" --ladder-sha256 "$UK_LADDER_SHA256" \ --ledger-facts "$UK_LEDGER_FACTS" \ diff --git a/docs/uk-dense-release-assembly-runbook-762.md b/docs/uk-dense-release-assembly-runbook-762.md index ef36300e4..59c399a06 100644 --- a/docs/uk-dense-release-assembly-runbook-762.md +++ b/docs/uk-dense-release-assembly-runbook-762.md @@ -2,7 +2,9 @@ The dense line `microcosm-uk-2024-25-dense` is the spine cloned K=15 times through the OA geography ladder and calibrated to the national and local -target surfaces in one solve (`tools/build_uk_rowwise_candidate.py`). It ships +target surfaces in one solve (`microcosm-build-uk --release-role dense`, the +graph driver `tools/build_uk_full.py`; `tools/build_uk_rowwise_candidate.py` +is a stub over it). It ships on the **inspect lane only**: a constant release id, an immutable per-cut tag, `dataset_role: non_default_local_area`, an empty `default_datasets` map, and `--no-latest` at publication, so it can never displace the default artifact. @@ -36,7 +38,7 @@ digest mismatch, or doctrine constants that are not the ruled ones. ## 2. Run the release candidate ```bash -uv run --no-sync python tools/build_uk_rowwise_candidate.py --release-role dense \ +uv run --no-sync python tools/build_uk_full.py --release-role dense \ --release-candidate --input-h5 --input-sha256 \ --ladder build/uk/uk_oa_ladder_2021.npz --ladder-sha256 \ --ledger-facts --ledger-facts-sha256 \ @@ -53,7 +55,14 @@ the doctrine (bound 10, `grain_equal`, K=15, 1500 epochs), resolves the engine in a single block, and runs the rotated holdout. Best-effort staging telemetry uploads every 300 s by default on this driver (the Hub allows about 128 commits per hour per repository). -Expect about 3.5 hours and 10 GB at K=15. +Expect about 3.5 hours and 10 GB at K=15 (measured on the rowwise tool; the +graph driver's dense build has not been timed yet). Since microcosm#901 the +dense role runs through the [full-build graph](uk-full-build-graph.md): the +candidate directory also holds `candidate.json`, `certification.json`, +`operations.json` and the `build.json` completion marker, the graph store sits +at `/.graph-store` unless `--graph-store` says otherwise, and +`--input-h5` must be a spine whose gate manifest matches the branch's gate +declarations. ## 3. Pre-flight the finished run, then score it diff --git a/docs/uk-full-build-graph.md b/docs/uk-full-build-graph.md new file mode 100644 index 000000000..25117b4b0 --- /dev/null +++ b/docs/uk-full-build-graph.md @@ -0,0 +1,113 @@ +# UK full-build graph + +The UK has one calibration build. It constructs the canonical FRS spine, samples the pool when requested, expands linked entities into K geographic copies, assigns locations, constructs the selected contribution matrix, calibrates, optionally selects exactly k households and refits, evaluates gates and diagnostics, and packages a checked H5. It is registered on the shared executable graph (`microcosm.graph`) and served by one driver, `microcosm-build-uk` (`tools/build_uk_full.py`), which carries the release roles of microcosm#823: `--release-role dense` runs this graph, and `--release-role national` is validated by the same posture-aware validator and dispatched to the retained calibration seam (see "The national role" below). `tools/build_uk_rowwise_candidate.py` is a stub over the driver, so the runbook commands that name it keep working. + +**The default is to calibrate all applicable geographies together.** An omitted selector and `--target-geographies all` have the same target scope. `--target-geographies country` explicitly selects country-level rows in the same graph; regional rows are not country-level rows. No failure, size request or performance setting changes the selector implicitly. The geographic pool copies K (`--n-clones`), the exported household count k (`--dataset-households`) and the target scope are independent settings. This registration preserves the current UK algorithms and does not establish native candidate acceptance. + +## Run the dense build + +From a canonical spine checkpoint with its `.build.json` and `.spine_gates.json` sidecars: + +```bash +uv run --no-sync python tools/build_uk_full.py --release-role dense \ + --input-h5 /data/uk/spine.h5 --input-sha256 \ + --ladder /data/uk/ladder.npz --ladder-sha256 \ + --ledger-facts /data/chronicle/uk-artifact \ + --ledger-facts-sha256 --ledger-manifest-sha256 \ + --out /data/uk/full-build +``` + +`--release-role` is required. The dense role supplies every unset solve default from the local doctrine (epochs, learning rate, seed, K, weight rule, constituency vintage, selection) and refuses the national role's flags (`--target-loss-cap`, `--allow-unpinned-feed`, `--incumbent-h5` and `--incumbent-sha256`). `--ladder` with `--ladder-sha256` is required by the dense role, `--input-sha256` by every `--input-h5` build, and the three Ledger arguments (`--ledger-facts`, `--ledger-facts-sha256`, `--ledger-manifest-sha256`) by every build. The supplied hashes must agree with the committed Chronicle pins: a target-scope filter does not authorise a different source, and the dense role has no unpinned-feed override. + +The checkpoint must bind the exact frame content, the current spine stage roster and the gate-report bytes. The bound-spine node compares the checkpoint's gate report digests with the branch's own gate declarations and refuses a spine whose gate manifest differs from them, so `--input-h5` needs a spine built by a branch with the same declarations; every acceptance spine on disk when this registration landed predates them and is not admitted. Historical candidate H5 files and reviewed-bypass sidecars are not alternate build sources. Chronicle facts and manifest must match the independently reviewed national and local feed declarations; filtering targets does not relax source validation. + +The target registry binds Census household and demographic rows from the reviewed Chronicle feed, including the approved Northern Ireland constituency geography. The OA ladder supplies geographic assignment and lookup support. Its household counts are not a second source of calibration targets. Source receipts retain the Chronicle identity and the paired ladder digest, so target values and the geography used to assign households can be audited separately. + +To include raw spine construction in the same execution, pass `--spine-request /data/uk/spine-request.json` instead of `--input-h5`. This file is a JSON array of the raw-source arguments accepted by `uk_runtime.spine_build` (the arguments of `tools/build_uk_frs_spine.py`): + +```json +[ + "--frs-raw-dir", "/data/frs/2024-25", + "--spi-tab", "/data/spi/put2223uk.tab", + "--hmrc-ods", "/data/hmrc/collated.ods", + "--was-tab", "/data/was/household.tab", + "--nts-household-tab", "/data/nts/household.tab", + "--nts-individual-tab", "/data/nts/individual.tab", + "--nts-trip-tab", "/data/nts/trip.tab", + "--nts-stage-tab", "/data/nts/stage.tab", + "--nts-ticket-tab", "/data/nts/ticket.tab", + "--lcfs-hh-tab", "/data/lcfs/household.tab", + "--lcfs-person-tab", "/data/lcfs/person.tab", + "--etb-tab", "/data/etb/household.tab" +] +``` + +The request declares existing source adapters and lazy transforms; the spine stages execute in the same graph, and the driver supplies `--spine-h5` when the request omits it. A spine request has no checkpoint H5 to pin, so `--input-sha256` is not required on that path. The spine-only command remains available to materialise an execution checkpoint. + +For smaller outputs, add `--dataset-households 100000` to use the common informed L0 search, exact-count draw and refit. `--baseline-pi-floor` floors the inclusion probability the refit's Horvitz-Thompson baseline divides each selected row's dense weight by (0, the default, is the untrimmed baseline), and `--no-size-checkpoint` skips materialising `size_selection_checkpoint.{npz,json}` into `--out`. `--n-clones K` sets the number of geographic copies in the pool; `--candidate-clone-counts 5,10,15` with `--dry-run` prints the compiled operation inventory for each K without solving. K and k are independent, and neither narrows target scope. A country-only run may explicitly request a smaller K, but this is never inferred. `--households-only` binds only the Chronicle census-household constituency targets (a target-selection family filter). + +`--sample-fraction` samples before geographic cloning. Raw-spine sampling in the JSON request instead occurs at the original ingest boundary, before enrichment. An already sampled spine cannot be sampled a second time. The effective sample fraction controls development gate and target-admission policy. + +## The national role + +`--release-role national` builds the certified national line. The driver parses and validates it with the same posture-aware validator (the national role refuses the dense role's arguments, `--release-candidate` among them, and requires a bound `--input-h5` with `--input-sha256`; a `--spine-request` is refused because the seam reads a pinned checkpoint) and then, before any graph is prepared, dispatches to `uk_runtime.national_role.run_national_role`. That module is the rowwise tool's national branch moved unchanged: the calibration seam of microcosm#823 (`uk_runtime.calibration_run.run_uk_calibration`, the national calibration stage, the release-cut battery and the certifier stay exactly as they were). Its outputs (`microcosm_uk_2024_25.h5`, `build_record.json`, `microcosm_uk_2024_25.terminal_gates.json`, `calibration_diagnostics.json`, the registries and `rowwise_candidate_manifest.json`), its Logbook row, staging telemetry, staged bundle and incumbent evaluation are the seam's own; the [national calibration runbook](uk-national-calibration-runbook-623.md) and the [national release assembly runbook](uk-national-release-assembly-runbook-806.md) describe them. Parity with the previous national line is by identity: the same engine runs the same code. A posture-driven national path through this graph is planned as the next change on this line and is not part of this registration. + +## Graph owners and shared contracts + +The spine graph is the 33-stage source roster declared by the UK spec, built by `uk_runtime.spine_build` (the spine tool moved into the package; `tools/build_uk_frs_spine.py` is a six-line shim over it). The superseded `frs_hmrc_retained_leaves` and `hmrc_spi_income` stages and the `UK_SPINE_EXCLUSIONS` list that hid them are removed; the active HMRC path is `frs_hmrc_spine_leaves`, `spi_support_channel`, `spi_income_band_donors` and `hmrc_spi_income_spine`, and the FRS HMRC leaf columns come from `uk_runtime.frs_hmrc_source`. The spine's assembled and transferred gate batteries are graph nodes (`uk_runtime.graph_evidence`); stage evidence and fit-weight records are read back from the content store through the shared `stage_evidence` artifact rather than from in-memory collectors; the HMRC replay sidecar is rebuilt from the SPI stage's checkpoint metadata; the spine sidecar records the operation inventory and the graph manifest, and the graph manifest is saved as `spine.graph.json` under the checkpoint root. + +| Operation | Graph owner | +| --- | --- | +| Raw FRS and donor preparation, enrichment, support channels | The 33 UK spine stage nodes and their declared composite operations | +| Assembled and transferred spine gates | `spine.gates.assembled`, `spine.gates.transferred` | +| Bound checkpoint admission | `uk.full.spine_checkpoint`, when resuming a saved spine | +| Pool sample and mass normalization | `uk.full.sample`, `uk.full.normalize` | +| Linked entity expansion and ancestry | `uk.full.expand`, `uk.full.expand.owned` | +| Location draw, mapping and integrity | `uk.full.locations`, `uk.full.geography_mapping`, `uk.full.geography_gate` | +| Full pinned source/register compilation | `uk.full.target_compilation` | +| Explicit target selection and inclusion/exclusion receipt | `uk.full.target_selection` | +| Engine measures and ordered contribution problem | `uk.full.measures`, `uk.full.problem` | +| Complete original-pool checkpoint | `uk.full.pool` | +| Source/reference preflight and dense reference | `uk.full.gates.preflight`, `uk.full.dense` | +| Informed search, exact draw and refit (with `--dataset-households`) | `uk.full.size_search`, `uk.full.size_draw`, `uk.full.size_refit` | +| Selected population (with `--dataset-households`) and installed calibrated weights | `uk.full.selected`, `uk.full.calibrated` | +| Rotated local holdout and final gates/diagnostics | `uk.full.holdout`, `uk.full.gates.calibrated` | +| Export contract, H5 readback and package inventory | `uk.full.export.prepare`, `uk.full.export.readback`, `uk.full.package` | +| Gate, comparison and export evidence for certification | `uk.full.certification` | + +The modules under `uk_runtime` divide the roster: `graph_build` composes the graph over the bound spine, `graph_population` owns sample through the geography gate, `graph_targets` owns target compilation through the problem, `graph_calibration` owns the dense solve, the size nodes and the calibrated population, `graph_terminal` owns the gate nodes, the holdout, the export and the package, `full_certification` owns the terminal certification node, and `full_build_cli` resolves the request, executes graph endpoints and atomically materialises their stored artifacts. + +`operations.json` is generated from the compiled graph, including actual dependencies and artifact owners. It is the execution inventory, rather than a second manually maintained pipeline roster. Composite source/model stages preserve their existing numerical boundary; for example, WAS retains its joint donor/recipient encoding dependency. This registration does not change imputation order, RNG consumption, clone IDs or geography methodology. + +Shared machinery includes graph execution/storage/replay, typed artifacts, explicit same-kind weight updates, target selection receipts, ordered sparse calibration problem/solution/result codecs, exact-count selection, atomic artifact materialization and bundle publication. UK adapters retain source interpretation, entity relationships, geography mappings, measure bindings and gate prescriptions. + +The full target compiler preserves the unreduced band-edge register, reference-period compilations, approved exclusions and frozen-register completeness checks before target selection. A nonempty country-only problem can have zero local rows. Local holdout is then inapplicable, while source, identity, mass and geographic integrity checks remain applicable. Omitted rows are never reported as fitted constraints. + +## Replay and outputs + +The shared store defaults to `/.graph-store`. `--graph-store` can reuse another store. `--resume require` requires completed numerical nodes and evidence to be available; output materialization and byte readback still verify the recreated files. `--resume-size-checkpoint` imports a legacy size-search checkpoint only after validating invocation identity, ordered target/household axes, initial weights and recomputed losses. It skips the saved dense solve and search. New runs persist their intermediates as graph artifacts before drawing. + +Each attempt also keeps its checkpoint manifests and small stage, gate and provenance reports under `/uk-full-attempts/`. A failed run's `failure.json` links to this evidence, including a persisted spine gate verdict before downstream admission stops execution. A previously completed output bundle remains intact. + +The output bundle is named from the role's posture and the FRS release vintage: `microcosm_uk_2024_25_local.h5`, its signed gate report `microcosm_uk_2024_25_local.local_gates.json`, and the `.diagnostics.json`, `.targets.csv`, `.area_support.csv`, `.holdout.json` and `.target_selection.json` siblings on the same stem, beside `graph.json`, `operations.json`, the stored source/stage/gate evidence and the graph manifests. `candidate.json` is the immutable graph package inventory: it binds the dataset, its evidence, the target selector and the independent K/k request. Comparison inputs refer to this candidate identity. Adding comparison evidence does not rewrite the candidate package. `rowwise_candidate_manifest.json` is projected from the stored artifacts in the schema-4 shape the rowwise tool wrote (`graph_terminal.rowwise_candidate_manifest_from_graph`), so the dense release pre-flight and assembler read a graph build as they read a rowwise-tool build; it records the release role, the release verdict, `staging_delivery` and `staged_dataset`. + +The terminal graph node writes unsigned `certification.json` from those identified artifacts and any declared native or matched-size comparisons. `build.json` is the completion marker that binds both `candidate.json` and the certification artifact. Physical output bytes are checked against their declared artifacts. Bundle publication writes the completion marker last and rolls back handled failures or interrupts. A process kill or power loss can leave an absent completion marker; a directory without a valid bound marker is not a completed build. + +A non-dry dense run is wrapped in the rowwise tool's operational envelope: the Logbook attempt (a `uk-local-candidate` row spooled under `/logbook-spool` on every terminal outcome, chained through `--logbook-prev-row-digest` or `POPULACE_LOGBOOK_PREV_ROW_DIGEST`, with an error receipt on failure), version 2 staging telemetry with stage events around each graph phase and per-epoch `calibration_progress` rows from the dense solve, and the staged-dataset delivery of the published bundle under `staged//` in the private repository. `--staging-local-only`, `--no-staging`, `--staging-read-back` and `--no-staged-dataset` behave as in [UK staging operations](uk-staging-operations.md). Dry runs plan without solving or writing and record no Logbook row. + +Structural failures stop export. The maintained local statistical failure policy may still export an unreleasable diagnostic candidate with a nonzero process status. Missing evidence remains explicit. `--release-candidate` applies the maintained strictness and solve settings; it does not publish, sign or authorize a release. Fixture acceptance proves graph behavior. Native certification additionally requires measured incumbent comparison evidence supplied with `--native-scorecard`; exact-count promotion also requires the measured comparison at the requested k through `--matched-size-scorecard`. The certification node verifies their candidate and output identities before assessing readiness. + +`tools/build_uk_rowwise_dataset.py` stays the separate driver it was (its tests load it by path, and it still serves `--candidate-clone-counts`). `tools/calibrate_uk_national_dataset.py` was retired by microcosm#823 and does not forward. + +## Known behaviour changes + +Recorded for review in `experiments/901-uk-main-rebase-receipts.md` (R3): + +- A blocked *assembled* spine gate now fails inside `run_graph`, so the gate report lives in the content store on that path (the previous tool wrote `spine_gates.json` before raising); a blocked *transferred* gate still writes the file. +- `numerical_dependencies` pins installed versions into the H2 fixture. +- `_normalise_uk_local_bound_families` accepts an empty declaration (country-only scope). +- Per-epoch `calibration_progress` staging rows come from the dense solve only. +- The HMRC family names `spi_income_band_donors` (microcosm#1006) as a predecessor and admits its two operation kinds; the contract otherwise refused the drifted operation order. +- `tools/build_uk_rowwise_dataset.py` stays as it was on main (its tests load it by path; it still serves `--candidate-clone-counts`). + +The dense and national parity builds through this driver are owed once a spine can be built at main's head (receipts R4); the national role is parity by identity regardless. diff --git a/docs/uk-national-calibration-runbook-623.md b/docs/uk-national-calibration-runbook-623.md index ca059a55c..786da9b91 100644 --- a/docs/uk-national-calibration-runbook-623.md +++ b/docs/uk-national-calibration-runbook-623.md @@ -17,22 +17,26 @@ is the release-cut producer's job, not calibration's. ## Command shape -Calibration runs through the rowwise driver's national release role, -`tools/build_uk_rowwise_candidate.py --release-role national` (microcosm#823; -it replaced the retired `tools/calibrate_uk_national_dataset.py`). The role -delegates the build to the calibration seam library -(`uk_runtime.calibration_run.run_uk_calibration`): it is the only path that -builds the measure resolver from the input file and applies the committed +Calibration runs through the UK build driver's national release role, +`microcosm-build-uk --release-role national` (`tools/build_uk_full.py`; +`tools/build_uk_rowwise_candidate.py` is a stub over the same driver). The +role arrived with microcosm#823, which retired +`tools/calibrate_uk_national_dataset.py`, and moved into the graph driver with +microcosm#901. The driver validates the request with its posture-aware +validator and, before any graph is prepared, dispatches to +`uk_runtime.national_role`, which delegates the build to the calibration seam +library (`uk_runtime.calibration_run.run_uk_calibration`): it is the only path +that builds the measure resolver from the input file and applies the committed measure-exclusion register, and 187 of the activated references bind model outputs that no frame carries — so it is the only path on which this target -surface materializes. No cloning, no ladder, national targets only. The June -builder (`tools/build_uk_national_dataset.py`) constructs the calibration -stage without either and aborts on the first unmaterializable reference; it -also rebuilds SPI income onto its input, which a spine artifact already -carries. +surface materializes. No cloning, no ladder, national targets only; the role +builds from a bound `--input-h5` and refuses a `--spine-request`. The retired +June builder constructed the calibration stage without either and aborted on +the first unmaterializable reference; it also rebuilt SPI income onto its +input, which a spine artifact already carries. ```bash -uv run --no-sync python tools/build_uk_rowwise_candidate.py --release-role national \ +uv run --no-sync python tools/build_uk_full.py --release-role national \ --input-h5 data/ukds/acceptance/623-first-calibrated-candidate/input-spine.h5 \ --input-sha256 \ --ledger-facts \ @@ -100,7 +104,7 @@ side too. The receipt's `evaluation` block decides rule 1 (#578) on that surface: `verdict` is `passed` when the candidate's full loss is below the incumbent's, `failed` otherwise, and the release-cut certifier refuses any receipt whose verdict is not `passed` or whose surface does not close, so -publication never runs on an unpassed evaluation. The rowwise driver's +publication never runs on an unpassed evaluation. The build driver's national role runs this evaluation at the end of every build it is given an incumbent for (microcosm#965) and writes the same receipt as `score_vs_incumbent.json`. diff --git a/docs/uk-national-release-assembly-runbook-806.md b/docs/uk-national-release-assembly-runbook-806.md index 692b08989..a01db6506 100644 --- a/docs/uk-national-release-assembly-runbook-806.md +++ b/docs/uk-national-release-assembly-runbook-806.md @@ -27,11 +27,13 @@ reference together through the run. ## 1. Calibrate the national candidate -Use the rowwise driver's national release role and record the input digest -rather than relying on a mutable path: +Use the UK build driver's national release role (`microcosm-build-uk`, +`tools/build_uk_full.py`; `tools/build_uk_rowwise_candidate.py` is a stub +over the same driver, and the role dispatches to the retained calibration +seam) and record the input digest rather than relying on a mutable path: ```bash -uv run --no-sync python tools/build_uk_rowwise_candidate.py --release-role national \ +uv run --no-sync python tools/build_uk_full.py --release-role national \ --input-h5 \ --input-sha256 \ --ledger-facts \ diff --git a/docs/uk-staging-operations.md b/docs/uk-staging-operations.md index c99b52c3a..53ec90a10 100644 --- a/docs/uk-staging-operations.md +++ b/docs/uk-staging-operations.md @@ -115,9 +115,10 @@ read-only credential. ## Command modes and files -The two UK commands (`tools/build_uk_frs_spine.py` and -`tools/build_uk_rowwise_candidate.py` in either release role, `national` or -`dense`) support these staging modes: +The two UK commands (`tools/build_uk_frs_spine.py`, a shim over +`uk_runtime.spine_build`, and `microcosm-build-uk` / `tools/build_uk_full.py` +in either release role, `national` or `dense`; `tools/build_uk_rowwise_candidate.py` +is a stub over the same driver) support these staging modes: - Default: local version 2 files plus best-effort delivery to `policyengine/populace-uk-staging`. @@ -153,8 +154,8 @@ rejected before remote storage is called. ## Staged datasets -Telemetry is not the dataset. The rowwise candidate command (dense K=15 and -exact-count `--dataset-households` runs alike) also stages the bundle its +Telemetry is not the dataset. The build command (`microcosm-build-uk`; dense +K=15, exact-count `--dataset-households` and national runs alike) also stages the bundle its manifest vouches for, so a run can be inspected by the team without being published. The two destinations share one run id: @@ -242,7 +243,8 @@ role and is rebuilt, never grandfathered. ## Smoke verification -The UK spine command keeps fractional input sampling for scale tests. The +The UK spine command (`tools/build_uk_frs_spine.py`, a shim over the package's +`uk_runtime.spine_build`) keeps fractional input sampling for scale tests. The `--smoke` option marks its H5, sidecar, and staging records as non-release. It does not invoke national calibration, release certification, release assembly, or publication. diff --git a/experiments/901-uk-main-rebase-receipts.md b/experiments/901-uk-main-rebase-receipts.md new file mode 100644 index 000000000..348c372c0 --- /dev/null +++ b/experiments/901-uk-main-rebase-receipts.md @@ -0,0 +1,85 @@ +# #901 re-based onto main: receipts + +Date: 2026-09-25. Branch `uk-full-build-graph-registration`, new history on `origin/main` +8f628b1e7 (main after #1006). Old head kept locally as the tag `uk-901-pre-main-rebase` +(2da4f421, stacked on #893's 2300e56b). Plan: `repos/uk-901-main-rebase-plan.md` (approved +2026-09-25); audit: `repos/uk-901-rebase-to-main-audit.md`. + +## R0. What was ported and what was not + +- Nothing from #893's shared code was needed by #901: the graph-kernel amendments the UK graph + binds (platform-bitwise scope, keyed seeds, artifact inputs, rewrite ownership, the raw-bytes + codec, the typed-artifact scope rule) were on main before #893 branched (merge-base 15ebde806). +- Folded in: Max's #918 (17 commits, cherry-picked with `-x`, authorship kept), renumbered from + graph amendments 25/26 to 26/27 because main recorded the observer opt-in (#950/#951) as 25; + `docs/graph-interface.lock` re-recorded (decl.py `b25ae4a6…`, kernel.py `24045ab2…`). +- #901's own shared additions (country-agnostic): `artifact_files`, `stage_evidence`, the + gate-phase payload codec and `record_phase`, `calibrate.artifacts`, + `calibrate.target_selection`, the `TargetSpec` dict codec. +- Left with #893: table identity, survey allocation, the CD benchmark, the legacy-QRF fit family, + grouped bounds and target snapshots, the frame-checkpoint v4 schema, two performance fast paths. + +## R1. Phase-1 spine consolidation: licensed A/B + +Inputs identical on both sides: FRS 2024-25 raw tables, SPI 2022-23 PUT, HMRC collated tables +2023-24, WAS round 8, LCFS 2023-24 household and person, ETB 1977-2024, the five NTS tabs; no +sampling; `--no-staging`; checkpoints on. Script and outputs: +`data/ukds/acceptance/901-rebase-ab/` (`build_ab_spine.sh`, `spine-a/`, `spine-b/`, logs). + +- A = main 8f628b1e7, `tools/build_uk_frs_spine.py`; B = branch 80f706050, the six-line shim over + `uk_runtime/spine_build.py`. +- Both runs block at phase `transferred` on `uk_stage_student_loans_realization` (PLAN_5 + realization_deviation 1.1021725784754537 exceeds 1.0). That gate is main's own limitation at + this head, recorded by #1006; neither side reaches an H5. Wall: A 284 s, B 308 s. +- The two 26-gate reports (`spine-a.spine_gates.json`, `spine-b.spine_gates.json`) are identical + after removing the timestamped `release_id` and the signature that covers it: same gates, same + outcomes, same evidence digests, same `blocked_at_phase`. +- The two content stores hold 8,076 stored tables each (`values.npy`); the multiset of their + bytes is identical, with no table unique to either side. Node keys differ, as declared: the + artifact declarations on the spine nodes move every key. +- Owed: the full-H5 A/B once a spine can be built at main's head, and the dense and national + parity builds through the graph driver (see R4). + +## R2. Phase results and test counts (JUnit-counted) + +- Commit 2 (renumber, re-lock): `packages/microcosm-graph/tests` 690 passed / 1 skipped, twice + (before and after the renumbering); `tools/graph_acceptance_burndown.py --verify` ok. +- Commit 3 (shared additions): 66 passed (new suites, gate-register pins, registry codec). +- Phase 1 (spine consolidation, 80f706050): targeted 82 passed; `spine-uk` group 2,825 passed / + 21 skipped; H2 parity 1/1 after regeneration (oracle `edb1659b…`). +- Phase 2 (graph build beside the drivers, 3df839f48): targeted 367 passed; whole `uk` group + 2,599 passed / 20 skipped; lock-hash test 24 passed; spot-check 203 passed. +- Phase 3 (dense role, 9bc39f8aa): whole `uk` group 2,654 passed / 20 skipped; role files 214 + passed; both drivers' `--help` exit 0. +- Phase 4a (national dispatch, 142268fa2): whole `uk` group 2,571 passed / 20 skipped; + 90 + 140 + 145 targeted. +- Phase 4b (HMRC tail retirement, ffe6ed7d2): retirement files 458 passed / 2 skipped; + whole `uk` group 2,549 passed / 20 skipped; `shared-spec` 2,100 passed / 48 skipped; + microcosm-data contract 254; H2 3 passed / 1 skipped; spot-check 337 passed / 1 skipped. +- Registry re-point (7eaf6ff3d): `test_frame_serializer_registry` green. +- Derived surfaces regenerated with their tools, never pasted: H2 spine parity fixture + (`tools/graph_uk_spine_fixture.py`), release-input coverage manifest (`--check` current, 145 + required inputs, `source_stages.json` 73c3a14f…); gate-register digests did not move. + `uv.lock` relocked; `APPROVED_UV_LOCK_SHA256` = `3bd27a6c…`. + +## R3. Behaviour changes recorded for review + +- A blocked *assembled* spine gate now fails inside `run_graph`, so the gate report lives in + the content store on that path (main wrote `spine_gates.json` before raising); a blocked + *transferred* gate still writes the file (R1). +- `numerical_dependencies` pins installed versions into the H2 fixture. +- `_normalise_uk_local_bound_families` accepts an empty declaration (country-only scope). +- Per-epoch `calibration_progress` staging rows come from the dense solve only. +- The HMRC family names `spi_income_band_donors` (#1006) as a predecessor and admits its two + operation kinds; the contract otherwise refused the drifted operation order. +- `tools/build_uk_rowwise_dataset.py` stays main's (its tests load it by path; it still serves + `--candidate-clone-counts`). + +## R4. Blocker for the dense and national licensed parity + +The graph driver refuses every existing acceptance spine at its strict checkpoint gate +(`gates_manifest_sha256 differs from current declarations`), by #901's design, and a fresh spine +cannot be built at main's head (R1). Options put to María on 2026-09-25: wait for the #1012 +lane to settle main's fence; a non-release measurement flag that accepts a stale gate manifest +and records the mismatch; or a measurement-only branch over #1012. The national role is parity +by identity regardless (same engine, moved code); the dense comparison needs a spine. From 637f5d297ca7cd0426dc06930d44c75b799e47db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 26 Sep 2026 11:42:16 +0100 Subject: [PATCH 27/44] Register the UK chronicle source codec explicitly, and let the shared codec suite tolerate country codecs The wheels gate runs every package's tests in one process, and the shared test_graph_codecs asserted that the process-global SOURCE_CODECS held exactly the shipped raw-bytes codec; graph_targets registered the UK chronicle-consumer-facts codec at import, so the UK suite running first broke that assertion in both wheels jobs (the only failures of the main-based CI run). Two changes: the UK registration is now explicit and idempotent (register_uk_source_codecs, called by register_uk_target_kernels, so importing the module leaves the registry untouched and the driver registers before any graph runs), and the shared assertion checks the shipped codec's presence and behaviour rather than that it is alone, because the registry is extensible by design and the executor reads the global at run time. A UK test pins both: a subprocess import leaves the registry at the shipped set, and registration is idempotent. Verified: UK target/admission/CLI suites plus the shared codec suite in one process, 58 passed; ruff clean; ci_test_groups --verify ok. Co-Authored-By: Claude Fable 5.1 --- .../build/uk_runtime/graph_targets.py | 14 +++++- .../engine_free/uk/test_uk_source_codecs.py | 46 +++++++++++++++++++ .../engine_free/shared/test_graph_codecs.py | 5 +- 3 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_source_codecs.py diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py index 965d08daf..69909c473 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py @@ -66,7 +66,18 @@ TARGET_SELECTION_TYPE = ArtifactType("microcosm.uk.full-target-selection", 1) MEASURE_TYPE = ArtifactType("microcosm.uk.full-measured-contributions", 1) -SOURCE_CODECS.register_bytes(CHRONICLE_SOURCE_CODEC, load_chronicle_source_bytes) + +def register_uk_source_codecs(registry=SOURCE_CODECS) -> None: + """Register the UK raw-byte source codecs the target nodes bind. + + Registration is explicit and idempotent: importing this module leaves the + shared registry untouched (the shared codec suite asserts the shipped set), + and the driver's kernel registration is the one place the chronicle + consumer-facts codec joins ``SOURCE_CODECS`` before a graph runs. Re-registering + the same loader is a no-op by the registry's own rule. + """ + + registry.register_bytes(CHRONICLE_SOURCE_CODEC, load_chronicle_source_bytes) def registry_payload(registry: TargetRegistry) -> dict: @@ -665,6 +676,7 @@ def append_uk_target_nodes( def register_uk_target_kernels(registry: KernelRegistry) -> None: + register_uk_source_codecs() for kernel in ( UKFullTargetCompilationKernel(), UKFullTargetSelectionKernel(), diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_source_codecs.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_source_codecs.py new file mode 100644 index 000000000..3e9ad39e9 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_source_codecs.py @@ -0,0 +1,46 @@ +"""The UK raw-byte source codecs join the shared registry explicitly, never on import. + +The wheels gate runs every package's suite in one process, and the shared codec +suite asserts the shipped raw-byte set is exactly ``raw-bytes-v1``; an import-time +registration from a country adapter broke that invariant. Registration therefore +happens with the UK target kernels and is idempotent. +""" + +from __future__ import annotations + +import subprocess +import sys + +from microcosm.build.uk_runtime.full_targets import ( + CHRONICLE_SOURCE_CODEC, + load_chronicle_source_bytes, +) +from microcosm.build.uk_runtime.graph_targets import ( + register_uk_source_codecs, + register_uk_target_kernels, +) +from microcosm.graph import KernelRegistry +from microcosm.graph.codecs import SOURCE_CODECS, SourceCodecRegistry + + +def test_importing_the_uk_target_module_leaves_the_shared_registry_untouched(): + script = ( + "import microcosm.build.uk_runtime.graph_targets\n" + "from microcosm.graph.codecs import SOURCE_CODECS\n" + "assert SOURCE_CODECS.bytes_names() == ('raw-bytes-v1',), SOURCE_CODECS.bytes_names()\n" + ) + subprocess.run([sys.executable, "-c", script], check=True) + + +def test_registration_is_explicit_and_idempotent(): + registry = SourceCodecRegistry() + registry.register_bytes("raw-bytes-v1", SOURCE_CODECS.get("raw-bytes-v1")) + register_uk_source_codecs(registry) + register_uk_source_codecs(registry) + assert set(registry.bytes_names()) == {"raw-bytes-v1", CHRONICLE_SOURCE_CODEC} + assert registry.get(CHRONICLE_SOURCE_CODEC) is load_chronicle_source_bytes + + +def test_kernel_registration_registers_the_codec_on_the_shared_registry(): + register_uk_target_kernels(KernelRegistry()) + assert SOURCE_CODECS.get(CHRONICLE_SOURCE_CODEC) is load_chronicle_source_bytes diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_codecs.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_codecs.py index 1053a2eb4..a4833a234 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_codecs.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_codecs.py @@ -191,7 +191,10 @@ def test_raw_bytes_codec_reads_one_regular_file_verbatim(tmp_path: Path) -> None payload = b"\x00lookup\xff" * 3 source = tmp_path / "table.npz" source.write_bytes(payload) - assert SOURCE_CODECS.bytes_names() == ("raw-bytes-v1",) + # The shipped registry is process-global and extensible by design: a country + # adapter registers its own raw-byte codecs before its graphs run, so this + # checks the shipped codec is present, not that it is alone. + assert "raw-bytes-v1" in SOURCE_CODECS.bytes_names() assert "raw-bytes-v1" not in SOURCE_CODECS.names() assert SOURCE_CODECS.get("raw-bytes-v1") is load_raw_bytes assert load_source_bytes("raw-bytes-v1", source) == payload From 23006657305ae9cd267f820f71c70976770049b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 26 Sep 2026 13:03:42 +0100 Subject: [PATCH 28/44] Register the chronicle codec inside its own test instead of relying on the import side effect The wheels gate's second run failed on one test: test_chronicle_source_codec_validates_directory_manifest imported graph_targets only to trigger the registration that the previous commit made explicit, so in the wheels job's one-process ordering the codec was not yet installed. The test now calls register_uk_source_codecs itself; no other test or tool relied on the side effect. Verified: the test alone in a fresh process, test_uk_full_targets with the shared codec suite, and every test_uk_full_* file in alphabetical order followed by the shared suite, all green. Co-Authored-By: Claude Fable 5.1 --- .../tests/engine_free/uk/test_uk_full_targets.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py index fe8e979d2..5a1c50a06 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py @@ -200,9 +200,12 @@ def test_current_national_and_local_pins_share_one_reviewed_identity(): def test_chronicle_source_codec_validates_directory_manifest(tmp_path): - from microcosm.build.uk_runtime import graph_targets # noqa: F401 + from microcosm.build.uk_runtime.graph_targets import register_uk_source_codecs from microcosm.graph.codecs import SOURCE_CODECS + # Registration is explicit (never an import side effect), so this test + # registers before reading the process-global registry. + register_uk_source_codecs() payload = b'{"value": 1.0}\n' (tmp_path / "consumer_facts.jsonl").write_bytes(payload) manifest = tmp_path / "manifest.json" From e0cc4486700f26f3ab961a42cefc7e5a3f5f1c3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:15:42 +0100 Subject: [PATCH 29/44] Materialise a blocked assembled spine gate report before the driver fails (review item 1) The assembled battery runs as a graph node and the first post-checkpoint stage refuses on its stored verdict, so run_graph raised before the success-path materialisation and the operator was left with a traceback and a content store. require_uk_spine_gate_admission now raises SpineGateBlockedError (a ValueError, message unchanged) carrying the report it decoded from the gate node's persisted artifact; the executor chains it under NodeRejected, and spine_build.main's failure path walks that chain and applies the battery's write-before-block policy (record_phase, enforce) so spine_gates.json is on disk with blocked_at_phase "assembled" and the transferred phase unreached, and the GateBatteryBlockedError naming the report path becomes the error the receipt and stderr record, exactly as main's in-process tool behaved. Test: test_uk_frs_spine.py gains a driver case that arms the real spine gate declarations on the synthetic roster, has run_graph refuse admission on a stored assembled report through the real admission check, and asserts the sidecar, the receipt, the stderr line, the failed Logbook row and no H5. Docs "Known behaviour changes" and receipts R3 record the change as withdrawn. Verified: the new test + 2 neighbouring driver cases 3 passed; H2 parity 1 passed; gate-battery contract pins + spec-engine bundles 18 passed; coverage manifest --check current; ruff clean. Co-Authored-By: Claude Fable 5.1 --- docs/uk-full-build-graph.md | 2 +- experiments/901-uk-main-rebase-receipts.md | 8 +- .../build/uk_runtime/graph_evidence.py | 68 +++++++++- .../microcosm/build/uk_runtime/spine_build.py | 19 ++- .../tests/engine_free/uk/test_uk_frs_spine.py | 121 ++++++++++++++++++ 5 files changed, 209 insertions(+), 9 deletions(-) diff --git a/docs/uk-full-build-graph.md b/docs/uk-full-build-graph.md index 25117b4b0..33683d90a 100644 --- a/docs/uk-full-build-graph.md +++ b/docs/uk-full-build-graph.md @@ -103,7 +103,7 @@ Structural failures stop export. The maintained local statistical failure policy Recorded for review in `experiments/901-uk-main-rebase-receipts.md` (R3): -- A blocked *assembled* spine gate now fails inside `run_graph`, so the gate report lives in the content store on that path (the previous tool wrote `spine_gates.json` before raising); a blocked *transferred* gate still writes the file. +- A blocked *assembled* spine gate fails inside `run_graph` (the first post-checkpoint stage refuses on the stored verdict), and the driver materialises the stored assembled report into `spine_gates.json` (`blocked_at_phase: "assembled"`, the transferred phase `unreached`) before it fails, so the operator gets the same file the previous tool wrote before raising; a blocked *transferred* gate writes the file on the success path. The report is also in the content store either way. - `numerical_dependencies` pins installed versions into the H2 fixture. - `_normalise_uk_local_bound_families` accepts an empty declaration (country-only scope). - Per-epoch `calibration_progress` staging rows come from the dense solve only. diff --git a/experiments/901-uk-main-rebase-receipts.md b/experiments/901-uk-main-rebase-receipts.md index 348c372c0..e86097efe 100644 --- a/experiments/901-uk-main-rebase-receipts.md +++ b/experiments/901-uk-main-rebase-receipts.md @@ -64,9 +64,11 @@ sampling; `--no-staging`; checkpoints on. Script and outputs: ## R3. Behaviour changes recorded for review -- A blocked *assembled* spine gate now fails inside `run_graph`, so the gate report lives in - the content store on that path (main wrote `spine_gates.json` before raising); a blocked - *transferred* gate still writes the file (R1). +- Withdrawn 2026-09-28 (review item 1): a blocked *assembled* spine gate fails inside + `run_graph`, and the driver now materialises the stored assembled report into + `spine_gates.json` (`blocked_at_phase: "assembled"`, transferred `unreached`) before it + fails, as main's tool did; a blocked *transferred* gate writes the file on the success path + (R1). Pinned by `test_uk_frs_spine.py::test_driver_materializes_a_blocked_assembled_gate_report_before_failing`. - `numerical_dependencies` pins installed versions into the H2 fixture. - `_normalise_uk_local_bound_families` accepts an empty declaration (country-only scope). - Per-epoch `calibration_progress` staging rows come from the dense solve only. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py index f1f8c36c5..8eaf1b63c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py @@ -49,6 +49,30 @@ SPINE_GATE_REPORT_TYPE = ArtifactType("microcosm.gate-phase-report", 1) +class SpineGateBlockedError(ValueError): + """A stored spine gate report refused the stage that depends on it. + + Raised by :func:`require_uk_spine_gate_admission` with the report it + decoded from the gate node's persisted artifact (the executor loaded + those bytes from the store by their verified key). The executor wraps + the refusal in ``NodeRejected`` with this as its cause, so the driver + can find the report on the failure path and materialise it for the + operator before it re-raises: the graph holds the evidence, the + sidecar is how the operator reads it. + """ + + def __init__( + self, + report: gate_battery.GatePhaseReport, + blocking: Sequence[gate_battery.GateOutcome], + ) -> None: + self.report = report + super().__init__( + f"Stored {report.phase} spine gates block downstream execution: " + + ", ".join(outcome.entry.id for outcome in blocking) + ) + + def require_uk_spine_gate_admission( context: KernelContext, *, alias: str = "spine_gate" ) -> None: @@ -71,10 +95,46 @@ def require_uk_spine_gate_admission( synthetic_smoke=bool(context.params["spine_gate_synthetic_smoke"]), ) if blocking: - raise ValueError( - f"Stored {report.phase} spine gates block downstream execution: " - + ", ".join(outcome.entry.id for outcome in blocking) - ) + raise SpineGateBlockedError(report, blocking) + + +def blocked_spine_gate_report( + error: BaseException, +) -> gate_battery.GatePhaseReport | None: + """The stored phase report behind a graph failure, when a gate refused it. + + Walks the cause/context chain the executor builds around a kernel + refusal; ``None`` when the run failed for any other reason. + """ + seen: set[int] = set() + current: BaseException | None = error + while current is not None and id(current) not in seen: + seen.add(id(current)) + if isinstance(current, SpineGateBlockedError): + return current.report + current = current.__cause__ or current.__context__ + return None + + +def materialize_blocked_spine_gate_report( + error: BaseException, *, battery: GateBatteryRun +) -> None: + """Apply the write-before-block policy to a run ``run_graph`` refused. + + The success path restores every stored phase through + :func:`materialize_spine_gate_reports`; a run the assembled gate blocks + never returns a manifest, so the report reaches the operator from the + refusal itself. Recording it through ``battery`` writes the sidecar with + ``blocked_at_phase`` set and the later phase ``unreached``, then raises + :class:`~microcosm.build.gate_battery.GateBatteryBlockedError` naming the + report path, exactly as the in-process battery did. A failure that is not + a gate refusal, or a battery that already blocked, leaves nothing to do. + """ + report = blocked_spine_gate_report(error) + if report is None or battery.blocked_at_phase is not None: + return + battery.record_phase(report) + battery.enforce(report.phase, mode=BlockingMode.BLOCKS_ARTIFACT) def uk_spine_gate_manifest(spec: CountrySpec) -> GatesManifest | None: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py index 0f9325f41..a547da5f6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py @@ -21,7 +21,12 @@ normalize_sampled_household_mass, sample_frame_households, ) -from microcosm.build.gate_battery import BlockingMode, EvidenceContext, GateBatteryRun +from microcosm.build.gate_battery import ( + BlockingMode, + EvidenceContext, + GateBatteryBlockedError, + GateBatteryRun, +) from microcosm.build.logbook import canonical_json_bytes from microcosm.build.logbook_adoption import ( AttemptState, @@ -103,6 +108,7 @@ from microcosm.build.uk_runtime.graph_evidence import ( add_uk_spine_gate_nodes, load_spine_stage_artifacts, + materialize_blocked_spine_gate_report, materialize_spine_gate_reports, register_spine_gate_kernel, spine_sidecar_evidence, @@ -1634,6 +1640,7 @@ def main(argv: list[str] | None = None) -> int: code_pin = "unresolved-local-git-code-pin" spool_dir = args.spine_h5.parent / "logbook-spool" telemetry: StagingTelemetryV2 | None = None + spine_battery: GateBatteryRun | None = None try: _validate_args(args) # A crash between the H5 write and the sidecar writes must never @@ -1933,6 +1940,16 @@ def main(argv: list[str] | None = None) -> int: print(f"Wrote Logbook row: {spool_path}", file=sys.stderr) return 0 except Exception as error: + if spine_battery is not None: + # A run the assembled gate blocked never returned a manifest for + # the success-path materialisation; the refusal carries the + # stored report, and the battery writes ``spine_gates.json`` + # before the block error replaces the graph's own wrapper, so + # the receipt and the operator message both name the report. + try: + materialize_blocked_spine_gate_report(error, battery=spine_battery) + except GateBatteryBlockedError as blocked: + error = blocked if telemetry is not None and telemetry.status == "running": try: telemetry.fail(error) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py index d35ac7539..9d692c910 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py @@ -2008,6 +2008,127 @@ def _fail_graph(*args, **kwargs): assert str(tmp_path) not in serialized +def test_driver_materializes_a_blocked_assembled_gate_report_before_failing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A gate refusal inside ``run_graph`` still leaves ``spine_gates.json``. + + The assembled battery runs as a graph node and the first later stage + refuses on its stored verdict, so ``run_graph`` raises before the + success-path materialisation. The driver hands the operator the same + report file the in-process battery wrote before it raised: blocked at + ``assembled``, the transferred phase unreached, the block error in the + receipt and on stderr, and no H5. + """ + from microcosm.build.gate_battery import ( + GateOutcome, + GatePhaseReport, + GateStatus, + gate_phase_report_payload, + ) + from microcosm.build.gates import GateResult + from microcosm.build.uk_runtime.graph_evidence import ( + require_uk_spine_gate_admission, + ) + from microcosm.graph import NodeRejected + + raw_dir = tmp_path / "raw" + stage = _write_fixture(raw_dir) + output = tmp_path / "blocked.h5" + tool = _load_tool() + spec = _synthetic_spec(stage) + # The synthetic roster carries the real gate declarations, so the driver + # arms the spine battery and the graph gains its gate nodes. + spec.gates = load_country_spec("uk").gates + monkeypatch.setattr(tool, "load_country_spec", lambda country: spec) + monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) + _stub_policy_readers(monkeypatch) + spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) + + gates = tool._spine_gate_manifest_from_spec(spec) + assembled = [entry for entry in gates.gates if entry.phase == "assembled"] + failed = assembled[0] + report = GatePhaseReport( + "assembled", + tuple( + GateOutcome( + entry, + GateStatus.FAILED if entry is failed else GateStatus.PASSED, + GateResult( + name=entry.id, + passed=entry is not failed, + failures=( + ("synthetic assembled failure",) if entry is failed else () + ), + details={}, + ), + ) + for entry in assembled + ), + ) + stored = json.dumps(gate_phase_report_payload(report, gates=gates)).encode() + + def _refuse_admission(*_args, **_kwargs): + # What the first post-checkpoint stage does with the stored verdict, + # wrapped the way the executor wraps a kernel failure. + context = SimpleNamespace( + artifacts={"spine_gate": SimpleNamespace(payload=stored)}, + node=SimpleNamespace(artifact_inputs=()), + params={ + "spine_gate_phase": "assembled", + "spine_gate_release_candidate": False, + "spine_gate_synthetic_smoke": False, + }, + ) + try: + require_uk_spine_gate_admission(context) + except ValueError as refusal: + raise NodeRejected( + f"Node 'frs_age_tail' kernel 'uk.stage@1' failed: {refusal}" + ) from refusal + pytest.fail("the stored assembled verdict did not refuse admission") + + monkeypatch.setattr(tool, "run_graph", _refuse_admission) + + assert ( + tool.main( + [ + "--frs-raw-dir", + str(raw_dir), + "--spine-h5", + str(output), + "--spi-tab", + str(spi_tab), + "--hmrc-ods", + str(hmrc_ods), + "--no-staging", + ] + ) + == 1 + ) + + report_path = output.with_suffix(".spine_gates.json") + err = capsys.readouterr().err + assert f"Gate battery blocked at phase 'assembled' (report: {report_path})" in err + payload = json.loads(report_path.read_text(encoding="utf-8")) + assert payload["blocked_at_phase"] == "assembled" + assert payload["gates"][failed.id]["status"] == "failed" + transferred = [entry.id for entry in gates.gates if entry.phase == "transferred"] + assert transferred + assert {payload["gates"][gate_id]["status"] for gate_id in transferred} == { + "unreached" + } + assert not output.exists() + receipts = list((tmp_path / "logbook-receipts").rglob("error.json")) + assert len(receipts) == 1 + assert json.loads(receipts[0].read_text())["error_type"].endswith( + "GateBatteryBlockedError" + ) + rows = load_spool_rows(tmp_path / "logbook-spool") + assert len(rows) == 1 + assert rows[0].disposition == "failed" + + def test_driver_sampled_named_edge_aborts_with_receipt( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From b3e689abc8a153b62e8e0b48959e6db7f336898b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:20:52 +0100 Subject: [PATCH 30/44] Scope the empty local binding declaration to the country-only selection (review item 2) _normalise_uk_local_bound_families keeps main's refusal of a declaration that names nothing; an explicit allow_empty keyword (threaded through require_adjudicated_uk_local_binding, prepare_uk_full_solve and solve_uk_rowwise_weights_under_doctrine as allow_empty_local_binding, default False) is the only way past it. UKFullProblemInputs records whether the selection admitted no local-surface spec (country_only), and UKFullProblemKernel passes that flag, so the graph's country-only target selection is the one production caller that declares nothing; every other malformed declaration is refused with the keyword set as without it. Tests: test_uk_local_rowwise pins both behaviours (refused by default, admitted with allow_empty, other refusals intact); test_uk_full_solve_scope now says allow_empty_local_binding=True where it models the country-only scope and asserts the default refusal. Docs "Known behaviour changes" and receipts R3 record the change as withdrawn. Verified: test_uk_local_rowwise + test_uk_full_solve_scope 57 passed; engine lane test_uk_full_target_graph::test_explicit_country_filter_runs_same_full_graph_without_local_constraints 1 passed (the kernel path); ruff clean. Co-Authored-By: Claude Fable 5.1 --- docs/uk-full-build-graph.md | 2 +- experiments/901-uk-main-rebase-receipts.md | 7 +++- .../build/uk_runtime/graph_targets.py | 7 ++++ .../build/uk_runtime/local_rowwise.py | 28 ++++++++++++++-- .../uk/test_uk_full_solve_scope.py | 13 +++++++- .../engine_free/uk/test_uk_local_rowwise.py | 32 +++++++++++++++++++ 6 files changed, 84 insertions(+), 5 deletions(-) diff --git a/docs/uk-full-build-graph.md b/docs/uk-full-build-graph.md index 33683d90a..3011dc004 100644 --- a/docs/uk-full-build-graph.md +++ b/docs/uk-full-build-graph.md @@ -105,7 +105,7 @@ Recorded for review in `experiments/901-uk-main-rebase-receipts.md` (R3): - A blocked *assembled* spine gate fails inside `run_graph` (the first post-checkpoint stage refuses on the stored verdict), and the driver materialises the stored assembled report into `spine_gates.json` (`blocked_at_phase: "assembled"`, the transferred phase `unreached`) before it fails, so the operator gets the same file the previous tool wrote before raising; a blocked *transferred* gate writes the file on the success path. The report is also in the content store either way. - `numerical_dependencies` pins installed versions into the H2 fixture. -- `_normalise_uk_local_bound_families` accepts an empty declaration (country-only scope). +- `_normalise_uk_local_bound_families` keeps main's refusal of a declaration that names nothing; only the graph's country-only target selection (`UKFullProblemKernel`, no local-surface spec selected) passes `allow_empty_local_binding=True` through `prepare_uk_full_solve`, because its local surface is empty by construction. - Per-epoch `calibration_progress` staging rows come from the dense solve only. - The HMRC family names `spi_income_band_donors` (microcosm#1006) as a predecessor and admits its two operation kinds; the contract otherwise refused the drifted operation order. - `tools/build_uk_rowwise_dataset.py` stays as it was on main (its tests load it by path; it still serves `--candidate-clone-counts`). diff --git a/experiments/901-uk-main-rebase-receipts.md b/experiments/901-uk-main-rebase-receipts.md index e86097efe..41dbdfacd 100644 --- a/experiments/901-uk-main-rebase-receipts.md +++ b/experiments/901-uk-main-rebase-receipts.md @@ -70,7 +70,12 @@ sampling; `--no-staging`; checkpoints on. Script and outputs: fails, as main's tool did; a blocked *transferred* gate writes the file on the success path (R1). Pinned by `test_uk_frs_spine.py::test_driver_materializes_a_blocked_assembled_gate_report_before_failing`. - `numerical_dependencies` pins installed versions into the H2 fixture. -- `_normalise_uk_local_bound_families` accepts an empty declaration (country-only scope). +- Withdrawn 2026-09-28 (review item 2): `_normalise_uk_local_bound_families` keeps main's + refusal of a declaration that names nothing; only the graph's country-only target selection + (`UKFullProblemKernel`, no local-surface spec selected) passes `allow_empty_local_binding=True` + through `prepare_uk_full_solve`. Both behaviours pinned by + `test_uk_local_rowwise.py::test_rowwise_binding_refuses_an_empty_declaration_unless_allowed` + and `test_uk_full_solve_scope.py::test_zero_local_scope_uses_same_solver_and_has_no_fake_holdout`. - Per-epoch `calibration_progress` staging rows come from the dense solve only. - The HMRC family names `spi_income_band_donors` (#1006) as a predecessor and admits its two operation kinds; the contract otherwise refused the drifted operation order. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py index 69909c473..f58738c5d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py @@ -436,6 +436,11 @@ class UKFullProblemInputs: rung: object national: TargetRegistry selected: TargetRegistry + #: The selection admits no local-surface spec (an explicit country-level + #: filter), so the local binding declaration is empty by construction and + #: the binding check is told so; every other selection keeps the refusal + #: of a declaration that names nothing. + country_only: bool = False def reconstruct_uk_full_problem_inputs(context: KernelContext) -> UKFullProblemInputs: @@ -496,6 +501,7 @@ def reconstruct_uk_full_problem_inputs(context: KernelContext) -> UKFullProblemI rung, national, selected, + country_only=not local_specs, ) @@ -516,6 +522,7 @@ def run(self, context: KernelContext) -> KernelResult: bound_families=bound_families, national_rows=national_rows, target_weight_rule=str(context.params["target_weight_rule"]), + allow_empty_local_binding=inputs.country_only, ) problem = build_constraint_matrix(frame, prepared.target_set, "household") if problem.skipped: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py index e189eb8cd..e559ac9d3 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py @@ -755,8 +755,15 @@ def require_adjudicated_uk_local_binding( census: Mapping[str, Any] | None = None, register: Mapping[str, Any] | None = None, now: Any = None, + allow_empty: bool = False, ) -> dict[str, Any]: - """Require in-force review records before binding fenced UK local families.""" + """Require in-force review records before binding fenced UK local families. + + A declaration that names nothing is refused unless ``allow_empty`` is + passed: only the graph's country-only target selection, whose local + surface is empty by construction, has nothing to declare; every other + caller must name exactly what its matrix binds. + """ census_payload = ( local_target_census.load_uk_local_target_census() if census is None else census @@ -765,6 +772,7 @@ def require_adjudicated_uk_local_binding( declared, parsed = _normalise_uk_local_bound_families( bound_families, family_rows=family_rows, + allow_empty=allow_empty, ) derived = _derive_uk_local_bound_families_from_target_frame( target_frame, @@ -880,6 +888,7 @@ def _normalise_uk_local_bound_families( bound_families: Sequence[str], *, family_rows: Mapping[str, Mapping[str, Any]], + allow_empty: bool = False, ) -> tuple[tuple[str, ...], dict[str, tuple[str, str]]]: if isinstance(bound_families, str): raise ValueError( @@ -887,6 +896,11 @@ def _normalise_uk_local_bound_families( "sequence of family/area_type strings, not one string." ) declared = tuple(str(name) for name in bound_families) + if not declared and not allow_empty: + raise ValueError( + "UK local binding declarations: bound_families must name at " + "least one family/area_type pair." + ) blanks = [name for name in declared if not name.strip()] if blanks: raise ValueError( @@ -1118,8 +1132,15 @@ def prepare_uk_full_solve( bound_families: Sequence[str], national_rows: UKRowwiseNationalRows | None = None, target_weight_rule: str = "uniform", + allow_empty_local_binding: bool = False, ) -> UKPreparedFullSolve: - """Validate the selected surface with one doctrine for every geography.""" + """Validate the selected surface with one doctrine for every geography. + + ``allow_empty_local_binding`` admits a local declaration that names no + family: the graph's country-only target selection passes it, because its + local surface is empty by construction; any other caller keeps the + refusal of a declaration that names nothing. + """ _require_uniform_target_surface(problem) if not len(problem.targets) and ( @@ -1132,6 +1153,7 @@ def prepare_uk_full_solve( binding_adjudications = require_adjudicated_uk_local_binding( local_bound_families, problem.target_frame, + allow_empty=allow_empty_local_binding, ) national_families = ( () @@ -1295,6 +1317,7 @@ def solve_uk_rowwise_weights_under_doctrine( checkpoint_provenance: Mapping[str, Any] | None = None, progress: Callable[[str], None] | None = None, progress_events: Callable[[dict[str, object]], None] | None = None, + allow_empty_local_binding: bool = False, ) -> UKRowwiseDoctrineSolve: """Solve rowwise household weights under the reviewed doctrine. @@ -1344,6 +1367,7 @@ def solve_uk_rowwise_weights_under_doctrine( bound_families=bound_families, national_rows=national_rows, target_weight_rule=target_weight_rule, + allow_empty_local_binding=allow_empty_local_binding, ) doctrine = UK_LOCAL_SOLVE_DOCTRINE target_set = prepared.target_set diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_solve_scope.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_solve_scope.py index 98c454d8a..79cc284cf 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_solve_scope.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_solve_scope.py @@ -52,8 +52,18 @@ def test_zero_local_scope_uses_same_solver_and_has_no_fake_holdout(): frame = _clone_frame() local = empty_uk_local_problem(frame.table("household")["household_id"]) rows = national_rows() + # The country-only selection is the one caller that declares no local + # family, and it says so explicitly; nothing else may declare nothing. + with pytest.raises(ValueError, match="at least one family/area_type pair"): + prepare_uk_full_solve( + frame, local, bound_families=("national/fixture",), national_rows=rows + ) prepared = prepare_uk_full_solve( - frame, local, bound_families=("national/fixture",), national_rows=rows + frame, + local, + bound_families=("national/fixture",), + national_rows=rows, + allow_empty_local_binding=True, ) dense = solve_uk_dense_reference(prepared, epochs=8, seed=17) finished = finish_uk_full_solve(prepared, dense) @@ -64,6 +74,7 @@ def test_zero_local_scope_uses_same_solver_and_has_no_fake_holdout(): national_rows=rows, epochs=8, seed=17, + allow_empty_local_binding=True, ) np.testing.assert_array_equal(finished.weights, existing.weights) assert finished.diagnostics.empty diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_local_rowwise.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_local_rowwise.py index 8ac31204b..a952f9698 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_local_rowwise.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_local_rowwise.py @@ -699,6 +699,38 @@ def test_rowwise_binding_refuses_unknown_family_and_bad_area_type() -> None: ) +def test_rowwise_binding_refuses_an_empty_declaration_unless_allowed() -> None: + """A declaration that names nothing is refused for every caller by default. + + Only the graph's country-only target selection, whose local surface is + empty by construction, passes ``allow_empty``; the keyword excuses the + empty declaration alone and every other malformed declaration is still + refused (review item 2 on microcosm#901). + """ + from microcosm.build.uk_runtime.local_rowwise import empty_uk_local_problem + + problem = empty_uk_local_problem([1, 2, 3]) + with pytest.raises(ValueError, match="at least one family/area_type pair"): + require_adjudicated_uk_local_binding([], problem.target_frame) + with pytest.raises(ValueError, match="at least one family/area_type pair"): + require_adjudicated_uk_local_binding((), problem.target_frame, register={}) + + receipt = require_adjudicated_uk_local_binding( + [], problem.target_frame, allow_empty=True + ) + assert receipt["bound_families"] == [] + assert receipt["stood_on"] == {} + assert receipt["register_resource"] == "local_binding_adjudications.json" + with pytest.raises(ValueError, match="unknown census family"): + require_adjudicated_uk_local_binding( + ["not_a_family/constituency"], problem.target_frame, allow_empty=True + ) + with pytest.raises(ValueError, match="extra.*census_households/constituency"): + require_adjudicated_uk_local_binding( + ["census_households/constituency"], problem.target_frame, allow_empty=True + ) + + def test_rowwise_binding_refuses_expired_and_premature_adjudications() -> None: problem = build_uk_rowwise_local_matrix(_metrics(), _assigned(), _targets()) expired_register = { From d78ebef5f546050093e90a9013f2d0cdf1bb04ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:21:14 +0100 Subject: [PATCH 31/44] Reword the codec-registration docstring to what the shared suite asserts (review item 4) register_uk_source_codecs' docstring still said the shared codec suite asserts the shipped set; a46712b39 relaxed that suite to check the shipped raw-bytes codec's presence and behaviour while tolerating a country codec beside it. The docstring now states that, and that the registry itself refuses a name claimed by a different loader. Co-Authored-By: Claude Fable 5.1 --- .../src/microcosm/build/uk_runtime/graph_targets.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py index f58738c5d..240b6b66a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py @@ -71,10 +71,13 @@ def register_uk_source_codecs(registry=SOURCE_CODECS) -> None: """Register the UK raw-byte source codecs the target nodes bind. Registration is explicit and idempotent: importing this module leaves the - shared registry untouched (the shared codec suite asserts the shipped set), - and the driver's kernel registration is the one place the chronicle - consumer-facts codec joins ``SOURCE_CODECS`` before a graph runs. Re-registering - the same loader is a no-op by the registry's own rule. + shared registry untouched, and the driver's kernel registration is the one + place the chronicle consumer-facts codec joins ``SOURCE_CODECS`` before a + graph runs. The shared codec suite checks that the shipped raw-bytes codec + is present and behaves, and tolerates a country codec registered beside it + (it no longer asserts the shipped set is the whole registry); the registry + itself refuses a name claimed by a different loader, and re-registering the + same loader is a no-op by its own rule. """ registry.register_bytes(CHRONICLE_SOURCE_CODEC, load_chronicle_source_bytes) From 6e01a5ca4981b3c130d60e83c924b23ae3656dfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:38:02 +0100 Subject: [PATCH 32/44] Restore the retired candidate-tool contracts on the graph driver (review: the retired-tests hole) Of the 25 in-process tool tests retired in af01b990d, the review counted 10 lost, five of them the "a delivery-side problem must not fail a good build" contract. Every row is now either restored against full_build_cli.main over the synthetic dense build or shown covered by a named test; the 25-row mapping is receipts section R5. Restored in test_uk_full_build_cli.py (11 new tests, one extended): telemetry content refusal never aborts the solve (driving the driver's own _solve_observer), an invalid local bundle is a warning, --no-staging records both opt-outs, remote staging uploads telemetry and the bundle in one commit (with the re-stage and fetch tools), a remote staging failure still succeeds, --no-staged-dataset keeps telemetry remote and the bundle local, a refusal leaves the gate and error receipt pointers on the failed Logbook row (blocked gate, post-binding refusal, preparation failure), two release-blocking local failures partition as blocking with no diagnostics, a multi-block engine run is never releasable, --engine-blocks must equal --n-clones, an input under the output directory is refused before anything is written, and the local-only staging test gains the bundle-inventory assertions. test_support/microcosm_build/uk_full_build_cli.py: run_dense_main returns the status and bundle (graph_dense_bundle wraps it), gate_payload fails several gates with distinct lines, arguments(staging=None) requests the remote mode, plus local_ref/spool_rows/single_run_id/load_tool. arguments() now parses through the real parser bound at import: it went through cli.parse_args, which an earlier run in the same test had patched, so a second run reused the first run's output directory. Two residual gaps for a ruling (R5 rows 18, 19): the graph driver sets no telemetry sample block, and size search/refit epochs are not staged (R3). Verified: test_uk_full_build_cli + the helper's two consumer modules 77 passed; ci_test_groups --verify ok; ruff clean. Co-Authored-By: Claude Fable 5.1 --- experiments/901-uk-main-rebase-receipts.md | 128 ++++ .../engine_free/uk/test_uk_full_build_cli.py | 570 +++++++++++++++++- .../microcosm_build/uk_full_build_cli.py | 109 +++- 3 files changed, 791 insertions(+), 16 deletions(-) diff --git a/experiments/901-uk-main-rebase-receipts.md b/experiments/901-uk-main-rebase-receipts.md index 41dbdfacd..8b3f75460 100644 --- a/experiments/901-uk-main-rebase-receipts.md +++ b/experiments/901-uk-main-rebase-receipts.md @@ -90,3 +90,131 @@ cannot be built at main's head (R1). Options put to María on 2026-09-25: wait f lane to settle main's fence; a non-release measurement flag that accepts a stale gate manifest and records the mismatch; or a measurement-only branch over #1012. The national role is parity by identity regardless (same engine, moved code); the dense comparison needs a spine. + +## R5. The 25 retired in-process tool tests, mapped onto the graph driver (2026-09-28) + +Vahid's review counted 7 replaced, 8 partially covered and 10 lost. Re-derived here for all 25 +against the pre-retirement bodies (`af01b990d~1:packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py`), +each retired test is either restored on `full_build_cli.main` over the synthetic dense build +(`test_support/microcosm_build/uk_full_build_cli.py`: `run_dense_main`, multi-gate +`gate_payload`, `_FakeHub` from the rowwise helper, patches on `rowwise_staging`) or shown +covered by a named test. Unqualified `::` names below are in +`packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py`. Totals: 14 rows +restored (rows 3, 8, 9, 10, 12, 14, 15, 18, 20 to 25, through 11 new test functions and one +extended test; rows 9 and 10 live inside row 8's test), 10 rows covered by named tests (1, 2, 4, +5, 6, 7, 11, 13, 16, 17), 1 row superseded by a documented behaviour change (19, R3). Residual +gaps are named on their rows; rows 18 and 19 carry the two for a ruling. + +1. `candidate_build_writes_calibrated_h5_and_evidence`: covered. Manifest projection (schema 4, + dense role and release id, doctrine block, identity pins, solve, weights, output digests, + releasable) by `::test_dense_run_projects_the_rowwise_candidate_manifest`; the H5 and sidecars + written and replayed by `::test_cli_cold_and_required_replay_recreate_dataset_and_sidecars`; + the Logbook row (pipeline, rung, seed, iterating, artifact location, every verdict passed + with a `.local_gates.json#/gates/` receipt) by + `::test_main_runs_the_logbook_envelope_around_a_dense_build`; the real-solve calibrated + weights, mass record and solver parity by + `engine/uk/test_uk_full_target_graph.py::test_default_all_has_direct_matrix_and_solver_parity_and_replays`. + The seeded adjudication row values are pinned by + `test_uk_local_rowwise.py::test_committed_local_binding_register_references_committed_census` + rather than through the driver. +2. `candidate_dry_run_plans_without_solve_or_write`: covered by + `::test_dry_run_has_no_files_or_kernel_execution` (no files, no kernel execution, no Logbook + row) and `test_uk_rowwise_candidate.py::test_graph_driver_dry_run_prints_the_operation_inventory`; + the plan is the operation inventory by design, and the old plan's sampling, binding and + cross-grain blocks are graph artifacts asserted through row 1. +3. `candidate_sampling_rung_receipt_and_engine_block_validation`: the `--engine-blocks` must + equal `--n-clones` refusal restored as + `::test_engine_blocks_must_be_positive_and_equal_the_clone_count`; the sampled-rung receipt + covered by `test_uk_national_sampling.py::test_spine_sampling_is_stratified_keeps_families_and_normalizes` + and `::test_source_sampling_cannot_be_reapplied_as_pool_sampling` (the graph samples the + source spine, never the pool). The dry-run plan no longer carries a sampling block (row 2). +4. `candidate_f100_does_not_call_any_sampler`: covered by + `test_uk_national_sampling.py::test_full_fraction_is_a_structural_no_op` and + `::test_source_sampling_cannot_be_reapplied_as_pool_sampling`; the compact national sampler + is not reachable from the graph's population node. +5. `candidate_engine_surface_reuses_one_resolver`: covered by + `test_uk_full_measure.py::test_full_measure_reuses_one_resolver` (direct port). +6. `candidate_engine_surface_resolves_real_per_clone_blocks`: covered by + `test_uk_full_measure.py::test_full_measure_resolves_real_per_clone_blocks` (direct port, same + parametrisation). +7. `joint_candidate_f100_and_f001_end_to_end`: covered. The joint local/ladder/national matrix, + solver parity and replay by + `engine/uk/test_uk_full_target_graph.py::test_default_all_has_direct_matrix_and_solver_parity_and_replays`; + the unbound-bridge and fan-out receipts by `test_uk_ledger_targets.py`; the rowwise dataset + round trip and `clone_index` rename by `test_uk_rowwise_dataset.py::test_clone_uk_dataset_h5_roundtrip` + and `test_uk_ladder_rowwise_clone.py::test_inherited_clone_index_is_replaced_like_the_pre_frame_writer`. + Residual: the f001 leg's `rung_surface` counts are asserted through the driver only as a + present manifest key (row 1); a sampled-rung run through the driver is not repeated. +8. `candidate_refusal_records_receipt_and_reraises`: restored as + `::test_refusal_records_the_gate_and_error_receipt_pointers` (a blocked geography gate leaves + `{verdict: failed, receipt: #/gates/}` on the failed row; a raise + leaves `pipeline_error` with `#/error_type`). The graph driver returns the block as status 1 + instead of re-raising, by design. +9. `candidate_binding_adjudication_failure_records_failed_row`: restored inside the same test + (a refusal after `targets_bound` and before `solved` records the failed row and pointer); the + binding refusal itself by `test_uk_local_rowwise.py::test_rowwise_binding_refuses_unadjudicated_committed_fence`. +10. `candidate_setup_failure_records_failed_row`: restored inside the same test (a preparation + failure spools the failed row with the pointer and no manifest); `inputs_pinned` is not a + graph-driver phase, the pins ride on the prepared build's `inputs` record (row 1). +11. `households_only_targets_come_from_compiled_chronicle_registry`: covered by + `::test_households_only_binds_the_census_family_on_the_selection_node`, the uprating receipt + cases in `test_uk_ledger_targets.py`, and the compiled-registry problem in + `engine/uk/test_uk_full_target_graph.py` (both selection cases). +12. `candidate_dry_run_refuses_ladder_sidecar_collision`: restored as + `::test_input_inside_the_output_directory_is_refused_before_anything_is_written` (an input + under the output directory is refused by `_output_locations` before anything is written; the + colliding file keeps its bytes), beside + `::test_rejected_output_inside_source_never_writes_failure_sidecar` for the reverse direction. +13. `candidate_weight_ratio_failure_is_reported_and_blocks`: covered by + `::test_blocked_gate_projects_an_unreleasable_manifest` and the restored row 14 (the + weight-ratio failure line, the report's criticality and status, the failed row). +14. `candidate_block_partitions_failures_by_criticality`: restored as + `::test_blocked_gates_partition_failures_by_criticality`. +15. `candidate_multi_block_engine_run_is_never_releasable`: restored as + `::test_multi_block_engine_run_is_never_releasable` (end to end on the driver), beside + `test_uk_rowwise_candidate.py::test_release_verdict_requires_single_block_engine`. +16. `size_candidate_exports_compact_links_and_cannot_claim_dense_release`: covered by + `test_uk_full_calibration_graph.py::test_graph_preserves_numerical_path_and_complete_resume` + (size search and refit nodes, the size artifact, complete resume) and + `::test_changing_k_reuses_dense_but_source_bytes_invalidate_it`; + `test_uk_rowwise_candidate.py::test_dense_candidate_manifest_has_no_size_sidecars` and + `::test_size_cli_refuses_promotion_without_separate_certification`; the size outputs are + read by `test_uk_size_evaluation.py::test_weight_tables_size_and_dense_spine_paths`. + Residual: the K=300 export's link integrity and the two-seed manifest projection are not + repeated through the driver. +17. `size_candidate_checkpoints_before_the_draw_and_resumes_from_it`: covered by + `test_uk_full_calibration_graph.py::test_external_search_checkpoint_is_imported_without_repeating_solves` + and `::test_reused_draw_skips_rng_and_rejects_changed_binding`, and the size-checkpoint cases + in `test_uk_local_rowwise.py`. Residual: the `size_selection_checkpointed` and + `size_selection_resumed` Logbook phases and the stderr progress lines are not pinned through + the driver. +18. `candidate_build_stages_telemetry_locally_and_inventories_the_bundle`: restored by extending + `::test_main_stages_the_bundle_locally_with_staging_local_only` with the retired inventory + assertions (stdout manifest equals the on-disk one, run id equals build id, operation and + pipeline ids, artifacts, fit summary, staged files with digests, sha256sums, sidecar + inventory, sidecars never outputs, stage sequence). Residual for a ruling: the run manifest's + `sample` block is unset on the graph driver where the tool wrote `{"mode": "full"}`; per-epoch + rows are the dense solve's only (R3). +19. `size_candidate_stages_the_search_and_refit_phases`: superseded by design (R3: per-epoch + staging rows come from the dense solve only; the search and refit forward no epochs), pinned + by `::test_dense_run_projects_the_rowwise_candidate_manifest` (`graph.epoch_rows == + "dense_solve_only"`); the size-run manifest claims are rows 16 and 17. Not restorable as + written; flagged for a ruling with row 18. +20. `telemetry_content_refusal_never_aborts_the_solve`: restored as + `::test_telemetry_content_refusal_never_aborts_the_solve` (drives the driver's own + `_solve_observer` with synthetic epochs). +21. `invalid_local_telemetry_bundle_is_a_warning_not_the_runs_failure`: restored as + `::test_invalid_local_telemetry_bundle_is_a_warning_not_the_runs_failure`. +22. `no_staging_records_both_opt_outs`: restored as `::test_no_staging_records_both_opt_outs`. +23. `remote_staging_uploads_telemetry_and_the_bundle_in_one_commit`: restored as + `::test_remote_staging_uploads_telemetry_and_the_bundle_in_one_commit`, including the + re-stage (`tools/stage_uk_rowwise_candidate.py`) and fetch (`tools/fetch_uk_staged_dataset.py`) + legs. +24. `remote_staging_failure_is_recorded_and_the_build_still_succeeds`: restored as + `::test_remote_staging_failure_is_recorded_and_the_build_still_succeeds`. +25. `no_staged_dataset_keeps_telemetry_remote_and_the_bundle_local`: restored as + `::test_no_staged_dataset_keeps_telemetry_remote_and_the_bundle_local`. + +One helper defect surfaced while restoring row 8: `arguments()` parsed through `cli.parse_args`, +which an earlier `run_dense_main` in the same test had already patched, so a second run reused +the first run's output directory; the helper now parses through the real parser bound at import. diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py index f1e1efb29..286cd4b45 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py @@ -60,6 +60,19 @@ def test_dense_role_refuses_the_national_knobs(tmp_path, extra, needle): assert needle in str(excinfo.value) +def test_engine_blocks_must_be_positive_and_equal_the_clone_count(tmp_path): + """The per-clone engine block count is either one or the clone count.""" + with pytest.raises(ValueError, match="must equal --n-clones"): + cli.validate_cli_args( + arguments(tmp_path, "--n-clones", "4", "--engine-blocks", "2") + ) + with pytest.raises(ValueError, match="must be positive"): + cli.validate_cli_args(arguments(tmp_path, "--engine-blocks", "0")) + cli.validate_cli_args( + arguments(tmp_path, "--n-clones", "2", "--engine-blocks", "2") + ) + + def test_dense_role_requires_the_ladder_and_the_pins(tmp_path): argv = [ "--release-role", @@ -416,6 +429,28 @@ def test_rejected_output_inside_source_never_writes_failure_sidecar( assert not args.out.exists() +def test_input_inside_the_output_directory_is_refused_before_anything_is_written( + tmp_path, monkeypatch +): + """An input that would collide with a published file keeps its bytes. + + The candidate tool refused a ladder named as the manifest inside the + output directory; the graph driver refuses any input source under the + output directory before it writes, and the colliding file is untouched. + """ + args = arguments(tmp_path) + args.out.mkdir() + collision = args.out / cli.MANIFEST_FILENAME + collision.write_bytes(b"ladder stand-in living where the manifest goes") + build = prepared(tmp_path) + build = replace(build, sources={"fixture": collision}) + monkeypatch.setattr(cli, "parse_args", lambda argv: args) + monkeypatch.setattr(cli, "prepare_full_build", lambda args, **kwargs: build) + assert cli.main([]) == 1 + assert collision.read_bytes() == b"ladder stand-in living where the manifest goes" + assert list(args.out.iterdir()) == [collision] + + def test_main_runs_the_logbook_envelope_around_a_dense_build(tmp_path, monkeypatch): pytest.importorskip("tables") from microcosm.build.logbook import load_spool_rows @@ -525,8 +560,16 @@ def refuse_transferred(compiled, **kwargs): ) -def test_main_stages_the_bundle_locally_with_staging_local_only(tmp_path, monkeypatch): - """``--staging-local-only`` writes and validates the v2 bundle, uploads nothing.""" +def test_main_stages_the_bundle_locally_with_staging_local_only( + tmp_path, monkeypatch, capsys +): + """``--staging-local-only`` writes and validates the v2 bundle, uploads nothing. + + The second half restores the candidate tool's bundle-inventory contract on + the graph driver: the stdout manifest is the on-disk one, the run id is + the build id, the staged inventory names every output with its digest, the + local sums verify the directory, and the sidecars are never outputs. + """ pytest.importorskip("tables") from microcosm.build.staging_dataset import ( @@ -568,3 +611,526 @@ def test_main_stages_the_bundle_locally_with_staging_local_only(tmp_path, monkey staged = json.loads((artifacts / "staged_dataset.json").read_text()) assert staged["mode"] == "local_only" and staged["status"] == "skipped" assert (artifacts / "fit_summary.json").is_file() + + captured = capsys.readouterr() + # The stdout manifest is the on-disk manifest, evidence blocks included. + assert json.loads(captured.out)["staged_dataset"] == manifest["staged_dataset"] + assert "staged dataset: skipped (local_only)" in captured.err + run_id = runs[0] + rows = spool_rows(out) + assert rows[0].build_id == run_id + run_manifest = bundle["run_manifest"] + assert run_manifest["operation_id"] == "uk_rowwise_candidate" + assert run_manifest["pipeline"]["id"] == "uk-local-candidate" + assert run_manifest["non_release"] is True + # The candidate tool wrote ``sample == {"mode": "full"}``; the graph driver + # sets no sample block (receipts R5, residual gap for a ruling). + assert run_manifest["delivery"]["mode"] == "local_only" + assert run_manifest["delivery"]["upload_attempts"] == 0 + assert {a["logical_name"] for a in run_manifest["artifacts"]} == { + "fit_summary", + "staged_dataset", + } + fit_summary = json.loads((artifacts / "fit_summary.json").read_text()) + assert fit_summary["run_id"] == run_id + assert fit_summary["gates"]["uk_local_target_fit"] == "passed" + assert fit_summary["loss"]["final"] == manifest["solve"]["final_loss"] + assert staged == manifest["staged_dataset"] + # Every graph phase reports started then completed, in build order. The + # synthetic preflight gate binds no selection node, so target compilation + # only starts here; on the real graph it completes with the selected count. + events = bundle["events"] + completed = [e["stage_id"] for e in events if e["status"] == "completed"] + assert completed == [ + "calibration", + "gate_battery", + "output_bundle", + "dataset_staging", + "complete", + ] + assert [e["status"] for e in events if e["stage_id"] == "target_compilation"] == [ + "started" + ] + for stage in ("calibration", "output_bundle"): + transitions = [e["status"] for e in events if e["stage_id"] == stage] + assert transitions == ["started", "completed"], stage + # The manifest carries both receipts; the bundle inventory is the outputs. + assert manifest["staging_delivery"]["run_id"] == run_id + assert staged["prefix"] == f"staged/{run_id}" + assert set(staged["files"]) == { + Path(entry["path"]).name for entry in manifest["outputs"].values() + } + for entry in manifest["outputs"].values(): + assert staged["files"][Path(entry["path"]).name]["sha256"] == entry["sha256"] + # The local sums verify the directory as it is, evidence blocks included. + for line in (out / SHA256SUMS_FILENAME).read_text().splitlines(): + digest, name = line.split(" ") + assert hashlib.sha256((out / name).read_bytes()).hexdigest() == digest, name + inventory = json.loads((out / STAGED_MANIFEST_FILENAME).read_text()) + assert inventory["run_id"] == run_id and inventory["files"] == staged["files"] + assert inventory["summary"]["releasable"] is True + assert inventory["telemetry"] == { + "repository": None, + "prefix": f"runs/{run_id}", + "mode": "local_only", + } + # The sidecars are evidence about the outputs, never outputs themselves. + assert "sha256sums" not in manifest["outputs"] + assert "staged_manifest" not in manifest["outputs"] + + +# --------------------------------------------------------------------------- +# Delivery-side resilience and refusal receipts on the graph driver. These +# restore the in-process candidate tool's contracts (retired in af01b990d; +# mapping in experiments/901-uk-main-rebase-receipts.md, R5) against +# ``full_build_cli.main`` over the synthetic dense build: a delivery-side +# problem must never fail a good build, and a refusal must leave its receipt +# pointers on the Logbook row. +# --------------------------------------------------------------------------- + + +def _drive_epochs(args, telemetry, epochs: int = 2) -> None: + """Feed synthetic dense-solve epochs through the driver's own observer. + + The synthetic graph has no calibration kernel, so the thinned staging rows + the real solve would forward are produced here through the very callback + ``prepare_full_build`` hands the kernels. + """ + observer = cli._solve_observer(args, telemetry) + for epoch in range(1, epochs + 1): + observer( + { + "kind": "calibration_epoch", + "epoch": epoch, + "epochs": epochs, + "loss": 0.5 / epoch, + } + ) + + +def _remote_hub(monkeypatch, **kwargs): + from microcosm.build.uk_runtime import rowwise_staging + from test_support.microcosm_build.uk_rowwise_candidate import _FakeHub + + hub = _FakeHub(**kwargs) + monkeypatch.setattr(rowwise_staging, "_hub_api", lambda: hub) + monkeypatch.setattr(rowwise_staging, "_hub_token", lambda: "hf_test_token") + return hub + + +def test_no_staging_records_both_opt_outs(tmp_path, monkeypatch): + pytest.importorskip("tables") + status, out = run_dense_main(tmp_path, monkeypatch, staging="--no-staging") + assert status == 0 + assert not (out / "staging").exists() + assert not (out / "sha256sums.txt").exists() + manifest = json.loads((out / cli.MANIFEST_FILENAME).read_text()) + assert manifest["staging_delivery"]["mode"] == "disabled" + assert manifest["staging_delivery"]["opt_out_reason"] == "--no-staging" + assert manifest["staged_dataset"] == { + "contract_version": 1, + "mode": "disabled", + "repository": None, + "prefix": None, + "run_id": None, + "revision": None, + "status": "skipped", + "error_code": None, + "opt_out_reason": "--no-staging", + "files": {}, + } + rows = spool_rows(out) + assert "dataset_stage_skipped" in rows[0].phases_reached + + +def test_invalid_local_telemetry_bundle_is_a_warning_not_the_runs_failure( + tmp_path, monkeypatch, capsys +): + pytest.importorskip("tables") + from microcosm.build.staging_v2 import StagingContractError + from microcosm.build.uk_runtime import rowwise_staging + + class Invalid(rowwise_staging.StagingTelemetryV2): + def validate_local_bundle(self): + raise StagingContractError("synthetic bundle defect") + + monkeypatch.setattr(rowwise_staging, "StagingTelemetryV2", Invalid) + status, out = run_dense_main(tmp_path, monkeypatch, staging="--staging-local-only") + assert status == 0 + err = capsys.readouterr().err + assert "does not validate" in err and "synthetic bundle defect" in err + manifest = json.loads((out / cli.MANIFEST_FILENAME).read_text()) + assert manifest["staging_delivery"]["mode"] == "local_only" + assert manifest["staged_dataset"]["status"] == "skipped" + rows = spool_rows(out) + assert rows and rows[0].disposition == "iterating" + + +def test_telemetry_content_refusal_never_aborts_the_solve( + tmp_path, monkeypatch, capsys +): + pytest.importorskip("tables") + from microcosm.build.staging_v2 import StagingContentError, validate_v2_bundle + from microcosm.build.uk_runtime import rowwise_staging + + class Refusing(rowwise_staging.StagingTelemetryV2): + def calibration_progress(self, event): + raise StagingContentError("Staging file exceeds the 5242880-byte limit.") + + monkeypatch.setattr(rowwise_staging, "StagingTelemetryV2", Refusing) + status, out = run_dense_main( + tmp_path, monkeypatch, staging="--staging-local-only", on_prepare=_drive_epochs + ) + assert status == 0 + err = capsys.readouterr().err + assert err.count("no longer forwarded") == 1 + run_id = single_run_id(out) + bundle = validate_v2_bundle(out / "staging", run_id) + assert bundle["run_manifest"]["status"] == "completed" + assert not ( + out / "staging" / "runs" / run_id / "calibration_progress.json" + ).exists() + manifest = json.loads((out / cli.MANIFEST_FILENAME).read_text()) + assert manifest["staging_delivery"]["mode"] == "local_only" + assert spool_rows(out)[0].disposition == "iterating" + + +def test_remote_staging_uploads_telemetry_and_the_bundle_in_one_commit( + tmp_path, monkeypatch, capsys +): + pytest.importorskip("tables") + hub = _remote_hub(monkeypatch) + status, out = run_dense_main( + tmp_path, + monkeypatch, + "--staging-upload-interval-seconds", + "0", + staging=None, + on_prepare=_drive_epochs, + ) + assert status == 0 + err = capsys.readouterr().err + run_id = single_run_id(out) + manifest = json.loads((out / cli.MANIFEST_FILENAME).read_text()) + + # Telemetry went to runs// of the staging repository, artifacts + # included, under the fixed prefix and nothing else. + telemetry_paths = hub.paths("policyengine/populace-uk-staging") + assert telemetry_paths == sorted( + f"runs/{run_id}/{name}" + for name in ( + "run_manifest.json", + "progress.json", + "events.ndjson", + "calibration_progress.json", + "artifacts/fit_summary.json", + "artifacts/staged_dataset.json", + ) + ) + delivery = manifest["staging_delivery"] + assert delivery["mode"] == "local_and_remote" + assert delivery["configured_repository"] == "policyengine/populace-uk-staging" + assert delivery["upload_successes"] == delivery["upload_attempts"] > 0 + remote_progress = json.loads( + hub.files[("policyengine/populace-uk-staging", f"runs/{run_id}/progress.json")] + ) + assert remote_progress["status"] == "completed" + + # The bundle went to staged// of the private repository in one + # commit: every output, the manifest as built, and the two sidecars. + assert len(hub.commits) == 1 + commit = hub.commits[0] + assert commit["repo_id"] == "policyengine/populace-uk-private" + expected = {Path(e["path"]).name for e in manifest["outputs"].values()} | { + cli.MANIFEST_FILENAME, + "staged_manifest.json", + "sha256sums.txt", + } + assert commit["paths"] == sorted(f"staged/{run_id}/{name}" for name in expected) + assert hub.paths("policyengine/populace-uk-private") == commit["paths"] + staged = manifest["staged_dataset"] + assert staged["status"] == "uploaded" + assert staged["repository"] == "policyengine/populace-uk-private" + assert staged["prefix"] == f"staged/{run_id}" + assert staged["revision"] == hub.sha + assert ( + f"staged dataset: uploaded at policyengine/populace-uk-private/staged/{run_id}" + in err + ) + dataset_name = Path(manifest["outputs"]["dataset"]["path"]).name + remote_h5 = hub.files[ + ("policyengine/populace-uk-private", f"staged/{run_id}/{dataset_name}") + ] + assert remote_h5 == (out / dataset_name).read_bytes() + # The uploaded manifest is the one the bundle was built from; the local + # copy gained the two evidence blocks afterwards. + remote_manifest = json.loads( + hub.files[ + ( + "policyengine/populace-uk-private", + f"staged/{run_id}/{cli.MANIFEST_FILENAME}", + ) + ] + ) + assert ( + "staged_dataset" not in remote_manifest + and "staging_delivery" not in remote_manifest + ) + assert remote_manifest["outputs"] == manifest["outputs"] + rows = spool_rows(out) + assert "dataset_staged" in rows[0].phases_reached + assert rows[0].disposition == "iterating" + + def sums_verify() -> None: + for line in (out / "sha256sums.txt").read_text().splitlines(): + digest, name = line.split(" ") + assert hashlib.sha256((out / name).read_bytes()).hexdigest() == digest, name + + # Re-staging a directory whose record already says these outputs are + # uploaded touches nothing: the driver's record and revision stand, the + # sidecars keep their bytes, and no commit is made. + stager = load_tool("stage_uk_rowwise_candidate") + monkeypatch.setattr(stager, "_hub_api", lambda: hub) + sidecar_bytes = (out / "staged_manifest.json").read_bytes() + capsys.readouterr() + assert stager.main(["--run-dir", str(out)]) == 0 + assert "nothing to do" in capsys.readouterr().err + restaged = json.loads((out / cli.MANIFEST_FILENAME).read_text())["staged_dataset"] + assert restaged == staged + assert (out / "staged_manifest.json").read_bytes() == sidecar_bytes + sums_verify() + assert len(hub.commits) == 1 + + # A record that says the upload failed while the Hub already holds these + # outputs: the re-stage finds the bundle and records its own commit, not + # the repository head, which has moved on since. + manifest_path = out / cli.MANIFEST_FILENAME + manifest = json.loads(manifest_path.read_text()) + manifest["staged_dataset"] = { + **staged, + "status": "failed", + "revision": None, + "error_code": "UPLOAD_FAILED", + } + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True)) + hub.sha = "e" * 40 + assert stager.main(["--run-dir", str(out)]) == 0 + recovered = json.loads(manifest_path.read_text())["staged_dataset"] + assert recovered["status"] == "already_staged" + assert recovered["revision"] == staged["revision"] != hub.sha + assert len(hub.commits) == 1 + sums_verify() + + # Consumers fetch by run id and get digest-verified local files. + fetcher = load_tool("fetch_uk_staged_dataset") + monkeypatch.setattr(fetcher, "_hub_api", lambda: hub) + dest = tmp_path / "fetched" + capsys.readouterr() + assert fetcher.main(["--run-id", run_id, "--dest", str(dest)]) == 0 + listed = capsys.readouterr().out.splitlines() + assert str(dest / dataset_name) in listed + assert (dest / "sha256sums.txt").is_file() + assert (dest / dataset_name).read_bytes() == remote_h5 + + +def test_remote_staging_failure_is_recorded_and_the_build_still_succeeds( + tmp_path, monkeypatch, capsys +): + pytest.importorskip("tables") + hub = _remote_hub(monkeypatch, fail_commit=True) + status, out = run_dense_main(tmp_path, monkeypatch, staging=None) + assert status == 0 + err = capsys.readouterr().err + assert "staged dataset upload failed" in err and "do-not-record" not in err + manifest = json.loads((out / cli.MANIFEST_FILENAME).read_text()) + staged = manifest["staged_dataset"] + assert staged["status"] == "failed" and staged["error_code"] == "UPLOAD_FAILED" + assert staged["revision"] is None and staged["files"] + assert "do-not-record" not in json.dumps(manifest) + assert hub.paths("policyengine/populace-uk-private") == [] + # Telemetry still completed and recorded the outcome. + run_id = single_run_id(out) + progress = json.loads( + hub.files[("policyengine/populace-uk-staging", f"runs/{run_id}/progress.json")] + ) + assert progress["status"] == "completed" + events = [ + json.loads(line) + for line in hub.files[ + ("policyengine/populace-uk-staging", f"runs/{run_id}/events.ndjson") + ] + .decode() + .splitlines() + if line + ] + done = next( + e + for e in events + if e["stage_id"] == "dataset_staging" and e["status"] == "completed" + ) + assert done["details"]["status"] == "failed" + assert done["details"]["error_code"] == "UPLOAD_FAILED" + rows = spool_rows(out) + assert rows[0].disposition == "iterating" + assert "dataset_stage_failed" in rows[0].phases_reached + # The sidecars are in place for a later re-stage. + assert (out / "sha256sums.txt").is_file() and ( + out / "staged_manifest.json" + ).is_file() + + +def test_no_staged_dataset_keeps_telemetry_remote_and_the_bundle_local( + tmp_path, monkeypatch +): + pytest.importorskip("tables") + hub = _remote_hub(monkeypatch) + status, out = run_dense_main( + tmp_path, monkeypatch, "--no-staged-dataset", staging=None + ) + assert status == 0 + assert hub.paths("policyengine/populace-uk-private") == [] + assert hub.commits == [] + assert hub.paths("policyengine/populace-uk-staging") + manifest = json.loads((out / cli.MANIFEST_FILENAME).read_text()) + assert manifest["staging_delivery"]["mode"] == "local_and_remote" + assert manifest["staged_dataset"]["mode"] == "disabled" + assert manifest["staged_dataset"]["opt_out_reason"] == "--no-staged-dataset" + assert not (out / "sha256sums.txt").exists() + + +def test_refusal_records_the_gate_and_error_receipt_pointers(tmp_path, monkeypatch): + """A refusal leaves its receipt pointer on the failed Logbook row. + + The candidate tool re-raised a blocking gate and recorded both the gate + verdict and a pipeline error; the graph driver returns the block as a + non-zero status and records the failed gate's receipt pointer, and a run + that raises records the error receipt pointer. + """ + pytest.importorskip("tables") + geography = "uk_local_geography_ladder_post_calibration" + blocked = tmp_path / "blocked" + blocked.mkdir() + status, out = run_dense_main(blocked, monkeypatch, failed=geography) + assert status == 1 + gate_report_path = out / f"{STEM}.local_gates.json" + assert gate_report_path.exists() + rows = spool_rows(out) + assert len(rows) == 1 + row = rows[0] + assert row.disposition == "failed" + assert row.gate_verdicts[geography] == { + "verdict": "failed", + "receipt": f"{local_ref(gate_report_path)}#/gates/{geography}", + } + assert "pipeline_error" not in row.gate_verdicts + + # A refusal after the targets are bound (the binding adjudication's + # place on the candidate tool) and before the solve completes. + raised = tmp_path / "raised" + raised.mkdir() + original_run = cli.run_graph + + def refuse_numerical(compiled, **kwargs): + if "uk.full.gates.calibrated" in {node.id for node in compiled.graph.nodes}: + raise ValueError("census_disclosure_control_noise is not adjudicated") + return original_run(compiled, **kwargs) + + monkeypatch.setattr(cli, "run_graph", refuse_numerical) + status, out = run_dense_main(raised, monkeypatch) + assert status == 1 + rows = spool_rows(out) + assert len(rows) == 1 + row = rows[0] + assert row.disposition == "failed" + assert "targets_bound" in row.phases_reached + assert "solved" not in row.phases_reached + receipts = list((out / "logbook-receipts").rglob("error.json")) + assert len(receipts) == 1 + assert row.gate_verdicts["pipeline_error"] == { + "verdict": "error", + "receipt": f"{local_ref(receipts[0])}#/error_type", + } + assert not (out / cli.MANIFEST_FILENAME).exists() + + # A pre-graph setup failure (the ladder load's place on the candidate + # tool) still spools a failed row with its error receipt pointer. + setup = tmp_path / "setup" + setup.mkdir() + monkeypatch.setattr(cli, "run_graph", original_run) + + def refuse_setup(args, telemetry): + raise RuntimeError("ladder artifact refused to parse") + + status, out = run_dense_main(setup, monkeypatch, on_prepare=refuse_setup) + assert status == 1 + rows = spool_rows(out) + assert len(rows) == 1 + row = rows[0] + assert row.disposition == "failed" + assert "targets_bound" not in row.phases_reached + receipts = list((out / "logbook-receipts").rglob("error.json")) + assert len(receipts) == 1 + assert json.loads(receipts[0].read_text())["message"] == ( + "ladder artifact refused to parse" + ) + assert row.gate_verdicts["pipeline_error"] == { + "verdict": "error", + "receipt": f"{local_ref(receipts[0])}#/error_type", + } + assert not (out / cli.MANIFEST_FILENAME).exists() + + +def test_blocked_gates_partition_failures_by_criticality(tmp_path, monkeypatch, capsys): + """Two release-blocking local failures are both enforced and none is diagnostic.""" + pytest.importorskip("tables") + status, out = run_dense_main( + tmp_path, + monkeypatch, + failed=( + ("uk_local_area_support", "ESS 42.3 < 50"), + ("uk_local_weight_ratio", "ratio 578 > 100"), + ), + ) + assert status == 1 + assert "artifact unreleasable" in capsys.readouterr().err + manifest = json.loads((out / cli.MANIFEST_FILENAME).read_text()) + assert manifest["failing_gate_ids"] == [ + "uk_local_area_support", + "uk_local_weight_ratio", + ] + assert manifest["blocked_at_f100"] is True + assert manifest["blocking_failures"] == [ + "[uk_local_area_support] ESS 42.3 < 50", + "[uk_local_weight_ratio] ratio 578 > 100", + ] + assert manifest["diagnostic_failures"] == [] + assert manifest["releasable"] is False + report = json.loads( + Path(manifest["outputs"]["local_gate_report"]["path"]).read_text() + ) + outcomes = {outcome["id"]: outcome for outcome in report["report"]["outcomes"]} + for gate_id in ("uk_local_area_support", "uk_local_weight_ratio"): + assert outcomes[gate_id]["criticality"] == "release_blocking" + assert outcomes[gate_id]["status"] == "failed" + assert spool_rows(out)[0].disposition == "failed" + + +def test_multi_block_engine_run_is_never_releasable(tmp_path, monkeypatch): + """End to end: ``--engine-blocks K`` on f100 writes ``releasable: false``. + + Every release-blocking gate passes here; the posture alone withholds the + verdict, and the manifest names the leg (``single_block_engine``). + """ + pytest.importorskip("tables") + status, out = run_dense_main( + tmp_path, monkeypatch, "--n-clones", "2", "--engine-blocks", "2" + ) + assert status == 0 + manifest = json.loads((out / cli.MANIFEST_FILENAME).read_text()) + assert manifest["parameters"]["engine_blocks"] == 2 + assert manifest["blocking_failures"] == [] + assert manifest["releasable"] is False + posture = manifest["release_posture"] + assert posture["full_rung"] is True + assert posture["single_block_engine"] is False + assert posture["release_blocking_gates_passed"] is True diff --git a/test_support/microcosm_build/uk_full_build_cli.py b/test_support/microcosm_build/uk_full_build_cli.py index b407b2328..5b21b5169 100644 --- a/test_support/microcosm_build/uk_full_build_cli.py +++ b/test_support/microcosm_build/uk_full_build_cli.py @@ -6,6 +6,7 @@ # ruff: noqa: F401 import hashlib +import importlib.util import json from dataclasses import replace from pathlib import Path @@ -20,6 +21,8 @@ gate_phase_report_payload, ) from microcosm.build.gates import GateResult +from microcosm.build.logbook import LOGBOOK_ROW_FIELDS, load_spool_rows +from microcosm.build.logbook_adoption import local_artifact_reference from microcosm.build.uk_runtime import full_build_cli as cli from microcosm.build.uk_runtime.full_certification import FULL_CERTIFICATION_TYPE from microcosm.build.uk_runtime.full_gates import ( @@ -70,11 +73,19 @@ ) from microcosm.graph.canonical import canonical_json from test_support.microcosm_build.uk_graph_terminal import _frame +from test_support.paths import paths_for PIN = "0" * 64 STEM = "microcosm_uk_2024_25_local" +_TEST_PATHS = paths_for("microcosm-build") + +# The real parser, bound before any test patches ``cli.parse_args`` to return +# one prepared namespace: a test that runs ``main`` twice must still parse +# its second request rather than receive the first one's. +_PARSE_ARGS = cli.parse_args + def _placeholder(path: Path, payload: bytes) -> str: if not path.exists(): @@ -87,11 +98,12 @@ def arguments(tmp_path, *extra, role="dense", staging="--no-staging"): The pins are the stand-ins' real digests so the validator and a real preparation would both accept them; the Ledger pins are synthetic - because these tests never compile targets. + because these tests never compile targets. ``staging=None`` passes no + staging switch at all, which is the remote (``local_and_remote``) mode. """ spine = tmp_path / "spine.h5" ladder = tmp_path / "ladder.npz" - return cli.parse_args( + return _PARSE_ARGS( [ "--release-role", role, @@ -111,12 +123,40 @@ def arguments(tmp_path, *extra, role="dense", staging="--no-staging"): PIN, "--out", str(tmp_path / "out"), - staging, + *(() if staging is None else (staging,)), *extra, ] ) +def local_ref(path: Path) -> str: + """The Logbook receipt reference of a file, as the driver writes it.""" + return local_artifact_reference(path, repository_hint=cli.REPOSITORY) + + +def spool_rows(out: Path): + rows = load_spool_rows(out / "logbook-spool") + for row in rows: + assert frozenset(row.to_mapping()) == LOGBOOK_ROW_FIELDS + return rows + + +def single_run_id(out: Path) -> str: + runs = sorted(path.name for path in (out / "staging" / "runs").iterdir()) + assert len(runs) == 1, runs + return runs[0] + + +def load_tool(name: str): + """Execute ``tools/.py`` as a private module copy (its seams patchable).""" + root = _TEST_PATHS.repository + spec = importlib.util.spec_from_file_location(name, root / "tools" / f"{name}.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def _national_argv(tmp_path, *extra): return [ "--release-role", @@ -149,19 +189,35 @@ def _national_argv(tmp_path, *extra): } +def failure_lines(failed) -> dict[str, str]: + """Gate ids to fail with their failure line. + + ``failed`` is one gate id (line ``"synthetic failure"``), a tuple of + ``(gate_id, line)`` pairs, or ``None``; the pair form rides on a node + parameter (tuples of strings), so a synthetic evidence node can fail + several gates with distinct lines. + """ + if failed is None: + return {} + if isinstance(failed, str): + return {failed: "synthetic failure"} + return {str(gate_id): str(line) for gate_id, line in failed} + + def gate_payload(phase, failed=None): gates = uk_full_gate_manifest(SELECTION) + lines = failure_lines(failed) report = GatePhaseReport( phase, tuple( GateOutcome( entry, - GateStatus.FAILED if entry.id == failed else GateStatus.PASSED, + GateStatus.FAILED if entry.id in lines else GateStatus.PASSED, GateResult( name=entry.id, - passed=entry.id != failed, + passed=entry.id not in lines, details={}, - failures=("synthetic failure",) if entry.id == failed else (), + failures=(lines[entry.id],) if entry.id in lines else (), ), ) for entry in gates.gates @@ -505,20 +561,45 @@ def certification_service_fixture(monkeypatch): patch_certification(monkeypatch) +def run_dense_main( + tmp_path, + monkeypatch, + *extra, + staging="--no-staging", + failed=None, + on_prepare=None, +) -> tuple[int, Path]: + """Run the synthetic dense build through ``main``; return its status and bundle. + + ``failed`` fails those gates on the synthetic evidence nodes (see + :func:`failure_lines`); ``on_prepare(args, telemetry)`` runs inside the + patched preparation, where the driver's own solve observer can be driven + with synthetic epochs (the synthetic graph has no calibration kernel). + """ + monkeypatch.delenv("POPULACE_LOGBOOK_PREV_ROW_DIGEST", raising=False) + patch_certification(monkeypatch) + args = arguments(tmp_path, *extra, staging=staging) + build = prepared(tmp_path, failed) + + def prepare(args, *, telemetry=None, attempt=None): + if on_prepare is not None: + on_prepare(args, telemetry) + return build + + monkeypatch.setattr(cli, "parse_args", lambda argv: args) + monkeypatch.setattr(cli, "prepare_full_build", prepare) + return cli.main([]), args.out + + def graph_dense_bundle(tmp_path, monkeypatch, *extra, staging="--no-staging") -> Path: """Run the synthetic dense build through ``main`` and return its bundle. The other UK test modules feed the resulting ``rowwise_candidate_manifest.json`` to the release pre-flight and the dense assembler. """ - monkeypatch.delenv("POPULACE_LOGBOOK_PREV_ROW_DIGEST", raising=False) - patch_certification(monkeypatch) - args = arguments(tmp_path, *extra, staging=staging) - build = prepared(tmp_path) - monkeypatch.setattr(cli, "parse_args", lambda argv: args) - monkeypatch.setattr(cli, "prepare_full_build", lambda args, **kwargs: build) - assert cli.main([]) == 0 - return args.out + status, out = run_dense_main(tmp_path, monkeypatch, *extra, staging=staging) + assert status == 0 + return out def prepared(tmp_path, failed=None): From 6e5db107bde9b4d3a641c999e1d714787c4c4e4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:59:33 +0100 Subject: [PATCH 33/44] Stage the sample block and the size-phase epochs on the graph driver (receipts R5 rows 18 and 19) Two staging gaps against main's dense tool. The graph driver never set the telemetry's sampling evidence: rowwise_staging.stage_sample now writes {"mode": "full"} on the f100 rung and nothing below it, judged on the prepared build's effective fraction (pool times source spine), and the national seam calls the same helper. The size search and refit solved without an observer: UKSizeSearchKernel and UKSizeRefitKernel now take the same operational progress_callback as the dense kernel (instance state, never a node key or implementation hash) and register_uk_calibration_kernels hands one observer to all three, so the driver's _solve_observer stages the size_search/size_refit rows and prints their phase. graph.epoch_rows in the projected manifest names the three solves; the dense-only bullet leaves the doc and R3, and R5 rows 18 and 19 are restored, naming their tests. Co-Authored-By: Claude Fable 5.1 --- docs/uk-full-build-graph.md | 3 +- experiments/901-uk-main-rebase-receipts.md | 36 +++++---- .../build/uk_runtime/full_build_cli.py | 20 ++++- .../microcosm/build/uk_runtime/graph_build.py | 6 +- .../build/uk_runtime/graph_calibration.py | 53 +++++++++---- .../build/uk_runtime/graph_terminal.py | 12 +-- .../build/uk_runtime/national_role.py | 4 +- .../build/uk_runtime/rowwise_staging.py | 17 +++++ .../engine_free/uk/test_uk_full_build_cli.py | 51 ++++++++++++- .../uk/test_uk_full_calibration_graph.py | 76 +++++++++++++++++++ .../microcosm_build/uk_full_build_cli.py | 7 +- 11 files changed, 235 insertions(+), 50 deletions(-) diff --git a/docs/uk-full-build-graph.md b/docs/uk-full-build-graph.md index 3011dc004..a964933d8 100644 --- a/docs/uk-full-build-graph.md +++ b/docs/uk-full-build-graph.md @@ -93,7 +93,7 @@ The output bundle is named from the role's posture and the FRS release vintage: The terminal graph node writes unsigned `certification.json` from those identified artifacts and any declared native or matched-size comparisons. `build.json` is the completion marker that binds both `candidate.json` and the certification artifact. Physical output bytes are checked against their declared artifacts. Bundle publication writes the completion marker last and rolls back handled failures or interrupts. A process kill or power loss can leave an absent completion marker; a directory without a valid bound marker is not a completed build. -A non-dry dense run is wrapped in the rowwise tool's operational envelope: the Logbook attempt (a `uk-local-candidate` row spooled under `/logbook-spool` on every terminal outcome, chained through `--logbook-prev-row-digest` or `POPULACE_LOGBOOK_PREV_ROW_DIGEST`, with an error receipt on failure), version 2 staging telemetry with stage events around each graph phase and per-epoch `calibration_progress` rows from the dense solve, and the staged-dataset delivery of the published bundle under `staged//` in the private repository. `--staging-local-only`, `--no-staging`, `--staging-read-back` and `--no-staged-dataset` behave as in [UK staging operations](uk-staging-operations.md). Dry runs plan without solving or writing and record no Logbook row. +A non-dry dense run is wrapped in the rowwise tool's operational envelope: the Logbook attempt (a `uk-local-candidate` row spooled under `/logbook-spool` on every terminal outcome, chained through `--logbook-prev-row-digest` or `POPULACE_LOGBOOK_PREV_ROW_DIGEST`, with an error receipt on failure), version 2 staging telemetry with the run's sampling evidence (`sample: {"mode": "full"}` on the f100 rung, null below it, judged on the effective fraction), stage events around each graph phase and per-epoch `calibration_progress` rows from the dense solve and, with `--dataset-households`, the size search and refit (rows tagged with their `phase`), and the staged-dataset delivery of the published bundle under `staged//` in the private repository. `--staging-local-only`, `--no-staging`, `--staging-read-back` and `--no-staged-dataset` behave as in [UK staging operations](uk-staging-operations.md). Dry runs plan without solving or writing and record no Logbook row. Structural failures stop export. The maintained local statistical failure policy may still export an unreleasable diagnostic candidate with a nonzero process status. Missing evidence remains explicit. `--release-candidate` applies the maintained strictness and solve settings; it does not publish, sign or authorize a release. Fixture acceptance proves graph behavior. Native certification additionally requires measured incumbent comparison evidence supplied with `--native-scorecard`; exact-count promotion also requires the measured comparison at the requested k through `--matched-size-scorecard`. The certification node verifies their candidate and output identities before assessing readiness. @@ -106,7 +106,6 @@ Recorded for review in `experiments/901-uk-main-rebase-receipts.md` (R3): - A blocked *assembled* spine gate fails inside `run_graph` (the first post-checkpoint stage refuses on the stored verdict), and the driver materialises the stored assembled report into `spine_gates.json` (`blocked_at_phase: "assembled"`, the transferred phase `unreached`) before it fails, so the operator gets the same file the previous tool wrote before raising; a blocked *transferred* gate writes the file on the success path. The report is also in the content store either way. - `numerical_dependencies` pins installed versions into the H2 fixture. - `_normalise_uk_local_bound_families` keeps main's refusal of a declaration that names nothing; only the graph's country-only target selection (`UKFullProblemKernel`, no local-surface spec selected) passes `allow_empty_local_binding=True` through `prepare_uk_full_solve`, because its local surface is empty by construction. -- Per-epoch `calibration_progress` staging rows come from the dense solve only. - The HMRC family names `spi_income_band_donors` (microcosm#1006) as a predecessor and admits its two operation kinds; the contract otherwise refused the drifted operation order. - `tools/build_uk_rowwise_dataset.py` stays as it was on main (its tests load it by path; it still serves `--candidate-clone-counts`). diff --git a/experiments/901-uk-main-rebase-receipts.md b/experiments/901-uk-main-rebase-receipts.md index 8b3f75460..b78dcc011 100644 --- a/experiments/901-uk-main-rebase-receipts.md +++ b/experiments/901-uk-main-rebase-receipts.md @@ -76,7 +76,6 @@ sampling; `--no-staging`; checkpoints on. Script and outputs: through `prepare_uk_full_solve`. Both behaviours pinned by `test_uk_local_rowwise.py::test_rowwise_binding_refuses_an_empty_declaration_unless_allowed` and `test_uk_full_solve_scope.py::test_zero_local_scope_uses_same_solver_and_has_no_fake_holdout`. -- Per-epoch `calibration_progress` staging rows come from the dense solve only. - The HMRC family names `spi_income_band_donors` (#1006) as a predecessor and admits its two operation kinds; the contract otherwise refused the drifted operation order. - `tools/build_uk_rowwise_dataset.py` stays main's (its tests load it by path; it still serves @@ -99,11 +98,12 @@ each retired test is either restored on `full_build_cli.main` over the synthetic (`test_support/microcosm_build/uk_full_build_cli.py`: `run_dense_main`, multi-gate `gate_payload`, `_FakeHub` from the rowwise helper, patches on `rowwise_staging`) or shown covered by a named test. Unqualified `::` names below are in -`packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py`. Totals: 14 rows -restored (rows 3, 8, 9, 10, 12, 14, 15, 18, 20 to 25, through 11 new test functions and one -extended test; rows 9 and 10 live inside row 8's test), 10 rows covered by named tests (1, 2, 4, -5, 6, 7, 11, 13, 16, 17), 1 row superseded by a documented behaviour change (19, R3). Residual -gaps are named on their rows; rows 18 and 19 carry the two for a ruling. +`packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py`. Totals: 15 rows +restored (rows 3, 8, 9, 10, 12, 14, 15, 18, 19, 20 to 25, through 13 new test functions and +one extended test; rows 9 and 10 live inside row 8's test), 10 rows covered by named tests (1, +2, 4, 5, 6, 7, 11, 13, 16, 17). Residual gaps are named on their rows; the two gaps rows 18 and +19 carried for a ruling (the run manifest's `sample` block, the size phases' epoch rows) were +closed on 2026-09-28 by restoring the tool's behaviour on the driver. 1. `candidate_build_writes_calibrated_h5_and_evidence`: covered. Manifest projection (schema 4, dense role and release id, doctrine block, identity pins, solve, weights, output digests, @@ -192,14 +192,22 @@ gaps are named on their rows; rows 18 and 19 carry the two for a ruling. `::test_main_stages_the_bundle_locally_with_staging_local_only` with the retired inventory assertions (stdout manifest equals the on-disk one, run id equals build id, operation and pipeline ids, artifacts, fit summary, staged files with digests, sha256sums, sidecar - inventory, sidecars never outputs, stage sequence). Residual for a ruling: the run manifest's - `sample` block is unset on the graph driver where the tool wrote `{"mode": "full"}`; per-epoch - rows are the dense solve's only (R3). -19. `size_candidate_stages_the_search_and_refit_phases`: superseded by design (R3: per-epoch - staging rows come from the dense solve only; the search and refit forward no epochs), pinned - by `::test_dense_run_projects_the_rowwise_candidate_manifest` (`graph.epoch_rows == - "dense_solve_only"`); the size-run manifest claims are rows 16 and 17. Not restorable as - written; flagged for a ruling with row 18. + inventory, sidecars never outputs, stage sequence) and the run manifest's `sample` block: + `{"mode": "full"}` on the f100 rung, written through `rowwise_staging.stage_sample`, the + one helper the national seam and the dense driver now share (the driver judges the rung on + the effective fraction, pool times source spine); a rung below f100 stages a null sample, + as the tool did, pinned by `::test_sampled_run_stages_a_null_sample_block`. +19. `size_candidate_stages_the_search_and_refit_phases`: restored as + `test_uk_full_calibration_graph.py::test_size_kernels_forward_phased_epochs_to_the_registered_observer`: + the size search and refit kernels forward their epochs, tagged `size_search` and + `size_refit` by `dataset_size`, through the one observer + `register_uk_calibration_kernels(progress_callback=)` registers (the driver's + `_solve_observer`, so the staging rows and the stderr lines carry the phase); the observer is + instance state, so the implementation hashes are those of an unobserved registry and the + observed run replays under one without execution. The manifest projection records + `graph.epoch_rows == "dense_solve,size_search,size_refit"` (pinned by + `::test_dense_run_projects_the_rowwise_candidate_manifest`); the size-run manifest claims + are rows 16 and 17. 20. `telemetry_content_refusal_never_aborts_the_solve`: restored as `::test_telemetry_content_refusal_never_aborts_the_solve` (drives the driver's own `_solve_observer` with synthetic epochs). diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py index 2ba6bfc33..6e53a1eb1 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py @@ -19,8 +19,10 @@ A non-dry dense run is wrapped in the rowwise tool's operational envelope: the Logbook attempt (a spooled row under ``/logbook-spool`` on every terminal outcome, an error receipt on failure), the version 2 staging -telemetry with stage events around each graph phase and per-epoch rows from -the dense solve, and the staged-dataset delivery of the published bundle. +telemetry with the run's sampling evidence, stage events around each graph +phase and per-epoch rows from the dense solve and, with +``--dataset-households``, the size search and refit (rows tagged with their +``phase``), and the staged-dataset delivery of the published bundle. The bundle carries ``rowwise_candidate_manifest.json`` projected from the graph's stored artifacts (:func:`~microcosm.build.uk_runtime.graph_terminal.rowwise_candidate_manifest_from_graph`), @@ -141,6 +143,7 @@ replace_manifest, stage, stage_dataset, + stage_sample, staging_delivery, staging_epoch_every, thinned_epochs, @@ -437,7 +440,12 @@ def _stderr_progress(line: str) -> None: def _solve_observer(args: argparse.Namespace, telemetry): - """Readable stderr lines plus the thinned staging rows of the dense solve.""" + """Readable stderr lines plus the thinned staging rows of every solve. + + The dense solve, the informed size search and the refit share this one + observer; the size events carry the ``phase`` the stderr line names and + the staging row records. + """ from .solve_progress import uk_solve_progress_callback sinks = [uk_solve_progress_callback(_stderr_progress)] @@ -846,6 +854,12 @@ def execute_full_build( return _execute_full_build(prepared, args) from microcosm.build.artifact_files import publish_staged_bundle + # The rung is known once the source spine's own sampling has been read + # (preparation); it is staged before any graph phase, where the rowwise + # tool staged it after its sampling step. + stage_sample( + telemetry, sample_fraction=prepared.full.config.effective_sample_fraction + ) output, graph_store = _output_locations(prepared, args) output.parent.mkdir(parents=True, exist_ok=True) # Operational attempt identity never enters a scientific node/cache key. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py index 8fc0b3c0d..45ed25c88 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py @@ -345,8 +345,10 @@ def register_uk_full_kernels( ) -> KernelRegistry: """Extend the existing UK source/stage registry with the full build. - ``progress_callback`` is an operational observer of the dense solve's - epochs (staging telemetry, stderr progress); it never enters a node key. + ``progress_callback`` is an operational observer of the solves' epochs + (staging telemetry, stderr progress): the dense solve and, with a + ``dataset_households`` request, the size search and refit, whose events + carry their ``phase``; it never enters a node key. """ # Source graph registries already contain these primitive kernels. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py index 8d5b361d5..dccfc00ee 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py @@ -324,7 +324,23 @@ def run(self, context): ) -class UKDenseSolveKernel(_CalibrationKernel): +class _ObservedSolveKernel(_CalibrationKernel): + """A solving kernel whose epochs an operational observer may watch. + + ``progress_callback`` receives the calibrator's progress events (progress + lines, staging telemetry rows). It is registered like the gate kernels' + coverage engine: instance state, never part of the node key or the + implementation hash, so an observed and an unobserved solve share one + cache entry. The size kernels' events carry the ``phase`` + (``size_search``, ``size_refit``) that :mod:`.dataset_size` tags; the + dense solve's carry none. + """ + + def __init__(self, *, progress_callback=None): + self.progress_callback = progress_callback + + +class UKDenseSolveKernel(_ObservedSolveKernel): ref = "uk.full.dense@1" capabilities = Capabilities( Determinism.DETERMINISTIC, @@ -332,14 +348,6 @@ class UKDenseSolveKernel(_CalibrationKernel): dependencies=_DEPENDENCIES, ) - def __init__(self, *, progress_callback=None): - # An operational observer of the solve's epochs (progress lines, - # staging telemetry rows). It is registered like the gate kernels' - # coverage engine: instance state, never part of the node key or - # the implementation hash, so an observed and an unobserved solve - # share one cache entry. - self.progress_callback = progress_callback - def run(self, context): from .graph_terminal import decode_full_gate_report @@ -413,7 +421,7 @@ def _full_pool(frame, context): return k == frame.n("household") -class UKSizeSearchKernel(_CalibrationKernel): +class UKSizeSearchKernel(_ObservedSolveKernel): ref = "uk.full.size_search@1" capabilities = Capabilities( Determinism.DETERMINISTIC, @@ -441,7 +449,10 @@ def run(self, context): result_payload = context.artifacts["dense"].payload else: selection = dataset_size.select_uk_dataset_size( - frame, dense, **dict(context.params) + frame, + dense, + progress_callback=self.progress_callback, + **dict(context.params), ) metadata = { "method": "contribution_informed_l0", @@ -523,7 +534,7 @@ def run(self, context): ) -class UKSizeRefitKernel(_CalibrationKernel): +class UKSizeRefitKernel(_ObservedSolveKernel): ref = "uk.full.size_refit@1" capabilities = Capabilities( Determinism.DETERMINISTIC, @@ -558,7 +569,12 @@ def run(self, context): } ) compact = dataset_size.refit_uk_dataset_size( - frame, dense, selection=selection, draw=cached_draw, **dict(context.params) + frame, + dense, + selection=selection, + draw=cached_draw, + progress_callback=self.progress_callback, + **dict(context.params), ) result = compact.result ids = _axis(result.frame) @@ -860,12 +876,17 @@ def uk_calibration_nodes( def register_uk_calibration_kernels( registry: KernelRegistry, *, progress_callback=None ) -> KernelRegistry: - registry.register(UKDenseSolveKernel(progress_callback=progress_callback)) + """Register the calibration kernels; ``progress_callback`` observes every solve. + + The one observer is handed to the dense solve, the size search and the + refit (see :class:`_ObservedSolveKernel`); the import, draw, filter and + install kernels solve nothing and take none. + """ + for kernel in (UKDenseSolveKernel, UKSizeSearchKernel, UKSizeRefitKernel): + registry.register(kernel(progress_callback=progress_callback)) for kernel in ( UKSizeCheckpointImportKernel, - UKSizeSearchKernel, UKSizeDrawKernel, - UKSizeRefitKernel, UKSizeFilterKernel, UKInstallCalibrationKernel, ): diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py index a9e60bd54..5cc674348 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py @@ -1132,11 +1132,11 @@ def materialize_uk_terminal_artifacts( # necessity: main's ``_manifest`` cannot run without the live objects. The # projection adds one ``graph`` block naming the artifacts it stood on. # -# Per-epoch ``calibration_progress`` rows: the dense solve node forwards its -# epochs through the observer registered on ``UKDenseSolveKernel``; the size -# search and refit nodes solve through ``dataset_size`` without an observer, -# so a size run stages no epoch rows for those two phases (recorded in the -# manifest's ``graph.epoch_rows`` field). +# Per-epoch ``calibration_progress`` rows: the dense solve, size search and +# size refit nodes forward their epochs through the one observer registered +# on the calibration kernels (``register_uk_calibration_kernels``), the size +# rows tagged with their ``phase``; the manifest's ``graph.epoch_rows`` field +# names the three solves whose rows a size run stages. _LADDER_TARGET_PREFIX = "ons.census.households@" _NATIONAL_MATERIALIZATION = "uk_national_measure" @@ -1581,6 +1581,6 @@ def rowwise_candidate_manifest_from_graph( key: dict(value) for key, value in terminal_files.items() }, "enforcement": enforcement, - "epoch_rows": "dense_solve_only", + "epoch_rows": "dense_solve,size_search,size_refit", }, } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_role.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_role.py index 7d49f7310..786b10c90 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_role.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_role.py @@ -83,6 +83,7 @@ replace_manifest, stage, stage_dataset, + stage_sample, staging_delivery, staging_epoch_every, thinned_epochs, @@ -328,8 +329,7 @@ def _run_national_attempt( "completed", dataset_sha256=input_artifact["sha256"], ) - if telemetry is not None: - telemetry.set_sample({"mode": "full"}) + stage_sample(telemetry, sample_fraction=args.sample_fraction) frs_release = load_uk_frs_release() calibration_year = int(frs_release.calibration_year) args._calibration_year = calibration_year diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py index e2e07aa7b..ccb031c11 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py @@ -241,6 +241,23 @@ def stage( _stage = stage +def stage_sample( + telemetry: StagingTelemetryV2 | None, *, sample_fraction: float +) -> None: + """Record the run's sampling evidence on the staging telemetry. + + The contract's only sampling statement is ``{"mode": "full"}``, the f100 + rung; a rung below f100 stages a null sample, as the spine builder does. + The national seam and the dense graph driver both call this once their + rung is known (for the graph, the pool fraction times the source spine's + own fraction), so the two release roles stage the same evidence. + """ + + if telemetry is None or float(sample_fraction) != 1.0: + return + telemetry.set_sample({"mode": "full"}) + + def staging_epoch_every(args: argparse.Namespace) -> int: """The epoch stride that keeps the forwarded rows under the row budget. diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py index 286cd4b45..0cfe082b1 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py @@ -367,7 +367,7 @@ def test_dense_run_projects_the_rowwise_candidate_manifest(tmp_path): assert Path(manifest["outputs"]["local_gate_report"]["path"]).name == ( f"{STEM}.local_gates.json" ) - assert manifest["graph"]["epoch_rows"] == "dense_solve_only" + assert manifest["graph"]["epoch_rows"] == "dense_solve,size_search,size_refit" assert "uk.full.problem" in manifest["graph"]["artifacts"] # The staged bundle is every registered output plus the manifest; each # registered file is on disk beside it with its recorded digest. @@ -623,8 +623,10 @@ def test_main_stages_the_bundle_locally_with_staging_local_only( assert run_manifest["operation_id"] == "uk_rowwise_candidate" assert run_manifest["pipeline"]["id"] == "uk-local-candidate" assert run_manifest["non_release"] is True - # The candidate tool wrote ``sample == {"mode": "full"}``; the graph driver - # sets no sample block (receipts R5, residual gap for a ruling). + # The f100 rung stages the contract's one sampling statement, as the + # candidate tool wrote after its sampling step (receipts R5, row 18). + assert run_manifest["sample"] == {"mode": "full"} + assert bundle["progress"]["sample"] == {"mode": "full"} assert run_manifest["delivery"]["mode"] == "local_only" assert run_manifest["delivery"]["upload_attempts"] == 0 assert {a["logical_name"] for a in run_manifest["artifacts"]} == { @@ -679,6 +681,49 @@ def test_main_stages_the_bundle_locally_with_staging_local_only( assert "staged_manifest" not in manifest["outputs"] +def test_sampled_run_stages_a_null_sample_block(tmp_path, monkeypatch): + """A rung below f100 stages ``sample: null``, as the candidate tool did. + + The contract's only sampling statement is ``{"mode": "full"}``. The driver + judges the rung on the effective fraction (pool times source spine), read + from the prepared build's configuration, so a ``--sample-fraction 0.1`` + pool over a full spine stages no sample block while its Logbook row + carries the f010 rung. + """ + + pytest.importorskip("tables") + from microcosm.build.staging_v2 import validate_v2_bundle + + arguments(tmp_path) # writes the stand-ins the prepared build pins + build = prepared(tmp_path) + sampled = replace( + build, + full=replace( + build.full, config=replace(build.full.config, sample_fraction=0.1) + ), + ) + assert sampled.full.config.effective_sample_fraction == 0.1 + staging_dir = tmp_path / "staging-bundle" + status, out = run_dense_main( + tmp_path, + monkeypatch, + "--sample-fraction", + "0.1", + "--staging-dir", + str(staging_dir), + staging="--staging-local-only", + build=sampled, + ) + assert status == 0 + runs = sorted(path.name for path in (staging_dir / "runs").iterdir()) + assert len(runs) == 1 + bundle = validate_v2_bundle(staging_dir, runs[0]) + assert bundle["progress"]["status"] == "completed" + assert bundle["run_manifest"]["sample"] is None + assert bundle["progress"]["sample"] is None + assert spool_rows(out)[0].rung == "f010" + + # --------------------------------------------------------------------------- # Delivery-side resilience and refusal receipts on the graph driver. These # restore the in-process candidate tool's contracts (retired in af01b990d; diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_calibration_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_calibration_graph.py index f97b481e9..5fb4c2947 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_calibration_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_calibration_graph.py @@ -432,3 +432,79 @@ def test_blocking_preflight_refuses_before_dense_or_checkpoint_work(imported): artifacts["imported_dense"] = SimpleNamespace(payload=b"must not read") with pytest.raises(ValueError, match="refused by the source preflight"): UKDenseSolveKernel().run(SimpleNamespace(artifacts=artifacts)) + + +def test_size_kernels_forward_phased_epochs_to_the_registered_observer( + tmp_path, monkeypatch +): + """One observer sees the dense, search and refit epochs, tagged by phase. + + Restores the candidate tool's ``size_candidate_stages_the_search_and_refit_phases`` + contract on the graph (receipts R5, row 19): the size search and refit + kernels forward their epochs through the callback + ``register_uk_calibration_kernels`` registers, tagged ``size_search`` and + ``size_refit`` by ``dataset_size``, after the dense solve's untagged + epochs, and the driver's stderr line names the phase. The observer is + instance state: the implementation hashes are an unobserved registry's, + and the observed run replays under one without executing a kernel. + """ + from microcosm.build.uk_runtime.solve_progress import uk_solve_progress_callback + + events: list[dict] = [] + observer = events.append + observed = KernelRegistry() + observed.register(Source()) + register_uk_calibration_kernels(observed, progress_callback=observer) + unobserved = registry() + for ref in ("uk.full.dense@1", "uk.full.size_search@1", "uk.full.size_refit@1"): + assert observed.get(ref).progress_callback is observer + assert unobserved.get(ref).progress_callback is None + assert ( + observed.get(ref).implementation_hash() + == unobserved.get(ref).implementation_hash() + ) + graph, endpoints = compiled(2) + fixture = tmp_path / "fixture" + fixture.write_bytes(b"fixture") + store = ContentStore(tmp_path / "store") + first = run_graph( + graph, sources={"fixture": fixture}, store=store, kernels=observed + ) + assert not first.node("uk.full.size_search").hit + epochs = [event for event in events if event["kind"] == "calibration_epoch"] + assert list(dict.fromkeys(event.get("phase") for event in epochs)) == [ + None, + "size_search", + "size_refit", + ] + for event in epochs: + assert 1 <= int(event["epoch"]) <= int(event["epochs"]) == 2 + assert "loss" in event + # Only the size search reports anything but epochs (probes, the stop). + assert { + event["kind"] for event in events if event.get("phase") != "size_search" + } == {"calibration_epoch"} + lines: list[str] = [] + render = uk_solve_progress_callback(lines.append, every=1) + for event in events: + render(event) + assert any(" dense solve: epoch " in line for line in lines) + assert any(" search: epoch " in line or " probe " in line for line in lines) + assert any(" refit: epoch " in line for line in lines) + # The observer never entered a node key: the observed run replays under + # an unobserved registry without executing a kernel. + for kernel in unobserved.as_mapping().values(): + monkeypatch.setattr( + kernel, "run", lambda *a, **kw: pytest.fail("replay executed") + ) + replay = run_graph( + graph, + sources={"fixture": fixture}, + store=store, + kernels=unobserved, + resume="require", + ) + np.testing.assert_array_equal( + replay.population(endpoints.population).weights_for("household").values, + first.population(endpoints.population).weights_for("household").values, + ) diff --git a/test_support/microcosm_build/uk_full_build_cli.py b/test_support/microcosm_build/uk_full_build_cli.py index 5b21b5169..81a4483d8 100644 --- a/test_support/microcosm_build/uk_full_build_cli.py +++ b/test_support/microcosm_build/uk_full_build_cli.py @@ -568,18 +568,21 @@ def run_dense_main( staging="--no-staging", failed=None, on_prepare=None, + build=None, ) -> tuple[int, Path]: """Run the synthetic dense build through ``main``; return its status and bundle. ``failed`` fails those gates on the synthetic evidence nodes (see :func:`failure_lines`); ``on_prepare(args, telemetry)`` runs inside the patched preparation, where the driver's own solve observer can be driven - with synthetic epochs (the synthetic graph has no calibration kernel). + with synthetic epochs (the synthetic graph has no calibration kernel); + ``build`` replaces the synthetic prepared build (a :func:`prepared` with + an altered configuration, say). """ monkeypatch.delenv("POPULACE_LOGBOOK_PREV_ROW_DIGEST", raising=False) patch_certification(monkeypatch) args = arguments(tmp_path, *extra, staging=staging) - build = prepared(tmp_path, failed) + build = prepared(tmp_path, failed) if build is None else build def prepare(args, *, telemetry=None, attempt=None): if on_prepare is not None: From 28c051bff1f808e93c7927bb7b8f19c7eaba9897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 28 Sep 2026 13:49:49 +0100 Subject: [PATCH 34/44] Record the rebase onto main after #1012 in receipts R6 Co-Authored-By: Claude Fable 5.1 --- experiments/901-uk-main-rebase-receipts.md | 34 ++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/experiments/901-uk-main-rebase-receipts.md b/experiments/901-uk-main-rebase-receipts.md index b78dcc011..0d0be4b24 100644 --- a/experiments/901-uk-main-rebase-receipts.md +++ b/experiments/901-uk-main-rebase-receipts.md @@ -226,3 +226,37 @@ closed on 2026-09-28 by restoring the tool's behaviour on the driver. One helper defect surfaced while restoring row 8: `arguments()` parsed through `cli.parse_args`, which an earlier `run_dense_main` in the same test had already patched, so a second run reused the first run's output directory; the helper now parses through the real parser bound at import. + +## R6. Rebase onto main 937aca4ec after #1012 merged (2026-09-28) + +- Replay: `git rebase origin/main` over the 33-commit series (pre-rebase head ae3179cff, local tag + `uk-901-pre-1012-rebase`). Two stops. At the spine consolidation commit the shim-vs-tool conflict + on `tools/build_uk_frs_spine.py` and the `uk_spine.json` fixture were staged by rerere from the + trial merge (`repos/populace-901-1012trial`, 6103d2be2, never pushed); at the HMRC tail + retirement commit the coverage manifest and `test_country_spec` were staged by rerere and + `test_uk_graph.py` was resolved by hand (34 stages; the exclusion assertion stays retired). The + trial's two modify/delete stops did not recur because #1012 merged on #998's layout. +- Lifted as the recipe recorded: #1012's three spine-tool hunks into `uk_runtime/spine_build.py` + (the `UKSPIHousingShellStageTransform` import, the `impute_spi_housing_shell` and + `price_domestic_energy` seed branches in `_declared_seeds`, the `implementations["spi_housing_shell"]` + entry in `prepare_uk_spine_execution`), identical to main's delta on the tool; the H2 docstring count + in `test_support/microcosm_graph/acceptance_h_parity.py`. Main already carried the 34 counts in + `graph_kernels.py`, `tools/graph_uk_spine_fixture.py` and the three #1012 tests at their #998 + paths, so those lifts were moot. Both lifts are squashed into the commits they belong to + (spine consolidation, tail retirement), so each commit keeps "spine_build = main's tool moved" true. +- Derived surfaces regenerated with their tools and found unchanged: the release-input coverage + manifest (`--check` current, 145 required, 0 reviewed exclusions) equals the rerere resolution; + the H2 fixture (oracle identity `9a069cfae…` on this machine) equals main's #1012 fixture with the + rerere'd `uk_spine.json`. `uv lock --check` is quiet; `APPROVED_UV_LOCK_SHA256` stays `e299eef1…`. +- Roster: 34 stages, #1012's 36 minus the retired pair, in #1012's order (SPI block after + `frs_brma`, `spi_housing_shell` after `hmrc_spi_income_spine`). +- Verification (targeted to the files the rebase touched, JUnit-counted): the 24 files the rebase touched or #1012 added (spine, roster, coverage, gates, evidence, driver, staging, pins) 729 passed / 0 failed, after one test fix: `test_uk_graph_evidence` had hard-coded `was_wealth` as the stage the assembled gate admits; the gate is roster-derived (the stage after `frs_brma`, now `frs_hmrc_spine_leaves` because #1012 moved the SPI block ahead of the WAS transfer, exactly where main's tool places `UK_SPINE_ASSEMBLED_FINAL_STAGE`), so the test now derives it and pins the new value; H2 spine parity, the shared parity check and the interface lock 5 passed; + `tools/ci_test_groups.py --verify` ok (engine-free-shared 145, engine-free-us 142, engine-free-uk + 163, engine-us 78, engine-uk 34, integration-uk 1); `ruff check` and `ruff format --check` clean + on the edited files. The rest is CI on the pushed head. +- Found by the targeted run: `tests/engine_free/shared/test_gate_battery_contract_pins.py` was a + zero-byte file on this branch since the #998 re-homing (the national-dispatch commit's 17-line edit + to the flat file had been replayed as a delete), so its 20 tests collected nothing on the pushed + heads and CI. Restored from main's 585-line file with the branch's own hunk (the dense-line mirror + reads `UK_ROWWISE_DENSE_POSTURE.gate_policy_suffix` instead of loading the retired tool by path); + 20 passed on this tree, no digest moved. From 4ed235c0cf4a06786dc78deab3515a77c8aa6f22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:07:42 +0100 Subject: [PATCH 35/44] Compile the historical validation target periods best-effort; the calibration year stays fail-closed Found by the licensed 10 % smoke run before merge: the full-build target compilation compiled the national register for {2023, 2025, calibration year} and refused when any period had an unmatched reference. The pinned Chronicle feed (5324aa2) carries no OBR facts at or before 2023, so every full build failed at uk.full.target_compilation with 'obr.income_tax ... did not match a Ledger fact at or before target period 2023'. Main's rowwise tool compiles the calibration year only and never saw this. The calibration year is still fail-closed. The historical validation periods (2023 and 2025 nationally, 2025 locally) are compiled when the feed supports them and otherwise skipped and recorded in register_completeness (validation_periods, validation_periods_unsupported), so the terminal surface and the certifier keep whatever registries the feed can still produce. Co-Authored-By: Claude Fable 5.1 --- .../build/uk_runtime/full_targets.py | 58 ++++++++++++++++--- .../engine_free/uk/test_uk_full_targets.py | 38 +++++++++++- 2 files changed, 87 insertions(+), 9 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py index 46e81f077..42fb775b4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py @@ -7,6 +7,7 @@ from __future__ import annotations +from collections.abc import Mapping from datetime import date from pathlib import Path from typing import Any @@ -53,6 +54,21 @@ def load_uk_local_chronicle_pin() -> dict[str, Any]: return dict(_LEDGER_FACT_FEED_PIN) +#: Historical target periods compiled beside the calibration year so the +#: terminal surface and the certifier can compare against them; skipped, not +#: fatal, when the pinned feed no longer carries their facts. +_VALIDATION_PERIODS: frozenset[int] = frozenset({2023, 2025}) +_LOCAL_VALIDATION_PERIODS: frozenset[int] = frozenset({2025}) + + +def _unsupported_names(unsupported: object) -> tuple[str, ...]: + names = [] + for item in unsupported: + name = item.get("name") if isinstance(item, Mapping) else None + names.append(str(name if name is not None else item)) + return tuple(names) + + def load_uk_full_target_inputs( facts_path: str | Path, *, @@ -112,25 +128,43 @@ def load_uk_full_target_inputs( raise ValueError("calibration_year must be a positive integer.") evaluated_on = exclusion_evaluation_date(exclusions_evaluated_on) crosswalk = load_uk_local_area_crosswalk() + # The calibration year is fail-closed. The historical validation periods + # are compiled when the pinned feed still carries their facts and skipped + # (recorded below) when it does not: a feed that has moved past a period + # must not block the build the way a missing calibration-year fact does. national_registries = {} local_registries = {} - for period in sorted({2023, 2025, year}): + unsupported_validation: dict[str, dict[int, tuple[str, ...]]] = { + "national": {}, + "local": {}, + } + for period in sorted({*_VALIDATION_PERIODS, year}): compilation = compile_uk_target_registry(artifact.facts, target_period=period) if compilation.unsupported: - raise ValueError( - f"UK national target references failed to compile for {period}: " - f"{compilation.unsupported!r}." + if period == year: + raise ValueError( + f"UK national target references failed to compile for {period}: " + f"{compilation.unsupported!r}." + ) + unsupported_validation["national"][period] = _unsupported_names( + compilation.unsupported ) + continue national_registries[period] = compilation.registry - for period in sorted({2025, year}): + for period in sorted({*_LOCAL_VALIDATION_PERIODS, year}): compilation = compile_uk_local_target_registry( artifact.facts, target_period=period, crosswalk=crosswalk ) if compilation.unsupported: - raise ValueError( - f"UK local target references failed to compile for {period}: " - f"{compilation.unsupported!r}." + if period == year: + raise ValueError( + f"UK local target references failed to compile for {period}: " + f"{compilation.unsupported!r}." + ) + unsupported_validation["local"][period] = _unsupported_names( + compilation.unsupported ) + continue local_registries[period] = compilation.registry band_edges = national_registries[year] frozen_version = None @@ -180,6 +214,14 @@ def load_uk_full_target_inputs( "band_edge_registry_reconciled": True, "compiled_local_reference_count": len(local_registries[year].specs), "local_registry_version": local_registries[year].version, + "validation_periods": { + "national": sorted(p for p in national_registries if p != year), + "local": sorted(p for p in local_registries if p != year), + }, + "validation_periods_unsupported": { + scope: {str(p): list(names) for p, names in sorted(periods.items())} + for scope, periods in unsupported_validation.items() + }, }, "uk_ledger_compiled_registries": national_registries, "uk_ledger_compiled_local_registries": local_registries, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py index 5a1c50a06..7e8237c9f 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py @@ -113,6 +113,14 @@ def test_full_inputs_preserve_band_edges_and_validation_periods(prepared): ] assert result["register_completeness"]["compiled_reference_count"] == 2 assert result["register_completeness"]["approved_reference_count"] == 1 + assert result["register_completeness"]["validation_periods"] == { + "national": [2023, 2025], + "local": [2025], + } + assert result["register_completeness"]["validation_periods_unsupported"] == { + "national": {}, + "local": {}, + } assert result["register_completeness"]["compiled_local_reference_count"] == 1 assert result["local_source_pin"]["fact_row_count"] == 1 assert result["reviewed_unbound_higher_targets"] == { @@ -245,5 +253,33 @@ def test_validation_reference_compilation_is_fail_closed(prepared, monkeypatch): registry=prepared[0], unsupported=({"target": "missing"},) ), ) - with pytest.raises(ValueError, match="failed to compile for 2023"): + # 2023 is a validation period and is skipped; the calibration year is not. + with pytest.raises(ValueError, match="failed to compile for 2024"): _load() + + +def test_historical_validation_period_without_facts_is_recorded_not_fatal( + prepared, monkeypatch +): + national, approved, local, artifact, calls = prepared + + def compile_national(facts, *, target_period): + calls.append(("national", target_period)) + if target_period == 2023: + return SimpleNamespace( + registry=None, + unsupported=( + {"name": "obr.income_tax", "period": 2023, "reason": "no fact"}, + ), + ) + return SimpleNamespace(registry=national, unsupported=()) + + monkeypatch.setattr(runtime, "compile_uk_target_registry", compile_national) + result = _load() + assert set(result["uk_ledger_compiled_registries"]) == {2024, 2025} + completeness = result["register_completeness"] + assert completeness["validation_periods"] == {"national": [2025], "local": [2025]} + assert completeness["validation_periods_unsupported"] == { + "national": {"2023": ["obr.income_tax"]}, + "local": {}, + } From eeabbc4000fc1daadba7a3f42e0559db7558ae20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:20:45 +0100 Subject: [PATCH 36/44] Carry the transferred gate's synthetic-smoke posture onto the sample admission (raw-spine path) Found by the licensed 10 % end-to-end run: with --spine-request the graph admits uk.full.sample through the transferred spine gate, and the admission check reads spine_gate_phase, spine_gate_release_candidate and spine_gate_synthetic_smoke from the node's parameters; the population composer set only the first two, so the first real spine-request build failed at uk.full.sample with KeyError('spine_gate_synthetic_smoke') after the whole spine had run. The assembled admission on the first post-BRMA stage already carried all three. Pinned by a composition test over the packaged spec for both postures. Co-Authored-By: Claude Fable 5.1 --- .../build/uk_runtime/graph_population.py | 1 + .../engine_free/uk/test_uk_graph_evidence.py | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py index 683092dc3..0fdf81761 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py @@ -394,6 +394,7 @@ def append_uk_population_nodes( spine_gate_params = { "spine_gate_phase": "transferred", "spine_gate_release_candidate": bool(gate.params["release_candidate"]), + "spine_gate_synthetic_smoke": bool(gate.params["synthetic_smoke"]), } nodes = list(graph.nodes) nodes.append( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py index cb7332b4e..56fcb900b 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py @@ -219,3 +219,37 @@ def __call__(self, frame): context.params["spine_gate_release_candidate"] = True with pytest.raises(ValueError, match="block downstream"): require_uk_spine_gate_admission(context) + + +@pytest.mark.parametrize("synthetic_smoke", [False, True]) +def test_sample_admission_carries_the_transferred_gate_posture(synthetic_smoke): + """The raw-spine path admits ``uk.full.sample`` through the transferred gate + with every parameter the admission check reads (found by the first licensed + end-to-end run: ``spine_gate_synthetic_smoke`` was missing there).""" + from microcosm.build.uk_runtime.graph_population import append_uk_population_nodes + + country = load_country_spec("uk") + spine = uk_spine_graph(country) + gated = add_uk_spine_gate_nodes( + spine, + spec=country, + engine_identity="test-engine", + synthetic_smoke=synthetic_smoke, + ) + graph = append_uk_population_nodes( + gated, + population=uk_spine_endpoint(spine).population, + time_period="2024", + weight_kind="importance", + sample_fraction=0.1, + n_clones=2, + ) + gate = graph.node("spine.gates.transferred") + sample = graph.node("uk.full.sample") + assert {edge.producer for edge in sample.artifact_inputs} >= {gate.id} + assert sample.params["spine_gate_phase"] == "transferred" + assert sample.params["spine_gate_release_candidate"] is bool( + gate.params["release_candidate"] + ) + assert sample.params["spine_gate_synthetic_smoke"] is synthetic_smoke + assert gate.params["synthetic_smoke"] is synthetic_smoke From 8588fa73aacf3d5867a91d00f4dfe1313ad978ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:39:37 +0100 Subject: [PATCH 37/44] Put the whole national register beside the local cells for cross-grain reconciliation, as the rowwise tool does Found by the licensed 10 % smoke run: the graph's joint-surface helper filtered the national register to country rows (a September design note that predates microcosm#906), so the region rows that parent constituency and local-authority legs were absent and the shared reconciliation refused at uk.full.target_compilation: 'national_age_0_9_vs_local_age_0_10: region_over_constituency has unparented lower-grain leg(s) [E12000001 ... E12000009]'. Main's tool passes the whole register and records the same group among its inconsistencies in force. The two helpers now match the tool's, with a unit test that a region row survives the joint surface. Co-Authored-By: Claude Fable 5.1 --- .../build/uk_runtime/full_problem.py | 22 +++++------- .../uk/test_uk_full_target_graph.py | 35 +++++++++++++++++++ 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_problem.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_problem.py index 067e2bd6f..13b3ae0ed 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_problem.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_problem.py @@ -13,7 +13,6 @@ uk_area_region_codes, uk_local_target_surface, ) -from microcosm.build.uk_runtime.ledger_targets import _spec_geography from microcosm.build.uk_runtime.local_rowwise import ( build_uk_rowwise_local_surface_matrix, empty_uk_local_problem, @@ -27,7 +26,6 @@ def _national_contract_target_ids(registry: TargetRegistry) -> tuple[str, ...]: { str(spec.metadata.get("contract_target_id", spec.name)) for spec in registry.specs - if _spec_geography(spec)[0] == "country" } ) ) @@ -37,22 +35,18 @@ def _joint_surface_registry( local_registry: TargetRegistry, national_registry: TargetRegistry, ) -> TargetRegistry: - """Put country controls beside local cells for declared reconciliation. + """Put national controls beside local cells for cross-grain reconciliation. - Regional constraints stay in the national solve registry. They are outside - the country/constituency/LA reconciliation rule and cannot be passed as - country controls or silently assigned a new reconciliation policy. + The whole national register goes in, as the rowwise tool passes it: the + country rows parent the regions and, since microcosm#906 activated the + region grain, the region rows are the controls that parent constituency + and local-authority legs (``region_over_constituency``, ``region_over_la``). + Filtering to country rows leaves those legs unparented and the surface + refuses (found by the first licensed graph build). """ return TargetRegistry( - [ - *local_registry.specs, - *( - spec - for spec in national_registry.specs - if _spec_geography(spec)[0] == "country" - ), - ], + [*local_registry.specs, *national_registry.specs], country="uk", ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_target_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_target_graph.py index 5c6d58cd1..a09132880 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_target_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_target_graph.py @@ -96,3 +96,38 @@ def test_target_kernel_identity_includes_reconciliation_and_ladder_diagnostics( graph_targets.UKFullTargetCompilationKernel().implementation_hash() assert graph_targets.cross_grain in hashed assert graph_targets.ladder_targets in hashed + + +def test_joint_surface_keeps_region_rows_as_cross_grain_controls(): + """The joint surface carries the whole national register, region rows + included, as main's rowwise tool does; a country-only filter left the + region_over_constituency legs unparented on the first licensed graph build.""" + from microcosm.build.uk_runtime import full_problem + from microcosm.calibrate import TargetRegistry, TargetSpec + + def spec(name, level, geography_id): + return TargetSpec( + name=name, + entity="person", + value=1.0, + measure="age", + period=2025, + source="test", + family="age", + metadata={ + "contract_target_id": name, + "geography_level": level, + "geography_id": geography_id, + }, + ) + + national = TargetRegistry( + [spec("age_uk", "country", "K02000001"), spec("age_ne", "region", "E12000001")], + country="uk", + ) + local = TargetRegistry( + [spec("age_pcon", "constituency", "E14000530")], country="uk" + ) + joint = full_problem._joint_surface_registry(local, national) + assert [s.name for s in joint.specs] == ["age_pcon", "age_uk", "age_ne"] + assert full_problem._national_contract_target_ids(national) == ("age_ne", "age_uk") From ba2ab24bdd49840c9fd6364a3989523e71a8b435 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:00:29 +0100 Subject: [PATCH 38/44] Decode the stored local surface's hierarchy column before the problem assembly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by the licensed 10 % smoke run: the local target surface crosses the target_compilation → problem node boundary as JSON, so each spec's schema-8 CalibrationHierarchy comes back as a mapping; the problem assembly hands the row's hierarchy to Target, which refuses it ('Target hmrc.self_employment_income.amount@E14001063: hierarchy must be a CalibrationHierarchy or None'). _surface_records already encoded the column the way TargetSpec.to_dict does; _surface_frame now decodes it the way TargetSpec.from_dict does. Main's rowwise tool never serialises the frame. Pinned by a JSON round-trip test with a fixture hierarchy. Co-Authored-By: Claude Fable 5.1 --- .../build/uk_runtime/graph_targets.py | 23 ++++++++++- .../uk/test_uk_full_target_graph.py | 39 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py index 240b6b66a..f52ae4104 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py @@ -134,6 +134,27 @@ def encode(key, value): ] +def _surface_frame(payload: Mapping) -> pd.DataFrame: + """Rebuild the stored local surface, decoding what ``_surface_records`` encoded. + + The frame crosses a graph node boundary as JSON, so a spec's + ``CalibrationHierarchy`` comes back as a mapping; the problem assembly + hands each row's hierarchy to ``Target``, which refuses a mapping (found by + the first licensed graph build). Main's rowwise tool never serialises the + frame and never saw this. + """ + + surface = pd.DataFrame(payload["surface"], columns=payload["surface_columns"]) + if "hierarchy" in surface.columns: + surface["hierarchy"] = [ + CalibrationHierarchy.from_dict(dict(value)) + if isinstance(value, Mapping) + else value + for value in surface["hierarchy"] + ] + return surface + + def _local_specs(surface: pd.DataFrame) -> list[TargetSpec]: return [ TargetSpec( @@ -470,7 +491,7 @@ def reconstruct_uk_full_problem_inputs(context: KernelContext) -> UKFullProblemI grain: pd.DataFrame(arrays[f"metrics_{grain}"], columns=columns, index=ids) for grain, columns in metadata["grains"].items() } - surface = pd.DataFrame(full["surface"], columns=full["surface_columns"]) + surface = _surface_frame(full) surface = _selected_local_surface(surface, local_specs) ladder = load_uk_oa_ladder(context.sources["uk_ladder"]) _, local_problem, cross, bound_families, rung = build_uk_full_local_problem( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_target_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_target_graph.py index a09132880..6fb091520 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_target_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_target_graph.py @@ -131,3 +131,42 @@ def spec(name, level, geography_id): joint = full_problem._joint_surface_registry(local, national) assert [s.name for s in joint.specs] == ["age_pcon", "age_uk", "age_ne"] assert full_problem._national_contract_target_ids(national) == ("age_ne", "age_uk") + + +def test_stored_surface_round_trip_restores_the_hierarchy(): + """The local surface crosses the node boundary as JSON; its schema-8 + hierarchy must come back as a CalibrationHierarchy, or the problem assembly + refuses every hierarchy-bearing target (first licensed graph build).""" + import json + + import pandas as pd + + from microcosm.build.uk_runtime import graph_targets + from microcosm.calibrate.hierarchy import CalibrationHierarchy + from test_support.microcosm_build.uk_hierarchy_fixtures import uk_fixture_hierarchy + + name = "hmrc.self_employment_income.amount@E14001063" + hierarchy = uk_fixture_hierarchy( + name, level="constituency", geography_id="E14001063" + ) + frame = pd.DataFrame( + [ + {"target_name": name, "value": 1.5, "hierarchy": hierarchy}, + {"target_name": "plain@E14001063", "value": 2.0, "hierarchy": None}, + ] + ) + payload = json.loads( + json.dumps( + { + "surface": graph_targets._surface_records(frame), + "surface_columns": frame.columns.tolist(), + } + ) + ) + assert isinstance(payload["surface"][0]["hierarchy"], dict) + restored = graph_targets._surface_frame(payload) + assert restored.columns.tolist() == frame.columns.tolist() + assert isinstance(restored["hierarchy"][0], CalibrationHierarchy) + assert restored["hierarchy"][0] == hierarchy + assert restored["hierarchy"][1] is None + assert restored["value"].tolist() == [1.5, 2.0] From 3bd6f75ae42c1430bca96aa6cb4742c790f044a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:25:17 +0100 Subject: [PATCH 39/44] Hand the coverage engine to the full gate batteries, as the release-cut producer does Found by the licensed 10 % smoke run: the graph driver registered its gate kernels with the spine's PolicyEngineUKEngine as coverage_engine, whose variables() enumerates the frame's non-computed columns; the release input-coverage gate checks the coverage manifest against the engine's loadable inputs, so sixteen live inputs (employment_income, capital_gains, property_wealth, marital_status, ...) were reported as manifest drift and the preflight battery stopped the build. tools/certify_uk_release_cut.py hands the same batteries PolicyEngineUKCoverageEngine(); the driver now does too, and keeps the spine engine's provenance identity. On the spine-request path the prepared spine engine is no longer bound, only its identity. Co-Authored-By: Claude Fable 5.1 --- .../microcosm/build/uk_runtime/full_build_cli.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py index 6e53a1eb1..625062177 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py @@ -504,7 +504,7 @@ def prepare_full_build( time_period = str(frame.metadata["time_period"]) source_fraction = float((sidecar.get("sampling") or {}).get("fraction", 1.0)) stages = tuple(sidecar["stages"]) - engine = _rules_engine() + _rules_engine() # the uk extra must be installed; the identity is provenance engine_identity = hashlib.sha256( canonical_json(_rules_engine_provenance()) ).hexdigest() @@ -542,7 +542,7 @@ def prepare_full_build( time_period = prepared.frs_release.time_period source_fraction = raw.sample_fraction stages = prepared.stage_names - engine, engine_identity = prepared.engine, prepared.engine_identity + engine_identity = prepared.engine_identity inputs["dataset"] = { "path": None, "sha256": None, @@ -701,8 +701,17 @@ def prepare_full_build( kernels, progress_callback=None if args.dry_run else _solve_observer(args, telemetry), ) + # The gate batteries take the coverage engine, as the release-cut producer + # hands them (tools/certify_uk_release_cut.py): its variables() enumerates + # the loadable inputs the coverage manifest is checked against, whereas the + # spine adapter's enumerates the frame's non-computed columns (found by the + # first licensed graph build: sixteen live inputs reported as drift). + from .release_input_coverage import PolicyEngineUKCoverageEngine + register_uk_full_gate_kernels( - kernels, coverage_engine=engine, engine_identity=engine_identity + kernels, + coverage_engine=PolicyEngineUKCoverageEngine(), + engine_identity=engine_identity, ) register_uk_terminal_kernels(kernels) register_uk_full_certification_kernel(kernels) From 1de4aa17bd1fc517e6961d6a019a8c4e299ab957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:25:17 +0100 Subject: [PATCH 40/44] Keep a partial registry for a validation period the feed no longer fully compiles The preflight battery declares uk_ledger_compile_parity_production_2023, which reads the compiled 2023 registry; skipping the period (the previous change) left the gate with no registry and a KeyError. The release-cut producer keeps compilation.registry for every parity period whatever its unsupported list says; load_uk_full_target_inputs now does the same for the validation periods, still recording the unsupported references and still refusing on the calibration year. Co-Authored-By: Claude Fable 5.1 --- .../src/microcosm/build/uk_runtime/full_targets.py | 9 ++++----- .../tests/engine_free/uk/test_uk_full_targets.py | 13 ++++++++++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py index 42fb775b4..afff50fff 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py @@ -129,9 +129,10 @@ def load_uk_full_target_inputs( evaluated_on = exclusion_evaluation_date(exclusions_evaluated_on) crosswalk = load_uk_local_area_crosswalk() # The calibration year is fail-closed. The historical validation periods - # are compiled when the pinned feed still carries their facts and skipped - # (recorded below) when it does not: a feed that has moved past a period - # must not block the build the way a missing calibration-year fact does. + # keep whatever the pinned feed still compiles, as the release-cut producer + # does (tools/certify_uk_release_cut.py keeps ``compilation.registry`` for + # every parity period): the references the feed no longer carries are + # recorded below instead of blocking the build. national_registries = {} local_registries = {} unsupported_validation: dict[str, dict[int, tuple[str, ...]]] = { @@ -149,7 +150,6 @@ def load_uk_full_target_inputs( unsupported_validation["national"][period] = _unsupported_names( compilation.unsupported ) - continue national_registries[period] = compilation.registry for period in sorted({*_LOCAL_VALIDATION_PERIODS, year}): compilation = compile_uk_local_target_registry( @@ -164,7 +164,6 @@ def load_uk_full_target_inputs( unsupported_validation["local"][period] = _unsupported_names( compilation.unsupported ) - continue local_registries[period] = compilation.registry band_edges = national_registries[year] frozen_version = None diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py index 7e8237c9f..075132640 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py @@ -262,12 +262,13 @@ def test_historical_validation_period_without_facts_is_recorded_not_fatal( prepared, monkeypatch ): national, approved, local, artifact, calls = prepared + partial = _registry("retained") def compile_national(facts, *, target_period): calls.append(("national", target_period)) if target_period == 2023: return SimpleNamespace( - registry=None, + registry=partial, unsupported=( {"name": "obr.income_tax", "period": 2023, "reason": "no fact"}, ), @@ -276,9 +277,15 @@ def compile_national(facts, *, target_period): monkeypatch.setattr(runtime, "compile_uk_target_registry", compile_national) result = _load() - assert set(result["uk_ledger_compiled_registries"]) == {2024, 2025} + # The partial 2023 registry stays available to the parity gates, as the + # release-cut producer keeps it; the missing references are recorded. + assert set(result["uk_ledger_compiled_registries"]) == {2023, 2024, 2025} + assert result["uk_ledger_compiled_registries"][2023] is partial completeness = result["register_completeness"] - assert completeness["validation_periods"] == {"national": [2025], "local": [2025]} + assert completeness["validation_periods"] == { + "national": [2023, 2025], + "local": [2025], + } assert completeness["validation_periods_unsupported"] == { "national": {"2023": ["obr.income_tax"]}, "local": {}, From 85525577e4f8ef23b4aff51f566a6a117c9e2196 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:51:26 +0100 Subject: [PATCH 41/44] Declare the rotated holdout like the dense solve it rotates Found by the licensed 10 % smoke run, the first time the calibration segment ran on real data: the holdout kernel alone declared Numeric.PLATFORM_BITWISE while the dense-solve kernels (same solver, same dependencies) use the default class, so the calibrated gate battery and the certification node, both default-class consumers of the holdout report, were refused by amendment 19 ('A platform_bitwise artifact requires a platform_bitwise consumer'). The holdout now declares what the dense solve declares. A test walks every typed edge of the composed full graph on the synthetic spec and requires a compatible consumer, so a future declaration change is caught before a licensed build. Co-Authored-By: Claude Fable 5.1 --- .../build/uk_runtime/graph_terminal.py | 6 +++- .../engine_free/uk/test_uk_full_build_cli.py | 28 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py index 5cc674348..66ca8d152 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py @@ -944,9 +944,13 @@ def register_uk_full_gate_kernels( class UKFullHoldoutKernel(KernelBase): ref = "uk.full.rotated-holdout@1" + # Declared like the dense solve it rotates (graph_calibration's kernels: + # the default numeric class with the same solver dependencies). Declaring + # platform-bitwise here made the holdout report unreadable by the + # calibrated gate battery and the certification node, which claim the + # default class (found by the first licensed graph build). capabilities = Capabilities( Determinism.DETERMINISTIC, - numeric=Numeric.PLATFORM_BITWISE, seed_source=SeedSource.PARAM, dependencies=("policyengine-uk", "torch"), ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py index 0cfe082b1..11e175e79 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py @@ -1179,3 +1179,31 @@ def test_multi_block_engine_run_is_never_releasable(tmp_path, monkeypatch): assert posture["full_rung"] is True assert posture["single_block_engine"] is False assert posture["release_blocking_gates_passed"] is True + + +def test_every_typed_edge_of_the_full_graph_has_a_compatible_consumer(tmp_path): + """Amendment 19 refuses a typed artifact whose consumer claims a stronger + numeric class than its producer; the first licensed graph build found the + calibrated gate battery reading a platform-bitwise holdout report. Every + typed edge of the composed graph is checked here, on the synthetic spec.""" + from microcosm.graph.artifact_edges import numeric_scope, require_compatible_scope + + build = prepared(tmp_path) + graph, kernels = build.full.graph, build.kernels + incompatible = [] + typed_edges = 0 + for node in graph.nodes: + consumer = kernels.get(node.kernel).capabilities + for binding in node.artifact_inputs: + typed_edges += 1 + producer = kernels.get(graph.node(binding.producer).kernel).capabilities + try: + require_compatible_scope(numeric_scope(producer), consumer) + except Exception as error: # noqa: BLE001 - the message is the finding + incompatible.append( + (node.id, binding.producer, binding.name, str(error)) + ) + producers = {b.producer for n in graph.nodes for b in n.artifact_inputs} + assert {"uk.full.holdout", "uk.full.gates.calibrated"} <= producers + assert typed_edges > 0 + assert incompatible == [] From 4d80213188d6e30380f9f69ea293ada11efb93c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:19:20 +0100 Subject: [PATCH 42/44] Partition the calibration diagnostics against the registry that entered the solve Found by the licensed 10 % smoke run, the first dense solve on real data through the graph: below the f100 rung the problem drops cells unreachable in the sample (the rowwise tool's rung surface), so the selected registry is wider than the solved one; the calibrated gate battery handed the selected registry to the diagnostics, whose partition check found the local-authority UC rows neither compiled nor skipped and refused. The rowwise tool builds its diagnostics registry from the solve's own targets; the battery now keeps the selected specs that entered the problem, by (name, period), so the metadata and hierarchies the schema-8 diagnostics need stay attached. The failure had surfaced as a bare KeyError on the missing diagnostics artifact, because a GATE kernel keeps an exception inside its receipt and completes with no artifacts; the driver now names the node and the recorded exception before it reads the terminal artifacts. Both pinned by tests. Co-Authored-By: Claude Fable 5.1 --- .../build/uk_runtime/full_build_cli.py | 21 ++++++++++++ .../build/uk_runtime/graph_terminal.py | 20 ++++++++++- .../engine_free/uk/test_uk_full_build_cli.py | 33 +++++++++++++++++++ .../engine_free/uk/test_uk_graph_terminal.py | 21 +++++++++++- 4 files changed, 93 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py index 625062177..2c1eec2ef 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py @@ -463,6 +463,26 @@ def observer(event: dict[str, object]) -> None: return observer +def _require_gate_kernel_completed(manifest, node_id: str) -> None: + """Surface a gate battery that recorded an exception instead of a report. + + A GATE kernel preserves a failure inside its receipt rather than raising, + so the executor completes the node with no artifacts; the driver then met + the missing artifact as a bare KeyError (first licensed graph build). Name + the node and the recorded exception instead. + """ + + receipt = dict(manifest.nodes[node_id].receipt or {}) + evidence = receipt.get("evidence") + if receipt.get("outcome") == "fail" and isinstance(evidence, dict): + exception_type = evidence.get("exception_type") + if exception_type: + raise RuntimeError( + f"{node_id} failed inside the gate battery: {exception_type}: " + f"{evidence.get('message')}" + ) + + def prepare_full_build( args: argparse.Namespace, *, telemetry=None, attempt: dict | None = None ) -> PreparedUKFullBuild: @@ -1165,6 +1185,7 @@ def _execute_full_build( ) _persist_checkpoint(manifest, store, args, "numerical") _materialize_evidence(manifest, store, args.out) + _require_gate_kernel_completed(manifest, "uk.full.gates.calibrated") terminal_files = materialize_uk_terminal_artifacts( manifest, store, directory=args.out, stem=stem ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py index 66ca8d152..01191ac96 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py @@ -20,6 +20,7 @@ import numpy as np import pandas as pd +from microcosm.calibrate import TargetRegistry from microcosm.frame import Frame, engine_tables from microcosm.graph import ( ArtifactInput, @@ -733,6 +734,23 @@ def run(self, context: KernelContext) -> KernelResult: selected_registry = registry_from_payload( json.loads(context.artifacts["selection"].payload)["registry"] ) + # The diagnostics partition the registry that entered the solve, as + # the rowwise tool's diagnostics registry does: below the f100 rung + # the problem drops cells unreachable in the sample, so the selected + # registry is wider than the solved one (found by the first licensed + # graph build: the local-authority UC rows were neither compiled nor + # skipped and the battery refused). + solved = { + (target.name, target.period) for target in problem.problem.targets + } + target_registry = TargetRegistry( + [ + spec + for spec in selected_registry.specs + if (spec.name, spec.period) in solved + ], + country="uk", + ) holdout = json.loads(context.artifacts["holdout"].payload) complete_diagnostics = uk_calibration_diagnostics_payload( result, @@ -743,7 +761,7 @@ def run(self, context: KernelContext) -> KernelResult: problem.problem.targets, problem.target_metadata, strict=True ) }, - target_registry=selected_registry, + target_registry=target_registry, local_area_support=support_frame, rotated_holdout=holdout, build={ diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py index 11e175e79..feaf0b6d7 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py @@ -1207,3 +1207,36 @@ def test_every_typed_edge_of_the_full_graph_has_a_compatible_consumer(tmp_path): assert {"uk.full.holdout", "uk.full.gates.calibrated"} <= producers assert typed_edges > 0 assert incompatible == [] + + +def test_gate_battery_recorded_exception_is_named_not_a_missing_artifact(): + """A GATE kernel keeps a failure inside its receipt and completes with no + artifacts; the driver names the node and the exception (first licensed + graph build: a bare KeyError on the missing diagnostics artifact).""" + from types import SimpleNamespace + + import pytest + + failed = SimpleNamespace( + nodes={ + "uk.full.gates.calibrated": SimpleNamespace( + receipt={ + "outcome": "fail", + "evidence": { + "exception_type": "ValueError", + "message": "registry must exactly partition", + }, + } + ) + } + ) + with pytest.raises(RuntimeError, match="uk.full.gates.calibrated failed inside"): + cli._require_gate_kernel_completed(failed, "uk.full.gates.calibrated") + passed = SimpleNamespace( + nodes={ + "uk.full.gates.calibrated": SimpleNamespace( + receipt={"outcome": "fail", "evidence": {"blocking": ["a_gate"]}} + ) + } + ) + cli._require_gate_kernel_completed(passed, "uk.full.gates.calibrated") diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py index bcdba256e..8d2a39b33 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py @@ -507,6 +507,20 @@ def test_final_gate_kernel_owns_complete_diagnostics_and_reuses_decoded_result( }, ), ] + unsolved_spec = replace( + specs[1], + name="dropped_local", + metadata={ + **specs[1].metadata, + "geography_level": "la", + "geography_id": "E07000002", + "area_type": "la", + "area_code": "E07000002", + }, + hierarchy=uk_fixture_hierarchy( + "dropped_local", level="la", geography_id="E07000002" + ), + ) targets = TargetSet( [ replace(specs[0].to_target(), measure=lambda f: np.ones(2)), @@ -560,7 +574,12 @@ def artifact(payload, raw=False): "selection": artifact( { "receipt": selection, - "registry": registry_payload(TargetRegistry(specs, country="uk")), + # Below the f100 rung the problem drops cells unreachable in the + # sample, so the selected registry is wider than the solved one; + # the battery partitions the registry that entered the solve. + "registry": registry_payload( + TargetRegistry([*specs, unsolved_spec], country="uk") + ), } ), "preflight": artifact(preflight_payload(selection=selection), True), From 01bda8e0a9e45750e5f21fde6f3c316911b4d8aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:16:52 +0100 Subject: [PATCH 43/44] Receipts R7: the licensed 10 % dense smoke run before merge Method, settings, the four rungs (main's tool, the graph on the same spine, the graph end to end, the graph with the measurement-only export exception), the nine defects the run surfaced with their fixes, the exact parity of the solve, the local gates, the holdout and the H5 payload, and the full-run proxy. Co-Authored-By: Claude Fable 5.1 --- experiments/901-uk-main-rebase-receipts.md | 103 +++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/experiments/901-uk-main-rebase-receipts.md b/experiments/901-uk-main-rebase-receipts.md index 0d0be4b24..3f995ce2c 100644 --- a/experiments/901-uk-main-rebase-receipts.md +++ b/experiments/901-uk-main-rebase-receipts.md @@ -260,3 +260,106 @@ the first run's output directory; the helper now parses through the real parser heads and CI. Restored from main's 585-line file with the branch's own hunk (the dense-line mirror reads `UK_ROWWISE_DENSE_POSTURE.gate_policy_suffix` instead of loading the retired tool by path); 20 passed on this tree, no digest moved. + +## R7. Licensed 10 % dense smoke run before merge (2026-09-28, María's ask) + +Question put: has a dense licensed run been made end to end, and what does a full dense run cost +on the graph driver? Neither had been done (R4). Method: two measurement-only worktrees at main +937aca4ec (`repos/populace-main-1012`, main's tool) and at this branch (`repos/populace-901-measure`), +both carrying the #1014 lane's PLAN_5 relaxation patch uncommitted so a spine can be built at all; +harness and outputs under `data/ukds/acceptance/901-dense-10pct/` (`run_dense_10pct.sh`, the chain +scripts, `phase_timeline.py`, `compare_ab.sh`). Smoke settings ruled by María: K=2 clones, 250 +epochs, `--sample-fraction 0.1 --sample-seed 7`, seed 42, staging local-only (phase timings, no +upload); never a release. Reference: the runbook's 3.5 h / 10 GB at K=15, 1,500 epochs on the +rowwise tool; the graph driver had never been timed. + +- Spine A (main's tool, full data, no sampling): 413 s wall, 9.7 GB peak, 26/26 gates passed under + the measurement patch, H5 188 MB (`spine-a/`). +- A = main's tool on spine-a (`tool-10pct/`): exit 0, 581 s wall, 7.4 GB peak. f010 rung: 5,278 + households → K=2 → 10,556 rows, 18,729 targets; 250 epochs, loss 0.699 → 0.354; 4 of 6 local + gates fail (area support, per-family fit, target fit, weight ratio), `releasable: false`. + Phases: target_compilation 494 s, cloning 4 s, surface_resolution 10 s, calibration 10 s, + gate_battery 1 s, holdout 39 s, output_bundle 10 s. +- B = the graph driver on spine-a, same flags: exit 1 by the calibrated battery's verdict, 1,086 s wall, 7.7 GB peak (`graph-10pct/`). Every + node through the calibrated gate battery ran: the same f010 sample (10,556 rows, 18,729 targets), + the dense solve, the rotated holdout, the calibrated population and the 26-gate calibrated battery; + the battery classed nine national/source checks as structural stops (as the full-build enforcement + rule declares below f100: only the five local fit/support/weight checks are excused), so no export, + package or certification node ran. Phases: target_compilation 997 s (the graph's node also carries + the surface, the cross-grain reconciliation, the measures, the problem, the geography gate and the + preflight battery, which the tool splits over compile 494 s + cloning 4 s + surface 10 s), + calibration 76 s (dense 15.5 s, holdout 38.5 s, calibrated 2.2 s, battery 13.3 s; the tool: 10 s + solve + 39 s holdout + 1 s battery + 10 s bundle). +- B2 = the graph driver end to end (`--spine-request`, the graph builds the spine): exit 1 by the same battery verdict, 1,445 s wall, 9.9 GB peak (`graph-spine-request-10pct/`). The graph + built the 34-stage spine itself with the two gate nodes (assembled after `frs_brma`, transferred at the + endpoint): the 26 spine gates carry the same ids and the same outcomes as main's tool report on + spine-a (all passed), the same `gates_manifest_sha256`; the sampled pool and the solve are bit-identical to + the input-H5 rung and to main's tool (final loss 0.3535366112843803 again), which is the full-H5 + spine parity R1 left owed, established through the solve rather than a payload compare (the graph + materialises no spine H5 below the export). Node wall times: the 50 spine nodes sum to 296 s against + the tool's 413 s spine build; `uk.full.target_compilation` 860 s; preflight battery 72 s; measures + 12.6 s; problem 7.3 s; dense 15.3 s; holdout 39.5 s; calibrated battery 14.4 s. +- Parity A vs B: bit-identical where both sides produce the same object: initial loss 0.6991022825241089 and final + loss 0.3535366112843803 on both drivers, 18,729 target rows, 10,556 non-zero households, the same six + local gate outcomes (four failed, two passed), the same rotated holdout (mean 0.7248643006468352, worst + 0.7320742950761016, five folds). The H5 payload compare waits on the export rung below. +- B3 = the graph driver on spine-a with a measurement-only export exception (below f100 the national + checks are recorded without enforcement, as the tool's sampled rungs behave; `export-exception-measurement.patch`, + applied in the measurement worktree only): exit 0, 1,143 s wall, 8.6 GB peak (`graph-10pct-export/`). The whole + line ran: export, readback, package, certification readiness (`ready_for_external_review: false` at f010, as it + must be), the schema-4 manifest, `sha256sums.txt`, the local bundle. H5 payload against A + (`tools/compare_uk_h5_payload.py`, receipt `ab_payload_compare.json`): same 40,238,817 bytes, keys equal, + `person`, `benunit` and `time_period` payload-identical, `household` identical in rows, column order and + every value, with eleven geography code columns stored as pandas `string` on the graph against `str` on + the tool (the file digests differ for that alone); the six local gates agree. +- Full-run proxy: the solve path is byte-identical (same losses to the last digit at every rung), so the per-epoch + and per-row costs are the tool's; the graph adds a fixed cost per run, at this rung about 500 s + (target_compilation 860–997 s against the tool's 494 s compile + 14 s cloning and surface, of + which the historical validation-period registry compile and the preflight battery are the + identifiable parts) plus about 15 s across the calibration segment. Against the runbook's + 3.5 h / 10 GB reference at K=15 and 1,500 epochs on the tool, a full graph run projects to about + 3.7 h; peak memory 7.7 GB (input-H5) and 9.9 GB (spine-request) at this rung against the tool's + 7.4 GB. Not exploding; the fixed overhead is the target-compilation node, which is the one place + worth profiling before a full run. + +Defects the run surfaced, none visible to the unit suites or CI: + +1. Main (#971, merged 2026-09-23, not this branch): every real full-UK ladder is refused at the + locations step because `draw_uk_ladder_locations` compares the ladder's composite + local-authority vintage (`ew:2023_april_lad;scotland:2019_council_area;ni:2014_lgd`, the only + vintage the artifact tool writes) with the EW-only names-resource constant; the fixture ladders + carry the plain string, so the tests pass. Both drivers hit it. Measured with a second + uncommitted patch (`la-vintage-measurement.patch`) in both trees; fix drafted for main + (`repos/uk-971-ladder-vintage-defect-draft.md`). +2. This branch, fixed (1214fb616): `full_targets` compiled the national register for {2023, 2025, + calibration year} fail-closed; the pinned feed carries OBR facts from fiscal 2024 onward only, + so the build refused at `uk.full.target_compilation`. Historical validation periods are now + best-effort and recorded in `register_completeness`; the calibration year stays fail-closed. +3. This branch, fixed (265f8acf4): on the raw-spine path the sample node's transferred-gate + admission lacked `spine_gate_synthetic_smoke`, so the first end-to-end build failed at + `uk.full.sample` after the whole spine had run. +4. This branch, fixed (cd6829a71): the joint-surface helpers filtered the national register to + country rows (pre-#906), leaving the region legs unparented in the cross-grain reconciliation; + they now pass the whole register as the rowwise tool does. +5. This branch, fixed (da9b2472a): the local surface crosses the target_compilation → problem + boundary as JSON, so each spec's schema-8 hierarchy came back as a mapping and the problem + assembly refused it; the stored surface is now decoded the way ``TargetSpec.from_dict`` does. +6. This branch, fixed (f9562fc01): the driver handed the gate batteries the spine adapter as the + coverage engine, whose ``variables()`` enumerates the frame's non-computed columns, so the + release input-coverage gate read sixteen live inputs as manifest drift; the batteries now get + ``PolicyEngineUKCoverageEngine`` as the release-cut producer hands it. +7. This branch, fixed (7003aaee3): the preflight parity gate for production 2023 reads the 2023 + registry, which fix 2 had skipped; a validation period now keeps its partial registry, as the + release-cut producer does, with the unsupported references recorded. +8. This branch, fixed (86137886d): the calibration segment ran for the first time on real data and the + executor refused the holdout report at the calibrated gate battery and the certification node + (amendment 19: a platform-bitwise artifact needs a platform-bitwise consumer); the holdout kernel + alone declared that class while the dense solve it rotates declares the default; it now matches, + and a test walks every typed edge of the composed graph on the synthetic spec. +9. This branch, fixed (055591b54): the dense solve ran (250 epochs, loss 0.35355 against main's + 0.35354 on the same spine), the holdout and the calibrated population followed, and the + calibrated battery refused in its receipt: it handed the diagnostics the selected registry, wider + than the solved one below f100 (the rung drops cells unreachable in the sample), so the + local-authority UC rows were neither compiled nor skipped; it now partitions the specs that entered + the solve, as the rowwise tool's diagnostics registry does, and the driver names a battery's recorded + exception instead of tripping on the missing artifact. From c9ce92b43597b997df64e899452917fc15ba905c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:34:35 +0100 Subject: [PATCH 44/44] Follow microcosm#1007's typed calibration diagnostics in the graph terminal tests Rebase onto main 4b57d15a2 (#1007 unified calibration diagnostics, #1041, #1046). The calibrated gate battery now hands a typed rotated-holdout report to the diagnostics, so the kernel test's holdout fixture is a complete UKMeasuredRotatedHoldout; the synthetic graph driver's fake diagnostics declare the legacy schema, like the rowwise-tool candidate fixture in the assembler tests, because the assembler validates only the current schema, which the real gate kernel emits. The lock pin is re-recorded on the merged lock (696ea49c...), inside the graph-registration commit. Co-Authored-By: Claude Fable 5.1 --- .../engine_free/uk/test_uk_graph_terminal.py | 32 +++++++++++++++++-- .../microcosm_build/uk_full_build_cli.py | 6 +++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py index 8d2a39b33..0a629c0a0 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py @@ -595,7 +595,34 @@ def artifact(payload, raw=False): "uk_ledger_compiled_local_registries": {"2025": empty}, } ), - "holdout": artifact({"report_only": True, "outcome": "fixture"}), + "holdout": artifact( + { + "report_only": True, + "method": "rotated_folds", + "target_loss_cap": 10.0, + "loss_weight_scale": "held_local_grains_only", + "target_weight_rule": "grain_equal", + "population": "held_out_local_targets", + "grains": ["constituency", "local_authority"], + "n_folds": 2, + "seed": 20260529, + "solve_seed": 42, + "mean_holdout_loss": 0.1, + "worst_holdout_loss": 0.15, + "fold_losses": [0.05, 0.15], + "folds": [ + { + "fold": fold, + "n_train_targets": 1, + "n_holdout_targets": 1, + "holdout_target_indices": [fold], + "training_national_rows": 1, + "holdout_loss": loss, + } + for fold, loss in enumerate([0.05, 0.15]) + ], + } + ), } monkeypatch.setattr( geography_ladder, @@ -645,7 +672,8 @@ def forbidden(*args, **kwargs): phase, _ = decode_full_gate_report(stored.artifacts["gate_report"]) assert phase.phase == "terminal" document = json.loads(stored.artifacts["calibration_diagnostics"]) - assert document["uk_diagnostics"]["rotated_holdout"]["outcome"] == "fixture" + assert document["uk_diagnostics"]["rotated_holdout"]["method"] == "rotated_folds" + assert document["uk_diagnostics"]["rotated_holdout"]["worst_holdout_loss"] == 0.15 assert len(document["targets"]) == 2 assert ( len(pd.read_csv(__import__("io").BytesIO(stored.artifacts["area_support_csv"]))) diff --git a/test_support/microcosm_build/uk_full_build_cli.py b/test_support/microcosm_build/uk_full_build_cli.py index 81a4483d8..f2c473230 100644 --- a/test_support/microcosm_build/uk_full_build_cli.py +++ b/test_support/microcosm_build/uk_full_build_cli.py @@ -365,9 +365,13 @@ def surface_payload() -> bytes: def diagnostics_payload() -> bytes: + # A legacy-schema stand-in, like the rowwise-tool candidate fixture in the + # assembler tests: the assembler validates only the current schema + # (microcosm#1007), which the real gate kernel emits; this fake drives the + # driver's plumbing, not the diagnostics contract. return canonical_json( { - "schema_version": 8, + "schema_version": 6, "n_records": 2, "n_nonzero": 2, "initial_loss": 0.5,