From 90d6f449490446b56224cfa9944651d66d9d3f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 18 Sep 2026 18:41:40 +0100 Subject: [PATCH 01/22] Sign the v20 self-employment 20-30k deferral on the target-fit register (#796) The 2026-09-18 national line on main ce76b358 blocks on one cell: hmrc/self_employment_income_income_band_20_000_to_30_000@2025 at +25.34% vs the 25% fence. Receipts show a calibration-competition regression, not a data defect: spine-s improved the cell's raw error (+10.6% -> +6.4%), the SPI target is unchanged, and the 638-target surface (the #906 region fan-out plus the #939 CGT bands) pushed the band out where v18's 364-target solve left it at +10.5%. Signed by Maria for four weeks; the committed-register tests evaluate at the entry's window. Co-Authored-By: Claude Fable 5 --- .../uk/target_fit_reviewed_exclusions.json | 10 +++++++++- .../tests/test_uk_terminal_gates.py | 19 +++++++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk/target_fit_reviewed_exclusions.json b/packages/microcosm-build/src/microcosm/build/uk/target_fit_reviewed_exclusions.json index 1e9d92c1d..590e8e76c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/target_fit_reviewed_exclusions.json +++ b/packages/microcosm-build/src/microcosm/build/uk/target_fit_reviewed_exclusions.json @@ -1,5 +1,13 @@ { "schema_version": 2, "description": "Reviewed target-fit exclusions for the UK terminal battery (schema 2 per #610). Each entry defers one calibration target whose |relative error| breaches the 25% release bound, with the defect diagnosed, tracked, and owned elsewhere; the receipt records the reasoning, approver, adjudication reference, approval date, and expiry. The gate keeps the target BOUND in calibration (exclusion applies to the release fence only, never the solve), reports dormant entries, FAILS stale ones (targets back inside the bound), and FAILS expired ones until the adjudication is renewed, so this register cannot rot.", - "exclusions": {} + "exclusions": { + "hmrc/self_employment_income_income_band_20_000_to_30_000@2025": { + "reason": "Calibration-competition regression from the 2026-09-18 surface expansion, not a data defect: spine-s IMPROVED this cell's raw error (+10.6% on spine-m to +6.4%), the SPI 15.7bn target is unchanged, and the 638-target solve (region-grain fan-out #906: 108 age-by-region cells, 24 regional CGT rows, regional council-tax stock; CGT bands #939) pushed the band +6.4% -> +25.3% where v18's 364-target solve left it at +10.5%. The 30_000_to_40_000 neighbour shows the same signature inside the bound (+17.5%) and every other self_employment band fits within 0.2%; no aggregate self_employment_income anchor exists to hold banded mass against the regional reshuffle. Deferred four weeks to unblock the first post-#906/#939 national line while the anchor/loss-weight fix is scoped.", + "approved_by": "juaristi22", + "adjudication": "microcosm#796", + "approved_on": "2026-09-18", + "expires_on": "2026-10-16" + } + } } diff --git a/packages/microcosm-build/tests/test_uk_terminal_gates.py b/packages/microcosm-build/tests/test_uk_terminal_gates.py index b1fa9cdb6..adb2d0f27 100644 --- a/packages/microcosm-build/tests/test_uk_terminal_gates.py +++ b/packages/microcosm-build/tests/test_uk_terminal_gates.py @@ -389,7 +389,18 @@ def test_target_fit_out_of_force_exclusion_fails_even_without_a_breach() -> None def test_committed_target_fit_register_retains_only_live_deferrals() -> None: register = uk_default_target_fit_reviewed_exclusions() - assert register == {} + # The v20 (2026-09-18, main ce76b358) national line: the 638-target + # surface after the #906 region fan-out and #939 CGT bands pushes one + # national SPI band 0.34pp past the fence while spine-s itself improved + # the cell. One live deferral, signed for four weeks. + assert set(register) == { + "hmrc/self_employment_income_income_band_20_000_to_30_000@2025", + } + record = register["hmrc/self_employment_income_income_band_20_000_to_30_000@2025"] + assert record.approved_by == "juaristi22" + assert record.adjudication == "microcosm#796" + assert record.approved_on == "2026-09-18" + assert record.expires_on == "2026-10-16" # Aggregate errors from the fresh UC #882 development run: 1,500 epochs with @@ -414,7 +425,7 @@ def test_restored_fit_checks_leave_empty_payment_tail_cells_blocked() -> None: **empty_tail, }, reviewed_exclusions=uk_default_target_fit_reviewed_exclusions(), - now=date(2026, 9, 9), + now=date(2026, 9, 18), ) assert not fit.passed @@ -436,7 +447,7 @@ def test_restored_fit_checks_apply_if_a_later_run_breaches_again( fit = uk_target_fit_gate( {name: relative_error}, reviewed_exclusions=uk_default_target_fit_reviewed_exclusions(), - now=date(2026, 9, 9), + now=date(2026, 9, 18), ) assert fit.passed is passes @@ -451,7 +462,7 @@ def test_observed_liability_has_no_retired_cash_exemption() -> None: fit = uk_target_fit_gate( {"hmrc.cgt.liability_total@2025": 0.30}, reviewed_exclusions=register, - now=date(2026, 9, 15), + now=date(2026, 9, 18), ) assert not fit.passed assert fit.details["failing_targets"] == {"hmrc.cgt.liability_total@2025": 0.30} From 795b71b0ffb60f87c8f94f99ec942071518cd3a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 18 Sep 2026 19:13:41 +0100 Subject: [PATCH 02/22] Scoring prepare: drop injected measure inputs and take a contract band-edge register prepare_uk_target_frame (the scoring route) missed two things the calibration stage does: dropping the injected measure-input columns after materialization, and materializing with a caller-supplied band-edge register. Since the region fan-out, cross-grain measures (region, country, esa_contrib, esa_income) inject at two entities, so the prepared frame's global-column-uniqueness rule refused every score of a candidate export. The band-edge register matters whenever the scored registry is a pruned surface: per #803 a pruned roster must never redraw its own band edges. The scorer tool grows --band-edge-registry-json and threads both fixes. Debt: no unit fixture exists for CalibrationFrameAdapter or this route; the functional evidence is the v20 head-to-head this commit unblocks. A proper synthetic-frame test belongs in the upstream PR. Co-Authored-By: Claude Fable 5 --- .../build/uk_runtime/national_calibration.py | 22 +++++++++++- tools/score_uk_national_candidate.py | 36 +++++++++++++++++-- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py index cc07210b0..a98c96f4f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py @@ -466,6 +466,7 @@ def prepare_uk_target_frame( *, period: int | str, measure_resolver: object | None, + band_edge_registry: TargetRegistry | None = None, ) -> tuple[Frame, Mapping[str, Any] | None]: """Materialize a registry's measures onto a frame, for scoring. @@ -475,6 +476,15 @@ def prepare_uk_target_frame( measure that calibration deliberately strips before export. A skipped target refuses rather than quietly shrinking the surface both sides are compared on. + + ``band_edge_registry`` carries the full compiled contract register when + ``registry`` is a pruned scoring surface, so banded fan-out edges come + from the compiled roster exactly as they do in the calibration stage + (#803) — a pruned surface must never redraw its own band edges. Injected + measure-input columns are dropped after materialization, mirroring the + stage: cross-grain measures (a person-grain and a household-grain target + sharing one measure name since the region fan-out) would otherwise + violate the prepared frame's global-column-uniqueness rule. """ resolution = None @@ -484,16 +494,26 @@ def prepare_uk_target_frame( registry, measure_resolver, period=period, + band_edge_registry=band_edge_registry, ) adapter = CalibrationFrameAdapter(frame) + original_columns = { + entity: set(table.columns) for entity, table in adapter.tables.items() + } if resolution is not None: inject_measure_inputs(adapter, resolution.measure_inputs) - materialized = materialize_uk_ledger_targets(adapter, registry, period=period) + materialized = materialize_uk_ledger_targets( + adapter, registry, period=period, band_edge_registry=band_edge_registry + ) if materialized.skipped: raise RuntimeError( "target measures did not materialize for scoring: " f"{[skip.__dict__ for skip in materialized.skipped]}." ) + if resolution is not None: + drop_injected_measure_inputs( + adapter, resolution.measure_inputs, original_columns + ) return adapter.prepared_frame(), ( None if resolution is None else dict(resolution.receipt) ) diff --git a/tools/score_uk_national_candidate.py b/tools/score_uk_national_candidate.py index df75a54c6..dd7d885b4 100644 --- a/tools/score_uk_national_candidate.py +++ b/tools/score_uk_national_candidate.py @@ -47,6 +47,7 @@ def score_uk_national_candidate( measure_resolver_factory: Callable[[Path, Any], Any] | None = None, candidate_label: str | None = None, incumbent_label: str = "enhanced_frs_2024_25", + band_edge_registry: TargetRegistry | None = None, ) -> dict[str, Any]: """Return the #578 rule-1 score block on a shared target registry. @@ -64,10 +65,18 @@ def score_uk_national_candidate( candidate_pin = _verify_artifact("candidate", candidate_h5, candidate_sha256) incumbent_pin = _verify_artifact("incumbent", incumbent_h5, incumbent_sha256) candidate_frame, candidate_resolution = _scored_frame( - candidate_h5, target_registry, calibration_year, measure_resolver_factory + candidate_h5, + target_registry, + calibration_year, + measure_resolver_factory, + band_edge_registry=band_edge_registry, ) incumbent_frame, incumbent_resolution = _scored_frame( - incumbent_h5, target_registry, calibration_year, measure_resolver_factory + incumbent_h5, + target_registry, + calibration_year, + measure_resolver_factory, + band_edge_registry=band_edge_registry, ) candidate = score_targets( candidate_frame, @@ -227,11 +236,17 @@ def _scored_frame( registry: TargetRegistry, calibration_year: int, factory: Callable[[Path, Any], Any] | None, + *, + band_edge_registry: TargetRegistry | None = None, ) -> tuple[Any, Any]: frame, _provenance = load_uk_national_frame(h5_path) resolver = None if factory is None else factory(Path(h5_path), frame) return prepare_uk_target_frame( - frame, registry, period=calibration_year, measure_resolver=resolver + frame, + registry, + period=calibration_year, + measure_resolver=resolver, + band_edge_registry=band_edge_registry, ) @@ -305,6 +320,15 @@ def main(argv: list[str] | None = None) -> int: default="enhanced_frs_2024_25", help="Override the incumbent/reference label.", ) + parser.add_argument( + "--band-edge-registry-json", + type=Path, + help=( + "Full compiled contract register supplying banded fan-out edges " + "when --registry-json is a pruned scoring surface (#803: a " + "pruned surface must never redraw its own band edges)." + ), + ) parser.add_argument( "--no-measure-resolution", action="store_true", @@ -316,6 +340,11 @@ def main(argv: list[str] | None = None) -> int: ) args = parser.parse_args(argv) registry = _load_registry(args.registry_json) + band_edge_registry = ( + None + if args.band_edge_registry_json is None + else _load_registry(args.band_edge_registry_json) + ) calibration_year = args.calibration_year if calibration_year is None: from microcosm.build.uk_runtime.frs_release import load_uk_frs_release @@ -338,6 +367,7 @@ def main(argv: list[str] | None = None) -> int: measure_resolver_factory=factory, candidate_label=args.candidate_label, incumbent_label=args.incumbent_label, + band_edge_registry=band_edge_registry, ) _write_json(args.output_json, {"score_vs_enhanced_frs": score}) return 0 From c7c8193ca1ab0368b75869ec8c74f9c38950b18b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 21 Sep 2026 10:29:22 +0100 Subject: [PATCH 03/22] Retire six A16 exclusions the v20 national build measures inside the fence Evidence: the full 705-spec contract register resolved and scored on the green v20 candidate (spine-s, 638-target solve, all gates passed) at weights that never solved toward the excluded rows. 62 of the 67 excluded references measure (five are signed unresolvable counterfactuals); ten land inside the 25 percent fence; the six with margin retire, per Maria's instruction of 2026-09-21: hmrc/self_employment_income_count_..._500_000_to_1_000_000 +0.0 % dwp.uc.households_housing_element -2.8 % dwp.benefit_cap.capped_households_400_01_to_500 -3.8 % dwp/uc_payment_dist/LONE_PARENT_annual_payment_10_800_12_000 -8.1 % dwp.benefit_cap.capped_households_200_01_to_300 -10.1 % hmrc/employment_income_count_income_band_1_000_000_to_inf +16.9 % The any-tenure housing row's signed structural-bias adjudication (microcosm#921 review) is superseded by measurement: the bias is real and -2.8 %, inside the fence. Four uk-data#452 payment bands measuring 18-25 percent stay excluded: past retirements teach that binding-time competition can push a marginal cell over (the v20 self-employment band deferral is exactly that mechanism). Register 67 -> 61 entries; the census, tranche and element lockstep tests move with it. Co-Authored-By: Claude Fable 5 --- .../uk/calibration_measure_exclusions.json | 54 ------------------- .../tests/test_uk_measure_simulation.py | 42 ++++++++------- 2 files changed, 22 insertions(+), 74 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json b/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json index 86ea1d58e..3028c1384 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json +++ b/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json @@ -55,15 +55,6 @@ "approved_on": "2026-08-26", "expires_on": "2026-11-26" }, - { - "name": "hmrc/employment_income_count_income_band_1_000_000_to_inf", - "reason": "Zero-support channel on total income (ITA 2007 s.23, which excludes capital gains): zero in-band carrier records with total income above 1m, under both the SPI assessable-income concept and the component-sum proxy, measured identically on spine-i and spine-j (microcosm#807, receipts uk-candidate-eval out/excluded_cell_support_spine_{i,j}.json). The spine carries capital gains to 20.4m and thousands of records above 1m wealth; the premise is total-income support, never a wealth or gains cap, and a zero-support cell cannot be reached by reweighting.", - "tracking": "microcosm#757", - "approved_by": "juaristi22", - "adjudication": "microcosm#757 (the uk_target_fit dispositions, issue comment 5427936411); reason revised 2026-08-28 per microcosm#807 (the spine-i/j excluded-cell support re-measurement)", - "approved_on": "2026-08-26", - "expires_on": "2026-11-26" - }, { "name": "hmrc/dividend_income_income_band_1_000_000_to_inf", "reason": "Zero-support channel on total income (ITA 2007 s.23, which excludes capital gains): zero in-band carrier records with total income above 1m, under both the SPI assessable-income concept and the component-sum proxy, measured identically on spine-i and spine-j (microcosm#807, receipts uk-candidate-eval out/excluded_cell_support_spine_{i,j}.json). The spine carries capital gains to 20.4m and thousands of records above 1m wealth; the premise is total-income support, never a wealth or gains cap, and a zero-support cell cannot be reached by reweighting.", @@ -271,15 +262,6 @@ "approved_on": "2026-08-26", "expires_on": "2026-11-26" }, - { - "name": "dwp/uc_payment_dist/LONE_PARENT_annual_payment_10_800_to_12_000", - "reason": "One-sided undershoot: the annual-payment band composition inside the UC caseload is unreachable under the spine's current UC support (the #623-carried catalogue); every cell in this class undershoots on the rebind packet and two sit at -100% with final estimate 0.0. The payment distribution needs element/taper composition support beyond the would_claim_uc take-up lever.", - "tracking": "uk-data#452", - "approved_by": "juaristi22", - "adjudication": "microcosm#757 (the uk_target_fit dispositions, issue comment 5427936411)", - "approved_on": "2026-08-26", - "expires_on": "2026-11-26" - }, { "name": "dwp/uc_payment_dist/SINGLE_annual_payment_13_200_to_14_400", "reason": "One-sided undershoot: the annual-payment band composition inside the UC caseload is unreachable under the spine's current UC support (the #623-carried catalogue); every cell in this class undershoots on the rebind packet and two sit at -100% with final estimate 0.0. The payment distribution needs element/taper composition support beyond the would_claim_uc take-up lever.", @@ -352,15 +334,6 @@ "approved_on": "2026-08-26", "expires_on": "2026-11-26" }, - { - "name": "hmrc/self_employment_income_count_income_band_500_000_to_1_000_000", - "reason": "Sparse cell support: 4 in-band carrier records under the SPI assessable-income concept and 16 under the component-sum proxy (8,080 weighted carriers), measured identically on spine-i and spine-j (microcosm#807) - too few for the solver to hit the published value without distorting neighbouring cells. One of the three thin 500k-1m cells retained when the microcosm#807 revision lifted the three mid-band cells whose measured support was in the hundreds; re-measure on the next spine build before renewal.", - "tracking": "microcosm#757", - "approved_by": "juaristi22", - "adjudication": "microcosm#757 (the uk_target_fit dispositions, issue comment 5427936411); reason revised 2026-08-28 per microcosm#807 (the spine-i/j excluded-cell support re-measurement)", - "approved_on": "2026-08-26", - "expires_on": "2026-11-26" - }, { "name": "ons.savings_interest_income", "reason": "The spine carries £13.8bn of savings interest income against the ONS £95.5bn (-86% at initialization on spine-m, -89% after the solve): the FRS interest income capture, not the weights. Unreachable by reweighting within the stretch bound: a frame-side gap, not a weight question; while bound it pulls the joint solve toward weight concentration (housing benefit alone triples the max/median ratio under grain_equal). One-month window: correct the spine or re-adjudicate.", @@ -451,15 +424,6 @@ "approved_on": "2026-09-14", "expires_on": "2026-11-26" }, - { - "name": "dwp.uc.households_housing_element", - "reason": "Structural bias, not a weight question: DWP's any-tenure housing-entitlement count is social plus private plus 'Other or unknown' tenure, and the model has no other/unknown tenure (uc_housing_costs_element pays only council, housing-association and private renters), so the model's any-tenure count is identically the social plus private rows, which are bound and fit; the row overstates the reachable total by the other/unknown cell, 112,518 of 4,037,650 in calendar 2025 (2.8 percent; the published any-tenure mean, which the three tenure cells reproduce to the unit). Fitting all three housing rows would ask the solver to over-fill the tenure rows by that amount. Measured on every evaluation as the housing-element caseload diagnostic (Q50f -0.8 percent, D2 +0.6 percent at the baseline).", - "tracking": "microcosm#882", - "approved_by": "juaristi22", - "adjudication": "microcosm#882 (element sizing at the 10 September 2026 baseline, docs/uk-uc-baseline-2026-09-10.md; María's sign-off in review of microcosm#921 on 2026-09-15, setting the one-month window)", - "approved_on": "2026-09-15", - "expires_on": "2026-10-15" - }, { "name": "dwp.hb.households", "reason": "Housing Benefit caseload diagnostic (all client types and tenures, calendar-2025 mean of the Stat-Xplore Total items), measured on every evaluation but held out of the objective (#882, microcosm#867): the frame cannot reach the DWP caseload for the mechanisms signed on obr.housing_benefit (SPI-channel reporters in owner-occupied synthetic households, reporters drawn into Universal Credit, head-only rent, the means test). Q50f pays 235k GB units against a calendar-2025 published mean near 1.70m (-86 percent). Fitting it would move weight onto the few reporting units the engine pays. Retire when the row sits inside the 25 percent bound at the baseline.", @@ -496,15 +460,6 @@ "approved_on": "2026-09-16", "expires_on": "2026-12-08" }, - { - "name": "dwp.benefit_cap.capped_households_200_01_to_300", - "reason": "Support, not a weight question (#882): the benefit-cap amount band 200 .01 to 300 measures 22,125 against 16,240 (+36 percent) on the Q50f candidate at calibrated weights (November 2025 DWP Table 4, GB). The capped population rests on a handful of source households cloned many times: four of the fourteen bands are empty and 14.2 percent of capped units sit above GBP 1,300 a month against 0.6 percent published, a tail of large families outside London (4.45 children on average, full child elements, housing element near GBP 1,350 a month). The bound total (dwp.benefit_cap.capped_households) and the up-to-GBP-100 band, the one band inside the 25 percent bound, stay in the objective; fitting this band would move weight onto those clones (the #762 zero-band lesson). Measured on every evaluation with tools/diagnose_uk_benefit_cap.py; retire when support exists and the band sits inside the bound.", - "tracking": "microcosm#882", - "approved_by": "juaristi22", - "adjudication": "microcosm#882 (the 2026-09-16 status comment, issuecomment-5694598278, whose next-steps list this repair works, and the repairs plan María approved on 2026-09-16 with her population ruling on the U8 take-up entry; approver as author per the register convention). Expiry 2026-12-08 aligns the whole UC-repair tranche with the zero-band clock already on the register (microcosm#877), so the next spine re-adjudicates them together. Receipts: experiments/882-uc-repairs-receipts.md (R2 and R5, tools/diagnose_uk_benefit_cap.py on the Q50f candidate and the twins).", - "approved_on": "2026-09-16", - "expires_on": "2026-12-08" - }, { "name": "dwp.benefit_cap.capped_households_300_01_to_400", "reason": "Support, not a weight question (#882): the benefit-cap amount band 300 .01 to 400 measures 3,118 against 10,562 (-70 percent) on the Q50f candidate at calibrated weights (November 2025 DWP Table 4, GB). The capped population rests on a handful of source households cloned many times: four of the fourteen bands are empty and 14.2 percent of capped units sit above GBP 1,300 a month against 0.6 percent published, a tail of large families outside London (4.45 children on average, full child elements, housing element near GBP 1,350 a month). The bound total (dwp.benefit_cap.capped_households) and the up-to-GBP-100 band, the one band inside the 25 percent bound, stay in the objective; fitting this band would move weight onto those clones (the #762 zero-band lesson). Measured on every evaluation with tools/diagnose_uk_benefit_cap.py; retire when support exists and the band sits inside the bound.", @@ -514,15 +469,6 @@ "approved_on": "2026-09-16", "expires_on": "2026-12-08" }, - { - "name": "dwp.benefit_cap.capped_households_400_01_to_500", - "reason": "Support, not a weight question (#882): the benefit-cap amount band 400 .01 to 500 measures 10,933 against 7,813 (+40 percent) on the Q50f candidate at calibrated weights (November 2025 DWP Table 4, GB). The capped population rests on a handful of source households cloned many times: four of the fourteen bands are empty and 14.2 percent of capped units sit above GBP 1,300 a month against 0.6 percent published, a tail of large families outside London (4.45 children on average, full child elements, housing element near GBP 1,350 a month). The bound total (dwp.benefit_cap.capped_households) and the up-to-GBP-100 band, the one band inside the 25 percent bound, stay in the objective; fitting this band would move weight onto those clones (the #762 zero-band lesson). Measured on every evaluation with tools/diagnose_uk_benefit_cap.py; retire when support exists and the band sits inside the bound.", - "tracking": "microcosm#882", - "approved_by": "juaristi22", - "adjudication": "microcosm#882 (the 2026-09-16 status comment, issuecomment-5694598278, whose next-steps list this repair works, and the repairs plan María approved on 2026-09-16 with her population ruling on the U8 take-up entry; approver as author per the register convention). Expiry 2026-12-08 aligns the whole UC-repair tranche with the zero-band clock already on the register (microcosm#877), so the next spine re-adjudicates them together. Receipts: experiments/882-uc-repairs-receipts.md (R2 and R5, tools/diagnose_uk_benefit_cap.py on the Q50f candidate and the twins).", - "approved_on": "2026-09-16", - "expires_on": "2026-12-08" - }, { "name": "dwp.benefit_cap.capped_households_500_01_to_600", "reason": "Support, not a weight question (#882): the benefit-cap amount band 500 .01 to 600 measures 8,267 against 4,431 (+87 percent) on the Q50f candidate at calibrated weights (November 2025 DWP Table 4, GB). The capped population rests on a handful of source households cloned many times: four of the fourteen bands are empty and 14.2 percent of capped units sit above GBP 1,300 a month against 0.6 percent published, a tail of large families outside London (4.45 children on average, full child elements, housing element near GBP 1,350 a month). The bound total (dwp.benefit_cap.capped_households) and the up-to-GBP-100 band, the one band inside the 25 percent bound, stay in the objective; fitting this band would move weight onto those clones (the #762 zero-band lesson). Measured on every evaluation with tools/diagnose_uk_benefit_cap.py; retire when support exists and the band sits inside the bound.", diff --git a/packages/microcosm-build/tests/test_uk_measure_simulation.py b/packages/microcosm-build/tests/test_uk_measure_simulation.py index d0f6468cc..11789e611 100644 --- a/packages/microcosm-build/tests/test_uk_measure_simulation.py +++ b/packages/microcosm-build/tests/test_uk_measure_simulation.py @@ -571,9 +571,9 @@ def test_exclusion_applier_warns_within_week_of_expiry(): #: never absorbed. _PACKAGED_EXCLUSION_CENSUS = { "hmrc.salary_sacrifice.": 5, - "_1_000_000_to_inf": 11, + "_1_000_000_to_inf": 10, "slc.": 5, - "dwp/uc_payment_dist/": 18, + "dwp/uc_payment_dist/": 17, "obr.universal_credit_": 2, # microcosm#890 E1 (2026-09-11): the all-road-users obr.fuel_duties row # is signed out of the reference surface (target_reference_signed_ @@ -592,22 +592,25 @@ def test_exclusion_applier_warns_within_week_of_expiry(): # microcosm#882 (2026-09-15): three UC element rows were held out of the # objective, signed by María in review of microcosm#921 with a one-month # window. The carer and childcare rows retired on 2026-09-16 with their - # repairs (care hours and the childcare take-up draw on policyengine-uk - # 2.98.0); the any-tenure housing row stays held out as a structural bias. - "dwp.uc.households_": 1, + # repairs; the any-tenure housing row retired on 2026-09-21 on the v20 + # measurement (the signed structural bias measures -2.8 % at calibrated + # weights that never solved toward the row, inside the 25 % fence). + "dwp.uc.households_": 0, # microcosm#882 repairs (2026-09-16): the three Housing Benefit caseload # rows and the thirteen benefit-cap amount bands outside the 25 percent # bound are measured on every evaluation but held out of the objective; # the A16 rows obr.housing_benefit and dwp.jsa_claimants were # re-adjudicated the same day on the mechanism receipts. "dwp.hb.": 3, - "dwp.benefit_cap.capped_households_": 13, + # Two amount bands (200.01-300, 400.01-500) retired 2026-09-21 on the + # v20 measurement (-10.1 % and -3.8 % unbound); the tail bands stay out. + "dwp.benefit_cap.capped_households_": 11, } # The carer and childcare rows were retired on 2026-09-16 with the repairs -# (care hours and the childcare take-up draw on policyengine-uk 2.98.0); -# only the any-tenure housing row remains held out. -_UC_ELEMENT_REGISTER_ROWS = ("dwp.uc.households_housing_element",) +# (care hours and the childcare take-up draw on policyengine-uk 2.98.0); the +# any-tenure housing row retired on 2026-09-21 on the v20 measurement. +_UC_ELEMENT_REGISTER_ROWS = () _A16_UNREACHABLE_ROWS = ( "ons.savings_interest_income", @@ -622,7 +625,7 @@ def test_exclusion_applier_warns_within_week_of_expiry(): def test_packaged_exclusions_load(): exclusions = load_uk_calibration_measure_exclusions() names = [entry["name"] for entry in exclusions] - assert len(names) == len(set(names)) == 67 + assert len(names) == len(set(names)) == 61 band_h_region_cells = [ entry for entry in exclusions @@ -641,8 +644,9 @@ def test_packaged_exclusions_load(): matched = [name for name in names if marker in name] assert len(matched) == expected, (marker, matched) # The sparse HMRC band cells are whatever remains: hmrc/ band cells - # that are not the eleven 1m+ channel cells. After the #807 revision - # only the three genuinely thin 500k-1m cells stay excluded. + # that are not the 1m+ channel cells. After the #807 revision three + # genuinely thin 500k-1m cells stayed excluded; the self-employment one + # retired 2026-09-21 on the v20 measurement (+0.0 % unbound). sparse = [ name for name in names @@ -651,7 +655,6 @@ def test_packaged_exclusions_load(): assert sorted(sparse) == [ "hmrc/dividend_income_income_band_500_000_to_1_000_000", "hmrc/property_income_count_income_band_500_000_to_1_000_000", - "hmrc/self_employment_income_count_income_band_500_000_to_1_000_000", ], sparse # The 2026-08-26 tranche carries the uk_target_fit disposition @@ -663,7 +666,8 @@ def test_packaged_exclusions_load(): # the obr.fuel_duties entry (2026-09-11): the all-road-users row is # signed out of the reference surface and the cars receipts bind instead. tranche = [e for e in exclusions if e["approved_on"] == "2026-08-26"] - assert len(tranche) == 35 + # Three 2026-08-26 rows retired 2026-09-21 on the v20 measurement. + assert len(tranche) == 32 for entry in tranche: assert "5427936411" in entry["adjudication"], entry["name"] assert entry["expires_on"] == "2026-11-26", entry["name"] @@ -704,7 +708,7 @@ def test_packaged_exclusions_load(): for e in exclusions if e["approved_on"] == "2026-09-16" and e["name"] not in _A16_READJUDICATED_ROWS ] - assert len(repairs) == 16 + assert len(repairs) == 14 for entry in repairs: assert entry["expires_on"] == "2026-12-08", entry["name"] # Every entry of the tranche points at the ruling that exists (the @@ -738,15 +742,13 @@ def test_packaged_exclusions_load(): assert "docs/uk-uc-baseline-2026-09-10.md" in entry["adjudication"], entry[ "name" ] - housing = next( - e for e in elements if e["name"] == "dwp.uc.households_housing_element" - ) - assert "112,518 of 4,037,650" in housing["reason"] - # The four element rows that stay in the objective are not on the register. + # Every UC element row now rides in the objective; none is on the register + # (the any-tenure housing row retired 2026-09-21 on the v20 measurement). for riding in ( "dwp.uc.households_carer_element", "dwp.uc.households_childcare_element", "dwp.uc.households_lcwra_element", + "dwp.uc.households_housing_element", "dwp.uc.households_housing_element_social_rented", "dwp.uc.households_housing_element_private_rented", "dwp.uc.households_with_deduction", From b7beb71367ee19f17fda8651cd8033c59e77b69f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 21 Sep 2026 12:22:51 +0100 Subject: [PATCH 04/22] Evidence for the microcosm#968 BHC poverty diagnosis The memo as posted on #968 plus the two post-review sections (the direction of an entitlement-ranked take-up allocation, and the year basis), the six measurement scripts, every output JSON including the re-run #731 scorecards with their HBAI-basis block at 2024, and receipts.json with the five files' sha256 digests and sizes, the engine (policyengine-uk 2.99.1), the feed pin (c5e5bf8) and the DWP and ONS sources. Aggregates only; no household rows. Co-Authored-By: Claude Fable 5.1 --- docs/evidence/uk-poverty-968/README.md | 112 + .../uk-poverty-968/outputs/basis_swap.json | 372 +++ .../uk-poverty-968/outputs/poverty_audit.json | 2899 +++++++++++++++++ .../outputs/scorecard_d20_dense.json | 72 + .../outputs/scorecard_efrs_1_57_3.json | 72 + .../outputs/scorecard_t95_55k.json | 72 + .../outputs/scorecard_v20_national.json | 72 + .../outputs/scorecard_x50_55k.json | 72 + .../uk-poverty-968/outputs/tail_profile.json | 479 +++ .../outputs/takeup_direction.json | 38 + .../uk-poverty-968/outputs/takeup_probe.json | 82 + docs/evidence/uk-poverty-968/receipts.json | 78 + .../scripts/measure_968_basis_swap.py | 62 + .../scripts/measure_968_poverty_audit.py | 123 + .../scripts/measure_968_tail_profile.py | 56 + .../scripts/measure_968_takeup_direction.py | 34 + .../scripts/measure_968_takeup_probe.py | 50 + .../uk-poverty-968/scripts/scorecard_731.py | 239 ++ 18 files changed, 4984 insertions(+) create mode 100644 docs/evidence/uk-poverty-968/README.md create mode 100644 docs/evidence/uk-poverty-968/outputs/basis_swap.json create mode 100644 docs/evidence/uk-poverty-968/outputs/poverty_audit.json create mode 100644 docs/evidence/uk-poverty-968/outputs/scorecard_d20_dense.json create mode 100644 docs/evidence/uk-poverty-968/outputs/scorecard_efrs_1_57_3.json create mode 100644 docs/evidence/uk-poverty-968/outputs/scorecard_t95_55k.json create mode 100644 docs/evidence/uk-poverty-968/outputs/scorecard_v20_national.json create mode 100644 docs/evidence/uk-poverty-968/outputs/scorecard_x50_55k.json create mode 100644 docs/evidence/uk-poverty-968/outputs/tail_profile.json create mode 100644 docs/evidence/uk-poverty-968/outputs/takeup_direction.json create mode 100644 docs/evidence/uk-poverty-968/outputs/takeup_probe.json create mode 100644 docs/evidence/uk-poverty-968/receipts.json create mode 100644 docs/evidence/uk-poverty-968/scripts/measure_968_basis_swap.py create mode 100644 docs/evidence/uk-poverty-968/scripts/measure_968_poverty_audit.py create mode 100644 docs/evidence/uk-poverty-968/scripts/measure_968_tail_profile.py create mode 100644 docs/evidence/uk-poverty-968/scripts/measure_968_takeup_direction.py create mode 100644 docs/evidence/uk-poverty-968/scripts/measure_968_takeup_probe.py create mode 100644 docs/evidence/uk-poverty-968/scripts/scorecard_731.py diff --git a/docs/evidence/uk-poverty-968/README.md b/docs/evidence/uk-poverty-968/README.md new file mode 100644 index 000000000..4cbe5646d --- /dev/null +++ b/docs/evidence/uk-poverty-968/README.md @@ -0,0 +1,112 @@ +# microcosm#968: BHC poverty on the calibrated UK national line against HBAI + +Evidence for [PolicyEngine/microcosm#968](https://github.com/PolicyEngine/microcosm/issues/968), filed with PR #967. The memo below is the diagnosis as posted on the issue on 2026-09-21; the two sections after it record what the review added. Every number comes from the scripts in `scripts/` run on the files listed in `receipts.json` (digests, engine policyengine-uk 2.99.1, feed c5e5bf8); their JSON outputs are in `outputs/`. + + +Diagnosis memo, 2026-09-21. One engine throughout (policyengine-uk 2.99.1, the uk-data venv), the #731 reproduction conventions (person-weighted share of people in flagged households). Receipts: `scripts/` and `outputs/` in this directory, digests and sources in `receipts.json`. DWP sources: HBAI FYE 2024 summary tables (`HBAI_summary_results.ods`, tables 1.2b, 1.3a–1.6a), the HBAI table pack (chart 2.4 income bands, tables 2.1db, 3.5db, 3.6db, 4.5db, 4.6db), the HBAI quality and methodology report, and the FRS 2023/24 methodology tables M.6a, M.6b and M.8. + +## The answer in four lines + +- The scorecard's poverty row compares two different concepts. The engine's `in_poverty_bhc` is the **absolute** BHC flag (60 % of the 2010/11 median uprated by CPI: £381/week in 2024, £403/week in 2026). HBAI's "~17 %" is the **relative** BHC rate. HBAI's absolute BHC rate for FYE 2024 is 15 %, its relative rate 17 % (17.3 % from the published bands), its median £650/week. +- Under HBAI's own concept (relative, 60 % of the person-weighted median, computed within the file), the v20 national line measures 15.3 % at 2024 and 14.6 % at 2026. The gap to HBAI is about 2 points, not 8. +- Under the absolute concept the 15 → 9.3 gap is mostly an income-level effect, not a poverty effect: every engine file's median sits 15–20 % above HBAI's (£745 at spine-s design weights, £751 v20, £780 eFRS, all at 2024), so a fixed £381 line bites at 51 % of the engine's median where DWP's bites at 57 %. At 57 % of its own median v20 gives 13.3 % against HBAI's 15 %, the same ~2-point gap. +- The eFRS is not a clean reference: on the relative concept it measures 19.7 % (2.4 points above HBAI) with an effective sample of 1,166 households and 0.58 % of people in non-positive-income households. + +## 1. Definition audit against HBAI + +What the engine does versus DWP, checked line by line (`hbai_household_net_income.py`, `household_equivalisation_bhc.py`, `in_poverty_bhc.py`, `in_relative_poverty_bhc.py`, `poverty_threshold_bhc.py`): + +- Income concept: the engine's HBAI adds and subtracts match DWP's list (earnings, self-employment, investments, pensions, maintenance, all benefits and tax credits, benefits in kind; minus income tax, NI, council tax, pension contributions, maintenance paid, student loan repayments). No divergence found. Small engine-only additions (tax-free childcare, healthy start vouchers) are under £1bn. +- Equivalisation: the modified OECD scale with identical weights (0.67 / 0.33 / 0.20 / 0.33). The engine's adult threshold is 18 where HBAI's dependent child runs to 19 in full-time education, but both sides give that person 0.33, so no divergence. +- Unit: both count individuals in households. No divergence. +- Absolute line: the engine's CPI-uprated 2010/11 line (£372/week at 2023, £381 at 2024) is within 1–3 % of DWP's (60 % of the 2010/11 median restated in 2023/24 prices is £370/week). No material divergence. +- **Relative line: the engine's `in_relative_poverty_bhc` uses a household-weighted median.** HBAI's median is over individuals. On v20 at 2024 the household-weighted flag gives 13.4 % against 15.3 % person-weighted, a 2-point understatement. Its label also says "AHC". Neither the probe nor the scorecard uses this flag today, but anything that does is biased low. (There is also no `poverty_threshold_ahc` variable, only `poverty_line_ahc`; cosmetic.) +- **Concept: the probe reads the absolute flag and the issue text calls it relative.** This is the bulk of the "8.7 vs 17" gap. +- Year basis: HBAI FYE 2024 is April 2023 to March 2024; the files are on a 2024 basis and the engine's medians grow about 3.6 % a year (v20: £751 → £778 → £800). At most about 3 points of the median gap is timing. + +Where the engine's median level comes from is not a poverty question and is left open here, with two markers: the ONS household-finances basis puts the FYE 2024 equivalised median at £706/week (ONS "Average household income, UK: FYE 2024"), 9 % above HBAI on definitions alone, and the reported-benefit swap in section 3 barely moves the median (£745 → £741 at design weights), so the level difference lives in market incomes, taxes and uprating, not in simulated benefits. + +## 2. The right numbers, one engine + +Relative BHC, 60 % of the person-weighted median, share of individuals: + +- HBAI FYE 2024: 17 % published, 17.3 % from the £10 income bands. +- spine-s at design weights, 2024: 16.2 %. +- v20 national, calibrated, 2024: 15.3 %. 2025: 15.3 %. 2026: 14.6 %. +- eFRS 1.57.3, 2024: 19.7 %. 2026: 19.3 %. +- X50 55k local draw, 2026: 13.8 %. +- June populace_uk_2023, 2026: 20.0 %. + +Absolute BHC (engine line), share of individuals: + +- HBAI FYE 2024: 15 %. +- spine-s design 2024: 10.4 %. v20 2024: 9.3 %. v20 2026: 8.7 %. eFRS 2024: 11.2 %. June 2026: 15.1 %. + +Medians (person-weighted, £/week): HBAI £650; spine-s design £745; v20 £751 (2024), £800 (2026); eFRS £780 (2024). + +AHC, for completeness (HBAI relative 21 %, absolute 18 %): v20 2024 relative 19.5 %, absolute 13.1 %; eFRS 23.7 % and 14.6 %. + +The June file's 16.7 % "match" was two errors cancelling: an absolute reading against a relative benchmark, on a file whose relative rate is 20 %. + +## 3. Decomposing the microcosm layer (same records, same engine) + +spine-s and the v20 H5 are the same 52,846 households in the same order; only `household_weight` differs. At 2024: + +- Design weights, design line: 16.16 %. Calibrated weights, design line: 14.96 % (reweighting moves 1.20 points of mass out of the below-line region). Calibrated weights, calibrated line: 15.30 % (the median moves £745 → £751, adding 0.34 back). **Net calibration effect: −0.86 points.** At 2026 the same split gives 15.59 → 14.41 → 14.63, net −0.96. +- Absolute flag: 10.36 → 9.31 (−1.05). + +Basis swap (replace the engine's simulated means-tested benefits with the FRS-reported amounts on the same records; `measure_968_basis_swap.py`): + +- v20 calibrated weights: 15.30 → 15.94 % (+0.64). Universal Credit alone +0.29, child benefit +0.31, the rest under 0.1 each. Absolute flag 9.31 → 10.31. +- spine-s design weights: 16.16 → 16.21 %. At design weights the simulated-versus-reported basis is not the story. +- eFRS: 19.68 → 20.20 %. + +So the FRS under-reporting that the DWP itself documents (FRS 2023/24 captures 67 % of UC families, 74 % of Housing Benefit, 71 % of Pension Credit, 61 % of ESA, table M.6a) costs the calibrated line about 0.6 points of measured poverty and nothing at design weights. + +Take-up allocation (`measure_968_takeup_probe.py`, v20 at 2024, calibrated weights, claimant count held near the file's): + +- Baseline draw: 15.30 % (6.37m UC families, £76.8bn). +- Three alternative random draws of the same residual count: 15.40–15.59 %. +- Residual given to the poorest entitled units: 14.05 %. To the richest: 16.66 %. **The allocation alone spans 2.6 points at a fixed claimant count.** +- Reported claimants only, no residual: 16.12 % (4.67m families, £63.6bn). Everyone entitled claims: 14.85 % (8.61m). +- Child relative rate across the same runs: 17.6 % to 21.0 %. + +Reading: the residual gap to HBAI (about 2 points at 2024) is entirely inside the span of "who gets the drawn Universal Credit". The anchored-residual draw assigns the non-reporting claimants uniformly over entitled units; real non-take-up is not uniform (it concentrates in small entitlements), and HBAI's non-reporters are not uniform either. + +## 4. Bottom-decile composition and shape + +- Bottom of the distribution against HBAI's published bands (share of individuals below x % of the median, BHC): HBAI 6.2 / 11.0 / 17.3 / 24.5 / 33.1 at 40 / 50 / 60 / 70 / 80 %. v20 2024: 4.9 / 8.9 / 15.3 / 23.8 / 32.8. spine-s design: 5.1 / 9.8 / 16.2 / 24.0 / 32.7. eFRS: 7.1 / 11.8 / 19.7 / 27.1 / 34.8. v20's deficit is concentrated below half the median (deep poverty), where it is 1.3–2.1 points short; from 70 % of the median up it matches HBAI. +- The zero-income spike: HBAI has 0.66m individuals (1.0 %) at £0–10/week. spine-s design 0.34 %, v20 0.16 %, eFRS 0.62 %. Half of v20's deep-tail deficit is this spike; the engine's imputation fills incomes that the FRS leaves at zero. +- Quantile ratios to the median: HBAI p10 0.48, p30 0.765, p70 1.31, p90 1.96, mean/median 1.22. v20 calibrated: 0.52, 0.765, 1.27, 1.75, 1.13. spine-s design: 0.51, 0.77, 1.29, 1.85, 1.14. eFRS: 0.46, 0.73, 1.32, 1.87, 1.13. The engine files are compressed at both ends; calibration compresses the top further (p90 ratio 1.85 → 1.75). That is the scorecard's open Gini item, not a poverty item, but it shares a cause candidate (weights on the SPI-synthetic and top households). +- Income sources by quintile (share of gross, state support): HBAI 47 / 36 / 22 / 10 / 3. v20 2024: 41 / 31 / 20 / 13 / 4.5. The bottom two quintiles carry less state support and more market income than HBAI's; the top two carry more. Not a benefits-too-high-at-the-bottom signature. +- Groups, relative BHC, v20 2024 versus HBAI FYE 2024: children 18.8 vs 23 (the largest miss); pensioners 17.2 vs 19; working-age 13.6 vs 15. Children by family size: 1 child 9.9 vs 15, 2 children 15.3 vs 18, 3+ 32.3 vs 35. Tenure (household basis): social renters 20–22 vs 29, private renters 15.8 vs 22, owned outright 15.0 vs 18, mortgagors 12.5 vs 8. v20 has too few poor renters and too many poor mortgagors; the eFRS has the opposite renter error (social 31–44 %). Family typing here is household-level while HBAI's is benefit-unit-level, so single-adult categories are not comparable and are omitted. + +## 5. Attribution + +Against HBAI FYE 2024, relative BHC 17.3 %, v20 at 2024 measures 15.3 %: + +- Concept and level (the family-wide layer as the issue framed it): the "8.7 vs 17" gap is an absolute reading against a relative benchmark on files whose income level is 15–20 % above HBAI's. Not a poverty defect; a scorecard-basis error plus the open income-level question. +- Family-wide residual at design weights: about 1.1 points (16.2 vs 17.3), of which roughly half is the missing zero-income spike and the rest the year basis and the compressed bottom tail. +- Calibration: −0.9 points (reweighting at a fixed line −1.2, line shift +0.3). +- Simulated-versus-reported benefit basis: −0.6 points on the calibrated weights, ~0 at design weights. +- Take-up allocation: ±1.3 points around the draw at a fixed claimant count, which brackets the whole residual. + +## 6. Fix path as offered (rulings recorded below) + +- A. Scorecard and probe (definition fix, do first): make `scorecard_731.py` report the pair HBAI publishes, absolute BHC (benchmark 15 %) and relative BHC on the person-weighted median (17 %), plus the AHC pair (18 % / 21 %) and the median (£650). Re-state the #731 poverty row on that basis; the row then reads v20 14.6 % relative and 8.7 % absolute at 2026, against 17 % and 15 % at FYE 2024 with the caveat that HBAI's latest actual is FYE 2024 and the Resolution Foundation's outlook has typical incomes flat and child poverty rising to 2029/30. +- B. Engine (policyengine-uk#1865): `in_relative_poverty_bhc` should take the median over individuals (person-weighted), and its label should say BHC; consider a `poverty_threshold_ahc` twin of the BHC variable. Not a microcosm blocker. +- C. Microcosm, the substantive ~2-point signal: (i) the UC residual draw's allocation is the biggest single lever; a non-uniform residual (entitlement-size or income-ranked) is a modelling choice worth a spec, with the caseload bands unchanged; (ii) children at 18.8 vs 23 and the tenure pattern (too few poor renters) are the two rows to chase, starting from what the region × tenure and family-composition targets do to renter weights in the bottom two deciles; (iii) the zero-income spike is an imputation-policy question (whether households the FRS records at zero should be simulated into benefit receipt). +- D. Doctrine: do not put an HBAI poverty target on the surface. HBAI is a survey-reported-income statistic on a source that misses a third of UC families, and the DWP's own methodology report warns the bottom decile should not be read as living standards. Keep poverty a watched free outcome, but watch the right pair, with a T4-style diagnostic anchor (relative and absolute BHC/AHC, the median, the p10/p50 and p90/p50 ratios) cited to HBAI FYE 2024 tables 1.2b and 1.3a. +- Verdict on "the benchmark basis differs": partly. The concept and level differences are basis; the residual 2 points and the child and tenure pattern are real signals, bounded by the take-up allocation's span, and are the follow-up. + +## 7. After the review (2026-09-21) + +- **Direction of a realistic take-up allocation** (review point 3; `outputs/takeup_direction.json`). Ranking the drawn Universal Credit residual by entitlement size, largest first, which is DWP's finding that non-take-up concentrates in small entitlements, gives 14.9 % relative BHC at 2024 against the file's 15.3 % (children 18.7 % against 18.8 %; UC spend £84.3bn against £76.8bn at the same claimant count). Smallest first gives 15.9 %. So the correction the evidence favours moves measured poverty away from HBAI, not towards it, and the residual gap should be read as at least what the memo states. +- **Year basis** (review point 4). The scorecard probe now carries an `hbai_basis` block measured at 2024, the closest engine year to HBAI FYE 2024, with the same pair plus the p10 and p90 ratios to the median and the under-£10 share; the evaluation reports show that block in its own table beside HBAI FYE 2024 and keep the 2026 rows file against file only. +- **Position taken (2026-09-21).** The scorecard is made accurate on concept and year, and the evidence is filed here. Poverty is not chased as a benchmark: HBAI is a survey statistic from another producer, not an administrative total, and a poverty rate is not a calibration target. The income-level and tenure work the decomposition points at is already under way on its own merits and is not tied to moving a poverty rate; the residual 2-point relative gap and the children and tenure rows stay on record here rather than in a follow-up issue. + +## Receipts + +- `receipts.json`: the five files with sha256 digests and byte sizes, the engine, the feed pin, the DWP and ONS sources, and what each output holds. +- `scripts/`: `measure_968_poverty_audit.py`, `measure_968_basis_swap.py`, `measure_968_tail_profile.py`, `measure_968_takeup_probe.py`, `measure_968_takeup_direction.py`, and `scorecard_731.py` (the #731 scorecard probe with the HBAI pair and the 2024 block). +- `outputs/`: one JSON per script, plus the scorecard outputs for the national candidate, the enhanced FRS and the three K=20 local files. diff --git a/docs/evidence/uk-poverty-968/outputs/basis_swap.json b/docs/evidence/uk-poverty-968/outputs/basis_swap.json new file mode 100644 index 000000000..9953d3cff --- /dev/null +++ b/docs/evidence/uk-poverty-968/outputs/basis_swap.json @@ -0,0 +1,372 @@ +{ + "engine": "2.99.1", + "year": 2024, + "swap_set": [ + "universal_credit", + "housing_benefit", + "pension_credit", + "child_tax_credit", + "working_tax_credit", + "income_support", + "jsa_income", + "esa_income", + "council_tax_benefit", + "child_benefit" + ], + "files": { + "v20_national": { + "totals_bn": { + "universal_credit": { + "simulated": 76.83272818724737, + "reported": 61.65982094107585, + "simulated_recipient_hh_m": 5.899447441967964, + "reported_recipient_hh_m": 4.69722375107956 + }, + "housing_benefit": { + "simulated": 4.464219235857155, + "reported": 8.161163754658835, + "simulated_recipient_hh_m": 0.7608244295825958, + "reported_recipient_hh_m": 1.3749762283983231 + }, + "pension_credit": { + "simulated": 6.537695825669497, + "reported": 5.100636866745836, + "simulated_recipient_hh_m": 2.066473884399414, + "reported_recipient_hh_m": 1.3827790538253784 + }, + "child_tax_credit": { + "simulated": 0.1017876937232826, + "reported": 2.2963176066417685, + "simulated_recipient_hh_m": 0.015767602984428405, + "reported_recipient_hh_m": 0.298927515627861 + }, + "working_tax_credit": { + "simulated": 0.15224768097739344, + "reported": 0.33702950661060943, + "simulated_recipient_hh_m": 0.028848597001075744, + "reported_recipient_hh_m": 0.08318189950847625 + }, + "income_support": { + "simulated": 0.18812226463660675, + "reported": 1.025131292093722, + "simulated_recipient_hh_m": 0.022543715876579284, + "reported_recipient_hh_m": 0.25844708364868163 + }, + "jsa_income": { + "simulated": 0.06740073784475399, + "reported": 0.23748659140568668, + "simulated_recipient_hh_m": 0.017261854499816893, + "reported_recipient_hh_m": 0.053914079498291016 + }, + "esa_income": { + "simulated": 1.8593223369913088, + "reported": 4.119879551637238, + "simulated_recipient_hh_m": 0.1741251804587841, + "reported_recipient_hh_m": 0.4171803486907482 + }, + "council_tax_benefit": { + "simulated": 1.146668125699208, + "reported": 1.490041262990671, + "simulated_recipient_hh_m": 2.046592849905014, + "reported_recipient_hh_m": 3.083109921485901 + }, + "child_benefit": { + "simulated": 15.692417869013031, + "reported": 12.413568159230236, + "simulated_recipient_hh_m": 7.810235416875362, + "reported_recipient_hh_m": 5.987863538380146 + } + }, + "weightings": { + "calibrated": { + "engine_simulated": { + "abs_bhc_pct": 9.31103618296997, + "rel_bhc_pct": 15.301746413861888, + "median_week": 751.48046875, + "fixed_line_60pct_of_base_median_pct": null + }, + "all_reported": { + "abs_bhc_pct": 10.313734271166835, + "rel_bhc_pct": 15.941276893905556, + "median_week": 743.6017127403846, + "fixed_line_60pct_of_base_median_pct": 16.546027931767355 + }, + "only_universal_credit_reported": { + "abs_bhc_pct": 10.580153833924347, + "rel_bhc_pct": 15.587112687788679, + "median_week": 738.2648434448281, + "fixed_line_60pct_of_base_median_pct": 16.297318138004513 + }, + "only_housing_benefit_reported": { + "abs_bhc_pct": 9.14016077321755, + "rel_bhc_pct": 15.16678200312834, + "median_week": 754.4547852249873, + "fixed_line_60pct_of_base_median_pct": 15.058853549391705 + }, + "only_pension_credit_reported": { + "abs_bhc_pct": 9.318995885101302, + "rel_bhc_pct": 15.286388734873727, + "median_week": 751.1156843254604, + "fixed_line_60pct_of_base_median_pct": 15.29943831376547 + }, + "only_child_tax_credit_reported": { + "abs_bhc_pct": 9.1699525335271, + "rel_bhc_pct": 15.200603540728826, + "median_week": 752.8256460336538, + "fixed_line_60pct_of_base_median_pct": 15.178694664392356 + }, + "only_working_tax_credit_reported": { + "abs_bhc_pct": 9.313146841385175, + "rel_bhc_pct": 15.295077286349073, + "median_week": 751.3091606901123, + "fixed_line_60pct_of_base_median_pct": 15.296503576905119 + }, + "only_income_support_reported": { + "abs_bhc_pct": 9.302540797769172, + "rel_bhc_pct": 15.32238359179189, + "median_week": 753.5564903846154, + "fixed_line_60pct_of_base_median_pct": 15.279235753395055 + }, + "only_jsa_income_reported": { + "abs_bhc_pct": 9.293716044860428, + "rel_bhc_pct": 15.296462153660825, + "median_week": 751.48046875, + "fixed_line_60pct_of_base_median_pct": 15.296462153660825 + }, + "only_esa_income_reported": { + "abs_bhc_pct": 9.091234425217008, + "rel_bhc_pct": 15.202225069758379, + "median_week": 753.5740685096154, + "fixed_line_60pct_of_base_median_pct": 15.159441571540565 + }, + "only_council_tax_benefit_reported": { + "abs_bhc_pct": 9.306952741969605, + "rel_bhc_pct": 15.273189405312587, + "median_week": 751.5027749667146, + "fixed_line_60pct_of_base_median_pct": 15.273189405312587 + }, + "only_child_benefit_reported": { + "abs_bhc_pct": 9.620577762938007, + "rel_bhc_pct": 15.608389549912069, + "median_week": 749.7188096145738, + "fixed_line_60pct_of_base_median_pct": 15.775085185634453 + } + }, + "spine_s_design": { + "engine_simulated": { + "abs_bhc_pct": 10.36196031429083, + "rel_bhc_pct": 16.163689191086075, + "median_week": 745.38671875, + "fixed_line_60pct_of_base_median_pct": null + }, + "all_reported": { + "abs_bhc_pct": 10.919095975960676, + "rel_bhc_pct": 16.21300915534116, + "median_week": 741.3272317748576, + "fixed_line_60pct_of_base_median_pct": 16.4939436433749 + }, + "only_universal_credit_reported": { + "abs_bhc_pct": 10.864636764086667, + "rel_bhc_pct": 16.10896510569944, + "median_week": 739.8475186091435, + "fixed_line_60pct_of_base_median_pct": 16.396321770638878 + }, + "only_housing_benefit_reported": { + "abs_bhc_pct": 10.23609788093914, + "rel_bhc_pct": 16.075942196134008, + "median_week": 747.9852216918238, + "fixed_line_60pct_of_base_median_pct": 15.995373114349686 + }, + "only_pension_credit_reported": { + "abs_bhc_pct": 10.402997815065309, + "rel_bhc_pct": 16.21252096711801, + "median_week": 744.800732591508, + "fixed_line_60pct_of_base_median_pct": 16.255839298426686 + }, + "only_child_tax_credit_reported": { + "abs_bhc_pct": 10.259407566686674, + "rel_bhc_pct": 16.028604350894277, + "median_week": 746.2291623635363, + "fixed_line_60pct_of_base_median_pct": 16.007228768116725 + }, + "only_working_tax_credit_reported": { + "abs_bhc_pct": 10.397790904848254, + "rel_bhc_pct": 16.150295858379952, + "median_week": 745.3914368722677, + "fixed_line_60pct_of_base_median_pct": 16.150295858379952 + }, + "only_income_support_reported": { + "abs_bhc_pct": 10.347779684562006, + "rel_bhc_pct": 16.154662098810736, + "median_week": 745.7957172866228, + "fixed_line_60pct_of_base_median_pct": 16.145274797711195 + }, + "only_jsa_income_reported": { + "abs_bhc_pct": 10.334706377652573, + "rel_bhc_pct": 16.1498074991165, + "median_week": 745.6569691736897, + "fixed_line_60pct_of_base_median_pct": 16.1498074991165 + }, + "only_esa_income_reported": { + "abs_bhc_pct": 10.30149271253396, + "rel_bhc_pct": 16.103754306349042, + "median_week": 746.1148587740385, + "fixed_line_60pct_of_base_median_pct": 16.082378723571495 + }, + "only_council_tax_benefit_reported": { + "abs_bhc_pct": 10.348221388275173, + "rel_bhc_pct": 16.145950788540702, + "median_week": 745.7141002161189, + "fixed_line_60pct_of_base_median_pct": 16.144713154254536 + }, + "only_child_benefit_reported": { + "abs_bhc_pct": 10.610616558778073, + "rel_bhc_pct": 16.34191503228337, + "median_week": 741.5446213942307, + "fixed_line_60pct_of_base_median_pct": 16.583148536156827 + } + } + } + }, + "efrs_1_57_3": { + "totals_bn": { + "universal_credit": { + "simulated": 73.09709172161321, + "reported": 74.61322382625191, + "simulated_recipient_hh_m": 6.239545211998403, + "reported_recipient_hh_m": 5.840541549760684 + }, + "housing_benefit": { + "simulated": 12.244102252339145, + "reported": 14.713157282822568, + "simulated_recipient_hh_m": 1.5778565503483117, + "reported_recipient_hh_m": 2.062660035393119 + }, + "pension_credit": { + "simulated": 6.447777116554897, + "reported": 6.03124227029022, + "simulated_recipient_hh_m": 1.7548619712848663, + "reported_recipient_hh_m": 1.4655489025582522 + }, + "child_tax_credit": { + "simulated": 0.07133737308845621, + "reported": 1.3395116965943374, + "simulated_recipient_hh_m": 0.009244576636970044, + "reported_recipient_hh_m": 0.19743776312781872 + }, + "working_tax_credit": { + "simulated": 0.06874109296621335, + "reported": 0.13433514966076926, + "simulated_recipient_hh_m": 0.016961884278178217, + "reported_recipient_hh_m": 0.02922350984734297 + }, + "income_support": { + "simulated": 0.3825988032267648, + "reported": 0.2954680901833564, + "simulated_recipient_hh_m": 0.058967820897340775, + "reported_recipient_hh_m": 0.06506074785017968 + }, + "jsa_income": { + "simulated": 0.0018578498791179123, + "reported": 0.011457864562215168, + "simulated_recipient_hh_m": 0.00040267406916618347, + "reported_recipient_hh_m": 0.002600004368543625 + }, + "esa_income": { + "simulated": 2.1363262599735786, + "reported": 4.142687820719381, + "simulated_recipient_hh_m": 0.18741943308846654, + "reported_recipient_hh_m": 0.3894029940016568 + }, + "council_tax_benefit": { + "simulated": 2.26717084087797, + "reported": 2.3202796003310877, + "simulated_recipient_hh_m": 3.4095928315644266, + "reported_recipient_hh_m": 4.523823705864847 + }, + "child_benefit": { + "simulated": 16.475751731474123, + "reported": 13.38427200537357, + "simulated_recipient_hh_m": 8.321704258219615, + "reported_recipient_hh_m": 6.313253319340095 + } + }, + "weightings": { + "calibrated": { + "engine_simulated": { + "abs_bhc_pct": 11.248234644883528, + "rel_bhc_pct": 19.67636083543157, + "median_week": 780.1297292523311, + "fixed_line_60pct_of_base_median_pct": null + }, + "all_reported": { + "abs_bhc_pct": 10.748342162967067, + "rel_bhc_pct": 20.19765005842865, + "median_week": 780.1297292523311, + "fixed_line_60pct_of_base_median_pct": 20.19765005842865 + }, + "only_universal_credit_reported": { + "abs_bhc_pct": 11.080994993072013, + "rel_bhc_pct": 20.035017025030427, + "median_week": 780.1297292523311, + "fixed_line_60pct_of_base_median_pct": 20.035017025030427 + }, + "only_housing_benefit_reported": { + "abs_bhc_pct": 11.13165726795986, + "rel_bhc_pct": 19.660840381028184, + "median_week": 780.1297292523311, + "fixed_line_60pct_of_base_median_pct": 19.660840381028184 + }, + "only_pension_credit_reported": { + "abs_bhc_pct": 11.184496972415786, + "rel_bhc_pct": 19.62190850785528, + "median_week": 779.8224909855769, + "fixed_line_60pct_of_base_median_pct": 19.630151615180026 + }, + "only_child_tax_credit_reported": { + "abs_bhc_pct": 11.232347127528135, + "rel_bhc_pct": 19.608493341890863, + "median_week": 780.1297292523311, + "fixed_line_60pct_of_base_median_pct": 19.608493341890863 + }, + "only_working_tax_credit_reported": { + "abs_bhc_pct": 11.248234644883528, + "rel_bhc_pct": 19.676314107233647, + "median_week": 780.1297292523311, + "fixed_line_60pct_of_base_median_pct": 19.676314107233647 + }, + "only_income_support_reported": { + "abs_bhc_pct": 11.246721826301014, + "rel_bhc_pct": 19.6771246050495, + "median_week": 780.1297292523311, + "fixed_line_60pct_of_base_median_pct": 19.6771246050495 + }, + "only_jsa_income_reported": { + "abs_bhc_pct": 11.248234644883528, + "rel_bhc_pct": 19.67636083543157, + "median_week": 780.1297292523311, + "fixed_line_60pct_of_base_median_pct": 19.67636083543157 + }, + "only_esa_income_reported": { + "abs_bhc_pct": 11.028404036936275, + "rel_bhc_pct": 19.607005782280726, + "median_week": 780.1297292523311, + "fixed_line_60pct_of_base_median_pct": 19.607005782280726 + }, + "only_council_tax_benefit_reported": { + "abs_bhc_pct": 11.216962783614044, + "rel_bhc_pct": 19.644332982188086, + "median_week": 780.1297292523311, + "fixed_line_60pct_of_base_median_pct": 19.644332982188086 + }, + "only_child_benefit_reported": { + "abs_bhc_pct": 11.419842632142183, + "rel_bhc_pct": 20.111287786942928, + "median_week": 779.8224909855769, + "fixed_line_60pct_of_base_median_pct": 20.142793500550205 + } + } + } + } + } +} \ No newline at end of file diff --git a/docs/evidence/uk-poverty-968/outputs/poverty_audit.json b/docs/evidence/uk-poverty-968/outputs/poverty_audit.json new file mode 100644 index 000000000..4de647bcb --- /dev/null +++ b/docs/evidence/uk-poverty-968/outputs/poverty_audit.json @@ -0,0 +1,2899 @@ +{ + "engine": "2.99.1", + "files": { + "v20_national": { + "h5": "/Users/mariajuaristi/Desktop/PolicyEngine/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5", + "years": { + "2024": { + "bhc": { + "abs_rate_pct": 9.31103618296997, + "rel_rate_person_median_pct": 15.301746413861888, + "rel_rate_household_median_pct": 13.355658550011269, + "median_person_weighted_week": 751.48046875, + "median_household_weighted_week": 715.2810997596154, + "abs_threshold_week": 381.37943209134613, + "rel_line_person_week": 450.88828125, + "abs_threshold_over_person_median": 0.5075041174732409, + "share_below_pct_of_person_median": { + "40": 4.866116809189164, + "50": 8.899083268636325, + "60": 15.301746413861888, + "70": 23.76767736991377, + "80": 32.822889396729835 + }, + "share_below_abs_line_scaled": { + "90": 6.975293988224797, + "95": 7.977594856073962, + "100": 9.31103618296997, + "105": 10.700457901403947, + "110": 12.484983962653114 + }, + "by_group_abs_pct": { + "children": 10.562436526276183, + "working_age": 8.501985459054989, + "pensioners": 10.57590799235489 + }, + "by_group_rel_pct": { + "children": 18.75940248255886, + "working_age": 13.552832633740335, + "pensioners": 17.157864355662795 + }, + "group_pop_shares_pct": { + "children": 21.081650521464447, + "working_age": 60.852408020179105, + "pensioners": 18.065941458356452 + }, + "share_nonpositive_income_pct": 0.12655103103575885, + "person_quantiles_week": { + "p5": 303.5370905949519, + "p10": 389.5080003004808, + "p20": 494.11583533653845, + "p30": 575.0846980168269, + "p40": 664.0314002403846, + "p50": 751.48046875 + } + }, + "ahc": { + "abs_rate_pct": 13.066817529758456, + "rel_rate_person_median_pct": 19.51732487820331, + "rel_rate_household_median_pct": 18.624992595718805, + "median_person_weighted_week": 653.3710186298077, + "median_household_weighted_week": 638.1504657451923, + "abs_threshold_week": 326.76125411812575, + "rel_line_person_week": 392.02261117788464, + "abs_threshold_over_person_median": 0.5001159292363178, + "share_below_pct_of_person_median": { + "40": 8.148351559906608, + "50": 13.066817529758456, + "60": 19.51732487820331, + "70": 26.96916878981138, + "80": 34.929802781834965 + }, + "share_below_abs_line_scaled": { + "90": 10.196078019163515, + "95": 11.691041319095206, + "100": 13.066817529758456, + "105": 14.428285410356626, + "110": 16.125188367259234 + }, + "by_group_abs_pct": { + "children": 16.94963737314186, + "working_age": 13.252085213674231, + "pensioners": 7.911800511393495 + }, + "by_group_rel_pct": { + "children": 26.27755345435722, + "working_age": 18.200023030768143, + "pensioners": 16.065759017350224 + }, + "group_pop_shares_pct": { + "children": 21.081650521464447, + "working_age": 60.852408020179105, + "pensioners": 18.065941458356452 + }, + "share_nonpositive_income_pct": 1.1585762055090465, + "person_quantiles_week": { + "p5": 178.97040264423077, + "p10": 291.2607609675481, + "p20": 397.0943134014423, + "p30": 480.98482572115387, + "p40": 563.5982947716346, + "p50": 653.3710186298077 + } + }, + "decile_composition_person_weighted_weekly_per_household": { + "1": { + "market": 209.7408777427128, + "benefits": 185.64986204850382, + "taxes": 53.5106855421387, + "hbai_net": 341.8800548823869, + "uc": 86.50530807422567, + "state_pension": 42.047513890536464, + "hb": 0.9416238793841175, + "pension_credit": 2.575312364951102, + "child_benefit": 20.67391274245672, + "equiv_bhc": 283.76120129910726, + "persons_m": 6.924613445786714, + "equiv_factor_mean": 1.1878373701841825 + }, + "2": { + "market": 403.8767558442179, + "benefits": 241.42841141065443, + "taxes": 81.74639069878056, + "hbai_net": 563.5587760407568, + "uc": 105.0874249429487, + "state_pension": 62.39977578312995, + "hb": 3.0091407039992646, + "pension_credit": 3.1735653746555026, + "child_benefit": 25.358000246516674, + "equiv_bhc": 445.5344709632699, + "persons_m": 6.923021032057285, + "equiv_factor_mean": 1.2659378892350668 + }, + "3": { + "market": 515.0475341626217, + "benefits": 282.76211408586306, + "taxes": 107.6340343517, + "hbai_net": 690.1756135842884, + "uc": 106.48623436831716, + "state_pension": 82.91311021292469, + "hb": 1.8846128132581228, + "pension_credit": 6.061565400794136, + "child_benefit": 23.849700443272337, + "equiv_bhc": 536.6858310598434, + "persons_m": 6.925802743706226, + "equiv_factor_mean": 1.2861910987537617 + }, + "4": { + "market": 662.5601502599767, + "benefits": 246.31534964491328, + "taxes": 137.8039921763805, + "hbai_net": 771.0715082016343, + "uc": 77.09446026816627, + "state_pension": 76.98573627519137, + "hb": 2.4688567085509727, + "pension_credit": 4.783790759401175, + "child_benefit": 20.100138337081752, + "equiv_bhc": 619.6127045788005, + "persons_m": 6.922197212537289, + "equiv_factor_mean": 1.2445119301233563 + }, + "5": { + "market": 834.6668332380816, + "benefits": 233.5077275576045, + "taxes": 177.10419203752934, + "hbai_net": 891.0703671408488, + "uc": 64.54954057416786, + "state_pension": 81.64232298773109, + "hb": 1.6757863048250916, + "pension_credit": 3.2198813130203257, + "child_benefit": 18.75101686498397, + "equiv_bhc": 705.8434926936238, + "persons_m": 6.92499919271946, + "equiv_factor_mean": 1.2628717519913542 + }, + "6": { + "market": 1026.7526883715045, + "benefits": 223.04968764792787, + "taxes": 229.24353223059782, + "hbai_net": 1020.5588444963539, + "uc": 59.37171478843989, + "state_pension": 76.06756917295385, + "hb": 2.002129569879049, + "pension_credit": 3.6670639187304976, + "child_benefit": 17.277929386720988, + "equiv_bhc": 797.2807729190649, + "persons_m": 6.927222891370773, + "equiv_factor_mean": 1.2798050770893636 + }, + "7": { + "market": 1278.3357375811831, + "benefits": 209.78690954512584, + "taxes": 287.2151786241318, + "hbai_net": 1200.9074694589426, + "uc": 49.956669473160176, + "state_pension": 66.48620009047059, + "hb": 1.7453845957680594, + "pension_credit": 2.951371531952015, + "child_benefit": 15.03725316354566, + "equiv_bhc": 898.5878717562136, + "persons_m": 6.923978537809372, + "equiv_factor_mean": 1.3359968312626367 + }, + "8": { + "market": 1521.8042686805743, + "benefits": 202.98920763868108, + "taxes": 364.1279907341044, + "hbai_net": 1360.665482496997, + "uc": 58.65010684402825, + "state_pension": 66.98248772007165, + "hb": 0.471525203733026, + "pension_credit": 1.0774291706286923, + "child_benefit": 15.227886180573115, + "equiv_bhc": 1024.3175356423321, + "persons_m": 6.9121479408888815, + "equiv_factor_mean": 1.3282962556637723 + }, + "9": { + "market": 1983.3385240464454, + "benefits": 164.9697263166697, + "taxes": 516.4988836293776, + "hbai_net": 1631.8093665380145, + "uc": 42.26095160503408, + "state_pension": 57.65877707554444, + "hb": 0.4279518183168272, + "pension_credit": 1.7763086206023833, + "child_benefit": 12.381724480296178, + "equiv_bhc": 1186.8866160654845, + "persons_m": 6.937264704737664, + "equiv_factor_mean": 1.3781831134571274 + }, + "10": { + "market": 3698.645826330981, + "benefits": 103.62704185745325, + "taxes": 1337.4740930095031, + "hbai_net": 2464.79877219132, + "uc": 9.917974933073104, + "state_pension": 47.587995530307694, + "hb": 0.1967908778760721, + "pension_credit": 2.4423849866334084, + "child_benefit": 12.806900262633615, + "equiv_bhc": 1972.7081513537562, + "persons_m": 6.926900773700237, + "equiv_factor_mean": 1.2692959313716845 + } + }, + "totals_bn": { + "market": 1464.8448362239376, + "benefits": 307.23850180807904, + "taxes": 401.67094750633834, + "hbai_net": 1370.4123901551513, + "uc": 76.83272818724737, + "state_pension": 128.60448824558935, + "hb": 4.464219235857155, + "pension_credit": 6.537695825669497, + "child_benefit": 15.692417869013031 + }, + "population_m": 69.2481484753139, + "households_m": 29.010721861268284, + "engine_flags_pct": { + "in_poverty_bhc": 9.31103618296997, + "in_relative_poverty_bhc": 13.355658550011269, + "in_poverty_ahc": 13.066817529758456 + }, + "spine_s_design_weights": { + "bhc": { + "abs_rate_pct": 10.361960314290833, + "rel_rate_person_median_pct": 16.16368919108608, + "rel_rate_household_median_pct": 15.035711919547602, + "median_person_weighted_week": 745.38671875, + "median_household_weighted_week": 725.6853215144231, + "abs_threshold_week": 381.37943209134613, + "rel_line_person_week": 447.23203125, + "abs_threshold_over_person_median": 0.5116531090477605, + "share_below_pct_of_person_median": { + "40": 5.104617380892967, + "50": 9.773922037702388, + "60": 16.16368919108608, + "70": 24.000264044502885, + "80": 32.707630020630475 + }, + "share_below_abs_line_scaled": { + "90": 7.696497333364498, + "95": 9.033456035982717, + "100": 10.361960314290833, + "105": 11.833821078444462, + "110": 13.543869532897437 + }, + "by_group_abs_pct": { + "children": 10.917010494735573, + "working_age": 10.531785214037384, + "pensioners": 9.336453485997316 + }, + "by_group_rel_pct": { + "children": 18.446789492069485, + "working_age": 15.515752934377344, + "pensioners": 15.685309123112583 + }, + "group_pop_shares_pct": { + "children": 20.894629487752614, + "working_age": 58.16421342842894, + "pensioners": 20.94115708381844 + }, + "share_nonpositive_income_pct": 0.2859869392428203, + "person_quantiles_week": { + "p5": 295.76785982572113, + "p10": 376.1286433293269, + "p20": 486.14595853365387, + "p30": 571.8191856971154, + "p40": 660.3960336538462, + "p50": 745.38671875 + } + }, + "ahc": { + "abs_rate_pct": 13.773631159233107, + "rel_rate_person_median_pct": 19.825239154182615, + "rel_rate_household_median_pct": 19.859312379389234, + "median_person_weighted_week": 654.2380558894231, + "median_household_weighted_week": 654.4743088942307, + "abs_threshold_week": 326.76125411812575, + "rel_line_person_week": 392.54283353365383, + "abs_threshold_over_person_median": 0.49945314427467935, + "share_below_pct_of_person_median": { + "40": 8.956489801933476, + "50": 13.779349853942602, + "60": 19.825239154182615, + "70": 27.114978933716305, + "80": 34.85921822301734 + }, + "share_below_abs_line_scaled": { + "90": 10.903874104102165, + "95": 12.348497840192234, + "100": 13.773631159233107, + "105": 15.216163424785506, + "110": 16.71096567117924 + }, + "by_group_abs_pct": { + "children": 17.081780495201404, + "working_age": 15.080693522348342, + "pensioners": 6.842456900964894 + }, + "by_group_rel_pct": { + "children": 25.524602076052005, + "working_age": 20.166420529399772, + "pensioners": 13.190905514520214 + }, + "group_pop_shares_pct": { + "children": 20.894629487752614, + "working_age": 58.16421342842894, + "pensioners": 20.94115708381844 + }, + "share_nonpositive_income_pct": 1.5583880854793044, + "person_quantiles_week": { + "p5": 164.168701171875, + "p10": 279.2134352463942, + "p20": 393.73486328125, + "p30": 482.36831430288464, + "p40": 566.1413010817307, + "p50": 654.2380558894231 + } + }, + "decile_composition_person_weighted_weekly_per_household": { + "1": { + "market": 198.94082496708123, + "benefits": 171.01246297174072, + "taxes": 52.59416884116912, + "hbai_net": 317.3591197579969, + "uc": 83.20622443384974, + "state_pension": 38.66366032721232, + "hb": 0.6674791930897002, + "pension_credit": 3.4443040716182165, + "child_benefit": 19.751174682688113, + "equiv_bhc": 263.75750970613666, + "persons_m": 6.627048569279177, + "equiv_factor_mean": 1.1753207597860738 + }, + "2": { + "market": 398.9243639800398, + "benefits": 224.9339533818834, + "taxes": 80.79001308862834, + "hbai_net": 543.0683044090354, + "uc": 92.15785386619616, + "state_pension": 67.81227759655577, + "hb": 2.795812270487686, + "pension_credit": 4.781488527889011, + "child_benefit": 24.557551553430002, + "equiv_bhc": 434.193994111029, + "persons_m": 6.6319992261935985, + "equiv_factor_mean": 1.2505174014805662 + }, + "3": { + "market": 535.4524984374079, + "benefits": 239.54013503892318, + "taxes": 112.2095703977188, + "hbai_net": 662.7830619996373, + "uc": 81.02046372434611, + "state_pension": 82.90673328179363, + "hb": 2.1872951720572305, + "pension_credit": 3.370053053059126, + "child_benefit": 23.66593081593165, + "equiv_bhc": 530.4135205172363, + "persons_m": 6.628659314139337, + "equiv_factor_mean": 1.2506533692020965 + }, + "4": { + "market": 673.63085560077, + "benefits": 221.1890629180666, + "taxes": 145.76108343110997, + "hbai_net": 749.0588349774207, + "uc": 55.81873806144478, + "state_pension": 90.79346079881999, + "hb": 1.9627216764336501, + "pension_credit": 4.081846252655132, + "child_benefit": 21.009351420397852, + "equiv_bhc": 616.2017725678733, + "persons_m": 6.627502652476668, + "equiv_factor_mean": 1.2151819180723489 + }, + "5": { + "market": 847.2607440554614, + "benefits": 207.7248839599545, + "taxes": 191.4838943075372, + "hbai_net": 863.501733765406, + "uc": 45.81449174030248, + "state_pension": 88.92669417569451, + "hb": 1.9010302578032108, + "pension_credit": 3.0542439385616147, + "child_benefit": 19.55796584309726, + "equiv_bhc": 702.0116223895305, + "persons_m": 6.63076185957939, + "equiv_factor_mean": 1.230510064587136 + }, + "6": { + "market": 1043.8935740037082, + "benefits": 190.625101035878, + "taxes": 241.1939667250079, + "hbai_net": 993.3247086413802, + "uc": 38.454554158886076, + "state_pension": 81.8736710919489, + "hb": 2.27897589285833, + "pension_credit": 3.1413355492361514, + "child_benefit": 18.062919898177622, + "equiv_bhc": 791.826143705495, + "persons_m": 6.631437375670118, + "equiv_factor_mean": 1.255013721785733 + }, + "7": { + "market": 1244.932526273063, + "benefits": 173.65120740859172, + "taxes": 300.68312055245127, + "hbai_net": 1117.900614022017, + "uc": 26.457381852098575, + "state_pension": 83.16227057490977, + "hb": 1.6898951963394815, + "pension_credit": 2.4774315947061933, + "child_benefit": 16.182605062656425, + "equiv_bhc": 897.6331019093886, + "persons_m": 6.6286536772647855, + "equiv_factor_mean": 1.2459320446837456 + }, + "8": { + "market": 1515.794327168594, + "benefits": 157.9639951815206, + "taxes": 381.1857411438809, + "hbai_net": 1292.572579211799, + "uc": 20.481357665243873, + "state_pension": 79.67802820800924, + "hb": 0.5972974559402886, + "pension_credit": 1.4548021016801291, + "child_benefit": 14.764765770821198, + "equiv_bhc": 1034.4754782696568, + "persons_m": 6.628290781839843, + "equiv_factor_mean": 1.2488917113790854 + }, + "9": { + "market": 1938.6877746085643, + "benefits": 133.8820457432784, + "taxes": 530.3608778128201, + "hbai_net": 1542.208944006699, + "uc": 12.09027464941008, + "state_pension": 74.48361650882188, + "hb": 0.2164370370401927, + "pension_credit": 0.8594794503577574, + "child_benefit": 12.568136578454359, + "equiv_bhc": 1228.2773169082975, + "persons_m": 6.630071756433924, + "equiv_factor_mean": 1.2563227231577156 + }, + "10": { + "market": 3468.8780136305254, + "benefits": 102.46468337971685, + "taxes": 1246.9129105589361, + "hbai_net": 2324.4297882173787, + "uc": 4.154652807201615, + "state_pension": 63.86622724184254, + "hb": 0.32375106066148046, + "pension_credit": 1.6881351722223708, + "child_benefit": 11.62658545338353, + "equiv_bhc": 1957.3805804132012, + "persons_m": 6.631411039028923, + "equiv_factor_mean": 1.195880153502088 + } + }, + "totals_bn": { + "market": 1473.5271262516228, + "benefits": 273.11505337757023, + "taxes": 408.92363368698085, + "hbai_net": 1337.7185461054833, + "uc": 49.008402562651916, + "state_pension": 147.09701721913996, + "hb": 3.834187269606173, + "pension_credit": 5.762778493442082, + "child_benefit": 15.252532545538282 + }, + "population_m": 66.29583625190577, + "households_m": 29.36443299126203, + "engine_flags_pct": { + "in_poverty_bhc": 10.361960314290833, + "in_relative_poverty_bhc": 14.466576100377917, + "in_poverty_ahc": 13.773631159233107 + } + }, + "decomposition_bhc_rel": { + "design_weights_design_line_pct": 16.16368919108608, + "calibrated_weights_design_line_pct": 14.960303256735846, + "calibrated_weights_calibrated_line_pct": 15.301746413861888, + "design_median_week": 745.38671875, + "calibrated_median_week": 751.48046875, + "weight_scale_ratio_engine_vs_h5": 0.9999999997024301 + }, + "decomposition_bhc_abs": { + "design_weights_pct": 10.361960314290833, + "calibrated_weights_pct": 9.31103618296997 + } + }, + "2025": { + "bhc": { + "abs_rate_pct": 9.323482270722925, + "rel_rate_person_median_pct": 15.285052258148985, + "rel_rate_household_median_pct": 13.469155850513282, + "median_person_weighted_week": 778.4378004807693, + "median_household_weighted_week": 742.1250751201923, + "abs_threshold_week": 394.3464918870192, + "rel_line_person_week": 467.0626802884615, + "abs_threshold_over_person_median": 0.506587028075291, + "share_below_pct_of_person_median": { + "40": 4.883128815867428, + "50": 8.98832447842496, + "60": 15.285052258148985, + "70": 23.72498720959722, + "80": 32.743736252600755 + }, + "share_below_abs_line_scaled": { + "90": 6.952284186420167, + "95": 8.060600022952338, + "100": 9.323482270722925, + "105": 10.755105264044031, + "110": 12.413369028985919 + }, + "by_group_abs_pct": { + "children": 10.858290768761524, + "working_age": 8.546910208293452, + "pensioners": 10.148237718682198 + }, + "by_group_rel_pct": { + "children": 19.101403483420373, + "working_age": 13.596419346599667, + "pensioners": 16.519551784029346 + }, + "group_pop_shares_pct": { + "children": 21.081650514384382, + "working_age": 60.85240802533646, + "pensioners": 18.06594146027916 + }, + "share_nonpositive_income_pct": 0.12517331125128076, + "person_quantiles_week": { + "p5": 312.07825646033655, + "p10": 403.4509840745192, + "p20": 510.08841646634613, + "p30": 597.0752704326923, + "p40": 688.1170372596154, + "p50": 778.4378004807693 + } + }, + "ahc": { + "abs_rate_pct": 13.274205686667727, + "rel_rate_person_median_pct": 19.695280516960114, + "rel_rate_household_median_pct": 18.728185267342663, + "median_person_weighted_week": 673.1179387019231, + "median_household_weighted_week": 659.1632361778846, + "abs_threshold_week": 337.8712070861862, + "rel_line_person_week": 403.8707632211538, + "abs_threshold_over_person_median": 0.5019494915523351, + "share_below_pct_of_person_median": { + "40": 8.46954107014267, + "50": 13.175066079140105, + "60": 19.695280516960114, + "70": 27.154930518820525, + "80": 34.74227792570841 + }, + "share_below_abs_line_scaled": { + "90": 10.433635943233678, + "95": 11.96524931907333, + "100": 13.274205686667727, + "105": 14.894269172505812, + "110": 16.41345170826116 + }, + "by_group_abs_pct": { + "children": 17.311925599861734, + "working_age": 13.515722134654212, + "pensioners": 7.748965956311188 + }, + "by_group_rel_pct": { + "children": 26.905985767007866, + "working_age": 18.428379692385256, + "pensioners": 15.54827260686579 + }, + "group_pop_shares_pct": { + "children": 21.081650514384382, + "working_age": 60.85240802533646, + "pensioners": 18.06594146027916 + }, + "share_nonpositive_income_pct": 1.2300306776796177, + "person_quantiles_week": { + "p5": 178.89918870192307, + "p10": 296.72774564302887, + "p20": 407.52880859375, + "p30": 494.4497821514423, + "p40": 582.9683368389423, + "p50": 673.1179387019231 + } + }, + "decile_composition_person_weighted_weekly_per_household": { + "1": { + "market": 221.92540916016938, + "benefits": 191.07903147139663, + "taxes": 58.05885606429801, + "hbai_net": 354.94558327849006, + "uc": 91.04294241465324, + "state_pension": 42.231930305453766, + "hb": 0.9088239788239871, + "pension_credit": 2.3440917117244404, + "child_benefit": 21.565135442463404, + "equiv_bhc": 293.0713402277129, + "persons_m": 6.974002363422632, + "equiv_factor_mean": 1.1943502621106776 + }, + "2": { + "market": 417.1865302151592, + "benefits": 258.4029986725752, + "taxes": 87.66523196876626, + "hbai_net": 587.9242956962282, + "uc": 112.22902656386051, + "state_pension": 66.58016619994955, + "hb": 3.3993824297800566, + "pension_credit": 3.1301961663193536, + "child_benefit": 26.46089697234453, + "equiv_bhc": 460.8538870105919, + "persons_m": 6.955802734446764, + "equiv_factor_mean": 1.2759157135603258 + }, + "3": { + "market": 523.2619120450495, + "benefits": 286.1226153877204, + "taxes": 112.76425835677925, + "hbai_net": 696.6202687954963, + "uc": 104.5782868045406, + "state_pension": 86.95694307305703, + "hb": 1.9260741109793877, + "pension_credit": 5.890121950548572, + "child_benefit": 23.00834359311519, + "equiv_bhc": 554.9005637545578, + "persons_m": 6.989384446594238, + "equiv_factor_mean": 1.2563428337228368 + }, + "4": { + "market": 703.5603525070975, + "benefits": 256.03426145112655, + "taxes": 150.65566903122988, + "hbai_net": 808.9389455599871, + "uc": 81.33962380225542, + "state_pension": 80.16282464437793, + "hb": 2.939316229805306, + "pension_credit": 4.583743399498733, + "child_benefit": 21.347655491252826, + "equiv_bhc": 641.4993710755323, + "persons_m": 6.9467977743759155, + "equiv_factor_mean": 1.2620819288943725 + }, + "5": { + "market": 873.6402070225905, + "benefits": 246.35616974681506, + "taxes": 192.27315980871785, + "hbai_net": 927.723217369708, + "uc": 67.18761356642355, + "state_pension": 85.16272490902674, + "hb": 1.7899429613696196, + "pension_credit": 3.1933452124463844, + "child_benefit": 18.609641110091907, + "equiv_bhc": 731.7280721534592, + "persons_m": 7.006025578388691, + "equiv_factor_mean": 1.2681723870616592 + }, + "6": { + "market": 1079.3218887786722, + "benefits": 222.28904529073213, + "taxes": 249.79777379233337, + "hbai_net": 1051.8131605547644, + "uc": 55.97316768128731, + "state_pension": 82.30297680095939, + "hb": 2.138937679521919, + "pension_credit": 2.951014760781015, + "child_benefit": 17.15182310416914, + "equiv_bhc": 827.2793818603858, + "persons_m": 6.9675176125917435, + "equiv_factor_mean": 1.27072666125407 + }, + "7": { + "market": 1346.4047867166812, + "benefits": 213.60480426719363, + "taxes": 311.99719060408233, + "hbai_net": 1248.0123973070195, + "uc": 50.65713364800575, + "state_pension": 67.72260160968105, + "hb": 2.0822970185735312, + "pension_credit": 2.702840717594318, + "child_benefit": 15.70825416163373, + "equiv_bhc": 931.7738035585335, + "persons_m": 6.963589946418762, + "equiv_factor_mean": 1.3388474628709752 + }, + "8": { + "market": 1598.4408679856604, + "benefits": 214.12725894297895, + "taxes": 392.69507677592384, + "hbai_net": 1419.8730489725965, + "uc": 62.40410068097952, + "state_pension": 70.57623373627685, + "hb": 0.24690527229799428, + "pension_credit": 0.8856441870184659, + "child_benefit": 15.477063225936265, + "equiv_bhc": 1063.4885592927542, + "persons_m": 6.9749135331983565, + "equiv_factor_mean": 1.3343862979141625 + }, + "9": { + "market": 2071.54953427873, + "benefits": 165.3406500033413, + "taxes": 556.6699313934188, + "hbai_net": 1680.220256535401, + "uc": 41.41121879077054, + "state_pension": 60.941388061202716, + "hb": 0.4622213530235042, + "pension_credit": 1.8032290034155474, + "child_benefit": 12.303963247577435, + "equiv_bhc": 1231.8004319884349, + "persons_m": 6.974608459346294, + "equiv_factor_mean": 1.366550478716423 + }, + "10": { + "market": 3869.727287855322, + "benefits": 106.71308836922897, + "taxes": 1419.596047844269, + "hbai_net": 2556.8443301282628, + "uc": 10.0640279727355, + "state_pension": 49.29901285467216, + "hb": 0.2124731186521771, + "pension_credit": 2.337484438879809, + "child_benefit": 13.053788526545983, + "equiv_bhc": 2043.5672993073965, + "persons_m": 6.994092709913493, + "equiv_factor_mean": 1.2717695199286974 + } + }, + "totals_bn": { + "market": 1544.9873884610008, + "benefits": 320.78168396271457, + "taxes": 434.3171424491921, + "hbai_net": 1431.4519302533497, + "uc": 79.79920245274407, + "state_pension": 135.58027009173892, + "hb": 4.838911987967944, + "pension_credit": 6.187196380265217, + "child_benefit": 16.083866891768803 + }, + "population_m": 69.7467351586969, + "households_m": 29.219599065139295, + "engine_flags_pct": { + "in_poverty_bhc": 9.323482270722925, + "in_relative_poverty_bhc": 13.469155850513282, + "in_poverty_ahc": 13.274205686667727 + }, + "spine_s_design_weights": { + "bhc": { + "abs_rate_pct": 10.45081748529652, + "rel_rate_person_median_pct": 16.185258848065367, + "rel_rate_household_median_pct": 15.161379606650941, + "median_person_weighted_week": 773.0267427884615, + "median_household_weighted_week": 753.8106971153846, + "abs_threshold_week": 394.3464918870192, + "rel_line_person_week": 463.8160456730769, + "abs_threshold_over_person_median": 0.5101330524019555, + "share_below_pct_of_person_median": { + "40": 5.284494486645626, + "50": 9.849983205767465, + "60": 16.185258848065367, + "70": 24.12407970818208, + "80": 32.78811435643526 + }, + "share_below_abs_line_scaled": { + "90": 7.765509365017187, + "95": 9.081737739490817, + "100": 10.45081748529652, + "105": 11.942244823893104, + "110": 13.571895211034239 + }, + "by_group_abs_pct": { + "children": 11.317083081571889, + "working_age": 10.653944879213238, + "pensioners": 9.022288791998601 + }, + "by_group_rel_pct": { + "children": 18.869461725221402, + "working_age": 15.548190745725382, + "pensioners": 15.27648100742285 + }, + "group_pop_shares_pct": { + "children": 20.894629487752617, + "working_age": 58.16421342842896, + "pensioners": 20.941157083818442 + }, + "share_nonpositive_income_pct": 0.2826103836712187, + "person_quantiles_week": { + "p5": 303.88709435096155, + "p10": 388.5358698918269, + "p20": 503.52193509615387, + "p30": 592.5508939302885, + "p40": 684.8533653846154, + "p50": 773.0267427884615 + } + }, + "ahc": { + "abs_rate_pct": 14.049778120891391, + "rel_rate_person_median_pct": 20.168165641999142, + "rel_rate_household_median_pct": 20.221934042689348, + "median_person_weighted_week": 676.5720402644231, + "median_household_weighted_week": 677.2047025240385, + "abs_threshold_week": 337.8712070861862, + "rel_line_person_week": 405.94322415865383, + "abs_threshold_over_person_median": 0.49938689005554643, + "share_below_pct_of_person_median": { + "40": 9.248447874788972, + "50": 14.088966520173457, + "60": 20.168165641999142, + "70": 27.513280476377055, + "80": 34.917450618833286 + }, + "share_below_abs_line_scaled": { + "90": 11.252414897725245, + "95": 12.75302221855906, + "100": 14.049778120891391, + "105": 15.59106576375367, + "110": 17.048100294813732 + }, + "by_group_abs_pct": { + "children": 17.663354432660064, + "working_age": 15.374034406156294, + "pensioners": 6.766099250630969 + }, + "by_group_rel_pct": { + "children": 26.411317215281258, + "working_age": 20.542958535074927, + "pensioners": 12.897895323384688 + }, + "group_pop_shares_pct": { + "children": 20.894629487752617, + "working_age": 58.16421342842896, + "pensioners": 20.941157083818442 + }, + "share_nonpositive_income_pct": 1.6308242480217239, + "person_quantiles_week": { + "p5": 164.41248497596155, + "p10": 283.57117638221155, + "p20": 404.10903695913464, + "p30": 497.1654522235577, + "p40": 585.0177283653846, + "p50": 676.5720402644231 + } + }, + "decile_composition_person_weighted_weekly_per_household": { + "1": { + "market": 210.3617862856604, + "benefits": 175.4261117469092, + "taxes": 56.78814043553544, + "hbai_net": 328.9997577862635, + "uc": 85.97223144648014, + "state_pension": 37.72870827458308, + "hb": 2.20131988981467, + "pension_credit": 3.2669782924030994, + "child_benefit": 20.558844040894606, + "equiv_bhc": 271.5300105406725, + "persons_m": 6.676727409890088, + "equiv_factor_mean": 1.183208655325264 + }, + "2": { + "market": 412.4770912889353, + "benefits": 236.93785569457606, + "taxes": 86.37029141127965, + "hbai_net": 563.0446554377237, + "uc": 98.39905923084073, + "state_pension": 70.51296651105737, + "hb": 1.938088614004518, + "pension_credit": 4.830377735158439, + "child_benefit": 25.437878295571725, + "equiv_bhc": 448.79094044763485, + "persons_m": 6.677813911896572, + "equiv_factor_mean": 1.2546681539548596 + }, + "3": { + "market": 560.8394071248384, + "benefits": 243.0069741622645, + "taxes": 121.38137265380799, + "hbai_net": 682.4650093968914, + "uc": 79.83755724811576, + "state_pension": 87.34350224763465, + "hb": 2.594711134593452, + "pension_credit": 3.473948992894442, + "child_benefit": 23.793904497771397, + "equiv_bhc": 548.8990695036684, + "persons_m": 6.666908520454079, + "equiv_factor_mean": 1.2446284000292087 + }, + "4": { + "market": 707.446002261379, + "benefits": 226.24119046038138, + "taxes": 158.42645144286564, + "hbai_net": 775.2607420867321, + "uc": 56.901552452159905, + "state_pension": 93.56263197406727, + "hb": 2.426595566919263, + "pension_credit": 3.691007361244539, + "child_benefit": 21.421790728278065, + "equiv_bhc": 638.3548977107845, + "persons_m": 6.686656743645588, + "equiv_factor_mean": 1.2140300383571065 + }, + "5": { + "market": 894.9592910358934, + "benefits": 220.85803903064414, + "taxes": 209.3196121109151, + "hbai_net": 906.497719092489, + "uc": 47.84769198137398, + "state_pension": 95.29905750765442, + "hb": 1.9554681220101757, + "pension_credit": 3.0043446954065582, + "child_benefit": 20.008903576260312, + "equiv_bhc": 728.4741453685324, + "persons_m": 6.672835568656195, + "equiv_factor_mean": 1.2440298579869031 + }, + "6": { + "market": 1086.290145917138, + "benefits": 192.4578770548708, + "taxes": 258.8228823584319, + "hbai_net": 1019.9251395181606, + "uc": 36.36867207826757, + "state_pension": 87.43206551608031, + "hb": 2.4155016569160566, + "pension_credit": 2.7789808279864157, + "child_benefit": 17.91650724994517, + "equiv_bhc": 822.1284711426293, + "persons_m": 6.680984178751226, + "equiv_factor_mean": 1.2402612228510055 + }, + "7": { + "market": 1301.1212714432054, + "benefits": 180.84462049931406, + "taxes": 323.1537113972723, + "hbai_net": 1158.8121793892237, + "uc": 26.851419207596376, + "state_pension": 87.39567577573237, + "hb": 1.8283031419387923, + "pension_credit": 2.556383394105691, + "child_benefit": 16.467009925843985, + "equiv_bhc": 931.65710393482, + "persons_m": 6.675395867625039, + "equiv_factor_mean": 1.2443322407583126 + }, + "8": { + "market": 1586.354277118136, + "benefits": 160.44035333079043, + "taxes": 412.11165010338294, + "hbai_net": 1334.6829789889084, + "uc": 17.823681686881642, + "state_pension": 85.47766102200673, + "hb": 0.5316581545656278, + "pension_credit": 1.1762313418086638, + "child_benefit": 14.815578994447764, + "equiv_bhc": 1073.9420013080842, + "persons_m": 6.680139812433494, + "equiv_factor_mean": 1.242600887940923 + }, + "9": { + "market": 2036.343009028546, + "benefits": 141.04609542840836, + "taxes": 570.6311284834098, + "hbai_net": 1606.7579796462628, + "uc": 14.322385888109125, + "state_pension": 77.66387683387067, + "hb": 0.23226866849761837, + "pension_credit": 0.8648877529725376, + "child_benefit": 12.743837281151787, + "equiv_bhc": 1274.2309354867643, + "persons_m": 6.678110930165382, + "equiv_factor_mean": 1.2618399495010424 + }, + "10": { + "market": 3632.6597996311325, + "benefits": 106.40716686809347, + "taxes": 1329.3799954452275, + "hbai_net": 2409.686970782491, + "uc": 4.317780233181207, + "state_pension": 66.90326301846783, + "hb": 0.3498659112188177, + "pension_credit": 1.6972125130109765, + "child_benefit": 11.792191856811188, + "equiv_bhc": 2029.9552774056824, + "persons_m": 6.677593344186918, + "equiv_factor_mean": 1.194687740462182 + } + }, + "totals_bn": { + "market": 1553.7095252171562, + "benefits": 286.0873852713462, + "taxes": 442.3597828124515, + "hbai_net": 1397.4371280998746, + "uc": 50.60486958335118, + "state_pension": 155.38312770040827, + "hb": 4.195873377824732, + "pension_credit": 5.573042548460878, + "child_benefit": 15.633030889702873 + }, + "population_m": 66.77316628770457, + "households_m": 29.575856915347885, + "engine_flags_pct": { + "in_poverty_bhc": 10.45081748529652, + "in_relative_poverty_bhc": 14.625663553722859, + "in_poverty_ahc": 14.049778120891391 + } + }, + "decomposition_bhc_rel": { + "design_weights_design_line_pct": 16.185258848065367, + "calibrated_weights_design_line_pct": 15.025617165179494, + "calibrated_weights_calibrated_line_pct": 15.285052258148985, + "design_median_week": 773.0267427884615, + "calibrated_median_week": 778.4378004807693, + "weight_scale_ratio_engine_vs_h5": 1.0071999999233046 + }, + "decomposition_bhc_abs": { + "design_weights_pct": 10.45081748529652, + "calibrated_weights_pct": 9.323482270722925 + } + }, + "2026": { + "bhc": { + "abs_rate_pct": 8.740819809337934, + "rel_rate_person_median_pct": 14.63452283021983, + "rel_rate_household_median_pct": 12.759260476926432, + "median_person_weighted_week": 799.7987530048077, + "median_household_weighted_week": 762.9431340144231, + "abs_threshold_week": 403.41556490384613, + "rel_line_person_week": 479.8792518028846, + "abs_threshold_over_person_median": 0.5043963414399336, + "share_below_pct_of_person_median": { + "40": 4.68504748779905, + "50": 8.482655701288216, + "60": 14.63452283021983, + "70": 23.105278706980815, + "80": 32.11388842676875 + }, + "share_below_abs_line_scaled": { + "90": 6.533342876639281, + "95": 7.615886352112203, + "100": 8.740819809337934, + "105": 10.006807169966693, + "110": 11.657025935149596 + }, + "by_group_abs_pct": { + "children": 9.661984378129434, + "working_age": 8.240440662912821, + "pensioners": 9.351339139227507 + }, + "by_group_rel_pct": { + "children": 17.412147526264324, + "working_age": 13.283345592982245, + "pensioners": 15.944472576935235 + }, + "group_pop_shares_pct": { + "children": 21.08165051217406, + "working_age": 60.85240801933478, + "pensioners": 18.065941468491157 + }, + "share_nonpositive_income_pct": 0.11875704265628229, + "person_quantiles_week": { + "p5": 326.9440730168269, + "p10": 423.5251652644231, + "p20": 529.9164287860577, + "p30": 618.0621995192307, + "p40": 709.4474909855769, + "p50": 799.7987530048077 + } + }, + "ahc": { + "abs_rate_pct": 12.773078128628862, + "rel_rate_person_median_pct": 19.086946645184565, + "rel_rate_household_median_pct": 18.233173965439352, + "median_person_weighted_week": 691.2844801682693, + "median_household_weighted_week": 677.7602163461538, + "abs_threshold_week": 345.64151696038755, + "rel_line_person_week": 414.7706881009615, + "abs_threshold_over_person_median": 0.49999895394187516, + "share_below_pct_of_person_median": { + "40": 8.334524158867099, + "50": 12.773078128628862, + "60": 19.086946645184565, + "70": 26.808491094247906, + "80": 34.43522420294095 + }, + "share_below_abs_line_scaled": { + "90": 10.108761918360006, + "95": 11.465551208501946, + "100": 12.773078128628862, + "105": 14.213182921247322, + "110": 15.831964438699849 + }, + "by_group_abs_pct": { + "children": 16.052102290122054, + "working_age": 13.340015918258688, + "pensioners": 7.03704874591201 + }, + "by_group_rel_pct": { + "children": 25.15087090572062, + "working_age": 18.265129861174124, + "pensioners": 14.778950747438873 + }, + "group_pop_shares_pct": { + "children": 21.08165051217406, + "working_age": 60.85240801933478, + "pensioners": 18.065941468491157 + }, + "share_nonpositive_income_pct": 1.2624019834941527, + "person_quantiles_week": { + "p5": 185.48907001201923, + "p10": 308.82010591947113, + "p20": 422.70015775240387, + "p30": 512.1095628004807, + "p40": 599.0629131610577, + "p50": 691.2844801682693 + } + }, + "decile_composition_person_weighted_weekly_per_household": { + "1": { + "market": 227.16208371260922, + "benefits": 194.33159631525308, + "taxes": 61.555351005855464, + "hbai_net": 359.9383288917483, + "uc": 91.08557744820882, + "state_pension": 44.274263286904926, + "hb": 1.205496398593097, + "pension_credit": 2.6703013941444023, + "child_benefit": 20.248154137799684, + "equiv_bhc": 305.33939158364547, + "persons_m": 6.997711355805635, + "equiv_factor_mean": 1.1682662097070446 + }, + "2": { + "market": 439.12404854613186, + "benefits": 279.98054117888955, + "taxes": 94.65973198844895, + "hbai_net": 624.4448576880291, + "uc": 129.05771056383213, + "state_pension": 68.4667922163435, + "hb": 2.6206199294595147, + "pension_credit": 3.1467403218892116, + "child_benefit": 27.77383498324311, + "equiv_bhc": 480.1397546419985, + "persons_m": 7.004427216351509, + "equiv_factor_mean": 1.29995621409035 + }, + "3": { + "market": 559.2200528531984, + "benefits": 288.0975983188929, + "taxes": 124.373753339956, + "hbai_net": 722.9438988863774, + "uc": 106.16057965609365, + "state_pension": 88.5988606621818, + "hb": 2.4425623276118493, + "pension_credit": 5.933563277797118, + "child_benefit": 23.82880937034239, + "equiv_bhc": 576.2601613026749, + "persons_m": 6.9813964583725925, + "equiv_factor_mean": 1.2537580052929467 + }, + "4": { + "market": 701.2308235776723, + "benefits": 296.5497448377491, + "taxes": 154.78924719403227, + "hbai_net": 842.9913199786911, + "uc": 104.70138482027465, + "state_pension": 84.42978485612456, + "hb": 2.3077170221065195, + "pension_credit": 5.156296141427854, + "child_benefit": 23.98252858588805, + "equiv_bhc": 663.4326562873073, + "persons_m": 7.015239436738968, + "equiv_factor_mean": 1.2713381087030047 + }, + "5": { + "market": 903.2779543107698, + "benefits": 258.9947959709738, + "taxes": 204.70087655110459, + "hbai_net": 957.5718735037088, + "uc": 73.2177163028071, + "state_pension": 88.19273814940006, + "hb": 2.7256789550099136, + "pension_credit": 3.964556991196741, + "child_benefit": 20.12270337845587, + "equiv_bhc": 752.5409222251981, + "persons_m": 7.005933398739815, + "equiv_factor_mean": 1.272447559663959 + }, + "6": { + "market": 1121.2372530288226, + "benefits": 222.83986008433627, + "taxes": 265.98716230948946, + "hbai_net": 1078.0899520497355, + "uc": 49.866987158322324, + "state_pension": 85.907352037295, + "hb": 2.0929823538990413, + "pension_credit": 3.1725531769687616, + "child_benefit": 17.405143432142115, + "equiv_bhc": 851.2872366145358, + "persons_m": 7.000427022171021, + "equiv_factor_mean": 1.265715782117665 + }, + "7": { + "market": 1376.2432696149458, + "benefits": 235.21378202877384, + "taxes": 325.24143887215683, + "hbai_net": 1286.2156135757164, + "uc": 59.91344387146302, + "state_pension": 74.17737707937611, + "hb": 1.5128423207224875, + "pension_credit": 2.772364669106728, + "child_benefit": 16.914206307213966, + "equiv_bhc": 956.8328698331219, + "persons_m": 7.001322208899498, + "equiv_factor_mean": 1.3434461284543364 + }, + "8": { + "market": 1662.9748332655315, + "benefits": 224.7014802006747, + "taxes": 418.3107841216324, + "hbai_net": 1469.365530394715, + "uc": 64.1440716821363, + "state_pension": 75.6708721027551, + "hb": 1.2287505622990407, + "pension_credit": 1.629428831507148, + "child_benefit": 16.327020865900753, + "equiv_bhc": 1090.330532699896, + "persons_m": 7.002574889716625, + "equiv_factor_mean": 1.3465071029316398 + }, + "9": { + "market": 2145.211136891477, + "benefits": 143.65936292131363, + "taxes": 589.2193905628272, + "hbai_net": 1699.6511063590951, + "uc": 19.47120221358012, + "state_pension": 65.65053462299737, + "hb": 0.4478755943294463, + "pension_credit": 2.1516288219388127, + "child_benefit": 11.770166485221932, + "equiv_bhc": 1262.7513174957965, + "persons_m": 6.999710599377632, + "equiv_factor_mean": 1.3470071758662678 + }, + "10": { + "market": 4006.658752327156, + "benefits": 107.44218080808767, + "taxes": 1491.611817901835, + "hbai_net": 2622.489119723301, + "uc": 7.189431504867519, + "state_pension": 53.457268788782066, + "hb": 0.261008249828476, + "pension_credit": 2.6137955826177803, + "child_benefit": 13.320382475498674, + "equiv_bhc": 2097.649903417026, + "persons_m": 7.003030142487288, + "equiv_factor_mean": 1.2704674956777993 + } + }, + "totals_bn": { + "market": 1603.579104846361, + "benefits": 335.2848044417536, + "taxes": 460.3930309242946, + "hbai_net": 1478.470879574793, + "uc": 82.27460669261878, + "state_pension": 143.34295439981796, + "hb": 5.070113869468493, + "pension_credit": 6.911159084135606, + "child_benefit": 16.760951441502275 + }, + "population_m": 70.01177272866059, + "households_m": 29.33063353709817, + "engine_flags_pct": { + "in_poverty_bhc": 8.740819809337934, + "in_relative_poverty_bhc": 12.759260476926432, + "in_poverty_ahc": 12.773078128628862 + }, + "spine_s_design_weights": { + "bhc": { + "abs_rate_pct": 9.82998779211222, + "rel_rate_person_median_pct": 15.593000636976221, + "rel_rate_household_median_pct": 14.649745421649094, + "median_person_weighted_week": 796.302734375, + "median_household_weighted_week": 777.7744140625, + "abs_threshold_week": 403.41556490384613, + "rel_line_person_week": 477.781640625, + "abs_threshold_over_person_median": 0.506610799497603, + "share_below_pct_of_person_median": { + "40": 5.06469153215283, + "50": 9.559050347655436, + "60": 15.593000636976221, + "70": 23.4961814099124, + "80": 32.2545792670541 + }, + "share_below_abs_line_scaled": { + "90": 7.266547474482094, + "95": 8.571522735622365, + "100": 9.82998779211222, + "105": 11.127294458390907, + "110": 12.690004132791055 + }, + "by_group_abs_pct": { + "children": 10.112501056150053, + "working_age": 10.318144584288003, + "pensioners": 8.192243245007095 + }, + "by_group_rel_pct": { + "children": 17.306190842067977, + "working_age": 15.310514605499895, + "pensioners": 14.668223812343909 + }, + "group_pop_shares_pct": { + "children": 20.89462948775261, + "working_age": 58.16421342842894, + "pensioners": 20.94115708381844 + }, + "share_nonpositive_income_pct": 0.28046893625003594, + "person_quantiles_week": { + "p5": 317.43276742788464, + "p10": 405.8301532451923, + "p20": 525.0134465144231, + "p30": 615.8310922475962, + "p40": 706.3608774038462, + "p50": 796.302734375 + } + }, + "ahc": { + "abs_rate_pct": 13.518400637861628, + "rel_rate_person_median_pct": 19.58178855201802, + "rel_rate_household_median_pct": 19.689399387674857, + "median_person_weighted_week": 696.0784254807693, + "median_household_weighted_week": 698.642578125, + "abs_threshold_week": 345.64151696038755, + "rel_line_person_week": 417.6470552884615, + "abs_threshold_over_person_median": 0.4965554229347921, + "share_below_pct_of_person_median": { + "40": 9.103982577193506, + "50": 13.658741139901103, + "60": 19.58178855201802, + "70": 27.092595245945922, + "80": 34.68354648220801 + }, + "share_below_abs_line_scaled": { + "90": 10.811273362195935, + "95": 12.295968990227042, + "100": 13.518400637861628, + "105": 14.783539516569805, + "110": 16.38968056785799 + }, + "by_group_abs_pct": { + "children": 16.424182546839045, + "working_age": 15.150255338542117, + "pensioners": 6.086586778105001 + }, + "by_group_rel_pct": { + "children": 24.68517160123495, + "working_age": 20.360304835945602, + "pensioners": 12.327409787975261 + }, + "group_pop_shares_pct": { + "children": 20.89462948775261, + "working_age": 58.16421342842894, + "pensioners": 20.94115708381844 + }, + "share_nonpositive_income_pct": 1.6322841622082547, + "person_quantiles_week": { + "p5": 170.06912935697116, + "p10": 295.96982046274036, + "p20": 422.1852463942308, + "p30": 515.2130033052885, + "p40": 603.8011944110577, + "p50": 696.0784254807693 + } + }, + "decile_composition_person_weighted_weekly_per_household": { + "1": { + "market": 216.5210939943648, + "benefits": 180.8590327913275, + "taxes": 59.84335589184183, + "hbai_net": 337.5367708973127, + "uc": 88.30038138773108, + "state_pension": 39.84807643260239, + "hb": 2.347133369871548, + "pension_credit": 3.629501656067105, + "child_benefit": 19.921246024279757, + "equiv_bhc": 282.6464455844677, + "persons_m": 6.70242147111856, + "equiv_factor_mean": 1.1692031411521295 + }, + "2": { + "market": 432.4045193325062, + "benefits": 246.8178822378244, + "taxes": 93.15861493208605, + "hbai_net": 586.0637868649942, + "uc": 104.68362036805668, + "state_pension": 74.43603319821685, + "hb": 2.0885210976086825, + "pension_credit": 5.121742209974366, + "child_benefit": 25.310399725166185, + "equiv_bhc": 468.95374574594774, + "persons_m": 6.70204447566685, + "equiv_factor_mean": 1.248730979325275 + }, + "3": { + "market": 588.3684435914486, + "benefits": 255.24883223387008, + "taxes": 132.4220338178338, + "hbai_net": 711.1952415298315, + "uc": 90.28557205629085, + "state_pension": 88.25807862328317, + "hb": 2.5251967505381834, + "pension_credit": 3.430123592697613, + "child_benefit": 25.13659553395923, + "equiv_bhc": 571.0514156317196, + "persons_m": 6.700734650025831, + "equiv_factor_mean": 1.246926110513895 + }, + "4": { + "market": 732.4403575929445, + "benefits": 255.2709288012757, + "taxes": 169.37310593555773, + "hbai_net": 818.3381802678915, + "uc": 75.21743446355595, + "state_pension": 97.85621919115059, + "hb": 2.1976415235083198, + "pension_credit": 3.645716760316542, + "child_benefit": 24.230393645530345, + "equiv_bhc": 661.2701689084718, + "persons_m": 6.704530882246157, + "equiv_factor_mean": 1.23717395618559 + }, + "5": { + "market": 925.4018247008169, + "benefits": 232.55424286529376, + "taxes": 221.86344919117607, + "hbai_net": 936.092618815273, + "uc": 52.08161970923515, + "state_pension": 98.46760915116829, + "hb": 2.3213146614864133, + "pension_credit": 4.034696832146456, + "child_benefit": 21.226762131740053, + "equiv_bhc": 750.9121633505466, + "persons_m": 6.703194200822092, + "equiv_factor_mean": 1.2459600882330175 + }, + "6": { + "market": 1122.4943598088958, + "benefits": 197.62014756701154, + "taxes": 273.32924660765605, + "hbai_net": 1046.7852615666366, + "uc": 35.14059405848254, + "state_pension": 93.30010241437253, + "hb": 2.324365842300129, + "pension_credit": 3.0119138080503793, + "child_benefit": 18.407410193435, + "equiv_bhc": 846.7877441863633, + "persons_m": 6.702274371760789, + "equiv_factor_mean": 1.2359564775644512 + }, + "7": { + "market": 1347.2731014404885, + "benefits": 192.0901621921187, + "taxes": 343.5598621317364, + "hbai_net": 1195.803402469521, + "uc": 30.029205280417973, + "state_pension": 91.94029594852367, + "hb": 2.0919911931536848, + "pension_credit": 2.7429330782328445, + "child_benefit": 17.346710204828334, + "equiv_bhc": 958.9099980714722, + "persons_m": 6.702194061357594, + "equiv_factor_mean": 1.2477035895810729 + }, + "8": { + "market": 1630.461527851325, + "benefits": 170.94550440365907, + "taxes": 432.92088727451824, + "hbai_net": 1368.4861464116098, + "uc": 18.32560190030833, + "state_pension": 93.26343644237653, + "hb": 0.7193639687659945, + "pension_credit": 1.4652731225063946, + "child_benefit": 15.204661292833249, + "equiv_bhc": 1103.573685650061, + "persons_m": 6.7036487001444645, + "equiv_factor_mean": 1.2400578011069758 + }, + "9": { + "market": 2102.7915968019774, + "benefits": 143.5069877661796, + "taxes": 603.3062624901054, + "hbai_net": 1642.992321079044, + "uc": 11.131176486411682, + "state_pension": 83.90184036219625, + "hb": 0.19477527369310485, + "pension_credit": 1.1367944899587976, + "child_benefit": 13.085125288753982, + "equiv_bhc": 1307.3944414957534, + "persons_m": 6.701890785017252, + "equiv_factor_mean": 1.257682264921373 + }, + "10": { + "market": 3752.1327411291777, + "benefits": 113.06557891534905, + "taxes": 1394.8410978217948, + "hbai_net": 2470.357221848004, + "uc": 4.347332959800716, + "state_pension": 71.94069506854095, + "hb": 0.42702810585479656, + "pension_credit": 1.9158672135696944, + "child_benefit": 12.130330009729445, + "equiv_bhc": 2080.638708925674, + "persons_m": 6.703970711180707, + "equiv_factor_mean": 1.1948647059620576 + } + }, + "totals_bn": { + "market": 1612.805385230441, + "benefits": 301.62339235562916, + "taxes": 469.54223478017514, + "hbai_net": 1444.886543192967, + "uc": 53.874940992731446, + "state_pension": 164.67192059629252, + "hb": 4.398346810279494, + "pension_credit": 6.164083735703407, + "child_benefit": 16.29100361119545 + }, + "population_m": 67.0269043093403, + "households_m": 29.688245167082822, + "engine_flags_pct": { + "in_poverty_bhc": 9.82998779211222, + "in_relative_poverty_bhc": 13.925169185473022, + "in_poverty_ahc": 13.518400637861628 + } + }, + "decomposition_bhc_rel": { + "design_weights_design_line_pct": 15.593000636976221, + "calibrated_weights_design_line_pct": 14.411192671790495, + "calibrated_weights_calibrated_line_pct": 14.63452283021983, + "design_median_week": 796.302734375, + "calibrated_median_week": 799.7987530048077, + "weight_scale_ratio_engine_vs_h5": 1.011027359768289 + }, + "decomposition_bhc_abs": { + "design_weights_pct": 9.82998779211222, + "calibrated_weights_pct": 8.740819809337934 + } + } + } + }, + "efrs_1_57_3": { + "h5": "/Users/mariajuaristi/.cache/huggingface/hub/models--policyengine--policyengine-uk-data-private/snapshots/25af520a6651b8812fef56964a42a79a3f9f515a/enhanced_frs_2024_25.h5", + "years": { + "2024": { + "bhc": { + "abs_rate_pct": 11.248234644883528, + "rel_rate_person_median_pct": 19.67636083543157, + "rel_rate_household_median_pct": 18.32336759407664, + "median_person_weighted_week": 780.1297325721154, + "median_household_weighted_week": 756.0951772836538, + "abs_threshold_week": 381.37943209134613, + "rel_line_person_week": 468.07783954326925, + "abs_threshold_over_person_median": 0.4888666796917541, + "share_below_pct_of_person_median": { + "40": 7.146337809968621, + "50": 11.775091805879931, + "60": 19.67636083543157, + "70": 27.067102743756017, + "80": 34.819159408841756 + }, + "share_below_abs_line_scaled": { + "90": 8.815035911304177, + "95": 10.137185007718035, + "100": 11.248234644883528, + "105": 12.596001568459187, + "110": 14.245238248673846 + }, + "by_group_abs_pct": { + "children": 12.53077176093861, + "working_age": 11.341886851261055, + "pensioners": 9.305284734931432 + }, + "by_group_rel_pct": { + "children": 24.76065359849939, + "working_age": 17.28684657065702, + "pensioners": 21.53642491175725 + }, + "group_pop_shares_pct": { + "children": 22.041199987501038, + "working_age": 60.493596373124525, + "pensioners": 17.465203639374437 + }, + "share_nonpositive_income_pct": 0.5815278154779585, + "person_quantiles_week": { + "p5": 278.5449030949519, + "p10": 359.0090895432692, + "p20": 471.1292067307692, + "p30": 570.0135967548077, + "p40": 675.75390625, + "p50": 780.1297325721154 + } + }, + "ahc": { + "abs_rate_pct": 14.636882117887081, + "rel_rate_person_median_pct": 23.66888127953728, + "rel_rate_household_median_pct": 22.299445826202227, + "median_person_weighted_week": 682.5825570913462, + "median_household_weighted_week": 656.4424579326923, + "abs_threshold_week": 326.76125411812575, + "rel_line_person_week": 409.54953425480767, + "abs_threshold_over_person_median": 0.47871316183427925, + "share_below_pct_of_person_median": { + "40": 10.79241388655313, + "50": 15.789426166702736, + "60": 23.66888127953728, + "70": 30.131618777908265, + "80": 36.55219975106158 + }, + "share_below_abs_line_scaled": { + "90": 12.163253311872415, + "95": 13.31703792975646, + "100": 14.636882117887081, + "105": 15.926083525386197, + "110": 18.16162220961478 + }, + "by_group_abs_pct": { + "children": 18.406832267236265, + "working_age": 15.30255344756186, + "pensioners": 7.57351884618117 + }, + "by_group_rel_pct": { + "children": 32.58141885813711, + "working_age": 22.081855240330587, + "pensioners": 17.918126087070124 + }, + "group_pop_shares_pct": { + "children": 22.041199987501038, + "working_age": 60.493596373124525, + "pensioners": 17.465203639374437 + }, + "share_nonpositive_income_pct": 1.7138610026959502, + "person_quantiles_week": { + "p5": 146.8833289513221, + "p10": 253.63504732572116, + "p20": 379.76990685096155, + "p30": 475.6253004807692, + "p40": 578.3253831129807, + "p50": 682.5825570913462 + } + }, + "decile_composition_person_weighted_weekly_per_household": { + "1": { + "market": 201.11114140598968, + "benefits": 173.19090325528168, + "taxes": 95.00938007991967, + "hbai_net": 279.2926644238555, + "uc": 84.64104787598282, + "state_pension": 32.73469296908871, + "hb": 0.12701675104014482, + "pension_credit": 1.0606568896716324, + "child_benefit": 20.96934776629559, + "equiv_bhc": 230.7606919290198, + "persons_m": 6.910128566314966, + "equiv_factor_mean": 1.1504458899457846 + }, + "2": { + "market": 291.9175659840563, + "benefits": 282.634275988276, + "taxes": 59.097544391982595, + "hbai_net": 515.4542969155966, + "uc": 128.56296343317482, + "state_pension": 76.07559359804341, + "hb": 4.020281678041794, + "pension_credit": 4.179463907884714, + "child_benefit": 28.1215681709052, + "equiv_bhc": 422.08350478760445, + "persons_m": 6.923218229840457, + "equiv_factor_mean": 1.2206002426432279 + }, + "3": { + "market": 435.09336458961377, + "benefits": 284.5741743284019, + "taxes": 85.91648863688566, + "hbai_net": 633.7510494790184, + "uc": 116.57937333105916, + "state_pension": 71.19873958288967, + "hb": 4.427183405775146, + "pension_credit": 1.7226511816463383, + "child_benefit": 26.097061080922686, + "equiv_bhc": 521.8726620984175, + "persons_m": 6.8078717361290755, + "equiv_factor_mean": 1.214878793299162 + }, + "4": { + "market": 649.3945615631098, + "benefits": 226.2501348510077, + "taxes": 136.4575768945451, + "hbai_net": 739.1871213163231, + "uc": 65.91780969361818, + "state_pension": 86.29518311944548, + "hb": 1.893628751510577, + "pension_credit": 1.2193529746634226, + "child_benefit": 18.16518863336191, + "equiv_bhc": 621.3618507172408, + "persons_m": 7.028938577704921, + "equiv_factor_mean": 1.1920816893792385 + }, + "5": { + "market": 815.5479888418301, + "benefits": 219.18847978341412, + "taxes": 165.442209291435, + "hbai_net": 869.2942585941148, + "uc": 58.20121200871212, + "state_pension": 75.04171554074271, + "hb": 8.601090617350916, + "pension_credit": 2.918068545960314, + "child_benefit": 17.102301703322972, + "equiv_bhc": 731.007066437893, + "persons_m": 6.8218880381258575, + "equiv_factor_mean": 1.1918965001574697 + }, + "6": { + "market": 1039.959218061144, + "benefits": 188.39811066978447, + "taxes": 214.16918338367157, + "hbai_net": 1014.188144870832, + "uc": 31.195312343501705, + "state_pension": 73.06691545024387, + "hb": 9.128591462442612, + "pension_credit": 2.8921072387564077, + "child_benefit": 13.061452731255734, + "equiv_bhc": 828.8075890612306, + "persons_m": 6.64609677523014, + "equiv_factor_mean": 1.2234572326384632 + }, + "7": { + "market": 1200.1941140164652, + "benefits": 152.59842297144408, + "taxes": 256.777056141372, + "hbai_net": 1096.0154820697155, + "uc": 19.695719416615017, + "state_pension": 59.700137095628676, + "hb": 7.260230807552104, + "pension_credit": 5.197661583821304, + "child_benefit": 12.85763115468154, + "equiv_bhc": 949.5850419241118, + "persons_m": 7.247749828826055, + "equiv_factor_mean": 1.1519648824920303 + }, + "8": { + "market": 1776.7377173366697, + "benefits": 95.27788853210161, + "taxes": 413.2418381112336, + "hbai_net": 1458.773768000911, + "uc": 6.992842109271558, + "state_pension": 40.22304341648491, + "hb": 0.0, + "pension_credit": 0.08030411485747825, + "child_benefit": 16.744647577803796, + "equiv_bhc": 1098.6289379322818, + "persons_m": 6.899541079229623, + "equiv_factor_mean": 1.3312230213552319 + }, + "9": { + "market": 2101.1886129353584, + "benefits": 114.9579223128488, + "taxes": 515.4476839146556, + "hbai_net": 1700.6988589809155, + "uc": 5.937267578064827, + "state_pension": 37.716605136800176, + "hb": 0.0393471268135906, + "pension_credit": 0.16476057048080192, + "child_benefit": 16.821010398674716, + "equiv_bhc": 1303.8405115069038, + "persons_m": 6.944625464099169, + "equiv_factor_mean": 1.3094210519094338 + }, + "10": { + "market": 3781.7808674796734, + "benefits": 101.71995015877461, + "taxes": 1350.1982149300686, + "hbai_net": 2533.302596297898, + "uc": 4.659273316206283, + "state_pension": 42.47437016113914, + "hb": 0.8463633207246434, + "pension_credit": 1.823569668021381, + "child_benefit": 13.792883003266839, + "equiv_bhc": 2091.1939594748083, + "persons_m": 6.949319931458503, + "equiv_factor_mean": 1.2061431384562689 + } + }, + "totals_bn": { + "market": 1580.5557396343675, + "benefits": 296.7170666991969, + "taxes": 428.04769895445526, + "hbai_net": 1449.2251081525528, + "uc": 73.09709172161321, + "state_pension": 119.29508571145112, + "hb": 12.244102252339145, + "pension_credit": 6.447777116554897, + "child_benefit": 16.475751731474123 + }, + "population_m": 69.17937822695876, + "households_m": 31.116924677781938, + "engine_flags_pct": { + "in_poverty_bhc": 11.248234644883528, + "in_relative_poverty_bhc": 18.32336759407664, + "in_poverty_ahc": 14.636882117887081 + } + }, + "2025": { + "bhc": { + "abs_rate_pct": 11.42799645172305, + "rel_rate_person_median_pct": 20.077452424323738, + "rel_rate_household_median_pct": 18.299262107818574, + "median_person_weighted_week": 813.4892578125, + "median_household_weighted_week": 784.3092698317307, + "abs_threshold_week": 394.3464918870192, + "rel_line_person_week": 488.09355468749993, + "abs_threshold_over_person_median": 0.4847593107098061, + "share_below_pct_of_person_median": { + "40": 7.323950923711547, + "50": 12.151242042807754, + "60": 20.077452424323738, + "70": 27.286061636979987, + "80": 35.15903256530913 + }, + "share_below_abs_line_scaled": { + "90": 8.805470604418515, + "95": 10.157973287527229, + "100": 11.42799645172305, + "105": 13.00699086241165, + "110": 14.369865566403197 + }, + "by_group_abs_pct": { + "children": 13.007408286794687, + "working_age": 11.56536758215088, + "pensioners": 8.958961070449618 + }, + "by_group_rel_pct": { + "children": 25.894443932341787, + "working_age": 17.56578267080024, + "pensioners": 21.435951506399192 + }, + "group_pop_shares_pct": { + "children": 22.04119997437577, + "working_age": 60.49359635126953, + "pensioners": 17.465203674354708 + }, + "share_nonpositive_income_pct": 0.5838224466197276, + "person_quantiles_week": { + "p5": 282.70297475961536, + "p10": 372.5285456730769, + "p20": 487.14806189903845, + "p30": 591.6826547475962, + "p40": 700.0447716346154, + "p50": 813.4892578125 + } + }, + "ahc": { + "abs_rate_pct": 14.840663693140588, + "rel_rate_person_median_pct": 23.958998986851757, + "rel_rate_household_median_pct": 22.037153391057977, + "median_person_weighted_week": 706.3498347355769, + "median_household_weighted_week": 674.5321514423077, + "abs_threshold_week": 337.8712070861862, + "rel_line_person_week": 423.8099008413461, + "abs_threshold_over_person_median": 0.4783340923590203, + "share_below_pct_of_person_median": { + "40": 11.029931837128567, + "50": 16.630328511745613, + "60": 23.958998986851757, + "70": 30.160143231381337, + "80": 36.498491659869295 + }, + "share_below_abs_line_scaled": { + "90": 12.432288894397505, + "95": 13.542250567931394, + "100": 14.840663693140588, + "105": 16.849399076549396, + "110": 18.595881718727714 + }, + "by_group_abs_pct": { + "children": 18.664204266272606, + "working_age": 15.586057996673565, + "pensioners": 7.433535387333312 + }, + "by_group_rel_pct": { + "children": 33.465426958827855, + "working_age": 22.329254352146243, + "pensioners": 17.606713153065076 + }, + "group_pop_shares_pct": { + "children": 22.04119997437577, + "working_age": 60.49359635126953, + "pensioners": 17.465203674354708 + }, + "share_nonpositive_income_pct": 1.788761807085206, + "person_quantiles_week": { + "p5": 148.54150390625, + "p10": 261.22787710336536, + "p20": 386.7255108173077, + "p30": 492.06591796875, + "p40": 595.3966346153846, + "p50": 706.3498347355769 + } + }, + "decile_composition_person_weighted_weekly_per_household": { + "1": { + "market": 206.5436065586196, + "benefits": 180.2002213691758, + "taxes": 98.92562852533077, + "hbai_net": 287.8181982921596, + "uc": 89.39431649965, + "state_pension": 33.411493136709105, + "hb": 0.1778481714901296, + "pension_credit": 0.9928710705482342, + "child_benefit": 21.518195139081765, + "equiv_bhc": 237.60242117173357, + "persons_m": 6.888798299313158, + "equiv_factor_mean": 1.1538667782332979 + }, + "2": { + "market": 309.3798048722266, + "benefits": 294.2737467984344, + "taxes": 65.14953231771788, + "hbai_net": 538.5040188058621, + "uc": 138.05944534547942, + "state_pension": 73.27053832772839, + "hb": 3.078869761490375, + "pension_credit": 3.8346760961530952, + "child_benefit": 30.436425195504494, + "equiv_bhc": 434.48287211979425, + "persons_m": 6.9981435327114765, + "equiv_factor_mean": 1.2384047315262312 + }, + "3": { + "market": 471.19522047120006, + "benefits": 289.2646675532569, + "taxes": 96.47326616816775, + "hbai_net": 663.9866218010179, + "uc": 108.43764448924993, + "state_pension": 84.14853356485972, + "hb": 4.731429056074473, + "pension_credit": 1.8399590959413494, + "child_benefit": 25.40776346452504, + "equiv_bhc": 540.4959848245983, + "persons_m": 7.013822983975127, + "equiv_factor_mean": 1.2248802663780014 + }, + "4": { + "market": 668.9333410734778, + "benefits": 230.1396828627762, + "taxes": 147.41296663875204, + "hbai_net": 751.6600577081497, + "uc": 68.72401233461646, + "state_pension": 84.92786114921729, + "hb": 3.165050412166117, + "pension_credit": 1.278992819102135, + "child_benefit": 17.723918658332085, + "equiv_bhc": 645.1283511378105, + "persons_m": 6.969912924239293, + "equiv_factor_mean": 1.1652697620620789 + }, + "5": { + "market": 868.5834815841637, + "benefits": 243.46775476438583, + "taxes": 183.56430523359734, + "hbai_net": 928.4869338619311, + "uc": 60.67900677738088, + "state_pension": 93.55687870830462, + "hb": 5.492776735116658, + "pension_credit": 2.1386167453262606, + "child_benefit": 17.331218358942166, + "equiv_bhc": 759.3142760989002, + "persons_m": 6.9384520655929744, + "equiv_factor_mean": 1.2227941537464841 + }, + "6": { + "market": 1073.657657229047, + "benefits": 177.52373123161186, + "taxes": 227.18380442912388, + "hbai_net": 1023.9975841867907, + "uc": 28.73823308493361, + "state_pension": 64.42682503121794, + "hb": 13.372535578825493, + "pension_credit": 3.1022767473608117, + "child_benefit": 12.930534052933805, + "equiv_bhc": 862.0125033075631, + "persons_m": 6.730596364961549, + "equiv_factor_mean": 1.1855674202250672 + }, + "7": { + "market": 1258.761667359511, + "benefits": 160.7796562630709, + "taxes": 277.5872116297802, + "hbai_net": 1141.9541129300537, + "uc": 20.209443803683317, + "state_pension": 63.76844764755755, + "hb": 7.874441421474954, + "pension_credit": 5.257183272990661, + "child_benefit": 13.198077035360232, + "equiv_bhc": 988.1087893053443, + "persons_m": 7.157556822282254, + "equiv_factor_mean": 1.1531041954946313 + }, + "8": { + "market": 1861.7921864613907, + "benefits": 98.64599751373822, + "taxes": 444.35960232890216, + "hbai_net": 1516.0785856199113, + "uc": 8.875648282993232, + "state_pension": 41.41767707168272, + "hb": 0.0, + "pension_credit": 0.0683192228020821, + "child_benefit": 17.23108427834429, + "equiv_bhc": 1141.682646209089, + "persons_m": 7.001363813009918, + "equiv_factor_mean": 1.3311304710978873 + }, + "9": { + "market": 2207.4240137649626, + "benefits": 117.85828575702133, + "taxes": 560.0839166369858, + "hbai_net": 1765.198387222653, + "uc": 4.073936976826436, + "state_pension": 40.4603470172768, + "hb": 0.04254729940019537, + "pension_credit": 0.16858258190368763, + "child_benefit": 16.958449677682662, + "equiv_bhc": 1354.5318423979074, + "persons_m": 6.9856157651087045, + "equiv_factor_mean": 1.307356853661007 + }, + "10": { + "market": 3954.309964073249, + "benefits": 105.33274965362587, + "taxes": 1437.3738575395666, + "hbai_net": 2622.268855295911, + "uc": 6.0713645731969414, + "state_pension": 44.08425394868458, + "hb": 0.9148758696627696, + "pension_credit": 1.8480122427482857, + "child_benefit": 13.964834614965355, + "equiv_bhc": 2165.4633854015547, + "persons_m": 6.993207431136698, + "equiv_factor_mean": 1.2055572535602062 + } + }, + "totals_bn": { + "market": 1667.1460356974935, + "benefits": 310.4323968936116, + "taxes": 463.21768622917017, + "hbai_net": 1514.360747769021, + "uc": 75.8460574968688, + "state_pension": 126.08517079849511, + "hb": 13.180874097602773, + "pension_credit": 6.246992213224758, + "child_benefit": 16.88672788803331 + }, + "population_m": 69.67747000233115, + "households_m": 31.340966658856033, + "engine_flags_pct": { + "in_poverty_bhc": 11.42799645172305, + "in_relative_poverty_bhc": 18.299262107818574, + "in_poverty_ahc": 14.840663693140588 + } + }, + "2026": { + "bhc": { + "abs_rate_pct": 10.990541354418099, + "rel_rate_person_median_pct": 19.266248791847566, + "rel_rate_household_median_pct": 17.40638006260177, + "median_person_weighted_week": 838.8731971153846, + "median_household_weighted_week": 808.1431790865385, + "abs_threshold_week": 403.41556490384613, + "rel_line_person_week": 503.3239182692308, + "abs_threshold_over_person_median": 0.48090172184671376, + "share_below_pct_of_person_median": { + "40": 6.964452154926855, + "50": 11.799017462581117, + "60": 19.266248791847566, + "70": 27.03627035118985, + "80": 35.29031478213309 + }, + "share_below_abs_line_scaled": { + "90": 8.333747070837845, + "95": 9.763330870031446, + "100": 10.990541354418099, + "105": 12.289194641854346, + "110": 13.595856772649595 + }, + "by_group_abs_pct": { + "children": 12.060585546664607, + "working_age": 11.293892083819705, + "pensioners": 8.589433680444026 + }, + "by_group_rel_pct": { + "children": 23.233950328000113, + "working_age": 17.465638242003386, + "pensioners": 20.49569365079658 + }, + "group_pop_shares_pct": { + "children": 22.04119995771672, + "working_age": 60.493596361560144, + "pensioners": 17.46520368072314 + }, + "share_nonpositive_income_pct": 0.5823169605509905, + "person_quantiles_week": { + "p5": 298.6914625901442, + "p10": 386.3477313701923, + "p20": 510.75484525240387, + "p30": 614.5469876802885, + "p40": 720.1217698317307, + "p50": 838.8731971153846 + } + }, + "ahc": { + "abs_rate_pct": 14.35820392159011, + "rel_rate_person_median_pct": 23.319916335514826, + "rel_rate_household_median_pct": 21.021968431503204, + "median_person_weighted_week": 727.8550180288462, + "median_household_weighted_week": 697.349609375, + "abs_threshold_week": 345.64151696038755, + "rel_line_person_week": 436.7130108173077, + "abs_threshold_over_person_median": 0.4748768757498477, + "share_below_pct_of_person_median": { + "40": 10.921918753494213, + "50": 15.65888369072899, + "60": 23.319916335514826, + "70": 29.843474355495868, + "80": 37.08983231057727 + }, + "share_below_abs_line_scaled": { + "90": 12.25386829196866, + "95": 13.072577428601893, + "100": 14.35820392159011, + "105": 15.521179411036231, + "110": 17.505771535983364 + }, + "by_group_abs_pct": { + "children": 17.132312941218093, + "working_age": 15.608981779354245, + "pensioners": 6.524986229277347 + }, + "by_group_rel_pct": { + "children": 31.27705755909658, + "working_age": 22.217004399288825, + "pensioners": 17.09806898149333 + }, + "group_pop_shares_pct": { + "children": 22.04119995771672, + "working_age": 60.493596361560144, + "pensioners": 17.46520368072314 + }, + "share_nonpositive_income_pct": 1.7953276480336935, + "person_quantiles_week": { + "p5": 150.74431903545673, + "p10": 272.2326096754808, + "p20": 406.37391075721155, + "p30": 511.56434044471155, + "p40": 612.8059645432693, + "p50": 727.8550180288462 + } + }, + "decile_composition_person_weighted_weekly_per_household": { + "1": { + "market": 217.17518749088907, + "benefits": 184.633030100687, + "taxes": 103.50886135188048, + "hbai_net": 298.29935464286746, + "uc": 93.4517725685493, + "state_pension": 33.99278122896131, + "hb": 0.19007940152606073, + "pension_credit": 1.1050147104953263, + "child_benefit": 21.39770234090579, + "equiv_bhc": 250.78146662663403, + "persons_m": 6.994168231892809, + "equiv_factor_mean": 1.138929584529245 + }, + "2": { + "market": 328.3656049940518, + "benefits": 302.386054320098, + "taxes": 70.81631687175235, + "hbai_net": 559.9353432089416, + "uc": 140.0767516284737, + "state_pension": 80.84998695290084, + "hb": 3.191772675641288, + "pension_credit": 4.221861900762251, + "child_benefit": 28.83924683066738, + "equiv_bhc": 456.7696992649037, + "persons_m": 6.982137473540694, + "equiv_factor_mean": 1.2224964314996425 + }, + "3": { + "market": 492.56446730903394, + "benefits": 318.2869862357891, + "taxes": 102.83283877084472, + "hbai_net": 708.0186127201088, + "uc": 128.69583387089477, + "state_pension": 84.23171086111903, + "hb": 4.643718545065168, + "pension_credit": 2.0100569301228592, + "child_benefit": 28.26969750635778, + "equiv_bhc": 563.1931619012132, + "persons_m": 7.00631888147226, + "equiv_factor_mean": 1.2544578764810466 + }, + "4": { + "market": 680.442758700862, + "benefits": 257.9016270716654, + "taxes": 156.25059304735376, + "hbai_net": 782.0937920221129, + "uc": 90.0321642083949, + "state_pension": 85.97400918674401, + "hb": 3.5383601291142908, + "pension_credit": 1.2100720994757856, + "child_benefit": 20.583735066297038, + "equiv_bhc": 668.1179664457802, + "persons_m": 6.992706390345693, + "equiv_factor_mean": 1.1695078346479089 + }, + "5": { + "market": 887.6413750026476, + "benefits": 233.177719775035, + "taxes": 190.64142686810698, + "hbai_net": 930.1776697184155, + "uc": 66.86317938615421, + "state_pension": 76.63296132493569, + "hb": 5.178011522632004, + "pension_credit": 2.1097422703826045, + "child_benefit": 18.463421863787158, + "equiv_bhc": 781.2727649952989, + "persons_m": 6.893514225133866, + "equiv_factor_mean": 1.1921907058077097 + }, + "6": { + "market": 1110.7630432527885, + "benefits": 211.6745240706726, + "taxes": 242.93794893587523, + "hbai_net": 1079.4996182989455, + "uc": 25.71247031410301, + "state_pension": 94.90664771588253, + "hb": 14.204899417410827, + "pension_credit": 4.108927351518745, + "child_benefit": 13.014145900013606, + "equiv_bhc": 886.916268851828, + "persons_m": 6.830232895453229, + "equiv_factor_mean": 1.217123284808575 + }, + "7": { + "market": 1313.3151082739105, + "benefits": 152.51025163068095, + "taxes": 296.86347517884974, + "hbai_net": 1168.9618824223462, + "uc": 19.103850010094167, + "state_pension": 57.330489430271, + "hb": 8.311504868927926, + "pension_credit": 5.895208370343582, + "child_benefit": 13.686396316260735, + "equiv_bhc": 1013.8366848773644, + "persons_m": 6.917864602829874, + "equiv_factor_mean": 1.150886227032361 + }, + "8": { + "market": 1920.0395327402514, + "benefits": 117.00555979918742, + "taxes": 475.70163520407453, + "hbai_net": 1561.34345488208, + "uc": 8.117551775376608, + "state_pension": 57.21993481979527, + "hb": 0.10900981969034768, + "pension_credit": 0.1030208469928841, + "child_benefit": 17.705759616691836, + "equiv_bhc": 1171.3186722834748, + "persons_m": 7.327650623415619, + "equiv_factor_mean": 1.3354694538376977 + }, + "9": { + "market": 2255.8433202764068, + "benefits": 120.48488685406038, + "taxes": 576.9173586427644, + "hbai_net": 1799.41085994081, + "uc": 3.30709379717955, + "state_pension": 42.22650195657857, + "hb": 0.04472701603137648, + "pension_credit": 0.21110358811635038, + "child_benefit": 17.340403896597035, + "equiv_bhc": 1391.8642121573585, + "persons_m": 6.989199944393009, + "equiv_factor_mean": 1.2956526284024878 + }, + "10": { + "market": 4088.5764885931485, + "benefits": 110.16421306240906, + "taxes": 1510.342078474451, + "hbai_net": 2688.3986275746306, + "uc": 6.369421568675801, + "state_pension": 46.66634550773403, + "hb": 0.9603397111013148, + "pension_credit": 2.096854864882865, + "child_benefit": 14.472188881899793, + "equiv_bhc": 2220.277523549659, + "persons_m": 7.008453531275034, + "equiv_factor_mean": 1.2053981336421704 + } + }, + "totals_bn": { + "market": 1729.8980805954045, + "benefits": 326.70094359393113, + "taxes": 490.76655579386147, + "hbai_net": 1565.8324692048793, + "uc": 80.23370658660285, + "state_pension": 133.75304733886628, + "hb": 13.655758845867721, + "pension_credit": 6.979933595314538, + "child_benefit": 17.597690265609145 + }, + "population_m": 69.94224679975208, + "households_m": 31.460063417191506, + "engine_flags_pct": { + "in_poverty_bhc": 10.990541354418099, + "in_relative_poverty_bhc": 17.40638006260177, + "in_poverty_ahc": 14.35820392159011 + } + } + } + }, + "x50_55k": { + "h5": "/Users/mariajuaristi/Desktop/PolicyEngine/data/ukds/acceptance/spine-s-main-rebuild/l0-k20-h55000-e2000-p50-f001-s42/microcosm_uk_2025_local.h5", + "years": { + "2026": { + "bhc": { + "abs_rate_pct": 9.898335651152065, + "rel_rate_person_median_pct": 13.761413065566503, + "rel_rate_household_median_pct": 14.182390243485104, + "median_person_weighted_week": 764.9750600961538, + "median_household_weighted_week": 771.6796123798077, + "abs_threshold_week": 403.41556490384613, + "rel_line_person_week": 458.9850360576923, + "abs_threshold_over_person_median": 0.5273578002048049, + "share_below_pct_of_person_median": { + "40": 4.6410406786191665, + "50": 8.47276441535702, + "60": 13.761413065566503, + "70": 22.095990292333802, + "80": 31.819168702084454 + }, + "share_below_abs_line_scaled": { + "90": 7.121604438334858, + "95": 8.514945104835684, + "100": 9.898335651152065, + "105": 10.85961994281597, + "110": 12.256168766342473 + }, + "by_group_abs_pct": { + "children": 11.849284415521133, + "working_age": 9.575876004783042, + "pensioners": 8.660530812793754 + }, + "by_group_rel_pct": { + "children": 16.66730672089322, + "working_age": 12.791575785767183, + "pensioners": 13.636841888804778 + }, + "group_pop_shares_pct": { + "children": 21.214911502314063, + "working_age": 61.322612324314804, + "pensioners": 17.462476173371126 + }, + "share_nonpositive_income_pct": 0.21286435420285027, + "person_quantiles_week": { + "p5": 316.2426006610577, + "p10": 405.27794471153845, + "p20": 516.1306340144231, + "p30": 594.0160006009615, + "p40": 680.1556490384615, + "p50": 764.9750600961538 + } + }, + "ahc": { + "abs_rate_pct": 15.142914851505248, + "rel_rate_person_median_pct": 18.88035846934363, + "rel_rate_household_median_pct": 20.110181811333273, + "median_person_weighted_week": 648.2629206730769, + "median_household_weighted_week": 668.8716195913462, + "abs_threshold_week": 345.64149803635786, + "rel_line_person_week": 388.95775240384614, + "abs_threshold_over_person_median": 0.5331810396890909, + "share_below_pct_of_person_median": { + "40": 8.746906424040072, + "50": 13.24004217765539, + "60": 18.88035846934363, + "70": 25.532034552857745, + "80": 33.938825847425576 + }, + "share_below_abs_line_scaled": { + "90": 12.063087092418153, + "95": 13.796303004659505, + "100": 15.142914851505248, + "105": 16.542674289884037, + "110": 18.31883935208468 + }, + "by_group_abs_pct": { + "children": 19.824473485085658, + "working_age": 15.469408314422003, + "pensioners": 8.308816240727806 + }, + "by_group_rel_pct": { + "children": 25.196529207693953, + "working_age": 18.652440565359125, + "pensioners": 12.007307890021023 + }, + "group_pop_shares_pct": { + "children": 21.214911502314063, + "working_age": 61.322612324314804, + "pensioners": 17.462476173371126 + }, + "share_nonpositive_income_pct": 1.501742040646369, + "person_quantiles_week": { + "p5": 167.00809420072116, + "p10": 282.50942758413464, + "p20": 398.88931039663464, + "p30": 486.7474834735577, + "p40": 563.9139498197115, + "p50": 648.2629206730769 + } + }, + "decile_composition_person_weighted_weekly_per_household": { + "1": { + "market": 226.78557065707767, + "benefits": 200.50417010515915, + "taxes": 60.94514602906767, + "hbai_net": 366.34459502625486, + "uc": 102.39499000954126, + "state_pension": 33.58937699293079, + "hb": 1.5274102767577404, + "pension_credit": 3.4569990446883674, + "child_benefit": 24.029235609900937, + "equiv_bhc": 290.0852306213882, + "persons_m": 6.981793302788295, + "equiv_factor_mean": 1.2525572196888357 + }, + "2": { + "market": 432.5652802588821, + "benefits": 261.99029474623114, + "taxes": 86.3176476395537, + "hbai_net": 608.237927182129, + "uc": 121.91765698546124, + "state_pension": 61.017924698084705, + "hb": 0.22645350485755728, + "pension_credit": 8.662173671646133, + "child_benefit": 26.114120943183153, + "equiv_bhc": 469.85636146471825, + "persons_m": 6.987255800244387, + "equiv_factor_mean": 1.2926896380686295 + }, + "3": { + "market": 585.681422141985, + "benefits": 298.08270819510903, + "taxes": 122.66264168370208, + "hbai_net": 761.1014888971453, + "uc": 125.51313037123484, + "state_pension": 73.88089883284273, + "hb": 0.39932626674222865, + "pension_credit": 5.478198965107041, + "child_benefit": 29.54086753537109, + "equiv_bhc": 557.4652918590592, + "persons_m": 6.982507039329279, + "equiv_factor_mean": 1.3671489156307213 + }, + "4": { + "market": 674.8151516024361, + "benefits": 301.87294176579, + "taxes": 141.57526835334693, + "hbai_net": 835.1128236158618, + "uc": 114.67549988453369, + "state_pension": 82.48323562796428, + "hb": 0.23446878178687808, + "pension_credit": 4.558547011662106, + "child_benefit": 24.63406810198514, + "equiv_bhc": 639.144028060765, + "persons_m": 6.989213967676409, + "equiv_factor_mean": 1.3046156884769748 + }, + "5": { + "market": 849.1661230427623, + "benefits": 301.3637311008272, + "taxes": 186.10222485647898, + "hbai_net": 964.4276301272228, + "uc": 99.73557498686516, + "state_pension": 85.26736963746526, + "hb": 2.034777240733876, + "pension_credit": 4.745996904445794, + "child_benefit": 23.994524596495804, + "equiv_bhc": 726.5214456372148, + "persons_m": 6.978149088190954, + "equiv_factor_mean": 1.3262501034542702 + }, + "6": { + "market": 1070.5436403402878, + "benefits": 199.14118716342557, + "taxes": 249.1419851844194, + "hbai_net": 1020.542843789211, + "uc": 61.70671837685442, + "state_pension": 59.78102269385106, + "hb": 2.055998172492743, + "pension_credit": 1.557469861579282, + "child_benefit": 17.300978815831964, + "equiv_bhc": 812.934216042944, + "persons_m": 6.991809102800302, + "equiv_factor_mean": 1.2566737222080502 + }, + "7": { + "market": 1278.4206385086948, + "benefits": 201.60258691073568, + "taxes": 304.17596892271456, + "hbai_net": 1175.847257378897, + "uc": 41.78755942467304, + "state_pension": 75.83992019803519, + "hb": 1.6705664220260599, + "pension_credit": 2.675368576096329, + "child_benefit": 14.908353588133236, + "equiv_bhc": 920.6460926192121, + "persons_m": 6.986492431338628, + "equiv_factor_mean": 1.2763825636444435 + }, + "8": { + "market": 1533.2973620420237, + "benefits": 216.86806487455553, + "taxes": 385.5550322384625, + "hbai_net": 1364.6103968380694, + "uc": 41.60013910805023, + "state_pension": 82.43118140211453, + "hb": 3.814364924266895, + "pension_credit": 7.299027629745025, + "child_benefit": 15.97495981315305, + "equiv_bhc": 1054.0585985059918, + "persons_m": 6.985358188248719, + "equiv_factor_mean": 1.2943992296963829 + }, + "9": { + "market": 1985.3772140413607, + "benefits": 177.88614450709622, + "taxes": 539.9930322276078, + "hbai_net": 1623.2703237429205, + "uc": 27.071044310130898, + "state_pension": 69.12212996105339, + "hb": 0.10121462040412199, + "pension_credit": 1.6886314558670663, + "child_benefit": 11.363847476413582, + "equiv_bhc": 1237.5607908475342, + "persons_m": 6.9718223995286355, + "equiv_factor_mean": 1.310695143047705 + }, + "10": { + "market": 3578.6065341279705, + "benefits": 95.4538777524351, + "taxes": 1252.0109967011388, + "hbai_net": 2422.049415323388, + "uc": 6.582693301495249, + "state_pension": 51.4683213619999, + "hb": 0.0, + "pension_credit": 2.179300951886241, + "child_benefit": 8.604791319623768, + "equiv_bhc": 2004.443311238711, + "persons_m": 7.000194492915988, + "equiv_factor_mean": 1.2197546046110495 + } + }, + "totals_bn": { + "market": 1555.0335600009557, + "benefits": 313.02094370584996, + "taxes": 430.37649821840137, + "hbai_net": 1437.6780058566596, + "uc": 82.22809434103732, + "state_pension": 126.8990244550201, + "hb": 3.317437955688143, + "pension_credit": 6.8887940950277216, + "child_benefit": 17.02169456902163 + }, + "population_m": 69.8545958130616, + "households_m": 28.949249626461988, + "engine_flags_pct": { + "in_poverty_bhc": 9.898335651152065, + "in_relative_poverty_bhc": 14.182390243485104, + "in_poverty_ahc": 15.142914851505248 + } + } + } + }, + "june_populace_2023": { + "h5": "/Users/mariajuaristi/.cache/huggingface/hub/datasets--policyengine--populace-uk-private/snapshots/a75a9a831d6b07aaffbd09713f2a1124f5c0f08f/populace_uk_2023.h5", + "years": { + "2026": { + "bhc": { + "abs_rate_pct": 15.077655028350934, + "rel_rate_person_median_pct": 19.9653104490307, + "rel_rate_household_median_pct": 20.062657809669606, + "median_person_weighted_week": 774.8520883413462, + "median_household_weighted_week": 777.7077824519231, + "abs_threshold_week": 403.41556490384613, + "rel_line_person_week": 464.9112530048077, + "abs_threshold_over_person_median": 0.5206355780332222, + "share_below_pct_of_person_median": { + "40": 8.519248957380594, + "50": 13.435757466854772, + "60": 19.9653104490307, + "70": 27.039989457755826, + "80": 36.33221035616302 + }, + "share_below_abs_line_scaled": { + "90": 11.875522179557535, + "95": 13.236898804906824, + "100": 15.077655028350934, + "105": 16.705621887495166, + "110": 18.377165378377025 + }, + "by_group_abs_pct": { + "children": 20.531318129174334, + "working_age": 13.92404446398752, + "pensioners": 12.485493160785921 + }, + "by_group_rel_pct": { + "children": 26.462986572724713, + "working_age": 17.869068109382045, + "pensioners": 19.213496871056847 + }, + "group_pop_shares_pct": { + "children": 21.493655595024208, + "working_age": 59.978393343419725, + "pensioners": 18.527951061556042 + }, + "share_nonpositive_income_pct": 0.5030499816588749, + "person_quantiles_week": { + "p5": 238.689453125, + "p10": 332.50600961538464, + "p20": 465.7186748798077, + "p30": 573.2874849759615, + "p40": 658.5960787259615, + "p50": 774.8520883413462 + } + }, + "ahc": { + "abs_rate_pct": 18.67102913153246, + "rel_rate_person_median_pct": 22.894706652017586, + "rel_rate_household_median_pct": 24.45759014858302, + "median_person_weighted_week": 657.083984375, + "median_household_weighted_week": 683.8571213942307, + "abs_threshold_week": 345.64151696038755, + "rel_line_person_week": 394.250390625, + "abs_threshold_over_person_median": 0.5260233473642675, + "share_below_pct_of_person_median": { + "40": 13.250303300823477, + "50": 17.442367823624785, + "60": 22.894706652017586, + "70": 28.91374854102166, + "80": 37.18690464279808 + }, + "share_below_abs_line_scaled": { + "90": 16.149812652945787, + "95": 17.417103926233025, + "100": 18.67102913153246, + "105": 20.350259755837545, + "110": 21.677107937757427 + }, + "by_group_abs_pct": { + "children": 27.283743705974235, + "working_age": 17.706649692545746, + "pensioners": 11.801581395266917 + }, + "by_group_rel_pct": { + "children": 32.28026837401778, + "working_age": 21.511456559866392, + "pensioners": 16.484666345640946 + }, + "group_pop_shares_pct": { + "children": 21.493655595024208, + "working_age": 59.978393343419725, + "pensioners": 18.527951061556042 + }, + "share_nonpositive_income_pct": 2.5257600362245136, + "person_quantiles_week": { + "p5": 104.91811899038461, + "p10": 208.923095703125, + "p20": 359.45797025240387, + "p30": 471.4865159254808, + "p40": 552.3515625, + "p50": 657.083984375 + } + }, + "decile_composition_person_weighted_weekly_per_household": { + "1": { + "market": 192.41016051552455, + "benefits": 150.79231822351306, + "taxes": 66.81756293979706, + "hbai_net": 276.38491638014887, + "uc": 49.15467758290101, + "state_pension": 24.488135385896193, + "hb": 3.5604695781457276, + "pension_credit": 2.3285531989116945, + "child_benefit": 29.328195657812653, + "equiv_bhc": 211.86480486090736, + "persons_m": 6.9848169395283755, + "equiv_factor_mean": 1.29809677754189 + }, + "2": { + "market": 354.1341397881384, + "benefits": 223.44782916930217, + "taxes": 83.68668397390533, + "hbai_net": 493.8952851334798, + "uc": 67.3220465365862, + "state_pension": 80.23830218889059, + "hb": 4.4308322405164615, + "pension_credit": 4.741108179008472, + "child_benefit": 23.82614948695291, + "equiv_bhc": 402.1729436278801, + "persons_m": 7.00118443094854, + "equiv_factor_mean": 1.2262989811392546 + }, + "3": { + "market": 556.632622811934, + "benefits": 258.17081901403236, + "taxes": 136.36667126118138, + "hbai_net": 678.4367695078582, + "uc": 84.77011466292946, + "state_pension": 89.36089861025708, + "hb": 3.8964651133766783, + "pension_credit": 3.117225508998537, + "child_benefit": 27.608767043462404, + "equiv_bhc": 519.5107459766431, + "persons_m": 6.99451649911505, + "equiv_factor_mean": 1.3061692314913116 + }, + "4": { + "market": 848.2760045155354, + "benefits": 281.3228812102635, + "taxes": 209.4191676262908, + "hbai_net": 920.1797124840627, + "uc": 42.099343050218266, + "state_pension": 109.37083761225517, + "hb": 5.149882516648047, + "pension_credit": 4.4412704438561565, + "child_benefit": 19.998267502029243, + "equiv_bhc": 611.0266747182308, + "persons_m": 6.979360866723657, + "equiv_factor_mean": 1.5132915723365596 + }, + "5": { + "market": 948.9933869198776, + "benefits": 195.9364727407791, + "taxes": 242.5714300085018, + "hbai_net": 902.3584332981175, + "uc": 34.392960493227186, + "state_pension": 76.49898322568231, + "hb": 1.8256118812485516, + "pension_credit": 3.68063107631643, + "child_benefit": 22.21792890345546, + "equiv_bhc": 715.3038686799762, + "persons_m": 7.008374375624135, + "equiv_factor_mean": 1.262932968146229 + }, + "6": { + "market": 1148.8225921391327, + "benefits": 157.35643261343245, + "taxes": 281.0433368506133, + "hbai_net": 1025.13568779924, + "uc": 14.353908613956103, + "state_pension": 86.39542975026453, + "hb": 3.0004722506140737, + "pension_credit": 2.4959339890897314, + "child_benefit": 18.631596558868885, + "equiv_bhc": 829.1331245002488, + "persons_m": 6.982665246898342, + "equiv_factor_mean": 1.2371469013695935 + }, + "7": { + "market": 1471.0762682232973, + "benefits": 122.81948555986916, + "taxes": 405.9838797414041, + "hbai_net": 1187.9118696194143, + "uc": 12.721999082211747, + "state_pension": 60.2906601774029, + "hb": 0.6938991063649785, + "pension_credit": 0.8818068931576245, + "child_benefit": 16.506044084064328, + "equiv_bhc": 967.8657557054256, + "persons_m": 6.9773167014677675, + "equiv_factor_mean": 1.2281326422787462 + }, + "8": { + "market": 1854.887223782822, + "benefits": 142.78115355522485, + "taxes": 526.4848096805848, + "hbai_net": 1471.183571524258, + "uc": 4.717924340920427, + "state_pension": 87.45969074566347, + "hb": 0.9985781489546814, + "pension_credit": 1.116283724207655, + "child_benefit": 15.512747231350373, + "equiv_bhc": 1148.4214264830998, + "persons_m": 7.00714998068811, + "equiv_factor_mean": 1.2813436412135033 + }, + "9": { + "market": 2361.5710939001538, + "benefits": 117.95646590942496, + "taxes": 774.9780623399857, + "hbai_net": 1704.5495046912804, + "uc": 2.454489434742221, + "state_pension": 90.0681388273189, + "hb": 0.21336377763018627, + "pension_credit": 0.3834198093872897, + "child_benefit": 10.62139867632979, + "equiv_bhc": 1406.9661888423277, + "persons_m": 7.007432859110534, + "equiv_factor_mean": 1.2149391312303208 + }, + "10": { + "market": 4541.069669897678, + "benefits": 59.167761573549996, + "taxes": 1753.4018292886476, + "hbai_net": 2846.835601182208, + "uc": 0.18982007963290973, + "state_pension": 42.30381751641475, + "hb": 0.04466298013837484, + "pension_credit": 0.6411675655872788, + "child_benefit": 10.201602875482214, + "equiv_bhc": 2507.254550217403, + "persons_m": 6.99376969599843, + "equiv_factor_mean": 1.1511523332031428 + } + }, + "totals_bn": { + "market": 1890.9255019421119, + "benefits": 265.4267905577881, + "taxes": 605.2669583006755, + "hbai_net": 1551.0853348574149, + "uc": 38.93968181980096, + "state_pension": 145.09702216981432, + "hb": 5.235245191738161, + "pension_credit": 5.725827904638691, + "child_benefit": 17.43470226349069 + }, + "population_m": 69.93658759610295, + "households_m": 29.47058318248134, + "engine_flags_pct": { + "in_poverty_bhc": 15.077655028350934, + "in_relative_poverty_bhc": 20.062657809669606, + "in_poverty_ahc": 18.67102913153246 + } + } + } + } + } +} \ No newline at end of file diff --git a/docs/evidence/uk-poverty-968/outputs/scorecard_d20_dense.json b/docs/evidence/uk-poverty-968/outputs/scorecard_d20_dense.json new file mode 100644 index 000000000..a287a526c --- /dev/null +++ b/docs/evidence/uk-poverty-968/outputs/scorecard_d20_dense.json @@ -0,0 +1,72 @@ +{ + "probe": "microcosm#731 scorecard", + "label": "D20", + "h5": "/Users/mariajuaristi/Desktop/PolicyEngine/data/ukds/acceptance/spine-s-main-rebuild/dense-k20-e2000/microcosm_uk_2025_local.h5", + "year": 2026, + "engine": { + "package": "policyengine-uk", + "version": "2.99.1" + }, + "conventions": "taxpayers: income_tax>0 persons; uc_families: universal_credit>0 benunits; poverty_bhc_absolute (= poverty_bhc_pct): person-weighted in_poverty_bhc, the engine's CPI-uprated 2010/11 line; poverty_*_relative: individuals under 60% of the person-weighted median equivalised HBAI net income of the same file (microcosm#968); median_equiv_bhc_income_week: person-weighted, couple-no-children equivalent; gini/top-1%: household-weighted household_net_income", + "hbai_fye_2024_reference": { + "poverty_bhc_absolute_pct": 15, + "poverty_bhc_relative_pct": 17, + "poverty_ahc_absolute_pct": 18, + "poverty_ahc_relative_pct": 21, + "median_equiv_bhc_income_week": 650, + "child_poverty_bhc_relative_pct": 23, + "year": "FYE 2024 (April 2023 to March 2024); compare with the hbai_basis block, not with the 2026 rows", + "source": "DWP HBAI FYE 2024 summary tables 1.2b, 1.3a, 1.4a (March 2025)" + }, + "rows": { + "population_m": 69.98817909054051, + "households_m": 29.276851533892593, + "state_pension_bn": 131.5003306295686, + "income_tax_bn": 284.4527214178233, + "income_taxpayers_m": 39.3057451370966, + "universal_credit_bn": 83.45939403518328, + "uc_families_m": 6.807705388853736, + "child_benefit_bn": 17.021080907275007, + "pension_credit_bn": 6.885914901712243, + "council_tax_bn": 48.26098708939038, + "poverty_bhc_pct": 9.965536057830626, + "poverty_bhc_absolute_pct": 9.965536057830626, + "poverty_bhc_relative_pct": 14.8504340218774, + "poverty_ahc_absolute_pct": 14.292058323518788, + "poverty_ahc_relative_pct": 18.91293695626547, + "median_equiv_bhc_income_week": 779.7072566105769, + "child_poverty_bhc_relative_pct": 17.421881972279195, + "p10_over_median_bhc": 0.518261879251364, + "p90_over_median_bhc": 1.777658632243306, + "zero_income_share_pct": 0.18713627236317096, + "top_1pct_net_income_share_pct": 8.5942241262201, + "gini_net_hh_income": 0.37403745897677565 + }, + "hbai_basis": { + "year": 2024, + "rows": { + "poverty_bhc_absolute_pct": 10.537252183995344, + "poverty_bhc_relative_pct": 15.377218262723245, + "poverty_ahc_absolute_pct": 14.745902000182891, + "poverty_ahc_relative_pct": 19.534787135940757, + "child_poverty_bhc_relative_pct": 18.686424972759223, + "median_equiv_bhc_income_week": 730.7856069711538, + "p10_over_median_bhc": 0.5112471780546036, + "p90_over_median_bhc": 1.7815920654738617, + "zero_income_share_pct": 0.1905185120838884 + }, + "hbai": { + "label": "HBAI FYE 2024 (April 2023 to March 2024), survey-reported incomes", + "poverty_bhc_absolute_pct": 15, + "poverty_bhc_relative_pct": 17, + "poverty_ahc_absolute_pct": 18, + "poverty_ahc_relative_pct": 21, + "child_poverty_bhc_relative_pct": 23, + "median_equiv_bhc_income_week": 650, + "p10_over_median_bhc": 0.48, + "p90_over_median_bhc": 1.955, + "zero_income_share_pct": 1.0, + "source": "DWP HBAI FYE 2024 (March 2025): summary tables 1.2b (quintile medians: bottom 312, top 1271 on a 650 median) and 1.3a-1.4a; chart 2.4 income bands (0.66m individuals at 0-10 GBP/week)" + } + } +} \ No newline at end of file diff --git a/docs/evidence/uk-poverty-968/outputs/scorecard_efrs_1_57_3.json b/docs/evidence/uk-poverty-968/outputs/scorecard_efrs_1_57_3.json new file mode 100644 index 000000000..b70f15068 --- /dev/null +++ b/docs/evidence/uk-poverty-968/outputs/scorecard_efrs_1_57_3.json @@ -0,0 +1,72 @@ +{ + "probe": "microcosm#731 scorecard", + "label": "eFRS", + "h5": "/Users/mariajuaristi/.cache/huggingface/hub/models--policyengine--policyengine-uk-data-private/snapshots/25af520a6651b8812fef56964a42a79a3f9f515a/enhanced_frs_2024_25.h5", + "year": 2026, + "engine": { + "package": "policyengine-uk", + "version": "2.99.1" + }, + "conventions": "taxpayers: income_tax>0 persons; uc_families: universal_credit>0 benunits; poverty_bhc_absolute (= poverty_bhc_pct): person-weighted in_poverty_bhc, the engine's CPI-uprated 2010/11 line; poverty_*_relative: individuals under 60% of the person-weighted median equivalised HBAI net income of the same file (microcosm#968); median_equiv_bhc_income_week: person-weighted, couple-no-children equivalent; gini/top-1%: household-weighted household_net_income", + "hbai_fye_2024_reference": { + "poverty_bhc_absolute_pct": 15, + "poverty_bhc_relative_pct": 17, + "poverty_ahc_absolute_pct": 18, + "poverty_ahc_relative_pct": 21, + "median_equiv_bhc_income_week": 650, + "child_poverty_bhc_relative_pct": 23, + "year": "FYE 2024 (April 2023 to March 2024); compare with the hbai_basis block, not with the 2026 rows", + "source": "DWP HBAI FYE 2024 summary tables 1.2b, 1.3a, 1.4a (March 2025)" + }, + "rows": { + "population_m": 69.94224679975208, + "households_m": 31.460063417191506, + "state_pension_bn": 133.75304733886628, + "income_tax_bn": 312.48697156978466, + "income_taxpayers_m": 43.227384603161276, + "universal_credit_bn": 80.23370658660285, + "uc_families_m": 6.381444614622757, + "child_benefit_bn": 17.597690265609145, + "pension_credit_bn": 6.979933595314537, + "council_tax_bn": 52.39058821056049, + "poverty_bhc_pct": 10.990541354418097, + "poverty_bhc_absolute_pct": 10.990541354418097, + "poverty_bhc_relative_pct": 19.266248791847566, + "poverty_ahc_absolute_pct": 14.358203921590109, + "poverty_ahc_relative_pct": 23.319916335514826, + "median_equiv_bhc_income_week": 838.8731971153846, + "child_poverty_bhc_relative_pct": 23.23395032800011, + "p10_over_median_bhc": 0.46055557943526865, + "p90_over_median_bhc": 1.8630417262167012, + "zero_income_share_pct": 0.6210200962352113, + "top_1pct_net_income_share_pct": 6.939483191160551, + "gini_net_hh_income": 0.3757644454470449 + }, + "hbai_basis": { + "year": 2024, + "rows": { + "poverty_bhc_absolute_pct": 11.248234644883526, + "poverty_bhc_relative_pct": 19.67636083543157, + "poverty_ahc_absolute_pct": 14.636882117887081, + "poverty_ahc_relative_pct": 23.66888127953728, + "child_poverty_bhc_relative_pct": 24.760653598499392, + "median_equiv_bhc_income_week": 780.1297325721154, + "p10_over_median_bhc": 0.460191522709439, + "p90_over_median_bhc": 1.8742793392101578, + "zero_income_share_pct": 0.6179569441376033 + }, + "hbai": { + "label": "HBAI FYE 2024 (April 2023 to March 2024), survey-reported incomes", + "poverty_bhc_absolute_pct": 15, + "poverty_bhc_relative_pct": 17, + "poverty_ahc_absolute_pct": 18, + "poverty_ahc_relative_pct": 21, + "child_poverty_bhc_relative_pct": 23, + "median_equiv_bhc_income_week": 650, + "p10_over_median_bhc": 0.48, + "p90_over_median_bhc": 1.955, + "zero_income_share_pct": 1.0, + "source": "DWP HBAI FYE 2024 (March 2025): summary tables 1.2b (quintile medians: bottom 312, top 1271 on a 650 median) and 1.3a-1.4a; chart 2.4 income bands (0.66m individuals at 0-10 GBP/week)" + } + } +} \ No newline at end of file diff --git a/docs/evidence/uk-poverty-968/outputs/scorecard_t95_55k.json b/docs/evidence/uk-poverty-968/outputs/scorecard_t95_55k.json new file mode 100644 index 000000000..315368b61 --- /dev/null +++ b/docs/evidence/uk-poverty-968/outputs/scorecard_t95_55k.json @@ -0,0 +1,72 @@ +{ + "probe": "microcosm#731 scorecard", + "label": "T95", + "h5": "/Users/mariajuaristi/Desktop/PolicyEngine/data/ukds/acceptance/spine-s-main-rebuild/l0-k20-h55000-e2000-tol1-scratch-s42/microcosm_uk_2025_local.h5", + "year": 2026, + "engine": { + "package": "policyengine-uk", + "version": "2.99.1" + }, + "conventions": "taxpayers: income_tax>0 persons; uc_families: universal_credit>0 benunits; poverty_bhc_absolute (= poverty_bhc_pct): person-weighted in_poverty_bhc, the engine's CPI-uprated 2010/11 line; poverty_*_relative: individuals under 60% of the person-weighted median equivalised HBAI net income of the same file (microcosm#968); median_equiv_bhc_income_week: person-weighted, couple-no-children equivalent; gini/top-1%: household-weighted household_net_income", + "hbai_fye_2024_reference": { + "poverty_bhc_absolute_pct": 15, + "poverty_bhc_relative_pct": 17, + "poverty_ahc_absolute_pct": 18, + "poverty_ahc_relative_pct": 21, + "median_equiv_bhc_income_week": 650, + "child_poverty_bhc_relative_pct": 23, + "year": "FYE 2024 (April 2023 to March 2024); compare with the hbai_basis block, not with the 2026 rows", + "source": "DWP HBAI FYE 2024 summary tables 1.2b, 1.3a, 1.4a (March 2025)" + }, + "rows": { + "population_m": 69.99767346691662, + "households_m": 29.07548092880127, + "state_pension_bn": 126.51702359684938, + "income_tax_bn": 286.1820736323539, + "income_taxpayers_m": 39.27711377333081, + "universal_credit_bn": 82.59845038442386, + "uc_families_m": 6.826297144149538, + "child_benefit_bn": 16.992810474767495, + "pension_credit_bn": 6.927671937966923, + "council_tax_bn": 47.437445619518435, + "poverty_bhc_pct": 10.079247744643098, + "poverty_bhc_absolute_pct": 10.079247744643098, + "poverty_bhc_relative_pct": 14.054363197623282, + "poverty_ahc_absolute_pct": 15.183955847390598, + "poverty_ahc_relative_pct": 18.957445677219102, + "median_equiv_bhc_income_week": 765.5464993990385, + "child_poverty_bhc_relative_pct": 17.08647705447216, + "p10_over_median_bhc": 0.5243763710074276, + "p90_over_median_bhc": 1.8179756460508363, + "zero_income_share_pct": 0.2760933818597608, + "top_1pct_net_income_share_pct": 8.5331935862987, + "gini_net_hh_income": 0.362298039276458 + }, + "hbai_basis": { + "year": 2024, + "rows": { + "poverty_bhc_absolute_pct": 10.304322103981097, + "poverty_bhc_relative_pct": 14.503810981224126, + "poverty_ahc_absolute_pct": 15.363252888698115, + "poverty_ahc_relative_pct": 19.047233791313353, + "child_poverty_bhc_relative_pct": 17.916180160606977, + "median_equiv_bhc_income_week": 719.6479116586538, + "p10_over_median_bhc": 0.5224258929036499, + "p90_over_median_bhc": 1.8159519154217145, + "zero_income_share_pct": 0.275382444831545 + }, + "hbai": { + "label": "HBAI FYE 2024 (April 2023 to March 2024), survey-reported incomes", + "poverty_bhc_absolute_pct": 15, + "poverty_bhc_relative_pct": 17, + "poverty_ahc_absolute_pct": 18, + "poverty_ahc_relative_pct": 21, + "child_poverty_bhc_relative_pct": 23, + "median_equiv_bhc_income_week": 650, + "p10_over_median_bhc": 0.48, + "p90_over_median_bhc": 1.955, + "zero_income_share_pct": 1.0, + "source": "DWP HBAI FYE 2024 (March 2025): summary tables 1.2b (quintile medians: bottom 312, top 1271 on a 650 median) and 1.3a-1.4a; chart 2.4 income bands (0.66m individuals at 0-10 GBP/week)" + } + } +} \ No newline at end of file diff --git a/docs/evidence/uk-poverty-968/outputs/scorecard_v20_national.json b/docs/evidence/uk-poverty-968/outputs/scorecard_v20_national.json new file mode 100644 index 000000000..ed2b59062 --- /dev/null +++ b/docs/evidence/uk-poverty-968/outputs/scorecard_v20_national.json @@ -0,0 +1,72 @@ +{ + "probe": "microcosm#731 scorecard", + "label": "V20", + "h5": "/Users/mariajuaristi/Desktop/PolicyEngine/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5", + "year": 2026, + "engine": { + "package": "policyengine-uk", + "version": "2.99.1" + }, + "conventions": "taxpayers: income_tax>0 persons; uc_families: universal_credit>0 benunits; poverty_bhc_absolute (= poverty_bhc_pct): person-weighted in_poverty_bhc, the engine's CPI-uprated 2010/11 line; poverty_*_relative: individuals under 60% of the person-weighted median equivalised HBAI net income of the same file (microcosm#968); median_equiv_bhc_income_week: person-weighted, couple-no-children equivalent; gini/top-1%: household-weighted household_net_income", + "hbai_fye_2024_reference": { + "poverty_bhc_absolute_pct": 15, + "poverty_bhc_relative_pct": 17, + "poverty_ahc_absolute_pct": 18, + "poverty_ahc_relative_pct": 21, + "median_equiv_bhc_income_week": 650, + "child_poverty_bhc_relative_pct": 23, + "year": "FYE 2024 (April 2023 to March 2024); compare with the hbai_basis block, not with the 2026 rows", + "source": "DWP HBAI FYE 2024 summary tables 1.2b, 1.3a, 1.4a (March 2025)" + }, + "rows": { + "population_m": 70.01177272866059, + "households_m": 29.33063353709817, + "state_pension_bn": 143.34295439981796, + "income_tax_bn": 308.9927890724263, + "income_taxpayers_m": 40.28271853523135, + "universal_credit_bn": 82.27460669261879, + "uc_families_m": 6.4363085692987445, + "child_benefit_bn": 16.760951441502275, + "pension_credit_bn": 6.9111590841356065, + "council_tax_bn": 48.767579616322, + "poverty_bhc_pct": 8.740819809337934, + "poverty_bhc_absolute_pct": 8.740819809337934, + "poverty_bhc_relative_pct": 14.63452283021983, + "poverty_ahc_absolute_pct": 12.77307812862886, + "poverty_ahc_relative_pct": 19.086946645184565, + "median_equiv_bhc_income_week": 799.7987530048077, + "child_poverty_bhc_relative_pct": 17.41214752626432, + "p10_over_median_bhc": 0.5295396669140309, + "p90_over_median_bhc": 1.7424408427563236, + "zero_income_share_pct": 0.1550300104130417, + "top_1pct_net_income_share_pct": 9.106001360340661, + "gini_net_hh_income": 0.376326181202968 + }, + "hbai_basis": { + "year": 2024, + "rows": { + "poverty_bhc_absolute_pct": 9.31103618296997, + "poverty_bhc_relative_pct": 15.301746413861888, + "poverty_ahc_absolute_pct": 13.066817529758456, + "poverty_ahc_relative_pct": 19.51732487820331, + "child_poverty_bhc_relative_pct": 18.759402482558862, + "median_equiv_bhc_income_week": 751.48046875, + "p10_over_median_bhc": 0.518320856626363, + "p90_over_median_bhc": 1.750322380461325, + "zero_income_share_pct": 0.1570949869082417 + }, + "hbai": { + "label": "HBAI FYE 2024 (April 2023 to March 2024), survey-reported incomes", + "poverty_bhc_absolute_pct": 15, + "poverty_bhc_relative_pct": 17, + "poverty_ahc_absolute_pct": 18, + "poverty_ahc_relative_pct": 21, + "child_poverty_bhc_relative_pct": 23, + "median_equiv_bhc_income_week": 650, + "p10_over_median_bhc": 0.48, + "p90_over_median_bhc": 1.955, + "zero_income_share_pct": 1.0, + "source": "DWP HBAI FYE 2024 (March 2025): summary tables 1.2b (quintile medians: bottom 312, top 1271 on a 650 median) and 1.3a-1.4a; chart 2.4 income bands (0.66m individuals at 0-10 GBP/week)" + } + } +} \ No newline at end of file diff --git a/docs/evidence/uk-poverty-968/outputs/scorecard_x50_55k.json b/docs/evidence/uk-poverty-968/outputs/scorecard_x50_55k.json new file mode 100644 index 000000000..9955ab905 --- /dev/null +++ b/docs/evidence/uk-poverty-968/outputs/scorecard_x50_55k.json @@ -0,0 +1,72 @@ +{ + "probe": "microcosm#731 scorecard", + "label": "X50", + "h5": "/Users/mariajuaristi/Desktop/PolicyEngine/data/ukds/acceptance/spine-s-main-rebuild/l0-k20-h55000-e2000-p50-f001-s42/microcosm_uk_2025_local.h5", + "year": 2026, + "engine": { + "package": "policyengine-uk", + "version": "2.99.1" + }, + "conventions": "taxpayers: income_tax>0 persons; uc_families: universal_credit>0 benunits; poverty_bhc_absolute (= poverty_bhc_pct): person-weighted in_poverty_bhc, the engine's CPI-uprated 2010/11 line; poverty_*_relative: individuals under 60% of the person-weighted median equivalised HBAI net income of the same file (microcosm#968); median_equiv_bhc_income_week: person-weighted, couple-no-children equivalent; gini/top-1%: household-weighted household_net_income", + "hbai_fye_2024_reference": { + "poverty_bhc_absolute_pct": 15, + "poverty_bhc_relative_pct": 17, + "poverty_ahc_absolute_pct": 18, + "poverty_ahc_relative_pct": 21, + "median_equiv_bhc_income_week": 650, + "child_poverty_bhc_relative_pct": 23, + "year": "FYE 2024 (April 2023 to March 2024); compare with the hbai_basis block, not with the 2026 rows", + "source": "DWP HBAI FYE 2024 summary tables 1.2b, 1.3a, 1.4a (March 2025)" + }, + "rows": { + "population_m": 69.8545958130616, + "households_m": 28.949249626461988, + "state_pension_bn": 126.89902445502013, + "income_tax_bn": 283.7876768258048, + "income_taxpayers_m": 39.16611166404715, + "universal_credit_bn": 82.22809434103732, + "uc_families_m": 6.798939695234622, + "child_benefit_bn": 17.021694569021633, + "pension_credit_bn": 6.888794095027721, + "council_tax_bn": 47.34146258104452, + "poverty_bhc_pct": 9.898335651152065, + "poverty_bhc_absolute_pct": 9.898335651152065, + "poverty_bhc_relative_pct": 13.761413065566503, + "poverty_ahc_absolute_pct": 15.142914851505248, + "poverty_ahc_relative_pct": 18.88035846934363, + "median_equiv_bhc_income_week": 764.9750600961538, + "child_poverty_bhc_relative_pct": 16.66730672089322, + "p10_over_median_bhc": 0.5297923629831761, + "p90_over_median_bhc": 1.813250416267813, + "zero_income_share_pct": 0.24859435237322716, + "top_1pct_net_income_share_pct": 8.567420998102795, + "gini_net_hh_income": 0.35952114565293447 + }, + "hbai_basis": { + "year": 2024, + "rows": { + "poverty_bhc_absolute_pct": 10.157043611790538, + "poverty_bhc_relative_pct": 14.19988945514989, + "poverty_ahc_absolute_pct": 15.302262179471166, + "poverty_ahc_relative_pct": 18.941239907605603, + "child_poverty_bhc_relative_pct": 17.590719607829104, + "median_equiv_bhc_income_week": 719.0164513221154, + "p10_over_median_bhc": 0.5260240585978421, + "p90_over_median_bhc": 1.8099987389708267, + "zero_income_share_pct": 0.24338356970329345 + }, + "hbai": { + "label": "HBAI FYE 2024 (April 2023 to March 2024), survey-reported incomes", + "poverty_bhc_absolute_pct": 15, + "poverty_bhc_relative_pct": 17, + "poverty_ahc_absolute_pct": 18, + "poverty_ahc_relative_pct": 21, + "child_poverty_bhc_relative_pct": 23, + "median_equiv_bhc_income_week": 650, + "p10_over_median_bhc": 0.48, + "p90_over_median_bhc": 1.955, + "zero_income_share_pct": 1.0, + "source": "DWP HBAI FYE 2024 (March 2025): summary tables 1.2b (quintile medians: bottom 312, top 1271 on a 650 median) and 1.3a-1.4a; chart 2.4 income bands (0.66m individuals at 0-10 GBP/week)" + } + } +} \ No newline at end of file diff --git a/docs/evidence/uk-poverty-968/outputs/tail_profile.json b/docs/evidence/uk-poverty-968/outputs/tail_profile.json new file mode 100644 index 000000000..cd54312e9 --- /dev/null +++ b/docs/evidence/uk-poverty-968/outputs/tail_profile.json @@ -0,0 +1,479 @@ +{ + "v20_national|calibrated|2024": { + "median_week": 751.48046875, + "rel_bhc_pct": 15.301746413861888, + "abs_bhc_pct": 9.31103618296997, + "share_below_abs_week_pct": { + "10": 0.1570949869082417, + "50": 0.2551341267391073, + "100": 0.4812751817780037, + "150": 0.7673984333203583, + "200": 1.5403930912468216, + "260": 3.1462492073877795, + "325": 5.975847775079704, + "370": 8.550000053929296, + "390": 10.011772922162303, + "455": 15.658543878351837 + }, + "share_in_band_pct_of_median": { + "-1000-0": 0.12272392226015867, + "0-20": 0.6446745110601996, + "20-40": 4.098718375868806, + "40-50": 4.032966459447161, + "50-60": 6.402663145225563, + "60-70": 8.465930956051883, + "70-80": 9.055212026816069 + }, + "by_family_type": { + "couple_wa_no_children": { + "pop_share_pct": 14.647785354337602, + "rel_rate_pct": 12.118022704191924, + "share_of_poor_pct": 11.600126592687051 + }, + "couple_with_children": { + "pop_share_pct": 29.11331168610235, + "rel_rate_pct": 16.844081745166154, + "share_of_poor_pct": 32.04777994941586 + }, + "lone_parent": { + "pop_share_pct": 5.645372157324529, + "rel_rate_pct": 26.76304228588916, + "share_of_poor_pct": 9.873862086041846 + }, + "other_3plus_adults": { + "pop_share_pct": 26.13077878375383, + "rel_rate_pct": 6.768883889523009, + "share_of_poor_pct": 11.559217016549717 + }, + "pensioner_couple_or_mixed": { + "pop_share_pct": 12.084163343042189, + "rel_rate_pct": 11.721410564555232, + "share_of_poor_pct": 9.256684566712636 + }, + "single_pensioner": { + "pop_share_pct": 5.90000929983278, + "rel_rate_pct": 31.83565700269337, + "share_of_poor_pct": 12.275113395685423 + }, + "single_wa_no_children": { + "pop_share_pct": 6.478579375606723, + "rel_rate_pct": 31.61924529366143, + "share_of_poor_pct": 13.387216392907463 + } + }, + "by_n_children": { + "0": { + "pop_share_pct": 53.355506925128516, + "rel_rate_pct": 14.805766067180816, + "share_of_poor_pct": 51.626078002028834 + }, + "1": { + "pop_share_pct": 17.626600950950934, + "rel_rate_pct": 9.06709883955612, + "share_of_poor_pct": 10.444698840578274 + }, + "2": { + "pop_share_pct": 19.659957663754515, + "rel_rate_pct": 14.644645552262636, + "share_of_poor_pct": 18.815702715956206 + }, + "3": { + "pop_share_pct": 9.357934460166037, + "rel_rate_pct": 31.253717806636622, + "share_of_poor_pct": 19.113520441436687 + } + }, + "children_rel_rate_by_family_children": { + "1": { + "rate": 9.941021956134142, + "children_m": 3.746010797958374 + }, + "2": { + "rate": 15.282912976286513, + "children_m": 6.694207773974418 + }, + "3": { + "rate": 32.29961753425295, + "children_m": 4.158434082217694 + } + }, + "by_work": { + "workless_hh": { + "pop_share_pct": 21.412543667608492, + "rel_rate_pct": 28.916518281929026, + "share_of_poor_pct": 40.464414562908416 + }, + "working_hh": { + "pop_share_pct": 78.58745633239151, + "rel_rate_pct": 11.592160803704454, + "share_of_poor_pct": 59.535585437091584 + }, + "any_ft_earner": { + "pop_share_pct": 72.31334086160223, + "rel_rate_pct": 8.782801101409499, + "share_of_poor_pct": 41.50596099217339 + }, + "working_no_ft": { + "pop_share_pct": 6.274115470789274, + "rel_rate_pct": 43.97189412240729, + "share_of_poor_pct": 18.029624444918188 + } + }, + "by_tenure": { + "OWNED_OUTRIGHT": { + "pop_share_pct": 31.02199025160368, + "rel_rate_pct": 15.04080978049108, + "share_of_poor_pct": 30.492980459010173 + }, + "OWNED_WITH_MORTGAGE": { + "pop_share_pct": 33.31314630912093, + "rel_rate_pct": 12.47052289309537, + "share_of_poor_pct": 27.14934246411161 + }, + "RENT_FROM_COUNCIL": { + "pop_share_pct": 9.030729940440967, + "rel_rate_pct": 21.925029562667692, + "share_of_poor_pct": 12.939635487441398 + }, + "RENT_FROM_HA": { + "pop_share_pct": 8.847695317940182, + "rel_rate_pct": 19.154885290697564, + "share_of_poor_pct": 11.075637010207991 + }, + "RENT_PRIVATELY": { + "pop_share_pct": 17.786438180894244, + "rel_rate_pct": 15.780046608393372, + "share_of_poor_pct": 18.342404579228827 + } + }, + "children": { + "rel_rate_pct": 18.759402482558862, + "in_workless_hh_rate": 42.66198787542584, + "share_children_in_workless_hh": 9.243876076543387 + }, + "uc": { + "hh_with_uc_m": 5.899447441967964, + "persons_in_uc_hh_rel_poor_pct": 26.51749661703434, + "share_of_rel_poor_in_uc_hh_pct": 42.52589356650309, + "would_claim_false_eligible_hh_m": null + } + }, + "v20_national|spine_s_design|2024": { + "median_week": 745.38671875, + "rel_bhc_pct": 16.163689191086075, + "abs_bhc_pct": 10.36196031429083, + "share_below_abs_week_pct": { + "10": 0.33553979878860196, + "50": 0.5510786812655849, + "100": 0.9149299757858308, + "150": 1.3805037551782011, + "200": 2.1209895120244466, + "260": 3.674056351547778, + "325": 6.440320827761611, + "370": 9.51290886415279, + "390": 11.009116240639742, + "455": 16.867538927810777 + }, + "share_in_band_pct_of_median": { + "-1000-0": 0.25573012699728276, + "0-20": 1.115600352237792, + "20-40": 3.733286901657891, + "40-50": 4.669304656809421, + "50-60": 6.389767153383688, + "60-70": 7.836574853416804, + "70-80": 8.707365976127592 + }, + "by_family_type": { + "couple_wa_no_children": { + "pop_share_pct": 15.865273291311711, + "rel_rate_pct": 13.254902153766942, + "share_of_poor_pct": 13.010188616784378 + }, + "couple_with_children": { + "pop_share_pct": 30.156317016120095, + "rel_rate_pct": 15.06041642894951, + "share_of_poor_pct": 28.097959992737792 + }, + "lone_parent": { + "pop_share_pct": 6.1981205065012235, + "rel_rate_pct": 27.43353879612035, + "share_of_poor_pct": 10.519651631998826 + }, + "other_3plus_adults": { + "pop_share_pct": 19.15260122085622, + "rel_rate_pct": 11.258995319514508, + "share_of_poor_pct": 13.34095483728234 + }, + "pensioner_couple_or_mixed": { + "pop_share_pct": 14.432487218035021, + "rel_rate_pct": 9.787615865730485, + "share_of_poor_pct": 8.739319298164533 + }, + "single_pensioner": { + "pop_share_pct": 6.953419462753164, + "rel_rate_pct": 29.56232637059488, + "share_of_poor_pct": 12.717347699491588 + }, + "single_wa_no_children": { + "pop_share_pct": 7.241781284422556, + "rel_rate_pct": 30.29852046598832, + "share_of_poor_pct": 13.574577923540547 + } + }, + "by_n_children": { + "0": { + "pop_share_pct": 55.649196751672896, + "rel_rate_pct": 15.605646167257866, + "share_of_poor_pct": 53.72793696612589 + }, + "1": { + "pop_share_pct": 15.080928301391449, + "rel_rate_pct": 13.900647870982548, + "share_of_poor_pct": 12.969481868086557 + }, + "2": { + "pop_share_pct": 19.88373384645115, + "rel_rate_pct": 13.908729486507251, + "share_of_poor_pct": 17.109799129552233 + }, + "3": { + "pop_share_pct": 9.3861411004845, + "rel_rate_pct": 27.885271824775707, + "share_of_poor_pct": 16.19278203623532 + } + }, + "children_rel_rate_by_family_children": { + "1": { + "rate": 13.639135841317946, + "children_m": 3.192228549816739 + }, + "2": { + "rate": 14.440696754940682, + "children_m": 6.57995780342556 + }, + "3": { + "rate": 29.104764571986596, + "children_m": 3.913224613094256 + } + }, + "by_work": { + "workless_hh": { + "pop_share_pct": 22.00594460681432, + "rel_rate_pct": 29.58400255964338, + "share_of_poor_pct": 40.2769388769484 + }, + "working_hh": { + "pop_share_pct": 77.99405539318566, + "rel_rate_pct": 12.377161216540411, + "share_of_poor_pct": 59.72306112305161 + }, + "any_ft_earner": { + "pop_share_pct": 71.12734579092692, + "rel_rate_pct": 8.800893768300087, + "share_of_poor_pct": 38.727805696258436 + }, + "working_no_ft": { + "pop_share_pct": 6.866709602258747, + "rel_rate_pct": 49.42116426396102, + "share_of_poor_pct": 20.995255426793165 + } + }, + "by_tenure": { + "OWNED_OUTRIGHT": { + "pop_share_pct": 32.94218866348762, + "rel_rate_pct": 15.79331334274209, + "share_of_poor_pct": 32.187349163154025 + }, + "OWNED_WITH_MORTGAGE": { + "pop_share_pct": 33.0559172136907, + "rel_rate_pct": 12.562272449240586, + "share_of_poor_pct": 25.69075866213326 + }, + "RENT_FROM_COUNCIL": { + "pop_share_pct": 8.178926153998212, + "rel_rate_pct": 24.57754721780149, + "share_of_poor_pct": 12.436390069394534 + }, + "RENT_FROM_HA": { + "pop_share_pct": 7.935741928199471, + "rel_rate_pct": 19.12714121519747, + "share_of_poor_pct": 9.390681465945445 + }, + "RENT_PRIVATELY": { + "pop_share_pct": 17.887226040623993, + "rel_rate_pct": 18.33929823767218, + "share_of_poor_pct": 20.294820639372727 + } + }, + "children": { + "rel_rate_pct": 18.446789492069488, + "in_workless_hh_rate": 55.38332509394963, + "share_children_in_workless_hh": 7.758305095634709 + }, + "uc": { + "hh_with_uc_m": 4.894761128126905, + "persons_in_uc_hh_rel_poor_pct": 31.304910105215566, + "share_of_rel_poor_in_uc_hh_pct": 43.30216121380436, + "would_claim_false_eligible_hh_m": null + } + }, + "efrs_1_57_3|calibrated|2024": { + "median_week": 780.1297325721154, + "rel_bhc_pct": 19.67636083543157, + "abs_bhc_pct": 11.248234644883526, + "share_below_abs_week_pct": { + "10": 0.6179569441376033, + "50": 0.786336544701641, + "100": 1.1635474640343062, + "150": 1.736853790820671, + "200": 2.338498637813447, + "260": 4.252136285450512, + "325": 7.783945755077334, + "370": 10.660510343808868, + "390": 11.772593754364621, + "455": 18.45029639107173 + }, + "share_in_band_pct_of_median": { + "-1000-0": 0.5761874143119385, + "0-20": 1.232787796567757, + "20-40": 5.337362599088925, + "40-50": 4.628753995911311, + "50-60": 7.901269029551639, + "60-70": 7.390741908324446, + "70-80": 7.752056665085738 + }, + "by_family_type": { + "couple_wa_no_children": { + "pop_share_pct": 21.271520323545918, + "rel_rate_pct": 11.490883666260077, + "share_of_poor_pct": 12.422447803569764 + }, + "couple_with_children": { + "pop_share_pct": 29.306545654837908, + "rel_rate_pct": 18.36859798423391, + "share_of_poor_pct": 27.35872552565473 + }, + "lone_parent": { + "pop_share_pct": 8.804561431987969, + "rel_rate_pct": 37.8868674455644, + "share_of_poor_pct": 16.953198545199392 + }, + "other_3plus_adults": { + "pop_share_pct": 14.84584058934961, + "rel_rate_pct": 11.826591871624919, + "share_of_poor_pct": 8.923179398361059 + }, + "pensioner_couple_or_mixed": { + "pop_share_pct": 11.593695395825643, + "rel_rate_pct": 14.116832610247318, + "share_of_poor_pct": 8.317912982280182 + }, + "single_pensioner": { + "pop_share_pct": 6.0026127946652315, + "rel_rate_pct": 35.65255732077882, + "share_of_poor_pct": 10.876426719664218 + }, + "single_wa_no_children": { + "pop_share_pct": 8.175223809787722, + "rel_rate_pct": 36.45889899660404, + "share_of_poor_pct": 15.148109025270655 + } + }, + "by_n_children": { + "0": { + "pop_share_pct": 51.99199639250276, + "rel_rate_pct": 18.961277044407733, + "share_of_poor_pct": 50.10248876483717 + }, + "1": { + "pop_share_pct": 20.439674009909893, + "rel_rate_pct": 12.269472141511116, + "share_of_poor_pct": 12.745446830521985 + }, + "2": { + "pop_share_pct": 17.641048299520516, + "rel_rate_pct": 16.262193657893885, + "share_of_poor_pct": 14.580040799946529 + }, + "3": { + "pop_share_pct": 9.927281298066822, + "rel_rate_pct": 44.73886332991615, + "share_of_poor_pct": 22.572023604694316 + } + }, + "children_rel_rate_by_family_children": { + "1": { + "rate": 13.77486331475688, + "children_m": 4.521870235350698 + }, + "2": { + "rate": 17.619101768950316, + "children_m": 6.26379436050281 + }, + "3": { + "rate": 45.917796640548744, + "children_m": 4.462300509260222 + } + }, + "by_work": { + "workless_hh": { + "pop_share_pct": 25.7404989185242, + "rel_rate_pct": 38.35850233071385, + "share_of_poor_pct": 50.1803659740768 + }, + "working_hh": { + "pop_share_pct": 74.2595010814758, + "rel_rate_pct": 13.200588227864348, + "share_of_poor_pct": 49.8196340259232 + }, + "any_ft_earner": { + "pop_share_pct": 67.51475193552405, + "rel_rate_pct": 9.521934075861637, + "share_of_poor_pct": 32.6722518688813 + }, + "working_no_ft": { + "pop_share_pct": 6.744749145951758, + "rel_rate_pct": 50.0238142892988, + "share_of_poor_pct": 17.1473821570419 + } + }, + "by_tenure": { + "OWNED_OUTRIGHT": { + "pop_share_pct": 31.12760611012155, + "rel_rate_pct": 20.552162137366263, + "share_of_poor_pct": 32.5131061111311 + }, + "OWNED_WITH_MORTGAGE": { + "pop_share_pct": 31.901579246685017, + "rel_rate_pct": 9.353240521323821, + "share_of_poor_pct": 15.164549288352832 + }, + "RENT_FROM_COUNCIL": { + "pop_share_pct": 9.208246869580119, + "rel_rate_pct": 43.72553272597473, + "share_of_poor_pct": 20.462904863975 + }, + "RENT_FROM_HA": { + "pop_share_pct": 6.847437604355351, + "rel_rate_pct": 30.678994638281825, + "share_of_poor_pct": 10.676389974090426 + }, + "RENT_PRIVATELY": { + "pop_share_pct": 20.91513016925796, + "rel_rate_pct": 19.928411984426557, + "share_of_poor_pct": 21.18304976245064 + } + }, + "children": { + "rel_rate_pct": 24.760653598499392, + "in_workless_hh_rate": 62.51705849828974, + "share_children_in_workless_hh": 16.46014725389038 + }, + "uc": { + "hh_with_uc_m": 6.239545211998403, + "persons_in_uc_hh_rel_poor_pct": 42.32590334869225, + "share_of_rel_poor_in_uc_hh_pct": 44.298514411532, + "would_claim_false_eligible_hh_m": null + } + } +} \ No newline at end of file diff --git a/docs/evidence/uk-poverty-968/outputs/takeup_direction.json b/docs/evidence/uk-poverty-968/outputs/takeup_direction.json new file mode 100644 index 000000000..61a477d6f --- /dev/null +++ b/docs/evidence/uk-poverty-968/outputs/takeup_direction.json @@ -0,0 +1,38 @@ +[ + { + "label": "baseline draw (v20 file)", + "uc_families_m": 6.366628264195442, + "uc_bn": 76.8327281872474, + "rel_bhc_pct": 15.301746413861888, + "abs_bhc_pct": 9.31103618296997, + "child_rel_pct": 18.759402482558862, + "median_week": 751.48046875 + }, + { + "label": "everyone entitled claims", + "uc_families_m": 8.608132566101313, + "uc_bn": 88.91328865039067, + "rel_bhc_pct": 14.84606087017068, + "abs_bhc_pct": 8.902365521388054, + "child_rel_pct": 18.27018418397132, + "median_week": 754.9448617788462 + }, + { + "label": "residual to the LARGEST entitlements (DWP take-up pattern)", + "uc_families_m": 6.573080968585253, + "uc_bn": 84.2527051892229, + "rel_bhc_pct": 14.877240034148723, + "abs_bhc_pct": 8.831401289249339, + "child_rel_pct": 18.650685879964268, + "median_week": 755.2512019230769 + }, + { + "label": "residual to the SMALLEST entitlements (opposite)", + "uc_families_m": 6.673590072171211, + "uc_bn": 68.11996854422748, + "rel_bhc_pct": 15.902036846491308, + "abs_bhc_pct": 10.054853150827075, + "child_rel_pct": 20.022619001341596, + "median_week": 745.8589242788462 + } +] \ No newline at end of file diff --git a/docs/evidence/uk-poverty-968/outputs/takeup_probe.json b/docs/evidence/uk-poverty-968/outputs/takeup_probe.json new file mode 100644 index 000000000..30219e5a4 --- /dev/null +++ b/docs/evidence/uk-poverty-968/outputs/takeup_probe.json @@ -0,0 +1,82 @@ +[ + { + "label": "baseline draw (v20 file)", + "claim_true_units": 23609, + "uc_families_m": 6.366628264195442, + "uc_bn": 76.8327281872474, + "rel_bhc_pct": 15.301746413861888, + "abs_bhc_pct": 9.31103618296997, + "child_rel_pct": 18.759402482558862, + "median_week": 751.48046875 + }, + { + "label": "everyone eligible claims (full take-up)", + "claim_true_units": 61213, + "uc_families_m": 8.608132566101313, + "uc_bn": 88.91328865039067, + "rel_bhc_pct": 14.84606087017068, + "abs_bhc_pct": 8.902365521388054, + "child_rel_pct": 18.27018418397132, + "median_week": 754.9448617788462 + }, + { + "label": "alternative random draw seed 1 (same pool count)", + "claim_true_units": 23609, + "uc_families_m": 6.657901403939485, + "uc_bn": 76.35815771604855, + "rel_bhc_pct": 15.39889594138504, + "abs_bhc_pct": 9.518166877962985, + "child_rel_pct": 19.24463304922288, + "median_week": 750.5739182692307 + }, + { + "label": "alternative random draw seed 2 (same pool count)", + "claim_true_units": 23609, + "uc_families_m": 6.569224635575295, + "uc_bn": 75.43458077496157, + "rel_bhc_pct": 15.588309795497496, + "abs_bhc_pct": 9.713137948514358, + "child_rel_pct": 19.379405973636253, + "median_week": 750.7356520432693 + }, + { + "label": "alternative random draw seed 3 (same pool count)", + "claim_true_units": 23609, + "uc_families_m": 6.649282045759678, + "uc_bn": 76.07217339162284, + "rel_bhc_pct": 15.466480688055007, + "abs_bhc_pct": 9.531617693484717, + "child_rel_pct": 19.524596436629892, + "median_week": 751.2119891826923 + }, + { + "label": "residual to the POOREST entitled units (lower bound)", + "claim_true_units": 23609, + "uc_families_m": 6.282719685167551, + "uc_bn": 76.86378259339959, + "rel_bhc_pct": 14.048162923518687, + "abs_bhc_pct": 8.472403549297532, + "child_rel_pct": 17.56077482060928, + "median_week": 745.8589242788462 + }, + { + "label": "residual to the RICHEST entitled units (upper bound)", + "claim_true_units": 23609, + "uc_families_m": 6.95246468857956, + "uc_bn": 75.46329913799686, + "rel_bhc_pct": 16.657074318118408, + "abs_bhc_pct": 10.561862325491255, + "child_rel_pct": 21.044435243000358, + "median_week": 755.2287409855769 + }, + { + "label": "anchored reporters only (no residual draw)", + "claim_true_units": 20543, + "uc_families_m": 4.66540806167984, + "uc_bn": 63.571263130751404, + "rel_bhc_pct": 16.11736042406712, + "abs_bhc_pct": 10.566948570893295, + "child_rel_pct": 20.17302317579314, + "median_week": 744.3145282451923 + } +] \ No newline at end of file diff --git a/docs/evidence/uk-poverty-968/receipts.json b/docs/evidence/uk-poverty-968/receipts.json new file mode 100644 index 000000000..d2c5d87cf --- /dev/null +++ b/docs/evidence/uk-poverty-968/receipts.json @@ -0,0 +1,78 @@ +{ + "purpose": "Receipts for the microcosm#968 BHC poverty diagnosis (memo in README.md): the files measured, their digests, the engine, the feed pin and the DWP sources. Outputs are aggregates only; no household rows.", + "date": "2026-09-21", + "engine": { + "package": "policyengine-uk", + "version": "2.99.1", + "environment": "policyengine-uk-data .venv (the evaluation legs' engine); microcosm builds stay locked at 2.98.0" + }, + "feed": { + "chronicle_artifact": "chronicle-uk-artifact-c5e5bf8", + "note": "the spine-s build and the v20 calibration consumed this feed; the poverty measurements do not read the feed" + }, + "files": { + "v20_national": { + "path": "/Users/mariajuaristi/Desktop/PolicyEngine/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5", + "sha256": "8883e59256792525afcff2fe17af7f659ba3982cd0ac7a95a4005bac5ee7b389", + "bytes": 176040337, + "time_period": 2024 + }, + "spine_s_design": { + "path": "/Users/mariajuaristi/Desktop/PolicyEngine/data/ukds/acceptance/spine-s-main-rebuild/spine-s.h5", + "sha256": "4d9752fdcd92c0dc9fc682c15257788e771ced260da5d5b012ac23edfa048997", + "bytes": 175920391, + "time_period": 2024 + }, + "efrs_1_57_3": { + "path": "/Users/mariajuaristi/.cache/huggingface/hub/models--policyengine--policyengine-uk-data-private/snapshots/25af520a6651b8812fef56964a42a79a3f9f515a/enhanced_frs_2024_25.h5", + "sha256": "ef34c1ae28219367981fbc3c1144f58ea1f8a77554165fe02ff395b04c5ffea5", + "bytes": 127549424, + "time_period": 2024 + }, + "x50_55k_local": { + "path": "/Users/mariajuaristi/Desktop/PolicyEngine/data/ukds/acceptance/spine-s-main-rebuild/l0-k20-h55000-e2000-p50-f001-s42/microcosm_uk_2025_local.h5", + "sha256": "9dddfcb297184cc382c51402d8f178797e7e02dbe9cb87e0417c2d8c201dc89b", + "bytes": 243477508, + "time_period": 2025 + }, + "june_populace_uk_2023": { + "path": "/Users/mariajuaristi/.cache/huggingface/hub/datasets--policyengine--populace-uk-private/snapshots/a75a9a831d6b07aaffbd09713f2a1124f5c0f08f/populace_uk_2023.h5", + "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "bytes": 1315880118, + "time_period": 2023 + } + }, + "identity": "spine-s.h5 and the v20 microcosm_uk_2024.h5 hold the same 52,846 households in the same order (asserted in measure_968_poverty_audit.py); only household_weight differs, which is what makes the design-versus-calibrated decomposition exact", + "hbai_sources": [ + { + "title": "DWP, Households Below Average Income FYE 1995 to FYE 2024, summary tables (HBAI_summary_results.ods): 1.2b quintile medians, 1.3a-1.6a low-income shares", + "url": "https://www.gov.uk/government/statistics/households-below-average-income-for-financial-years-ending-1995-to-2024", + "published": "2025-03-27" + }, + { + "title": "DWP HBAI FYE 2024 table pack (HBAI_2324_ods_table_pack.zip): chart 2.4 income bands, tables 2.1db, 2.2db, 3.5db, 3.6db, 4.5db, 4.6db", + "url": "https://assets.publishing.service.gov.uk/media/6836dae29411f0341f323689/HBAI_2324_ods_table_pack.zip" + }, + { + "title": "DWP, HBAI series quality and methodology information report FYE 2024 (income definition, individual unit of analysis, absolute line uprating)", + "url": "https://www.gov.uk/government/statistics/households-below-average-income-for-financial-years-ending-1995-to-2024/households-below-average-income-series-quality-and-methodology-information-report-fye-2024" + }, + { + "title": "DWP, Family Resources Survey 2023/24 methodology tables (ch1-methodology-and-standard-errors.ods): M.6a, M.6b, M.8 benefit receipt against administrative data", + "url": "https://assets.publishing.service.gov.uk/media/681c890de26cd2f713d8709b/ch1-methodology-and-standard-errors.ods" + }, + { + "title": "ONS, Household disposable income and inequality, FYE 2024 (median \u00a336,700 a year)", + "url": "https://www.ons.gov.uk/peoplepopulationandcommunity/personalandhouseholdfinances/incomeandwealth/bulletins/householddisposableincomeandinequality/financialyearending2024" + } + ], + "outputs": { + "poverty_audit.json": "measure_968_poverty_audit.py: concept pair, medians, shape, groups, decile composition, and the spine-s design-versus-v20 decomposition (2024/2025/2026)", + "basis_swap.json": "measure_968_basis_swap.py: simulated means-tested benefits replaced by FRS-reported amounts, same records, year 2024", + "tail_profile.json": "measure_968_tail_profile.py: bottom-tail bands and family-type, family-size, work and tenure breakdowns, year 2024", + "takeup_probe.json": "measure_968_takeup_probe.py: who receives the drawn UC residual at a fixed claimant count (random redraws, poorest-first, richest-first, reporters only, full take-up)", + "takeup_direction.json": "measure_968_takeup_direction.py: the residual ranked by entitlement size (largest first = DWP's non-take-up pattern) and its opposite", + "scorecard_*.json": "scorecard_731.py at 2026 with the HBAI-basis block at 2024, one file each" + }, + "scripts_note": "The scripts are copies of the evaluation stack's (uk-candidate-eval, not a public repository); paths inside them are the author's local run trees and HF cache. They run under the policyengine-uk-data .venv with policyengine-uk 2.99.1." +} \ No newline at end of file diff --git a/docs/evidence/uk-poverty-968/scripts/measure_968_basis_swap.py b/docs/evidence/uk-poverty-968/scripts/measure_968_basis_swap.py new file mode 100644 index 000000000..f0fb9ca6e --- /dev/null +++ b/docs/evidence/uk-poverty-968/scripts/measure_968_basis_swap.py @@ -0,0 +1,62 @@ +"""microcosm#968 experiment: HBAI-basis swap. Replace the engine's simulated means-tested +benefits with the FRS-reported amounts (same records, same weights, year 2024) and re-measure +BHC poverty. Quantifies the 'benchmark basis differs' layer.""" +import sys, json, time, warnings +import numpy as np, pandas as pd +warnings.filterwarnings("ignore") +from importlib.metadata import version +from policyengine_uk import Microsimulation +ROOT = "/Users/mariajuaristi/Desktop/PolicyEngine"; HF = "/Users/mariajuaristi/.cache/huggingface/hub" +FILES = { + "v20_national": f"{ROOT}/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5", + "efrs_1_57_3": f"{HF}/models--policyengine--policyengine-uk-data-private/snapshots/25af520a6651b8812fef56964a42a79a3f9f515a/enhanced_frs_2024_25.h5", +} +SPINE_S = f"{ROOT}/data/ukds/acceptance/spine-s-main-rebuild/spine-s.h5" +# engine variable -> reported person-level input column(s) +SWAP = {"universal_credit": ["universal_credit_reported"], "housing_benefit": ["housing_benefit_reported"], "pension_credit": ["pension_credit_reported"], + "child_tax_credit": ["child_tax_credit_reported"], "working_tax_credit": ["working_tax_credit_reported"], "income_support": ["income_support_reported"], + "jsa_income": ["jsa_income_reported"], "esa_income": ["esa_income_reported"], "council_tax_benefit": ["council_tax_benefit_reported"], "child_benefit": ["child_benefit_reported"]} +YEAR = 2024 +def wmedian(x, w): + o = np.argsort(x); c = np.cumsum(w[o]); return float(x[o][np.searchsorted(c, c[-1]/2)]) +def wshare(m, w): return float(w[m].sum()/w.sum()) +out = {"engine": version("policyengine-uk"), "year": YEAR, "swap_set": list(SWAP), "files": {}} +for name, path in FILES.items(): + t0=time.time(); sim = Microsimulation(dataset=path) + hh = lambda v: np.asarray(sim.calculate(v, YEAR, map_to="household").values, float) + with pd.HDFStore(path, "r") as st: person = st["person"]; h5hh = st["household"] + hid = h5hh["household_id"].to_numpy() + p_idx = pd.Index(hid).get_indexer(person["person_household_id"].to_numpy()); assert (p_idx>=0).all() + hw = hh("household_weight"); pw = hw[p_idx] + assert len(hw) == len(hid) and np.allclose(hw, h5hh["household_weight"].to_numpy(float)), "sim household order != H5 order" + assert len(np.asarray(sim.calculate("age", YEAR).values)) == len(person), "sim person count != H5" + equiv = hh("household_equivalisation_bhc"); thr = float(hh("poverty_threshold_bhc")[0]) + base_income = hh("hbai_household_net_income") + h5_p_idx = p_idx + def reported_hh(cols): + v = sum(pd.to_numeric(person[c], errors="coerce").fillna(0).to_numpy(float) for c in cols if c in person) + return np.bincount(h5_p_idx, weights=v, minlength=len(hid)) + sim_amt = {k: hh(k) for k in SWAP}; rep_amt = {k: reported_hh(cols) for k, cols in SWAP.items()} + weightings = {"calibrated": hw} + if name == "v20_national": + with pd.HDFStore(SPINE_S, "r") as st: design = st["household"]["household_weight"].to_numpy(float) + weightings["spine_s_design"] = design * hw.sum()/design.sum() + entry = {"totals_bn": {k: {"simulated": float((sim_amt[k]*hw).sum()/1e9), "reported": float((rep_amt[k]*hw).sum()/1e9), + "simulated_recipient_hh_m": float(hw[sim_amt[k]>0].sum()/1e6), "reported_recipient_hh_m": float(hw[rep_amt[k]>0].sum()/1e6)} for k in SWAP}, "weightings": {}} + for wname, w in weightings.items(): + pwx = w[p_idx] + def rates(income): + eq = income/equiv; eqp = eq[p_idx]; med = wmedian(eqp, pwx) + return {"abs_bhc_pct": 100*wshare(eqp < thr, pwx), "rel_bhc_pct": 100*wshare(eqp < 0.6*med, pwx), "median_week": med/52, "fixed_line_60pct_of_base_median_pct": None} + base = rates(base_income); base_med = base["median_week"]*52 + res = {"engine_simulated": base} + full = base_income - sum(sim_amt.values()) + sum(rep_amt.values()) + r = rates(full); r["fixed_line_60pct_of_base_median_pct"] = 100*wshare((full/equiv)[p_idx] < 0.6*base_med, pwx); res["all_reported"] = r + for k in SWAP: + inc = base_income - sim_amt[k] + rep_amt[k]; r = rates(inc); r["fixed_line_60pct_of_base_median_pct"] = 100*wshare((inc/equiv)[p_idx] < 0.6*base_med, pwx); res[f"only_{k}_reported"] = r + entry["weightings"][wname] = res + print(name, wname, "abs", round(base["abs_bhc_pct"],2), "->", round(res["all_reported"]["abs_bhc_pct"],2), "| rel", round(base["rel_bhc_pct"],2), "->", round(res["all_reported"]["rel_bhc_pct"],2), flush=True) + out["files"][name] = entry + json.dump(out, open(sys.argv[1], "w"), indent=1) + print(name, "done", round(time.time()-t0), "s", flush=True) +print("ALL DONE") diff --git a/docs/evidence/uk-poverty-968/scripts/measure_968_poverty_audit.py b/docs/evidence/uk-poverty-968/scripts/measure_968_poverty_audit.py new file mode 100644 index 000000000..a8596e035 --- /dev/null +++ b/docs/evidence/uk-poverty-968/scripts/measure_968_poverty_audit.py @@ -0,0 +1,123 @@ +"""microcosm#968: poverty concept audit + decomposition across UK files (one engine).""" +import sys, json, time, warnings +import numpy as np, pandas as pd +warnings.filterwarnings("ignore") +from importlib.metadata import version +from policyengine_uk import Microsimulation + +ROOT = "/Users/mariajuaristi/Desktop/PolicyEngine" +HF = "/Users/mariajuaristi/.cache/huggingface/hub" +FILES = { + "v20_national": (f"{ROOT}/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5", [2024, 2025, 2026]), + "efrs_1_57_3": (f"{HF}/models--policyengine--policyengine-uk-data-private/snapshots/25af520a6651b8812fef56964a42a79a3f9f515a/enhanced_frs_2024_25.h5", [2024, 2025, 2026]), + "x50_55k": (f"{ROOT}/data/ukds/acceptance/spine-s-main-rebuild/l0-k20-h55000-e2000-p50-f001-s42/microcosm_uk_2025_local.h5", [2026]), + "june_populace_2023": (f"{HF}/datasets--policyengine--populace-uk-private/snapshots/a75a9a831d6b07aaffbd09713f2a1124f5c0f08f/populace_uk_2023.h5", [2026]), +} +SPINE_S = f"{ROOT}/data/ukds/acceptance/spine-s-main-rebuild/spine-s.h5" +OUT = sys.argv[1] if len(sys.argv) > 1 else "poverty_audit.json" +ONLY = sys.argv[2].split(",") if len(sys.argv) > 2 else None + +BENEFITS = ["child_benefit","council_tax_benefit","esa_income","esa_contrib","housing_benefit","income_support","jsa_income","jsa_contrib","pension_credit","universal_credit","working_tax_credit","child_tax_credit","attendance_allowance","afcs","bsp","carers_allowance","dla","iidb","incapacity_benefit","pip","sda","state_pension","maternity_allowance","statutory_sick_pay","statutory_maternity_pay","ssmg","cost_of_living_support_payment","winter_fuel_allowance","tax_free_childcare","healthy_start_vouchers","scottish_child_payment","carer_support_payment","free_school_meals","free_school_fruit_veg","free_school_milk","free_tv_licence_value"] +MARKET = ["employment_income","self_employment_income","savings_interest_income","dividend_income","miscellaneous_income","property_income","private_pension_income","private_transfer_income","maintenance_income"] +TAXES = ["income_tax","national_insurance","council_tax","domestic_rates","student_loan_repayments","employee_pension_contributions","personal_pension_contributions","maintenance_expenses","external_child_payments"] + +def wmedian(x, w): + o = np.argsort(x); c = np.cumsum(w[o]); return float(x[o][np.searchsorted(c, c[-1] / 2)]) +def wshare(mask, w): return float(w[mask].sum() / w.sum()) +def wq(x, w, qs): + o = np.argsort(x); c = np.cumsum(w[o]) / w[o].sum(); return [float(x[o][np.searchsorted(c, q)]) for q in qs] + +def measure(sim, year, hw, p_idx, hh_arrays, per_arrays): + A = hh_arrays; P = per_arrays + pw = hw[p_idx] + bhc = A["equiv_bhc"][p_idx]; ahc = A["equiv_ahc"][p_idx] + thr_b = float(A["thr_bhc"][0]); thr_a = float(A["thr_ahc"][0]) + med_b_p = wmedian(bhc, pw); med_a_p = wmedian(ahc, pw) + med_b_h = wmedian(A["equiv_bhc"], hw); med_a_h = wmedian(A["equiv_ahc"], hw) + child = P["age"] < 18; pens = P["age"] >= 66; wa = (~child) & (~pens) + def block(inc, thr_abs, med_p, med_h): + rel_p = inc < 0.6 * med_p; rel_h = inc < 0.6 * med_h; abs_ = inc < thr_abs + d = { + "abs_rate_pct": 100*wshare(abs_, pw), "rel_rate_person_median_pct": 100*wshare(rel_p, pw), + "rel_rate_household_median_pct": 100*wshare(rel_h, pw), + "median_person_weighted_week": med_p/52, "median_household_weighted_week": med_h/52, + "abs_threshold_week": thr_abs/52, "rel_line_person_week": 0.6*med_p/52, + "abs_threshold_over_person_median": thr_abs/med_p, + "share_below_pct_of_person_median": {str(q): 100*wshare(inc < q/100*med_p, pw) for q in (40,50,60,70,80)}, + "share_below_abs_line_scaled": {str(q): 100*wshare(inc < q/100*thr_abs, pw) for q in (90,95,100,105,110)}, + "by_group_abs_pct": {"children": 100*wshare(abs_ & child, pw)/wshare(child,pw), "working_age": 100*wshare(abs_ & wa, pw)/wshare(wa,pw), "pensioners": 100*wshare(abs_ & pens, pw)/wshare(pens,pw)}, + "by_group_rel_pct": {"children": 100*wshare(rel_p & child, pw)/wshare(child,pw), "working_age": 100*wshare(rel_p & wa, pw)/wshare(wa,pw), "pensioners": 100*wshare(rel_p & pens, pw)/wshare(pens,pw)}, + "group_pop_shares_pct": {"children": 100*wshare(child,pw), "working_age": 100*wshare(wa,pw), "pensioners": 100*wshare(pens,pw)}, + "share_nonpositive_income_pct": 100*wshare(inc <= 0, pw), + "person_quantiles_week": dict(zip(["p5","p10","p20","p30","p40","p50"], [v/52 for v in wq(inc, pw, [.05,.1,.2,.3,.4,.5])])), + } + return d + out = {"bhc": block(bhc, thr_b, med_b_p, med_b_h), "ahc": block(ahc, thr_a, med_a_p, med_a_h)} + # decile composition on person-weighted equivalised BHC deciles (household-level amounts, weekly per household) + edges = wq(bhc, pw, [i/10 for i in range(1,10)]) + dec = np.searchsorted(edges, bhc, side="right") # 0..9 per person + comp = {} + for d in range(10): + m = dec == d; w = pw[m]; hh = p_idx[m] + comp[str(d+1)] = {k: float((A[k][hh]*w).sum()/w.sum()/52) for k in ("market","benefits","taxes","hbai_net","uc","state_pension","hb","pension_credit","child_benefit","equiv_bhc")} + comp[str(d+1)]["persons_m"] = float(w.sum()/1e6) + comp[str(d+1)]["equiv_factor_mean"] = float((A["equiv_factor"][hh]*w).sum()/w.sum()) + out["decile_composition_person_weighted_weekly_per_household"] = comp + out["totals_bn"] = {k: float((A[k]*hw).sum()/1e9) for k in ("market","benefits","taxes","hbai_net","uc","state_pension","hb","pension_credit","child_benefit")} + out["population_m"] = float(pw.sum()/1e6); out["households_m"] = float(hw.sum()/1e6) + out["engine_flags_pct"] = {"in_poverty_bhc": 100*wshare(A["flag_abs_bhc"][p_idx] > 0, pw), "in_relative_poverty_bhc": 100*wshare(A["flag_rel_bhc"][p_idx] > 0, pw), "in_poverty_ahc": 100*wshare(A["flag_abs_ahc"][p_idx] > 0, pw)} + return out + +def load_arrays(sim, year): + hh = lambda v: np.asarray(sim.calculate(v, year, map_to="household").values, dtype=float) + A = {"equiv_bhc": hh("equiv_hbai_household_net_income"), "equiv_ahc": hh("equiv_hbai_household_net_income_ahc"), + "thr_bhc": hh("poverty_threshold_bhc"), "thr_ahc": hh("poverty_line_ahc")/hh("household_equivalisation_ahc"), "equiv_factor": hh("household_equivalisation_bhc"), + "hbai_net": hh("hbai_household_net_income"), "flag_abs_bhc": hh("in_poverty_bhc"), "flag_rel_bhc": hh("in_relative_poverty_bhc"), "flag_abs_ahc": hh("in_poverty_ahc"), + "hw": hh("household_weight")} + A["market"] = sum(hh(v) for v in MARKET); A["benefits"] = sum(hh(v) for v in BENEFITS); A["taxes"] = sum(hh(v) for v in TAXES) + A["uc"] = hh("universal_credit"); A["state_pension"] = hh("state_pension"); A["hb"] = hh("housing_benefit"); A["pension_credit"] = hh("pension_credit"); A["child_benefit"] = hh("child_benefit") + P = {"age": np.asarray(sim.calculate("age", year).values, dtype=float), "pw": np.asarray(sim.calculate("person_weight", year).values, dtype=float)} + hid = np.asarray(sim.calculate("household_id", year).values); hid_p = np.asarray(sim.calculate("household_id", year, map_to="person").values) + p_idx = pd.Index(hid).get_indexer(hid_p) + assert (p_idx >= 0).all(), "person->household mapping failed" + return A, P, p_idx + +res = {"engine": version("policyengine-uk"), "files": {}} +for name, (path, years) in FILES.items(): + if ONLY and name not in ONLY: continue + t0 = time.time(); print("==", name, flush=True) + sim = Microsimulation(dataset=path) + entry = {"h5": path, "years": {}} + for y in years: + t1 = time.time() + A, P, p_idx = load_arrays(sim, y) + hw = A["hw"] + assert np.allclose(P["pw"], hw[p_idx]), "person_weight != household weight mapping" + entry["years"][str(y)] = measure(sim, y, hw, p_idx, A, P) + print(" year", y, "done", round(time.time()-t1), "s; abs BHC", round(entry["years"][str(y)]["bhc"]["abs_rate_pct"],2), "rel(person med)", round(entry["years"][str(y)]["bhc"]["rel_rate_person_median_pct"],2), flush=True) + if name == "v20_national": + # decomposition against spine-s design weights (same records, same order — verified) + with pd.HDFStore(SPINE_S, "r") as st: design = st["household"]["household_weight"].to_numpy(float) + with pd.HDFStore(path, "r") as st: cal = st["household"]["household_weight"].to_numpy(float) + ratio = hw.sum()/cal.sum() # engine may scale weights by year; apply same scaling to design + dw = design * ratio + dm = measure(sim, y, dw, p_idx, A, P) + entry["years"][str(y)]["spine_s_design_weights"] = dm + # fixed-line decomposition: design line applied to calibrated weights + pw_c = hw[p_idx]; pw_d = dw[p_idx]; bhc = A["equiv_bhc"][p_idx] + med_d = wmedian(bhc, pw_d); med_c = wmedian(bhc, pw_c) + entry["years"][str(y)]["decomposition_bhc_rel"] = { + "design_weights_design_line_pct": 100*wshare(bhc < 0.6*med_d, pw_d), + "calibrated_weights_design_line_pct": 100*wshare(bhc < 0.6*med_d, pw_c), + "calibrated_weights_calibrated_line_pct": 100*wshare(bhc < 0.6*med_c, pw_c), + "design_median_week": med_d/52, "calibrated_median_week": med_c/52, + "weight_scale_ratio_engine_vs_h5": ratio, + } + thr = float(A["thr_bhc"][0]) + entry["years"][str(y)]["decomposition_bhc_abs"] = {"design_weights_pct": 100*wshare(bhc < thr, pw_d), "calibrated_weights_pct": 100*wshare(bhc < thr, pw_c)} + print(" decomposition:", json.dumps(entry["years"][str(y)]["decomposition_bhc_rel"]), flush=True) + json.dump(res | {"files": res["files"] | {name: entry}}, open(OUT, "w"), indent=1) + res["files"][name] = entry + json.dump(res, open(OUT, "w"), indent=1) + print(" file done", round(time.time()-t0), "s", flush=True) +print("ALL DONE") diff --git a/docs/evidence/uk-poverty-968/scripts/measure_968_tail_profile.py b/docs/evidence/uk-poverty-968/scripts/measure_968_tail_profile.py new file mode 100644 index 000000000..705cdf159 --- /dev/null +++ b/docs/evidence/uk-poverty-968/scripts/measure_968_tail_profile.py @@ -0,0 +1,56 @@ +"""microcosm#968: bottom-tail profile and relative-poverty breakdowns (year 2024) for HBAI comparison.""" +import sys, json, warnings +import numpy as np, pandas as pd +warnings.filterwarnings("ignore") +from policyengine_uk import Microsimulation +ROOT="/Users/mariajuaristi/Desktop/PolicyEngine"; HF="/Users/mariajuaristi/.cache/huggingface/hub" +FILES={"v20_national": f"{ROOT}/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5", + "efrs_1_57_3": f"{HF}/models--policyengine--policyengine-uk-data-private/snapshots/25af520a6651b8812fef56964a42a79a3f9f515a/enhanced_frs_2024_25.h5"} +SPINE_S=f"{ROOT}/data/ukds/acceptance/spine-s-main-rebuild/spine-s.h5" +YEARS=[int(y) for y in (sys.argv[2].split(",") if len(sys.argv)>2 else ["2024"])] +def wmedian(x,w): + o=np.argsort(x); c=np.cumsum(w[o]); return float(x[o][np.searchsorted(c,c[-1]/2)]) +out={} +for name,path in FILES.items(): + sim=Microsimulation(dataset=path) + with pd.HDFStore(path,"r") as st: person=st["person"]; h5hh=st["household"] + hid=h5hh["household_id"].to_numpy(); p_idx=pd.Index(hid).get_indexer(person["person_household_id"].to_numpy()); assert (p_idx>=0).all() + tenure = h5hh["tenure_type"].astype(str).to_numpy() if "tenure_type" in h5hh else np.array(["?"]*len(hid)) + for YEAR in YEARS: + hh=lambda v: np.asarray(sim.calculate(v,YEAR,map_to="household").values,float) + pp=lambda v: np.asarray(sim.calculate(v,YEAR).values,float) + hw=hh("household_weight"); assert np.allclose(hw, h5hh["household_weight"].to_numpy(float)) or YEAR!=2024 + age=pp("age"); emp=pp("employment_income")+pp("self_employment_income"); is_child=age<18; pens=age>=66; adult=age>=18 + n_child=np.bincount(p_idx,weights=is_child,minlength=len(hid)); n_adult=np.bincount(p_idx,weights=adult,minlength=len(hid)) + n_pens=np.bincount(p_idx,weights=pens,minlength=len(hid)); n_work=np.bincount(p_idx,weights=(emp>1000),minlength=len(hid)) + n_ft=np.bincount(p_idx,weights=(emp>=12000),minlength=len(hid)) + equiv=hh("equiv_hbai_household_net_income"); thr=float(hh("poverty_threshold_bhc")[0]) + uc=hh("universal_credit"); wc=np.asarray(sim.calculate("would_claim_uc",YEAR,map_to="household").values,float) + ftype=np.full(len(hid),"other_3plus_adults",dtype=object) + ftype[(n_adult==1)&(n_child==0)&(n_pens==0)]="single_wa_no_children"; ftype[(n_adult==2)&(n_child==0)&(n_pens==0)]="couple_wa_no_children" + ftype[(n_adult==1)&(n_child>0)]="lone_parent"; ftype[(n_adult==2)&(n_child>0)]="couple_with_children" + ftype[(n_adult==1)&(n_pens==1)&(n_child==0)]="single_pensioner"; ftype[(n_adult==2)&(n_pens>=1)&(n_child==0)]="pensioner_couple_or_mixed" + weightings={"calibrated":hw} + if name=="v20_national": + with pd.HDFStore(SPINE_S,"r") as st: design=st["household"]["household_weight"].to_numpy(float) + weightings["spine_s_design"]=design*hw.sum()/design.sum() + for wname,w in weightings.items(): + pw=w[p_idx]; eqp=equiv[p_idx]; med=wmedian(eqp,pw); rel=eqp<0.6*med + tot=pw.sum() + r={"median_week":med/52,"rel_bhc_pct":100*pw[rel].sum()/tot,"abs_bhc_pct":100*pw[eqp=a/100*med)&(eqp=3),k) for k in (0,1,2,3)} + r["children_rel_rate_by_family_children"]={} + for k in (1,2,3): + m=((n_child==k) if k<3 else (n_child>=3))[p_idx]&is_child; r["children_rel_rate_by_family_children"][str(k)]={"rate":100*pw[m&rel].sum()/max(pw[m].sum(),1e-9),"children_m":pw[m].sum()/1e6} + r["by_work"]={"workless_hh":grp(n_work==0,"w0"),"working_hh":grp(n_work>0,"w1"),"any_ft_earner":grp(n_ft>0,"ft"),"working_no_ft":grp((n_work>0)&(n_ft==0),"pt")} + r["by_tenure"]={t:grp(tenure==t,t) for t in sorted(set(tenure))} + r["children"]={"rel_rate_pct":100*pw[is_child&rel].sum()/pw[is_child].sum(),"in_workless_hh_rate":100*pw[is_child&rel&(n_work==0)[p_idx]].sum()/max(pw[is_child&(n_work==0)[p_idx]].sum(),1e-9),"share_children_in_workless_hh":100*pw[is_child&(n_work==0)[p_idx]].sum()/pw[is_child].sum()} + r["uc"]={"hh_with_uc_m":w[uc>0].sum()/1e6,"persons_in_uc_hh_rel_poor_pct":100*pw[(uc>0)[p_idx]&rel].sum()/max(pw[(uc>0)[p_idx]].sum(),1e-9),"share_of_rel_poor_in_uc_hh_pct":100*pw[(uc>0)[p_idx]&rel].sum()/pw[rel].sum(),"would_claim_false_eligible_hh_m":None} + out[f"{name}|{wname}|{YEAR}"]=r + print(name,wname,YEAR,"rel",round(r["rel_bhc_pct"],2),"child",round(r["children"]["rel_rate_pct"],2),"bands",{k:round(v,2) for k,v in r["share_in_band_pct_of_median"].items()},flush=True) +json.dump(out,open(sys.argv[1],"w"),indent=1); print("DONE") diff --git a/docs/evidence/uk-poverty-968/scripts/measure_968_takeup_direction.py b/docs/evidence/uk-poverty-968/scripts/measure_968_takeup_direction.py new file mode 100644 index 000000000..d979c7189 --- /dev/null +++ b/docs/evidence/uk-poverty-968/scripts/measure_968_takeup_direction.py @@ -0,0 +1,34 @@ +"""microcosm#968 follow-up (Vahid point 3): which way does a realistic residual allocation move poverty? +Residual UC claimants among entitled non-reporters ranked by entitlement size (largest first = DWP's +finding that non-take-up concentrates in small entitlements), v20 at 2024, calibrated weights.""" +import sys, json, warnings, numpy as np, pandas as pd +warnings.filterwarnings("ignore") +from policyengine_uk import Microsimulation +PATH="/Users/mariajuaristi/Desktop/PolicyEngine/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5"; YEAR=2024 +def wmedian(x,w): + o=np.argsort(x); c=np.cumsum(w[o]); return float(x[o][np.searchsorted(c,c[-1]/2)]) +with pd.HDFStore(PATH,"r") as st: person=st["person"]; h5hh=st["household"]; bu=st["benunit"] +hid=h5hh["household_id"].to_numpy(); p_idx=pd.Index(hid).get_indexer(person["person_household_id"].to_numpy()) +bid=bu["benunit_id"].to_numpy(); b_of_p=pd.Index(bid).get_indexer(person["person_benunit_id"].to_numpy()) +base_claim=bu["would_claim_uc"].to_numpy(bool) +anchor=np.zeros(len(bid),bool); anchor[np.unique(b_of_p[pd.to_numeric(person["universal_credit_reported"],errors="coerce").fillna(0).to_numpy()>0])]=True +def run(claim,label): + sim=Microsimulation(dataset=PATH); sim.set_input("would_claim_uc",YEAR,claim.astype(bool)) + hh=lambda v: np.asarray(sim.calculate(v,YEAR,map_to="household").values,float) + hw=hh("household_weight"); pw=hw[p_idx]; eq=hh("equiv_hbai_household_net_income")[p_idx]; thr=float(hh("poverty_threshold_bhc")[0]) + uc_b=np.asarray(sim.calculate("universal_credit",YEAR).values,float); bw=np.asarray(sim.calculate("benunit_weight",YEAR).values,float) + med=wmedian(eq,pw); age=np.asarray(sim.calculate("age",YEAR).values,float); child=age<18 + r={"label":label,"uc_families_m":float(bw[uc_b>0].sum()/1e6),"uc_bn":float((uc_b*bw).sum()/1e9),"rel_bhc_pct":100*float(pw[eq<0.6*med].sum()/pw.sum()), + "abs_bhc_pct":100*float(pw[eq0; pool=entitled&~anchor; n=int((base_claim&pool).sum()) +order=np.argsort(-uc_all); pool_order=[i for i in order if pool[i]] +largest=np.zeros(len(bid),bool); largest[pool_order[:n]]=True +smallest=np.zeros(len(bid),bool); smallest[pool_order[-n:]]=True +res.append(run(anchor|largest|(base_claim&~pool),"residual to the LARGEST entitlements (DWP take-up pattern)")[0]) +res.append(run(anchor|smallest|(base_claim&~pool),"residual to the SMALLEST entitlements (opposite)")[0]) +# mean entitlement of the pool by chosen set +w_b=None +json.dump(res,open(sys.argv[1],"w"),indent=1); print("pool",int(pool.sum()),"n",n,"DONE") diff --git a/docs/evidence/uk-poverty-968/scripts/measure_968_takeup_probe.py b/docs/evidence/uk-poverty-968/scripts/measure_968_takeup_probe.py new file mode 100644 index 000000000..4ffb0b6bc --- /dev/null +++ b/docs/evidence/uk-poverty-968/scripts/measure_968_takeup_probe.py @@ -0,0 +1,50 @@ +"""microcosm#968 plan item 4: does WHO claims UC (at a fixed claimant count) move measured poverty? v20 at 2024, calibrated weights.""" +import sys, json, warnings, numpy as np, pandas as pd +warnings.filterwarnings("ignore") +from policyengine_uk import Microsimulation +PATH="/Users/mariajuaristi/Desktop/PolicyEngine/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5"; YEAR=2024 +def wmedian(x,w): + o=np.argsort(x); c=np.cumsum(w[o]); return float(x[o][np.searchsorted(c,c[-1]/2)]) +with pd.HDFStore(PATH,"r") as st: person=st["person"]; h5hh=st["household"]; bu=st["benunit"] +hid=h5hh["household_id"].to_numpy(); p_idx=pd.Index(hid).get_indexer(person["person_household_id"].to_numpy()) +bid=bu["benunit_id"].to_numpy(); b_of_p=pd.Index(bid).get_indexer(person["person_benunit_id"].to_numpy()) +# household index of each benunit (first member) +first_p=pd.Series(np.arange(len(person))).groupby(b_of_p).first().reindex(np.arange(len(bid))).to_numpy(); b_hh=p_idx[first_p] +base_claim=bu["would_claim_uc"].to_numpy(bool) +anchor=np.zeros(len(bid),bool); anchor[np.unique(b_of_p[pd.to_numeric(person["universal_credit_reported"],errors="coerce").fillna(0).to_numpy()>0])]=True +def run(claim, label): + sim=Microsimulation(dataset=PATH) + sim.set_input("would_claim_uc", YEAR, claim.astype(bool)) + hh=lambda v: np.asarray(sim.calculate(v,YEAR,map_to="household").values,float) + hw=hh("household_weight"); pw=hw[p_idx]; eq=hh("equiv_hbai_household_net_income")[p_idx]; thr=float(hh("poverty_threshold_bhc")[0]) + uc_b=np.asarray(sim.calculate("universal_credit",YEAR).values,float); bw=np.asarray(sim.calculate("benunit_weight",YEAR).values,float) + med=wmedian(eq,pw); age=np.asarray(sim.calculate("age",YEAR).values,float); child=age<18 + r={"label":label,"claim_true_units":int(claim.sum()),"uc_families_m":float(bw[uc_b>0].sum()/1e6),"uc_bn":float((uc_b*bw).sum()/1e9), + "rel_bhc_pct":100*float(pw[eq<0.6*med].sum()/pw.sum()),"abs_bhc_pct":100*float(pw[eq 0 +pool = entitled & ~anchor # non-anchored entitled units: the draw chooses among these +n_base_pool = int((base_claim & pool).sum()) +print("entitled units", int(entitled.sum()), "anchored", int(anchor.sum()), "anchored&entitled", int((anchor&entitled).sum()), "pool", int(pool.sum()), "base draws in pool", n_base_pool, flush=True) +# pre-UC equivalised income of the unit's household under full take-up minus its UC (proxy for need ranking) +sim_hh_eq_no_uc = None +rng=np.random.default_rng(12345) +for seed in (1,2,3): + rng=np.random.default_rng(seed); pick=np.zeros(len(bid),bool); idx=np.flatnonzero(pool); pick[rng.choice(idx,size=n_base_pool,replace=False)]=True + res.append(run(anchor|pick|(base_claim&~pool), f"alternative random draw seed {seed} (same pool count)")[0]) +# need-ordered: rank pool by household equivalised income excluding their own UC (from the full-take-up run) +hh_eq_full = eq_all # person-level; take household-level via first member +hh_level_eq_full = np.zeros(len(hid)); hh_level_eq_full[p_idx]=eq_all +with_uc_removed = hh_level_eq_full[b_hh] - uc_all/np.maximum(1e-9,1) # crude: subtract unit UC (not equivalised) — ordering only +order=np.argsort(with_uc_removed); pool_order=[i for i in order if pool[i]] +poorest=np.zeros(len(bid),bool); poorest[pool_order[:n_base_pool]]=True +richest=np.zeros(len(bid),bool); richest[pool_order[-n_base_pool:]]=True +res.append(run(anchor|poorest|(base_claim&~pool), "residual to the POOREST entitled units (lower bound)")[0]) +res.append(run(anchor|richest|(base_claim&~pool), "residual to the RICHEST entitled units (upper bound)")[0]) +res.append(run(anchor|(base_claim&~pool), "anchored reporters only (no residual draw)")[0]) +json.dump(res, open(sys.argv[1],"w"), indent=1); print("DONE") diff --git a/docs/evidence/uk-poverty-968/scripts/scorecard_731.py b/docs/evidence/uk-poverty-968/scripts/scorecard_731.py new file mode 100644 index 000000000..e0f616e8d --- /dev/null +++ b/docs/evidence/uk-poverty-968/scripts/scorecard_731.py @@ -0,0 +1,239 @@ +"""The microcosm#731 benchmark scorecard, replayed on a local H5 at 2026. + +Vahid's scorecard (microcosm#731) compared populace_uk_2023 and +enhanced_frs_2024_25 against approximate admin benchmarks at 2026 through the +policyengine.py wrapper. This probe computes the same rows for any local H5 +directly through policyengine_uk, so the v15 candidate can sit in the same +table. Run under the DRIVER TREE's venv (the publication engine): + + cd && uv run --no-sync python scorecard_731.py \ + --h5 --label microcosm_uk_2024_v15 --year 2026 \ + --out out/v15/scorecard_731_candidate.json + +Conventions, matching the #731 reproduction code: taxpayers = persons with +income_tax > 0; UC families = benunits with universal_credit > 0; Gini and the +top-1% share are household-weighted over household_net_income (Vahid's "net hh +income" label). All outputs are weighted aggregates. + +Poverty (microcosm#968). The engine's ``in_poverty_bhc`` is the ABSOLUTE line +(60 % of the 2010/11 median uprated by CPI), not the relative line HBAI's +headline ~17 % refers to, so the scorecard reports the pair HBAI publishes: + +* ``poverty_bhc_absolute_pct`` — the engine flag, person-weighted share of + individuals (HBAI FYE 2024 absolute BHC: 15 %). ``poverty_bhc_pct`` is kept + as the same value for older reports that read that key. +* ``poverty_bhc_relative_pct`` — individuals in households whose equivalised + HBAI net income is under 60 % of the PERSON-weighted median of the same + file (HBAI FYE 2024: 17 %). The engine's ``in_relative_poverty_bhc`` uses a + household-weighted median, which understates this by ~2 points, so the + median is taken here directly. +* ``poverty_ahc_absolute_pct`` / ``poverty_ahc_relative_pct`` — the AHC pair + (HBAI FYE 2024: 18 % / 21 %). +* ``median_equiv_bhc_income_week`` — the person-weighted median, £/week for a + couple with no children (HBAI FYE 2024: £650). +* ``child_poverty_bhc_relative_pct`` — children under 18 (HBAI FYE 2024: 23 %). + +HBAI reference: DWP, Households Below Average Income FYE 2024, summary tables +1.2b and 1.3a–1.4a (March 2025). +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np + + +def gini(values: np.ndarray, weights: np.ndarray) -> float: + order = np.argsort(values) + v, w = values[order], weights[order] + cum_w = np.cumsum(w) + cum_vw = np.cumsum(v * w) + total_w, total_vw = cum_w[-1], cum_vw[-1] + # Weighted Gini via the Lorenz-curve trapezoid rule. + lorenz = cum_vw / total_vw + prev_lorenz = np.concatenate(([0.0], lorenz[:-1])) + return float(1.0 - ((w / total_w) * (lorenz + prev_lorenz)).sum()) + + +def weighted_median(values: np.ndarray, weights: np.ndarray) -> float: + order = np.argsort(values) + cum = np.cumsum(weights[order]) + return float(values[order][np.searchsorted(cum, cum[-1] / 2.0)]) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--h5", required=True, type=Path) + parser.add_argument("--label", required=True) + parser.add_argument("--year", type=int, default=2026) + parser.add_argument("--out", required=True, type=Path) + parser.add_argument( + "--hbai-year", type=int, default=2024, + help="year at which the HBAI-basis block is measured (HBAI FYE 2024 = April 2023 to March 2024; 0 disables)", + ) + args = parser.parse_args() + + from importlib.metadata import version + + from policyengine_uk import Microsimulation + + sim = Microsimulation(dataset=str(args.h5)) + year = args.year + + def measure(year): + def person(var): + return np.asarray(sim.calculate(var, year).values, dtype=float) + + def to_person(var): + return np.asarray(sim.calculate(var, year, map_to="person").values, dtype=float) + + pw = np.asarray(sim.calculate("person_weight", year).values, dtype=float) + bw = np.asarray(sim.calculate("benunit_weight", year).values, dtype=float) + hw = np.asarray(sim.calculate("household_weight", year).values, dtype=float) + + income_tax = person("income_tax") + uc = np.asarray(sim.calculate("universal_credit", year).values, dtype=float) + net = np.asarray( + sim.calculate("household_net_income", year).values, dtype=float + ) + poverty_abs_bhc = to_person("in_poverty_bhc") + poverty_abs_ahc = to_person("in_poverty_ahc") + equiv_bhc = to_person("equiv_hbai_household_net_income") + equiv_ahc = to_person("equiv_hbai_household_net_income_ahc") + is_child = person("age") < 18 + + median_bhc = weighted_median(equiv_bhc, pw) + median_ahc = weighted_median(equiv_ahc, pw) + rel_bhc = equiv_bhc < 0.6 * median_bhc + rel_ahc = equiv_ahc < 0.6 * median_ahc + + def share(mask, weights=pw): + return float(100.0 * weights[mask].sum() / weights.sum()) + + def quantile(values, weights, q): + order = np.argsort(values) + cum = np.cumsum(weights[order]) / weights[order].sum() + return float(values[order][np.searchsorted(cum, q)]) + + order = np.argsort(net)[::-1] + top1_cut = np.searchsorted(np.cumsum(hw[order]), hw.sum() * 0.01) + top1_share = float( + (net[order][: top1_cut + 1] * hw[order][: top1_cut + 1]).sum() + / (net * hw).sum() + ) + + poverty_bhc_absolute = share(poverty_abs_bhc > 0) + rows = { + "population_m": float(pw.sum() / 1e6), + "households_m": float(hw.sum() / 1e6), + "state_pension_bn": float((person("state_pension") * pw).sum() / 1e9), + "income_tax_bn": float((income_tax * pw).sum() / 1e9), + "income_taxpayers_m": float(pw[income_tax > 0].sum() / 1e6), + "universal_credit_bn": float((uc * bw).sum() / 1e9), + "uc_families_m": float(bw[uc > 0].sum() / 1e6), + "child_benefit_bn": float( + (np.asarray(sim.calculate("child_benefit", year).values, float) * bw).sum() + / 1e9 + ), + "pension_credit_bn": float( + (np.asarray(sim.calculate("pension_credit", year).values, float) * bw).sum() + / 1e9 + ), + "council_tax_bn": float( + (np.asarray(sim.calculate("council_tax", year).values, float) * hw).sum() + / 1e9 + ), + # microcosm#968: the engine flag is the absolute line; keep the old key + # (same value) for reports that read it, and add the HBAI pair. + "poverty_bhc_pct": poverty_bhc_absolute, + "poverty_bhc_absolute_pct": poverty_bhc_absolute, + "poverty_bhc_relative_pct": share(rel_bhc), + "poverty_ahc_absolute_pct": share(poverty_abs_ahc > 0), + "poverty_ahc_relative_pct": share(rel_ahc), + "median_equiv_bhc_income_week": median_bhc / 52.0, + "child_poverty_bhc_relative_pct": float( + 100.0 * pw[is_child & rel_bhc].sum() / pw[is_child].sum() + ), + "p10_over_median_bhc": quantile(equiv_bhc, pw, 0.10) / median_bhc, + "p90_over_median_bhc": quantile(equiv_bhc, pw, 0.90) / median_bhc, + "zero_income_share_pct": share(equiv_bhc < 10.0 * 52.0), + "top_1pct_net_income_share_pct": 100.0 * top1_share, + "gini_net_hh_income": gini(net, hw), + } + return rows + + rows = measure(year) + HBAI_KEYS = ( + "poverty_bhc_absolute_pct", "poverty_bhc_relative_pct", "poverty_ahc_absolute_pct", + "poverty_ahc_relative_pct", "child_poverty_bhc_relative_pct", "median_equiv_bhc_income_week", + "p10_over_median_bhc", "p90_over_median_bhc", "zero_income_share_pct", + ) + hbai_basis = None + if args.hbai_year: + # HBAI FYE 2024 is April 2023 to March 2024; the closest engine year is 2024. The + # files are measured at that year so the comparison does not conflate years + # (microcosm#968, review point 4). p10/p90 are ratios to the person-weighted + # median; zero_income_share is individuals under £10/week equivalised (HBAI's + # £0-10 band, 1.0 % of individuals in FYE 2024). + hb_rows = measure(args.hbai_year) if args.hbai_year != year else rows + hbai_basis = { + "year": args.hbai_year, + "rows": {k: hb_rows[k] for k in HBAI_KEYS}, + "hbai": { + "label": "HBAI FYE 2024 (April 2023 to March 2024), survey-reported incomes", + "poverty_bhc_absolute_pct": 15, + "poverty_bhc_relative_pct": 17, + "poverty_ahc_absolute_pct": 18, + "poverty_ahc_relative_pct": 21, + "child_poverty_bhc_relative_pct": 23, + "median_equiv_bhc_income_week": 650, + "p10_over_median_bhc": 0.48, + "p90_over_median_bhc": 1.955, + "zero_income_share_pct": 1.0, + "source": ( + "DWP HBAI FYE 2024 (March 2025): summary tables 1.2b (quintile medians: " + "bottom 312, top 1271 on a 650 median) and 1.3a-1.4a; chart 2.4 income " + "bands (0.66m individuals at 0-10 GBP/week)" + ), + }, + } + + payload = { + "probe": "microcosm#731 scorecard", + "label": args.label, + "h5": str(args.h5), + "year": year, + "engine": {"package": "policyengine-uk", "version": version("policyengine-uk")}, + "conventions": ( + "taxpayers: income_tax>0 persons; uc_families: universal_credit>0 " + "benunits; poverty_bhc_absolute (= poverty_bhc_pct): person-weighted " + "in_poverty_bhc, the engine's CPI-uprated 2010/11 line; " + "poverty_*_relative: individuals under 60% of the person-weighted " + "median equivalised HBAI net income of the same file (microcosm#968); " + "median_equiv_bhc_income_week: person-weighted, couple-no-children " + "equivalent; gini/top-1%: household-weighted household_net_income" + ), + "hbai_fye_2024_reference": { + "poverty_bhc_absolute_pct": 15, + "poverty_bhc_relative_pct": 17, + "poverty_ahc_absolute_pct": 18, + "poverty_ahc_relative_pct": 21, + "median_equiv_bhc_income_week": 650, + "child_poverty_bhc_relative_pct": 23, + "year": "FYE 2024 (April 2023 to March 2024); compare with the hbai_basis block, not with the 2026 rows", + "source": "DWP HBAI FYE 2024 summary tables 1.2b, 1.3a, 1.4a (March 2025)", + }, + "rows": rows, + "hbai_basis": hbai_basis, + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(payload, indent=2), encoding="utf-8") + print(json.dumps({args.label: rows}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From cf2d24fcef87e36c90711cf5c2da13fa7b7e31de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:33:23 +0100 Subject: [PATCH 05/22] Bring the #968 evidence receipts through lint and the incumbent-reference guard The evidence scripts are formatted and import-sorted (receipts pin output and dataset shas, not script bytes); the style rules that would edit their frozen measurement logic are waived for docs/evidence/*/scripts/ with the reason recorded in pyproject. The six evidence files that name the retired data package do so only as sha-locked provenance (the incumbent H5's HF cache path, the evaluation venv) and join the guard's allowlist beside the parity references that set the precedent. Co-Authored-By: Claude Fable 5 --- .../scripts/measure_968_basis_swap.py | 154 ++++++-- .../scripts/measure_968_poverty_audit.py | 351 ++++++++++++++---- .../scripts/measure_968_tail_profile.py | 216 ++++++++--- .../scripts/measure_968_takeup_direction.py | 125 +++++-- .../scripts/measure_968_takeup_probe.py | 169 +++++++-- .../uk-poverty-968/scripts/scorecard_731.py | 32 +- .../microcosm-build/tests/test_us_plan.py | 11 + pyproject.toml | 4 + 8 files changed, 843 insertions(+), 219 deletions(-) diff --git a/docs/evidence/uk-poverty-968/scripts/measure_968_basis_swap.py b/docs/evidence/uk-poverty-968/scripts/measure_968_basis_swap.py index f0fb9ca6e..ef1a1c125 100644 --- a/docs/evidence/uk-poverty-968/scripts/measure_968_basis_swap.py +++ b/docs/evidence/uk-poverty-968/scripts/measure_968_basis_swap.py @@ -1,62 +1,154 @@ """microcosm#968 experiment: HBAI-basis swap. Replace the engine's simulated means-tested benefits with the FRS-reported amounts (same records, same weights, year 2024) and re-measure BHC poverty. Quantifies the 'benchmark basis differs' layer.""" -import sys, json, time, warnings -import numpy as np, pandas as pd + +import json +import sys +import time +import warnings + +import numpy as np +import pandas as pd + warnings.filterwarnings("ignore") from importlib.metadata import version + from policyengine_uk import Microsimulation -ROOT = "/Users/mariajuaristi/Desktop/PolicyEngine"; HF = "/Users/mariajuaristi/.cache/huggingface/hub" + +ROOT = "/Users/mariajuaristi/Desktop/PolicyEngine" +HF = "/Users/mariajuaristi/.cache/huggingface/hub" FILES = { - "v20_national": f"{ROOT}/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5", - "efrs_1_57_3": f"{HF}/models--policyengine--policyengine-uk-data-private/snapshots/25af520a6651b8812fef56964a42a79a3f9f515a/enhanced_frs_2024_25.h5", + "v20_national": f"{ROOT}/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5", + "efrs_1_57_3": f"{HF}/models--policyengine--policyengine-uk-data-private/snapshots/25af520a6651b8812fef56964a42a79a3f9f515a/enhanced_frs_2024_25.h5", } SPINE_S = f"{ROOT}/data/ukds/acceptance/spine-s-main-rebuild/spine-s.h5" # engine variable -> reported person-level input column(s) -SWAP = {"universal_credit": ["universal_credit_reported"], "housing_benefit": ["housing_benefit_reported"], "pension_credit": ["pension_credit_reported"], - "child_tax_credit": ["child_tax_credit_reported"], "working_tax_credit": ["working_tax_credit_reported"], "income_support": ["income_support_reported"], - "jsa_income": ["jsa_income_reported"], "esa_income": ["esa_income_reported"], "council_tax_benefit": ["council_tax_benefit_reported"], "child_benefit": ["child_benefit_reported"]} +SWAP = { + "universal_credit": ["universal_credit_reported"], + "housing_benefit": ["housing_benefit_reported"], + "pension_credit": ["pension_credit_reported"], + "child_tax_credit": ["child_tax_credit_reported"], + "working_tax_credit": ["working_tax_credit_reported"], + "income_support": ["income_support_reported"], + "jsa_income": ["jsa_income_reported"], + "esa_income": ["esa_income_reported"], + "council_tax_benefit": ["council_tax_benefit_reported"], + "child_benefit": ["child_benefit_reported"], +} YEAR = 2024 + + def wmedian(x, w): - o = np.argsort(x); c = np.cumsum(w[o]); return float(x[o][np.searchsorted(c, c[-1]/2)]) -def wshare(m, w): return float(w[m].sum()/w.sum()) -out = {"engine": version("policyengine-uk"), "year": YEAR, "swap_set": list(SWAP), "files": {}} + o = np.argsort(x) + c = np.cumsum(w[o]) + return float(x[o][np.searchsorted(c, c[-1] / 2)]) + + +def wshare(m, w): + return float(w[m].sum() / w.sum()) + + +out = { + "engine": version("policyengine-uk"), + "year": YEAR, + "swap_set": list(SWAP), + "files": {}, +} for name, path in FILES.items(): - t0=time.time(); sim = Microsimulation(dataset=path) + t0 = time.time() + sim = Microsimulation(dataset=path) hh = lambda v: np.asarray(sim.calculate(v, YEAR, map_to="household").values, float) - with pd.HDFStore(path, "r") as st: person = st["person"]; h5hh = st["household"] + with pd.HDFStore(path, "r") as st: + person = st["person"] + h5hh = st["household"] hid = h5hh["household_id"].to_numpy() - p_idx = pd.Index(hid).get_indexer(person["person_household_id"].to_numpy()); assert (p_idx>=0).all() - hw = hh("household_weight"); pw = hw[p_idx] - assert len(hw) == len(hid) and np.allclose(hw, h5hh["household_weight"].to_numpy(float)), "sim household order != H5 order" - assert len(np.asarray(sim.calculate("age", YEAR).values)) == len(person), "sim person count != H5" - equiv = hh("household_equivalisation_bhc"); thr = float(hh("poverty_threshold_bhc")[0]) + p_idx = pd.Index(hid).get_indexer(person["person_household_id"].to_numpy()) + assert (p_idx >= 0).all() + hw = hh("household_weight") + pw = hw[p_idx] + assert len(hw) == len(hid) and np.allclose( + hw, h5hh["household_weight"].to_numpy(float) + ), "sim household order != H5 order" + assert len(np.asarray(sim.calculate("age", YEAR).values)) == len(person), ( + "sim person count != H5" + ) + equiv = hh("household_equivalisation_bhc") + thr = float(hh("poverty_threshold_bhc")[0]) base_income = hh("hbai_household_net_income") h5_p_idx = p_idx + def reported_hh(cols): - v = sum(pd.to_numeric(person[c], errors="coerce").fillna(0).to_numpy(float) for c in cols if c in person) + v = sum( + pd.to_numeric(person[c], errors="coerce").fillna(0).to_numpy(float) + for c in cols + if c in person + ) return np.bincount(h5_p_idx, weights=v, minlength=len(hid)) - sim_amt = {k: hh(k) for k in SWAP}; rep_amt = {k: reported_hh(cols) for k, cols in SWAP.items()} + + sim_amt = {k: hh(k) for k in SWAP} + rep_amt = {k: reported_hh(cols) for k, cols in SWAP.items()} weightings = {"calibrated": hw} if name == "v20_national": - with pd.HDFStore(SPINE_S, "r") as st: design = st["household"]["household_weight"].to_numpy(float) - weightings["spine_s_design"] = design * hw.sum()/design.sum() - entry = {"totals_bn": {k: {"simulated": float((sim_amt[k]*hw).sum()/1e9), "reported": float((rep_amt[k]*hw).sum()/1e9), - "simulated_recipient_hh_m": float(hw[sim_amt[k]>0].sum()/1e6), "reported_recipient_hh_m": float(hw[rep_amt[k]>0].sum()/1e6)} for k in SWAP}, "weightings": {}} + with pd.HDFStore(SPINE_S, "r") as st: + design = st["household"]["household_weight"].to_numpy(float) + weightings["spine_s_design"] = design * hw.sum() / design.sum() + entry = { + "totals_bn": { + k: { + "simulated": float((sim_amt[k] * hw).sum() / 1e9), + "reported": float((rep_amt[k] * hw).sum() / 1e9), + "simulated_recipient_hh_m": float(hw[sim_amt[k] > 0].sum() / 1e6), + "reported_recipient_hh_m": float(hw[rep_amt[k] > 0].sum() / 1e6), + } + for k in SWAP + }, + "weightings": {}, + } for wname, w in weightings.items(): pwx = w[p_idx] + def rates(income): - eq = income/equiv; eqp = eq[p_idx]; med = wmedian(eqp, pwx) - return {"abs_bhc_pct": 100*wshare(eqp < thr, pwx), "rel_bhc_pct": 100*wshare(eqp < 0.6*med, pwx), "median_week": med/52, "fixed_line_60pct_of_base_median_pct": None} - base = rates(base_income); base_med = base["median_week"]*52 + eq = income / equiv + eqp = eq[p_idx] + med = wmedian(eqp, pwx) + return { + "abs_bhc_pct": 100 * wshare(eqp < thr, pwx), + "rel_bhc_pct": 100 * wshare(eqp < 0.6 * med, pwx), + "median_week": med / 52, + "fixed_line_60pct_of_base_median_pct": None, + } + + base = rates(base_income) + base_med = base["median_week"] * 52 res = {"engine_simulated": base} full = base_income - sum(sim_amt.values()) + sum(rep_amt.values()) - r = rates(full); r["fixed_line_60pct_of_base_median_pct"] = 100*wshare((full/equiv)[p_idx] < 0.6*base_med, pwx); res["all_reported"] = r + r = rates(full) + r["fixed_line_60pct_of_base_median_pct"] = 100 * wshare( + (full / equiv)[p_idx] < 0.6 * base_med, pwx + ) + res["all_reported"] = r for k in SWAP: - inc = base_income - sim_amt[k] + rep_amt[k]; r = rates(inc); r["fixed_line_60pct_of_base_median_pct"] = 100*wshare((inc/equiv)[p_idx] < 0.6*base_med, pwx); res[f"only_{k}_reported"] = r + inc = base_income - sim_amt[k] + rep_amt[k] + r = rates(inc) + r["fixed_line_60pct_of_base_median_pct"] = 100 * wshare( + (inc / equiv)[p_idx] < 0.6 * base_med, pwx + ) + res[f"only_{k}_reported"] = r entry["weightings"][wname] = res - print(name, wname, "abs", round(base["abs_bhc_pct"],2), "->", round(res["all_reported"]["abs_bhc_pct"],2), "| rel", round(base["rel_bhc_pct"],2), "->", round(res["all_reported"]["rel_bhc_pct"],2), flush=True) + print( + name, + wname, + "abs", + round(base["abs_bhc_pct"], 2), + "->", + round(res["all_reported"]["abs_bhc_pct"], 2), + "| rel", + round(base["rel_bhc_pct"], 2), + "->", + round(res["all_reported"]["rel_bhc_pct"], 2), + flush=True, + ) out["files"][name] = entry json.dump(out, open(sys.argv[1], "w"), indent=1) - print(name, "done", round(time.time()-t0), "s", flush=True) + print(name, "done", round(time.time() - t0), "s", flush=True) print("ALL DONE") diff --git a/docs/evidence/uk-poverty-968/scripts/measure_968_poverty_audit.py b/docs/evidence/uk-poverty-968/scripts/measure_968_poverty_audit.py index a8596e035..5e1b53ccb 100644 --- a/docs/evidence/uk-poverty-968/scripts/measure_968_poverty_audit.py +++ b/docs/evidence/uk-poverty-968/scripts/measure_968_poverty_audit.py @@ -1,123 +1,342 @@ """microcosm#968: poverty concept audit + decomposition across UK files (one engine).""" -import sys, json, time, warnings -import numpy as np, pandas as pd + +import json +import sys +import time +import warnings + +import numpy as np +import pandas as pd + warnings.filterwarnings("ignore") from importlib.metadata import version + from policyengine_uk import Microsimulation ROOT = "/Users/mariajuaristi/Desktop/PolicyEngine" HF = "/Users/mariajuaristi/.cache/huggingface/hub" FILES = { - "v20_national": (f"{ROOT}/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5", [2024, 2025, 2026]), - "efrs_1_57_3": (f"{HF}/models--policyengine--policyengine-uk-data-private/snapshots/25af520a6651b8812fef56964a42a79a3f9f515a/enhanced_frs_2024_25.h5", [2024, 2025, 2026]), - "x50_55k": (f"{ROOT}/data/ukds/acceptance/spine-s-main-rebuild/l0-k20-h55000-e2000-p50-f001-s42/microcosm_uk_2025_local.h5", [2026]), - "june_populace_2023": (f"{HF}/datasets--policyengine--populace-uk-private/snapshots/a75a9a831d6b07aaffbd09713f2a1124f5c0f08f/populace_uk_2023.h5", [2026]), + "v20_national": ( + f"{ROOT}/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5", + [2024, 2025, 2026], + ), + "efrs_1_57_3": ( + f"{HF}/models--policyengine--policyengine-uk-data-private/snapshots/25af520a6651b8812fef56964a42a79a3f9f515a/enhanced_frs_2024_25.h5", + [2024, 2025, 2026], + ), + "x50_55k": ( + f"{ROOT}/data/ukds/acceptance/spine-s-main-rebuild/l0-k20-h55000-e2000-p50-f001-s42/microcosm_uk_2025_local.h5", + [2026], + ), + "june_populace_2023": ( + f"{HF}/datasets--policyengine--populace-uk-private/snapshots/a75a9a831d6b07aaffbd09713f2a1124f5c0f08f/populace_uk_2023.h5", + [2026], + ), } SPINE_S = f"{ROOT}/data/ukds/acceptance/spine-s-main-rebuild/spine-s.h5" OUT = sys.argv[1] if len(sys.argv) > 1 else "poverty_audit.json" ONLY = sys.argv[2].split(",") if len(sys.argv) > 2 else None -BENEFITS = ["child_benefit","council_tax_benefit","esa_income","esa_contrib","housing_benefit","income_support","jsa_income","jsa_contrib","pension_credit","universal_credit","working_tax_credit","child_tax_credit","attendance_allowance","afcs","bsp","carers_allowance","dla","iidb","incapacity_benefit","pip","sda","state_pension","maternity_allowance","statutory_sick_pay","statutory_maternity_pay","ssmg","cost_of_living_support_payment","winter_fuel_allowance","tax_free_childcare","healthy_start_vouchers","scottish_child_payment","carer_support_payment","free_school_meals","free_school_fruit_veg","free_school_milk","free_tv_licence_value"] -MARKET = ["employment_income","self_employment_income","savings_interest_income","dividend_income","miscellaneous_income","property_income","private_pension_income","private_transfer_income","maintenance_income"] -TAXES = ["income_tax","national_insurance","council_tax","domestic_rates","student_loan_repayments","employee_pension_contributions","personal_pension_contributions","maintenance_expenses","external_child_payments"] +BENEFITS = [ + "child_benefit", + "council_tax_benefit", + "esa_income", + "esa_contrib", + "housing_benefit", + "income_support", + "jsa_income", + "jsa_contrib", + "pension_credit", + "universal_credit", + "working_tax_credit", + "child_tax_credit", + "attendance_allowance", + "afcs", + "bsp", + "carers_allowance", + "dla", + "iidb", + "incapacity_benefit", + "pip", + "sda", + "state_pension", + "maternity_allowance", + "statutory_sick_pay", + "statutory_maternity_pay", + "ssmg", + "cost_of_living_support_payment", + "winter_fuel_allowance", + "tax_free_childcare", + "healthy_start_vouchers", + "scottish_child_payment", + "carer_support_payment", + "free_school_meals", + "free_school_fruit_veg", + "free_school_milk", + "free_tv_licence_value", +] +MARKET = [ + "employment_income", + "self_employment_income", + "savings_interest_income", + "dividend_income", + "miscellaneous_income", + "property_income", + "private_pension_income", + "private_transfer_income", + "maintenance_income", +] +TAXES = [ + "income_tax", + "national_insurance", + "council_tax", + "domestic_rates", + "student_loan_repayments", + "employee_pension_contributions", + "personal_pension_contributions", + "maintenance_expenses", + "external_child_payments", +] + def wmedian(x, w): - o = np.argsort(x); c = np.cumsum(w[o]); return float(x[o][np.searchsorted(c, c[-1] / 2)]) -def wshare(mask, w): return float(w[mask].sum() / w.sum()) + o = np.argsort(x) + c = np.cumsum(w[o]) + return float(x[o][np.searchsorted(c, c[-1] / 2)]) + + +def wshare(mask, w): + return float(w[mask].sum() / w.sum()) + + def wq(x, w, qs): - o = np.argsort(x); c = np.cumsum(w[o]) / w[o].sum(); return [float(x[o][np.searchsorted(c, q)]) for q in qs] + o = np.argsort(x) + c = np.cumsum(w[o]) / w[o].sum() + return [float(x[o][np.searchsorted(c, q)]) for q in qs] + def measure(sim, year, hw, p_idx, hh_arrays, per_arrays): - A = hh_arrays; P = per_arrays + A = hh_arrays + P = per_arrays pw = hw[p_idx] - bhc = A["equiv_bhc"][p_idx]; ahc = A["equiv_ahc"][p_idx] - thr_b = float(A["thr_bhc"][0]); thr_a = float(A["thr_ahc"][0]) - med_b_p = wmedian(bhc, pw); med_a_p = wmedian(ahc, pw) - med_b_h = wmedian(A["equiv_bhc"], hw); med_a_h = wmedian(A["equiv_ahc"], hw) - child = P["age"] < 18; pens = P["age"] >= 66; wa = (~child) & (~pens) + bhc = A["equiv_bhc"][p_idx] + ahc = A["equiv_ahc"][p_idx] + thr_b = float(A["thr_bhc"][0]) + thr_a = float(A["thr_ahc"][0]) + med_b_p = wmedian(bhc, pw) + med_a_p = wmedian(ahc, pw) + med_b_h = wmedian(A["equiv_bhc"], hw) + med_a_h = wmedian(A["equiv_ahc"], hw) + child = P["age"] < 18 + pens = P["age"] >= 66 + wa = (~child) & (~pens) + def block(inc, thr_abs, med_p, med_h): - rel_p = inc < 0.6 * med_p; rel_h = inc < 0.6 * med_h; abs_ = inc < thr_abs + rel_p = inc < 0.6 * med_p + rel_h = inc < 0.6 * med_h + abs_ = inc < thr_abs d = { - "abs_rate_pct": 100*wshare(abs_, pw), "rel_rate_person_median_pct": 100*wshare(rel_p, pw), - "rel_rate_household_median_pct": 100*wshare(rel_h, pw), - "median_person_weighted_week": med_p/52, "median_household_weighted_week": med_h/52, - "abs_threshold_week": thr_abs/52, "rel_line_person_week": 0.6*med_p/52, - "abs_threshold_over_person_median": thr_abs/med_p, - "share_below_pct_of_person_median": {str(q): 100*wshare(inc < q/100*med_p, pw) for q in (40,50,60,70,80)}, - "share_below_abs_line_scaled": {str(q): 100*wshare(inc < q/100*thr_abs, pw) for q in (90,95,100,105,110)}, - "by_group_abs_pct": {"children": 100*wshare(abs_ & child, pw)/wshare(child,pw), "working_age": 100*wshare(abs_ & wa, pw)/wshare(wa,pw), "pensioners": 100*wshare(abs_ & pens, pw)/wshare(pens,pw)}, - "by_group_rel_pct": {"children": 100*wshare(rel_p & child, pw)/wshare(child,pw), "working_age": 100*wshare(rel_p & wa, pw)/wshare(wa,pw), "pensioners": 100*wshare(rel_p & pens, pw)/wshare(pens,pw)}, - "group_pop_shares_pct": {"children": 100*wshare(child,pw), "working_age": 100*wshare(wa,pw), "pensioners": 100*wshare(pens,pw)}, - "share_nonpositive_income_pct": 100*wshare(inc <= 0, pw), - "person_quantiles_week": dict(zip(["p5","p10","p20","p30","p40","p50"], [v/52 for v in wq(inc, pw, [.05,.1,.2,.3,.4,.5])])), + "abs_rate_pct": 100 * wshare(abs_, pw), + "rel_rate_person_median_pct": 100 * wshare(rel_p, pw), + "rel_rate_household_median_pct": 100 * wshare(rel_h, pw), + "median_person_weighted_week": med_p / 52, + "median_household_weighted_week": med_h / 52, + "abs_threshold_week": thr_abs / 52, + "rel_line_person_week": 0.6 * med_p / 52, + "abs_threshold_over_person_median": thr_abs / med_p, + "share_below_pct_of_person_median": { + str(q): 100 * wshare(inc < q / 100 * med_p, pw) + for q in (40, 50, 60, 70, 80) + }, + "share_below_abs_line_scaled": { + str(q): 100 * wshare(inc < q / 100 * thr_abs, pw) + for q in (90, 95, 100, 105, 110) + }, + "by_group_abs_pct": { + "children": 100 * wshare(abs_ & child, pw) / wshare(child, pw), + "working_age": 100 * wshare(abs_ & wa, pw) / wshare(wa, pw), + "pensioners": 100 * wshare(abs_ & pens, pw) / wshare(pens, pw), + }, + "by_group_rel_pct": { + "children": 100 * wshare(rel_p & child, pw) / wshare(child, pw), + "working_age": 100 * wshare(rel_p & wa, pw) / wshare(wa, pw), + "pensioners": 100 * wshare(rel_p & pens, pw) / wshare(pens, pw), + }, + "group_pop_shares_pct": { + "children": 100 * wshare(child, pw), + "working_age": 100 * wshare(wa, pw), + "pensioners": 100 * wshare(pens, pw), + }, + "share_nonpositive_income_pct": 100 * wshare(inc <= 0, pw), + "person_quantiles_week": dict( + zip( + ["p5", "p10", "p20", "p30", "p40", "p50"], + [v / 52 for v in wq(inc, pw, [0.05, 0.1, 0.2, 0.3, 0.4, 0.5])], + ) + ), } return d - out = {"bhc": block(bhc, thr_b, med_b_p, med_b_h), "ahc": block(ahc, thr_a, med_a_p, med_a_h)} + + out = { + "bhc": block(bhc, thr_b, med_b_p, med_b_h), + "ahc": block(ahc, thr_a, med_a_p, med_a_h), + } # decile composition on person-weighted equivalised BHC deciles (household-level amounts, weekly per household) - edges = wq(bhc, pw, [i/10 for i in range(1,10)]) + edges = wq(bhc, pw, [i / 10 for i in range(1, 10)]) dec = np.searchsorted(edges, bhc, side="right") # 0..9 per person comp = {} for d in range(10): - m = dec == d; w = pw[m]; hh = p_idx[m] - comp[str(d+1)] = {k: float((A[k][hh]*w).sum()/w.sum()/52) for k in ("market","benefits","taxes","hbai_net","uc","state_pension","hb","pension_credit","child_benefit","equiv_bhc")} - comp[str(d+1)]["persons_m"] = float(w.sum()/1e6) - comp[str(d+1)]["equiv_factor_mean"] = float((A["equiv_factor"][hh]*w).sum()/w.sum()) + m = dec == d + w = pw[m] + hh = p_idx[m] + comp[str(d + 1)] = { + k: float((A[k][hh] * w).sum() / w.sum() / 52) + for k in ( + "market", + "benefits", + "taxes", + "hbai_net", + "uc", + "state_pension", + "hb", + "pension_credit", + "child_benefit", + "equiv_bhc", + ) + } + comp[str(d + 1)]["persons_m"] = float(w.sum() / 1e6) + comp[str(d + 1)]["equiv_factor_mean"] = float( + (A["equiv_factor"][hh] * w).sum() / w.sum() + ) out["decile_composition_person_weighted_weekly_per_household"] = comp - out["totals_bn"] = {k: float((A[k]*hw).sum()/1e9) for k in ("market","benefits","taxes","hbai_net","uc","state_pension","hb","pension_credit","child_benefit")} - out["population_m"] = float(pw.sum()/1e6); out["households_m"] = float(hw.sum()/1e6) - out["engine_flags_pct"] = {"in_poverty_bhc": 100*wshare(A["flag_abs_bhc"][p_idx] > 0, pw), "in_relative_poverty_bhc": 100*wshare(A["flag_rel_bhc"][p_idx] > 0, pw), "in_poverty_ahc": 100*wshare(A["flag_abs_ahc"][p_idx] > 0, pw)} + out["totals_bn"] = { + k: float((A[k] * hw).sum() / 1e9) + for k in ( + "market", + "benefits", + "taxes", + "hbai_net", + "uc", + "state_pension", + "hb", + "pension_credit", + "child_benefit", + ) + } + out["population_m"] = float(pw.sum() / 1e6) + out["households_m"] = float(hw.sum() / 1e6) + out["engine_flags_pct"] = { + "in_poverty_bhc": 100 * wshare(A["flag_abs_bhc"][p_idx] > 0, pw), + "in_relative_poverty_bhc": 100 * wshare(A["flag_rel_bhc"][p_idx] > 0, pw), + "in_poverty_ahc": 100 * wshare(A["flag_abs_ahc"][p_idx] > 0, pw), + } return out + def load_arrays(sim, year): - hh = lambda v: np.asarray(sim.calculate(v, year, map_to="household").values, dtype=float) - A = {"equiv_bhc": hh("equiv_hbai_household_net_income"), "equiv_ahc": hh("equiv_hbai_household_net_income_ahc"), - "thr_bhc": hh("poverty_threshold_bhc"), "thr_ahc": hh("poverty_line_ahc")/hh("household_equivalisation_ahc"), "equiv_factor": hh("household_equivalisation_bhc"), - "hbai_net": hh("hbai_household_net_income"), "flag_abs_bhc": hh("in_poverty_bhc"), "flag_rel_bhc": hh("in_relative_poverty_bhc"), "flag_abs_ahc": hh("in_poverty_ahc"), - "hw": hh("household_weight")} - A["market"] = sum(hh(v) for v in MARKET); A["benefits"] = sum(hh(v) for v in BENEFITS); A["taxes"] = sum(hh(v) for v in TAXES) - A["uc"] = hh("universal_credit"); A["state_pension"] = hh("state_pension"); A["hb"] = hh("housing_benefit"); A["pension_credit"] = hh("pension_credit"); A["child_benefit"] = hh("child_benefit") - P = {"age": np.asarray(sim.calculate("age", year).values, dtype=float), "pw": np.asarray(sim.calculate("person_weight", year).values, dtype=float)} - hid = np.asarray(sim.calculate("household_id", year).values); hid_p = np.asarray(sim.calculate("household_id", year, map_to="person").values) + hh = lambda v: np.asarray( + sim.calculate(v, year, map_to="household").values, dtype=float + ) + A = { + "equiv_bhc": hh("equiv_hbai_household_net_income"), + "equiv_ahc": hh("equiv_hbai_household_net_income_ahc"), + "thr_bhc": hh("poverty_threshold_bhc"), + "thr_ahc": hh("poverty_line_ahc") / hh("household_equivalisation_ahc"), + "equiv_factor": hh("household_equivalisation_bhc"), + "hbai_net": hh("hbai_household_net_income"), + "flag_abs_bhc": hh("in_poverty_bhc"), + "flag_rel_bhc": hh("in_relative_poverty_bhc"), + "flag_abs_ahc": hh("in_poverty_ahc"), + "hw": hh("household_weight"), + } + A["market"] = sum(hh(v) for v in MARKET) + A["benefits"] = sum(hh(v) for v in BENEFITS) + A["taxes"] = sum(hh(v) for v in TAXES) + A["uc"] = hh("universal_credit") + A["state_pension"] = hh("state_pension") + A["hb"] = hh("housing_benefit") + A["pension_credit"] = hh("pension_credit") + A["child_benefit"] = hh("child_benefit") + P = { + "age": np.asarray(sim.calculate("age", year).values, dtype=float), + "pw": np.asarray(sim.calculate("person_weight", year).values, dtype=float), + } + hid = np.asarray(sim.calculate("household_id", year).values) + hid_p = np.asarray(sim.calculate("household_id", year, map_to="person").values) p_idx = pd.Index(hid).get_indexer(hid_p) assert (p_idx >= 0).all(), "person->household mapping failed" return A, P, p_idx + res = {"engine": version("policyengine-uk"), "files": {}} for name, (path, years) in FILES.items(): - if ONLY and name not in ONLY: continue - t0 = time.time(); print("==", name, flush=True) + if ONLY and name not in ONLY: + continue + t0 = time.time() + print("==", name, flush=True) sim = Microsimulation(dataset=path) entry = {"h5": path, "years": {}} for y in years: t1 = time.time() A, P, p_idx = load_arrays(sim, y) hw = A["hw"] - assert np.allclose(P["pw"], hw[p_idx]), "person_weight != household weight mapping" + assert np.allclose(P["pw"], hw[p_idx]), ( + "person_weight != household weight mapping" + ) entry["years"][str(y)] = measure(sim, y, hw, p_idx, A, P) - print(" year", y, "done", round(time.time()-t1), "s; abs BHC", round(entry["years"][str(y)]["bhc"]["abs_rate_pct"],2), "rel(person med)", round(entry["years"][str(y)]["bhc"]["rel_rate_person_median_pct"],2), flush=True) + print( + " year", + y, + "done", + round(time.time() - t1), + "s; abs BHC", + round(entry["years"][str(y)]["bhc"]["abs_rate_pct"], 2), + "rel(person med)", + round(entry["years"][str(y)]["bhc"]["rel_rate_person_median_pct"], 2), + flush=True, + ) if name == "v20_national": # decomposition against spine-s design weights (same records, same order — verified) - with pd.HDFStore(SPINE_S, "r") as st: design = st["household"]["household_weight"].to_numpy(float) - with pd.HDFStore(path, "r") as st: cal = st["household"]["household_weight"].to_numpy(float) - ratio = hw.sum()/cal.sum() # engine may scale weights by year; apply same scaling to design + with pd.HDFStore(SPINE_S, "r") as st: + design = st["household"]["household_weight"].to_numpy(float) + with pd.HDFStore(path, "r") as st: + cal = st["household"]["household_weight"].to_numpy(float) + ratio = ( + hw.sum() / cal.sum() + ) # engine may scale weights by year; apply same scaling to design dw = design * ratio dm = measure(sim, y, dw, p_idx, A, P) entry["years"][str(y)]["spine_s_design_weights"] = dm # fixed-line decomposition: design line applied to calibrated weights - pw_c = hw[p_idx]; pw_d = dw[p_idx]; bhc = A["equiv_bhc"][p_idx] - med_d = wmedian(bhc, pw_d); med_c = wmedian(bhc, pw_c) + pw_c = hw[p_idx] + pw_d = dw[p_idx] + bhc = A["equiv_bhc"][p_idx] + med_d = wmedian(bhc, pw_d) + med_c = wmedian(bhc, pw_c) entry["years"][str(y)]["decomposition_bhc_rel"] = { - "design_weights_design_line_pct": 100*wshare(bhc < 0.6*med_d, pw_d), - "calibrated_weights_design_line_pct": 100*wshare(bhc < 0.6*med_d, pw_c), - "calibrated_weights_calibrated_line_pct": 100*wshare(bhc < 0.6*med_c, pw_c), - "design_median_week": med_d/52, "calibrated_median_week": med_c/52, + "design_weights_design_line_pct": 100 * wshare(bhc < 0.6 * med_d, pw_d), + "calibrated_weights_design_line_pct": 100 + * wshare(bhc < 0.6 * med_d, pw_c), + "calibrated_weights_calibrated_line_pct": 100 + * wshare(bhc < 0.6 * med_c, pw_c), + "design_median_week": med_d / 52, + "calibrated_median_week": med_c / 52, "weight_scale_ratio_engine_vs_h5": ratio, } thr = float(A["thr_bhc"][0]) - entry["years"][str(y)]["decomposition_bhc_abs"] = {"design_weights_pct": 100*wshare(bhc < thr, pw_d), "calibrated_weights_pct": 100*wshare(bhc < thr, pw_c)} - print(" decomposition:", json.dumps(entry["years"][str(y)]["decomposition_bhc_rel"]), flush=True) - json.dump(res | {"files": res["files"] | {name: entry}}, open(OUT, "w"), indent=1) + entry["years"][str(y)]["decomposition_bhc_abs"] = { + "design_weights_pct": 100 * wshare(bhc < thr, pw_d), + "calibrated_weights_pct": 100 * wshare(bhc < thr, pw_c), + } + print( + " decomposition:", + json.dumps(entry["years"][str(y)]["decomposition_bhc_rel"]), + flush=True, + ) + json.dump( + res | {"files": res["files"] | {name: entry}}, open(OUT, "w"), indent=1 + ) res["files"][name] = entry json.dump(res, open(OUT, "w"), indent=1) - print(" file done", round(time.time()-t0), "s", flush=True) + print(" file done", round(time.time() - t0), "s", flush=True) print("ALL DONE") diff --git a/docs/evidence/uk-poverty-968/scripts/measure_968_tail_profile.py b/docs/evidence/uk-poverty-968/scripts/measure_968_tail_profile.py index 705cdf159..772acdb1c 100644 --- a/docs/evidence/uk-poverty-968/scripts/measure_968_tail_profile.py +++ b/docs/evidence/uk-poverty-968/scripts/measure_968_tail_profile.py @@ -1,56 +1,170 @@ """microcosm#968: bottom-tail profile and relative-poverty breakdowns (year 2024) for HBAI comparison.""" -import sys, json, warnings -import numpy as np, pandas as pd + +import json +import sys +import warnings + +import numpy as np +import pandas as pd + warnings.filterwarnings("ignore") from policyengine_uk import Microsimulation -ROOT="/Users/mariajuaristi/Desktop/PolicyEngine"; HF="/Users/mariajuaristi/.cache/huggingface/hub" -FILES={"v20_national": f"{ROOT}/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5", - "efrs_1_57_3": f"{HF}/models--policyengine--policyengine-uk-data-private/snapshots/25af520a6651b8812fef56964a42a79a3f9f515a/enhanced_frs_2024_25.h5"} -SPINE_S=f"{ROOT}/data/ukds/acceptance/spine-s-main-rebuild/spine-s.h5" -YEARS=[int(y) for y in (sys.argv[2].split(",") if len(sys.argv)>2 else ["2024"])] -def wmedian(x,w): - o=np.argsort(x); c=np.cumsum(w[o]); return float(x[o][np.searchsorted(c,c[-1]/2)]) -out={} -for name,path in FILES.items(): - sim=Microsimulation(dataset=path) - with pd.HDFStore(path,"r") as st: person=st["person"]; h5hh=st["household"] - hid=h5hh["household_id"].to_numpy(); p_idx=pd.Index(hid).get_indexer(person["person_household_id"].to_numpy()); assert (p_idx>=0).all() - tenure = h5hh["tenure_type"].astype(str).to_numpy() if "tenure_type" in h5hh else np.array(["?"]*len(hid)) + +ROOT = "/Users/mariajuaristi/Desktop/PolicyEngine" +HF = "/Users/mariajuaristi/.cache/huggingface/hub" +FILES = { + "v20_national": f"{ROOT}/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5", + "efrs_1_57_3": f"{HF}/models--policyengine--policyengine-uk-data-private/snapshots/25af520a6651b8812fef56964a42a79a3f9f515a/enhanced_frs_2024_25.h5", +} +SPINE_S = f"{ROOT}/data/ukds/acceptance/spine-s-main-rebuild/spine-s.h5" +YEARS = [int(y) for y in (sys.argv[2].split(",") if len(sys.argv) > 2 else ["2024"])] + + +def wmedian(x, w): + o = np.argsort(x) + c = np.cumsum(w[o]) + return float(x[o][np.searchsorted(c, c[-1] / 2)]) + + +out = {} +for name, path in FILES.items(): + sim = Microsimulation(dataset=path) + with pd.HDFStore(path, "r") as st: + person = st["person"] + h5hh = st["household"] + hid = h5hh["household_id"].to_numpy() + p_idx = pd.Index(hid).get_indexer(person["person_household_id"].to_numpy()) + assert (p_idx >= 0).all() + tenure = ( + h5hh["tenure_type"].astype(str).to_numpy() + if "tenure_type" in h5hh + else np.array(["?"] * len(hid)) + ) for YEAR in YEARS: - hh=lambda v: np.asarray(sim.calculate(v,YEAR,map_to="household").values,float) - pp=lambda v: np.asarray(sim.calculate(v,YEAR).values,float) - hw=hh("household_weight"); assert np.allclose(hw, h5hh["household_weight"].to_numpy(float)) or YEAR!=2024 - age=pp("age"); emp=pp("employment_income")+pp("self_employment_income"); is_child=age<18; pens=age>=66; adult=age>=18 - n_child=np.bincount(p_idx,weights=is_child,minlength=len(hid)); n_adult=np.bincount(p_idx,weights=adult,minlength=len(hid)) - n_pens=np.bincount(p_idx,weights=pens,minlength=len(hid)); n_work=np.bincount(p_idx,weights=(emp>1000),minlength=len(hid)) - n_ft=np.bincount(p_idx,weights=(emp>=12000),minlength=len(hid)) - equiv=hh("equiv_hbai_household_net_income"); thr=float(hh("poverty_threshold_bhc")[0]) - uc=hh("universal_credit"); wc=np.asarray(sim.calculate("would_claim_uc",YEAR,map_to="household").values,float) - ftype=np.full(len(hid),"other_3plus_adults",dtype=object) - ftype[(n_adult==1)&(n_child==0)&(n_pens==0)]="single_wa_no_children"; ftype[(n_adult==2)&(n_child==0)&(n_pens==0)]="couple_wa_no_children" - ftype[(n_adult==1)&(n_child>0)]="lone_parent"; ftype[(n_adult==2)&(n_child>0)]="couple_with_children" - ftype[(n_adult==1)&(n_pens==1)&(n_child==0)]="single_pensioner"; ftype[(n_adult==2)&(n_pens>=1)&(n_child==0)]="pensioner_couple_or_mixed" - weightings={"calibrated":hw} - if name=="v20_national": - with pd.HDFStore(SPINE_S,"r") as st: design=st["household"]["household_weight"].to_numpy(float) - weightings["spine_s_design"]=design*hw.sum()/design.sum() - for wname,w in weightings.items(): - pw=w[p_idx]; eqp=equiv[p_idx]; med=wmedian(eqp,pw); rel=eqp<0.6*med - tot=pw.sum() - r={"median_week":med/52,"rel_bhc_pct":100*pw[rel].sum()/tot,"abs_bhc_pct":100*pw[eqp=a/100*med)&(eqp= 66 + adult = age >= 18 + n_child = np.bincount(p_idx, weights=is_child, minlength=len(hid)) + n_adult = np.bincount(p_idx, weights=adult, minlength=len(hid)) + n_pens = np.bincount(p_idx, weights=pens, minlength=len(hid)) + n_work = np.bincount(p_idx, weights=(emp > 1000), minlength=len(hid)) + n_ft = np.bincount(p_idx, weights=(emp >= 12000), minlength=len(hid)) + equiv = hh("equiv_hbai_household_net_income") + thr = float(hh("poverty_threshold_bhc")[0]) + uc = hh("universal_credit") + wc = np.asarray( + sim.calculate("would_claim_uc", YEAR, map_to="household").values, float + ) + ftype = np.full(len(hid), "other_3plus_adults", dtype=object) + ftype[(n_adult == 1) & (n_child == 0) & (n_pens == 0)] = "single_wa_no_children" + ftype[(n_adult == 2) & (n_child == 0) & (n_pens == 0)] = "couple_wa_no_children" + ftype[(n_adult == 1) & (n_child > 0)] = "lone_parent" + ftype[(n_adult == 2) & (n_child > 0)] = "couple_with_children" + ftype[(n_adult == 1) & (n_pens == 1) & (n_child == 0)] = "single_pensioner" + ftype[(n_adult == 2) & (n_pens >= 1) & (n_child == 0)] = ( + "pensioner_couple_or_mixed" + ) + weightings = {"calibrated": hw} + if name == "v20_national": + with pd.HDFStore(SPINE_S, "r") as st: + design = st["household"]["household_weight"].to_numpy(float) + weightings["spine_s_design"] = design * hw.sum() / design.sum() + for wname, w in weightings.items(): + pw = w[p_idx] + eqp = equiv[p_idx] + med = wmedian(eqp, pw) + rel = eqp < 0.6 * med + tot = pw.sum() + r = { + "median_week": med / 52, + "rel_bhc_pct": 100 * pw[rel].sum() / tot, + "abs_bhc_pct": 100 * pw[eqp < thr].sum() / tot, + } + r["share_below_abs_week_pct"] = { + str(c): 100 * pw[eqp < c * 52].sum() / tot + for c in (10, 50, 100, 150, 200, 260, 325, 370, 390, 455) + } + r["share_in_band_pct_of_median"] = { + f"{a}-{b}": 100 + * pw[(eqp >= a / 100 * med) & (eqp < b / 100 * med)].sum() + / tot + for a, b in ( + (-1000, 0), + (0, 20), + (20, 40), + (40, 50), + (50, 60), + (60, 70), + (70, 80), + ) + } + def grp(mask_hh, label): - m=mask_hh[p_idx]; return {"pop_share_pct":100*pw[m].sum()/tot, "rel_rate_pct":100*pw[m&rel].sum()/max(pw[m].sum(),1e-9), "share_of_poor_pct":100*pw[m&rel].sum()/pw[rel].sum()} - r["by_family_type"]={t:grp(ftype==t,t) for t in sorted(set(ftype))} - r["by_n_children"]={str(k):grp((n_child==k) if k<3 else (n_child>=3),k) for k in (0,1,2,3)} - r["children_rel_rate_by_family_children"]={} - for k in (1,2,3): - m=((n_child==k) if k<3 else (n_child>=3))[p_idx]&is_child; r["children_rel_rate_by_family_children"][str(k)]={"rate":100*pw[m&rel].sum()/max(pw[m].sum(),1e-9),"children_m":pw[m].sum()/1e6} - r["by_work"]={"workless_hh":grp(n_work==0,"w0"),"working_hh":grp(n_work>0,"w1"),"any_ft_earner":grp(n_ft>0,"ft"),"working_no_ft":grp((n_work>0)&(n_ft==0),"pt")} - r["by_tenure"]={t:grp(tenure==t,t) for t in sorted(set(tenure))} - r["children"]={"rel_rate_pct":100*pw[is_child&rel].sum()/pw[is_child].sum(),"in_workless_hh_rate":100*pw[is_child&rel&(n_work==0)[p_idx]].sum()/max(pw[is_child&(n_work==0)[p_idx]].sum(),1e-9),"share_children_in_workless_hh":100*pw[is_child&(n_work==0)[p_idx]].sum()/pw[is_child].sum()} - r["uc"]={"hh_with_uc_m":w[uc>0].sum()/1e6,"persons_in_uc_hh_rel_poor_pct":100*pw[(uc>0)[p_idx]&rel].sum()/max(pw[(uc>0)[p_idx]].sum(),1e-9),"share_of_rel_poor_in_uc_hh_pct":100*pw[(uc>0)[p_idx]&rel].sum()/pw[rel].sum(),"would_claim_false_eligible_hh_m":None} - out[f"{name}|{wname}|{YEAR}"]=r - print(name,wname,YEAR,"rel",round(r["rel_bhc_pct"],2),"child",round(r["children"]["rel_rate_pct"],2),"bands",{k:round(v,2) for k,v in r["share_in_band_pct_of_median"].items()},flush=True) -json.dump(out,open(sys.argv[1],"w"),indent=1); print("DONE") + m = mask_hh[p_idx] + return { + "pop_share_pct": 100 * pw[m].sum() / tot, + "rel_rate_pct": 100 * pw[m & rel].sum() / max(pw[m].sum(), 1e-9), + "share_of_poor_pct": 100 * pw[m & rel].sum() / pw[rel].sum(), + } + + r["by_family_type"] = {t: grp(ftype == t, t) for t in sorted(set(ftype))} + r["by_n_children"] = { + str(k): grp((n_child == k) if k < 3 else (n_child >= 3), k) + for k in (0, 1, 2, 3) + } + r["children_rel_rate_by_family_children"] = {} + for k in (1, 2, 3): + m = ((n_child == k) if k < 3 else (n_child >= 3))[p_idx] & is_child + r["children_rel_rate_by_family_children"][str(k)] = { + "rate": 100 * pw[m & rel].sum() / max(pw[m].sum(), 1e-9), + "children_m": pw[m].sum() / 1e6, + } + r["by_work"] = { + "workless_hh": grp(n_work == 0, "w0"), + "working_hh": grp(n_work > 0, "w1"), + "any_ft_earner": grp(n_ft > 0, "ft"), + "working_no_ft": grp((n_work > 0) & (n_ft == 0), "pt"), + } + r["by_tenure"] = {t: grp(tenure == t, t) for t in sorted(set(tenure))} + r["children"] = { + "rel_rate_pct": 100 * pw[is_child & rel].sum() / pw[is_child].sum(), + "in_workless_hh_rate": 100 + * pw[is_child & rel & (n_work == 0)[p_idx]].sum() + / max(pw[is_child & (n_work == 0)[p_idx]].sum(), 1e-9), + "share_children_in_workless_hh": 100 + * pw[is_child & (n_work == 0)[p_idx]].sum() + / pw[is_child].sum(), + } + r["uc"] = { + "hh_with_uc_m": w[uc > 0].sum() / 1e6, + "persons_in_uc_hh_rel_poor_pct": 100 + * pw[(uc > 0)[p_idx] & rel].sum() + / max(pw[(uc > 0)[p_idx]].sum(), 1e-9), + "share_of_rel_poor_in_uc_hh_pct": 100 + * pw[(uc > 0)[p_idx] & rel].sum() + / pw[rel].sum(), + "would_claim_false_eligible_hh_m": None, + } + out[f"{name}|{wname}|{YEAR}"] = r + print( + name, + wname, + YEAR, + "rel", + round(r["rel_bhc_pct"], 2), + "child", + round(r["children"]["rel_rate_pct"], 2), + "bands", + {k: round(v, 2) for k, v in r["share_in_band_pct_of_median"].items()}, + flush=True, + ) +json.dump(out, open(sys.argv[1], "w"), indent=1) +print("DONE") diff --git a/docs/evidence/uk-poverty-968/scripts/measure_968_takeup_direction.py b/docs/evidence/uk-poverty-968/scripts/measure_968_takeup_direction.py index d979c7189..bd2cfcb05 100644 --- a/docs/evidence/uk-poverty-968/scripts/measure_968_takeup_direction.py +++ b/docs/evidence/uk-poverty-968/scripts/measure_968_takeup_direction.py @@ -1,34 +1,103 @@ """microcosm#968 follow-up (Vahid point 3): which way does a realistic residual allocation move poverty? Residual UC claimants among entitled non-reporters ranked by entitlement size (largest first = DWP's finding that non-take-up concentrates in small entitlements), v20 at 2024, calibrated weights.""" -import sys, json, warnings, numpy as np, pandas as pd + +import json +import sys +import warnings + +import numpy as np +import pandas as pd + warnings.filterwarnings("ignore") from policyengine_uk import Microsimulation -PATH="/Users/mariajuaristi/Desktop/PolicyEngine/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5"; YEAR=2024 -def wmedian(x,w): - o=np.argsort(x); c=np.cumsum(w[o]); return float(x[o][np.searchsorted(c,c[-1]/2)]) -with pd.HDFStore(PATH,"r") as st: person=st["person"]; h5hh=st["household"]; bu=st["benunit"] -hid=h5hh["household_id"].to_numpy(); p_idx=pd.Index(hid).get_indexer(person["person_household_id"].to_numpy()) -bid=bu["benunit_id"].to_numpy(); b_of_p=pd.Index(bid).get_indexer(person["person_benunit_id"].to_numpy()) -base_claim=bu["would_claim_uc"].to_numpy(bool) -anchor=np.zeros(len(bid),bool); anchor[np.unique(b_of_p[pd.to_numeric(person["universal_credit_reported"],errors="coerce").fillna(0).to_numpy()>0])]=True -def run(claim,label): - sim=Microsimulation(dataset=PATH); sim.set_input("would_claim_uc",YEAR,claim.astype(bool)) - hh=lambda v: np.asarray(sim.calculate(v,YEAR,map_to="household").values,float) - hw=hh("household_weight"); pw=hw[p_idx]; eq=hh("equiv_hbai_household_net_income")[p_idx]; thr=float(hh("poverty_threshold_bhc")[0]) - uc_b=np.asarray(sim.calculate("universal_credit",YEAR).values,float); bw=np.asarray(sim.calculate("benunit_weight",YEAR).values,float) - med=wmedian(eq,pw); age=np.asarray(sim.calculate("age",YEAR).values,float); child=age<18 - r={"label":label,"uc_families_m":float(bw[uc_b>0].sum()/1e6),"uc_bn":float((uc_b*bw).sum()/1e9),"rel_bhc_pct":100*float(pw[eq<0.6*med].sum()/pw.sum()), - "abs_bhc_pct":100*float(pw[eq0; pool=entitled&~anchor; n=int((base_claim&pool).sum()) -order=np.argsort(-uc_all); pool_order=[i for i in order if pool[i]] -largest=np.zeros(len(bid),bool); largest[pool_order[:n]]=True -smallest=np.zeros(len(bid),bool); smallest[pool_order[-n:]]=True -res.append(run(anchor|largest|(base_claim&~pool),"residual to the LARGEST entitlements (DWP take-up pattern)")[0]) -res.append(run(anchor|smallest|(base_claim&~pool),"residual to the SMALLEST entitlements (opposite)")[0]) + +PATH = "/Users/mariajuaristi/Desktop/PolicyEngine/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5" +YEAR = 2024 + + +def wmedian(x, w): + o = np.argsort(x) + c = np.cumsum(w[o]) + return float(x[o][np.searchsorted(c, c[-1] / 2)]) + + +with pd.HDFStore(PATH, "r") as st: + person = st["person"] + h5hh = st["household"] + bu = st["benunit"] +hid = h5hh["household_id"].to_numpy() +p_idx = pd.Index(hid).get_indexer(person["person_household_id"].to_numpy()) +bid = bu["benunit_id"].to_numpy() +b_of_p = pd.Index(bid).get_indexer(person["person_benunit_id"].to_numpy()) +base_claim = bu["would_claim_uc"].to_numpy(bool) +anchor = np.zeros(len(bid), bool) +anchor[ + np.unique( + b_of_p[ + pd.to_numeric(person["universal_credit_reported"], errors="coerce") + .fillna(0) + .to_numpy() + > 0 + ] + ) +] = True + + +def run(claim, label): + sim = Microsimulation(dataset=PATH) + sim.set_input("would_claim_uc", YEAR, claim.astype(bool)) + hh = lambda v: np.asarray(sim.calculate(v, YEAR, map_to="household").values, float) + hw = hh("household_weight") + pw = hw[p_idx] + eq = hh("equiv_hbai_household_net_income")[p_idx] + thr = float(hh("poverty_threshold_bhc")[0]) + uc_b = np.asarray(sim.calculate("universal_credit", YEAR).values, float) + bw = np.asarray(sim.calculate("benunit_weight", YEAR).values, float) + med = wmedian(eq, pw) + age = np.asarray(sim.calculate("age", YEAR).values, float) + child = age < 18 + r = { + "label": label, + "uc_families_m": float(bw[uc_b > 0].sum() / 1e6), + "uc_bn": float((uc_b * bw).sum() / 1e9), + "rel_bhc_pct": 100 * float(pw[eq < 0.6 * med].sum() / pw.sum()), + "abs_bhc_pct": 100 * float(pw[eq < thr].sum() / pw.sum()), + "child_rel_pct": 100 + * float(pw[child & (eq < 0.6 * med)].sum() / pw[child].sum()), + "median_week": med / 52, + } + print(json.dumps(r), flush=True) + return r, uc_b + + +res = [] +r0, _ = run(base_claim, "baseline draw (v20 file)") +res.append(r0) +r_all, uc_all = run(np.ones(len(bid), bool), "everyone entitled claims") +res.append(r_all) +entitled = uc_all > 0 +pool = entitled & ~anchor +n = int((base_claim & pool).sum()) +order = np.argsort(-uc_all) +pool_order = [i for i in order if pool[i]] +largest = np.zeros(len(bid), bool) +largest[pool_order[:n]] = True +smallest = np.zeros(len(bid), bool) +smallest[pool_order[-n:]] = True +res.append( + run( + anchor | largest | (base_claim & ~pool), + "residual to the LARGEST entitlements (DWP take-up pattern)", + )[0] +) +res.append( + run( + anchor | smallest | (base_claim & ~pool), + "residual to the SMALLEST entitlements (opposite)", + )[0] +) # mean entitlement of the pool by chosen set -w_b=None -json.dump(res,open(sys.argv[1],"w"),indent=1); print("pool",int(pool.sum()),"n",n,"DONE") +w_b = None +json.dump(res, open(sys.argv[1], "w"), indent=1) +print("pool", int(pool.sum()), "n", n, "DONE") diff --git a/docs/evidence/uk-poverty-968/scripts/measure_968_takeup_probe.py b/docs/evidence/uk-poverty-968/scripts/measure_968_takeup_probe.py index 4ffb0b6bc..ded2b87d6 100644 --- a/docs/evidence/uk-poverty-968/scripts/measure_968_takeup_probe.py +++ b/docs/evidence/uk-poverty-968/scripts/measure_968_takeup_probe.py @@ -1,50 +1,149 @@ """microcosm#968 plan item 4: does WHO claims UC (at a fixed claimant count) move measured poverty? v20 at 2024, calibrated weights.""" -import sys, json, warnings, numpy as np, pandas as pd + +import json +import sys +import warnings + +import numpy as np +import pandas as pd + warnings.filterwarnings("ignore") from policyengine_uk import Microsimulation -PATH="/Users/mariajuaristi/Desktop/PolicyEngine/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5"; YEAR=2024 -def wmedian(x,w): - o=np.argsort(x); c=np.cumsum(w[o]); return float(x[o][np.searchsorted(c,c[-1]/2)]) -with pd.HDFStore(PATH,"r") as st: person=st["person"]; h5hh=st["household"]; bu=st["benunit"] -hid=h5hh["household_id"].to_numpy(); p_idx=pd.Index(hid).get_indexer(person["person_household_id"].to_numpy()) -bid=bu["benunit_id"].to_numpy(); b_of_p=pd.Index(bid).get_indexer(person["person_benunit_id"].to_numpy()) + +PATH = "/Users/mariajuaristi/Desktop/PolicyEngine/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5" +YEAR = 2024 + + +def wmedian(x, w): + o = np.argsort(x) + c = np.cumsum(w[o]) + return float(x[o][np.searchsorted(c, c[-1] / 2)]) + + +with pd.HDFStore(PATH, "r") as st: + person = st["person"] + h5hh = st["household"] + bu = st["benunit"] +hid = h5hh["household_id"].to_numpy() +p_idx = pd.Index(hid).get_indexer(person["person_household_id"].to_numpy()) +bid = bu["benunit_id"].to_numpy() +b_of_p = pd.Index(bid).get_indexer(person["person_benunit_id"].to_numpy()) # household index of each benunit (first member) -first_p=pd.Series(np.arange(len(person))).groupby(b_of_p).first().reindex(np.arange(len(bid))).to_numpy(); b_hh=p_idx[first_p] -base_claim=bu["would_claim_uc"].to_numpy(bool) -anchor=np.zeros(len(bid),bool); anchor[np.unique(b_of_p[pd.to_numeric(person["universal_credit_reported"],errors="coerce").fillna(0).to_numpy()>0])]=True +first_p = ( + pd.Series(np.arange(len(person))) + .groupby(b_of_p) + .first() + .reindex(np.arange(len(bid))) + .to_numpy() +) +b_hh = p_idx[first_p] +base_claim = bu["would_claim_uc"].to_numpy(bool) +anchor = np.zeros(len(bid), bool) +anchor[ + np.unique( + b_of_p[ + pd.to_numeric(person["universal_credit_reported"], errors="coerce") + .fillna(0) + .to_numpy() + > 0 + ] + ) +] = True + + def run(claim, label): - sim=Microsimulation(dataset=PATH) + sim = Microsimulation(dataset=PATH) sim.set_input("would_claim_uc", YEAR, claim.astype(bool)) - hh=lambda v: np.asarray(sim.calculate(v,YEAR,map_to="household").values,float) - hw=hh("household_weight"); pw=hw[p_idx]; eq=hh("equiv_hbai_household_net_income")[p_idx]; thr=float(hh("poverty_threshold_bhc")[0]) - uc_b=np.asarray(sim.calculate("universal_credit",YEAR).values,float); bw=np.asarray(sim.calculate("benunit_weight",YEAR).values,float) - med=wmedian(eq,pw); age=np.asarray(sim.calculate("age",YEAR).values,float); child=age<18 - r={"label":label,"claim_true_units":int(claim.sum()),"uc_families_m":float(bw[uc_b>0].sum()/1e6),"uc_bn":float((uc_b*bw).sum()/1e9), - "rel_bhc_pct":100*float(pw[eq<0.6*med].sum()/pw.sum()),"abs_bhc_pct":100*float(pw[eq 0].sum() / 1e6), + "uc_bn": float((uc_b * bw).sum() / 1e9), + "rel_bhc_pct": 100 * float(pw[eq < 0.6 * med].sum() / pw.sum()), + "abs_bhc_pct": 100 * float(pw[eq < thr].sum() / pw.sum()), + "child_rel_pct": 100 + * float(pw[child & (eq < 0.6 * med)].sum() / pw[child].sum()), + "median_week": med / 52, + } + print(json.dumps(r), flush=True) + return r, uc_b, eq + + +res = [] r0, uc0, eq0 = run(base_claim, "baseline draw (v20 file)") res.append(r0) # entitled set: everyone claims -r_all, uc_all, eq_all = run(np.ones(len(bid),bool), "everyone eligible claims (full take-up)"); res.append(r_all) +r_all, uc_all, eq_all = run( + np.ones(len(bid), bool), "everyone eligible claims (full take-up)" +) +res.append(r_all) entitled = uc_all > 0 -pool = entitled & ~anchor # non-anchored entitled units: the draw chooses among these +pool = entitled & ~anchor # non-anchored entitled units: the draw chooses among these n_base_pool = int((base_claim & pool).sum()) -print("entitled units", int(entitled.sum()), "anchored", int(anchor.sum()), "anchored&entitled", int((anchor&entitled).sum()), "pool", int(pool.sum()), "base draws in pool", n_base_pool, flush=True) +print( + "entitled units", + int(entitled.sum()), + "anchored", + int(anchor.sum()), + "anchored&entitled", + int((anchor & entitled).sum()), + "pool", + int(pool.sum()), + "base draws in pool", + n_base_pool, + flush=True, +) # pre-UC equivalised income of the unit's household under full take-up minus its UC (proxy for need ranking) sim_hh_eq_no_uc = None -rng=np.random.default_rng(12345) -for seed in (1,2,3): - rng=np.random.default_rng(seed); pick=np.zeros(len(bid),bool); idx=np.flatnonzero(pool); pick[rng.choice(idx,size=n_base_pool,replace=False)]=True - res.append(run(anchor|pick|(base_claim&~pool), f"alternative random draw seed {seed} (same pool count)")[0]) +rng = np.random.default_rng(12345) +for seed in (1, 2, 3): + rng = np.random.default_rng(seed) + pick = np.zeros(len(bid), bool) + idx = np.flatnonzero(pool) + pick[rng.choice(idx, size=n_base_pool, replace=False)] = True + res.append( + run( + anchor | pick | (base_claim & ~pool), + f"alternative random draw seed {seed} (same pool count)", + )[0] + ) # need-ordered: rank pool by household equivalised income excluding their own UC (from the full-take-up run) hh_eq_full = eq_all # person-level; take household-level via first member -hh_level_eq_full = np.zeros(len(hid)); hh_level_eq_full[p_idx]=eq_all -with_uc_removed = hh_level_eq_full[b_hh] - uc_all/np.maximum(1e-9,1) # crude: subtract unit UC (not equivalised) — ordering only -order=np.argsort(with_uc_removed); pool_order=[i for i in order if pool[i]] -poorest=np.zeros(len(bid),bool); poorest[pool_order[:n_base_pool]]=True -richest=np.zeros(len(bid),bool); richest[pool_order[-n_base_pool:]]=True -res.append(run(anchor|poorest|(base_claim&~pool), "residual to the POOREST entitled units (lower bound)")[0]) -res.append(run(anchor|richest|(base_claim&~pool), "residual to the RICHEST entitled units (upper bound)")[0]) -res.append(run(anchor|(base_claim&~pool), "anchored reporters only (no residual draw)")[0]) -json.dump(res, open(sys.argv[1],"w"), indent=1); print("DONE") +hh_level_eq_full = np.zeros(len(hid)) +hh_level_eq_full[p_idx] = eq_all +with_uc_removed = hh_level_eq_full[b_hh] - uc_all / np.maximum( + 1e-9, 1 +) # crude: subtract unit UC (not equivalised) — ordering only +order = np.argsort(with_uc_removed) +pool_order = [i for i in order if pool[i]] +poorest = np.zeros(len(bid), bool) +poorest[pool_order[:n_base_pool]] = True +richest = np.zeros(len(bid), bool) +richest[pool_order[-n_base_pool:]] = True +res.append( + run( + anchor | poorest | (base_claim & ~pool), + "residual to the POOREST entitled units (lower bound)", + )[0] +) +res.append( + run( + anchor | richest | (base_claim & ~pool), + "residual to the RICHEST entitled units (upper bound)", + )[0] +) +res.append( + run(anchor | (base_claim & ~pool), "anchored reporters only (no residual draw)")[0] +) +json.dump(res, open(sys.argv[1], "w"), indent=1) +print("DONE") diff --git a/docs/evidence/uk-poverty-968/scripts/scorecard_731.py b/docs/evidence/uk-poverty-968/scripts/scorecard_731.py index e0f616e8d..942cdaa98 100644 --- a/docs/evidence/uk-poverty-968/scripts/scorecard_731.py +++ b/docs/evidence/uk-poverty-968/scripts/scorecard_731.py @@ -71,7 +71,9 @@ def main() -> int: parser.add_argument("--year", type=int, default=2026) parser.add_argument("--out", required=True, type=Path) parser.add_argument( - "--hbai-year", type=int, default=2024, + "--hbai-year", + type=int, + default=2024, help="year at which the HBAI-basis block is measured (HBAI FYE 2024 = April 2023 to March 2024; 0 disables)", ) args = parser.parse_args() @@ -88,7 +90,9 @@ def person(var): return np.asarray(sim.calculate(var, year).values, dtype=float) def to_person(var): - return np.asarray(sim.calculate(var, year, map_to="person").values, dtype=float) + return np.asarray( + sim.calculate(var, year, map_to="person").values, dtype=float + ) pw = np.asarray(sim.calculate("person_weight", year).values, dtype=float) bw = np.asarray(sim.calculate("benunit_weight", year).values, dtype=float) @@ -135,15 +139,21 @@ def quantile(values, weights, q): "universal_credit_bn": float((uc * bw).sum() / 1e9), "uc_families_m": float(bw[uc > 0].sum() / 1e6), "child_benefit_bn": float( - (np.asarray(sim.calculate("child_benefit", year).values, float) * bw).sum() + ( + np.asarray(sim.calculate("child_benefit", year).values, float) * bw + ).sum() / 1e9 ), "pension_credit_bn": float( - (np.asarray(sim.calculate("pension_credit", year).values, float) * bw).sum() + ( + np.asarray(sim.calculate("pension_credit", year).values, float) * bw + ).sum() / 1e9 ), "council_tax_bn": float( - (np.asarray(sim.calculate("council_tax", year).values, float) * hw).sum() + ( + np.asarray(sim.calculate("council_tax", year).values, float) * hw + ).sum() / 1e9 ), # microcosm#968: the engine flag is the absolute line; keep the old key @@ -167,9 +177,15 @@ def quantile(values, weights, q): rows = measure(year) HBAI_KEYS = ( - "poverty_bhc_absolute_pct", "poverty_bhc_relative_pct", "poverty_ahc_absolute_pct", - "poverty_ahc_relative_pct", "child_poverty_bhc_relative_pct", "median_equiv_bhc_income_week", - "p10_over_median_bhc", "p90_over_median_bhc", "zero_income_share_pct", + "poverty_bhc_absolute_pct", + "poverty_bhc_relative_pct", + "poverty_ahc_absolute_pct", + "poverty_ahc_relative_pct", + "child_poverty_bhc_relative_pct", + "median_equiv_bhc_income_week", + "p10_over_median_bhc", + "p90_over_median_bhc", + "zero_income_share_pct", ) hbai_basis = None if args.hbai_year: diff --git a/packages/microcosm-build/tests/test_us_plan.py b/packages/microcosm-build/tests/test_us_plan.py index b2c7c8cff..18dfbdeba 100644 --- a/packages/microcosm-build/tests/test_us_plan.py +++ b/packages/microcosm-build/tests/test_us_plan.py @@ -1028,6 +1028,17 @@ def test_no_incumbent_data_package_references_in_live_tree(self) -> None: "uk_data_target_parity.json", "tools/extract_uk_local_registry_fixture.py", "UK_COVERAGE_PROGRESS.md", + # The microcosm#968 poverty-diagnosis evidence names the retired + # data package only as frozen provenance: the incumbent H5's + # sha-locked HF cache path inside recorded receipts and the + # measurement scripts, and the evaluation venv the legs ran + # under. Nothing is imported or executed from it. + "docs/evidence/uk-poverty-968/receipts.json", + "docs/evidence/uk-poverty-968/outputs/poverty_audit.json", + "docs/evidence/uk-poverty-968/outputs/scorecard_efrs_1_57_3.json", + "docs/evidence/uk-poverty-968/scripts/measure_968_basis_swap.py", + "docs/evidence/uk-poverty-968/scripts/measure_968_poverty_audit.py", + "docs/evidence/uk-poverty-968/scripts/measure_968_tail_profile.py", } checked_suffixes = {".py", ".toml", ".md", ".json"} offenders: list[tuple[str, str]] = [] diff --git a/pyproject.toml b/pyproject.toml index 976f2aa31..d73826774 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,10 @@ ignore = ["E501"] # defaults before numpy/torch load, which requires statements above the # import block. "tools/build_us_fiscal_refresh_release.py" = ["E402"] +# Evidence scripts are frozen receipts of one-off measurements: formatted and +# import-sorted, but their logic stays exactly as it produced the committed +# outputs, so style rules that would edit that logic are waived. +"docs/evidence/*/scripts/*.py" = ["B023", "B905", "E402", "E731", "N806"] [tool.ruff.lint.isort] # microcosm is a PEP 420 namespace package (no top-level __init__.py), so From 09b564ff9a4bd267d041c8109f5e527b5832e12c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 21 Sep 2026 16:05:22 +0100 Subject: [PATCH 06/22] Score the UK national candidate on the surface both artifacts can materialize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rule-1 scorer refused a frozen register it could not materialize on both arms, and 119 of the v20 register's 638 targets bind admin-basis inputs the enhanced FRS never carried (the UC family measures, the CGT asset type, the ONS household type), so no candidate built on the current surface could be scored, and therefore certified, in this repository; the v20 head-to-head ran on a pruning scorer outside the tree (María's direction, 2026-09-21). The score core moves to microcosm.build.uk_runtime.candidate_score, shared by the command, the rowwise driver's national role and the certifier. A target whose measure the incumbent cannot materialize is pruned from both arms and reported (a stderr warning naming every absent measure and the rows it removed; incumbent_unresolvable_pruned in the receipt); a measure the candidate cannot materialize still refuses; band edges come from the full register (#803). The receipt's evaluation block decides rule 1 (#578) on that surface with a verdict, and the receipt is the score block itself (the certifier reads artifacts.candidate.sha256 at the top level, #823 B0). The certifier refuses a receipt whose verdict is not passed, whose surface does not close over its pruned rows, or which carries no evaluation, and summarises the verdict into the certification. --no-prune-incumbent- unresolvable restores the refusal. Co-Authored-By: Claude Fable 5.1 --- ...didate-score-common-surface-967.changed.md | 1 + .../uk-score-receipt-shape-823.fixed.md | 1 + docs/uk-national-calibration-runbook-623.md | 19 + ...k-national-release-assembly-runbook-806.md | 16 +- .../build/uk_runtime/candidate_score.py | 611 ++++++++++++++++++ .../build/uk_runtime/release_certification.py | 61 ++ .../tests/test_score_uk_national_candidate.py | 66 +- .../tests/test_uk_candidate_score.py | 325 ++++++++++ .../tests/test_uk_release_certification.py | 54 ++ .../tests/uk_certification_fixtures.py | 49 +- tools/score_uk_national_candidate.py | 344 ++-------- 11 files changed, 1239 insertions(+), 308 deletions(-) create mode 100644 changelog.d/uk-candidate-score-common-surface-967.changed.md create mode 100644 changelog.d/uk-score-receipt-shape-823.fixed.md create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py create mode 100644 packages/microcosm-build/tests/test_uk_candidate_score.py diff --git a/changelog.d/uk-candidate-score-common-surface-967.changed.md b/changelog.d/uk-candidate-score-common-surface-967.changed.md new file mode 100644 index 000000000..3adc177af --- /dev/null +++ b/changelog.d/uk-candidate-score-common-surface-967.changed.md @@ -0,0 +1 @@ +The UK national scorer (`tools/score_uk_national_candidate.py`, core now in `microcosm.build.uk_runtime.candidate_score`) scores the candidate and the incumbent on the surface both can materialize: a target whose measure the incumbent lacks is pruned from both arms and reported (stderr warning naming every absent measure; `incumbent_unresolvable_pruned` in the receipt) instead of refusing the score, band edges come from the full register, and the receipt's `evaluation` block decides rule 1 (#578) with a `verdict` the release-cut certifier requires to be `passed`. `--no-prune-incumbent-unresolvable` restores the refusal.\n \ No newline at end of file diff --git a/changelog.d/uk-score-receipt-shape-823.fixed.md b/changelog.d/uk-score-receipt-shape-823.fixed.md new file mode 100644 index 000000000..3e19e007b --- /dev/null +++ b/changelog.d/uk-score-receipt-shape-823.fixed.md @@ -0,0 +1 @@ +Write the UK rule-1 score receipt as the score block itself instead of wrapping it under a `score_vs_enhanced_frs` key, so `tools/certify_uk_release_cut.py` finds `artifacts.candidate.sha256` where it reads it and a CLI-produced receipt can be certified.\n \ No newline at end of file diff --git a/docs/uk-national-calibration-runbook-623.md b/docs/uk-national-calibration-runbook-623.md index afdc2fc1b..52be5e7dc 100644 --- a/docs/uk-national-calibration-runbook-623.md +++ b/docs/uk-national-calibration-runbook-623.md @@ -86,6 +86,25 @@ uv run --no-sync python tools/score_uk_national_candidate.py \ Both artifacts are verified against the supplied digests before they are read, and both sides are scored on the same frozen register. +A target whose measure the incumbent cannot materialize (the admin-basis UC +family measures, the CGT asset type, the ONS household type: inputs the +enhanced FRS never carried) is pruned from **both** arms and reported, never +refused: the scorer warns on stderr naming every absent measure and the rows +it removed, and the receipt carries them under +`incumbent_unresolvable_pruned` (`n_pruned`, `n_scored`, `n_surface`, the +rows with their family and measure). The score stands on the common surface +with band edges from the full register (#803). A measure the *candidate* +cannot materialize still refuses: that is a defect. +`--no-prune-incumbent-unresolvable` restores the refusal on the incumbent +side too. The receipt's `evaluation` block decides rule 1 (#578) on that +surface: `verdict` is `passed` when the candidate's full loss is below the +incumbent's, `failed` otherwise, and the release-cut certifier refuses any +receipt whose verdict is not `passed` or whose surface does not close, so +publication never runs on an unpassed evaluation. The rowwise driver's +national role runs this evaluation at the end of every build it is given an +incumbent for (microcosm#965) and writes the same receipt as +`score_vs_incumbent.json`. + ## Evidence directory `data/ukds/acceptance/623-first-calibrated-candidate/` should contain: diff --git a/docs/uk-national-release-assembly-runbook-806.md b/docs/uk-national-release-assembly-runbook-806.md index 0ce8fda7a..3c3ead7fc 100644 --- a/docs/uk-national-release-assembly-runbook-806.md +++ b/docs/uk-national-release-assembly-runbook-806.md @@ -68,10 +68,14 @@ per-cut tag from that suffix. ## 2. Score and certify the cut -First create the rule-1 score receipt against the pinned incumbent, following -the scoring section of -`docs/uk-national-calibration-runbook-623.md`. Then run the release-cut battery -and compose the signed certification: +The rule-1 score receipt is the build's own when the national role was given +`--incumbent-h5` (`score_vs_incumbent.json` beside the candidate); otherwise +create it against the pinned incumbent following the scoring section of +`docs/uk-national-calibration-runbook-623.md`. Its `evaluation.verdict` must +be `passed`: the certifier refuses a receipt whose verdict is anything else, +whose scored surface does not close over its pruned rows, or which carries no +evaluation block. Then run the release-cut battery and compose the signed +certification: ```bash uv run --no-sync python tools/certify_uk_release_cut.py \ @@ -86,7 +90,7 @@ uv run --no-sync python tools/certify_uk_release_cut.py \ --ledger-facts-sha256 \ --ledger-manifest-sha256 \ --input-mass-reference \ - --score-receipt /score_vs_enhanced_frs.json \ + --score-receipt /score_vs_incumbent.json \ --release-id microcosm-uk-2024-25-national ``` @@ -110,7 +114,7 @@ uv run --no-sync python tools/assemble_uk_release_dir.py \ --diagnostics-json /calibration_diagnostics.json \ --seam-gate-report /microcosm_uk_2024_25.terminal_gates.json \ --release-cut-gate-json /microcosm_uk_2024_25.release_cut_gates.json \ - --score-receipt /score_vs_enhanced_frs.json \ + --score-receipt /score_vs_incumbent.json \ --out-dir releases ``` diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py new file mode 100644 index 000000000..1b20ec4b6 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py @@ -0,0 +1,611 @@ +"""Score a UK national candidate against the incumbent on one frozen register. + +The #578 rule-1 score core, shared by the scorer command +(``tools/score_uk_national_candidate.py``), the rowwise driver's national +role (which evaluates every build at its end) and the release-cut certifier +(which refuses a receipt whose verdict is not ``passed``). + +Both artifacts are materialized through the route the calibration seam +uses. A target whose measure the *candidate* cannot materialize refuses: +that is a defect. A target whose measure the *incumbent* cannot materialize +is structural — the incumbent artifact never carried the input (the +admin-basis UC family measures, the CGT asset type, the ONS household type +on the enhanced FRS) — so :func:`evaluate_uk_candidate_against_incumbent` +prunes it from both arms, scores the common surface with band edges from +the full register (#803), and reports every pruned measure loudly in the +receipt rather than quietly shrinking the surface or refusing the score. +""" + +from __future__ import annotations + +import hashlib +import re +from collections.abc import Callable, Mapping +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +from microcosm.build.target_materialization import MeasureResolutionError +from microcosm.build.uk_runtime.national_calibration import prepare_uk_target_frame +from microcosm.build.uk_runtime.national_frame import load_uk_national_frame +from microcosm.calibrate import TargetRegistry, score_targets + +UK_SCORE_LOSS_CAP = 10.0 +UK_SCORE_HOLDOUT_BASIS = "none_declared" + + +def _holdout_loss(basis: str) -> float | None: + """The holdout loss for a declared basis, or None when none exists. + + June's frozen fixture carries a genuinely different holdout value + (0.1239 against a 0.0159 train loss) because it held rows out. This + register declares no split, so the holdout keys report absence rather + than the fitted loss wearing a holdout name — a copied value reads + downstream as perfect generalization from a measurement never made. + Declaring a basis without computing its split fails loudly here rather + than falling back to the fitted loss. + """ + + if basis == "none_declared": + return None + raise NotImplementedError( + f"holdout basis {basis!r} declares a split this scorer does not compute." + ) + + +def score_uk_national_candidate( + *, + candidate_h5: str | Path, + incumbent_h5: str | Path, + candidate_sha256: str, + incumbent_sha256: str, + target_registry: TargetRegistry, + calibration_year: int, + measure_resolver_factory: Callable[[Path, Any], Any] | None = None, + candidate_label: str | None = None, + incumbent_label: str = "enhanced_frs_2024_25", + band_edge_registry: TargetRegistry | None = None, +) -> dict[str, Any]: + """Return the #578 rule-1 score block on a shared target registry. + + Both artifacts are verified against their declared digests before a byte + is read, and both sides are materialized through the same route the + calibration seam uses, so the score names the artifacts it actually + measured rather than two labels supplied on the command line. + """ + + if target_registry.country != "uk" or not target_registry.specs: + raise ValueError("UK candidate scoring requires a non-empty UK registry.") + candidate_path = Path(candidate_h5) + if candidate_label is None: + candidate_label = candidate_path.stem + candidate_pin = _verify_artifact("candidate", candidate_h5, candidate_sha256) + incumbent_pin = _verify_artifact("incumbent", incumbent_h5, incumbent_sha256) + candidate_frame, candidate_resolution = _scored_frame( + candidate_h5, + target_registry, + calibration_year, + measure_resolver_factory, + band_edge_registry=band_edge_registry, + ) + incumbent_frame, incumbent_resolution = _scored_frame( + incumbent_h5, + target_registry, + calibration_year, + measure_resolver_factory, + band_edge_registry=band_edge_registry, + ) + candidate = score_targets( + candidate_frame, + target_registry.to_target_set(), + target_loss_cap=UK_SCORE_LOSS_CAP, + ) + incumbent = score_targets( + incumbent_frame, + target_registry.to_target_set(), + target_loss_cap=UK_SCORE_LOSS_CAP, + ) + candidate_errors = _relative_errors(candidate) + incumbent_errors = _relative_errors(incumbent) + names = sorted(candidate_errors) + missing = sorted(set(names) ^ set(incumbent_errors)) + if missing: + raise RuntimeError( + "candidate and incumbent scores produced different target rows: " + f"{missing[:10]}." + ) + wins = _target_wins(candidate_errors, incumbent_errors) + return { + "candidate_train_loss": float(candidate.final_loss), + "candidate_holdout_loss": _holdout_loss(UK_SCORE_HOLDOUT_BASIS), + "candidate_full_loss": float(candidate.final_loss), + "incumbent_train_loss": float(incumbent.final_loss), + "incumbent_holdout_loss": _holdout_loss(UK_SCORE_HOLDOUT_BASIS), + "incumbent_full_loss": float(incumbent.final_loss), + "candidate_target_wins": wins["candidate"], + "incumbent_target_wins": wins["incumbent"], + "holdout_basis": UK_SCORE_HOLDOUT_BASIS, + "loss": { + "objective": "relative_error_loss", + "target_loss_cap": UK_SCORE_LOSS_CAP, + "train_equals_full": True, + }, + "register": { + "country": target_registry.country, + "version": target_registry.version, + "n_specs": len(target_registry), + }, + "artifacts": { + "candidate": {"label": candidate_label, **candidate_pin}, + "incumbent": {"label": incumbent_label, **incumbent_pin}, + }, + "measure_resolution": { + "candidate": candidate_resolution, + "incumbent": incumbent_resolution, + }, + "target_drift": _target_drift( + target_registry, candidate_errors, incumbent_errors + ), + "signed_asymmetries": [ + { + "id": "incumbent_own_registry", + "description": ( + "The incumbent was calibrated to its own registry; both " + "artifacts are rescored here on the supplied frozen register." + ), + }, + { + "id": "national_direct_vs_collapsed_local", + "description": ( + "Candidate national weights are a direct national solve; " + "incumbent national weights are the published national " + "surface being compared on the same frozen register." + ), + }, + ], + "target_wins_by_family": _target_wins_by_family( + target_registry, + candidate_errors, + incumbent_errors, + ), + } + + +def _relative_errors(result) -> dict[str, float]: + return { + diagnostic.name: float(diagnostic.relative_error) + for diagnostic in result.diagnostics + } + + +def _target_wins( + candidate_errors: dict[str, float], + incumbent_errors: dict[str, float], +) -> dict[str, int]: + candidate = 0 + incumbent = 0 + for name, candidate_error in candidate_errors.items(): + incumbent_error = incumbent_errors[name] + if abs(candidate_error) < abs(incumbent_error): + candidate += 1 + elif abs(incumbent_error) < abs(candidate_error): + incumbent += 1 + return {"candidate": candidate, "incumbent": incumbent} + + +def _target_wins_by_family( + registry: TargetRegistry, + candidate_errors: dict[str, float], + incumbent_errors: dict[str, float], +) -> dict[str, dict[str, int]]: + by_row_name = {spec.to_target().row_name: spec.family for spec in registry.specs} + result: dict[str, dict[str, int]] = {} + for name, family in by_row_name.items(): + bucket = result.setdefault( + family, + {"candidate_target_wins": 0, "incumbent_target_wins": 0, "ties": 0}, + ) + candidate_error = abs(candidate_errors[name]) + incumbent_error = abs(incumbent_errors[name]) + if candidate_error < incumbent_error: + bucket["candidate_target_wins"] += 1 + elif incumbent_error < candidate_error: + bucket["incumbent_target_wins"] += 1 + else: + bucket["ties"] += 1 + return result + + +def _load_registry(path: str | Path) -> TargetRegistry: + """Load the frozen register through its own validating loader. + + ``TargetRegistry.from_json`` checks the format revision and re-derives the + content hash, so a hand-edited or drifted register refuses here instead of + silently deciding rule 1 on a surface nobody reviewed. + """ + + return TargetRegistry.from_json(path) + + +def _sha256_file(path: str | Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _verify_artifact(role: str, path: str | Path, expected: str) -> dict[str, object]: + measured = _sha256_file(path) + if measured != expected: + raise ValueError( + f"{role} artifact sha mismatch: measured {measured}, pinned {expected}" + ) + return { + "path": str(path), + "sha256": measured, + "size_bytes": Path(path).stat().st_size, + } + + +def _scored_frame( + h5_path: str | Path, + registry: TargetRegistry, + calibration_year: int, + factory: Callable[[Path, Any], Any] | None, + *, + band_edge_registry: TargetRegistry | None = None, +) -> tuple[Any, Any]: + frame, _provenance = load_uk_national_frame(h5_path) + resolver = None if factory is None else factory(Path(h5_path), frame) + return prepare_uk_target_frame( + frame, + registry, + period=calibration_year, + measure_resolver=resolver, + band_edge_registry=band_edge_registry, + ) + + +def _default_measure_resolver_factory(scratch_dir: Path, year: int): + def build(h5_path: Path, frame: Any): + from microcosm.build.uk_runtime.measure_simulation import UKMeasureResolver + + return UKMeasureResolver( + simulation_source=h5_path, + scratch_dir=scratch_dir, + year=year, + frame=frame, + ) + + return build + + +def _target_drift( + registry: TargetRegistry, + candidate_errors: dict[str, float], + incumbent_errors: dict[str, float], +) -> list[dict[str, object]]: + """Per-target relative errors, the auditable half of the score block.""" + + families = {spec.to_target().row_name: spec.family for spec in registry.specs} + rows = [] + for name in sorted(candidate_errors): + candidate = candidate_errors[name] + incumbent = incumbent_errors[name] + # Relative errors are signed; the win is decided on magnitude, the + # same rule the aggregate counts use. + rows.append( + { + "target": name, + "family": families.get(name), + "candidate_relative_error": candidate, + "incumbent_relative_error": incumbent, + "winner": ( + "candidate" + if abs(candidate) < abs(incumbent) + else "incumbent" + if abs(incumbent) < abs(candidate) + else "tie" + ), + } + ) + return rows + + +#: The rule the verdict decides: microcosm#578 rule 1, on the surface both +#: artifacts can materialize. +UK_EVALUATION_RULE = ( + "microcosm#578 rule 1: the candidate beats the incumbent on the frozen " + "comparison register, both artifacts rescored on the surface both can " + "materialize; rows the incumbent cannot materialize are pruned from both " + "arms and listed, never a failure." +) +UK_EVALUATION_SCHEMA_VERSION = 1 +UK_EVALUATION_VERDICT_PASSED = "passed" +UK_EVALUATION_VERDICT_FAILED = "failed" +INCUMBENT_UNRESOLVABLE_NOTE = ( + "Rows pruned from BOTH arms because their measures cannot be computed on " + "the incumbent (inputs the incumbent artifact does not carry). The " + "candidate's fit on these rows is measured by the calibration diagnostics " + "and the terminal gates instead." +) + +_FAILED_MEASURE = re.compile( + r"provider (?:failed computing|does not know) ([a-z_0-9]+)\.([A-Za-z0-9_]+)" +) +_UNMATERIALIZABLE = re.compile( + r"([a-z_0-9]+)\.([A-Za-z0-9_]+) remained unmaterializable" +) + + +@dataclass(frozen=True) +class PrunedTarget: + """One target dropped from both arms because the incumbent lacks its measure.""" + + name: str + family: str + unresolvable_measure: str + reason: str + + +def _failing_measure(error: MeasureResolutionError) -> tuple[str, str] | None: + message = str(error) + for pattern in (_FAILED_MEASURE, _UNMATERIALIZABLE): + match = pattern.search(message) + if match is not None: + return match.group(1), match.group(2) + return None + + +def _binding_of(spec: Any, contract_targets: Mapping[str, Any]) -> Mapping[str, Any]: + target = contract_targets.get(str(spec.metadata.get("contract_target_id"))) + if not isinstance(target, Mapping): + return {} + bindings = target.get("bindings") + policyengine = ( + bindings.get("policyengine") if isinstance(bindings, Mapping) else None + ) + return policyengine if isinstance(policyengine, Mapping) else {} + + +def _packaged_contract_targets() -> Mapping[str, Any]: + from microcosm.build.uk_runtime.measure_simulation import _uk_contract_targets + + return _uk_contract_targets() + + +def prune_incumbent_unresolvable( + incumbent_h5: str | Path, + registry: TargetRegistry, + calibration_year: int, + measure_resolver_factory: Callable[[Path, Any], Any] | None, + *, + band_edge_registry: TargetRegistry | None = None, + contract_targets: Mapping[str, Any] | None = None, + max_rounds: int = 10, +) -> tuple[TargetRegistry, dict[str, PrunedTarget]]: + """Drop every target whose measure the incumbent cannot materialize. + + Probes the incumbent's resolution on the scoring surface; on each + :class:`MeasureResolutionError` naming a measure, drops the targets the + resolution receipt's skip rows name for it, else every target whose + contract binding gates or values through it, and records each drop with + the failing measure. A failure that names no target refuses rather than + pruning blindly. Raw-column scoring (no resolver) has nothing to probe. + """ + + if measure_resolver_factory is None: + return registry, {} + pruned: dict[str, PrunedTarget] = {} + surface = registry + # The probes stand on the full register's band edges too (#803): a + # pruned surface never redraws its own edges, not even to be probed. + edges = registry if band_edge_registry is None else band_edge_registry + for _round in range(max_rounds): + try: + _scored_frame( + incumbent_h5, + surface, + calibration_year, + measure_resolver_factory, + band_edge_registry=edges, + ) + except MeasureResolutionError as error: + failing = _failing_measure(error) + if failing is None: + raise + entity, variable = failing + receipt = getattr(error, "receipt", None) or {} + skips = receipt.get("skips", []) if isinstance(receipt, Mapping) else [] + skip_names = { + str(skip.get("name")) + for skip in skips + if isinstance(skip, Mapping) + and ( + variable == str(skip.get("measure", "")) + or variable in str(skip.get("reason", "")) + ) + } + dropped = [spec for spec in surface.specs if spec.name in skip_names] + if not dropped: + contract = ( + _packaged_contract_targets() + if contract_targets is None + else contract_targets + ) + dropped = [ + spec + for spec in surface.specs + if variable + in { + str(_binding_of(spec, contract).get("gated_variable", "")), + str(_binding_of(spec, contract).get("value_variable", "")), + } + ] + if not dropped: + raise MeasureResolutionError( + f"{entity}.{variable} is unresolvable on the incumbent but " + "neither the resolution receipt nor any contract binding " + "names a target for it; refusing to prune blindly.", + receipt=receipt if isinstance(receipt, Mapping) else {}, + ) from error + reason = str(error).splitlines()[0][:300] + for spec in dropped: + pruned[spec.name] = PrunedTarget( + name=spec.name, + family=spec.family, + unresolvable_measure=f"{entity}.{variable}", + reason=reason, + ) + surface = TargetRegistry( + [spec for spec in surface.specs if spec.name not in pruned], + country=surface.country, + ) + if not surface.specs: + raise RuntimeError( + "every target is unresolvable on the incumbent; nothing " + "remains to score." + ) from error + continue + return surface, pruned + raise RuntimeError( + f"the incumbent's measure resolution still fails after {max_rounds} " + "pruning rounds." + ) + + +def pruned_block( + pruned: Mapping[str, PrunedTarget], *, n_scored: int, n_surface: int +) -> dict[str, Any]: + families: dict[str, int] = {} + for row in pruned.values(): + families[row.family] = families.get(row.family, 0) + 1 + return { + "n_pruned": len(pruned), + "n_scored": int(n_scored), + "n_surface": int(n_surface), + "rows": {name: asdict(row) for name, row in sorted(pruned.items())}, + "measures": sorted({row.unresolvable_measure for row in pruned.values()}), + "families": dict(sorted(families.items())), + "note": INCUMBENT_UNRESOLVABLE_NOTE, + } + + +def evaluation_block(score: Mapping[str, Any]) -> dict[str, Any]: + """The machine-checkable verdict the certifier reads from a receipt.""" + + candidate_loss = float(score["candidate_full_loss"]) + incumbent_loss = float(score["incumbent_full_loss"]) + pruned = score.get("incumbent_unresolvable_pruned") or {} + n_scored = int(pruned.get("n_scored", score["register"]["n_specs"])) + n_pruned = int(pruned.get("n_pruned", 0)) + n_surface = int(pruned.get("n_surface", n_scored + n_pruned)) + rule_1_passed = candidate_loss < incumbent_loss + return { + "schema_version": UK_EVALUATION_SCHEMA_VERSION, + "rule": UK_EVALUATION_RULE, + "scored_surface": { + "n_scored": n_scored, + "n_pruned": n_pruned, + "n_surface": n_surface, + }, + "rule_1": { + "passed": rule_1_passed, + "candidate_full_loss": candidate_loss, + "incumbent_full_loss": incumbent_loss, + "candidate_target_wins": int(score["candidate_target_wins"]), + "incumbent_target_wins": int(score["incumbent_target_wins"]), + }, + "verdict": ( + UK_EVALUATION_VERDICT_PASSED + if rule_1_passed + else UK_EVALUATION_VERDICT_FAILED + ), + } + + +def pruned_warning(score: Mapping[str, Any]) -> str | None: + """The loud warning for a receipt that pruned rows, or None.""" + + pruned = score.get("incumbent_unresolvable_pruned") or {} + if not pruned.get("n_pruned"): + return None + lines = [ + f"warning: {pruned['n_pruned']} of {pruned['n_surface']} targets cannot be " + "materialized on the incumbent and were pruned from BOTH arms; the " + f"score stands on {pruned['n_scored']} common targets:" + ] + rows = pruned.get("rows", {}) + for measure in pruned.get("measures", []): + names = sorted( + name + for name, row in rows.items() + if row.get("unresolvable_measure") == measure + ) + lines.append( + f" - {measure}: {len(names)} target(s), e.g. {', '.join(names[:3])}" + ) + lines.append( + " the candidate's fit on these rows is measured by the calibration " + "diagnostics and the terminal gates, not by this score." + ) + return "\n".join(lines) + + +def evaluate_uk_candidate_against_incumbent( + *, + candidate_h5: str | Path, + incumbent_h5: str | Path, + candidate_sha256: str, + incumbent_sha256: str, + target_registry: TargetRegistry, + calibration_year: int, + measure_resolver_factory: Callable[[Path, Any], Any] | None = None, + candidate_label: str | None = None, + incumbent_label: str = "enhanced_frs_2024_25", + band_edge_registry: TargetRegistry | None = None, + prune_incumbent_unresolvable_measures: bool = True, + contract_targets: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Score on the common resolvable surface and decide the rule-1 verdict. + + Returns the score block with ``incumbent_unresolvable_pruned`` (every + row dropped, its family and the absent measure) and ``evaluation`` + (rule, counts, rule-1 losses and wins, ``verdict``). With pruning off the + strict scorer's refusal stands. + """ + + surface = target_registry + pruned: dict[str, PrunedTarget] = {} + if prune_incumbent_unresolvable_measures: + surface, pruned = prune_incumbent_unresolvable( + incumbent_h5, + target_registry, + calibration_year, + measure_resolver_factory, + band_edge_registry=band_edge_registry, + contract_targets=contract_targets, + ) + # A pruned surface must never redraw its own band edges (#803): the + # edges come from the full register when the caller supplied none. + edges = band_edge_registry + if pruned and edges is None: + edges = target_registry + score = score_uk_national_candidate( + candidate_h5=candidate_h5, + incumbent_h5=incumbent_h5, + candidate_sha256=candidate_sha256, + incumbent_sha256=incumbent_sha256, + target_registry=surface, + calibration_year=calibration_year, + measure_resolver_factory=measure_resolver_factory, + candidate_label=candidate_label, + incumbent_label=incumbent_label, + band_edge_registry=edges, + ) + score["incumbent_unresolvable_pruned"] = pruned_block( + pruned, n_scored=len(surface.specs), n_surface=len(target_registry.specs) + ) + score["evaluation"] = evaluation_block(score) + return score diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py index 278ecccea..e581ae8b7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py @@ -449,6 +449,7 @@ def compose_uk_release_certification( "score_receipt": { "filename": score_receipt_path.name, "sha256": hashlib.sha256(score_receipt_bytes).hexdigest(), + "evaluation": _score_receipt_summary(score_receipt), }, "exclusions_evaluated_on": exclusions_evaluated_on.isoformat(), "shippable": True, @@ -728,6 +729,10 @@ def _verify_identity_join( ) +#: The verdict a certifiable score receipt must carry (candidate_score). +UK_SCORE_RECEIPT_VERDICT_PASSED = "passed" + + def _verify_score_receipt(receipt: Mapping[str, Any], *, candidate_sha256: str) -> None: artifacts = receipt.get("artifacts") scored = ( @@ -742,6 +747,62 @@ def _verify_score_receipt(receipt: Mapping[str, Any], *, candidate_sha256: str) f"({candidate_sha256!r}); the rule-1 score must be measured on " "this candidate's bytes." ) + # Rule 1 (microcosm#578) is decided by the scorer on the surface both + # artifacts can materialize and signed into the receipt; the certifier + # refuses anything short of a passed verdict, so publication never runs + # on an unpassed evaluation. Pruned rows are reported, never a failure, + # but the surface must close over them. + evaluation = receipt.get("evaluation") + if not isinstance(evaluation, Mapping): + raise UKReleaseCertificationError( + "the score receipt carries no evaluation block; re-score the " + "candidate with tools/score_uk_national_candidate.py so the " + "rule-1 verdict is signed run evidence." + ) + surface = evaluation.get("scored_surface") + counts = ( + {key: surface.get(key) for key in ("n_scored", "n_pruned", "n_surface")} + if isinstance(surface, Mapping) + else {} + ) + if ( + len(counts) != 3 + or not all(isinstance(value, int) and value >= 0 for value in counts.values()) + or counts["n_scored"] + counts["n_pruned"] != counts["n_surface"] + or counts["n_scored"] == 0 + ): + raise UKReleaseCertificationError( + "the score receipt's scored surface does not close: n_scored + " + "n_pruned must equal n_surface with at least one scored target." + ) + verdict = evaluation.get("verdict") + if verdict != UK_SCORE_RECEIPT_VERDICT_PASSED: + raise UKReleaseCertificationError( + f"the score receipt's evaluation verdict is {verdict!r}, not " + f"{UK_SCORE_RECEIPT_VERDICT_PASSED!r}: rule 1 (microcosm#578) is " + "not met on the common surface, so the cut cannot be certified." + ) + + +def _score_receipt_summary(receipt: Mapping[str, Any]) -> dict[str, Any]: + evaluation = receipt["evaluation"] + surface = evaluation["scored_surface"] + rule_1 = evaluation.get("rule_1") + losses = ( + { + "candidate_full_loss": rule_1.get("candidate_full_loss"), + "incumbent_full_loss": rule_1.get("incumbent_full_loss"), + } + if isinstance(rule_1, Mapping) + else {} + ) + return { + "verdict": evaluation["verdict"], + "n_scored": surface["n_scored"], + "n_pruned": surface["n_pruned"], + "n_surface": surface["n_surface"], + **losses, + } def _status_census(payload: Mapping[str, Any]) -> dict[str, int]: diff --git a/packages/microcosm-build/tests/test_score_uk_national_candidate.py b/packages/microcosm-build/tests/test_score_uk_national_candidate.py index 4a71cfa79..79fdfca92 100644 --- a/packages/microcosm-build/tests/test_score_uk_national_candidate.py +++ b/packages/microcosm-build/tests/test_score_uk_national_candidate.py @@ -10,6 +10,10 @@ uk_national_frame, write_uk_national_frame, ) +from microcosm.build.uk_runtime.release_certification import ( + UKReleaseCertificationError, + _verify_score_receipt, +) from microcosm.calibrate import TargetRegistry, TargetSpec from microcosm.frame import WeightKind from tools.score_uk_national_candidate import ( @@ -170,8 +174,22 @@ def test_score_uk_national_candidate_cli_writes_score_block(tmp_path) -> None: == 0 ) - payload = json.loads(output_json.read_text(encoding="utf-8")) - score = payload["score_vs_enhanced_frs"] + score = json.loads(output_json.read_text(encoding="utf-8")) + # The receipt is the score block itself: the release-cut certifier reads + # ``artifacts.candidate.sha256`` and ``evaluation.verdict`` at the top + # level, so a wrapper key would leave every CLI receipt unverifiable. + assert "score_vs_enhanced_frs" not in score + assert score["artifacts"]["candidate"]["sha256"] == _sha256_file(candidate) + # Raw-column scoring has nothing to prune; the surface still closes and + # the verdict is decided: this candidate fits worse than the incumbent. + assert score["incumbent_unresolvable_pruned"]["n_pruned"] == 0 + assert score["evaluation"]["scored_surface"] == { + "n_scored": 2, + "n_pruned": 0, + "n_surface": 2, + } + assert score["evaluation"]["rule_1"]["passed"] is False + assert score["evaluation"]["verdict"] == "failed" assert score["artifacts"]["candidate"]["label"] == "explicit_candidate" assert score["artifacts"]["incumbent"]["label"] == "explicit_incumbent" assert score["holdout_basis"] == "none_declared" @@ -185,6 +203,50 @@ def test_score_uk_national_candidate_cli_writes_score_block(tmp_path) -> None: assert _load_registry(registry_json).version == _registry().version +def test_verify_score_receipt_accepts_the_cli_receipt(tmp_path) -> None: + """The certifier's receipt check binds to the file the CLI writes.""" + + pytest.importorskip("tables") + candidate = tmp_path / "candidate.h5" + incumbent = tmp_path / "incumbent.h5" + registry_json = tmp_path / "registry.json" + output_json = tmp_path / "score_vs_incumbent.json" + # The candidate hits both targets; the incumbent misses both. + _write(candidate, measure_a=[5.0, 5.0], measure_b=[10.0, 10.0]) + _write(incumbent, measure_a=[4.0, 4.0], measure_b=[6.0, 6.0]) + _registry().to_json(registry_json) + candidate_sha = _sha256_file(candidate) + + assert ( + main( + [ + "--candidate-h5", + str(candidate), + "--candidate-sha256", + candidate_sha, + "--incumbent-h5", + str(incumbent), + "--incumbent-sha256", + _sha256_file(incumbent), + "--registry-json", + str(registry_json), + "--output-json", + str(output_json), + "--calibration-year", + "2025", + "--no-measure-resolution", + ] + ) + == 0 + ) + + receipt = json.loads(output_json.read_text(encoding="utf-8")) + assert receipt["evaluation"]["verdict"] == "passed" + _verify_score_receipt(receipt, candidate_sha256=candidate_sha) + with pytest.raises(UKReleaseCertificationError, match="artifacts.candidate.sha256"): + _verify_score_receipt(receipt, candidate_sha256="0" * 64) + + def test_scorer_refuses_artifacts_that_do_not_match_their_pins(tmp_path) -> None: pytest.importorskip("tables") candidate = tmp_path / "candidate.h5" diff --git a/packages/microcosm-build/tests/test_uk_candidate_score.py b/packages/microcosm-build/tests/test_uk_candidate_score.py new file mode 100644 index 000000000..dd85522a0 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_candidate_score.py @@ -0,0 +1,325 @@ +"""The UK candidate score on the common resolvable surface (microcosm#967). + +The pruning loop is exercised against a stand-in for the scoring frame that +fails the incumbent's resolution the way the real resolver does — a +``MeasureResolutionError`` naming the absent measure, with the resolution +receipt's skip rows — while both frames score on raw columns, so the suite +needs no engine. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.target_materialization import MeasureResolutionError +from microcosm.build.uk_runtime import candidate_score +from microcosm.build.uk_runtime.candidate_score import ( + _sha256_file, + evaluate_uk_candidate_against_incumbent, + evaluation_block, + prune_incumbent_unresolvable, + pruned_warning, +) +from microcosm.build.uk_runtime.national_frame import ( + uk_national_frame, + write_uk_national_frame, +) +from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.frame import WeightKind + +MEASURE_B = "household.measure_b" + + +def _registry() -> TargetRegistry: + return TargetRegistry( + [ + TargetSpec( + name="target_a", + entity="household", + measure="measure_a", + value=10.0, + source="synthetic", + family="family_a", + metadata={"contract_target_id": "test.target_a"}, + ), + TargetSpec( + name="target_b", + entity="household", + measure="measure_b", + value=20.0, + source="synthetic", + family="family_b", + metadata={"contract_target_id": "test.target_b"}, + ), + ], + country="uk", + ) + + +def _write(path: Path, *, measure_a, measure_b=None) -> None: + n = 2 + person = pd.DataFrame( + { + "person_id": np.arange(n), + "person_household_id": np.arange(n), + "person_benunit_id": np.arange(n), + } + ) + benunit = pd.DataFrame({"benunit_id": np.arange(n)}) + household = {"household_id": np.arange(n), "measure_a": measure_a} + if measure_b is not None: + household["measure_b"] = measure_b + frame = uk_national_frame( + person=person, + benunit=benunit, + household=pd.DataFrame(household), + time_period="2025", + weight_kind=WeightKind.CALIBRATED, + household_weights=np.ones(n), + ) + write_uk_national_frame(frame, path) + + +def _twins(tmp_path: Path) -> tuple[Path, Path]: + candidate = tmp_path / "candidate.h5" + incumbent = tmp_path / "incumbent.h5" + # The candidate hits target_a exactly and carries measure_b; the + # incumbent misses target_a and never carried measure_b. + _write(candidate, measure_a=[5.0, 5.0], measure_b=[10.0, 10.0]) + _write(incumbent, measure_a=[4.0, 4.0]) + return candidate, incumbent + + +def _resolver_factory(path, frame): + # A resolver is present (so the incumbent is probed) but scoring stays + # on raw columns: the real resolver is stood in for by ``_intercept``. + return None + + +def _intercept(monkeypatch, *, failing_h5: Path, message: str, receipt: dict): + """Fail the scoring frame for ``failing_h5`` while ``target_b`` is on the + surface, the way the resolver fails on a measure the artifact lacks.""" + + original = candidate_score._scored_frame + calls: list[dict] = [] + + def fake(h5_path, registry, calibration_year, factory, *, band_edge_registry=None): + calls.append( + { + "h5": Path(h5_path), + "targets": [spec.name for spec in registry.specs], + "band_edge_registry": band_edge_registry, + } + ) + if Path(h5_path) == failing_h5 and any( + spec.name == "target_b" for spec in registry.specs + ): + raise MeasureResolutionError(message, receipt=receipt) + return original(h5_path, registry, calibration_year, None) + + monkeypatch.setattr(candidate_score, "_scored_frame", fake) + return calls + + +_SKIP_RECEIPT = { + "skips": [ + {"name": "target_b", "measure": "measure_b", "reason": "'household.measure_b'"} + ] +} + + +def test_prunes_the_target_the_incumbent_cannot_materialize_and_scores_the_rest( + monkeypatch, tmp_path +): + pytest.importorskip("tables") + candidate, incumbent = _twins(tmp_path) + full = _registry() + calls = _intercept( + monkeypatch, + failing_h5=incumbent, + message=f"provider does not know {MEASURE_B}", + receipt=_SKIP_RECEIPT, + ) + + score = evaluate_uk_candidate_against_incumbent( + candidate_h5=candidate, + incumbent_h5=incumbent, + candidate_sha256=_sha256_file(candidate), + incumbent_sha256=_sha256_file(incumbent), + target_registry=full, + calibration_year=2025, + measure_resolver_factory=_resolver_factory, + ) + + pruned = score["incumbent_unresolvable_pruned"] + assert pruned["n_pruned"] == 1 + assert pruned["n_scored"] == 1 + assert pruned["n_surface"] == 2 + assert pruned["measures"] == [MEASURE_B] + assert pruned["families"] == {"family_b": 1} + assert pruned["rows"]["target_b"] == { + "name": "target_b", + "family": "family_b", + "unresolvable_measure": MEASURE_B, + "reason": f"provider does not know {MEASURE_B}", + } + # Both arms were scored on the common surface only. + assert score["register"]["n_specs"] == 1 + assert [row["target"] for row in score["target_drift"]] == ["target_a@0"] + # The candidate hit its target and the incumbent missed: rule 1 passes. + assert score["evaluation"]["verdict"] == "passed" + assert score["evaluation"]["rule_1"]["passed"] is True + assert score["evaluation"]["scored_surface"] == { + "n_scored": 1, + "n_pruned": 1, + "n_surface": 2, + } + # A pruned surface never redraws its own band edges (#803): the scoring + # calls carry the full register as the band-edge register. + scoring_calls = [call for call in calls if call["targets"] == ["target_a"]] + assert scoring_calls and all( + call["band_edge_registry"] is full for call in scoring_calls + ) + warning = pruned_warning(score) + assert warning is not None + assert MEASURE_B in warning and "1 target(s)" in warning and "target_b" in warning + assert json.dumps(score) # the receipt is serialisable as written + + +def test_binding_fallback_names_the_target_when_the_receipt_does_not( + monkeypatch, tmp_path +): + pytest.importorskip("tables") + candidate, incumbent = _twins(tmp_path) + _intercept( + monkeypatch, + failing_h5=incumbent, + message=f"provider failed computing {MEASURE_B}: no such column", + receipt={"skips": []}, + ) + contract = { + "test.target_b": {"bindings": {"policyengine": {"value_variable": "measure_b"}}} + } + + surface, pruned = prune_incumbent_unresolvable( + incumbent, _registry(), 2025, _resolver_factory, contract_targets=contract + ) + + assert [spec.name for spec in surface.specs] == ["target_a"] + assert pruned["target_b"].unresolvable_measure == MEASURE_B + + +def test_refuses_to_prune_blindly(monkeypatch, tmp_path): + pytest.importorskip("tables") + candidate, incumbent = _twins(tmp_path) + _intercept( + monkeypatch, + failing_h5=incumbent, + message=f"provider does not know {MEASURE_B}", + receipt={"skips": []}, + ) + + with pytest.raises(MeasureResolutionError, match="refusing to prune blindly"): + prune_incumbent_unresolvable( + incumbent, _registry(), 2025, _resolver_factory, contract_targets={} + ) + + +def test_a_failure_naming_no_measure_is_re_raised(monkeypatch, tmp_path): + pytest.importorskip("tables") + candidate, incumbent = _twins(tmp_path) + _intercept( + monkeypatch, + failing_h5=incumbent, + message="measure materialization made no progress on round 0", + receipt={"skips": []}, + ) + + with pytest.raises(MeasureResolutionError, match="made no progress"): + prune_incumbent_unresolvable( + incumbent, _registry(), 2025, _resolver_factory, contract_targets={} + ) + + +def test_pruning_off_restores_the_refusal(monkeypatch, tmp_path): + pytest.importorskip("tables") + candidate, incumbent = _twins(tmp_path) + _intercept( + monkeypatch, + failing_h5=incumbent, + message=f"provider does not know {MEASURE_B}", + receipt=_SKIP_RECEIPT, + ) + + with pytest.raises(MeasureResolutionError, match="does not know"): + evaluate_uk_candidate_against_incumbent( + candidate_h5=candidate, + incumbent_h5=incumbent, + candidate_sha256=_sha256_file(candidate), + incumbent_sha256=_sha256_file(incumbent), + target_registry=_registry(), + calibration_year=2025, + measure_resolver_factory=_resolver_factory, + prune_incumbent_unresolvable_measures=False, + ) + + +def test_a_candidate_side_gap_still_refuses(monkeypatch, tmp_path): + pytest.importorskip("tables") + candidate = tmp_path / "candidate.h5" + incumbent = tmp_path / "incumbent.h5" + _write(candidate, measure_a=[5.0, 5.0]) + _write(incumbent, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) + _intercept( + monkeypatch, + failing_h5=candidate, + message=f"provider does not know {MEASURE_B}", + receipt=_SKIP_RECEIPT, + ) + + # Only the incumbent is probed for pruning; a measure the candidate + # cannot materialize is a defect and refuses as before. + with pytest.raises(MeasureResolutionError, match="does not know"): + evaluate_uk_candidate_against_incumbent( + candidate_h5=candidate, + incumbent_h5=incumbent, + candidate_sha256=_sha256_file(candidate), + incumbent_sha256=_sha256_file(incumbent), + target_registry=_registry(), + calibration_year=2025, + measure_resolver_factory=_resolver_factory, + ) + + +def test_verdict_fails_when_the_incumbent_fits_better(tmp_path): + pytest.importorskip("tables") + candidate = tmp_path / "candidate.h5" + incumbent = tmp_path / "incumbent.h5" + _write(candidate, measure_a=[5.0, 5.0], measure_b=[5.0, 5.0]) + _write(incumbent, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) + + score = evaluate_uk_candidate_against_incumbent( + candidate_h5=candidate, + incumbent_h5=incumbent, + candidate_sha256=_sha256_file(candidate), + incumbent_sha256=_sha256_file(incumbent), + target_registry=_registry(), + calibration_year=2025, + ) + + assert score["incumbent_unresolvable_pruned"]["n_pruned"] == 0 + assert pruned_warning(score) is None + assert score["evaluation"]["scored_surface"] == { + "n_scored": 2, + "n_pruned": 0, + "n_surface": 2, + } + assert score["evaluation"]["rule_1"]["candidate_full_loss"] == pytest.approx(0.25) + assert score["evaluation"]["rule_1"]["incumbent_full_loss"] == pytest.approx(0.1) + assert score["evaluation"]["verdict"] == "failed" + assert evaluation_block(score) == score["evaluation"] diff --git a/packages/microcosm-build/tests/test_uk_release_certification.py b/packages/microcosm-build/tests/test_uk_release_certification.py index 086987c3c..d6404b17c 100644 --- a/packages/microcosm-build/tests/test_uk_release_certification.py +++ b/packages/microcosm-build/tests/test_uk_release_certification.py @@ -42,6 +42,7 @@ def _load_fixture_module(): _FIXTURES = _load_fixture_module() _TEST_KEY = _FIXTURES.TEST_KEY _green_certification_inputs_fixture = _FIXTURES.green_certification_inputs +green_score_receipt = _FIXTURES.green_score_receipt _signing_key_fixture = _FIXTURES.signing_key _sha = _FIXTURES.sha256 _stub_registry = _FIXTURES.stub_registry @@ -285,6 +286,59 @@ def test_compose_refuses_score_receipt_scored_on_another_artifact( compose_uk_release_certification(**green_certification_inputs) +def test_compose_refuses_a_score_receipt_whose_verdict_is_not_passed( + green_certification_inputs, +): + receipt = green_score_receipt(green_certification_inputs["candidate_sha256"]) + receipt["evaluation"]["verdict"] = "failed" + receipt["evaluation"]["rule_1"]["passed"] = False + green_certification_inputs["score_receipt_path"].write_text( + json.dumps(receipt), encoding="utf-8" + ) + with pytest.raises(UKReleaseCertificationError, match="verdict is 'failed'"): + compose_uk_release_certification(**green_certification_inputs) + + +def test_compose_refuses_a_score_receipt_without_an_evaluation( + green_certification_inputs, +): + # A receipt from the strict scorer carries the cross-pin but no verdict: + # rule 1 was never decided on a closed surface, so it cannot certify. + receipt = green_score_receipt(green_certification_inputs["candidate_sha256"]) + del receipt["evaluation"] + green_certification_inputs["score_receipt_path"].write_text( + json.dumps(receipt), encoding="utf-8" + ) + with pytest.raises(UKReleaseCertificationError, match="no evaluation block"): + compose_uk_release_certification(**green_certification_inputs) + + +def test_compose_refuses_a_score_receipt_whose_surface_does_not_close( + green_certification_inputs, +): + receipt = green_score_receipt(green_certification_inputs["candidate_sha256"]) + receipt["evaluation"]["scored_surface"]["n_pruned"] = 5 + green_certification_inputs["score_receipt_path"].write_text( + json.dumps(receipt), encoding="utf-8" + ) + with pytest.raises(UKReleaseCertificationError, match="does not close"): + compose_uk_release_certification(**green_certification_inputs) + + +def test_certification_summarises_the_score_receipt_verdict( + green_certification_inputs, +): + certification = compose_uk_release_certification(**green_certification_inputs) + assert certification["score_receipt"]["evaluation"] == { + "verdict": "passed", + "n_scored": 305, + "n_pruned": 2, + "n_surface": 307, + "candidate_full_loss": 0.0096, + "incumbent_full_loss": 0.211, + } + + def test_compose_refuses_absent_signing_key(green_certification_inputs, monkeypatch): monkeypatch.delenv(gate_signing_key_env("uk")) with pytest.raises(UKReleaseCertificationError, match="must be set"): diff --git a/packages/microcosm-build/tests/uk_certification_fixtures.py b/packages/microcosm-build/tests/uk_certification_fixtures.py index c2f810369..35efdc67e 100644 --- a/packages/microcosm-build/tests/uk_certification_fixtures.py +++ b/packages/microcosm-build/tests/uk_certification_fixtures.py @@ -106,6 +106,44 @@ def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() +def green_score_receipt(candidate_sha: str) -> dict: + """A score receipt the certifier accepts: measured on the candidate's + bytes, scored on a closed common surface, rule 1 passed.""" + + return { + "artifacts": { + "candidate": {"sha256": candidate_sha, "size_bytes": 15}, + "incumbent": {"sha256": "9" * 64, "size_bytes": 1}, + }, + "candidate_full_loss": 0.0096, + "incumbent_full_loss": 0.211, + "candidate_target_wins": 293, + "incumbent_target_wins": 12, + "incumbent_unresolvable_pruned": { + "n_pruned": 2, + "n_scored": 305, + "n_surface": 307, + "rows": {}, + "measures": ["benunit.uc_calibration_child_count"], + "families": {"dwp_universal_credit": 2}, + "note": "pruned from both arms", + }, + "evaluation": { + "schema_version": 1, + "rule": "microcosm#578 rule 1 on the common surface", + "scored_surface": {"n_scored": 305, "n_pruned": 2, "n_surface": 307}, + "rule_1": { + "passed": True, + "candidate_full_loss": 0.0096, + "incumbent_full_loss": 0.211, + "candidate_target_wins": 293, + "incumbent_target_wins": 12, + }, + "verdict": "passed", + }, + } + + @pytest.fixture(name="green_certification_inputs") def green_certification_inputs(tmp_path: Path): """Three green signed parts plus a closed identity join.""" @@ -177,16 +215,7 @@ def green_certification_inputs(tmp_path: Path): } score_receipt = tmp_path / "score_vs_enhanced_frs.json" score_receipt.write_text( - json.dumps( - { - "artifacts": { - "candidate": {"sha256": candidate_sha, "size_bytes": 15}, - "incumbent": {"sha256": "9" * 64, "size_bytes": 1}, - }, - "candidate_target_wins": 293, - } - ), - encoding="utf-8", + json.dumps(green_score_receipt(candidate_sha)), encoding="utf-8" ) return { "release_id": "uk-757-first-certified-cut", diff --git a/tools/score_uk_national_candidate.py b/tools/score_uk_national_candidate.py index dd7d885b4..7657454d9 100644 --- a/tools/score_uk_national_candidate.py +++ b/tools/score_uk_national_candidate.py @@ -1,299 +1,37 @@ -"""Score a UK national candidate against the incumbent on one frozen register.""" +"""Score a UK national candidate against the incumbent on one frozen register. + +The score core lives in ``microcosm.build.uk_runtime.candidate_score``; this +command loads the frozen register, verifies both artifacts against their +digests, scores on the surface both can materialize (pruning the rows the +incumbent cannot, loudly, unless ``--no-prune-incumbent-unresolvable``) and +writes the receipt the release-cut certifier reads. +""" from __future__ import annotations import argparse -import hashlib import json -from collections.abc import Callable +import sys from pathlib import Path from typing import Any -from microcosm.build.uk_runtime.national_calibration import prepare_uk_target_frame -from microcosm.build.uk_runtime.national_frame import load_uk_national_frame -from microcosm.calibrate import TargetRegistry, score_targets - -UK_SCORE_LOSS_CAP = 10.0 -UK_SCORE_HOLDOUT_BASIS = "none_declared" - - -def _holdout_loss(basis: str) -> float | None: - """The holdout loss for a declared basis, or None when none exists. - - June's frozen fixture carries a genuinely different holdout value - (0.1239 against a 0.0159 train loss) because it held rows out. This - register declares no split, so the holdout keys report absence rather - than the fitted loss wearing a holdout name — a copied value reads - downstream as perfect generalization from a measurement never made. - Declaring a basis without computing its split fails loudly here rather - than falling back to the fitted loss. - """ - - if basis == "none_declared": - return None - raise NotImplementedError( - f"holdout basis {basis!r} declares a split this scorer does not compute." - ) - - -def score_uk_national_candidate( - *, - candidate_h5: str | Path, - incumbent_h5: str | Path, - candidate_sha256: str, - incumbent_sha256: str, - target_registry: TargetRegistry, - calibration_year: int, - measure_resolver_factory: Callable[[Path, Any], Any] | None = None, - candidate_label: str | None = None, - incumbent_label: str = "enhanced_frs_2024_25", - band_edge_registry: TargetRegistry | None = None, -) -> dict[str, Any]: - """Return the #578 rule-1 score block on a shared target registry. - - Both artifacts are verified against their declared digests before a byte - is read, and both sides are materialized through the same route the - calibration seam uses, so the score names the artifacts it actually - measured rather than two labels supplied on the command line. - """ - - if target_registry.country != "uk" or not target_registry.specs: - raise ValueError("UK candidate scoring requires a non-empty UK registry.") - candidate_path = Path(candidate_h5) - if candidate_label is None: - candidate_label = candidate_path.stem - candidate_pin = _verify_artifact("candidate", candidate_h5, candidate_sha256) - incumbent_pin = _verify_artifact("incumbent", incumbent_h5, incumbent_sha256) - candidate_frame, candidate_resolution = _scored_frame( - candidate_h5, - target_registry, - calibration_year, - measure_resolver_factory, - band_edge_registry=band_edge_registry, - ) - incumbent_frame, incumbent_resolution = _scored_frame( - incumbent_h5, - target_registry, - calibration_year, - measure_resolver_factory, - band_edge_registry=band_edge_registry, - ) - candidate = score_targets( - candidate_frame, - target_registry.to_target_set(), - target_loss_cap=UK_SCORE_LOSS_CAP, - ) - incumbent = score_targets( - incumbent_frame, - target_registry.to_target_set(), - target_loss_cap=UK_SCORE_LOSS_CAP, - ) - candidate_errors = _relative_errors(candidate) - incumbent_errors = _relative_errors(incumbent) - names = sorted(candidate_errors) - missing = sorted(set(names) ^ set(incumbent_errors)) - if missing: - raise RuntimeError( - "candidate and incumbent scores produced different target rows: " - f"{missing[:10]}." - ) - wins = _target_wins(candidate_errors, incumbent_errors) - return { - "candidate_train_loss": float(candidate.final_loss), - "candidate_holdout_loss": _holdout_loss(UK_SCORE_HOLDOUT_BASIS), - "candidate_full_loss": float(candidate.final_loss), - "incumbent_train_loss": float(incumbent.final_loss), - "incumbent_holdout_loss": _holdout_loss(UK_SCORE_HOLDOUT_BASIS), - "incumbent_full_loss": float(incumbent.final_loss), - "candidate_target_wins": wins["candidate"], - "incumbent_target_wins": wins["incumbent"], - "holdout_basis": UK_SCORE_HOLDOUT_BASIS, - "loss": { - "objective": "relative_error_loss", - "target_loss_cap": UK_SCORE_LOSS_CAP, - "train_equals_full": True, - }, - "register": { - "country": target_registry.country, - "version": target_registry.version, - "n_specs": len(target_registry), - }, - "artifacts": { - "candidate": {"label": candidate_label, **candidate_pin}, - "incumbent": {"label": incumbent_label, **incumbent_pin}, - }, - "measure_resolution": { - "candidate": candidate_resolution, - "incumbent": incumbent_resolution, - }, - "target_drift": _target_drift( - target_registry, candidate_errors, incumbent_errors - ), - "signed_asymmetries": [ - { - "id": "incumbent_own_registry", - "description": ( - "The incumbent was calibrated to its own registry; both " - "artifacts are rescored here on the supplied frozen register." - ), - }, - { - "id": "national_direct_vs_collapsed_local", - "description": ( - "Candidate national weights are a direct national solve; " - "incumbent national weights are the published national " - "surface being compared on the same frozen register." - ), - }, - ], - "target_wins_by_family": _target_wins_by_family( - target_registry, - candidate_errors, - incumbent_errors, - ), - } - - -def _relative_errors(result) -> dict[str, float]: - return { - diagnostic.name: float(diagnostic.relative_error) - for diagnostic in result.diagnostics - } - - -def _target_wins( - candidate_errors: dict[str, float], - incumbent_errors: dict[str, float], -) -> dict[str, int]: - candidate = 0 - incumbent = 0 - for name, candidate_error in candidate_errors.items(): - incumbent_error = incumbent_errors[name] - if abs(candidate_error) < abs(incumbent_error): - candidate += 1 - elif abs(incumbent_error) < abs(candidate_error): - incumbent += 1 - return {"candidate": candidate, "incumbent": incumbent} - - -def _target_wins_by_family( - registry: TargetRegistry, - candidate_errors: dict[str, float], - incumbent_errors: dict[str, float], -) -> dict[str, dict[str, int]]: - by_row_name = {spec.to_target().row_name: spec.family for spec in registry.specs} - result: dict[str, dict[str, int]] = {} - for name, family in by_row_name.items(): - bucket = result.setdefault( - family, - {"candidate_target_wins": 0, "incumbent_target_wins": 0, "ties": 0}, - ) - candidate_error = abs(candidate_errors[name]) - incumbent_error = abs(incumbent_errors[name]) - if candidate_error < incumbent_error: - bucket["candidate_target_wins"] += 1 - elif incumbent_error < candidate_error: - bucket["incumbent_target_wins"] += 1 - else: - bucket["ties"] += 1 - return result - - -def _load_registry(path: str | Path) -> TargetRegistry: - """Load the frozen register through its own validating loader. - - ``TargetRegistry.from_json`` checks the format revision and re-derives the - content hash, so a hand-edited or drifted register refuses here instead of - silently deciding rule 1 on a surface nobody reviewed. - """ - - return TargetRegistry.from_json(path) - - -def _sha256_file(path: str | Path) -> str: - digest = hashlib.sha256() - with Path(path).open("rb") as stream: - for chunk in iter(lambda: stream.read(1 << 20), b""): - digest.update(chunk) - return digest.hexdigest() - - -def _verify_artifact(role: str, path: str | Path, expected: str) -> dict[str, object]: - measured = _sha256_file(path) - if measured != expected: - raise ValueError( - f"{role} artifact sha mismatch: measured {measured}, pinned {expected}" - ) - return { - "path": str(path), - "sha256": measured, - "size_bytes": Path(path).stat().st_size, - } - - -def _scored_frame( - h5_path: str | Path, - registry: TargetRegistry, - calibration_year: int, - factory: Callable[[Path, Any], Any] | None, - *, - band_edge_registry: TargetRegistry | None = None, -) -> tuple[Any, Any]: - frame, _provenance = load_uk_national_frame(h5_path) - resolver = None if factory is None else factory(Path(h5_path), frame) - return prepare_uk_target_frame( - frame, - registry, - period=calibration_year, - measure_resolver=resolver, - band_edge_registry=band_edge_registry, - ) - - -def _default_measure_resolver_factory(scratch_dir: Path, year: int): - def build(h5_path: Path, frame: Any): - from microcosm.build.uk_runtime.measure_simulation import UKMeasureResolver - - return UKMeasureResolver( - simulation_source=h5_path, - scratch_dir=scratch_dir, - year=year, - frame=frame, - ) - - return build - - -def _target_drift( - registry: TargetRegistry, - candidate_errors: dict[str, float], - incumbent_errors: dict[str, float], -) -> list[dict[str, object]]: - """Per-target relative errors, the auditable half of the score block.""" - - families = {spec.to_target().row_name: spec.family for spec in registry.specs} - rows = [] - for name in sorted(candidate_errors): - candidate = candidate_errors[name] - incumbent = incumbent_errors[name] - # Relative errors are signed; the win is decided on magnitude, the - # same rule the aggregate counts use. - rows.append( - { - "target": name, - "family": families.get(name), - "candidate_relative_error": candidate, - "incumbent_relative_error": incumbent, - "winner": ( - "candidate" - if abs(candidate) < abs(incumbent) - else "incumbent" - if abs(incumbent) < abs(candidate) - else "tie" - ), - } - ) - return rows +from microcosm.build.uk_runtime.candidate_score import ( # noqa: F401 - re-exports + UK_SCORE_HOLDOUT_BASIS, + UK_SCORE_LOSS_CAP, + _default_measure_resolver_factory, + _holdout_loss, + _load_registry, + _relative_errors, + _scored_frame, + _sha256_file, + _target_drift, + _target_wins, + _target_wins_by_family, + _verify_artifact, + evaluate_uk_candidate_against_incumbent, + pruned_warning, + score_uk_national_candidate, +) def _write_json(path: str | Path, payload: dict[str, Any]) -> Path: @@ -329,6 +67,15 @@ def main(argv: list[str] | None = None) -> int: "pruned surface must never redraw its own band edges)." ), ) + parser.add_argument( + "--no-prune-incumbent-unresolvable", + action="store_true", + help=( + "Refuse the score when the incumbent cannot materialize a target " + "instead of pruning that target from both arms and reporting it. " + "The default prunes and warns, naming every absent measure." + ), + ) parser.add_argument( "--no-measure-resolution", action="store_true", @@ -357,7 +104,7 @@ def main(argv: list[str] | None = None) -> int: args.output_json.parent, int(calibration_year) ) ) - score = score_uk_national_candidate( + score = evaluate_uk_candidate_against_incumbent( candidate_h5=args.candidate_h5, incumbent_h5=args.incumbent_h5, candidate_sha256=args.candidate_sha256, @@ -368,8 +115,25 @@ def main(argv: list[str] | None = None) -> int: candidate_label=args.candidate_label, incumbent_label=args.incumbent_label, band_edge_registry=band_edge_registry, + prune_incumbent_unresolvable_measures=not args.no_prune_incumbent_unresolvable, + ) + warning = pruned_warning(score) + if warning is not None: + print(warning, file=sys.stderr, flush=True) + # The receipt is the score block itself: the release-cut certifier reads + # ``artifacts.candidate.sha256`` and ``evaluation.verdict`` at the top + # level, and the assembler copies the bytes into the release directory + # unchanged. + _write_json(args.output_json, score) + evaluation = score["evaluation"] + print( + f"evaluation {evaluation['verdict']}: candidate loss " + f"{evaluation['rule_1']['candidate_full_loss']:.6f} vs incumbent " + f"{evaluation['rule_1']['incumbent_full_loss']:.6f} on " + f"{evaluation['scored_surface']['n_scored']} of " + f"{evaluation['scored_surface']['n_surface']} targets", + flush=True, ) - _write_json(args.output_json, {"score_vs_enhanced_frs": score}) return 0 From a93dd39f6ce37e43def29708444e34d5fae62ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 21 Sep 2026 16:20:48 +0100 Subject: [PATCH 07/22] Key the pruned targets of the score receipt so the staging content policy accepts it The staging telemetry refuses any collection keyed rows; the receipt rides a national run as a reviewed aggregate artifact, so the pruned targets are keyed pruned_targets. Co-Authored-By: Claude Fable 5.1 --- docs/uk-national-calibration-runbook-623.md | 4 ++-- .../src/microcosm/build/uk_runtime/candidate_score.py | 9 ++++++--- .../microcosm-build/tests/test_uk_candidate_score.py | 2 +- .../microcosm-build/tests/uk_certification_fixtures.py | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/uk-national-calibration-runbook-623.md b/docs/uk-national-calibration-runbook-623.md index 52be5e7dc..dfbd9f447 100644 --- a/docs/uk-national-calibration-runbook-623.md +++ b/docs/uk-national-calibration-runbook-623.md @@ -91,8 +91,8 @@ family measures, the CGT asset type, the ONS household type: inputs the enhanced FRS never carried) is pruned from **both** arms and reported, never refused: the scorer warns on stderr naming every absent measure and the rows it removed, and the receipt carries them under -`incumbent_unresolvable_pruned` (`n_pruned`, `n_scored`, `n_surface`, the -rows with their family and measure). The score stands on the common surface +`incumbent_unresolvable_pruned` (`n_pruned`, `n_scored`, `n_surface`, +`pruned_targets` with each one's family and measure, `measures`, `families`). The score stands on the common surface with band edges from the full register (#803). A measure the *candidate* cannot materialize still refuses: that is a defect. `--no-prune-incumbent-unresolvable` restores the refusal on the incumbent diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py index 1b20ec4b6..4aa5e403f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py @@ -485,7 +485,10 @@ def pruned_block( "n_pruned": len(pruned), "n_scored": int(n_scored), "n_surface": int(n_surface), - "rows": {name: asdict(row) for name, row in sorted(pruned.items())}, + # Keyed ``pruned_targets``, never ``rows``: the staging telemetry's + # content policy refuses row-level collections by key name, and this + # receipt rides the run as a reviewed aggregate artifact. + "pruned_targets": {name: asdict(row) for name, row in sorted(pruned.items())}, "measures": sorted({row.unresolvable_measure for row in pruned.values()}), "families": dict(sorted(families.items())), "note": INCUMBENT_UNRESOLVABLE_NOTE, @@ -536,11 +539,11 @@ def pruned_warning(score: Mapping[str, Any]) -> str | None: "materialized on the incumbent and were pruned from BOTH arms; the " f"score stands on {pruned['n_scored']} common targets:" ] - rows = pruned.get("rows", {}) + targets = pruned.get("pruned_targets", {}) for measure in pruned.get("measures", []): names = sorted( name - for name, row in rows.items() + for name, row in targets.items() if row.get("unresolvable_measure") == measure ) lines.append( diff --git a/packages/microcosm-build/tests/test_uk_candidate_score.py b/packages/microcosm-build/tests/test_uk_candidate_score.py index dd85522a0..83621d3c0 100644 --- a/packages/microcosm-build/tests/test_uk_candidate_score.py +++ b/packages/microcosm-build/tests/test_uk_candidate_score.py @@ -162,7 +162,7 @@ def test_prunes_the_target_the_incumbent_cannot_materialize_and_scores_the_rest( assert pruned["n_surface"] == 2 assert pruned["measures"] == [MEASURE_B] assert pruned["families"] == {"family_b": 1} - assert pruned["rows"]["target_b"] == { + assert pruned["pruned_targets"]["target_b"] == { "name": "target_b", "family": "family_b", "unresolvable_measure": MEASURE_B, diff --git a/packages/microcosm-build/tests/uk_certification_fixtures.py b/packages/microcosm-build/tests/uk_certification_fixtures.py index 35efdc67e..8ef215ba5 100644 --- a/packages/microcosm-build/tests/uk_certification_fixtures.py +++ b/packages/microcosm-build/tests/uk_certification_fixtures.py @@ -123,7 +123,7 @@ def green_score_receipt(candidate_sha: str) -> dict: "n_pruned": 2, "n_scored": 305, "n_surface": 307, - "rows": {}, + "pruned_targets": {}, "measures": ["benunit.uc_calibration_child_count"], "families": {"dwp_universal_credit": 2}, "note": "pruned from both arms", From 800eae8c4f9d8930c5b6f628b1c23694a66a2a0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 21 Sep 2026 16:41:05 +0100 Subject: [PATCH 08/22] Certifier pins carry the byte size the Logbook digest requires tools/certify_uk_release_cut.py had never run end to end on a real candidate: its Logbook pin roles carried a digest alone, and role_pins_digest requires exactly sha256 and size_bytes, so every certification refused before a gate ran (found by the first rehearsal certification of a national cut, 2026-09-21). The pins carry the candidate file's and the consumer facts file's sizes, as the builds' do; a stub-battery test pins the digest. Co-Authored-By: Claude Fable 5.1 --- .../uk-certifier-logbook-pins-967.fixed.md | 1 + .../test_uk_certify_release_cut_command.py | 102 ++++++++++++++++++ tools/certify_uk_release_cut.py | 19 +++- 3 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 changelog.d/uk-certifier-logbook-pins-967.fixed.md create mode 100644 packages/microcosm-build/tests/test_uk_certify_release_cut_command.py diff --git a/changelog.d/uk-certifier-logbook-pins-967.fixed.md b/changelog.d/uk-certifier-logbook-pins-967.fixed.md new file mode 100644 index 000000000..f62c5f99b --- /dev/null +++ b/changelog.d/uk-certifier-logbook-pins-967.fixed.md @@ -0,0 +1 @@ +`tools/certify_uk_release_cut.py` no longer refuses every certification up front: its Logbook pin roles now carry the byte size beside the digest, the shape `role_pins_digest` requires, so the release-cut battery actually runs on a real candidate. diff --git a/packages/microcosm-build/tests/test_uk_certify_release_cut_command.py b/packages/microcosm-build/tests/test_uk_certify_release_cut_command.py new file mode 100644 index 000000000..b54384540 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_certify_release_cut_command.py @@ -0,0 +1,102 @@ +"""The release-cut certifier command's Logbook pins (microcosm#823 E1). + +The certifier had never run end to end on a real candidate: its Logbook pin +roles carried a digest without the byte size the pin digest requires, so +every certification refused up front. The command is exercised here with +its battery stubbed, on the pins alone. +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +from microcosm.build.logbook_adoption import role_pins_digest +from tools import certify_uk_release_cut as certify + + +def test_certifier_pins_carry_the_byte_sizes_the_logbook_digest_requires( + monkeypatch, tmp_path: Path, capsys +) -> None: + candidate = tmp_path / "microcosm_uk_2024_25.h5" + candidate.write_bytes(b"candidate bytes") + ledger = tmp_path / "ledger" + ledger.mkdir() + (ledger / "consumer_facts.jsonl").write_text('{"fact": 1}\n' * 3) + candidate_sha = hashlib.sha256(candidate.read_bytes()).hexdigest() + # The parser refuses aliased paths, so every evidence file is its own. + placeholders = {} + for name in ( + "spine.h5", + "calibration_diagnostics.json", + "build_record.json", + "terminal_gates.json", + "input_mass_reference.json", + "score_vs_incumbent.json", + ): + placeholders[name] = tmp_path / name + placeholders[name].write_text("{}") + facts_sha = "1" * 64 + captured: dict[str, object] = {} + + def fake_run(args, state): + captured["state"] = state + return {"shippable": True} + + monkeypatch.setattr(certify, "_run", fake_run) + monkeypatch.setattr(certify, "resolve_predecessor", lambda digest: None) + monkeypatch.setattr(certify, "record_terminal_attempt", lambda **kwargs: None) + + assert ( + certify.main( + [ + "--candidate-h5", + str(candidate), + "--candidate-sha256", + candidate_sha, + "--candidate-name", + "microcosm_uk_2024_25", + "--spine-h5", + str(placeholders["spine.h5"]), + "--diagnostics-json", + str(placeholders["calibration_diagnostics.json"]), + "--build-record-json", + str(placeholders["build_record.json"]), + "--seam-gate-report", + str(placeholders["terminal_gates.json"]), + "--ledger-facts", + str(ledger), + "--ledger-facts-sha256", + facts_sha, + "--ledger-manifest-sha256", + "2" * 64, + "--input-mass-reference", + str(placeholders["input_mass_reference.json"]), + "--score-receipt", + str(placeholders["score_vs_incumbent.json"]), + "--release-id", + "microcosm-uk-2024-25-national", + "--release-cut-gate-json", + str(tmp_path / "release_cut_gates.json"), + "--certification-json", + str(tmp_path / "release_certification.json"), + ] + ) + == 0 + ) + + state = captured["state"] + assert state.input_pins_digest == role_pins_digest( + { + "candidate_h5": { + "sha256": candidate_sha, + "size_bytes": len(b"candidate bytes"), + }, + "ledger_facts": { + "sha256": facts_sha, + "size_bytes": (ledger / "consumer_facts.jsonl").stat().st_size, + }, + } + ) + assert state.build_id.startswith("uk-frs-release-certification-attempt-") + assert '"shippable": true' in capsys.readouterr().out diff --git a/tools/certify_uk_release_cut.py b/tools/certify_uk_release_cut.py index 5d18e892f..7e18d1d2c 100644 --- a/tools/certify_uk_release_cut.py +++ b/tools/certify_uk_release_cut.py @@ -73,15 +73,30 @@ _LOCAL_COMPILE_PARITY_PERIOD = 2025 +def _ledger_facts_size(path: Path) -> int: + """The consumer facts file's size: the artifact directory holds it.""" + + facts = path / "consumer_facts.jsonl" if path.is_dir() else path + return facts.stat().st_size + + def main(argv: list[str] | None = None) -> int: args = _parse_args(argv) started_at = time.perf_counter() started_ts = datetime.now(UTC) code_pin = git_code_pin(_REPOSITORY) predecessor = resolve_predecessor(args.logbook_prev_row_digest) + # Logbook pin roles carry exactly the digest and the byte size, as the + # builds' do; a digest alone refused every real certification up front. source_pins = { - "candidate_h5": {"sha256": args.candidate_sha256}, - "ledger_facts": {"sha256": args.ledger_facts_sha256}, + "candidate_h5": { + "sha256": args.candidate_sha256, + "size_bytes": args.candidate_h5.stat().st_size, + }, + "ledger_facts": { + "sha256": args.ledger_facts_sha256, + "size_bytes": _ledger_facts_size(args.ledger_facts), + }, } state = AttemptState( build_id=f"{_PIPELINE}-attempt-{started_ts.strftime('%Y%m%dT%H%M%SZ')}", From 48e17777bde198ecd509b7c3d6127a1535a5632c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:49:56 +0100 Subject: [PATCH 09/22] Retire the two stale input-mass exclusions the national cut measures inside the fence The first release-cut certification of a national cut (2026-09-21) failed uk_input_mass_parity on two reviewed exclusions that are now inside the +/-452 % fence, and the register's rule fails a stale entry by design. charitable_investment_gifts (microcosm#630, signed for a +15,169 % drift the build no longer shows) and owned_land (microcosm#714, signed for seed-to-seed realization variance, expiring 2026-09-26) both retire; the realization-variance revisit stays registered on microcosm#145. The efrs-post-calibration block is empty, the contract's input-mass evidence digest is re-pinned to the empty register (17545916b692...) with its mirrored test constants, and the bindings' expired-register test injects its own lapsing receipt instead of relying on the register. Co-Authored-By: Claude Fable 5.1 --- ...t-mass-exclusion-retirement-967.changed.md | 1 + .../uk/input_mass_reviewed_exclusions.json | 17 +---------- .../tests/test_uk_battery_bindings.py | 28 ++++++++++++++++++- .../tests/test_uk_weighted_integrity.py | 8 +++--- .../src/microcosm/data/contract.py | 2 +- .../microcosm-data/tests/test_contract.py | 26 ++--------------- 6 files changed, 36 insertions(+), 46 deletions(-) create mode 100644 changelog.d/uk-input-mass-exclusion-retirement-967.changed.md diff --git a/changelog.d/uk-input-mass-exclusion-retirement-967.changed.md b/changelog.d/uk-input-mass-exclusion-retirement-967.changed.md new file mode 100644 index 000000000..ed7a2b0b1 --- /dev/null +++ b/changelog.d/uk-input-mass-exclusion-retirement-967.changed.md @@ -0,0 +1 @@ +Retire the two reviewed input-mass parity exclusions on the `efrs-post-calibration` reference: the first national candidate certified under microcosm#823 measured `charitable_investment_gifts` (signed microcosm#630 for a +15,169 % drift) and `owned_land` (signed microcosm#714 for seed-to-seed realization variance, revisit registered on microcosm#145) inside the ±452 % fence, and the gate fails a stale exclusion by design. The register's `efrs-post-calibration` block is empty and the contract's input-mass evidence digest is re-pinned (`17545916…`).\n \ No newline at end of file diff --git a/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json b/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json index 3a7d346bb..0679de266 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json +++ b/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json @@ -2,21 +2,6 @@ "schema_version": 3, "description": "Reviewed input-mass parity exclusions for the UK terminal battery, scoped per named comparison reference (microcosm#630): an exclusion suppresses only when its reference is the armed one, so a column excluded on a channel-blind reference is compared normally against a channel-aware one with no register change. Each entry is a complete approval receipt (schema-2 fields per #610). The gate reports dormant entries, FAILS stale ones, and FAILS expired ones until the adjudication is renewed, so this register cannot rot.", "references": { - "efrs-post-calibration": { - "charitable_investment_gifts": { - "reason": "SPI-channel-exclusive column on a channel-blind reference: the efrs-post-calibration incumbent structurally lacks the SPI clone channel, so its reference mass is survey-side scraps while the staged candidate's mass is the admin-captured SPI channel functioning as designed (microcosm#630 case 2). Compared normally against any future channel-aware reference.", - "approved_by": "juaristi22", - "adjudication": "microcosm#630", - "approved_on": "2026-08-20", - "expires_on": "2027-02-20" - }, - "owned_land": { - "reason": "Sparse heavy-tailed WAS donor column (0.7 percent weighted nonzero share) whose weighted total is dominated by a handful of large farm/estate records: the spine-e stability receipt (data/ukds/acceptance/757-swap/owned_land_stability_receipt_spine_e.json) measures a 53.8 percent national and 96.7 percent West Midlands swing between adjacent seeds on the 25-stage candidate \u2014 the realization-variance class the archived incumbent data repo records at uk-data#448 (4.6x Wales swing across releases), reproduced from the E5 instrument's method. Register parity at this grain stays not meaningful; the one-month expiry keeps the end-of-workstream revisit registered on microcosm#145 live (winsorised donor or separate land imputation are the candidate remedies).", - "approved_by": "juaristi22", - "adjudication": "microcosm#714", - "approved_on": "2026-08-26", - "expires_on": "2026-09-26" - } - } + "efrs-post-calibration": {} } } diff --git a/packages/microcosm-build/tests/test_uk_battery_bindings.py b/packages/microcosm-build/tests/test_uk_battery_bindings.py index 3e594bb6b..0c79b4d82 100644 --- a/packages/microcosm-build/tests/test_uk_battery_bindings.py +++ b/packages/microcosm-build/tests/test_uk_battery_bindings.py @@ -186,7 +186,15 @@ def _fixture_coverage_registry(): } -def _run_battery(tables, *, parity=None, fit_records=None, armed=True, clock=CLOCK): +def _run_battery( + tables, + *, + parity=None, + fit_records=None, + armed=True, + clock=CLOCK, + extra_artifacts=None, +): person, benunit, household = tables frame = uk_national_frame( person=person, benunit=benunit, household=household, time_period="2023" @@ -207,6 +215,8 @@ def _run_battery(tables, *, parity=None, fit_records=None, armed=True, clock=CLO artifacts["aggregate_admin"] = { "nhs_spending_total": 202_000_000_000, } + if extra_artifacts: + artifacts.update(extra_artifacts) # Small synthetic totals exercise battery behavior without disclosing # the licensed 131-column reference (same patch as the legacy tests); # the binding's declared-pin check compares spec to runtime constant and @@ -739,11 +749,27 @@ def test_every_exclusion_gate_shares_the_injected_clock(self) -> None: assert set(stamps.values()) == {CLOCK.isoformat()}, stamps def test_an_expired_register_fails_closed(self) -> None: + # The committed input-mass register carries no entry since the + # 2026-09-21 retirements, so the expiry discipline is exercised on an + # injected receipt that lapses before the clock below; the other two + # registers still expire from their committed entries. + lapsed = { + "efrs-post-calibration": { + "owned_land": { + "reason": "injected: a reviewed exclusion whose window has closed", + "approved_by": "juaristi22", + "adjudication": "microcosm#714", + "approved_on": "2026-08-26", + "expires_on": "2026-09-26", + } + } + } battery = _run_battery( _tables(), parity=_parity(), fit_records=(FitWeightRecord("spi_qrf", "importance"),), clock=date(2027, 3, 1), + extra_artifacts={"reviewed_input_mass_exclusions": lapsed}, ) failed = {o.entry.id for o in battery.outcomes if o.status is GateStatus.FAILED} assert { diff --git a/packages/microcosm-build/tests/test_uk_weighted_integrity.py b/packages/microcosm-build/tests/test_uk_weighted_integrity.py index 9f3066303..e7eb15ea3 100644 --- a/packages/microcosm-build/tests/test_uk_weighted_integrity.py +++ b/packages/microcosm-build/tests/test_uk_weighted_integrity.py @@ -659,10 +659,10 @@ def test_committed_exclusion_registers_load() -> None: ) assert set(input_mass) == {"efrs-post-calibration"} - assert set(input_mass["efrs-post-calibration"]) == { - "charitable_investment_gifts", - "owned_land", - } + # Both former entries retired 2026-09-21: the first national candidate + # measured charitable_investment_gifts and owned_land inside the fence, + # and the gate fails a stale exclusion by design. + assert input_mass["efrs-post-calibration"] == {} assert qrf_tail == {} assert uk_default_input_mass_reviewed_exclusions() is ( uk_default_input_mass_reviewed_exclusions() diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index f086ea672..01c43c525 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -576,7 +576,7 @@ def compatibility_claim_declarer_error(declared_by: object) -> str | None: # canonical hash; this pins the wrapped digest so the entry's evidence line # still binds the enhanced-FRS incumbent totals. _UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256 = ( - "c9211cbb923e13f4850b834b5bdb1ff1de87fe9237c332b5de63f01ed417aa2d" + "17545916b6926c77e9f8fc90876266cc3f8e4a381079bafc8d1c63fa8df43c04" ) # The degenerate binding's evidence payload digests the resolved exclusion # records; for a release that must be the committed register, so its digest diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index cb73c6d9a..48eb5223f 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -86,29 +86,7 @@ "SPI-channel-exclusive columns are comparable only through per-reference " "reviewed exclusions." ) -UK_INPUT_MASS_REVIEWED_EXCLUSIONS = { - "charitable_investment_gifts": { - "reason": ( - "SPI-channel-exclusive column on a channel-blind reference: the " - "efrs-post-calibration incumbent structurally lacks the SPI clone " - "channel, so its reference mass is survey-side scraps while the " - "staged candidate's mass is the admin-captured SPI channel " - "functioning as designed (microcosm#630 case 2). Compared " - "normally against any future channel-aware reference." - ), - "approved_by": "juaristi22", - "adjudication": "microcosm#630", - "approved_on": "2026-08-20", - "expires_on": "2027-02-20", - }, - "owned_land": { - "reason": "Sparse heavy-tailed WAS donor column (0.7 percent weighted nonzero share) whose weighted total is dominated by a handful of large farm/estate records: the spine-e stability receipt (data/ukds/acceptance/757-swap/owned_land_stability_receipt_spine_e.json) measures a 53.8 percent national and 96.7 percent West Midlands swing between adjacent seeds on the 25-stage candidate \u2014 the realization-variance class the archived incumbent data repo records at uk-data#448 (4.6x Wales swing across releases), reproduced from the E5 instrument's method. Register parity at this grain stays not meaningful; the one-month expiry keeps the end-of-workstream revisit registered on microcosm#145 live (winsorised donor or separate land imputation are the candidate remedies).", - "approved_by": "juaristi22", - "adjudication": "microcosm#714", - "approved_on": "2026-08-26", - "expires_on": "2026-09-26", - }, -} +UK_INPUT_MASS_REVIEWED_EXCLUSIONS: dict[str, dict[str, str]] = {} GIT_COMMIT = "5fa48f07436a806ad75ff76fd22cfb8613bddbe0" DATASET_SHA = "d" * 64 CALIBRATION_SHA = "a" * 64 @@ -150,7 +128,7 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: "6f0243bcda09dad26945376230c44ec3cf55d4e417c3a25e29bae8c59bc1a69d" ) UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256 = ( - "c9211cbb923e13f4850b834b5bdb1ff1de87fe9237c332b5de63f01ed417aa2d" + "17545916b6926c77e9f8fc90876266cc3f8e4a381079bafc8d1c63fa8df43c04" ) #: Spec entry id -> (neutral gate name, phase, legacy detail-schema name). UK_GATE_BATTERY_ENTRIES = { From 821c1348b4054383d1a9269c99b1deb96690b589 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:49:57 +0100 Subject: [PATCH 10/22] Retire the June-path CGT family; a stage that records no receipt requires none hmrc_cgt_gains, the capital-gains wrapper driven from the certified June H5, goes with its cgt_source_stages.json manifest, its package registration and its coverage family (superseded_by the spine stage but still demanded at the release cut). The FRS spine's hmrc_cgt_gains_spine is the only CGT gains stage: UK_CGT_SPINE_STAGE_NAME labels the allocation evidence, the spine's receipt reason is the shared imputation's default, and the manifest tests re-anchor on the spine stage's own operations. The coverage-manifest generator no longer invents a mass-change reason for a source stage that declares none (was_wealth, lcfs_consumption, etb_vat, etb_services, regional_property_uprating: column-writing fits that pass the household weights through untouched). Those families carry weights_pass_through semantics and no required reason; the loader accepts exactly that pair and refuses a pass-through family with a reason or a receipted family without one. The first release cut failed all five on a record nothing ever wrote. Manifest regenerated (145 required columns, 0 reviewed exclusions; --check clean). Co-Authored-By: Claude Fable 5.1 --- .../uk-retire-june-cgt-family-967.changed.md | 1 + .../microcosm/build/uk/cgt_source_stages.json | 190 ------------------ .../microcosm/build/uk/country_package.json | 5 - .../uk/release_input_coverage_manifest.json | 45 +---- .../build/uk_runtime/cgt_imputation.py | 65 ++---- .../uk_runtime/release_input_coverage.py | 43 ++-- .../tests/test_country_spec.py | 2 - .../tests/test_uk_cgt_imputation.py | 40 ++-- .../tests/test_uk_cgt_source_manifest.py | 119 +++++------ .../tests/test_uk_release_input_coverage.py | 56 ++++-- ...uild_uk_release_input_coverage_manifest.py | 182 +++-------------- 11 files changed, 187 insertions(+), 561 deletions(-) create mode 100644 changelog.d/uk-retire-june-cgt-family-967.changed.md delete mode 100644 packages/microcosm-build/src/microcosm/build/uk/cgt_source_stages.json diff --git a/changelog.d/uk-retire-june-cgt-family-967.changed.md b/changelog.d/uk-retire-june-cgt-family-967.changed.md new file mode 100644 index 000000000..60b374b28 --- /dev/null +++ b/changelog.d/uk-retire-june-cgt-family-967.changed.md @@ -0,0 +1 @@ +Retire the June-path UK capital gains stage `hmrc_cgt_gains` (the wrapper driven from the certified June H5, its `cgt_source_stages.json` manifest and its coverage family): the FRS spine's `hmrc_cgt_gains_spine` is the only CGT gains stage, its receipt reason is the default for the shared imputation, and the release contract no longer carries the superseded June family. Source-stage families whose stage declares no mass-change reason (the E5 fits and the regional uprating, which pass the household weights through untouched) require no mass receipt and record `weights_pass_through` semantics; the generator used to invent a reason nothing wrote, failing five families on the first release cut. diff --git a/packages/microcosm-build/src/microcosm/build/uk/cgt_source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/cgt_source_stages.json deleted file mode 100644 index a3749b572..000000000 --- a/packages/microcosm-build/src/microcosm/build/uk/cgt_source_stages.json +++ /dev/null @@ -1,190 +0,0 @@ -{ - "version": 1, - "country": "uk", - "policy": "The UK capital gains family is source-manifest-defined. The vendored joint distribution must be checked against the pinned Chronicle feed identity before it is read, the stage redraws amounts only (incidence and household weights pass through unchanged), policy amounts come from the policyengine-uk parameter tree through the uk extra's deferred import, and the published band facts are fenced from calibration as policy-endogenous and proxy-conditioned.", - "stages": [ - { - "stage": "hmrc_cgt_gains", - "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 3 2024-25 (individual taxpayers and gains by size of gain and taxable income) conditioned on the 2024-25 Tables 2.1a, 5, 6 and 1, all vendored from the pinned Chronicle feed", - "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", - "grain": "person", - "base_candidate": { - "filename": "populace_uk_2023.h5", - "tier": "frs", - "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", - "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", - "size_bytes": 1315880118, - "runtime_sha256_required": true - }, - "artifacts": [ - { - "role": "policy_parameters", - "kind": "versioned_parameter_tree", - "dependency": "policyengine-uk>=2.98.0 via microcosm-build[uk]", - "parameters": [ - "gov.hmrc.income_tax.allowances.personal_allowance.amount", - "gov.hmrc.income_tax.allowances.personal_allowance.maximum_ANI", - "gov.hmrc.income_tax.allowances.personal_allowance.reduction_rate", - "gov.hmrc.cgt.annual_exempt_amount" - ], - "instant_rule": "raw dated parameter files evaluated at 1 June of the build period's tax year", - "runtime_sha256_required": false, - "dependency_discipline": "deferred inside uk_cgt_policy_parameters; the base package never imports policyengine-uk at import time" - }, - { - "role": "cgt_conditioning_facts", - "kind": "public_aggregate_reference", - "resource": "hmrc_cgt_conditioning_facts.json", - "format": "json", - "runtime_sha256_required": true - } - ], - "operations": [ - { - "kind": "verify_certified_candidate", - "artifact": "base_candidate", - "runtime_sha256_required": true, - "fail_on_mismatch": true - }, - { - "kind": "verify_vendored_fact_resource", - "artifact_role": "cgt_conditioning_facts", - "resource": "hmrc_cgt_conditioning_facts.json", - "feed_pin": "chronicle_feed.json", - "record_sets": [ - "hmrc.cgt_table3_2026.ty2024.", - "hmrc.cgt_size_of_gain_2026.table2_1a.ty2024.", - "hmrc.cgt_table6_2026.age.ty2024", - "hmrc.cgt_table5_2026.country_region.ty2024", - "hmrc.cgt_statistics_2026.table1.ty2024" - ], - "source_vintage": "2024-25", - "mapped_build_period": 2024, - "period_mapping": "published_tax_year_equals_build_period", - "require_before_source_read": true, - "runtime_sha256_required": true, - "fail_on_mismatch": true - }, - { - "kind": "taxable_income_proxy", - "components": [ - "employment_income", - "self_employment_income", - "state_pension_reported", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "miscellaneous_income" - ], - "components_semantics": "Persisted leaves of the model's total_income concept (ITA 2007 s.23); state_pension_reported stands in for social_security_income, whose other taxable benefits are not persisted; reliefs such as pension contributions and Gift Aid are not deducted.", - "allowance": "tapered Personal Allowance from the policy_parameters artifact", - "fail_on_missing_component": true - }, - { - "kind": "rake_allocation_targets", - "resource": "hmrc_cgt_conditioning_facts.json", - "joint_margin": "Table 3 2024-25 reconciled cells (2026 release; individuals with a liability by size of gain and taxable income), every income column rescaled onto its published All-row taxpayer total; the same tax year as the Table 1 observations the calibration fits, so no vintage move is applied", - "band_aggregation": "Table 2.1a bands 0, 3,000 and 6,000 fold into Table 3 band 0; 10,000 and 12,300 fold into 10,000; the remaining bands map one to one; the folded rows must agree with Table 3's row totals within publication rounding and the difference is reported per band", - "age_margin": "Table 6 2024-25 individual taxpayers by age band, folded to the age groups; the 0-15 band, which adult carriers cannot hold, is spread across the groups by the normalisation rather than folded into the youngest group", - "gains_margin": "Table 6 2024-25 individual gains by age band, folded to the age groups the same way, as one linear constraint per group, the raked people in the group's cells times their Table 3 cell means summing to the group's gains, so each age group carries its published gains as well as its count instead of the frame's band mix (the amounts-consistent tilt; microcosm#725)", - "region_margin": "Table 5 2024-25 taxpayers by country and region (all taxpayers) times the Table 1 2024-25 individuals/total taxpayer share, folded to the region groups", - "margin_normalization": "every count margin is rescaled onto the joint's total taxpayer mass and the gains margin onto the joint's gains at cell means before raking, so rounding cannot make the margins mutually infeasible", - "seed_joint": "joint cell mass times the frame's household-weighted (age group, region group | income band) shares among gainers", - "age_group_lower_bounds": [ - 16, - 25, - 35, - 45, - 55, - 65, - 75 - ], - "region_groups": { - "LONDON": "london", - "SOUTH_EAST": "south_east_and_east_of_england", - "EAST_OF_ENGLAND": "south_east_and_east_of_england", - "NORTH_EAST": "rest_of_england", - "NORTH_WEST": "rest_of_england", - "YORKSHIRE": "rest_of_england", - "EAST_MIDLANDS": "rest_of_england", - "WEST_MIDLANDS": "rest_of_england", - "SOUTH_WEST": "rest_of_england", - "WALES": "wales_scotland_northern_ireland", - "SCOTLAND": "wales_scotland_northern_ireland", - "NORTHERN_IRELAND": "wales_scotland_northern_ireland" - }, - "gains_tilt": "exponential tilt per age group; the group's cells are multiplied by exp(lambda x cell mean / largest cell mean) with lambda solved by bisection so the group's people-weighted mean gain meets its target while the group's count is held, alternated with the three count margins until every margin holds to the tolerance", - "solver": "microcosm.build.uk_runtime.cgt_imputation._rake_with_gains_tilt (proportional scaling of the count margins and the gains tilt on the four-way array; the generic mean-raking helper scales columns independently and cannot hold a people-weighted gains constraint)", - "rake_rounds": 500, - "rake_tolerance": 1e-09, - "unattainable_policy": "a group whose target mean gain lies outside the means of its seeded cells takes the nearest attainable tilt and is reported", - "zero_seed_policy": "a cell with no frame support keeps a zero target and is reported; its joint mass is met from the income band's pooled fallback where support exists", - "fallback_policy": "pooled_income_band_rank_walk", - "conditioning_dimensions": [ - "income band", - "age group", - "region group" - ], - "reporting_grain": "Table 6 age bands and the twelve region-tier areas" - }, - { - "kind": "rank_preserving_allocation", - "within": "income band x age group x region group cell, then the income band's pooled unassigned gainers", - "ordering": "existing gains descending, person_id ascending on ties", - "band_order": "highest gain band first", - "suppressed_cell_allocation": "count implied by the cell's published gains at the band-total mean", - "column_reconciliation": "every income column rescales onto its published All-row taxpayer total", - "shortfall_policy": "proportional scale-down inside a cell that holds less gainer mass than its raked target; the income band's net shortfall after every cell walk, apportioned to gain bands in proportion to their positive shortfalls, is walked over the income band's pooled unassigned gainers, scaled down proportionally when the pool is short, so cell-walk rounding overshoot is netted rather than compounded", - "minimum_allocation_people": 1, - "weights": "household_weight mapped to persons; no person splits across bands", - "rounding": "systematic with one seeded offset per walk: a person joins the band whose cumulative boundary first covers (1 - offset) of their weight, so a band boundary smaller than a person's weight still receives that person on the share of walks it implies and the rounding is unbiased across the conditioning cells; each cell walk's signed rounding error is carried into the next cell's boundaries for the same gain band (error diffusion), so each gain band's walked total within an income band stays within one person's weight of its raked total instead of accumulating the overshoot of many cell walks", - "cell_order": "income band ascending, age group ascending, region group ascending, gain bands highest first inside a cell; then the pooled walk per income band" - }, - { - "kind": "within_band_draws", - "bounded_band_family": "truncated exponential matched to the cell's published mean", - "open_band_family": "Pareto with alpha = mean / (mean - lower bound)", - "quantile_scheme": "stratified by weight: the persons a (gain band, income band) plan receives across every cell and the pooled walk take the quantile strata that partition (0, 1) in proportion to their household weights, in ascending prior-gain order with one seeded jitter per plan, so the plan's weighted realised mean sits on its published mean rather than carrying n independent draws or the weight-rank mix of equal strata", - "open_band_realization": "each open-band person takes the conditional mean of their weight-proportional quantile stratum, which averages back to the published mean exactly under the household weights; an iid draw from a Pareto with infinite variance sits well below it on a few dozen carriers, and equal strata on unequal weights put the tail's mass on whichever rows happen to be light", - "mean_repair_margin": 0.02, - "mean_repair_reason": "Published counts round to the nearest thousand and amounts to the nearest million; two cells of the 2024-25 table (gains of GBP 1m to 2m at taxable incomes of GBP 37,700 to 49,999 and GBP 100,000 to 125,139) imply a mean below their own band, and repaired means clamp just inside the violated boundary.", - "bottom_band_floor": "annual exempt amount plus one pound, since every allocated person is a taxpayer with a liability", - "seed_base": 552, - "seed_mixing": "seed combined with the build period; each walk draws its rounding offset in cell order then pooled-fallback order, then each plan draws its quantile jitter, income bands ascending and gain bands highest first", - "deterministic": true, - "cell_means": "ratios of the published Table 3 2024-25 cell gains to cell taxpayers, both rounded by the publisher (counts to the nearest thousand, amounts to the nearest million), so derived rather than published numbers; suppressed-count cells take the band-total mean; every mean is repaired into its band" - }, - { - "kind": "sub_aea_remainder", - "policy": "gainers beyond the published taxpayer mass keep their existing amounts capped at the annual exempt amount", - "rationale": "Table 3 covers only individuals with a CGT liability; remaining gainers are treated as sub-AEA gainers rather than invented into the liability distribution or deleted." - }, - { - "kind": "record_mass_conservation_receipt", - "entity": "household", - "reason": "Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.", - "declared_factor": 1, - "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this reason." - }, - { - "kind": "classify_cgt_band_facts_with_reviewed_fence", - "calibration_permitted": false, - "fact_fence_id": "cgt_band_facts_policy_endogenous_proxy_conditioned", - "fenced_fact_count": 76, - "fenced_fact_composition": "60 joint cells, 10 gain-band row totals, 6 income-column totals (Table 3 2024-25, 2026 release)", - "classification_rationale": "The taxpayer count is endogenous to policy (liability requires gains above the annual exempt amount computed by the model under simulation), the income conditioning is an arithmetic proxy that can move a person one band, and the published surface needs rounding and suppression reconciliation before any per-band fact is exact. The SPI 208-fact fence is the precedent.", - "calibrated_facts_unchanged": "The calibrated CGT facts are the FY2024-25 individual Table 1 references (hmrc.cgt.taxpayers_total, hmrc.cgt.gains_total, hmrc.cgt.liability_total; UK_CGT_TARGET_COVERAGE_REQUIREMENTS) and the Table 2.1a size-of-gain, Table 6 age-band and Table 5 region rows declared in uk_population_targets.json (microcosm#467, #725); the 76 Table 3 2024-25 joint and marginal cells stay fenced and condition the imputation only, and Table 3's all-gains, all-incomes pair restates the Table 1 individual observations already bound.", - "promotion_path": "A separately reviewed target profile may lift specific band facts after the reconciliation and proxy adequacy are adjudicated; the Table 2.1a, 6 and 5 rows were promoted under microcosm#467 and #725 on the FY2024-25 individual observations, while the Table 3 joint cells remain fenced because the taxable-income proxy is still an arithmetic approximation.", - "adjudication": "https://github.com/PolicyEngine/microcosm/issues/552" - } - ], - "outputs": [ - "capital_gains" - ], - "notes": "Amounts-only replacement: who has gains comes from the candidate, how much they have comes from the published joint distribution, and household weights pass through unchanged, the weight kind carries through, and the stage appends the mass-conservation receipt the terminal family gate requires. Only persons with positive existing gains are redrawn; the candidate's net losses (negative amounts) and zeros pass through byte-identical, since Table 3 covers only individuals with a liability and says nothing about losses. The stage runs after hmrc_spi_income so the taxable-income proxy sees the restored income surface. Requires the microcosm-build uk extra at build time for the policy_parameters artifact. The allocation targets are raked to the 2024-25 Table 6 age and Table 5 region margins from the vendored conditioning resource and walked inside income x age x region cells (microcosm#725).", - "output_weight_kind": "importance", - "output_weight_kind_rationale": "The stage preserves whatever kind it receives; it runs after hmrc_spi_income, whose prior allocation establishes importance-kind household weights, so the final frame carries importance." - } - ] -} diff --git a/packages/microcosm-build/src/microcosm/build/uk/country_package.json b/packages/microcosm-build/src/microcosm/build/uk/country_package.json index 4255b9f48..8211cc0ae 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/uk/country_package.json @@ -32,11 +32,6 @@ "kind": "vintages", "schema_id": "vintages.schema.json" }, - { - "path": "cgt_source_stages.json", - "kind": "legacy_json", - "schema_id": "legacy_json" - }, { "path": "degenerate_reviewed_exclusions.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 29faa7edd..17d30c643 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -508,7 +508,7 @@ "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, - "mass_change_semantics": "mass_conserving", + "mass_change_semantics": "weights_pass_through", "output_weight_kind": "importance", "outputs": [ "dfe_education_spending", @@ -522,7 +522,6 @@ "nhs_admitted_patient_spending", "nhs_outpatient_spending" ], - "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", "source_manifest_sha256": "dd405553c28d556cf1444301913705931ebe01fa156ac827188286953dc7738e", @@ -537,12 +536,11 @@ "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, - "mass_change_semantics": "mass_conserving", + "mass_change_semantics": "weights_pass_through", "output_weight_kind": "importance", "outputs": [ "full_rate_vat_expenditure_rate" ], - "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", "source_manifest_sha256": "dd405553c28d556cf1444301913705931ebe01fa156ac827188286953dc7738e", @@ -574,36 +572,6 @@ "stage": "hmrc_cgt_asset_type_spine", "status": "required_at_build" }, - "hmrc_cgt_gains": { - "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", - "base_candidate_tier": "frs", - "calibration_permitted": false, - "effective_mass_requirements": {}, - "fact_fence_id": "cgt_band_facts_policy_endogenous_proxy_conditioned", - "fenced_fact_count": 76, - "output_weight_kind": "importance", - "outputs": [ - "capital_gains" - ], - "required_mass_change_reason": "Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.", - "source_manifest": "cgt_source_stages.json", - "source_manifest_sha256": "1b7e6c10728255c3fe2d6d0a33288fae50b282e348e0db9832b7f6073e66d9b7", - "source_vintages": { - "conditioning_resource": "hmrc_cgt_conditioning_facts.json", - "conditioning_resource_sha256": "f5bb0ad93fff3332b7ae8fb580f002d13207b3f95e44a3103c50108279d94797", - "conditioning_surface": "2024-25", - "hmrc_surface": "2024-25", - "mapped_build_period": "2024" - }, - "stage": "hmrc_cgt_gains", - "status": "required_at_build", - "superseded_by": { - "reason": "The FRS spine build executes hmrc_cgt_gains_spine, which applies the same HMRC Table 3 amounts redraw directly in source_stages.json before calibration.", - "source_manifest": "source_stages.json", - "source_manifest_sha256": "dd405553c28d556cf1444301913705931ebe01fa156ac827188286953dc7738e", - "stage": "hmrc_cgt_gains_spine" - } - }, "hmrc_cgt_gains_spine": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", @@ -728,7 +696,7 @@ "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, - "mass_change_semantics": "mass_conserving", + "mass_change_semantics": "weights_pass_through", "output_weight_kind": "importance", "outputs": [ "food_and_non_alcoholic_beverages_consumption", @@ -751,7 +719,6 @@ "gas_consumption", "has_fuel_consumption" ], - "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", "source_manifest_sha256": "dd405553c28d556cf1444301913705931ebe01fa156ac827188286953dc7738e", @@ -766,10 +733,9 @@ "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, - "mass_change_semantics": "mass_conserving", + "mass_change_semantics": "weights_pass_through", "output_weight_kind": "importance", "outputs": [], - "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [ "main_residence_value", "property_wealth" @@ -833,7 +799,7 @@ "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, - "mass_change_semantics": "mass_conserving", + "mass_change_semantics": "weights_pass_through", "output_weight_kind": "importance", "outputs": [ "owned_land", @@ -853,7 +819,6 @@ "mortgage_debt", "consumer_debt" ], - "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", "source_manifest_sha256": "dd405553c28d556cf1444301913705931ebe01fa156ac827188286953dc7738e", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py index bab9d8692..6694c7b92 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py @@ -101,7 +101,6 @@ load_hmrc_cgt_joint_distribution, ) from microcosm.build.uk_runtime.national_frame import ( - UKNationalStage, uk_household_weight_kind, uk_national_frame, uk_time_period, @@ -112,9 +111,8 @@ __all__ = [ "UK_CGT_IMPUTATION_SEED", - "UK_CGT_MASS_CONSERVATION_REASON", "UK_CGT_SPINE_MASS_CONSERVATION_REASON", - "UK_CGT_IMPUTATION_STAGE_NAME", + "UK_CGT_SPINE_STAGE_NAME", "UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS", "UKCGTImputationSummary", "UKCGTPolicyParameters", @@ -129,31 +127,25 @@ "impute_uk_capital_gains", "impute_uk_capital_gains_with_report", "summarize_uk_cgt_imputation", - "uk_capital_gains_imputation_stage", "uk_cgt_spine_stage_transform", "uk_cgt_policy_parameters", "uk_cgt_taxable_income_proxy", ] -UK_CGT_IMPUTATION_STAGE_NAME = "hmrc_cgt_gains" - -#: The reviewed mass-conservation receipt this stage records. The terminal -#: family gate requires a valid mass-conserving MassChangeRecord carrying -#: exactly this reason, so a build whose CGT stage silently moved household -#: mass — or never ran — fails by name. -UK_CGT_MASS_CONSERVATION_REASON = ( - "Amounts-only capital gains redraw: household weights pass through " - "unchanged and total household mass is conserved." -) +#: The FRS spine's capital-gains amounts stage: since microcosm#823 the only +#: CGT gains stage. The June-path wrapper (``hmrc_cgt_gains``, driven from +#: the certified June H5) is retired; its family left the release contract. +UK_CGT_SPINE_STAGE_NAME = "hmrc_cgt_gains_spine" #: Base seed for the stage's draws. Combined with the build period so two #: periods draw differently while each build is reproducible. UK_CGT_IMPUTATION_SEED = 552 -#: The spine projection records the same conservation invariant under its -#: own reason so the terminal family validator can never satisfy the -#: certified and spine families with one shared record (adversarial-review -#: finding on the E8 PR: reason strings are the receipt identity). +#: The reviewed mass-conservation receipt the spine stage records. The +#: terminal family gate requires a valid mass-conserving MassChangeRecord +#: carrying exactly this reason, so a build whose CGT stage silently moved +#: household mass, or never ran, fails by name (reason strings are the +#: receipt identity: adversarial-review finding on the E8 PR). UK_CGT_SPINE_MASS_CONSERVATION_REASON = ( "Amounts-only capital gains redraw on the source spine: household " "weights pass through unchanged and total household mass is conserved." @@ -608,7 +600,7 @@ class UKCGTImputationSummary: def evidence(self) -> dict[str, object]: evidence: dict[str, object] = { - "stage": UK_CGT_IMPUTATION_STAGE_NAME, + "stage": UK_CGT_SPINE_STAGE_NAME, "rows": self.rows.to_dict(orient="records"), "taxpayer_mass": self.taxpayer_mass, "published_taxpayer_mass": self.published_taxpayer_mass, @@ -1218,7 +1210,7 @@ def impute_uk_capital_gains_with_report( *, conditioning: HMRCCGTConditioningFacts, seed: int = UK_CGT_IMPUTATION_SEED, - mass_change_reason: str = UK_CGT_MASS_CONSERVATION_REASON, + mass_change_reason: str = UK_CGT_SPINE_MASS_CONSERVATION_REASON, ) -> tuple[Frame, UKCGTAllocationReport]: """Redraw gainers' amounts, conditioned on income, age and region.""" @@ -1520,7 +1512,7 @@ def impute_uk_capital_gains( *, conditioning: HMRCCGTConditioningFacts | None = None, seed: int = UK_CGT_IMPUTATION_SEED, - mass_change_reason: str = UK_CGT_MASS_CONSERVATION_REASON, + mass_change_reason: str = UK_CGT_SPINE_MASS_CONSERVATION_REASON, ) -> Frame: """Redraw gainers' amounts from the published joint distribution. @@ -1641,37 +1633,6 @@ def summarize_uk_cgt_imputation( ) -def uk_capital_gains_imputation_stage( - *, - parameters: UKCGTPolicyParameters | None = None, - distribution: HMRCCapitalGainsJointDistribution | None = None, - conditioning: HMRCCGTConditioningFacts | None = None, - seed: int = UK_CGT_IMPUTATION_SEED, - mass_change_reason: str = UK_CGT_MASS_CONSERVATION_REASON, -) -> UKNationalStage: - """Build the national stage that redraws capital gains amounts. - - The joint and the conditioning facts default to the committed vendored - 2024-25 resource, which is checked against the pinned Chronicle feed - before a row is read. Parameters default to the policyengine-uk tree at - the dataset's build period, resolved when the stage runs. - """ - - def transform(frame: Frame) -> Frame: - joint = distribution or load_hmrc_cgt_joint_distribution() - resolved = parameters or uk_cgt_policy_parameters(uk_time_period(frame)) - return impute_uk_capital_gains( - frame, - joint, - resolved, - conditioning=conditioning or load_hmrc_cgt_conditioning_facts(), - seed=seed, - mass_change_reason=mass_change_reason, - ) - - return UKNationalStage(name=UK_CGT_IMPUTATION_STAGE_NAME, transform=transform) - - def uk_cgt_spine_stage_transform( stage: SourceStageSpec, *, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py index 4ebb3a528..97c804a9a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py @@ -251,6 +251,14 @@ def _source_stage_base_candidate_tier( return validate_uk_release_tier(base_candidate.get("tier")) +#: A family whose stage writes columns and passes the household weights +#: through untouched: no mass receipt is recorded, so none is required. +_WEIGHTS_PASS_THROUGH = "weights_pass_through" +_MASS_CHANGE_SEMANTICS = frozenset( + {"mass_conserving", "mass_increasing_support", _WEIGHTS_PASS_THROUGH} +) + + def _parse_family_coverage( raw: object, *, @@ -312,9 +320,7 @@ def _parse_family_coverage( f"{resource}: family {name!r} superseded_by must be an object." ) superseding_stage = str(superseded_by.get("stage", "")).strip() - superseding_manifest = str( - superseded_by.get("source_manifest", "") - ).strip() + superseding_manifest = str(superseded_by.get("source_manifest", "")).strip() superseding_sha = str( superseded_by.get("source_manifest_sha256", "") ).strip() @@ -362,22 +368,29 @@ def _parse_family_coverage( required_mass_change_reason = str( raw_family.get("required_mass_change_reason", "") ).strip() - if not required_mass_change_reason: - raise ValueError( - f"{resource}: family {name!r} needs a reviewed " - "required_mass_change_reason." - ) mass_change_semantics = str( raw_family.get("mass_change_semantics", "mass_conserving") ).strip() - if mass_change_semantics not in { - "mass_conserving", - "mass_increasing_support", - }: + if mass_change_semantics not in _MASS_CHANGE_SEMANTICS: raise ValueError( f"{resource}: family {name!r} has invalid " f"mass_change_semantics {mass_change_semantics!r}." ) + # A stage that records a mass receipt is bound to it by reason; a + # weights-pass-through stage records none, so the contract carries + # none (a reason it demanded would fail every build by construction). + if mass_change_semantics == _WEIGHTS_PASS_THROUGH: + if required_mass_change_reason: + raise ValueError( + f"{resource}: family {name!r} declares " + f"{_WEIGHTS_PASS_THROUGH!r} semantics and must not require " + "a mass-change reason." + ) + elif not required_mass_change_reason: + raise ValueError( + f"{resource}: family {name!r} needs a reviewed " + "required_mass_change_reason." + ) raw_requirements = raw_family.get("effective_mass_requirements", {}) if not isinstance(raw_requirements, Mapping): @@ -445,7 +458,11 @@ def _parse_family_coverage( ), "base_candidate_tier": base_candidate_tier, "output_weight_kind": output_weight_kind, - "required_mass_change_reason": required_mass_change_reason, + **( + {"required_mass_change_reason": required_mass_change_reason} + if required_mass_change_reason + else {} + ), "mass_change_semantics": mass_change_semantics, "effective_mass_requirements": requirements, } diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index bc8094311..937f899bb 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -916,7 +916,6 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: row.path for row in spec.resource_rows if row.kind == "legacy_json" ) assert legacy_rows == ( - "cgt_source_stages.json", "degenerate_reviewed_exclusions.json", "target_fit_reviewed_exclusions.json", "efrs_parity_known_gaps.json", @@ -1023,7 +1022,6 @@ def test_uk_package_loads(self) -> None: "spec/sources.yaml", "spec/spine.yaml", "spec/vintages.yaml", - "cgt_source_stages.json", "degenerate_reviewed_exclusions.json", "target_fit_reviewed_exclusions.json", "efrs_parity_known_gaps.json", diff --git a/packages/microcosm-build/tests/test_uk_cgt_imputation.py b/packages/microcosm-build/tests/test_uk_cgt_imputation.py index 326e242d1..6103a583a 100644 --- a/packages/microcosm-build/tests/test_uk_cgt_imputation.py +++ b/packages/microcosm-build/tests/test_uk_cgt_imputation.py @@ -8,10 +8,10 @@ from microcosm.build.uk_runtime import cgt_imputation from microcosm.build.uk_runtime.cgt_imputation import ( UK_CGT_AGE_GROUP_LOWER_BOUNDS, - UK_CGT_IMPUTATION_STAGE_NAME, - UK_CGT_MASS_CONSERVATION_REASON, UK_CGT_REGION_GROUP_LABELS, UK_CGT_REGION_GROUPS, + UK_CGT_SPINE_MASS_CONSERVATION_REASON, + UK_CGT_SPINE_STAGE_NAME, UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS, UKCGTPolicyParameters, _band_plans, @@ -26,7 +26,6 @@ impute_uk_capital_gains, impute_uk_capital_gains_with_report, summarize_uk_cgt_imputation, - uk_capital_gains_imputation_stage, uk_cgt_spine_stage_transform, uk_cgt_taxable_income_proxy, ) @@ -351,7 +350,7 @@ def test_non_gainers_stay_at_zero_and_weights_pass_through(self) -> None: assert result.mass_log[:-1] == frame.mass_log receipt = result.mass_log[-1] assert receipt.entity == "household" - assert receipt.reason == UK_CGT_MASS_CONSERVATION_REASON + assert receipt.reason == UK_CGT_SPINE_MASS_CONSERVATION_REASON assert receipt.old_total == receipt.new_total assert receipt.declared_factor == 1.0 @@ -376,7 +375,7 @@ def test_loss_makers_pass_through_byte_identical(self) -> None: assert drawn[3] == 0.0 assert (drawn[4:] > 0.0).all() receipt = result.mass_log[-1] - assert receipt.reason == UK_CGT_MASS_CONSERVATION_REASON + assert receipt.reason == UK_CGT_SPINE_MASS_CONSERVATION_REASON def test_remainder_keeps_existing_amounts_capped_at_the_aea(self) -> None: # One income band holds far more gainer mass than the published @@ -461,15 +460,18 @@ def test_summary_reports_achieved_against_published(self) -> None: assert {"rows", "age_rows", "region_rows", "age_by_band_rows"} <= set(evidence) +def _spine_stage(): + spec = load_country_spec("uk") + assert spec.sources is not None + return spec.sources.stage_map()[UK_CGT_SPINE_STAGE_NAME] + + class TestStage: - def test_stage_runs_end_to_end_on_the_vendored_surface(self) -> None: - """The factory's own transform path, on the committed resource. + """The spine stage is the only CGT gains stage; the June wrapper is retired.""" - Regression test for the transform keeping a retired carrier type in - its signature: with postponed annotation evaluation, only running the - stage exercises the closure. - """ - stage = uk_capital_gains_imputation_stage(parameters=PARAMETERS) + def test_stage_runs_end_to_end_on_the_vendored_surface(self) -> None: + """The spine transform's own path, on the committed resource.""" + transform = uk_cgt_spine_stage_transform(_spine_stage(), parameters=PARAMETERS) incomes = [20_000.0, 55_000.0, 80_000.0, 120_000.0, 180_000.0, 400_000.0] frame = _frame( 60, @@ -477,16 +479,16 @@ def test_stage_runs_end_to_end_on_the_vendored_surface(self) -> None: incomes=[incomes[i % 6] for i in range(60)], ) - result = stage.run(frame) + result = transform(frame) drawn = result.table("person")["capital_gains"].to_numpy() assert (drawn >= 0).all() assert drawn.max() > 0 + assert result.mass_log[-1].reason == UK_CGT_SPINE_MASS_CONSERVATION_REASON - def test_stage_carries_the_reviewed_name(self) -> None: - stage = uk_capital_gains_imputation_stage(parameters=PARAMETERS) - - assert stage.name == UK_CGT_IMPUTATION_STAGE_NAME + def test_the_spine_declares_the_reviewed_stage(self) -> None: + assert UK_CGT_SPINE_STAGE_NAME == "hmrc_cgt_gains_spine" + assert _spine_stage().stage == UK_CGT_SPINE_STAGE_NAME def test_stage_checks_the_feed_pin_before_reading( self, monkeypatch: pytest.MonkeyPatch @@ -505,11 +507,11 @@ def test_stage_checks_the_feed_pin_before_reading( monkeypatch.setattr( hmrc_capital_gains, "load_vendored_resource", lambda _name: stale ) - stage = uk_capital_gains_imputation_stage(parameters=PARAMETERS) + transform = uk_cgt_spine_stage_transform(_spine_stage(), parameters=PARAMETERS) frame = _frame(1, gains=[10_000.0], incomes=[20_000.0]) with pytest.raises(ValueError, match="differs from the committed UK pin"): - stage.run(frame) + transform(frame) def test_cgt_spine_parsed_inputs_match_the_resource_resolution( diff --git a/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py b/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py index 318f076d0..ac4860b1c 100644 --- a/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py +++ b/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py @@ -1,18 +1,15 @@ -"""Contract tests for the UK capital gains source manifest.""" +"""Contract tests for the UK capital gains stage manifest and family.""" from __future__ import annotations -import json -from pathlib import Path - +from microcosm.build.country_spec import load_country_spec from microcosm.build.uk_runtime.cgt_asset_type import ( UK_CGT_ASSET_TYPE_MASS_CONSERVATION_REASON, ) from microcosm.build.uk_runtime.cgt_imputation import ( UK_CGT_IMPUTATION_SEED, - UK_CGT_IMPUTATION_STAGE_NAME, - UK_CGT_MASS_CONSERVATION_REASON, UK_CGT_SPINE_MASS_CONSERVATION_REASON, + UK_CGT_SPINE_STAGE_NAME, UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS, ) from microcosm.build.uk_runtime.cgt_structure import ( @@ -35,31 +32,24 @@ STUDENT_LOANS_MASS_CHANGE_REASON, ) -_MANIFEST_PATH = ( - Path(__file__).resolve().parents[1] - / "src" - / "microcosm" - / "build" - / "uk" - / "cgt_source_stages.json" -) +def _stage(): + """The spine's CGT amounts stage: since microcosm#823 the only one.""" + spec = load_country_spec("uk") + assert spec.sources is not None + return spec.sources.stage_map()[UK_CGT_SPINE_STAGE_NAME] -def _stage() -> dict: - payload = json.loads(_MANIFEST_PATH.read_text(encoding="utf-8")) - stages = payload["stages"] - assert len(stages) == 1 - return stages[0] + +def _operations() -> dict[str, dict]: + return {op.kind: dict(op.parameters) for op in _stage().operations} def test_manifest_names_the_resource_the_code_reads() -> None: """One provenance, declared once: the manifest repeats the module's roster.""" - artifacts = {artifact["role"]: artifact for artifact in _stage()["artifacts"]} + artifacts = {artifact["role"]: artifact for artifact in _stage().artifacts} assert "published_fact_surface" not in artifacts surface = artifacts["cgt_conditioning_facts"] - verify = {operation["kind"]: operation for operation in _stage()["operations"]}[ - "verify_vendored_fact_resource" - ] + verify = _operations()["verify_vendored_fact_resource"] assert surface["resource"] == HMRC_CGT_CONDITIONING_RESOURCE assert surface["runtime_sha256_required"] is True @@ -72,9 +62,9 @@ def test_manifest_names_the_resource_the_code_reads() -> None: def test_manifest_operations_match_the_stage_implementation() -> None: - operations = {operation["kind"]: operation for operation in _stage()["operations"]} + operations = _operations() - assert _stage()["stage"] == UK_CGT_IMPUTATION_STAGE_NAME + assert _stage().stage == UK_CGT_SPINE_STAGE_NAME proxy = operations["taxable_income_proxy"] assert tuple(proxy["components"]) == UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS draws = operations["within_band_draws"] @@ -86,9 +76,7 @@ def test_manifest_operations_match_the_stage_implementation() -> None: def test_band_facts_stay_fenced_from_calibration() -> None: - fence = {operation["kind"]: operation for operation in _stage()["operations"]}[ - "classify_cgt_band_facts_with_reviewed_fence" - ] + fence = _operations()["classify_cgt_band_facts_with_reviewed_fence"] assert fence["calibration_permitted"] is False assert fence["fenced_fact_count"] == 76 @@ -97,25 +85,52 @@ def test_band_facts_stay_fenced_from_calibration() -> None: def test_receipt_reason_matches_the_stage_constant() -> None: """The gate matches on the exact string, so one source of truth.""" - receipt = {operation["kind"]: operation for operation in _stage()["operations"]}[ - "record_mass_conservation_receipt" - ] + receipt = _operations()["record_mass_conservation_receipt"] - assert receipt["reason"] == UK_CGT_MASS_CONSERVATION_REASON + assert receipt["reason"] == UK_CGT_SPINE_MASS_CONSERVATION_REASON assert receipt["declared_factor"] == 1.0 def test_family_coverage_carries_the_stage_as_required_at_build() -> None: manifest = load_uk_release_input_coverage_manifest() - family = manifest.family_coverage[UK_CGT_IMPUTATION_STAGE_NAME] + family = manifest.family_coverage[UK_CGT_SPINE_STAGE_NAME] assert family["status"] == "required_at_build" - assert family["source_manifest"] == _MANIFEST_PATH.name + assert family["source_manifest"] == "source_stages.json" assert family["calibration_permitted"] is False assert family["outputs"] == ["capital_gains"] assert family["output_weight_kind"] == "importance" - assert family["required_mass_change_reason"] == UK_CGT_MASS_CONSERVATION_REASON - assert UK_CGT_IMPUTATION_STAGE_NAME in manifest.required_build_stages + assert ( + family["required_mass_change_reason"] == UK_CGT_SPINE_MASS_CONSERVATION_REASON + ) + assert UK_CGT_SPINE_STAGE_NAME in manifest.required_build_stages + + +def test_the_june_path_cgt_family_is_retired() -> None: + """One CGT gains family, the spine's (microcosm#823).""" + manifest = load_uk_release_input_coverage_manifest() + assert "hmrc_cgt_gains" not in manifest.family_coverage + assert "hmrc_cgt_gains" not in manifest.required_build_stages + assert "hmrc_cgt_gains" not in load_country_spec("uk").sources.stage_map() + + +def test_stages_without_a_declared_receipt_require_none() -> None: + """A column-writing stage that leaves the weights untouched records no + mass receipt, so the contract demands none; the weight-kind check still + covers it. The first release cut failed five families on a reason the + generator had invented.""" + manifest = load_uk_release_input_coverage_manifest() + for name in ( + "was_wealth", + "lcfs_consumption", + "etb_vat", + "etb_services", + "regional_property_uprating", + ): + family = manifest.family_coverage[name] + assert "required_mass_change_reason" not in family, name + assert family["mass_change_semantics"] == "weights_pass_through", name + assert family["output_weight_kind"] == "importance", name def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: @@ -137,9 +152,6 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: spi_reason = str( manifest.family_coverage["hmrc_spi_income"]["required_mass_change_reason"] ) - e5_reason = str( - manifest.family_coverage["was_wealth"]["required_mass_change_reason"] - ) compliant = SimpleNamespace( household_weight_kind=WeightKind.IMPORTANCE, time_period="2024", @@ -165,13 +177,6 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: declared_factor=None, reason=CGT_DONOR_MASS_CHANGE_REASON, ), - MassChangeRecord( - entity="household", - old_total=100.0, - new_total=100.0, - declared_factor=1.0, - reason=UK_CGT_MASS_CONSERVATION_REASON, - ), MassChangeRecord( entity="household", old_total=100.0, @@ -200,13 +205,6 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: declared_factor=1.0, reason=STUDENT_LOANS_MASS_CHANGE_REASON, ), - MassChangeRecord( - entity="household", - old_total=100.0, - new_total=100.0, - declared_factor=1.0, - reason=e5_reason, - ), ), ) @@ -219,7 +217,7 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: mass_log=compliant.mass_log[:1], ) _, failures = _family_build_state_diagnostics(missing_receipt, manifest) - assert any("hmrc_cgt_gains" in failure for failure in failures) + assert any("hmrc_cgt_gains_spine" in failure for failure in failures) wrong_kind = SimpleNamespace( household_weight_kind=WeightKind.DESIGN, @@ -228,16 +226,5 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: ) _, failures = _family_build_state_diagnostics(wrong_kind, manifest) assert any( - "hmrc_cgt_gains" in failure and "kind" in failure for failure in failures + "hmrc_cgt_gains_spine" in failure and "kind" in failure for failure in failures ) - - -def test_certified_and_spine_families_require_distinct_receipts() -> None: - """One record must never satisfy both CGT families (review finding).""" - manifest = load_uk_release_input_coverage_manifest() - families = manifest.family_coverage - certified = families["hmrc_cgt_gains"]["required_mass_change_reason"] - spine = families["hmrc_cgt_gains_spine"]["required_mass_change_reason"] - assert certified == UK_CGT_MASS_CONSERVATION_REASON - assert spine == UK_CGT_SPINE_MASS_CONSERVATION_REASON - assert certified != spine diff --git a/packages/microcosm-build/tests/test_uk_release_input_coverage.py b/packages/microcosm-build/tests/test_uk_release_input_coverage.py index ce8ec7b0a..61bbf803d 100644 --- a/packages/microcosm-build/tests/test_uk_release_input_coverage.py +++ b/packages/microcosm-build/tests/test_uk_release_input_coverage.py @@ -27,6 +27,16 @@ from microcosm.frame import EntitySchema, Frame, MassChangeRecord, WeightKind, Weights _REPO_ROOT = Path(__file__).resolve().parents[3] +_SHIPPED_MANIFEST = ( + _REPO_ROOT + / "packages" + / "microcosm-build" + / "src" + / "microcosm" + / "build" + / "uk" + / "release_input_coverage_manifest.json" +) def _person_frame(columns: dict[str, np.ndarray]) -> Frame: @@ -604,7 +614,6 @@ def test_shipped_manifest_is_current(self) -> None: assert manifest.required_build_stages == frozenset( { "hmrc_spi_income", - "hmrc_cgt_gains", "cgt_incidence_clone", "cgt_band_donors", "hmrc_cgt_gains_spine", @@ -691,18 +700,33 @@ def test_candidate_tier_must_be_present_and_ratified( with pytest.raises(ValueError, match="candidate_evidence.tier"): load_uk_release_input_coverage_manifest(str(bad)) - def test_deferred_family_requires_restoration_status(self, tmp_path) -> None: - source = ( - _REPO_ROOT - / "packages" - / "microcosm-build" - / "src" - / "microcosm" - / "build" - / "uk" - / "release_input_coverage_manifest.json" + def test_pass_through_family_must_not_require_a_reason(self, tmp_path) -> None: + payload = json.loads(_SHIPPED_MANIFEST.read_text(encoding="utf-8")) + family = payload["family_coverage"]["was_wealth"] + assert family["mass_change_semantics"] == "weights_pass_through" + family["required_mass_change_reason"] = "invented receipt" + bad = tmp_path / "pass_through_with_reason.json" + bad.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="must not require a mass-change"): + load_uk_release_input_coverage_manifest(str(bad)) + + def test_receipted_family_requires_a_reason(self, tmp_path) -> None: + payload = json.loads(_SHIPPED_MANIFEST.read_text(encoding="utf-8")) + family = payload["family_coverage"]["hmrc_spi_income"] + # The loader defaults an absent semantics key to mass_conserving. + assert family.get("mass_change_semantics", "mass_conserving") == ( + "mass_conserving" ) - payload = json.loads(source.read_text(encoding="utf-8")) + family.pop("required_mass_change_reason") + bad = tmp_path / "receipted_without_reason.json" + bad.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="needs a reviewed"): + load_uk_release_input_coverage_manifest(str(bad)) + + def test_deferred_family_requires_restoration_status(self, tmp_path) -> None: + payload = json.loads(_SHIPPED_MANIFEST.read_text(encoding="utf-8")) payload["family_coverage"]["hmrc_spi_income"]["status"] = ( "deferred_until_restored" ) @@ -733,7 +757,7 @@ def test_spine_posture_satisfies_superseded_required_families(self) -> None: spine_stages = tuple( stage for stage in manifest.required_build_stages - if stage not in {"hmrc_spi_income", "hmrc_cgt_gains"} + if stage != "hmrc_spi_income" ) result = assert_uk_release_input_coverage_build_stages( (*spine_stages, "hmrc_spi_income_spine"), @@ -744,10 +768,7 @@ def test_spine_posture_satisfies_superseded_required_families(self) -> None: manifest.family_coverage["hmrc_spi_income"]["superseded_by"]["stage"] == "hmrc_spi_income_spine" ) - assert ( - manifest.family_coverage["hmrc_cgt_gains"]["superseded_by"]["stage"] - == "hmrc_cgt_gains_spine" - ) + assert "hmrc_cgt_gains" not in manifest.family_coverage def test_supersession_does_not_hide_a_genuinely_missing_family(self) -> None: manifest = load_uk_release_input_coverage_manifest() @@ -757,7 +778,6 @@ def test_supersession_does_not_hide_a_genuinely_missing_family(self) -> None: if stage not in { "hmrc_spi_income", - "hmrc_cgt_gains", "student_loans", } ) diff --git a/tools/build_uk_release_input_coverage_manifest.py b/tools/build_uk_release_input_coverage_manifest.py index 699d7ea41..5f300a454 100644 --- a/tools/build_uk_release_input_coverage_manifest.py +++ b/tools/build_uk_release_input_coverage_manifest.py @@ -44,7 +44,6 @@ KNOWN_GAPS_PATH = UK_PACKAGE_DIR / "efrs_parity_known_gaps.json" MANIFEST_PATH = UK_PACKAGE_DIR / "release_input_coverage_manifest.json" HMRC_SOURCE_STAGES_PATH = UK_PACKAGE_DIR / "hmrc_income_source_stages.json" -CGT_SOURCE_STAGES_PATH = UK_PACKAGE_DIR / "cgt_source_stages.json" SOURCE_STAGES_PATH = UK_PACKAGE_DIR / "source_stages.json" CANDIDATE_REPO_ID = "policyengine/populace-uk-private" @@ -813,9 +812,6 @@ def build_manifest( stage_name="student_loans", candidate_source=candidate_source, ), - "hmrc_cgt_gains": _cgt_family_coverage_contract( - candidate_source=candidate_source, - ), "hmrc_spi_income": _hmrc_family_coverage_contract( candidate_source=candidate_source ), @@ -861,132 +857,6 @@ def build_manifest( } -def _cgt_family_coverage_contract( - *, - candidate_source: dict[str, Any], -) -> dict[str, Any]: - """Validate the CGT source manifest and emit its family contract. - - The stage redraws capital gains amounts only, so unlike the SPI family it - moves no mass and declares no distributional effective-mass requirement: - ``capital_gains`` already carries hard release status from the candidate, - and the stage replaces its values in place. - """ - - payload = _load(CGT_SOURCE_STAGES_PATH) - stages = payload.get("stages") - if not isinstance(stages, list) or len(stages) != 1: - raise ValueError( - f"{CGT_SOURCE_STAGES_PATH}: expected exactly one source stage." - ) - stage = stages[0] - if not isinstance(stage, dict) or stage.get("stage") != "hmrc_cgt_gains": - raise ValueError(f"{CGT_SOURCE_STAGES_PATH}: expected hmrc_cgt_gains stage.") - base_candidate = stage.get("base_candidate") - if not isinstance(base_candidate, dict): - raise ValueError(f"{CGT_SOURCE_STAGES_PATH}: base_candidate must be an object.") - source_tier = validate_uk_release_tier(candidate_source.get("tier")) - base_candidate_tier = validate_uk_release_tier(base_candidate.get("tier")) - if base_candidate_tier != source_tier: - raise ValueError( - "CGT source-stage base candidate tier disagrees with the certified " - f"candidate evidence: {base_candidate_tier!r} != {source_tier!r}." - ) - artifacts = { - artifact["role"]: artifact - for artifact in stage.get("artifacts", []) - if isinstance(artifact, dict) and isinstance(artifact.get("role"), str) - } - operations = { - operation["kind"]: operation - for operation in stage.get("operations", []) - if isinstance(operation, dict) and isinstance(operation.get("kind"), str) - } - required_artifacts = { - "policy_parameters", - "cgt_conditioning_facts", - } - missing_artifacts = sorted(required_artifacts - set(artifacts)) - required_operations = { - "verify_certified_candidate", - "verify_vendored_fact_resource", - "taxable_income_proxy", - "rake_allocation_targets", - "rank_preserving_allocation", - "within_band_draws", - "sub_aea_remainder", - "record_mass_conservation_receipt", - "classify_cgt_band_facts_with_reviewed_fence", - } - missing_operations = sorted(required_operations - set(operations)) - if missing_artifacts or missing_operations: - raise ValueError( - f"{CGT_SOURCE_STAGES_PATH}: incomplete CGT family contract; " - f"missing_artifacts={missing_artifacts}, " - f"missing_operations={missing_operations}." - ) - surface = artifacts["cgt_conditioning_facts"] - verify = operations["verify_vendored_fact_resource"] - fence = operations["classify_cgt_band_facts_with_reviewed_fence"] - if not bool(verify.get("require_before_source_read")): - raise ValueError( - f"{CGT_SOURCE_STAGES_PATH}: the vendored resource must be checked " - "against the feed pin before it is read." - ) - if verify.get("resource") != surface.get("resource"): - raise ValueError( - f"{CGT_SOURCE_STAGES_PATH}: the verification must name the " - "cgt_conditioning_facts resource." - ) - if bool(fence.get("calibration_permitted", True)): - raise ValueError( - f"{CGT_SOURCE_STAGES_PATH}: the band-fact fence must keep " - "calibration_permitted false; promotion goes through a separately " - "reviewed target profile." - ) - if not bool(surface.get("runtime_sha256_required")): - raise ValueError( - f"{CGT_SOURCE_STAGES_PATH}: cgt_conditioning_facts must require its " - "runtime sha256." - ) - return { - "status": "required_at_build", - "stage": "hmrc_cgt_gains", - "source_manifest": CGT_SOURCE_STAGES_PATH.name, - "source_manifest_sha256": _sha256(CGT_SOURCE_STAGES_PATH), - "superseded_by": { - "stage": "hmrc_cgt_gains_spine", - "source_manifest": SOURCE_STAGES_PATH.name, - "source_manifest_sha256": _sha256(SOURCE_STAGES_PATH), - "reason": ( - "The FRS spine build executes hmrc_cgt_gains_spine, which " - "applies the same HMRC Table 3 amounts redraw directly in " - "source_stages.json before calibration." - ), - }, - "base_candidate_sha256": str(base_candidate["sha256"]), - "base_candidate_tier": base_candidate_tier, - "source_vintages": { - "hmrc_surface": str(verify["source_vintage"]), - "mapped_build_period": str(verify["mapped_build_period"]), - "conditioning_surface": str(verify["source_vintage"]), - "conditioning_resource": str(surface["resource"]), - "conditioning_resource_sha256": _sha256( - UK_PACKAGE_DIR / str(surface["resource"]) - ), - }, - "output_weight_kind": str(stage["output_weight_kind"]), - "required_mass_change_reason": str( - operations["record_mass_conservation_receipt"]["reason"] - ), - "calibration_permitted": bool(fence["calibration_permitted"]), - "fact_fence_id": str(fence["fact_fence_id"]), - "fenced_fact_count": int(fence["fenced_fact_count"]), - "outputs": list(stage.get("outputs", [])), - "effective_mass_requirements": {}, - } - - def _source_stage_family_coverage_contract( *, stage_name: str, @@ -1016,23 +886,25 @@ def _source_stage_family_coverage_contract( for operation in operations if isinstance(operation.get("reason"), str) and operation.get("reason") ] - required_mass_change_reason = ( - declared_reasons[-1] - if declared_reasons - else ( - "E5 source-stage transform preserves household rows and typed " - "household weights; total household mass is conserved." - ) - ) - mass_change_semantics = ( - "mass_increasing_support" - if any( - operation.get("kind") == "stack_band_donor_households" - for operation in operations - ) - else "mass_conserving" - ) - return { + # A stage that declares a mass-change reason on one of its operations + # records that receipt on the frame, and the terminal family gate looks + # for it by that exact string. A stage that declares none (the E5 fits + # and the regional uprating: column-writing transforms that pass the + # household weights through untouched) leaves no receipt to look for, + # so the contract requires none; inventing a generic reason here made + # every such family fail the first release cut on a record nothing ever + # wrote. The weight-kind check still covers those stages. + required_mass_change_reason = declared_reasons[-1] if declared_reasons else None + if any( + operation.get("kind") == "stack_band_donor_households" + for operation in operations + ): + mass_change_semantics = "mass_increasing_support" + elif declared_reasons: + mass_change_semantics = "mass_conserving" + else: + mass_change_semantics = "weights_pass_through" + contract: dict[str, Any] = { "status": "required_at_build", "stage": stage_name, "source_manifest": SOURCE_STAGES_PATH.name, @@ -1044,12 +916,14 @@ def _source_stage_family_coverage_contract( "source": str(stage.get("source", "")), }, "output_weight_kind": "importance", - "required_mass_change_reason": required_mass_change_reason, "mass_change_semantics": mass_change_semantics, "outputs": list(stage.get("outputs", [])), "rewrites": list(stage.get("rewrites", [])), "effective_mass_requirements": {}, } + if required_mass_change_reason is not None: + contract["required_mass_change_reason"] = required_mass_change_reason + return contract def _cgt_spine_family_coverage_contract( @@ -1349,17 +1223,13 @@ def main() -> int: else: known_gaps = _load(KNOWN_GAPS_PATH) manifest = build_manifest(reference=reference, known_gaps_payload=known_gaps) - generic_fallback_reason = ( - "E5 source-stage transform preserves household rows and typed " - "household weights; total household mass is conserved." - ) declared_reasons: dict[str, str] = {} for family_name, family in manifest.get("family_coverage", {}).items(): reason = str(family.get("required_mass_change_reason", "")).strip() - if not reason or reason == generic_fallback_reason: - # The pre-E8 families share the generic fallback (standing - # follow-up); every stage-declared reason must be unique so a - # receipt identifies exactly one family. + if not reason: + # A family whose stage records no receipt requires none; every + # declared reason must be unique so a receipt identifies exactly + # one family. continue if reason in declared_reasons: raise ValueError( From ea907789d86ad213e1ec039b10368151ef45186c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:49:57 +0100 Subject: [PATCH 11/22] Certifier evidence: enum domains on both UK engines, the build-state half on the spine frame Two release-cut gates failed for want of certifier evidence, not of the candidate. The enum-domain gates read engine._variable(column), which only the frame adapter had, so the certifier's coverage engine failed them closed: both engines now expose enum_domain(name) and the evaluator resolves through it (falling back to _variable). uk_release_input_coverage re-evaluated the spine-stage contract (importance weights, stage receipts) on the calibrated release frame and failed 13 families on the weight kind: the gate takes a build_state_frame, the binding hands it the spine frame from the battery artifacts, the certifier loads the spine it already receives (now pinned by --spine-sha256, bytes verified, recorded under source_pins.spine_h5), and the column and effective-mass halves stay on the release frame. The gate's details name which frame the build-state half read; runbook 806 carries the new flag. Co-Authored-By: Claude Fable 5.1 --- .../uk-certifier-spine-frame-967.fixed.md | 1 + .../uk-enum-domain-accessor-967.fixed.md | 1 + ...k-national-release-assembly-runbook-806.md | 1 + .../build/uk_runtime/battery_bindings.py | 37 +++++- .../build/uk_runtime/release_certification.py | 8 ++ .../uk_runtime/release_input_coverage.py | 33 ++++- .../tests/test_uk_battery_bindings.py | 70 +++++++++++ .../test_uk_certify_release_cut_command.py | 8 ++ .../tests/test_uk_release_certification.py | 1 + .../tests/test_uk_release_input_coverage.py | 117 ++++++++++++++++++ .../frame/adapters/policyengine_uk.py | 5 + tools/certify_uk_release_cut.py | 15 +++ 12 files changed, 291 insertions(+), 6 deletions(-) create mode 100644 changelog.d/uk-certifier-spine-frame-967.fixed.md create mode 100644 changelog.d/uk-enum-domain-accessor-967.fixed.md diff --git a/changelog.d/uk-certifier-spine-frame-967.fixed.md b/changelog.d/uk-certifier-spine-frame-967.fixed.md new file mode 100644 index 000000000..9eb8917d2 --- /dev/null +++ b/changelog.d/uk-certifier-spine-frame-967.fixed.md @@ -0,0 +1 @@ +`tools/certify_uk_release_cut.py` pins the spine (`--spine-sha256`, a Logbook pin role) and loads its frame, and the release-cut battery hands it to `uk_release_input_coverage` as the evidence for the stage families' build state: typed `importance` weights and each stage's mass receipt are a spine-stage contract, and the first release-cut run evaluated them on the calibrated candidate, where the weights are `calibrated` by construction, failing all thirteen families. The coverage and effective-mass halves still read the release frame; the gate's details name which frame the build state came from.\n \ No newline at end of file diff --git a/changelog.d/uk-enum-domain-accessor-967.fixed.md b/changelog.d/uk-enum-domain-accessor-967.fixed.md new file mode 100644 index 000000000..872223fc3 --- /dev/null +++ b/changelog.d/uk-enum-domain-accessor-967.fixed.md @@ -0,0 +1 @@ +The UK enum-domain gates (`uk_uc_deduction_combination_enum_domain`, `uk_student_loan_plan_enum_domain`) resolve their domain through a public `enum_domain(column)` accessor carried by both engine adapters, the frame adapter's `PolicyEngineUKEngine` and the release cut's `PolicyEngineUKCoverageEngine`; the first release-cut run failed both gates with an `AttributeError` because the coverage engine lacked the private lookup the evaluator read, and an engine with neither now names itself in the refusal.\n \ No newline at end of file diff --git a/docs/uk-national-release-assembly-runbook-806.md b/docs/uk-national-release-assembly-runbook-806.md index 3c3ead7fc..aea405fe1 100644 --- a/docs/uk-national-release-assembly-runbook-806.md +++ b/docs/uk-national-release-assembly-runbook-806.md @@ -83,6 +83,7 @@ uv run --no-sync python tools/certify_uk_release_cut.py \ --candidate-sha256 \ --candidate-name microcosm_uk_2024_25 \ --spine-h5 \ + --spine-sha256 \ --diagnostics-json /calibration_diagnostics.json \ --build-record-json /build_record.json \ --seam-gate-report /microcosm_uk_2024_25.terminal_gates.json \ diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py index 15f56992e..5cc4d9d9a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py @@ -213,8 +213,17 @@ def _evaluate_release_input_coverage( f"unknown release_input_coverage check {check!r}; the declared " "preflight mode is 'manifest_current'." ) + # The release cut supplies the spine frame the stages produced, so the + # family build-state half reads importance weights and stage receipts + # where they live; the coverage halves read the release frame. + spine_frame = context.artifacts.get("spine_frame") return uk_release_input_coverage_gate( - _uk_gate_surface(context.frame), engine, manifest=manifest + _uk_gate_surface(context.frame), + engine, + manifest=manifest, + build_state_frame=None + if spine_frame is None + else _uk_gate_surface(spine_frame), ) @@ -463,6 +472,28 @@ def _evaluate_take_up_signal( return uk_take_up_signal_gate(context.frame, **dict(parameters)) +def _engine_enum_domain(engine: Any, column: str) -> Any: + """Resolve an enum column's domain from whichever engine adapter is armed. + + The public ``enum_domain`` accessor is the contract (both UK adapters + carry it); the private ``_variable`` lookup stays as the fallback for an + engine that predates it. An adapter with neither names itself in the + failure instead of surfacing as an AttributeError that reads like a + gate crash — the first release-cut run failed both enum gates that way. + """ + + accessor = getattr(engine, "enum_domain", None) + if callable(accessor): + return accessor(column) + lookup = getattr(engine, "_variable", None) + if callable(lookup): + return getattr(lookup(column), "possible_values", None) + raise ValueError( + f"{column} enum domain could not be resolved: the armed rules engine " + f"({type(engine).__name__}) exposes neither enum_domain nor _variable." + ) + + def _evaluate_enum_domain( context: EvidenceContext, parameters: Mapping[str, Any] ) -> GateResult: @@ -472,9 +503,7 @@ def _evaluate_enum_domain( column = columns[0] domain = context.artifacts.get(f"{column}_enum_domain") if domain is None: - engine = context.artifacts["rules_engine"] - variable = engine._variable(column) - domain = getattr(variable, "possible_values", None) + domain = _engine_enum_domain(context.artifacts["rules_engine"], column) if domain is None: raise ValueError(f"{column} enum domain could not be resolved from evidence.") matches = [ diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py index e581ae8b7..b0d8398da 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py @@ -264,9 +264,15 @@ def run_uk_release_cut_battery( input_mass_reference: Mapping[str, Any], exclusions_evaluated_on: date, gate_registry: Mapping[str, Any] | None = None, + spine_frame: Any | None = None, ) -> dict[str, Any]: """Run the 18 national gates over the calibrated candidate, signed. + ``spine_frame`` is the frame the spine build produced, the evidence the + input-coverage gate's family build-state half reads (typed importance + weights and each stage's mass receipt); the calibrated candidate is the + evidence for everything else. + Always release-candidate strict: this battery exists to certify a cut, so an ``evidence_absent`` gap blocks rather than being tolerated, and a blocked phase persists its report and raises before any composition. @@ -306,6 +312,8 @@ def run_uk_release_cut_battery( } if fit_weight_records is not None: terminal_artifacts["fit_weight_records"] = fit_weight_records + if spine_frame is not None: + terminal_artifacts["spine_frame"] = spine_frame battery.run_phase( "terminal", EvidenceContext(frame=frame, artifacts=terminal_artifacts) ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py index 97c804a9a..67a5322ce 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py @@ -584,6 +584,23 @@ def _tax_benefit_system(self) -> Any: self._system = CountryTaxBenefitSystem() return self._system + def _variable(self, name: str) -> Any: + variables = self._tax_benefit_system().variables + if name not in variables: + raise ValueError(f"Unknown PolicyEngine-UK variable {name!r}.") + return variables[name] + + def enum_domain(self, name: str) -> Any: + """The Enum an engine variable's stored values must belong to. + + The enum-domain gates resolve their domain through this accessor, so + every engine adapter the batteries are handed (the frame adapter on + the spine build, this coverage engine on the release cut) answers the + same question by the same name. + """ + + return getattr(self._variable(name), "possible_values", None) + def variables(self) -> list[str]: variables = self._tax_benefit_system().variables pure_inputs = { @@ -1089,8 +1106,19 @@ def uk_release_input_coverage_gate( engine: Any, *, manifest: UKReleaseInputCoverageManifest | None = None, + build_state_frame: Any | None = None, ) -> GateResult: - """Enforce required signal on rows carrying reviewed effective mass.""" + """Enforce required signal on rows carrying reviewed effective mass. + + The column-coverage and effective-mass halves read ``frame``, the + release frame. The family build-state half (each stage family's typed + ``importance`` weights and its mass receipt) is a spine-stage contract: + it reads ``build_state_frame`` when the caller supplies the spine frame + the stages produced, and falls back to ``frame`` for a build whose + terminal frame is that spine. A calibrated release frame carries + ``calibrated`` weights by construction, so evaluating the build-state + half on it fails every family on the kind alone. + """ manifest = manifest or load_uk_release_input_coverage_manifest() required = manifest.required_columns reviewed = manifest.reviewed_exclusions @@ -1179,7 +1207,7 @@ def uk_release_input_coverage_gate( ) failures.extend(family_failures) family_build_state, family_build_failures = _family_build_state_diagnostics( - frame, + frame if build_state_frame is None else build_state_frame, manifest, ) failures.extend(family_build_failures) @@ -1196,6 +1224,7 @@ def uk_release_input_coverage_gate( "wrong_entity_columns": wrong_entities, "family_effective_mass": family_diagnostics, "family_build_state": family_build_state, + "family_build_state_frame": "release" if build_state_frame is None else "spine", } return GateResult( name=base.name, diff --git a/packages/microcosm-build/tests/test_uk_battery_bindings.py b/packages/microcosm-build/tests/test_uk_battery_bindings.py index 0c79b4d82..bdae13a59 100644 --- a/packages/microcosm-build/tests/test_uk_battery_bindings.py +++ b/packages/microcosm-build/tests/test_uk_battery_bindings.py @@ -1364,3 +1364,73 @@ def test_measured_local_quality_failure_blocks_release( assert report["shippable"] is False assert report["gates"][gate_id]["status"] == "failed" assert report["gates"][gate_id]["criticality"] == "release_blocking" + + +class TestEnumDomainResolution: + """The enum gates resolve their domain from whichever engine is armed.""" + + def _frame_with_plans(self): + import enum + + person, benunit, household = _tables() + plans = ["NONE", "PLAN_2"] * len(person) + person["student_loan_plan"] = plans[: len(person)] + frame = uk_national_frame( + person=person, benunit=benunit, household=household, time_period="2023" + ) + + class StudentLoanPlan(enum.Enum): + NONE = "NONE" + PLAN_2 = "PLAN_2" + + return frame, StudentLoanPlan + + def test_public_accessor_is_the_contract(self) -> None: + from microcosm.build.uk_runtime.battery_bindings import _evaluate_enum_domain + + frame, domain = self._frame_with_plans() + + class Accessor: + def enum_domain(self, column): + assert column == "student_loan_plan" + return domain + + result = _evaluate_enum_domain( + EvidenceContext(frame=frame, artifacts={"rules_engine": Accessor()}), + {"columns": ["student_loan_plan"]}, + ) + assert result.passed + + def test_private_variable_lookup_still_resolves(self) -> None: + from types import SimpleNamespace + + from microcosm.build.uk_runtime.battery_bindings import _evaluate_enum_domain + + frame, domain = self._frame_with_plans() + + class Legacy: + def _variable(self, column): + return SimpleNamespace(possible_values=domain) + + result = _evaluate_enum_domain( + EvidenceContext(frame=frame, artifacts={"rules_engine": Legacy()}), + {"columns": ["student_loan_plan"]}, + ) + assert result.passed + + def test_an_engine_with_neither_names_itself(self) -> None: + from microcosm.build.uk_runtime.battery_bindings import _evaluate_enum_domain + + frame, _domain = self._frame_with_plans() + + class Bare: + pass + + # The first release-cut run failed both enum gates with an + # AttributeError from inside the evaluator; the refusal now names the + # adapter and the missing accessor. + with pytest.raises(ValueError, match="Bare.*neither enum_domain nor _variable"): + _evaluate_enum_domain( + EvidenceContext(frame=frame, artifacts={"rules_engine": Bare()}), + {"columns": ["student_loan_plan"]}, + ) diff --git a/packages/microcosm-build/tests/test_uk_certify_release_cut_command.py b/packages/microcosm-build/tests/test_uk_certify_release_cut_command.py index b54384540..72cd1986f 100644 --- a/packages/microcosm-build/tests/test_uk_certify_release_cut_command.py +++ b/packages/microcosm-build/tests/test_uk_certify_release_cut_command.py @@ -58,6 +58,8 @@ def fake_run(args, state): "microcosm_uk_2024_25", "--spine-h5", str(placeholders["spine.h5"]), + "--spine-sha256", + hashlib.sha256(placeholders["spine.h5"].read_bytes()).hexdigest(), "--diagnostics-json", str(placeholders["calibration_diagnostics.json"]), "--build-record-json", @@ -96,6 +98,12 @@ def fake_run(args, state): "sha256": facts_sha, "size_bytes": (ledger / "consumer_facts.jsonl").stat().st_size, }, + "spine_h5": { + "sha256": hashlib.sha256( + placeholders["spine.h5"].read_bytes() + ).hexdigest(), + "size_bytes": placeholders["spine.h5"].stat().st_size, + }, } ) assert state.build_id.startswith("uk-frs-release-certification-attempt-") diff --git a/packages/microcosm-build/tests/test_uk_release_certification.py b/packages/microcosm-build/tests/test_uk_release_certification.py index d6404b17c..d9988bf84 100644 --- a/packages/microcosm-build/tests/test_uk_release_certification.py +++ b/packages/microcosm-build/tests/test_uk_release_certification.py @@ -366,6 +366,7 @@ def test_release_cut_battery_runs_and_signs(tmp_path: Path, monkeypatch): input_mass_reference={}, exclusions_evaluated_on=date(2026, 8, 27), gate_registry=_stub_registry(), + spine_frame=object(), ) assert payload["posture"] == "release_cut" assert payload["release_candidate"] is True diff --git a/packages/microcosm-build/tests/test_uk_release_input_coverage.py b/packages/microcosm-build/tests/test_uk_release_input_coverage.py index 61bbf803d..ee60461e7 100644 --- a/packages/microcosm-build/tests/test_uk_release_input_coverage.py +++ b/packages/microcosm-build/tests/test_uk_release_input_coverage.py @@ -64,6 +64,7 @@ def _weighted_person_frame( *, weight_kind: WeightKind = WeightKind.DESIGN, mass_log: tuple[MassChangeRecord, ...] = (), + time_period: str | None = None, ) -> Frame: weights = np.asarray(household_weights, dtype=float) n = len(weights) @@ -85,6 +86,7 @@ def _weighted_person_frame( EntitySchema(group_entities=("benunit", "household")), {"household": Weights(values=weights, kind=weight_kind)}, mass_log=mass_log, + metadata=None if time_period is None else {"time_period": time_period}, ) @@ -990,3 +992,118 @@ def test_default_us_coverage_path_is_unchanged() -> None: ).read_text(encoding="utf-8") assert "us_release_input_coverage_gate" in us_builder assert "uk_release_input_coverage_gate" not in us_builder + + +@pytest.mark.requires_uk +def test_coverage_engine_resolves_enum_domains_for_the_release_cut() -> None: + """The certifier arms this engine; the terminal enum gates read it.""" + + from microcosm.build.uk_runtime.release_input_coverage import ( + PolicyEngineUKCoverageEngine, + ) + + engine = PolicyEngineUKCoverageEngine() + plans = engine.enum_domain("student_loan_plan") + combinations = engine.enum_domain("uc_deduction_combination") + assert {"NONE", "PLAN_1", "PLAN_2"} <= set(plans.__members__) + assert {"NONE", "ADVANCE_ONLY", "ALL_THREE"} <= set(combinations.__members__) + with pytest.raises(ValueError, match="Unknown PolicyEngine-UK variable"): + engine.enum_domain("not_a_variable") + + +class TestFamilyBuildStateFrame: + """The family build-state half reads the spine frame when supplied.""" + + def _contract(self): + family = _hmrc_family_coverage() + family["hmrc_spi_income"].update( + { + "output_weight_kind": "importance", + "required_mass_change_reason": "reviewed SPI allocation", + } + ) + return _manifest( + (UKReleaseInputColumn("gift_aid", "required"),), + family_coverage=family, + ) + + def _frames(self): + receipt = MassChangeRecord( + entity="household", + old_total=2_000.0, + new_total=2_000.0, + declared_factor=1.0, + reason="reviewed SPI allocation", + ) + columns = { + "gift_aid": np.asarray([0.0, 100.0]), + "person_support_channel": np.asarray(["frs", "spi"]), + } + spine = _weighted_person_frame( + columns, + np.asarray([1_000.0, 1_000.0]), + weight_kind=WeightKind.IMPORTANCE, + mass_log=(receipt,), + time_period="2024", + ) + release = _weighted_person_frame( + columns, + np.asarray([900.0, 1_100.0]), + weight_kind=WeightKind.CALIBRATED, + mass_log=( + receipt, + MassChangeRecord( + entity="household", + old_total=2_000.0, + new_total=2_000.0, + declared_factor=1.0, + reason="national calibration", + ), + ), + time_period="2024", + ) + return spine, release + + def test_gate_reads_build_state_from_the_spine_frame(self) -> None: + spine, release = self._frames() + engine = _StubEngine({"gift_aid": 0.0}) + + on_release = uk_release_input_coverage_gate( + release, engine, manifest=self._contract() + ) + assert not on_release.passed + assert any( + "expected reviewed kind 'importance'" in f for f in on_release.failures + ) + assert on_release.details["family_build_state_frame"] == "release" + + result = uk_release_input_coverage_gate( + release, engine, manifest=self._contract(), build_state_frame=spine + ) + assert result.passed, result.failures + assert result.details["family_build_state_frame"] == "spine" + state = result.details["family_build_state"]["hmrc_spi_income"] + assert state["actual_output_weight_kind"] == "importance" + assert state["valid_mass_change_records"] == 1 + # The coverage half still measured the release frame's weights. + assert result.details["effective_mass_by_column"]["gift_aid"][ + "effective_signal_mass_share" + ] == pytest.approx(1_100.0 / 2_000.0) + + def test_binding_hands_the_spine_frame_to_the_build_state_half(self) -> None: + from microcosm.build.gate_battery import EvidenceContext + from microcosm.build.uk_runtime.battery_bindings import ( + _evaluate_release_input_coverage, + ) + + spine, release = self._frames() + artifacts = { + "coverage_engine": _StubEngine({"gift_aid": 0.0}), + "coverage_manifest": self._contract(), + "spine_frame": spine, + } + result = _evaluate_release_input_coverage( + EvidenceContext(frame=release, artifacts=artifacts), {} + ) + assert result.passed, result.failures + assert result.details["family_build_state_frame"] == "spine" diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py b/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py index 00d134dfc..e324c6afd 100644 --- a/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py @@ -136,6 +136,11 @@ def _variable(self, name: str) -> Any: raise ValueError(f"Unknown PolicyEngine-UK variable {name!r}.") return variables[name] + def enum_domain(self, name: str) -> Any: + """The Enum an engine variable's stored values must belong to (gates).""" + + return getattr(self._variable(name), "possible_values", None) + def _build_dataset(self, bundle: Frame, period: int | str) -> Any: from policyengine_uk.data import UKSingleYearDataset diff --git a/tools/certify_uk_release_cut.py b/tools/certify_uk_release_cut.py index 7e18d1d2c..11b186631 100644 --- a/tools/certify_uk_release_cut.py +++ b/tools/certify_uk_release_cut.py @@ -97,6 +97,10 @@ def main(argv: list[str] | None = None) -> int: "sha256": args.ledger_facts_sha256, "size_bytes": _ledger_facts_size(args.ledger_facts), }, + "spine_h5": { + "sha256": args.spine_sha256, + "size_bytes": args.spine_h5.stat().st_size, + }, } state = AttemptState( build_id=f"{_PIPELINE}-attempt-{started_ts.strftime('%Y%m%dT%H%M%SZ')}", @@ -173,6 +177,12 @@ def _run(args: argparse.Namespace, state: AttemptState) -> dict[str, object]: "error: --candidate-h5 sha mismatch: " f"measured {measured}, pinned {args.candidate_sha256}" ) + spine_measured = hashlib.sha256(args.spine_h5.read_bytes()).hexdigest() + if spine_measured != args.spine_sha256: + raise SystemExit( + "error: --spine-h5 sha mismatch: " + f"measured {spine_measured}, pinned {args.spine_sha256}" + ) sidecar_path = args.spine_h5.with_suffix(".build.json") if not sidecar_path.is_file(): raise SystemExit(f"error: spine build sidecar absent: {sidecar_path}") @@ -223,6 +233,9 @@ def _run(args: argparse.Namespace, state: AttemptState) -> dict[str, object]: append_phase(state, "registries_compiled") frame, _provenance = load_uk_national_frame(args.candidate_h5) + # The spine frame is the evidence for the stage families' build state + # (importance weights, mass receipts); the candidate is calibrated. + spine_frame, _spine_provenance = load_uk_national_frame(args.spine_h5) engine = PolicyEngineUKCoverageEngine() parity_evidence = uk_release_parity_evidence( frame, @@ -243,6 +256,7 @@ def _run(args: argparse.Namespace, state: AttemptState) -> dict[str, object]: fit_weight_records=rehydrate_uk_fit_weight_records(sidecar), input_mass_reference=load_uk_input_mass_reference(args.input_mass_reference), exclusions_evaluated_on=evaluated_on, + spine_frame=spine_frame, ) append_phase(state, "release_cut_gates_evaluated") for gate_id, payload in report["gates"].items(): @@ -289,6 +303,7 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: help="The dataset name the certification certifies, e.g. microcosm_uk_2024.", ) parser.add_argument("--spine-h5", required=True, type=Path) + parser.add_argument("--spine-sha256", required=True, type=_sha256) parser.add_argument("--diagnostics-json", required=True, type=Path) parser.add_argument("--build-record-json", required=True, type=Path) parser.add_argument("--seam-gate-report", required=True, type=Path) From acb0b0438f911f7dccb2ce61430bf50634bfaeb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:49:57 +0100 Subject: [PATCH 12/22] Spine builder: collect the fit-weight records through the run proxies _collect_fit_weight_records probed the stage implementation object for fit_weight_records, and the telemetry wrapper (ObservedTransform) and the graph proxies the runner binds since 2026-09-01 hid it, so every telemetry-enabled spine sidecar since then lacks the block and uk_weights_audit is evidence-absent at the release cut. The probe now walks the wrapper chain to the transform that owns the hook; the test wraps a recording transform in both proxies. Existing spines need a rebuild to carry the records. Co-Authored-By: Claude Fable 5.1 --- .../uk-spine-fit-weight-records-967.fixed.md | 1 + .../tests/test_uk_frs_spine.py | 52 +++++++++++++++++++ tools/build_uk_frs_spine.py | 33 +++++++++--- 3 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 changelog.d/uk-spine-fit-weight-records-967.fixed.md diff --git a/changelog.d/uk-spine-fit-weight-records-967.fixed.md b/changelog.d/uk-spine-fit-weight-records-967.fixed.md new file mode 100644 index 000000000..2880ea4e5 --- /dev/null +++ b/changelog.d/uk-spine-fit-weight-records-967.fixed.md @@ -0,0 +1 @@ +The UK spine builder finds each fitting stage's `fit_weight_records` through the telemetry and graph proxies a staged run wraps every stage in, so the sidecar carries the weights-audit evidence again; every telemetry-enabled spine built since the graph driver (2026-09-01) silently lacked the block and the release-cut `uk_weights_audit` gate reported it absent.\n \ No newline at end of file diff --git a/packages/microcosm-build/tests/test_uk_frs_spine.py b/packages/microcosm-build/tests/test_uk_frs_spine.py index 098ec8225..3cf68d376 100644 --- a/packages/microcosm-build/tests/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/test_uk_frs_spine.py @@ -2416,3 +2416,55 @@ def test_care_hours_map_the_hourtot_band_codes_and_refuse_unknown_codes() -> Non frs_care_hours(pd.DataFrame({"hourtot": [11]})) with pytest.raises(ValueError, match="must be integral"): frs_care_hours(pd.DataFrame({"hourtot": [2.5]})) + + +def test_collect_fit_weight_records_sees_through_the_run_proxies(): + """A staged run wraps every stage in ObservedTransform, and the graph + driver in a source transform that proxies attribute reads; the collector + must find the hook through both, or every telemetry-enabled spine loses + its weights-audit evidence (as every one since 2026-09-01 did).""" + + from microcosm.build.observation import ObservedTransform + + tool = _load_tool() + record = SimpleNamespace( + fit_name="uk_was_2018_20_wealth:savings", weight_kind="design" + ) + + class _Fit: + fit_weight_records = (record,) + + def __call__(self, frame): + return frame + + class _GraphLike: + """The graph source transform after run_with_sources: the hook + copied into its own __dict__, reads proxied to the inner stage.""" + + def __init__(self, inner): + self.transform = inner + self.fit_weight_records = inner.fit_weight_records + + def run_with_sources(self, frame, sources): + return frame + + def __getattr__(self, name): + return getattr(self.__dict__["transform"], name) + + def observed(transform, stage_id): + return ObservedTransform( + transform, stage_id=stage_id, produced_column_count=1, observer=None + ) + + implementations = { + "frs_spine": observed(lambda frame: frame, "frs_spine"), + "was_wealth": observed(_Fit(), "was_wealth"), + "etb_vat": observed(_GraphLike(_Fit()), "etb_vat"), + } + records = tool._collect_fit_weight_records( + stage_names=("frs_spine", "was_wealth", "etb_vat"), + implementations=implementations, + ) + expected = [{"fit_name": "uk_was_2018_20_wealth:savings", "weight_kind": "design"}] + assert records == {"was_wealth": expected, "etb_vat": expected} + assert "frs_spine" not in records diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index e1627525f..68255437f 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -677,6 +677,29 @@ def _collect_stage_evidence( return evidence_by_stage +def _fit_weight_hook_holder(implementation: object) -> object | None: + """The object declaring ``fit_weight_records``, seen through the proxies. + + Detects the hook without evaluating it (a raising property must count as + a fitting stage with unreadable records, not vanish), and looks through + the wrappers a staged run puts around every stage: ``ObservedTransform`` + (telemetry) and the graph's source transform both proxy attribute reads + through ``__getattr__``, which a class-level probe never consults — so + every telemetry-enabled spine since the graph driver silently lost the + block and the release-cut weights audit found no evidence. + """ + + seen: set[int] = set() + candidate: object | None = implementation + while candidate is not None and id(candidate) not in seen: + seen.add(id(candidate)) + declared = getattr(type(candidate), "fit_weight_records", None) is not None + if declared or "fit_weight_records" in getattr(candidate, "__dict__", {}): + return candidate + candidate = getattr(candidate, "__dict__", {}).get("transform") + return None + + def _collect_fit_weight_records( *, stage_names: Sequence[str], @@ -700,15 +723,11 @@ def _collect_fit_weight_records( implementation = implementations.get(stage_name) if implementation is None: continue - # Detect the hook without evaluating it: a raising property must - # count as a fitting stage with unreadable records, not vanish. - exposes_records = getattr( - type(implementation), "fit_weight_records", None - ) is not None or "fit_weight_records" in getattr(implementation, "__dict__", {}) - if not exposes_records: + holder = _fit_weight_hook_holder(implementation) + if holder is None: continue try: - records = tuple(implementation.fit_weight_records or ()) + records = tuple(holder.fit_weight_records or ()) except Exception: # noqa: BLE001 - unreadable records fail the audit records_by_stage[stage_name] = [] continue From d5e2d504dc8826f555b0418144710a57a29e0383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:04:25 +0100 Subject: [PATCH 13/22] Re-pin the two register count tests to this branch's deferral and A16 retirements CI on this branch failed two count pins that trail its own register commits. The bindings suite's shared exclusion clock (2026-09-15) sat before the v20 self-employment 20-30k deferral's approved_on (2026-09-18, 92edabdd), so uk_target_fit refused the not-yet-in-force receipt and the fully-armed battery passed 18 gates instead of 19; the clock moves to 2026-09-18, inside every register's window (earliest expiry 2026-10-16), as the terminal-gates suite already did. The UC payment registry test pinned 82 active / 18 excluded rows; the A16 retirements (4173ab38) returned the LONE_PARENT 10,800-12,000 band to the active set, so it is 83 / 17. Co-Authored-By: Claude Fable 5.1 --- .../microcosm-build/tests/test_uk_battery_bindings.py | 10 ++++++---- packages/microcosm-build/tests/test_uk_uc_family.py | 9 ++++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/packages/microcosm-build/tests/test_uk_battery_bindings.py b/packages/microcosm-build/tests/test_uk_battery_bindings.py index bdae13a59..88cdc41b9 100644 --- a/packages/microcosm-build/tests/test_uk_battery_bindings.py +++ b/packages/microcosm-build/tests/test_uk_battery_bindings.py @@ -54,10 +54,12 @@ from microcosm.frame import engine_tables KEY = base64.b64encode(b"\x07" * 32).decode("ascii") -#: The shared exclusion-expiry clock, fixed inside the committed register's -#: validity window (latest approval 2026-09-05, expires 2027-02-10) so the suite -#: never drifts across an expiry boundary. -CLOCK = date(2026, 9, 15) +#: The shared exclusion-expiry clock, fixed inside the committed registers' +#: validity window (latest approval 2026-09-18, the v20 self-employment +#: 20-30k deferral on the target-fit register; earliest expiry 2026-10-16, +#: the same entry) so the suite never drifts across an approval or expiry +#: boundary. Move it forward when a register gains a later approval. +CLOCK = date(2026, 9, 18) VALIDATE_REFERENCE = ( "microcosm.build.uk_runtime.weighted_integrity." diff --git a/packages/microcosm-build/tests/test_uk_uc_family.py b/packages/microcosm-build/tests/test_uk_uc_family.py index 88bda9de3..f2a740545 100644 --- a/packages/microcosm-build/tests/test_uk_uc_family.py +++ b/packages/microcosm-build/tests/test_uk_uc_family.py @@ -491,9 +491,12 @@ def test_full_uc_payment_registry_matches_independent_relationship_and_band_case ] full_registry = TargetRegistry(specs, country="uk") active = [spec for spec in specs if spec.name not in excluded] + # 83 active / 17 excluded since the 2026-09-21 A16 retirements: the + # LONE_PARENT 10,800-12,000 payment-band row measured inside the fence on + # the v20 national build and left the exclusion register. assert len(specs) == 100 - assert len(active) == 82 - assert len({s.name for s in specs} & excluded) == 18 + assert len(active) == 83 + assert len({s.name for s in specs} & excluded) == 17 registry = TargetRegistry(active, country="uk") result = materialize_uk_ledger_targets( adapter, registry, period=2025, band_edge_registry=full_registry @@ -503,7 +506,7 @@ def test_full_uc_payment_registry_matches_independent_relationship_and_band_case adapter.to_frame(), registry.to_target_set(), weight_entity="household" ) assert not problem.skipped - assert len(problem.names) == 82 + assert len(problem.names) == 83 assert not {f"{name}@2025" for name in excluded}.intersection(problem.names) expected_family = np.asarray(expected_families)[bu.benunit_id.to_numpy()] uc = bu.universal_credit.to_numpy() From 06d7a1f489a748d039465467c57d1882c7442746 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 22 Sep 2026 11:02:23 +0100 Subject: [PATCH 14/22] Keep the any-tenure housing row as a measured-not-fit diagnostic; file the A16 evidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2026-09-21 retirement of dwp.uc.households_housing_element read the v20 measurement (-2.8 % unbound) as noise inside the fence. It is the structural other/unknown-tenure gap the entry records (112,518 of 4,037,650, 2.79 %), which the engine's housing element cannot pay, so retiring it would put a known unreachable total back into the objective and lose the only record of why (review finding). The entry is restored verbatim with the measurement added to its reason; María's #921 reading of 2026-09-15 stands to its 2026-10-15 expiry. Register 61 -> 62 and the 2026-09-15 tranche assertions return. The retirement evidence moves into the tree: docs/evidence/uk-a16-v20/ carries the A16 rows measured on the v20 candidate (receipts.json with the candidate's digest, the output, a non-runnable copy of the script from the private evaluation stack). The input-mass exclusion changelog now records the measured numbers (charitable_investment_gifts -99.0 %, owned_land -73.5 %) and why neither signed reason is confirmed. Co-Authored-By: Claude Fable 5.1 --- ...16-housing-element-restored-967.changed.md | 1 + ...t-mass-exclusion-retirement-967.changed.md | 2 +- docs/evidence/uk-a16-v20/README.md | 27 + .../uk-a16-v20/outputs/a16_rows_on_v20.json | 616 ++++++++++++++++++ docs/evidence/uk-a16-v20/receipts.json | 34 + .../scripts/measure_a16_rows_on_candidate.py | 153 +++++ .../uk/calibration_measure_exclusions.json | 9 + .../tests/test_uk_measure_simulation.py | 27 +- 8 files changed, 857 insertions(+), 12 deletions(-) create mode 100644 changelog.d/uk-a16-housing-element-restored-967.changed.md create mode 100644 docs/evidence/uk-a16-v20/README.md create mode 100644 docs/evidence/uk-a16-v20/outputs/a16_rows_on_v20.json create mode 100644 docs/evidence/uk-a16-v20/receipts.json create mode 100644 docs/evidence/uk-a16-v20/scripts/measure_a16_rows_on_candidate.py diff --git a/changelog.d/uk-a16-housing-element-restored-967.changed.md b/changelog.d/uk-a16-housing-element-restored-967.changed.md new file mode 100644 index 000000000..a6ddcc552 --- /dev/null +++ b/changelog.d/uk-a16-housing-element-restored-967.changed.md @@ -0,0 +1 @@ +Restore the `dwp.uc.households_housing_element` entry on the UK calibration measure-exclusion register (retired on 2026-09-21 as measured inside the fence): the −2.8 % the v20 candidate measures unbound is the structural other/unknown-tenure gap the entry records (112,518 of 4,037,650), so the row stays a measured-not-fit diagnostic under María's #921 reading to its 2026-10-15 expiry. The A16 measurement receipt (`a16_rows_on_v20.json` and its script) is filed under `docs/evidence/uk-a16-v20/`. diff --git a/changelog.d/uk-input-mass-exclusion-retirement-967.changed.md b/changelog.d/uk-input-mass-exclusion-retirement-967.changed.md index ed7a2b0b1..36e07f6a1 100644 --- a/changelog.d/uk-input-mass-exclusion-retirement-967.changed.md +++ b/changelog.d/uk-input-mass-exclusion-retirement-967.changed.md @@ -1 +1 @@ -Retire the two reviewed input-mass parity exclusions on the `efrs-post-calibration` reference: the first national candidate certified under microcosm#823 measured `charitable_investment_gifts` (signed microcosm#630 for a +15,169 % drift) and `owned_land` (signed microcosm#714 for seed-to-seed realization variance, revisit registered on microcosm#145) inside the ±452 % fence, and the gate fails a stale exclusion by design. The register's `efrs-post-calibration` block is empty and the contract's input-mass evidence digest is re-pinned (`17545916…`).\n \ No newline at end of file +Retire the two reviewed input-mass exclusions the first release-cut certification (2026-09-21) reports as stale, since the register's rule fails a stale entry: on the candidate measured, `charitable_investment_gifts` carries £5.9m against the enhanced FRS reference's £597m (−99.0 %, nonzero on 0.02 % of persons) and `owned_land` £105.5bn against £397.6bn (−73.5 %), both inside the ±452 % fence. Neither signed reason is confirmed by the measurement: microcosm#630 was signed for the opposite sign (the candidate carrying the admin SPI channel the incumbent lacks, +15,169 %), and the column has instead all but emptied, a data finding raised for adjudication rather than a resolved asymmetry; microcosm#714 was signed for seed-to-seed realization variance, which a level comparison does not measure, and its revisit stays registered on microcosm#145 (the entry was expiring 2026-09-26). The `efrs-post-calibration` block is empty, the contract's input-mass evidence digest is re-pinned, and the bindings' expired-register test injects its own lapsing receipt. diff --git a/docs/evidence/uk-a16-v20/README.md b/docs/evidence/uk-a16-v20/README.md new file mode 100644 index 000000000..5b89ef66d --- /dev/null +++ b/docs/evidence/uk-a16-v20/README.md @@ -0,0 +1,27 @@ +# A16 exclusion rows measured on the v20 national candidate + +Evidence for the 2026-09-21 retirements on [PolicyEngine/microcosm#967](https://github.com/PolicyEngine/microcosm/pull/967) (commit `4173ab38`), filed on the review's ask that the retirement evidence live in the repository. `receipts.json` carries the digests of the candidate measured, the register and the two files here. + +## What was measured + +Every entry on the calibration measure-exclusion register (67 on 2026-09-21) is compiled in the contract register but pruned from the calibration surface, so the candidate never solved toward it. `scripts/measure_a16_rows_on_candidate.py` resolves and scores the full frozen contract register (705 specs) on the v20 national candidate's calibrated weights, pruning loudly only the rows whose measures cannot resolve at all, and reports each excluded row's relative error. A row inside the 25 % fence at weights that never saw it is evidence its exclusion can be retired; the retirement itself remains a signed register change. + +Result (`outputs/a16_rows_on_v20.json`): 67 excluded entries, 62 measured, 5 unresolvable on the candidate, 10 inside the 25 % fence. + +## The rows retired in `4173ab38` + +- `hmrc/employment_income_count_income_band_1_000_000_to_inf` (microcosm#757): +16.9 % unbound +- `dwp/uc_payment_dist/LONE_PARENT_annual_payment_10_800_to_12_000` (uk-data#452): -8.1 % unbound +- `hmrc/self_employment_income_count_income_band_500_000_to_1_000_000` (microcosm#757): +0.0 % unbound +- `dwp.benefit_cap.capped_households_200_01_to_300` (microcosm#882): -10.1 % unbound +- `dwp.benefit_cap.capped_households_400_01_to_500` (microcosm#882): -3.8 % unbound + +## The row measured inside the fence but kept + +- `dwp.uc.households_housing_element` (microcosm#882): -2.8 % unbound. This is not noise inside the fence: DWP's any-tenure count includes an "other or unknown" tenure the engine's housing element cannot pay, 112,518 of 4,037,650 in calendar 2025 (2.79 %), so the model's any-tenure count is identically social plus private and the measured gap is the structural gap the entry records. María signed that reading in review of microcosm#921 on 2026-09-15; `4173ab38` had retired the entry on the measurement, and this PR restores it (the entry stays a measured-not-fit diagnostic to its 2026-10-15 expiry). + +Four uk-data#452 payment bands measuring 18–25 % unbound stay excluded: binding-time competition can push a marginal cell over the fence, and the v20 self-employment 20–30k deferral on the target-fit register is that mechanism in action. + +## Provenance + +The evaluation stack is not a public repository, so the script is copied rather than linked (it is untracked there; the receipt digest is of the copy as taken, before this repository's formatter). Paths inside the output are machine-local. The candidate is the v20 national build (`microcosm_uk_2024.h5`, sha `8883e5925679…`), the same file the #968 poverty evidence measured. diff --git a/docs/evidence/uk-a16-v20/outputs/a16_rows_on_v20.json b/docs/evidence/uk-a16-v20/outputs/a16_rows_on_v20.json new file mode 100644 index 000000000..95cfeb8c7 --- /dev/null +++ b/docs/evidence/uk-a16-v20/outputs/a16_rows_on_v20.json @@ -0,0 +1,616 @@ +{ + "candidate_h5": "/Users/mariajuaristi/Desktop/PolicyEngine/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5", + "contract_register": { + "path": "/Users/mariajuaristi/Desktop/PolicyEngine/repos/uk-candidate-eval/out/frozen_scoring_register.json", + "n_specs": 705 + }, + "excluded_entries": 67, + "measured": 62, + "unresolvable_on_candidate": 5, + "within_25pct": 10, + "rows": [ + { + "name": "dwp.benefit_cap.capped_households_1000_01_to_1100", + "tracking": "microcosm#882", + "expires_on": "2026-12-08", + "relative_error": 0.35345815125125757, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp.benefit_cap.capped_households_100_01_to_200", + "tracking": "microcosm#882", + "expires_on": "2026-12-08", + "relative_error": -0.4481693760335293, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp.benefit_cap.capped_households_1100_01_to_1200", + "tracking": "microcosm#882", + "expires_on": "2026-12-08", + "relative_error": 4.715556897615132, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp.benefit_cap.capped_households_1200_01_to_1300", + "tracking": "microcosm#882", + "expires_on": "2026-12-08", + "relative_error": -1.0, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp.benefit_cap.capped_households_1300_01_and_above", + "tracking": "microcosm#882", + "expires_on": "2026-12-08", + "relative_error": 14.595738461143092, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp.benefit_cap.capped_households_200_01_to_300", + "tracking": "microcosm#882", + "expires_on": "2026-12-08", + "relative_error": -0.10144728298844963, + "unresolvable": null, + "within_25pct": true, + "within_10pct": false + }, + { + "name": "dwp.benefit_cap.capped_households_300_01_to_400", + "tracking": "microcosm#882", + "expires_on": "2026-12-08", + "relative_error": -0.5473043813418312, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp.benefit_cap.capped_households_400_01_to_500", + "tracking": "microcosm#882", + "expires_on": "2026-12-08", + "relative_error": -0.037687252078366984, + "unresolvable": null, + "within_25pct": true, + "within_10pct": true + }, + { + "name": "dwp.benefit_cap.capped_households_500_01_to_600", + "tracking": "microcosm#882", + "expires_on": "2026-12-08", + "relative_error": 0.648402512571231, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp.benefit_cap.capped_households_600_01_to_700", + "tracking": "microcosm#882", + "expires_on": "2026-12-08", + "relative_error": -0.47596102164859794, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp.benefit_cap.capped_households_700_01_to_800", + "tracking": "microcosm#882", + "expires_on": "2026-12-08", + "relative_error": 2.3904674461838074, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp.benefit_cap.capped_households_800_01_to_900", + "tracking": "microcosm#882", + "expires_on": "2026-12-08", + "relative_error": 1.9122029959437359, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp.benefit_cap.capped_households_900_01_to_1000", + "tracking": "microcosm#882", + "expires_on": "2026-12-08", + "relative_error": -1.0, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp.hb.households", + "tracking": "microcosm#867", + "expires_on": "2026-12-08", + "relative_error": -0.5623232070339688, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp.hb.households_private_rented", + "tracking": "microcosm#867", + "expires_on": "2026-12-08", + "relative_error": -0.6434371885583391, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp.hb.households_social_rented", + "tracking": "microcosm#867", + "expires_on": "2026-12-08", + "relative_error": -0.544383524266314, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp.jsa_claimants", + "tracking": "microcosm#869", + "expires_on": "2026-12-08", + "relative_error": -0.3869038162500086, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp.uc.households_housing_element", + "tracking": "microcosm#882", + "expires_on": "2026-10-15", + "relative_error": -0.028268421565201853, + "unresolvable": null, + "within_25pct": true, + "within_10pct": true + }, + { + "name": "dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_26_400_to_27_600", + "tracking": "uk-data#452", + "expires_on": "2026-11-26", + "relative_error": -0.9099841702063353, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_27_600_to_28_800", + "tracking": "microcosm#736", + "expires_on": "2026-12-08", + "relative_error": -1.0, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_28_800_to_30_000", + "tracking": "microcosm#736", + "expires_on": "2026-12-08", + "relative_error": -1.0, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp/uc_payment_dist/LONE_PARENT_annual_payment_10_800_to_12_000", + "tracking": "uk-data#452", + "expires_on": "2026-11-26", + "relative_error": -0.08142506186312688, + "unresolvable": null, + "within_25pct": true, + "within_10pct": true + }, + { + "name": "dwp/uc_payment_dist/LONE_PARENT_annual_payment_12_000_to_13_200", + "tracking": "uk-data#452", + "expires_on": "2026-11-26", + "relative_error": -0.46236892861033185, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp/uc_payment_dist/LONE_PARENT_annual_payment_13_200_to_14_400", + "tracking": "uk-data#452", + "expires_on": "2026-11-26", + "relative_error": -0.5944516357917213, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp/uc_payment_dist/LONE_PARENT_annual_payment_14_400_to_15_600", + "tracking": "uk-data#452", + "expires_on": "2026-11-26", + "relative_error": -0.33940014584956163, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp/uc_payment_dist/LONE_PARENT_annual_payment_15_600_to_16_800", + "tracking": "uk-data#452", + "expires_on": "2026-11-26", + "relative_error": -0.5269852986018834, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp/uc_payment_dist/LONE_PARENT_annual_payment_16_800_to_18_000", + "tracking": "uk-data#452", + "expires_on": "2026-11-26", + "relative_error": -0.22678277327011268, + "unresolvable": null, + "within_25pct": true, + "within_10pct": false + }, + { + "name": "dwp/uc_payment_dist/LONE_PARENT_annual_payment_18_000_to_19_200", + "tracking": "uk-data#452", + "expires_on": "2026-11-26", + "relative_error": -0.5062625691621834, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp/uc_payment_dist/LONE_PARENT_annual_payment_19_200_to_20_400", + "tracking": "uk-data#452", + "expires_on": "2026-11-26", + "relative_error": -0.5112559600656904, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp/uc_payment_dist/LONE_PARENT_annual_payment_21_600_to_22_800", + "tracking": "uk-data#452", + "expires_on": "2026-11-26", + "relative_error": -0.44527555223077536, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp/uc_payment_dist/SINGLE_annual_payment_13_200_to_14_400", + "tracking": "uk-data#452", + "expires_on": "2026-11-26", + "relative_error": -0.299546792665799, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp/uc_payment_dist/SINGLE_annual_payment_14_400_to_15_600", + "tracking": "uk-data#452", + "expires_on": "2026-11-26", + "relative_error": -0.1863542251473777, + "unresolvable": null, + "within_25pct": true, + "within_10pct": false + }, + { + "name": "dwp/uc_payment_dist/SINGLE_annual_payment_21_600_to_22_800", + "tracking": "uk-data#452", + "expires_on": "2026-11-26", + "relative_error": 1.7516978114669297, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp/uc_payment_dist/SINGLE_annual_payment_27_600_to_28_800", + "tracking": "uk-data#452", + "expires_on": "2026-11-26", + "relative_error": 6.518936973392257, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "dwp/uc_payment_dist/SINGLE_annual_payment_8_400_to_9_600", + "tracking": "uk-data#452", + "expires_on": "2026-11-26", + "relative_error": 0.22327907268654476, + "unresolvable": null, + "within_25pct": true, + "within_10pct": false + }, + { + "name": "dwp/uc_payment_dist/SINGLE_annual_payment_9_600_to_10_800", + "tracking": "uk-data#452", + "expires_on": "2026-11-26", + "relative_error": -0.24164051304899195, + "unresolvable": null, + "within_25pct": true, + "within_10pct": false + }, + { + "name": "hmrc.salary_sacrifice.it_relief_additional_rate", + "tracking": "microcosm#623", + "expires_on": "2026-11-25", + "relative_error": null, + "unresolvable": "counterfactual_delta", + "within_25pct": null, + "within_10pct": null + }, + { + "name": "hmrc.salary_sacrifice.it_relief_basic_rate", + "tracking": "microcosm#623", + "expires_on": "2026-11-25", + "relative_error": null, + "unresolvable": "counterfactual_delta", + "within_25pct": null, + "within_10pct": null + }, + { + "name": "hmrc.salary_sacrifice.it_relief_higher_rate", + "tracking": "microcosm#623", + "expires_on": "2026-11-25", + "relative_error": null, + "unresolvable": "counterfactual_delta", + "within_25pct": null, + "within_10pct": null + }, + { + "name": "hmrc.salary_sacrifice.nics_relief_employee", + "tracking": "microcosm#623", + "expires_on": "2026-11-25", + "relative_error": null, + "unresolvable": "counterfactual_delta", + "within_25pct": null, + "within_10pct": null + }, + { + "name": "hmrc.salary_sacrifice.nics_relief_employer", + "tracking": "microcosm#623", + "expires_on": "2026-11-25", + "relative_error": null, + "unresolvable": "counterfactual_delta", + "within_25pct": null, + "within_10pct": null + }, + { + "name": "hmrc/dividend_income_count_income_band_1_000_000_to_inf", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": -1.0, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "hmrc/dividend_income_income_band_1_000_000_to_inf", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": -1.0, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "hmrc/dividend_income_income_band_500_000_to_1_000_000", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": -0.4247510888323754, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "hmrc/employment_income_count_income_band_1_000_000_to_inf", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": 0.1686265697711481, + "unresolvable": null, + "within_25pct": true, + "within_10pct": false + }, + { + "name": "hmrc/employment_income_income_band_1_000_000_to_inf", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": -0.2526732563272649, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "hmrc/private_pension_income_count_income_band_1_000_000_to_inf", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": -1.0, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "hmrc/private_pension_income_income_band_1_000_000_to_inf", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": -1.0, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "hmrc/property_income_count_income_band_1_000_000_to_inf", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": -1.0, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "hmrc/property_income_count_income_band_500_000_to_1_000_000", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": 0.8612986896221454, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "hmrc/self_employment_income_count_income_band_1_000_000_to_inf", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": -0.28272758345170457, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "hmrc/self_employment_income_count_income_band_500_000_to_1_000_000", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": 0.00012426687876383464, + "unresolvable": null, + "within_25pct": true, + "within_10pct": true + }, + { + "name": "hmrc/self_employment_income_income_band_1_000_000_to_inf", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": -0.3926640883194083, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "hmrc/state_pension_count_income_band_1_000_000_to_inf", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": 11.270578982597055, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "hmrc/state_pension_income_band_1_000_000_to_inf", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": 2.31990580083485, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "mhclg.council_tax_stock.band_h@E12000001", + "tracking": "microcosm#796", + "expires_on": "2026-11-26", + "relative_error": -1.0, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "mhclg.council_tax_stock.band_h@E12000002", + "tracking": "microcosm#796", + "expires_on": "2026-11-26", + "relative_error": -1.0, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "mhclg.council_tax_stock.band_h@E12000003", + "tracking": "microcosm#796", + "expires_on": "2026-11-26", + "relative_error": -1.0, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "obr.housing_benefit", + "tracking": "microcosm#867", + "expires_on": "2026-12-08", + "relative_error": -0.6030554418834341, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "obr.universal_credit_in_cap", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": -0.9686001602613119, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "obr.universal_credit_outside_cap", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": 4.991470053051877, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "ons.savings_interest_income", + "tracking": "microcosm#866", + "expires_on": "2026-10-03", + "relative_error": -0.8212790831838676, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "slc.borrowers.plan_2_above_threshold", + "tracking": "microcosm#868", + "expires_on": "2026-10-03", + "relative_error": -0.6382088891683673, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "slc.borrowers.plan_2_liable", + "tracking": "microcosm#868", + "expires_on": "2026-10-03", + "relative_error": -0.5974809773570987, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "slc.borrowers.plan_5_liable", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": -0.830589799167799, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "slc.repayments.england_plan_5", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": -0.9410493483613644, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + }, + { + "name": "slc.repayments.england_postgraduate", + "tracking": "microcosm#757", + "expires_on": "2026-11-26", + "relative_error": -1.0, + "unresolvable": null, + "within_25pct": false, + "within_10pct": false + } + ] +} diff --git a/docs/evidence/uk-a16-v20/receipts.json b/docs/evidence/uk-a16-v20/receipts.json new file mode 100644 index 000000000..c4dfb22cc --- /dev/null +++ b/docs/evidence/uk-a16-v20/receipts.json @@ -0,0 +1,34 @@ +{ + "purpose": "Receipts for the 2026-09-21 A16 retirements on PR #967 (microcosm#796 register discipline): the calibration measure-exclusion rows measured on the v20 national candidate at calibrated weights that never solved toward them. Aggregates only; no household rows.", + "date": "2026-09-21", + "measured_by": "scripts/measure_a16_rows_on_candidate.py (non-runnable copy from the private evaluation stack, uk-candidate-eval; run from a microcosm tree with uv run --no-sync)", + "candidate": { + "label": "v20 national (spine-s, feed c5e5bf8, 1,500 epochs, family_equal, learning rate 0.02, seed 0)", + "path": "/Users/mariajuaristi/Desktop/PolicyEngine/runs/uk-623-first-calibrated/spine-assessment-v20/microcosm_uk_2024.h5", + "sha256": "8883e59256792525afcff2fe17af7f659ba3982cd0ac7a95a4005bac5ee7b389", + "bytes": 176040337, + "time_period": 2024 + }, + "contract_register": { + "path": "/Users/mariajuaristi/Desktop/PolicyEngine/repos/uk-candidate-eval/out/frozen_scoring_register.json", + "n_specs": 705, + "note": "the full compiled contract register the evaluation stack freezes for scoring (every packaged reference, exclusions included), resolved and scored on the candidate's calibrated weights" + }, + "files": { + "outputs/a16_rows_on_v20.json": { + "sha256": "ef8836a7deb4b82ca9d83e0d1d50b8d1ab86edf095b082e0f0593de8eac3468a", + "bytes": 17147 + }, + "scripts/measure_a16_rows_on_candidate.py": { + "sha256": "a088459fd4d12b100c7e74b198175c4c8441493371d4dab638df0388084bd2fb", + "bytes": 5884, + "note": "digest of the copy as taken from the evaluation stack, before the repository's formatter" + } + }, + "summary": { + "excluded_entries": 67, + "measured": 62, + "unresolvable_on_candidate": 5, + "within_25pct": 10 + } +} diff --git a/docs/evidence/uk-a16-v20/scripts/measure_a16_rows_on_candidate.py b/docs/evidence/uk-a16-v20/scripts/measure_a16_rows_on_candidate.py new file mode 100644 index 000000000..2f2d01992 --- /dev/null +++ b/docs/evidence/uk-a16-v20/scripts/measure_a16_rows_on_candidate.py @@ -0,0 +1,153 @@ +"""Measure the A16-excluded rows on a calibrated national candidate. + +Retirement evidence for the calibration measure-exclusion register: each +excluded reference is compiled in the contract register but pruned from the +calibration surface, so the candidate never solved toward it. This tool +resolves and scores the FULL contract register on the candidate's calibrated +weights (pruning, loudly, only the rows whose measures cannot resolve at all +— the signed unresolvable-counterfactual entries) and reports every excluded +row's relative error. A row inside the 25 % fence at weights that never saw +it is evidence its exclusion can be retired; the retirement itself remains a +signed register change. + +Run from the driver tree: cd && uv run --no-sync python \ + .../measure_a16_rows_on_candidate.py --tree ... +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +_FAILED_MEASURE = re.compile( + r"provider (?:failed computing|does not know) ([a-z_]+)\.([A-Za-z0-9_]+)" +) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tree", required=True, type=Path) + parser.add_argument("--candidate-h5", required=True, type=Path) + parser.add_argument("--contract-json", required=True, type=Path) + parser.add_argument("--out", required=True, type=Path) + parser.add_argument("--calibration-year", type=int, default=2025) + parser.add_argument("--max-prune-rounds", type=int, default=10) + args = parser.parse_args(argv) + + sys.path.insert(0, str(args.tree / "tools")) + import score_uk_national_candidate as scorer # noqa: E402 + + from microcosm.build.target_materialization import ( # noqa: E402 + MeasureResolutionError, + ) + from microcosm.build.uk_runtime.measure_simulation import ( # noqa: E402 + load_uk_calibration_measure_exclusions, + ) + from microcosm.calibrate import ( # noqa: E402 + TargetRegistry, + score_targets, + ) + + contract = TargetRegistry.from_json(args.contract_json) + exclusions = load_uk_calibration_measure_exclusions() + excluded = {e["name"]: e for e in exclusions} + year = int(args.calibration_year) + factory = scorer._default_measure_resolver_factory(args.out.parent, year) + + registry = contract + pruned: dict[str, str] = {} + for _round in range(args.max_prune_rounds): + try: + frame, _resolution = scorer._scored_frame( + args.candidate_h5, + registry, + year, + factory, + band_edge_registry=contract, + ) + except MeasureResolutionError as error: + receipt = getattr(error, "receipt", None) or {} + match = _FAILED_MEASURE.search(str(error)) + if match is not None: + variable = match.group(2) + label = f"{match.group(1)}.{variable}" + names = { + str(skip.get("name")) + for skip in receipt.get("skips", []) + if variable in str(skip.get("reason", "")) + or variable == str(skip.get("measure", "")) + } + elif "counterfactual" in str(error): + label = "counterfactual_delta" + names = { + str(skip.get("name")) + for skip in receipt.get("skips", []) + if "counterfactual" in str(skip.get("reason", "")) + } + else: + raise + drop = [s for s in registry.specs if s.name in names] + if not drop: + raise + for s in drop: + pruned[s.name] = label + registry = TargetRegistry( + [s for s in registry.specs if s.name not in pruned], + country=registry.country, + ) + print(f"prune: {label} unresolvable -> dropped {len(drop)}", flush=True) + continue + break + else: + raise RuntimeError("resolution still failing after max rounds") + + result = score_targets(frame, registry.to_target_set()) + errors = {d.name: float(d.relative_error) for d in result.diagnostics} + # The exclusion register keys by spec name; scoring diagnostics key by the + # compiled row name. Bridge through the contract register. + row_name = {s.name: s.to_target().row_name for s in contract.specs} + rows = [] + for name, entry in sorted(excluded.items()): + rel = errors.get(row_name.get(name, name)) + rows.append( + { + "name": name, + "tracking": entry["tracking"], + "expires_on": entry["expires_on"], + "relative_error": rel, + "unresolvable": pruned.get(name), + "within_25pct": None if rel is None else abs(rel) <= 0.25, + "within_10pct": None if rel is None else abs(rel) <= 0.10, + } + ) + measured = [r for r in rows if r["relative_error"] is not None] + within = [r for r in measured if r["within_25pct"]] + payload = { + "candidate_h5": str(args.candidate_h5), + "contract_register": { + "path": str(args.contract_json), + "n_specs": len(contract.specs), + }, + "excluded_entries": len(excluded), + "measured": len(measured), + "unresolvable_on_candidate": len(rows) - len(measured), + "within_25pct": len(within), + "rows": rows, + } + args.out.write_text(json.dumps(payload, indent=1) + "\n", encoding="utf-8") + print( + f"wrote {args.out}: {len(excluded)} excluded entries, {len(measured)} " + f"measured on the candidate, {len(within)} within 25%" + ) + for r in sorted(within, key=lambda r: abs(r["relative_error"])): + print( + f" RETIRABLE? {r['name']:64s} {100 * r['relative_error']:+7.1f}% [{r['tracking']}]" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json b/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json index 3028c1384..af1ed8e8a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json +++ b/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json @@ -424,6 +424,15 @@ "approved_on": "2026-09-14", "expires_on": "2026-11-26" }, + { + "name": "dwp.uc.households_housing_element", + "reason": "Structural bias, not a weight question: DWP's any-tenure housing-entitlement count is social plus private plus 'Other or unknown' tenure, and the model has no other/unknown tenure (uc_housing_costs_element pays only council, housing-association and private renters), so the model's any-tenure count is identically the social plus private rows, which are bound and fit; the row overstates the reachable total by the other/unknown cell, 112,518 of 4,037,650 in calendar 2025 (2.8 percent; the published any-tenure mean, which the three tenure cells reproduce to the unit). Fitting all three housing rows would ask the solver to over-fill the tenure rows by that amount. Measured on every evaluation as the housing-element caseload diagnostic (Q50f -0.8 percent, D2 +0.6 percent at the baseline; v20 -2.8 percent unbound, the structural gap itself, docs/evidence/uk-a16-v20/).", + "tracking": "microcosm#882", + "approved_by": "juaristi22", + "adjudication": "microcosm#882 (element sizing at the 10 September 2026 baseline, docs/uk-uc-baseline-2026-09-10.md; María's sign-off in review of microcosm#921 on 2026-09-15, setting the one-month window)", + "approved_on": "2026-09-15", + "expires_on": "2026-10-15" + }, { "name": "dwp.hb.households", "reason": "Housing Benefit caseload diagnostic (all client types and tenures, calendar-2025 mean of the Stat-Xplore Total items), measured on every evaluation but held out of the objective (#882, microcosm#867): the frame cannot reach the DWP caseload for the mechanisms signed on obr.housing_benefit (SPI-channel reporters in owner-occupied synthetic households, reporters drawn into Universal Credit, head-only rent, the means test). Q50f pays 235k GB units against a calendar-2025 published mean near 1.70m (-86 percent). Fitting it would move weight onto the few reporting units the engine pays. Retire when the row sits inside the 25 percent bound at the baseline.", diff --git a/packages/microcosm-build/tests/test_uk_measure_simulation.py b/packages/microcosm-build/tests/test_uk_measure_simulation.py index 11789e611..964a79d96 100644 --- a/packages/microcosm-build/tests/test_uk_measure_simulation.py +++ b/packages/microcosm-build/tests/test_uk_measure_simulation.py @@ -592,10 +592,12 @@ def test_exclusion_applier_warns_within_week_of_expiry(): # microcosm#882 (2026-09-15): three UC element rows were held out of the # objective, signed by María in review of microcosm#921 with a one-month # window. The carer and childcare rows retired on 2026-09-16 with their - # repairs; the any-tenure housing row retired on 2026-09-21 on the v20 - # measurement (the signed structural bias measures -2.8 % at calibrated - # weights that never solved toward the row, inside the 25 % fence). - "dwp.uc.households_": 0, + # repairs; the any-tenure housing row stays held out as the signed + # structural bias: the v20 measurement (-2.8 % unbound, at weights that + # never solved toward the row) reproduces the 2.79 % other/unknown-tenure + # gap the entry records, so it is a diagnostic, not a fit target + # (docs/evidence/uk-a16-v20/). + "dwp.uc.households_": 1, # microcosm#882 repairs (2026-09-16): the three Housing Benefit caseload # rows and the thirteen benefit-cap amount bands outside the 25 percent # bound are measured on every evaluation but held out of the objective; @@ -608,9 +610,9 @@ def test_exclusion_applier_warns_within_week_of_expiry(): } # The carer and childcare rows were retired on 2026-09-16 with the repairs -# (care hours and the childcare take-up draw on policyengine-uk 2.98.0); the -# any-tenure housing row retired on 2026-09-21 on the v20 measurement. -_UC_ELEMENT_REGISTER_ROWS = () +# (care hours and the childcare take-up draw on policyengine-uk 2.98.0); +# only the any-tenure housing row remains held out (structural bias). +_UC_ELEMENT_REGISTER_ROWS = ("dwp.uc.households_housing_element",) _A16_UNREACHABLE_ROWS = ( "ons.savings_interest_income", @@ -625,7 +627,7 @@ def test_exclusion_applier_warns_within_week_of_expiry(): def test_packaged_exclusions_load(): exclusions = load_uk_calibration_measure_exclusions() names = [entry["name"] for entry in exclusions] - assert len(names) == len(set(names)) == 61 + assert len(names) == len(set(names)) == 62 band_h_region_cells = [ entry for entry in exclusions @@ -742,13 +744,16 @@ def test_packaged_exclusions_load(): assert "docs/uk-uc-baseline-2026-09-10.md" in entry["adjudication"], entry[ "name" ] - # Every UC element row now rides in the objective; none is on the register - # (the any-tenure housing row retired 2026-09-21 on the v20 measurement). + housing = next( + e for e in elements if e["name"] == "dwp.uc.households_housing_element" + ) + assert "112,518 of 4,037,650" in housing["reason"] + # The other UC element rows ride in the objective and are not on the + # register. for riding in ( "dwp.uc.households_carer_element", "dwp.uc.households_childcare_element", "dwp.uc.households_lcwra_element", - "dwp.uc.households_housing_element", "dwp.uc.households_housing_element_social_rented", "dwp.uc.households_housing_element_private_rented", "dwp.uc.households_with_deduction", From 2bbd93ce048c288ff1ab804ad8e61bcd15471e26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 22 Sep 2026 11:02:23 +0100 Subject: [PATCH 15/22] Test the scoring route on a synthetic cross-grain surface prepare_uk_target_frame had no test, and the defect it fixed (every score of a candidate export refused since the region fan-out, because one injected input name at two entities met the prepared frame's column-uniqueness rule) went unseen. The fixture injects one input at the person and the household entity: both targets materialize, the injected inputs are dropped from both tables, the same injection without the drop is refused, and a spy asserts the band-edge register reaches materialization (#803). Co-Authored-By: Claude Fable 5.1 --- .../uk-scoring-route-test-967.added.md | 1 + .../tests/test_uk_scoring_route.py | 192 ++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 changelog.d/uk-scoring-route-test-967.added.md create mode 100644 packages/microcosm-build/tests/test_uk_scoring_route.py diff --git a/changelog.d/uk-scoring-route-test-967.added.md b/changelog.d/uk-scoring-route-test-967.added.md new file mode 100644 index 000000000..6289de564 --- /dev/null +++ b/changelog.d/uk-scoring-route-test-967.added.md @@ -0,0 +1 @@ +`test_uk_scoring_route.py` drives `prepare_uk_target_frame` on a synthetic cross-grain surface (one input name injected at the person and the household entity): both targets materialize, the injected inputs are dropped from both tables, the same injection without the drop is refused by the prepared frame, and the band-edge register reaches materialization. diff --git a/packages/microcosm-build/tests/test_uk_scoring_route.py b/packages/microcosm-build/tests/test_uk_scoring_route.py new file mode 100644 index 000000000..4379a46c1 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_scoring_route.py @@ -0,0 +1,192 @@ +"""The scoring route through ``prepare_uk_target_frame`` on a synthetic surface. + +The route is the calibration stage's resolve-inject-materialize path without +the solve. Since the region fan-out, cross-grain measures inject one input +name at two entities, which the prepared frame's global-column-uniqueness +rule refuses unless the injected inputs are dropped after materialization; +every score of a candidate export was refused that way and nothing in the +suite caught it (review finding on PR #967). This fixture is that case. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime import national_calibration +from microcosm.build.uk_runtime.national_calibration import ( + CalibrationFrameAdapter, + inject_measure_inputs, + prepare_uk_target_frame, +) +from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.frame import WeightKind + +PEOPLE = "test/people_in_scope" +HOUSEHOLDS = "test/households_in_scope" + + +class _CrossGrainResolver: + """One input name, ``in_scope``, resolved at the person and the household + entity: the shape the region fan-out produces.""" + + contract_targets = { + "test.people_in_scope": { + "target_id": "test.people_in_scope", + "family": "test", + "bindings": { + "policyengine": { + "metric_name": PEOPLE, + "from_entity": "person", + "value_variable": "in_scope", + } + }, + }, + "test.households_in_scope": { + "target_id": "test.households_in_scope", + "family": "test", + "bindings": { + "policyengine": { + "metric_name": HOUSEHOLDS, + "from_entity": "household", + "value_variable": "in_scope", + } + }, + }, + } + + def __init__(self) -> None: + self.computed: list[tuple[str, str]] = [] + + def knows(self, entity: str, variable: str) -> bool: + return variable == "in_scope" and entity in {"person", "household"} + + def compute(self, entity: str, variable: str): + self.computed.append((entity, variable)) + if entity == "person": + return np.array([1.0, 0.0, 1.0, 1.0]), "stub_person" + return np.array([1.0, 0.0]), "stub_household" + + def receipt(self): + return {"provider": "stub"} + + +def _frame(): + return uk_national_frame( + person=pd.DataFrame( + { + "person_id": [11, 12, 21, 22], + "person_benunit_id": [1, 1, 2, 2], + "person_household_id": [1, 1, 2, 2], + } + ), + benunit=pd.DataFrame({"benunit_id": [1, 2]}), + household=pd.DataFrame({"household_id": [1, 2]}), + time_period="2025", + weight_kind=WeightKind.CALIBRATED, + household_weights=np.array([10.0, 20.0]), + ) + + +@pytest.fixture(autouse=True) +def _synthetic_contract(monkeypatch): + """The materializer reads the packaged contract; this surface is synthetic.""" + from microcosm.build.uk_runtime import ledger_targets + + monkeypatch.setattr( + ledger_targets, + "_uk_contract_targets", + lambda: dict(_CrossGrainResolver.contract_targets), + ) + + +def _registry() -> TargetRegistry: + return TargetRegistry( + [ + TargetSpec( + name="test.people_in_scope", + entity="person", + measure=PEOPLE, + value=50.0, + period=2025, + source="synthetic", + family="test", + metadata={"contract_target_id": "test.people_in_scope"}, + ), + TargetSpec( + name="test.households_in_scope", + entity="household", + measure=HOUSEHOLDS, + value=10.0, + period=2025, + source="synthetic", + family="test", + metadata={"contract_target_id": "test.households_in_scope"}, + ), + ], + country="uk", + ) + + +def test_cross_grain_measures_materialize_and_the_injected_inputs_are_dropped(): + resolver = _CrossGrainResolver() + + prepared, receipt = prepare_uk_target_frame( + _frame(), _registry(), period=2025, measure_resolver=resolver + ) + + assert sorted(resolver.computed) == [ + ("household", "in_scope"), + ("person", "in_scope"), + ] + person = prepared.table("person") + household = prepared.table("household") + assert person[PEOPLE].tolist() == [1.0, 0.0, 1.0, 1.0] + assert household[HOUSEHOLDS].tolist() == [1.0, 0.0] + # The injected inputs are gone from both tables, as the stage drops them. + assert "in_scope" not in person.columns + assert "in_scope" not in household.columns + assert receipt is not None + assert set(receipt["attached"]) == {"person.in_scope", "household.in_scope"} + + +def test_the_prepared_frame_refuses_the_injected_inputs_if_they_stayed(): + """The defect the route fixes: one name on two tables is refused.""" + adapter = CalibrationFrameAdapter(_frame()) + inject_measure_inputs( + adapter, + { + ("person", "in_scope"): np.array([1.0, 0.0, 1.0, 1.0]), + ("household", "in_scope"): np.array([1.0, 0.0]), + }, + ) + with pytest.raises(ValueError, match="in_scope"): + adapter.prepared_frame() + + +def test_the_band_edge_register_reaches_materialization(monkeypatch): + """A pruned scoring surface never redraws its own band edges (#803).""" + seen: list[object] = [] + original = national_calibration.materialize_uk_ledger_targets + + def spy(adapter, registry, *, period, band_edge_registry=None): + seen.append(band_edge_registry) + return original( + adapter, registry, period=period, band_edge_registry=band_edge_registry + ) + + monkeypatch.setattr(national_calibration, "materialize_uk_ledger_targets", spy) + full = _registry() + surface = TargetRegistry([full.specs[0]], country="uk") + + prepare_uk_target_frame( + _frame(), + surface, + period=2025, + measure_resolver=_CrossGrainResolver(), + band_edge_registry=full, + ) + + assert seen and all(edges is full for edges in seen) From 4d686cfe3f35f2998acf13aa4e05f05a9d8f8247 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 22 Sep 2026 11:02:23 +0100 Subject: [PATCH 16/22] Prune from both arms only under a reviewed incumbent-unresolvable register MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accepting any n_pruned as long as the surface closed made the scorer's inference the doctrine (review finding: at v21 the 120 pruned rows are exactly the candidate's admin-basis differentiators). Pruning now stands on uk/incumbent_unresolvable_measures.json (reviewed-exclusion schema, keyed entity.variable): the scorer prunes a measure only under a signed, in-force entry and refuses an unlisted one; the receipt records the register digest and the entry each pruned row stands on; the certifier re-checks every pruned measure against the committed register. Skip rows match the failing measure exactly ('entity.variable'), never by substring. The register ships empty: the six measures the enhanced FRS never carried are María's to sign, and until then the evaluation refuses on the v21 surface. Co-Authored-By: Claude Fable 5.1 --- ...mbent-unresolvable-register-967.changed.md | 1 + docs/uk-national-calibration-runbook-623.md | 2 +- .../microcosm/build/uk/country_package.json | 5 + .../uk/incumbent_unresolvable_measures.json | 5 + .../build/uk_runtime/candidate_score.py | 183 ++++++++++++++++-- .../build/uk_runtime/release_certification.py | 37 +++- .../tests/test_country_spec.py | 2 + .../tests/test_uk_candidate_score.py | 117 +++++++++++ .../tests/test_uk_release_certification.py | 36 +++- .../tests/uk_certification_fixtures.py | 17 +- 10 files changed, 378 insertions(+), 27 deletions(-) create mode 100644 changelog.d/uk-incumbent-unresolvable-register-967.changed.md create mode 100644 packages/microcosm-build/src/microcosm/build/uk/incumbent_unresolvable_measures.json diff --git a/changelog.d/uk-incumbent-unresolvable-register-967.changed.md b/changelog.d/uk-incumbent-unresolvable-register-967.changed.md new file mode 100644 index 000000000..22fc1d493 --- /dev/null +++ b/changelog.d/uk-incumbent-unresolvable-register-967.changed.md @@ -0,0 +1 @@ +Pruning a target from both arms of the UK rule-1 evaluation now stands on a reviewed register, `uk/incumbent_unresolvable_measures.json` (reviewed-exclusion schema, keyed `entity.variable`): the scorer prunes a measure only under a signed, in-force entry and refuses an unlisted one, the receipt records the register digest and the entries each pruned row stands on, and the release-cut certifier re-checks every pruned measure against the committed register. Skip rows are matched to the failing measure exactly (`'entity.variable'`), never by substring. The register ships empty: the six measures the enhanced FRS is known not to carry are María's to sign. diff --git a/docs/uk-national-calibration-runbook-623.md b/docs/uk-national-calibration-runbook-623.md index dfbd9f447..e1c142f46 100644 --- a/docs/uk-national-calibration-runbook-623.md +++ b/docs/uk-national-calibration-runbook-623.md @@ -95,7 +95,7 @@ it removed, and the receipt carries them under `pruned_targets` with each one's family and measure, `measures`, `families`). The score stands on the common surface with band edges from the full register (#803). A measure the *candidate* cannot materialize still refuses: that is a defect. -`--no-prune-incumbent-unresolvable` restores the refusal on the incumbent +`--no-prune-incumbent-unresolvable` restores the refusal on the incumbent A measure may be pruned only under a signed, in-force entry of the reviewed incumbent-unresolvable register (`packages/microcosm-build/src/microcosm/build/uk/incumbent_unresolvable_measures.json`, keyed `entity.variable`, reviewed-exclusion schema); an unlisted measure refuses the evaluation, the receipt records the register digest and the entries used, and the certifier re-checks every pruned measure against the committed register. side too. The receipt's `evaluation` block decides rule 1 (#578) on that surface: `verdict` is `passed` when the candidate's full loss is below the incumbent's, `failed` otherwise, and the release-cut certifier refuses any diff --git a/packages/microcosm-build/src/microcosm/build/uk/country_package.json b/packages/microcosm-build/src/microcosm/build/uk/country_package.json index 8211cc0ae..de9dd56ef 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/uk/country_package.json @@ -192,6 +192,11 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, + { + "path": "incumbent_unresolvable_measures.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, { "path": "spine_swap_signed_differences.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/uk/incumbent_unresolvable_measures.json b/packages/microcosm-build/src/microcosm/build/uk/incumbent_unresolvable_measures.json new file mode 100644 index 000000000..7c5754973 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/incumbent_unresolvable_measures.json @@ -0,0 +1,5 @@ +{ + "schema_version": 2, + "description": "Measures the incumbent comparison artifact (the enhanced FRS) is known not to carry, keyed entity.variable. A target bound through a listed measure may be pruned from BOTH arms of the rule-1 evaluation (microcosm#578 rule 1 on the common surface, microcosm#823); an unlisted unresolvable measure refuses the evaluation rather than pruning. Every entry is a complete approval receipt in the reviewed-exclusion schema, in force from approved_on through expires_on.", + "exclusions": {} +} diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py index 4aa5e403f..b570da421 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py @@ -22,12 +22,18 @@ import re from collections.abc import Callable, Mapping from dataclasses import asdict, dataclass +from datetime import date from pathlib import Path from typing import Any from microcosm.build.target_materialization import MeasureResolutionError from microcosm.build.uk_runtime.national_calibration import prepare_uk_target_frame from microcosm.build.uk_runtime.national_frame import load_uk_national_frame +from microcosm.build.uk_runtime.weighted_integrity import ( + UKReviewedExclusion, + exclusion_evaluation_date, + load_uk_reviewed_exclusion_register, +) from microcosm.calibrate import TargetRegistry, score_targets UK_SCORE_LOSS_CAP = 10.0 @@ -319,18 +325,109 @@ def _target_drift( "microcosm#578 rule 1: the candidate beats the incumbent on the frozen " "comparison register, both artifacts rescored on the surface both can " "materialize; rows the incumbent cannot materialize are pruned from both " - "arms and listed, never a failure." + "arms under the reviewed incumbent-unresolvable register and listed." ) UK_EVALUATION_SCHEMA_VERSION = 1 UK_EVALUATION_VERDICT_PASSED = "passed" UK_EVALUATION_VERDICT_FAILED = "failed" INCUMBENT_UNRESOLVABLE_NOTE = ( "Rows pruned from BOTH arms because their measures cannot be computed on " - "the incumbent (inputs the incumbent artifact does not carry). The " + "the incumbent (inputs the incumbent artifact does not carry), each under " + "a signed entry of the reviewed incumbent-unresolvable register. The " "candidate's fit on these rows is measured by the calibration diagnostics " "and the terminal gates instead." ) +#: The reviewed register of measures the incumbent is known not to carry +#: (``microcosm.build.uk`` resource, reviewed-exclusion schema 2, keyed +#: ``entity.variable``). Pruning a target from both arms is a doctrine +#: decision, not a scorer's inference: only a measure with a signed, in-force +#: entry may be pruned, and an unlisted one refuses the evaluation. +UK_INCUMBENT_UNRESOLVABLE_REGISTER = "incumbent_unresolvable_measures.json" + +#: A materialization skip's reason names the missing column as +#: ``'entity.variable'`` (or ``'variable'``); the prune loop matches that +#: exactly, never by substring, so ``measure_b`` cannot claim ``measure_bb``. +_SKIP_COLUMN = re.compile(r"^'(?:([a-z_0-9]+)\.)?([A-Za-z_0-9]+)'$") + + +def load_uk_incumbent_unresolvable_measures( + source: str | Path | None = None, +) -> dict[str, UKReviewedExclusion]: + """The reviewed incumbent-unresolvable register (packaged unless overridden).""" + + return load_uk_reviewed_exclusion_register( + source, resource=UK_INCUMBENT_UNRESOLVABLE_REGISTER + ) + + +def uk_incumbent_unresolvable_register_digest( + source: str | Path | None = None, +) -> dict[str, str]: + """Name and digest of the register a receipt was evaluated under.""" + + if source is None: + from importlib import resources as importlib_resources + + raw = ( + importlib_resources.files("microcosm.build.uk") + .joinpath(UK_INCUMBENT_UNRESOLVABLE_REGISTER) + .read_bytes() + ) + name = UK_INCUMBENT_UNRESOLVABLE_REGISTER + else: + raw = Path(source).read_bytes() + name = str(source) + return {"resource": name, "sha256": hashlib.sha256(raw).hexdigest()} + + +def _skip_names_the_measure( + skip: Mapping[str, Any], entity: str, variable: str +) -> bool: + if variable == str(skip.get("measure", "")): + return True + match = _SKIP_COLUMN.match(str(skip.get("reason", "")).strip()) + if match is None: + return False + skip_entity, skip_variable = match.group(1), match.group(2) + return skip_variable == variable and skip_entity in (None, entity) + + +def _require_reviewed_unresolvable( + reviewed: Mapping[str, UKReviewedExclusion], + entity: str, + variable: str, + *, + evaluated_on: date, +) -> UKReviewedExclusion: + key = f"{entity}.{variable}" + record = reviewed.get(key) + if record is None: + raise MeasureResolutionError( + f"{key} is unresolvable on the incumbent but is not on the reviewed " + f"incumbent-unresolvable register ({UK_INCUMBENT_UNRESOLVABLE_REGISTER}); " + "a target may be pruned from both arms only under a signed entry, " + "so the evaluation refuses rather than pruning.", + receipt={}, + ) + if record.premature(evaluated_on): + raise MeasureResolutionError( + f"the reviewed incumbent-unresolvable entry for {key} is not yet in " + f"force (takes force {record.approved_on}, evaluated on " + f"{evaluated_on.isoformat()}); correct the receipt's approved_on or " + "wait for it.", + receipt={}, + ) + if record.expired(evaluated_on): + raise MeasureResolutionError( + f"the reviewed incumbent-unresolvable entry for {key} expired " + f"{record.expires_on} (evaluated on {evaluated_on.isoformat()}); renew " + "the adjudication or remove the entry.", + receipt={}, + ) + return record + + _FAILED_MEASURE = re.compile( r"provider (?:failed computing|does not know) ([a-z_0-9]+)\.([A-Za-z0-9_]+)" ) @@ -347,6 +444,7 @@ class PrunedTarget: family: str unresolvable_measure: str reason: str + adjudication: str = "" def _failing_measure(error: MeasureResolutionError) -> tuple[str, str] | None: @@ -384,19 +482,31 @@ def prune_incumbent_unresolvable( band_edge_registry: TargetRegistry | None = None, contract_targets: Mapping[str, Any] | None = None, max_rounds: int = 10, + reviewed_measures: Mapping[str, UKReviewedExclusion] | None = None, + evaluated_on: date | None = None, ) -> tuple[TargetRegistry, dict[str, PrunedTarget]]: """Drop every target whose measure the incumbent cannot materialize. Probes the incumbent's resolution on the scoring surface; on each - :class:`MeasureResolutionError` naming a measure, drops the targets the - resolution receipt's skip rows name for it, else every target whose + :class:`MeasureResolutionError` naming a measure, requires a signed, + in-force entry for that measure on the reviewed incumbent-unresolvable + register (``reviewed_measures``, the packaged register by default, + evaluated on ``evaluated_on``, today by default), then drops the targets + the resolution receipt's skip rows name for it, else every target whose contract binding gates or values through it, and records each drop with - the failing measure. A failure that names no target refuses rather than - pruning blindly. Raw-column scoring (no resolver) has nothing to probe. + the failing measure and the entry's adjudication. An unlisted measure + refuses; a failure that names no target refuses rather than pruning + blindly. Raw-column scoring (no resolver) has nothing to probe. """ if measure_resolver_factory is None: return registry, {} + reviewed = ( + load_uk_incumbent_unresolvable_measures() + if reviewed_measures is None + else reviewed_measures + ) + now = exclusion_evaluation_date(evaluated_on) pruned: dict[str, PrunedTarget] = {} surface = registry # The probes stand on the full register's band edges too (#803): a @@ -416,16 +526,16 @@ def prune_incumbent_unresolvable( if failing is None: raise entity, variable = failing + record = _require_reviewed_unresolvable( + reviewed, entity, variable, evaluated_on=now + ) receipt = getattr(error, "receipt", None) or {} skips = receipt.get("skips", []) if isinstance(receipt, Mapping) else [] skip_names = { str(skip.get("name")) for skip in skips if isinstance(skip, Mapping) - and ( - variable == str(skip.get("measure", "")) - or variable in str(skip.get("reason", "")) - ) + and _skip_names_the_measure(skip, entity, variable) } dropped = [spec for spec in surface.specs if spec.name in skip_names] if not dropped: @@ -457,6 +567,7 @@ def prune_incumbent_unresolvable( family=spec.family, unresolvable_measure=f"{entity}.{variable}", reason=reason, + adjudication=record.adjudication, ) surface = TargetRegistry( [spec for spec in surface.specs if spec.name not in pruned], @@ -476,11 +587,23 @@ def prune_incumbent_unresolvable( def pruned_block( - pruned: Mapping[str, PrunedTarget], *, n_scored: int, n_surface: int + pruned: Mapping[str, PrunedTarget], + *, + n_scored: int, + n_surface: int, + reviewed_measures: Mapping[str, UKReviewedExclusion] | None = None, + register: Mapping[str, str] | None = None, ) -> dict[str, Any]: families: dict[str, int] = {} for row in pruned.values(): families[row.family] = families.get(row.family, 0) + 1 + measures = sorted({row.unresolvable_measure for row in pruned.values()}) + reviewed = {} if reviewed_measures is None else reviewed_measures + entries_used = { + measure: reviewed[measure].policy_payload() + for measure in measures + if measure in reviewed + } return { "n_pruned": len(pruned), "n_scored": int(n_scored), @@ -489,8 +612,16 @@ def pruned_block( # content policy refuses row-level collections by key name, and this # receipt rides the run as a reviewed aggregate artifact. "pruned_targets": {name: asdict(row) for name, row in sorted(pruned.items())}, - "measures": sorted({row.unresolvable_measure for row in pruned.values()}), + "measures": measures, "families": dict(sorted(families.items())), + "reviewed_register": { + **( + {"resource": UK_INCUMBENT_UNRESOLVABLE_REGISTER, "sha256": None} + if register is None + else dict(register) + ), + "entries_used": entries_used, + }, "note": INCUMBENT_UNRESOLVABLE_NOTE, } @@ -570,17 +701,31 @@ def evaluate_uk_candidate_against_incumbent( band_edge_registry: TargetRegistry | None = None, prune_incumbent_unresolvable_measures: bool = True, contract_targets: Mapping[str, Any] | None = None, + reviewed_measures: Mapping[str, UKReviewedExclusion] | None = None, + evaluated_on: date | None = None, ) -> dict[str, Any]: """Score on the common resolvable surface and decide the rule-1 verdict. Returns the score block with ``incumbent_unresolvable_pruned`` (every - row dropped, its family and the absent measure) and ``evaluation`` - (rule, counts, rule-1 losses and wins, ``verdict``). With pruning off the - strict scorer's refusal stands. + row dropped, its family, the absent measure and the register entry it + was pruned under) and ``evaluation`` (rule, counts, rule-1 losses and + wins, ``verdict``). Pruning stands on the reviewed incumbent-unresolvable + register (the packaged one unless ``reviewed_measures`` is supplied); + with pruning off the strict scorer's refusal stands. """ surface = target_registry pruned: dict[str, PrunedTarget] = {} + register_digest = ( + uk_incumbent_unresolvable_register_digest() + if reviewed_measures is None + else None + ) + reviewed = ( + load_uk_incumbent_unresolvable_measures() + if reviewed_measures is None + else reviewed_measures + ) if prune_incumbent_unresolvable_measures: surface, pruned = prune_incumbent_unresolvable( incumbent_h5, @@ -589,6 +734,8 @@ def evaluate_uk_candidate_against_incumbent( measure_resolver_factory, band_edge_registry=band_edge_registry, contract_targets=contract_targets, + reviewed_measures=reviewed, + evaluated_on=evaluated_on, ) # A pruned surface must never redraw its own band edges (#803): the # edges come from the full register when the caller supplied none. @@ -608,7 +755,11 @@ def evaluate_uk_candidate_against_incumbent( band_edge_registry=edges, ) score["incumbent_unresolvable_pruned"] = pruned_block( - pruned, n_scored=len(surface.specs), n_surface=len(target_registry.specs) + pruned, + n_scored=len(surface.specs), + n_surface=len(target_registry.specs), + reviewed_measures=reviewed, + register=register_digest, ) score["evaluation"] = evaluation_block(score) return score diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py index b0d8398da..2cf821295 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py @@ -348,6 +348,7 @@ def compose_uk_release_certification( score_receipt_path: Path, exclusions_evaluated_on: date, certification_path: Path, + reviewed_unresolvable_measures: Mapping[str, Any] | None = None, ) -> dict[str, Any]: """Verify the three scoped parts and compose the signed certification. @@ -418,7 +419,11 @@ def compose_uk_release_certification( ) score_receipt_bytes = score_receipt_path.read_bytes() score_receipt = json.loads(score_receipt_bytes) - _verify_score_receipt(score_receipt, candidate_sha256=candidate_sha256) + _verify_score_receipt( + score_receipt, + candidate_sha256=candidate_sha256, + reviewed_unresolvable_measures=reviewed_unresolvable_measures, + ) full_digests = _full_manifest_digests() run_config = build_record.get("run_config", {}) @@ -741,7 +746,12 @@ def _verify_identity_join( UK_SCORE_RECEIPT_VERDICT_PASSED = "passed" -def _verify_score_receipt(receipt: Mapping[str, Any], *, candidate_sha256: str) -> None: +def _verify_score_receipt( + receipt: Mapping[str, Any], + *, + candidate_sha256: str, + reviewed_unresolvable_measures: Mapping[str, Any] | None = None, +) -> None: artifacts = receipt.get("artifacts") scored = ( artifacts.get("candidate", {}).get("sha256") @@ -790,6 +800,29 @@ def _verify_score_receipt(receipt: Mapping[str, Any], *, candidate_sha256: str) f"{UK_SCORE_RECEIPT_VERDICT_PASSED!r}: rule 1 (microcosm#578) is " "not met on the common surface, so the cut cannot be certified." ) + # Every pruned measure must stand on the committed reviewed register: + # the scorer's inference is re-checked here against the register of + # record, so a receipt scored under another register cannot certify. + pruned = receipt.get("incumbent_unresolvable_pruned") + measures = pruned.get("measures", []) if isinstance(pruned, Mapping) else [] + if measures: + from microcosm.build.uk_runtime.candidate_score import ( + load_uk_incumbent_unresolvable_measures, + ) + + reviewed = ( + load_uk_incumbent_unresolvable_measures() + if reviewed_unresolvable_measures is None + else reviewed_unresolvable_measures + ) + unlisted = sorted(str(m) for m in measures if str(m) not in reviewed) + if unlisted: + raise UKReleaseCertificationError( + "the score receipt pruned targets through measure(s) not on the " + f"reviewed incumbent-unresolvable register: {unlisted}; pruning " + "from both arms stands only on a signed entry, so the cut " + "cannot be certified on this receipt." + ) def _score_receipt_summary(receipt: Mapping[str, Any]) -> dict[str, Any]: diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index 937f899bb..ef6146120 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -948,6 +948,7 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "take_up_contract.json", "target_reference_signed_exclusions.json", "input_mass_reviewed_exclusions.json", + "incumbent_unresolvable_measures.json", "spine_swap_signed_differences.json", "spine_candidate_acceptance.json", "ledger_compile_parity_incumbent_2025_signed_differences.json", @@ -1054,6 +1055,7 @@ def test_uk_package_loads(self) -> None: "take_up_contract.json", "target_reference_signed_exclusions.json", "input_mass_reviewed_exclusions.json", + "incumbent_unresolvable_measures.json", "spine_swap_signed_differences.json", "spine_candidate_acceptance.json", "ledger_compile_parity_incumbent_2025_signed_differences.json", diff --git a/packages/microcosm-build/tests/test_uk_candidate_score.py b/packages/microcosm-build/tests/test_uk_candidate_score.py index 83621d3c0..7fe7f4854 100644 --- a/packages/microcosm-build/tests/test_uk_candidate_score.py +++ b/packages/microcosm-build/tests/test_uk_candidate_score.py @@ -10,6 +10,7 @@ from __future__ import annotations import json +from datetime import date from pathlib import Path import numpy as np @@ -29,6 +30,7 @@ uk_national_frame, write_uk_national_frame, ) +from microcosm.build.uk_runtime.weighted_integrity import UKReviewedExclusion from microcosm.calibrate import TargetRegistry, TargetSpec from microcosm.frame import WeightKind @@ -126,6 +128,29 @@ def fake(h5_path, registry, calibration_year, factory, *, band_edge_registry=Non return calls +_REVIEWED = { + "household.measure_b": UKReviewedExclusion( + reason="the incumbent twin never carried measure_b", + approved_by="juaristi22", + adjudication="microcosm#823 (test register)", + approved_on="2026-09-01", + expires_on="2099-12-31", + ) +} + + +@pytest.fixture(autouse=True) +def _reviewed_register(monkeypatch): + """Stand in for the packaged register: measure_b is a signed, in-force + entry. Tests that need another register override the loader again.""" + + monkeypatch.setattr( + candidate_score, + "load_uk_incumbent_unresolvable_measures", + lambda source=None: dict(_REVIEWED), + ) + + _SKIP_RECEIPT = { "skips": [ {"name": "target_b", "measure": "measure_b", "reason": "'household.measure_b'"} @@ -167,6 +192,7 @@ def test_prunes_the_target_the_incumbent_cannot_materialize_and_scores_the_rest( "family": "family_b", "unresolvable_measure": MEASURE_B, "reason": f"provider does not know {MEASURE_B}", + "adjudication": "microcosm#823 (test register)", } # Both arms were scored on the common surface only. assert score["register"]["n_specs"] == 1 @@ -323,3 +349,94 @@ def test_verdict_fails_when_the_incumbent_fits_better(tmp_path): assert score["evaluation"]["rule_1"]["incumbent_full_loss"] == pytest.approx(0.1) assert score["evaluation"]["verdict"] == "failed" assert evaluation_block(score) == score["evaluation"] + + +def _evaluate(candidate, incumbent, **overrides): + kwargs = { + "candidate_h5": candidate, + "incumbent_h5": incumbent, + "candidate_sha256": _sha256_file(candidate), + "incumbent_sha256": _sha256_file(incumbent), + "target_registry": _registry(), + "calibration_year": 2025, + "measure_resolver_factory": _resolver_factory, + } + kwargs.update(overrides) + return candidate_score.evaluate_uk_candidate_against_incumbent(**kwargs) + + +def test_an_unlisted_unresolvable_measure_refuses_instead_of_pruning( + monkeypatch, tmp_path +): + """Pruning is doctrine, not inference: no signed entry, no prune.""" + candidate, incumbent = _twins(tmp_path) + _intercept( + monkeypatch, + failing_h5=incumbent, + message="provider does not know household.measure_b", + receipt=_SKIP_RECEIPT, + ) + monkeypatch.setattr( + candidate_score, "load_uk_incumbent_unresolvable_measures", lambda s=None: {} + ) + with pytest.raises(MeasureResolutionError, match="not on the reviewed"): + _evaluate(candidate, incumbent) + + +def test_an_expired_or_premature_entry_refuses(monkeypatch, tmp_path): + candidate, incumbent = _twins(tmp_path) + _intercept( + monkeypatch, + failing_h5=incumbent, + message="provider does not know household.measure_b", + receipt=_SKIP_RECEIPT, + ) + with pytest.raises(MeasureResolutionError, match="expired"): + _evaluate(candidate, incumbent, evaluated_on=date(2100, 1, 1)) + with pytest.raises(MeasureResolutionError, match="not yet in force"): + _evaluate(candidate, incumbent, evaluated_on=date(2026, 8, 1)) + + +def test_pruned_rows_carry_the_register_entry_they_stand_on(monkeypatch, tmp_path): + candidate, incumbent = _twins(tmp_path) + _intercept( + monkeypatch, + failing_h5=incumbent, + message="provider does not know household.measure_b", + receipt=_SKIP_RECEIPT, + ) + score = _evaluate(candidate, incumbent, evaluated_on=date(2026, 9, 22)) + pruned = score["incumbent_unresolvable_pruned"] + assert pruned["measures"] == ["household.measure_b"] + assert pruned["pruned_targets"]["target_b"]["adjudication"] == ( + "microcosm#823 (test register)" + ) + register = pruned["reviewed_register"] + assert register["resource"] == "incumbent_unresolvable_measures.json" + assert len(register["sha256"]) == 64 + assert register["entries_used"] == { + "household.measure_b": _REVIEWED["household.measure_b"].policy_payload() + } + + +def test_a_skip_reason_is_matched_exactly_never_by_substring(monkeypatch, tmp_path): + """``measure_b`` must not claim a skip whose reason names ``measure_bb``: + with no skip naming the measure and no binding, the loop refuses to + prune blindly rather than dropping an unrelated row.""" + candidate, incumbent = _twins(tmp_path) + _intercept( + monkeypatch, + failing_h5=incumbent, + message="provider does not know household.measure_b", + receipt={ + "skips": [ + { + "name": "target_b", + "measure": "other", + "reason": "'household.measure_bb'", + } + ] + }, + ) + with pytest.raises(MeasureResolutionError, match="refusing to prune blindly"): + _evaluate(candidate, incumbent, contract_targets={}) diff --git a/packages/microcosm-build/tests/test_uk_release_certification.py b/packages/microcosm-build/tests/test_uk_release_certification.py index d9988bf84..f0ba4bdf3 100644 --- a/packages/microcosm-build/tests/test_uk_release_certification.py +++ b/packages/microcosm-build/tests/test_uk_release_certification.py @@ -325,14 +325,46 @@ def test_compose_refuses_a_score_receipt_whose_surface_does_not_close( compose_uk_release_certification(**green_certification_inputs) +def test_compose_refuses_a_score_receipt_pruned_through_an_unlisted_measure( + green_certification_inputs, +): + """Pruning from both arms is doctrine: a measure the scorer pruned must + carry a signed entry on the reviewed incumbent-unresolvable register.""" + receipt = green_score_receipt(green_certification_inputs["candidate_sha256"]) + receipt["incumbent_unresolvable_pruned"].update( + {"n_pruned": 1, "n_scored": 306, "measures": ["benunit.not_reviewed"]} + ) + receipt["evaluation"]["scored_surface"].update({"n_pruned": 1, "n_scored": 306}) + green_certification_inputs["score_receipt_path"].write_text( + json.dumps(receipt), encoding="utf-8" + ) + with pytest.raises(UKReleaseCertificationError, match="not on the reviewed"): + compose_uk_release_certification(**green_certification_inputs) + + from microcosm.build.uk_runtime.weighted_integrity import UKReviewedExclusion + + listed = { + "benunit.not_reviewed": UKReviewedExclusion( + reason="the incumbent never carried it", + approved_by="juaristi22", + adjudication="microcosm#823 (test)", + approved_on="2026-09-01", + expires_on="2027-03-01", + ) + } + compose_uk_release_certification( + **green_certification_inputs, reviewed_unresolvable_measures=listed + ) + + def test_certification_summarises_the_score_receipt_verdict( green_certification_inputs, ): certification = compose_uk_release_certification(**green_certification_inputs) assert certification["score_receipt"]["evaluation"] == { "verdict": "passed", - "n_scored": 305, - "n_pruned": 2, + "n_scored": 307, + "n_pruned": 0, "n_surface": 307, "candidate_full_loss": 0.0096, "incumbent_full_loss": 0.211, diff --git a/packages/microcosm-build/tests/uk_certification_fixtures.py b/packages/microcosm-build/tests/uk_certification_fixtures.py index 8ef215ba5..92d31f6c6 100644 --- a/packages/microcosm-build/tests/uk_certification_fixtures.py +++ b/packages/microcosm-build/tests/uk_certification_fixtures.py @@ -120,18 +120,23 @@ def green_score_receipt(candidate_sha: str) -> dict: "candidate_target_wins": 293, "incumbent_target_wins": 12, "incumbent_unresolvable_pruned": { - "n_pruned": 2, - "n_scored": 305, + "n_pruned": 0, + "n_scored": 307, "n_surface": 307, "pruned_targets": {}, - "measures": ["benunit.uc_calibration_child_count"], - "families": {"dwp_universal_credit": 2}, - "note": "pruned from both arms", + "measures": [], + "families": {}, + "reviewed_register": { + "resource": "incumbent_unresolvable_measures.json", + "sha256": None, + "entries_used": {}, + }, + "note": "nothing pruned", }, "evaluation": { "schema_version": 1, "rule": "microcosm#578 rule 1 on the common surface", - "scored_surface": {"n_scored": 305, "n_pruned": 2, "n_surface": 307}, + "scored_surface": {"n_scored": 307, "n_pruned": 0, "n_surface": 307}, "rule_1": { "passed": True, "candidate_full_loss": 0.0096, From 63370e97460bdd229926df1acfbaff6ce74943e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 22 Sep 2026 11:02:23 +0100 Subject: [PATCH 17/22] Every column-writing spine stage records its household-mass receipt f6b8ce0c dropped the mass assertion on five families (was_wealth, lcfs_consumption, etb_vat, etb_services, regional_property_uprating) rather than demand a receipt nothing wrote. The honest replacement is the positive check (review finding), in the codebase's own E8 pattern: each stage appends a MassChangeRecord under its own module constant (old = new household total, factor 1.0), declared by a record_mass_conservation_receipt operation in spec/sources.yaml (the packaged projection updated to match), and the coverage manifest requires exactly that record. The generator refuses a stage that declares none instead of inventing a reason; the loader again requires a reason on every family; weights_pass_through is gone. Each stage's transform test asserts the receipt. Spines built before this carry none of the five receipts, so their families fail the release-cut coverage gate until rebuilt (the weights audit already needs the rebuild). Co-Authored-By: Claude Fable 5.1 --- ...-pass-through-mass-receipts-967.changed.md | 1 + .../uk-retire-june-cgt-family-967.changed.md | 2 +- .../uk/release_input_coverage_manifest.json | 41 ++--- .../src/microcosm/build/uk/source_stages.json | 35 ++++ .../src/microcosm/build/uk/spec/sources.yaml | 25 +++ .../build/uk_runtime/etb_services.py | 14 +- .../src/microcosm/build/uk_runtime/etb_vat.py | 22 ++- .../build/uk_runtime/lcfs_consumption.py | 14 +- .../build/uk_runtime/national_frame.py | 20 +++ .../build/uk_runtime/regional_uprating.py | 14 +- .../uk_runtime/release_input_coverage.py | 39 ++--- .../microcosm/build/uk_runtime/was_wealth.py | 14 +- .../tests/test_uk_cgt_source_manifest.py | 153 ++++-------------- .../tests/test_uk_etb_services.py | 104 ++++++++++++ .../microcosm-build/tests/test_uk_etb_vat.py | 88 +++++++++- .../tests/test_uk_lcfs_consumption.py | 8 + .../tests/test_uk_regional_uprating.py | 65 ++++++++ .../tests/test_uk_release_input_coverage.py | 11 -- .../tests/test_uk_source_stages.py | 47 ++++++ .../tests/test_uk_was_wealth.py | 7 + ...uild_uk_release_input_coverage_manifest.py | 47 +++--- 21 files changed, 556 insertions(+), 215 deletions(-) create mode 100644 changelog.d/uk-pass-through-mass-receipts-967.changed.md diff --git a/changelog.d/uk-pass-through-mass-receipts-967.changed.md b/changelog.d/uk-pass-through-mass-receipts-967.changed.md new file mode 100644 index 000000000..5c2dd428d --- /dev/null +++ b/changelog.d/uk-pass-through-mass-receipts-967.changed.md @@ -0,0 +1 @@ +Every UK column-writing source stage (`was_wealth`, `lcfs_consumption`, `etb_vat`, `etb_services`, `regional_property_uprating`) now records a household-mass conservation receipt under its own module constant, declared by a `record_mass_conservation_receipt` operation in the sources spec; the coverage manifest requires exactly that record, so the terminal family gate asserts positively that each stage left household mass unchanged (the E8 pattern on every stage). The generator refuses a stage that declares no receipt instead of inventing one, and the loader again requires a reason on every family. Spines built before this change carry none of the five receipts, so their families fail the release-cut coverage gate until rebuilt. diff --git a/changelog.d/uk-retire-june-cgt-family-967.changed.md b/changelog.d/uk-retire-june-cgt-family-967.changed.md index 60b374b28..bd176a705 100644 --- a/changelog.d/uk-retire-june-cgt-family-967.changed.md +++ b/changelog.d/uk-retire-june-cgt-family-967.changed.md @@ -1 +1 @@ -Retire the June-path UK capital gains stage `hmrc_cgt_gains` (the wrapper driven from the certified June H5, its `cgt_source_stages.json` manifest and its coverage family): the FRS spine's `hmrc_cgt_gains_spine` is the only CGT gains stage, its receipt reason is the default for the shared imputation, and the release contract no longer carries the superseded June family. Source-stage families whose stage declares no mass-change reason (the E5 fits and the regional uprating, which pass the household weights through untouched) require no mass receipt and record `weights_pass_through` semantics; the generator used to invent a reason nothing wrote, failing five families on the first release cut. +Retire the June-path UK capital gains stage `hmrc_cgt_gains` (the wrapper driven from the certified June H5, its `cgt_source_stages.json` manifest and its coverage family): the FRS spine's `hmrc_cgt_gains_spine` is the only CGT gains stage, its receipt reason is the default for the shared imputation, and the release contract no longer carries the superseded June family. diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 17d30c643..83da82f42 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -473,7 +473,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "dd405553c28d556cf1444301913705931ebe01fa156ac827188286953dc7738e", + "source_manifest_sha256": "19e51cf5f4b11b1fb4780acfafb2ac79103f399624a94e1b08be53e92a891f00", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2026 release, Table 2.1a for 2024-25, via the vendored conditioning facts", "survey": "HMRC Capital Gains Tax statistics Table 2.1a (2024-25 individuals) and Advani-Summers capital-gains incidence" @@ -496,7 +496,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "dd405553c28d556cf1444301913705931ebe01fa156ac827188286953dc7738e", + "source_manifest_sha256": "19e51cf5f4b11b1fb4780acfafb2ac79103f399624a94e1b08be53e92a891f00", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -508,7 +508,7 @@ "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, - "mass_change_semantics": "weights_pass_through", + "mass_change_semantics": "mass_conserving", "output_weight_kind": "importance", "outputs": [ "dfe_education_spending", @@ -522,9 +522,10 @@ "nhs_admitted_patient_spending", "nhs_outpatient_spending" ], + "required_mass_change_reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "dd405553c28d556cf1444301913705931ebe01fa156ac827188286953dc7738e", + "source_manifest_sha256": "19e51cf5f4b11b1fb4780acfafb2ac79103f399624a94e1b08be53e92a891f00", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -536,14 +537,15 @@ "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, - "mass_change_semantics": "weights_pass_through", + "mass_change_semantics": "mass_conserving", "output_weight_kind": "importance", "outputs": [ "full_rate_vat_expenditure_rate" ], + "required_mass_change_reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "dd405553c28d556cf1444301913705931ebe01fa156ac827188286953dc7738e", + "source_manifest_sha256": "19e51cf5f4b11b1fb4780acfafb2ac79103f399624a94e1b08be53e92a891f00", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -564,7 +566,7 @@ "required_mass_change_reason": "CGT asset-type assignment on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "dd405553c28d556cf1444301913705931ebe01fa156ac827188286953dc7738e", + "source_manifest_sha256": "19e51cf5f4b11b1fb4780acfafb2ac79103f399624a94e1b08be53e92a891f00", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed" @@ -588,7 +590,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "dd405553c28d556cf1444301913705931ebe01fa156ac827188286953dc7738e", + "source_manifest_sha256": "19e51cf5f4b11b1fb4780acfafb2ac79103f399624a94e1b08be53e92a891f00", "source_vintages": { "conditioning_resource": "hmrc_cgt_conditioning_facts.json", "conditioning_resource_sha256": "f5bb0ad93fff3332b7ae8fb580f002d13207b3f95e44a3103c50108279d94797", @@ -605,7 +607,7 @@ "base_candidate_tier": "frs", "calibration_permitted": false, "canonical_source_manifest": "source_stages.json", - "canonical_source_manifest_sha256": "dd405553c28d556cf1444301913705931ebe01fa156ac827188286953dc7738e", + "canonical_source_manifest_sha256": "19e51cf5f4b11b1fb4780acfafb2ac79103f399624a94e1b08be53e92a891f00", "effective_mass_requirements": { "charitable_investment_gifts": { "mass_share_denominator": "all_person_effective_mass", @@ -688,7 +690,7 @@ "superseded_by": { "reason": "The FRS spine build executes hmrc_spi_income_spine, which supersedes the June retained-leaves/hmrc_spi_income pair inside source_stages.json.", "source_manifest": "source_stages.json", - "source_manifest_sha256": "dd405553c28d556cf1444301913705931ebe01fa156ac827188286953dc7738e", + "source_manifest_sha256": "19e51cf5f4b11b1fb4780acfafb2ac79103f399624a94e1b08be53e92a891f00", "stage": "hmrc_spi_income_spine" } }, @@ -696,7 +698,7 @@ "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, - "mass_change_semantics": "weights_pass_through", + "mass_change_semantics": "mass_conserving", "output_weight_kind": "importance", "outputs": [ "food_and_non_alcoholic_beverages_consumption", @@ -719,9 +721,10 @@ "gas_consumption", "has_fuel_consumption" ], + "required_mass_change_reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "dd405553c28d556cf1444301913705931ebe01fa156ac827188286953dc7738e", + "source_manifest_sha256": "19e51cf5f4b11b1fb4780acfafb2ac79103f399624a94e1b08be53e92a891f00", "source_vintages": { "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs and the vendored Chronicle facts for NEED 2023 mean kWh, the FY2024-25 Ofgem cap levels, road-fuel outturn (DESNZ, HMRC, ONS), the DfT VEH1103 licensed-car stock, NTS bus use and the DfT and devolved bus finance tables.", "survey": "Living Costs and Food Survey 2023-24" @@ -733,15 +736,16 @@ "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, - "mass_change_semantics": "weights_pass_through", + "mass_change_semantics": "mass_conserving", "output_weight_kind": "importance", "outputs": [], + "required_mass_change_reason": "Regional property uprating on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [ "main_residence_value", "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "dd405553c28d556cf1444301913705931ebe01fa156ac827188286953dc7738e", + "source_manifest_sha256": "19e51cf5f4b11b1fb4780acfafb2ac79103f399624a94e1b08be53e92a891f00", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -765,7 +769,7 @@ "employee_pension_contributions" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "dd405553c28d556cf1444301913705931ebe01fa156ac827188286953dc7738e", + "source_manifest_sha256": "19e51cf5f4b11b1fb4780acfafb2ac79103f399624a94e1b08be53e92a891f00", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -787,7 +791,7 @@ "student_loan_plan" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "dd405553c28d556cf1444301913705931ebe01fa156ac827188286953dc7738e", + "source_manifest_sha256": "19e51cf5f4b11b1fb4780acfafb2ac79103f399624a94e1b08be53e92a891f00", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -799,7 +803,7 @@ "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, - "mass_change_semantics": "weights_pass_through", + "mass_change_semantics": "mass_conserving", "output_weight_kind": "importance", "outputs": [ "owned_land", @@ -819,9 +823,10 @@ "mortgage_debt", "consumer_debt" ], + "required_mass_change_reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "dd405553c28d556cf1444301913705931ebe01fa156ac827188286953dc7738e", + "source_manifest_sha256": "19e51cf5f4b11b1fb4780acfafb2ac79103f399624a94e1b08be53e92a891f00", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index c95606d41..204c82b15 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -1350,6 +1350,13 @@ "everyone" ], "keyed_by": "entity_id" + }, + { + "kind": "record_mass_conservation_receipt", + "entity": "household", + "reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", + "declared_factor": 1.0, + "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason." } ], "outputs": [ @@ -1415,6 +1422,13 @@ "factor": "avg_house_price / unweighted_mean(main_residence_value | region, >0)", "owner_rows": "main_residence_value > 0", "skip_empty_or_nonpositive_regions": true + }, + { + "kind": "record_mass_conservation_receipt", + "entity": "household", + "reason": "Regional property uprating on the source spine: household weights pass through unchanged and total household mass is conserved.", + "declared_factor": 1.0, + "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason." } ], "outputs": [], @@ -1932,6 +1946,13 @@ "diesel_spending" ], "condition": "has_fuel_consumption == false" + }, + { + "kind": "record_mass_conservation_receipt", + "entity": "household", + "reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", + "declared_factor": 1.0, + "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason." } ], "outputs": [ @@ -2044,6 +2065,13 @@ { "kind": "support_clip", "range": "donor_realized" + }, + { + "kind": "record_mass_conservation_receipt", + "entity": "household", + "reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", + "declared_factor": 1.0, + "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason." } ], "outputs": [ @@ -2303,6 +2331,13 @@ "budget_resource": "etb_services_anchors.json", "age_bands": "half_open", "top_band_fold_in": "none" + }, + { + "kind": "record_mass_conservation_receipt", + "entity": "household", + "reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", + "declared_factor": 1.0, + "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason." } ], "outputs": [ diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 05cbc7dac..4b3e655fb 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -1018,6 +1018,11 @@ stages: - 18 <= age <= 55 - everyone keyed_by: entity_id + - kind: record_mass_conservation_receipt + entity: household + reason: 'WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.' + declared_factor: 1.0 + gate_coupling: The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason. outputs: - owned_land - property_wealth @@ -1073,6 +1078,11 @@ stages: factor: avg_house_price / unweighted_mean(main_residence_value | region, >0) owner_rows: main_residence_value > 0 skip_empty_or_nonpositive_regions: true + - kind: record_mass_conservation_receipt + entity: household + reason: 'Regional property uprating on the source spine: household weights pass through unchanged and total household mass is conserved.' + declared_factor: 1.0 + gate_coupling: The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason. outputs: [] rewrites: - main_residence_value @@ -1455,6 +1465,11 @@ stages: - petrol_spending - diesel_spending condition: has_fuel_consumption == false + - kind: record_mass_conservation_receipt + entity: household + reason: 'LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.' + declared_factor: 1.0 + gate_coupling: The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason. outputs: - food_and_non_alcoholic_beverages_consumption - alcohol_and_tobacco_consumption @@ -1545,6 +1560,11 @@ stages: n_estimators: 100 - kind: support_clip range: donor_realized + - kind: record_mass_conservation_receipt + entity: household + reason: 'ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.' + declared_factor: 1.0 + gate_coupling: The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason. outputs: - full_rate_vat_expenditure_rate nonnegative_outputs: [] @@ -1730,6 +1750,11 @@ stages: budget_resource: etb_services_anchors.json age_bands: half_open top_band_fold_in: none + - kind: record_mass_conservation_receipt + entity: household + reason: 'ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.' + declared_factor: 1.0 + gate_coupling: The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason. outputs: - dfe_education_spending - rail_subsidy_spending diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py index 132ade430..6c03b50f7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py @@ -24,6 +24,7 @@ ) from microcosm.build.uk_runtime.frs_spine import read_pinned_tab from microcosm.build.uk_runtime.national_frame import ( + uk_household_mass_conservation_receipt, uk_household_weight_kind, uk_national_frame, uk_time_period, @@ -127,6 +128,12 @@ def evidence(self) -> dict[str, object]: return evidence +#: The household-mass receipt this stage records (the manifest's +#: ``record_mass_conservation_receipt`` operation repeats it): the terminal +#: family gate requires exactly this reason on a valid mass-conserving record. +UK_ETB_SERVICES_MASS_CONSERVATION_REASON = "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved." + + @dataclass class UKETBServicesStageTransform: stage: SourceStageSpec @@ -200,7 +207,12 @@ def __call__(self, frame: Frame) -> Frame: time_period=uk_time_period(frame), weight_kind=uk_household_weight_kind(frame), household_weights=frame.weights_for("household").values, - mass_log=frame.mass_log, + mass_log=( + *frame.mass_log, + uk_household_mass_conservation_receipt( + frame, UK_ETB_SERVICES_MASS_CONSERVATION_REASON + ), + ), ) validate_uk_national_frame(result) self.last_fit_weight_records = records diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py index ebf892978..10055e7c4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py @@ -14,6 +14,7 @@ from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.uk_runtime.frs_spine import read_pinned_tab from microcosm.build.uk_runtime.national_frame import ( + uk_household_mass_conservation_receipt, uk_household_weight_kind, uk_national_frame, uk_time_period, @@ -59,6 +60,12 @@ def evidence(self) -> dict[str, object]: } +#: The household-mass receipt this stage records (the manifest's +#: ``record_mass_conservation_receipt`` operation repeats it): the terminal +#: family gate requires exactly this reason on a valid mass-conserving record. +UK_ETB_VAT_MASS_CONSERVATION_REASON = "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved." + + @dataclass class UKETBVATStageTransform: stage: SourceStageSpec @@ -104,7 +111,12 @@ def __call__(self, frame: Frame) -> Frame: time_period=uk_time_period(frame), weight_kind=uk_household_weight_kind(frame), household_weights=frame.weights_for("household").values, - mass_log=frame.mass_log, + mass_log=( + *frame.mass_log, + uk_household_mass_conservation_receipt( + frame, UK_ETB_VAT_MASS_CONSERVATION_REASON + ), + ), ) validate_uk_national_frame(result) self.last_fit_weight_records = (record,) @@ -169,7 +181,9 @@ def clean_etb_vat_table( train["weight"] = data["hhold_adj_weight"] denominator = data["expdis"] - data["totvat"] if (denominator == 0).any(): - raise ValueError("ETB VAT donor contains zero disposable-expenditure denominators.") + raise ValueError( + "ETB VAT donor contains zero disposable-expenditure denominators." + ) train["full_rate_vat_expenditure_rate"] = ( data["totvat"] * (1 - reduced_rate_share) / standard_rate ) / denominator @@ -199,9 +213,7 @@ def etb_vat_configuration( } if stage is not None: derive = next( - operation - for operation in stage.operations - if operation.kind == "derive" + operation for operation in stage.operations if operation.kind == "derive" ) for key, value in config.items(): declared = derive.parameters.get(key) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py index f4cfbbb9a..f464bed01 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -57,6 +57,7 @@ from microcosm.build.uk_runtime.frs_spine import read_pinned_tab from microcosm.build.uk_runtime.ledger_fact_vendoring import vendored_rows from microcosm.build.uk_runtime.national_frame import ( + uk_household_mass_conservation_receipt, uk_household_weight_kind, uk_national_frame, uk_time_period, @@ -283,6 +284,12 @@ def evidence(self) -> dict[str, object]: return evidence +#: The household-mass receipt this stage records (the manifest's +#: ``record_mass_conservation_receipt`` operation repeats it): the terminal +#: family gate requires exactly this reason on a valid mass-conserving record. +UK_LCFS_CONSUMPTION_MASS_CONSERVATION_REASON = "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved." + + @dataclass class UKLCFSConsumptionStageTransform: """Whole-stage callable for LCFS-trained consumption imputation.""" @@ -413,7 +420,12 @@ def __call__(self, frame: Frame) -> Frame: time_period=uk_time_period(frame), weight_kind=uk_household_weight_kind(frame), household_weights=frame.weights_for("household").values, - mass_log=frame.mass_log, + mass_log=( + *frame.mass_log, + uk_household_mass_conservation_receipt( + frame, UK_LCFS_CONSUMPTION_MASS_CONSERVATION_REASON + ), + ), ) validate_uk_national_frame(result) self.last_fit_weight_records = imputation.fit_weight_records diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py index 894b2fe74..2dd1348dc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py @@ -57,6 +57,7 @@ "load_uk_national_frame", "uk_household_weight_kind", "uk_national_frame", + "uk_household_mass_conservation_receipt", "uk_time_period", "validate_uk_national_frame", "write_uk_national_frame", @@ -364,6 +365,25 @@ def uk_national_frame( ) +def uk_household_mass_conservation_receipt( + frame: Frame, reason: str +) -> MassChangeRecord: + """The receipt a column-writing stage records: the household weights pass + through untouched, so old and new totals agree and the declared factor is + 1.0. The terminal family gate requires exactly this record under the + stage's own reason, so a build whose stage moved household mass, or never + ran, fails by name (the E8 pattern, now on every source stage).""" + + total = float(frame.weights_for("household").total) + return MassChangeRecord( + entity="household", + old_total=total, + new_total=total, + declared_factor=1.0, + reason=reason, + ) + + def uk_time_period(frame: Frame) -> str: """The frame's UK time period label (e.g. ``"2023"``), from metadata.""" diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/regional_uprating.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/regional_uprating.py index 5a3803015..d2996e962 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/regional_uprating.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/regional_uprating.py @@ -13,6 +13,7 @@ from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.uk_runtime.national_frame import ( + uk_household_mass_conservation_receipt, uk_household_weight_kind, uk_national_frame, uk_time_period, @@ -23,6 +24,12 @@ UK_REGIONAL_PROPERTY_REWRITES = ("main_residence_value", "property_wealth") +#: The household-mass receipt this stage records (the manifest's +#: ``record_mass_conservation_receipt`` operation repeats it): the terminal +#: family gate requires exactly this reason on a valid mass-conserving record. +UK_REGIONAL_PROPERTY_UPRATING_MASS_CONSERVATION_REASON = "Regional property uprating on the source spine: household weights pass through unchanged and total household mass is conserved." + + @dataclass(frozen=True) class UKRegionalPropertyUpratingStageTransform: """Whole-stage callable for regional property-value uprating.""" @@ -47,7 +54,12 @@ def __call__(self, frame: Frame) -> Frame: time_period=uk_time_period(frame), weight_kind=uk_household_weight_kind(frame), household_weights=frame.weights_for("household").values, - mass_log=frame.mass_log, + mass_log=( + *frame.mass_log, + uk_household_mass_conservation_receipt( + frame, UK_REGIONAL_PROPERTY_UPRATING_MASS_CONSERVATION_REASON + ), + ), ) validate_uk_national_frame(result) return result diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py index 67a5322ce..d8648137e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py @@ -251,14 +251,6 @@ def _source_stage_base_candidate_tier( return validate_uk_release_tier(base_candidate.get("tier")) -#: A family whose stage writes columns and passes the household weights -#: through untouched: no mass receipt is recorded, so none is required. -_WEIGHTS_PASS_THROUGH = "weights_pass_through" -_MASS_CHANGE_SEMANTICS = frozenset( - {"mass_conserving", "mass_increasing_support", _WEIGHTS_PASS_THROUGH} -) - - def _parse_family_coverage( raw: object, *, @@ -368,29 +360,22 @@ def _parse_family_coverage( required_mass_change_reason = str( raw_family.get("required_mass_change_reason", "") ).strip() + if not required_mass_change_reason: + raise ValueError( + f"{resource}: family {name!r} needs a reviewed " + "required_mass_change_reason." + ) mass_change_semantics = str( raw_family.get("mass_change_semantics", "mass_conserving") ).strip() - if mass_change_semantics not in _MASS_CHANGE_SEMANTICS: + if mass_change_semantics not in { + "mass_conserving", + "mass_increasing_support", + }: raise ValueError( f"{resource}: family {name!r} has invalid " f"mass_change_semantics {mass_change_semantics!r}." ) - # A stage that records a mass receipt is bound to it by reason; a - # weights-pass-through stage records none, so the contract carries - # none (a reason it demanded would fail every build by construction). - if mass_change_semantics == _WEIGHTS_PASS_THROUGH: - if required_mass_change_reason: - raise ValueError( - f"{resource}: family {name!r} declares " - f"{_WEIGHTS_PASS_THROUGH!r} semantics and must not require " - "a mass-change reason." - ) - elif not required_mass_change_reason: - raise ValueError( - f"{resource}: family {name!r} needs a reviewed " - "required_mass_change_reason." - ) raw_requirements = raw_family.get("effective_mass_requirements", {}) if not isinstance(raw_requirements, Mapping): @@ -458,11 +443,7 @@ def _parse_family_coverage( ), "base_candidate_tier": base_candidate_tier, "output_weight_kind": output_weight_kind, - **( - {"required_mass_change_reason": required_mass_change_reason} - if required_mass_change_reason - else {} - ), + "required_mass_change_reason": required_mass_change_reason, "mass_change_semantics": mass_change_semantics, "effective_mass_requirements": requirements, } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py index 59b88f689..b18882eb4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py @@ -15,6 +15,7 @@ from microcosm.build.uk_runtime.frs_brma import _benunit_household_map from microcosm.build.uk_runtime.frs_spine import read_pinned_tab from microcosm.build.uk_runtime.national_frame import ( + uk_household_mass_conservation_receipt, uk_household_weight_kind, uk_national_frame, uk_time_period, @@ -193,6 +194,12 @@ def evidence(self) -> dict[str, object]: } +#: The household-mass receipt this stage records (the manifest's +#: ``record_mass_conservation_receipt`` operation repeats it): the terminal +#: family gate requires exactly this reason on a valid mass-conserving record. +UK_WAS_WEALTH_MASS_CONSERVATION_REASON = "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved." + + @dataclass class UKWASWealthStageTransform: """Whole-stage callable for WAS-trained UK wealth imputation. @@ -263,7 +270,12 @@ def __call__(self, frame: Frame) -> Frame: time_period=uk_time_period(frame), weight_kind=uk_household_weight_kind(frame), household_weights=frame.weights_for("household").values, - mass_log=frame.mass_log, + mass_log=( + *frame.mass_log, + uk_household_mass_conservation_receipt( + frame, UK_WAS_WEALTH_MASS_CONSERVATION_REASON + ), + ), ) validate_uk_national_frame(result) self.last_result = UKWASWealthResult( diff --git a/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py b/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py index ac4860b1c..3c0fbdfdc 100644 --- a/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py +++ b/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py @@ -3,19 +3,12 @@ from __future__ import annotations from microcosm.build.country_spec import load_country_spec -from microcosm.build.uk_runtime.cgt_asset_type import ( - UK_CGT_ASSET_TYPE_MASS_CONSERVATION_REASON, -) from microcosm.build.uk_runtime.cgt_imputation import ( UK_CGT_IMPUTATION_SEED, UK_CGT_SPINE_MASS_CONSERVATION_REASON, UK_CGT_SPINE_STAGE_NAME, UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS, ) -from microcosm.build.uk_runtime.cgt_structure import ( - CGT_CLONE_MASS_CHANGE_REASON, - CGT_DONOR_MASS_CHANGE_REASON, -) from microcosm.build.uk_runtime.hmrc_capital_gains import ( HMRC_CGT_BUILD_PERIOD, HMRC_CGT_CONDITIONING_RECORD_SETS, @@ -25,12 +18,6 @@ from microcosm.build.uk_runtime.release_input_coverage import ( load_uk_release_input_coverage_manifest, ) -from microcosm.build.uk_runtime.salary_sacrifice import ( - SALSAC_MASS_CHANGE_REASON, -) -from microcosm.build.uk_runtime.student_loans import ( - STUDENT_LOANS_MASS_CHANGE_REASON, -) def _stage(): @@ -114,117 +101,39 @@ def test_the_june_path_cgt_family_is_retired() -> None: assert "hmrc_cgt_gains" not in load_country_spec("uk").sources.stage_map() -def test_stages_without_a_declared_receipt_require_none() -> None: - """A column-writing stage that leaves the weights untouched records no - mass receipt, so the contract demands none; the weight-kind check still - covers it. The first release cut failed five families on a reason the - generator had invented.""" - manifest = load_uk_release_input_coverage_manifest() - for name in ( - "was_wealth", - "lcfs_consumption", - "etb_vat", - "etb_services", - "regional_property_uprating", - ): - family = manifest.family_coverage[name] - assert "required_mass_change_reason" not in family, name - assert family["mass_change_semantics"] == "weights_pass_through", name - assert family["output_weight_kind"] == "importance", name - - -def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: - """The real manifest's family entries against a compliant final frame. - - The wiring first shipped a family whose declared weight kind and - mass-change coupling would have failed every real build at the terminal - gate — invisible because no test drove the shipped manifest through the - family diagnostics. This drives exactly that path. - """ - from types import SimpleNamespace - - from microcosm.build.uk_runtime.release_input_coverage import ( - _family_build_state_diagnostics, +def test_every_source_stage_family_requires_its_declared_receipt() -> None: + """A column-writing stage records a conservation receipt under its module + constant and the contract requires exactly it: the positive "household + mass unchanged" assertion, never an invented reason (review finding on + the first release cut) and never no assertion at all.""" + from microcosm.build.uk_runtime.etb_services import ( + UK_ETB_SERVICES_MASS_CONSERVATION_REASON, ) - from microcosm.frame import MassChangeRecord, WeightKind - - manifest = load_uk_release_input_coverage_manifest() - spi_reason = str( - manifest.family_coverage["hmrc_spi_income"]["required_mass_change_reason"] + from microcosm.build.uk_runtime.etb_vat import UK_ETB_VAT_MASS_CONSERVATION_REASON + from microcosm.build.uk_runtime.lcfs_consumption import ( + UK_LCFS_CONSUMPTION_MASS_CONSERVATION_REASON, ) - compliant = SimpleNamespace( - household_weight_kind=WeightKind.IMPORTANCE, - time_period="2024", - mass_log=( - MassChangeRecord( - entity="household", - old_total=100.0, - new_total=100.0, - declared_factor=1.0, - reason=spi_reason, - ), - MassChangeRecord( - entity="household", - old_total=100.0, - new_total=100.0, - declared_factor=1.0, - reason=CGT_CLONE_MASS_CHANGE_REASON, - ), - MassChangeRecord( - entity="household", - old_total=100.0, - new_total=110.0, - declared_factor=None, - reason=CGT_DONOR_MASS_CHANGE_REASON, - ), - MassChangeRecord( - entity="household", - old_total=100.0, - new_total=100.0, - declared_factor=1.0, - reason=UK_CGT_SPINE_MASS_CONSERVATION_REASON, - ), - MassChangeRecord( - entity="household", - old_total=100.0, - new_total=100.0, - declared_factor=1.0, - reason=UK_CGT_ASSET_TYPE_MASS_CONSERVATION_REASON, - ), - MassChangeRecord( - entity="household", - old_total=100.0, - new_total=100.0, - declared_factor=1.0, - reason=SALSAC_MASS_CHANGE_REASON, - ), - MassChangeRecord( - entity="household", - old_total=100.0, - new_total=100.0, - declared_factor=1.0, - reason=STUDENT_LOANS_MASS_CHANGE_REASON, - ), - ), + from microcosm.build.uk_runtime.regional_uprating import ( + UK_REGIONAL_PROPERTY_UPRATING_MASS_CONSERVATION_REASON, ) - - _, failures = _family_build_state_diagnostics(compliant, manifest) - assert failures == [] - - missing_receipt = SimpleNamespace( - household_weight_kind=WeightKind.IMPORTANCE, - time_period="2024", - mass_log=compliant.mass_log[:1], + from microcosm.build.uk_runtime.was_wealth import ( + UK_WAS_WEALTH_MASS_CONSERVATION_REASON, ) - _, failures = _family_build_state_diagnostics(missing_receipt, manifest) - assert any("hmrc_cgt_gains_spine" in failure for failure in failures) - wrong_kind = SimpleNamespace( - household_weight_kind=WeightKind.DESIGN, - time_period="2024", - mass_log=compliant.mass_log, - ) - _, failures = _family_build_state_diagnostics(wrong_kind, manifest) - assert any( - "hmrc_cgt_gains_spine" in failure and "kind" in failure for failure in failures - ) + manifest = load_uk_release_input_coverage_manifest() + expected = { + "was_wealth": UK_WAS_WEALTH_MASS_CONSERVATION_REASON, + "lcfs_consumption": UK_LCFS_CONSUMPTION_MASS_CONSERVATION_REASON, + "etb_vat": UK_ETB_VAT_MASS_CONSERVATION_REASON, + "etb_services": UK_ETB_SERVICES_MASS_CONSERVATION_REASON, + "regional_property_uprating": ( + UK_REGIONAL_PROPERTY_UPRATING_MASS_CONSERVATION_REASON + ), + } + for name, reason in expected.items(): + family = manifest.family_coverage[name] + assert family["required_mass_change_reason"] == reason, name + assert family["mass_change_semantics"] == "mass_conserving", name + assert family["output_weight_kind"] == "importance", name + for name, family in manifest.family_coverage.items(): + assert str(family["required_mass_change_reason"]).strip(), name diff --git a/packages/microcosm-build/tests/test_uk_etb_services.py b/packages/microcosm-build/tests/test_uk_etb_services.py index e2ae92005..75fbf005b 100644 --- a/packages/microcosm-build/tests/test_uk_etb_services.py +++ b/packages/microcosm-build/tests/test_uk_etb_services.py @@ -307,3 +307,107 @@ def materialize(self, frame, variables, period): assert result["is_SP_age"].tolist() == [1.0, 0.0] assert result["dla"].tolist() == [100.0, 0.0] assert result["hbai_household_net_income"].tolist() == [1e4, 2e4] + + +def _receipt_frame(): + import numpy as np + + from microcosm.build.uk_runtime.national_frame import uk_national_frame + from microcosm.frame import WeightKind + + return uk_national_frame( + person=pd.DataFrame( + { + "person_id": [1, 2, 3], + "person_benunit_id": [1, 2, 2], + "person_household_id": [1, 2, 2], + "age": [40, 35, 6], + } + ), + benunit=pd.DataFrame({"benunit_id": [1, 2]}), + household=pd.DataFrame({"household_id": [1, 2]}), + time_period="2024", + weight_kind=WeightKind.IMPORTANCE, + household_weights=np.array([10.0, 30.0]), + ) + + +def _receipt_stage(name: str, fit_kind: str): + from microcosm.build.source_manifest import SourceStageSpec + + return SourceStageSpec.from_mapping( + { + "stage": name, + "survey": "test", + "source": "test", + "grain": "household", + "artifacts": [], + "operations": [{"kind": fit_kind, "seed": 0}], + "outputs": [], + } + ) + + +def test_stage_transform_records_its_mass_conservation_receipt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from types import SimpleNamespace + + from microcosm.build.uk_runtime import etb_services as module + + frame = _receipt_frame() + household_columns = list(module.UK_ETB_SERVICES_HOUSEHOLD_OUTPUT_COLUMNS) + draws = pd.DataFrame( + {column: [1.0, 2.0] for column in [*household_columns, "rail_subsidy_spending"]} + ) + nhs = pd.DataFrame( + {column: [1.0, 2.0, 3.0] for column in module.UK_NHS_OUTPUT_COLUMNS} + ) + monkeypatch.setattr(module, "assert_rules_engine_country", lambda *_: None) + monkeypatch.setattr( + module, + "etb_services_configuration", + lambda _stage: { + "year": 2024, + "weeks_in_year": 52, + "rail_fare_index": 1.0, + "nhs_budget": 1.0, + }, + ) + monkeypatch.setattr(module, "etb_donor_uprating", lambda _stage: ({}, {})) + monkeypatch.setattr(module, "clean_etb_services_table", lambda raw, **_: raw) + monkeypatch.setattr(module, "recipient_predictors", lambda *_: pd.DataFrame()) + monkeypatch.setattr(module, "_qrf_seed", lambda _stage: 0) + monkeypatch.setattr( + module, + "impute_etb_services", + lambda *_, **__: ( + draws, + (module.FitWeightRecord("etb_services:test", "explicit"),), + ), + ) + monkeypatch.setattr(module, "support_clip_exempt", lambda _stage: ()) + monkeypatch.setattr( + module, + "support_clip_to_donor", + lambda d, *_a, **_k: SimpleNamespace( + clipped=d, receipt=SimpleNamespace(evidence=lambda: {}) + ), + ) + monkeypatch.setattr(module, "etb_bus_support_rake", lambda _s, d, **_: (d, {})) + monkeypatch.setattr( + module, "allocate_nhs_by_age_gender", lambda *_, **__: (nhs, {}) + ) + transform = module.UKETBServicesStageTransform( + stage=_receipt_stage("etb_services", "fit_weighted_qrf_chain"), + engine=object(), + donor=pd.DataFrame({"weight": [1.0]}), + ) + + result = transform(frame) + + assert result.weights_for("household").values.tolist() == [10.0, 30.0] + receipt = result.mass_log[-1] + assert receipt.reason == module.UK_ETB_SERVICES_MASS_CONSERVATION_REASON + assert receipt.old_total == receipt.new_total == 40.0 + assert receipt.declared_factor == 1.0 diff --git a/packages/microcosm-build/tests/test_uk_etb_vat.py b/packages/microcosm-build/tests/test_uk_etb_vat.py index 7b87eb51e..0a886d1b2 100644 --- a/packages/microcosm-build/tests/test_uk_etb_vat.py +++ b/packages/microcosm-build/tests/test_uk_etb_vat.py @@ -174,9 +174,7 @@ def materialize(self, frame, variables, period): } ) benunit = pd.DataFrame({"benunit_id": [100, 200], "benunit_household_id": [10, 20]}) - household = pd.DataFrame( - {"household_id": [10, 20], "household_weight": [1.0, 1.0]} - ) + household = pd.DataFrame({"household_id": [10, 20], "household_weight": [1.0, 1.0]}) frame = uk_national_frame( person=person, benunit=benunit, @@ -190,3 +188,87 @@ def materialize(self, frame, variables, period): assert result["is_child"].tolist() == [1.0, 0.0] assert result["is_SP_age"].tolist() == [1.0, 0.0] assert result["household_net_income"].tolist() == [1e4, 2e4] + + +def _receipt_frame(): + import numpy as np + + from microcosm.build.uk_runtime.national_frame import uk_national_frame + from microcosm.frame import WeightKind + + return uk_national_frame( + person=pd.DataFrame( + { + "person_id": [1, 2, 3], + "person_benunit_id": [1, 2, 2], + "person_household_id": [1, 2, 2], + "age": [40, 35, 6], + } + ), + benunit=pd.DataFrame({"benunit_id": [1, 2]}), + household=pd.DataFrame({"household_id": [1, 2]}), + time_period="2024", + weight_kind=WeightKind.IMPORTANCE, + household_weights=np.array([10.0, 30.0]), + ) + + +def _receipt_stage(name: str, fit_kind: str): + from microcosm.build.source_manifest import SourceStageSpec + + return SourceStageSpec.from_mapping( + { + "stage": name, + "survey": "test", + "source": "test", + "grain": "household", + "artifacts": [], + "operations": [{"kind": fit_kind, "seed": 0}], + "outputs": [], + } + ) + + +def test_stage_transform_records_its_mass_conservation_receipt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from types import SimpleNamespace + + from microcosm.build.uk_runtime import etb_vat as module + + frame = _receipt_frame() + donor = pd.DataFrame({"full_rate_vat_expenditure_rate": [0.1, 0.2]}) + imputed = pd.DataFrame({"full_rate_vat_expenditure_rate": [0.1, 0.2]}) + monkeypatch.setattr(module, "assert_rules_engine_country", lambda *_: None) + monkeypatch.setattr(module, "etb_vat_configuration", lambda _stage: {}) + monkeypatch.setattr(module, "clean_etb_vat_table", lambda raw, **_: donor) + monkeypatch.setattr(module, "recipient_predictors", lambda *_: pd.DataFrame()) + monkeypatch.setattr( + module, + "impute_etb_vat", + lambda *_, **__: (imputed, module.FitWeightRecord("etb_vat:test", "explicit")), + ) + monkeypatch.setattr( + module, + "support_clip_to_donor", + lambda draws, *_a, **_k: SimpleNamespace( + clipped=draws, receipt=SimpleNamespace(evidence=lambda: {}) + ), + ) + transform = module.UKETBVATStageTransform( + stage=_receipt_stage("etb_vat", "fit_weighted_qrf"), + engine=object(), + donor=donor, + ) + + result = transform(frame) + + assert result.table("household")["full_rate_vat_expenditure_rate"].tolist() == [ + 0.1, + 0.2, + ] + assert result.weights_for("household").values.tolist() == [10.0, 30.0] + receipt = result.mass_log[-1] + assert receipt.reason == module.UK_ETB_VAT_MASS_CONSERVATION_REASON + assert receipt.old_total == receipt.new_total == 40.0 + assert receipt.declared_factor == 1.0 diff --git a/packages/microcosm-build/tests/test_uk_lcfs_consumption.py b/packages/microcosm-build/tests/test_uk_lcfs_consumption.py index 5b81ee334..c942f48e9 100644 --- a/packages/microcosm-build/tests/test_uk_lcfs_consumption.py +++ b/packages/microcosm-build/tests/test_uk_lcfs_consumption.py @@ -691,6 +691,14 @@ def materialize(self, frame, variables, period): out = result.table("household") assert set(UK_LCFS_CONSUMPTION_OUTPUT_COLUMNS) <= set(out.columns) assert len(transform.fit_weight_records) == 18 + from microcosm.build.uk_runtime.lcfs_consumption import ( + UK_LCFS_CONSUMPTION_MASS_CONSERVATION_REASON, + ) + + receipt = result.mass_log[-1] + assert receipt.reason == UK_LCFS_CONSUMPTION_MASS_CONSERVATION_REASON + assert receipt.old_total == receipt.new_total > 0 + assert receipt.declared_factor == 1.0 assert set(UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS) <= set( transform.stage.operations[4].parameters["predictors"] ) diff --git a/packages/microcosm-build/tests/test_uk_regional_uprating.py b/packages/microcosm-build/tests/test_uk_regional_uprating.py index e263490ec..8d745bb2a 100644 --- a/packages/microcosm-build/tests/test_uk_regional_uprating.py +++ b/packages/microcosm-build/tests/test_uk_regional_uprating.py @@ -38,3 +38,68 @@ def test_regional_property_uprating_scales_owners_only_and_skips_missing_regions def test_regional_property_uprating_requires_columns() -> None: with pytest.raises(KeyError, match="property-uprating"): uprate_household_property_by_region(pd.DataFrame({"region": ["LONDON"]}), {}) + + +def test_stage_transform_rewrites_property_and_records_its_receipt() -> None: + import numpy as np + + from microcosm.build.source_manifest import SourceStageSpec + from microcosm.build.uk_runtime.national_frame import uk_national_frame + from microcosm.build.uk_runtime.regional_uprating import ( + UK_REGIONAL_PROPERTY_UPRATING_MASS_CONSERVATION_REASON, + UKRegionalPropertyUpratingStageTransform, + ) + from microcosm.frame import WeightKind + + frame = uk_national_frame( + person=pd.DataFrame( + { + "person_id": [1, 2], + "person_benunit_id": [1, 2], + "person_household_id": [1, 2], + } + ), + benunit=pd.DataFrame({"benunit_id": [1, 2]}), + household=pd.DataFrame( + { + "household_id": [1, 2], + "region": ["LONDON", "SCOTLAND"], + "main_residence_value": [100.0, 200.0], + "property_wealth": [150.0, 300.0], + } + ), + time_period="2024", + weight_kind=WeightKind.IMPORTANCE, + household_weights=np.array([10.0, 30.0]), + ) + stage = SourceStageSpec.from_mapping( + { + "stage": "regional_property_uprating", + "survey": "test", + "source": "test", + "grain": "household", + "artifacts": [], + "operations": [{"kind": "uprate_to_regional_reference"}], + "outputs": [], + "rewrites": ["main_residence_value", "property_wealth"], + } + ) + transform = UKRegionalPropertyUpratingStageTransform( + stage=stage, + resource={ + "values": [ + {"region": "LONDON", "avg_house_price": 200.0, "dwellings": 1}, + {"region": "SCOTLAND", "avg_house_price": 600.0, "dwellings": 1}, + ] + }, + ) + + result = transform(frame) + + household = result.table("household") + assert household["main_residence_value"].tolist() == pytest.approx([200.0, 600.0]) + assert result.weights_for("household").values.tolist() == [10.0, 30.0] + receipt = result.mass_log[-1] + assert receipt.reason == UK_REGIONAL_PROPERTY_UPRATING_MASS_CONSERVATION_REASON + assert receipt.old_total == receipt.new_total == 40.0 + assert receipt.declared_factor == 1.0 diff --git a/packages/microcosm-build/tests/test_uk_release_input_coverage.py b/packages/microcosm-build/tests/test_uk_release_input_coverage.py index ee60461e7..e678211ce 100644 --- a/packages/microcosm-build/tests/test_uk_release_input_coverage.py +++ b/packages/microcosm-build/tests/test_uk_release_input_coverage.py @@ -702,17 +702,6 @@ def test_candidate_tier_must_be_present_and_ratified( with pytest.raises(ValueError, match="candidate_evidence.tier"): load_uk_release_input_coverage_manifest(str(bad)) - def test_pass_through_family_must_not_require_a_reason(self, tmp_path) -> None: - payload = json.loads(_SHIPPED_MANIFEST.read_text(encoding="utf-8")) - family = payload["family_coverage"]["was_wealth"] - assert family["mass_change_semantics"] == "weights_pass_through" - family["required_mass_change_reason"] = "invented receipt" - bad = tmp_path / "pass_through_with_reason.json" - bad.write_text(json.dumps(payload), encoding="utf-8") - - with pytest.raises(ValueError, match="must not require a mass-change"): - load_uk_release_input_coverage_manifest(str(bad)) - def test_receipted_family_requires_a_reason(self, tmp_path) -> None: payload = json.loads(_SHIPPED_MANIFEST.read_text(encoding="utf-8")) family = payload["family_coverage"]["hmrc_spi_income"] diff --git a/packages/microcosm-build/tests/test_uk_source_stages.py b/packages/microcosm-build/tests/test_uk_source_stages.py index e6ac57c56..0ce3817f5 100644 --- a/packages/microcosm-build/tests/test_uk_source_stages.py +++ b/packages/microcosm-build/tests/test_uk_source_stages.py @@ -630,9 +630,11 @@ def test_e3_operation_kinds_are_declared_in_order(self) -> None: "fold_into", "support_clip", "allocate_within_group_waterfall", + "record_mass_conservation_receipt", ] assert [op.kind for op in stages["regional_property_uprating"].operations] == [ "uprate_to_regional_reference", + "record_mass_conservation_receipt", ] assert [op.kind for op in stages["lcfs_consumption"].operations] == [ "derive", @@ -648,12 +650,14 @@ def test_e3_operation_kinds_are_declared_in_order(self) -> None: "rake_to_vendored_facts", "fold_into", "zero_when_false", + "record_mass_conservation_receipt", ] assert [op.kind for op in stages["etb_vat"].operations] == [ "derive", "materialize_rules_engine_predictors", "fit_weighted_qrf", "support_clip", + "record_mass_conservation_receipt", ] assert [op.kind for op in stages["etb_services"].operations] == [ "derive", @@ -664,6 +668,7 @@ def test_e3_operation_kinds_are_declared_in_order(self) -> None: "rake_to_vendored_facts", "compute_ratio", "allocate_per_capita_from_cell_table", + "record_mass_conservation_receipt", ] assert [op.kind for op in stages["frs_hmrc_spine_leaves"].operations] == [ "retain_adjudicated_frs_hmrc_leaves", @@ -1032,3 +1037,45 @@ def test_stage2_outputs_are_backed_by_runtime_written_columns(self) -> None: "concept instead of a persisted column, or the runtime constant " "moved without the manifest following." ) + + +def test_every_column_writing_stage_declares_the_receipt_its_module_records() -> None: + """One source of truth per stage: the manifest's receipt reason is the + module constant the transform appends (the gate matches on the string).""" + from microcosm.build.uk_runtime.etb_services import ( + UK_ETB_SERVICES_MASS_CONSERVATION_REASON, + ) + from microcosm.build.uk_runtime.etb_vat import UK_ETB_VAT_MASS_CONSERVATION_REASON + from microcosm.build.uk_runtime.lcfs_consumption import ( + UK_LCFS_CONSUMPTION_MASS_CONSERVATION_REASON, + ) + from microcosm.build.uk_runtime.regional_uprating import ( + UK_REGIONAL_PROPERTY_UPRATING_MASS_CONSERVATION_REASON, + ) + from microcosm.build.uk_runtime.was_wealth import ( + UK_WAS_WEALTH_MASS_CONSERVATION_REASON, + ) + + spec = load_country_spec("uk") + assert spec.sources is not None + stages = spec.sources.stage_map() + expected = { + "was_wealth": UK_WAS_WEALTH_MASS_CONSERVATION_REASON, + "regional_property_uprating": ( + UK_REGIONAL_PROPERTY_UPRATING_MASS_CONSERVATION_REASON + ), + "lcfs_consumption": UK_LCFS_CONSUMPTION_MASS_CONSERVATION_REASON, + "etb_vat": UK_ETB_VAT_MASS_CONSERVATION_REASON, + "etb_services": UK_ETB_SERVICES_MASS_CONSERVATION_REASON, + } + for stage_name, reason in expected.items(): + receipts = [ + op + for op in stages[stage_name].operations + if op.kind == "record_mass_conservation_receipt" + ] + assert len(receipts) == 1, stage_name + assert receipts[0].parameters["reason"] == reason, stage_name + assert receipts[0].parameters["entity"] == "household", stage_name + assert receipts[0].parameters["declared_factor"] == 1.0, stage_name + assert len(set(expected.values())) == len(expected) diff --git a/packages/microcosm-build/tests/test_uk_was_wealth.py b/packages/microcosm-build/tests/test_uk_was_wealth.py index f15b3b43f..20312abb8 100644 --- a/packages/microcosm-build/tests/test_uk_was_wealth.py +++ b/packages/microcosm-build/tests/test_uk_was_wealth.py @@ -318,6 +318,13 @@ def fake_impute(*args, **kwargs): module.FitWeightRecord("uk_was_2018_20_wealth:test", "explicit"), ) assert transformed.table("household")["num_vehicles"].tolist() == [1, 3] + # The stage records its household-mass conservation receipt: the terminal + # family gate asserts positively that the weights passed through. + receipt = transformed.mass_log[-1] + assert receipt.reason == module.UK_WAS_WEALTH_MASS_CONSERVATION_REASON + assert receipt.entity == "household" + assert receipt.old_total == receipt.new_total > 0 + assert receipt.declared_factor == 1.0 assert transform.last_result is not None assert transform.checkpoint_metadata()["evidence"]["support_clip"]["columns"][ "num_vehicles" diff --git a/tools/build_uk_release_input_coverage_manifest.py b/tools/build_uk_release_input_coverage_manifest.py index 5f300a454..7a888328d 100644 --- a/tools/build_uk_release_input_coverage_manifest.py +++ b/tools/build_uk_release_input_coverage_manifest.py @@ -886,25 +886,30 @@ def _source_stage_family_coverage_contract( for operation in operations if isinstance(operation.get("reason"), str) and operation.get("reason") ] - # A stage that declares a mass-change reason on one of its operations - # records that receipt on the frame, and the terminal family gate looks - # for it by that exact string. A stage that declares none (the E5 fits - # and the regional uprating: column-writing transforms that pass the - # household weights through untouched) leaves no receipt to look for, - # so the contract requires none; inventing a generic reason here made - # every such family fail the first release cut on a record nothing ever - # wrote. The weight-kind check still covers those stages. - required_mass_change_reason = declared_reasons[-1] if declared_reasons else None - if any( - operation.get("kind") == "stack_band_donor_households" - for operation in operations - ): - mass_change_semantics = "mass_increasing_support" - elif declared_reasons: - mass_change_semantics = "mass_conserving" - else: - mass_change_semantics = "weights_pass_through" - contract: dict[str, Any] = { + # Every source stage records its household-mass receipt under its own + # reason (a column-writing stage records a conservation receipt: old and + # new totals agree, declared factor 1.0), so the terminal family gate + # asserts positively that the stage left household mass unchanged. A + # stage that declares none has no receipt to require, and the contract + # refuses it rather than inventing one (the first release cut failed + # five families on a reason the generator had invented). + if not declared_reasons: + raise ValueError( + f"source stage {stage_name!r} declares no " + "record_mass_conservation_receipt operation; every spine stage " + "records its household-mass receipt so the terminal family gate can " + "assert the mass was conserved." + ) + required_mass_change_reason = declared_reasons[-1] + mass_change_semantics = ( + "mass_increasing_support" + if any( + operation.get("kind") == "stack_band_donor_households" + for operation in operations + ) + else "mass_conserving" + ) + return { "status": "required_at_build", "stage": stage_name, "source_manifest": SOURCE_STAGES_PATH.name, @@ -916,14 +921,12 @@ def _source_stage_family_coverage_contract( "source": str(stage.get("source", "")), }, "output_weight_kind": "importance", + "required_mass_change_reason": required_mass_change_reason, "mass_change_semantics": mass_change_semantics, "outputs": list(stage.get("outputs", [])), "rewrites": list(stage.get("rewrites", [])), "effective_mass_requirements": {}, } - if required_mass_change_reason is not None: - contract["required_mass_change_reason"] = required_mass_change_reason - return contract def _cgt_spine_family_coverage_contract( From 2f97e0871bb901860f0f149a79dc13794a0a07ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 22 Sep 2026 11:15:20 +0100 Subject: [PATCH 18/22] Sign the six incumbent-unresolvable measures on the reviewed register MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit María's signature (2026-09-22, in force to 2026-12-22): the four administrative-basis UC family inputs the relationship-to-head reconstruction derives on the spine and the enhanced FRS never carried (dependent_children missing on that arm), the ONS census household type and the CGT asset type. Targets bound through them may be pruned from both arms of the rule-1 evaluation; every other unresolvable measure still refuses. Co-Authored-By: Claude Fable 5.1 --- ...mbent-unresolvable-register-967.changed.md | 2 +- .../uk/incumbent_unresolvable_measures.json | 45 ++++++++++++++++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/changelog.d/uk-incumbent-unresolvable-register-967.changed.md b/changelog.d/uk-incumbent-unresolvable-register-967.changed.md index 22fc1d493..1b936b3f6 100644 --- a/changelog.d/uk-incumbent-unresolvable-register-967.changed.md +++ b/changelog.d/uk-incumbent-unresolvable-register-967.changed.md @@ -1 +1 @@ -Pruning a target from both arms of the UK rule-1 evaluation now stands on a reviewed register, `uk/incumbent_unresolvable_measures.json` (reviewed-exclusion schema, keyed `entity.variable`): the scorer prunes a measure only under a signed, in-force entry and refuses an unlisted one, the receipt records the register digest and the entries each pruned row stands on, and the release-cut certifier re-checks every pruned measure against the committed register. Skip rows are matched to the failing measure exactly (`'entity.variable'`), never by substring. The register ships empty: the six measures the enhanced FRS is known not to carry are María's to sign. +Pruning a target from both arms of the UK rule-1 evaluation now stands on a reviewed register, `uk/incumbent_unresolvable_measures.json` (reviewed-exclusion schema, keyed `entity.variable`): the scorer prunes a measure only under a signed, in-force entry and refuses an unlisted one, the receipt records the register digest and the entries each pruned row stands on, and the release-cut certifier re-checks every pruned measure against the committed register. Skip rows are matched to the failing measure exactly (`'entity.variable'`), never by substring. The six measures the enhanced FRS is known not to carry (the four administrative-basis UC family inputs, the ONS household type, the CGT asset type) are signed by María on 2026-09-22, in force to 2026-12-22. diff --git a/packages/microcosm-build/src/microcosm/build/uk/incumbent_unresolvable_measures.json b/packages/microcosm-build/src/microcosm/build/uk/incumbent_unresolvable_measures.json index 7c5754973..c2d859344 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/incumbent_unresolvable_measures.json +++ b/packages/microcosm-build/src/microcosm/build/uk/incumbent_unresolvable_measures.json @@ -1,5 +1,48 @@ { "schema_version": 2, "description": "Measures the incumbent comparison artifact (the enhanced FRS) is known not to carry, keyed entity.variable. A target bound through a listed measure may be pruned from BOTH arms of the rule-1 evaluation (microcosm#578 rule 1 on the common surface, microcosm#823); an unlisted unresolvable measure refuses the evaluation rather than pruning. Every entry is a complete approval receipt in the reviewed-exclusion schema, in force from approved_on through expires_on.", - "exclusions": {} + "exclusions": { + "benunit.uc_calibration_administrative_family_type": { + "reason": "Administrative-basis UC family type (microcosm#832/#835/#937): derived from the relationship-to-head reconstruction (microcosm#791) the enhanced FRS never carried; 93 dwp_universal_credit rows bind through it.", + "approved_by": "juaristi22", + "adjudication": "microcosm#823 (rule-1 evaluation on the common surface; microcosm#578)", + "approved_on": "2026-09-22", + "expires_on": "2026-12-22" + }, + "benunit.uc_calibration_child_count": { + "reason": "Administrative-basis UC dependent-child count (microcosm#832/#835/#937): the same relationship-to-head derivation (microcosm#791) the enhanced FRS never carried.", + "approved_by": "juaristi22", + "adjudication": "microcosm#823 (rule-1 evaluation on the common surface; microcosm#578)", + "approved_on": "2026-09-22", + "expires_on": "2026-12-22" + }, + "benunit.uc_calibration_has_child_under_one": { + "reason": "Administrative-basis UC child-under-one flag (microcosm#832/#835/#937): the same relationship-to-head derivation (microcosm#791) the enhanced FRS never carried.", + "approved_by": "juaristi22", + "adjudication": "microcosm#823 (rule-1 evaluation on the common surface; microcosm#578)", + "approved_on": "2026-09-22", + "expires_on": "2026-12-22" + }, + "benunit.uc_tcl_affected_benunit_proxy": { + "reason": "Two-child-limit affected-benunit proxy (microcosm#840): a counterfactual the enhanced FRS carries no inputs for; 15 dwp_two_child_limit rows bind through it.", + "approved_by": "juaristi22", + "adjudication": "microcosm#823 (rule-1 evaluation on the common surface; microcosm#578)", + "approved_on": "2026-09-22", + "expires_on": "2026-12-22" + }, + "household.ons_household_type": { + "reason": "ONS census household type (microcosm#887): the enhanced FRS carries no household-type classification; 10 ons_household_composition rows bind through it.", + "approved_by": "juaristi22", + "adjudication": "microcosm#823 (rule-1 evaluation on the common surface; microcosm#578)", + "approved_on": "2026-09-22", + "expires_on": "2026-12-22" + }, + "person.capital_gains_asset_type": { + "reason": "CGT asset type (microcosm#939, the E8 spine stage): the enhanced FRS carries gains amounts only; 2 hmrc_cgt rows bind through it.", + "approved_by": "juaristi22", + "adjudication": "microcosm#823 (rule-1 evaluation on the common surface; microcosm#578)", + "approved_on": "2026-09-22", + "expires_on": "2026-12-22" + } + } } From d34df63d543cd8c8f50ca100a848f1e1c7807c4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:26:29 +0100 Subject: [PATCH 19/22] Regenerate the UK spine graph parity fixture for the stage receipts The graph pinned beside the charter-H2 fixture (uk_spine.json) and its data-only source bundle carry the five stages' new record_mass_conservation_receipt operations, so the parity test's graph-equality assertion holds again; the compiled order and the oracle identity are unchanged. Regenerated with tools/graph_uk_spine_fixture.py. Co-Authored-By: Claude Fable 5.1 --- .../parity/uk_spine/sources/fixture.json | 35 +++++++++++++++++++ .../fixtures/parity/uk_spine/uk_spine.json | 2 +- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index c93878903..d6199ba23 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -461,6 +461,13 @@ "kind": "allocate_per_capita_from_cell_table", "resource": "nhs_consumption_by_age_gender.json", "top_band_fold_in": "none" + }, + { + "declared_factor": 1, + "entity": "household", + "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason.", + "kind": "record_mass_conservation_receipt", + "reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved." } ], "outputs": [ @@ -539,6 +546,13 @@ { "kind": "support_clip", "range": "donor_realized" + }, + { + "declared_factor": 1, + "entity": "household", + "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason.", + "kind": "record_mass_conservation_receipt", + "reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved." } ], "outputs": [ @@ -3293,6 +3307,13 @@ ], "condition": "has_fuel_consumption == false", "kind": "zero_when_false" + }, + { + "declared_factor": 1, + "entity": "household", + "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason.", + "kind": "record_mass_conservation_receipt", + "reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved." } ], "outputs": [ @@ -3340,6 +3361,13 @@ "owner_rows": "main_residence_value > 0", "resource": "regional_land_values.json", "skip_empty_or_nonpositive_regions": true + }, + { + "declared_factor": 1, + "entity": "household", + "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason.", + "kind": "record_mass_conservation_receipt", + "reason": "Regional property uprating on the source spine: household weights pass through unchanged and total household mass is conserved." } ], "outputs": [], @@ -3917,6 +3945,13 @@ "18 <= age <= 55", "everyone" ] + }, + { + "declared_factor": 1, + "entity": "household", + "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason.", + "kind": "record_mass_conservation_receipt", + "reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved." } ], "outputs": [ diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json index c6a8c1b9d..bbf871806 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json @@ -1 +1 @@ -{"country":"uk","nodes":[{"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_savings_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"047d30937efb21e39259fdbed5597751e4f5a0c341ea583c7446e4c9c11eba48","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tax_free_savings_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"frs_relationships","stage_contract_sha256":"30628808d06d9829ebcbcf460cd355a01261e78c7c73e6006c75bec2e3d60c33","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_employment","stage_contract_sha256":"ddbefaf05b44788d794a6e4b0e8926c14318d66e50d2f11f50ca549538bbf60c","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","sic_industry_division"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"frs_council_tax","stage_contract_sha256":"3381cd9f7a736514d5073c57480e07f5098890f3ca9ed3865392043594771eb9","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_education","stage_contract_sha256":"836cb0f5a582fee1425190e96c9cb81bdef859bd236c8b6bc27660b6e3d0c2f8","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"stage":"frs_take_up","stage_contract_sha256":"067d31b7fbe733d1d182588705e9ce79df0c8e5f6e8f519314765af49b1b00d7","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_person_draws","stage_contract_sha256":"7e0af955541ec5669dcdf1de4e1e6ca642853e48b775ec4e25238b41491df324","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"frs_brma","stage_contract_sha256":"5b8f0b361310c7cd3efbaae3762b347649b7d4afb9e9943131d9bef9266fc3c8","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"was_wealth","stage_contract_sha256":"c499dcb7c49a0176229dbc363b400415cb5c1da44af266ba2155f9a62a0bd2ed","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"base":"frs_brma.checkpoint","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"regional_property_uprating","stage_contract_sha256":"4304356c6c5ba91a04883148cbf9078761ee926c8387aadb6c15b6b70ebb31f6","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"lcfs_consumption","stage_contract_sha256":"09edc53ce6c0fd1b6e7374ca84902d72dad5eff305c9ea9a312e8d4bb5d3d411","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"etb_vat","stage_contract_sha256":"99a6756256adfe8255672fafda89710c08ae56bd2b8f011b20c3b8381eabfa1e","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"etb_services","stage_contract_sha256":"841c690d7f23ac8661f663f59a15c20c2795a17149709b337e92d04848b3fe17","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employee_pension_contributions","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"2bb3d068003489b47cc8676ac26c203ce3c39a9b6e92f3ecd0a3c06acd6addc4","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","stage":"spi_support_channel","stage_contract_sha256":"4b10f1a4a215cdf2c406c9974de2d1e580eb3cd3a25ed34973c1b63abf4a54bb","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tax_free_savings_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"hmrc_spi_income_spine","stage_contract_sha256":"4c2edca1f77416a00cff10c7e9d0a9da50e332a2bb6a995ef9ac1bdd00ff97a9","time_period":"2024"},"population":"spi_support_channel","sources":["frs"],"structural":"none","weights":null},{"base":"spi_support_channel","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"uc_reporter_redraw","stage_contract_sha256":"7e94e034cad29c5bae566ffb875c42cddf3a1025c15496907e783ceff1144f8b","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_benunit_head","is_parent","person_support_channel","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"uc_capital_coherence","stage_contract_sha256":"059f27ee687ee15385b246d43068b17aeecbe3645ccc41fd52fcad06a006a081","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","stage":"cgt_incidence_clone","stage_contract_sha256":"ee30278543cc0297a5366d855b7753aa04af5518971594c193c52be36bf8a7b4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Run structural UK stage cgt_band_donors.","id":"cgt_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_band_donors@1","mass":"free","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_band_donor","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","stage":"cgt_band_donors","stage_contract_sha256":"6fde981ff62f0435f9d708ef4048f64b8edfdec34bab521a19dec677d2603694","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_band_donors.","id":"cgt_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_band_donor"]},"population":"cgt_band_donors","sources":[],"structural":"none","weights":null},{"base":"cgt_band_donors","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_capital_gains_clone","household_is_cgt_band_donor"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported","tax_free_savings_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"2914b5187377b195d201a55b10792d6f602ae85475580a262bf84d10d9766e5a","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"529706ff53425e28907eca2451a6d7c8ccb5b0f82643ed62eb3947ed4f71dfc3","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_capital_gains_clone","household_is_cgt_band_donor"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","capital_gains","capital_gains_asset_type","capital_gains_residential_property"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"salary_sacrifice","stage_contract_sha256":"a4af925af21cc6766eb7c8855b6743a3aa4eeeb38df12cfc83063d7ad6890539","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","current_education","employee_pension_contributions","highest_education","student_loan_repayments","student_loans"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"student_loans","stage_contract_sha256":"7c4c76398c2e80b41c7f941a5f06246b57d89b5eb0c36d42e3116ac35e82e203","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} +{"country":"uk","nodes":[{"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_savings_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"047d30937efb21e39259fdbed5597751e4f5a0c341ea583c7446e4c9c11eba48","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tax_free_savings_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"frs_relationships","stage_contract_sha256":"30628808d06d9829ebcbcf460cd355a01261e78c7c73e6006c75bec2e3d60c33","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_employment","stage_contract_sha256":"ddbefaf05b44788d794a6e4b0e8926c14318d66e50d2f11f50ca549538bbf60c","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","sic_industry_division"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"frs_council_tax","stage_contract_sha256":"3381cd9f7a736514d5073c57480e07f5098890f3ca9ed3865392043594771eb9","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_education","stage_contract_sha256":"836cb0f5a582fee1425190e96c9cb81bdef859bd236c8b6bc27660b6e3d0c2f8","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"stage":"frs_take_up","stage_contract_sha256":"067d31b7fbe733d1d182588705e9ce79df0c8e5f6e8f519314765af49b1b00d7","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_person_draws","stage_contract_sha256":"7e0af955541ec5669dcdf1de4e1e6ca642853e48b775ec4e25238b41491df324","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"frs_brma","stage_contract_sha256":"5b8f0b361310c7cd3efbaae3762b347649b7d4afb9e9943131d9bef9266fc3c8","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"was_wealth","stage_contract_sha256":"366e019c41ee2a28bcd61bbaa62a024ed0bb4b30b3ffc4d87ab23b340b4076bf","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"base":"frs_brma.checkpoint","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"regional_property_uprating","stage_contract_sha256":"acb342e8ca1770556c5682a9411c5ead5d61dd1420cfe6b0305ed14a3407fb25","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"lcfs_consumption","stage_contract_sha256":"e321f404b2e3a6c0c8bb29861290046db1d4987be071b5c9e3f6fca69a780daf","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"etb_vat","stage_contract_sha256":"5a2b777d88315bcccb72477c088a136c634131e0bc1a339b7287217ee5cfd8d3","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"etb_services","stage_contract_sha256":"a354a6bbf0be88a4001970a0879600511ff83f1194836dafd8c56fc11b31278e","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employee_pension_contributions","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"2bb3d068003489b47cc8676ac26c203ce3c39a9b6e92f3ecd0a3c06acd6addc4","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","stage":"spi_support_channel","stage_contract_sha256":"4b10f1a4a215cdf2c406c9974de2d1e580eb3cd3a25ed34973c1b63abf4a54bb","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tax_free_savings_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"hmrc_spi_income_spine","stage_contract_sha256":"4c2edca1f77416a00cff10c7e9d0a9da50e332a2bb6a995ef9ac1bdd00ff97a9","time_period":"2024"},"population":"spi_support_channel","sources":["frs"],"structural":"none","weights":null},{"base":"spi_support_channel","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"uc_reporter_redraw","stage_contract_sha256":"7e94e034cad29c5bae566ffb875c42cddf3a1025c15496907e783ceff1144f8b","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_benunit_head","is_parent","person_support_channel","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"uc_capital_coherence","stage_contract_sha256":"059f27ee687ee15385b246d43068b17aeecbe3645ccc41fd52fcad06a006a081","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","stage":"cgt_incidence_clone","stage_contract_sha256":"ee30278543cc0297a5366d855b7753aa04af5518971594c193c52be36bf8a7b4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Run structural UK stage cgt_band_donors.","id":"cgt_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_band_donors@1","mass":"free","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_band_donor","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","stage":"cgt_band_donors","stage_contract_sha256":"6fde981ff62f0435f9d708ef4048f64b8edfdec34bab521a19dec677d2603694","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_band_donors.","id":"cgt_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_band_donor"]},"population":"cgt_band_donors","sources":[],"structural":"none","weights":null},{"base":"cgt_band_donors","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_capital_gains_clone","household_is_cgt_band_donor"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported","tax_free_savings_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"2914b5187377b195d201a55b10792d6f602ae85475580a262bf84d10d9766e5a","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"529706ff53425e28907eca2451a6d7c8ccb5b0f82643ed62eb3947ed4f71dfc3","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_capital_gains_clone","household_is_cgt_band_donor"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","care_hours","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","relationship_to_head","ons_family_role","ons_family_index","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","capital_gains","capital_gains_asset_type","capital_gains_residential_property"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"salary_sacrifice","stage_contract_sha256":"a4af925af21cc6766eb7c8855b6743a3aa4eeeb38df12cfc83063d7ad6890539","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","current_education","employee_pension_contributions","highest_education","student_loan_repayments","student_loans"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"student_loans","stage_contract_sha256":"7c4c76398c2e80b41c7f941a5f06246b57d89b5eb0c36d42e3116ac35e82e203","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} From 4ee986fbb505990248e8e4d2349578b03c396ef4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:19:56 +0100 Subject: [PATCH 20/22] Bind the release cut's spine to the recorded parent; validate the candidate before pruning; re-check register windows at certification Adversarial review on #967 (2026-09-22): C1, A1, A2 and the wheels lanes. C1. --spine-sha256 proved only that the operator's hash matched the bytes; a spine regenerated at the same path after the solve pinned fine and lent another build's receipts to the candidate. The certifier now refuses, before any gate runs, a spine whose digest is not the parent the calibration recorded (build record input posture and input pin, the signed diagnostics' build block); compose_uk_release_certification takes spine_sha256 as a required argument, the identity join binds it to the build record, and the certification records it as parent_spine, now a required, digest-checked field of the data contract's certification set. A1. The evaluator materializes the candidate on the full comparison surface before the incumbent is probed, so an export missing a listed measure refuses as the strict scorer does, whether or not the incumbent shares the gap; the receipt records candidate_validated_on_full_surface. A2. _verify_score_receipt takes the certification's evaluation date and refuses a pruned measure whose entry is expired or not yet in force on it, so a receipt does not outlive the entry it was pruned under. The four scorer tests added in round 2 wrote H5 files without the pytables import-skip, which the wheels venv lacks; they carry it now. Co-Authored-By: Claude Fable 5.1 --- ...ertifier-parent-spine-binding-967.fixed.md | 1 + .../uk-certifier-register-window-967.fixed.md | 1 + ...uk-evaluation-candidate-first-967.fixed.md | 1 + docs/uk-national-calibration-runbook-623.md | 2 +- ...k-national-release-assembly-runbook-806.md | 2 + .../build/uk_runtime/candidate_score.py | 16 ++++ .../build/uk_runtime/release_certification.py | 45 +++++++++++ .../tests/test_uk_candidate_score.py | 36 +++++++++ .../test_uk_certify_release_cut_command.py | 80 +++++++++++++++++++ .../tests/test_uk_release_assembler.py | 11 ++- .../tests/test_uk_release_certification.py | 66 ++++++++++++--- .../tests/uk_certification_fixtures.py | 15 ++++ .../src/microcosm/data/contract.py | 12 +++ .../microcosm-data/tests/test_contract.py | 15 ++++ tools/certify_uk_release_cut.py | 40 ++++++++++ 15 files changed, 329 insertions(+), 14 deletions(-) create mode 100644 changelog.d/uk-certifier-parent-spine-binding-967.fixed.md create mode 100644 changelog.d/uk-certifier-register-window-967.fixed.md create mode 100644 changelog.d/uk-evaluation-candidate-first-967.fixed.md diff --git a/changelog.d/uk-certifier-parent-spine-binding-967.fixed.md b/changelog.d/uk-certifier-parent-spine-binding-967.fixed.md new file mode 100644 index 000000000..f824e56e0 --- /dev/null +++ b/changelog.d/uk-certifier-parent-spine-binding-967.fixed.md @@ -0,0 +1 @@ +`tools/certify_uk_release_cut.py` and `compose_uk_release_certification` bind the spine supplied as stage evidence to the parent the calibration recorded (the build record's input posture and input pin, the signed diagnostics' build block): a spine that pins correctly on its own hash but is not that parent is refused before any family build-state gate reads its receipts, and the certification records the parent under `parent_spine`. The data contract's certification field set gains `parent_spine` (a sha256 digest), so a published certification always names the spine its release cut measured. diff --git a/changelog.d/uk-certifier-register-window-967.fixed.md b/changelog.d/uk-certifier-register-window-967.fixed.md new file mode 100644 index 000000000..2a4a6997e --- /dev/null +++ b/changelog.d/uk-certifier-register-window-967.fixed.md @@ -0,0 +1 @@ +The release-cut certifier re-checks each pruned measure's reviewed incumbent-unresolvable entry against its window at the certification's own evaluation date, so a score receipt does not outlive the entry it was pruned under. diff --git a/changelog.d/uk-evaluation-candidate-first-967.fixed.md b/changelog.d/uk-evaluation-candidate-first-967.fixed.md new file mode 100644 index 000000000..411b3fde2 --- /dev/null +++ b/changelog.d/uk-evaluation-candidate-first-967.fixed.md @@ -0,0 +1 @@ +The UK rule-1 evaluator validates the candidate on the full comparison surface before pruning the incumbent's unresolvable measures, so an export missing a listed measure refuses (as the strict scorer does) instead of scoring on a surface reduced by the incumbent's gap; the receipt records `candidate_validated_on_full_surface`. diff --git a/docs/uk-national-calibration-runbook-623.md b/docs/uk-national-calibration-runbook-623.md index e1c142f46..ca059a55c 100644 --- a/docs/uk-national-calibration-runbook-623.md +++ b/docs/uk-national-calibration-runbook-623.md @@ -95,7 +95,7 @@ it removed, and the receipt carries them under `pruned_targets` with each one's family and measure, `measures`, `families`). The score stands on the common surface with band edges from the full register (#803). A measure the *candidate* cannot materialize still refuses: that is a defect. -`--no-prune-incumbent-unresolvable` restores the refusal on the incumbent A measure may be pruned only under a signed, in-force entry of the reviewed incumbent-unresolvable register (`packages/microcosm-build/src/microcosm/build/uk/incumbent_unresolvable_measures.json`, keyed `entity.variable`, reviewed-exclusion schema); an unlisted measure refuses the evaluation, the receipt records the register digest and the entries used, and the certifier re-checks every pruned measure against the committed register. +`--no-prune-incumbent-unresolvable` restores the refusal on the incumbent A measure may be pruned only under a signed, in-force entry of the reviewed incumbent-unresolvable register (`packages/microcosm-build/src/microcosm/build/uk/incumbent_unresolvable_measures.json`, keyed `entity.variable`, reviewed-exclusion schema); an unlisted measure refuses the evaluation, the receipt records the register digest and the entries used, and the certifier re-checks every pruned measure against the committed register, including each entry's window at the certification's own evaluation date. The candidate is validated on the full surface before any pruning, so an export missing a listed measure refuses rather than scoring on a reduced surface. side too. The receipt's `evaluation` block decides rule 1 (#578) on that surface: `verdict` is `passed` when the candidate's full loss is below the incumbent's, `failed` otherwise, and the release-cut certifier refuses any diff --git a/docs/uk-national-release-assembly-runbook-806.md b/docs/uk-national-release-assembly-runbook-806.md index aea405fe1..2e676d89b 100644 --- a/docs/uk-national-release-assembly-runbook-806.md +++ b/docs/uk-national-release-assembly-runbook-806.md @@ -95,6 +95,8 @@ uv run --no-sync python tools/certify_uk_release_cut.py \ --release-id microcosm-uk-2024-25-national ``` +The spine is stage evidence for the family build-state gates, so `--spine-sha256` must be the parent the calibration recorded (`build_record.input_posture.sha256`, `source_pins.input_h5.sha256`, the signed diagnostics' `build.input_posture`); a spine that pins correctly but is not that parent is refused before any gate runs, and the certification records it as `parent_spine`. + With the default paths, this writes `microcosm_uk_2024_25.release_cut_gates.json` and `microcosm_uk_2024_25.release_certification.json` next to the candidate. Continue diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py index b570da421..e4d891357 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py @@ -614,6 +614,9 @@ def pruned_block( "pruned_targets": {name: asdict(row) for name, row in sorted(pruned.items())}, "measures": measures, "families": dict(sorted(families.items())), + # The candidate materialized every target of the full surface before + # the incumbent's gaps were pruned; only the incumbent is pruned for. + "candidate_validated_on_full_surface": True, "reviewed_register": { **( {"resource": UK_INCUMBENT_UNRESOLVABLE_REGISTER, "sha256": None} @@ -727,6 +730,19 @@ def evaluate_uk_candidate_against_incumbent( else reviewed_measures ) if prune_incumbent_unresolvable_measures: + # The candidate is validated on the FULL surface before any pruning: + # a measure the candidate cannot materialize is a defect and refuses + # whether or not the incumbent shares the gap (review finding A1), + # so pruning only ever removes rows the incumbent cannot express. + _scored_frame( + candidate_h5, + target_registry, + calibration_year, + measure_resolver_factory, + band_edge_registry=( + target_registry if band_edge_registry is None else band_edge_registry + ), + ) surface, pruned = prune_incumbent_unresolvable( incumbent_h5, target_registry, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py index 2cf821295..75c6a202e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py @@ -340,6 +340,7 @@ def compose_uk_release_certification( candidate_name: str, candidate_path: Path, candidate_sha256: str, + spine_sha256: str, spine_report_path: Path, seam_report_path: Path, release_cut_report_path: Path, @@ -415,6 +416,7 @@ def compose_uk_release_certification( build_record=build_record, candidate_path=candidate_path, candidate_sha256=candidate_sha256, + spine_sha256=spine_sha256, release_id=release_id, ) score_receipt_bytes = score_receipt_path.read_bytes() @@ -423,6 +425,7 @@ def compose_uk_release_certification( score_receipt, candidate_sha256=candidate_sha256, reviewed_unresolvable_measures=reviewed_unresolvable_measures, + evaluated_on=exclusions_evaluated_on, ) full_digests = _full_manifest_digests() @@ -438,6 +441,9 @@ def compose_uk_release_certification( "sha256": candidate_sha256, "size_bytes": candidate_path.stat().st_size, }, + # The spine whose stage receipts the release cut measured: bound to + # the parent the calibration recorded, never an operator's choice. + "parent_spine": {"sha256": spine_sha256}, "parts": part_summaries, "spec": { "gates_manifest_sha256": full_digests["gates_manifest_sha256"], @@ -674,8 +680,25 @@ def _verify_identity_join( build_record: Mapping[str, Any], candidate_path: Path, candidate_sha256: str, + spine_sha256: str, release_id: str, ) -> None: + # The spine supplied as stage evidence must be the parent the + # calibration consumed: the build record carries that parent's digest + # twice (input posture and source pins), and both must name it. + recorded_parents = { + "input_posture.sha256": (build_record.get("input_posture") or {}).get("sha256"), + "source_pins.input_h5.sha256": ( + (build_record.get("source_pins") or {}).get("input_h5") or {} + ).get("sha256"), + } + if any(value != spine_sha256 for value in recorded_parents.values()): + raise UKReleaseCertificationError( + "the spine supplied as stage evidence is not the parent the " + f"calibration recorded: build record {recorded_parents}, supplied " + f"{spine_sha256!r}; the family build-state gates were measured on " + "another build's receipts." + ) sidecar_binding = spine_sidecar.get("spine_gate_report") if not isinstance(sidecar_binding, Mapping): raise UKReleaseCertificationError( @@ -751,6 +774,7 @@ def _verify_score_receipt( *, candidate_sha256: str, reviewed_unresolvable_measures: Mapping[str, Any] | None = None, + evaluated_on: date | None = None, ) -> None: artifacts = receipt.get("artifacts") scored = ( @@ -823,6 +847,27 @@ def _verify_score_receipt( "from both arms stands only on a signed entry, so the cut " "cannot be certified on this receipt." ) + # An entry suppresses only inside its window, at the certification's + # own evaluation date: a receipt scored while an entry was in force + # does not outlive the entry. + if evaluated_on is not None: + out_of_window = { + str(m): ( + f"expired {reviewed[str(m)].expires_on}" + if reviewed[str(m)].expired(evaluated_on) + else f"takes force {reviewed[str(m)].approved_on}" + ) + for m in measures + if reviewed[str(m)].expired(evaluated_on) + or reviewed[str(m)].premature(evaluated_on) + } + if out_of_window: + raise UKReleaseCertificationError( + "the score receipt pruned targets through reviewed " + "incumbent-unresolvable entries outside their window on " + f"{evaluated_on.isoformat()}: {out_of_window}; renew the " + "adjudication or re-score the candidate." + ) def _score_receipt_summary(receipt: Mapping[str, Any]) -> dict[str, Any]: diff --git a/packages/microcosm-build/tests/test_uk_candidate_score.py b/packages/microcosm-build/tests/test_uk_candidate_score.py index 7fe7f4854..8bece9c53 100644 --- a/packages/microcosm-build/tests/test_uk_candidate_score.py +++ b/packages/microcosm-build/tests/test_uk_candidate_score.py @@ -369,6 +369,7 @@ def test_an_unlisted_unresolvable_measure_refuses_instead_of_pruning( monkeypatch, tmp_path ): """Pruning is doctrine, not inference: no signed entry, no prune.""" + pytest.importorskip("tables") candidate, incumbent = _twins(tmp_path) _intercept( monkeypatch, @@ -384,6 +385,7 @@ def test_an_unlisted_unresolvable_measure_refuses_instead_of_pruning( def test_an_expired_or_premature_entry_refuses(monkeypatch, tmp_path): + pytest.importorskip("tables") candidate, incumbent = _twins(tmp_path) _intercept( monkeypatch, @@ -398,6 +400,7 @@ def test_an_expired_or_premature_entry_refuses(monkeypatch, tmp_path): def test_pruned_rows_carry_the_register_entry_they_stand_on(monkeypatch, tmp_path): + pytest.importorskip("tables") candidate, incumbent = _twins(tmp_path) _intercept( monkeypatch, @@ -423,6 +426,7 @@ def test_a_skip_reason_is_matched_exactly_never_by_substring(monkeypatch, tmp_pa """``measure_b`` must not claim a skip whose reason names ``measure_bb``: with no skip naming the measure and no binding, the loop refuses to prune blindly rather than dropping an unrelated row.""" + pytest.importorskip("tables") candidate, incumbent = _twins(tmp_path) _intercept( monkeypatch, @@ -440,3 +444,35 @@ def test_a_skip_reason_is_matched_exactly_never_by_substring(monkeypatch, tmp_pa ) with pytest.raises(MeasureResolutionError, match="refusing to prune blindly"): _evaluate(candidate, incumbent, contract_targets={}) + + +def test_a_gap_shared_by_both_artifacts_still_refuses_on_the_candidate( + monkeypatch, tmp_path +): + """Review A1: the candidate is validated on the full surface BEFORE the + incumbent's gaps are pruned, so an export missing a listed measure is a + defect even when the incumbent lacks it too; it never scores as a win.""" + pytest.importorskip("tables") + candidate = tmp_path / "candidate.h5" + incumbent = tmp_path / "incumbent.h5" + _write(candidate, measure_a=[5.0, 5.0]) + _write(incumbent, measure_a=[4.0, 4.0]) + original = candidate_score._scored_frame + calls: list[tuple[Path, list[str]]] = [] + + def fake(h5_path, registry, calibration_year, factory, *, band_edge_registry=None): + calls.append((Path(h5_path), [spec.name for spec in registry.specs])) + if any(spec.name == "target_b" for spec in registry.specs): + raise MeasureResolutionError( + f"provider does not know {MEASURE_B}", receipt=_SKIP_RECEIPT + ) + return original(h5_path, registry, calibration_year, None) + + monkeypatch.setattr(candidate_score, "_scored_frame", fake) + + with pytest.raises(MeasureResolutionError, match="does not know"): + _evaluate(candidate, incumbent, evaluated_on=date(2026, 9, 22)) + # The first probe was the candidate on the full surface; the incumbent + # was never asked, so nothing was pruned. + assert calls[0] == (candidate, ["target_a", "target_b"]) + assert all(path == candidate for path, _ in calls) diff --git a/packages/microcosm-build/tests/test_uk_certify_release_cut_command.py b/packages/microcosm-build/tests/test_uk_certify_release_cut_command.py index 72cd1986f..1e898dd76 100644 --- a/packages/microcosm-build/tests/test_uk_certify_release_cut_command.py +++ b/packages/microcosm-build/tests/test_uk_certify_release_cut_command.py @@ -9,8 +9,11 @@ from __future__ import annotations import hashlib +import json from pathlib import Path +import pytest + from microcosm.build.logbook_adoption import role_pins_digest from tools import certify_uk_release_cut as certify @@ -108,3 +111,80 @@ def fake_run(args, state): ) assert state.build_id.startswith("uk-frs-release-certification-attempt-") assert '"shippable": true' in capsys.readouterr().out + + +def test_certifier_refuses_a_spine_that_is_not_the_candidates_recorded_parent( + monkeypatch, tmp_path: Path +) -> None: + """Review C1: the supplied spine pins fine on its own hash but is not the + parent the build record and the signed diagnostics name; the certifier + refuses before any gate reads its receipts.""" + candidate = tmp_path / "microcosm_uk_2024_25.h5" + candidate.write_bytes(b"candidate bytes") + spine = tmp_path / "spine.h5" + spine.write_bytes(b"spine B bytes") + (tmp_path / "spine.build.json").write_text('{"stages": ["frs_spine"]}') + parent_a = "3" * 64 + diagnostics = tmp_path / "calibration_diagnostics.json" + diagnostics.write_text( + json.dumps({"build": {"input_posture": {"sha256": parent_a}}, "targets": []}) + ) + build_record = tmp_path / "build_record.json" + build_record.write_text( + json.dumps( + { + "artifacts": { + "diagnostics_json": { + "sha256": hashlib.sha256(diagnostics.read_bytes()).hexdigest() + } + }, + "input_posture": {"sha256": parent_a}, + "source_pins": {"input_h5": {"sha256": parent_a}}, + } + ) + ) + ledger = tmp_path / "ledger" + ledger.mkdir() + (ledger / "consumer_facts.jsonl").write_text('{"fact": 1}\n') + for name in ("terminal_gates.json", "input_mass_reference.json", "score.json"): + (tmp_path / name).write_text("{}") + monkeypatch.setattr(certify, "resolve_predecessor", lambda digest: None) + monkeypatch.setattr(certify, "record_terminal_attempt", lambda **kwargs: None) + + with pytest.raises(SystemExit, match="not the candidate's recorded parent"): + certify.main( + [ + "--candidate-h5", + str(candidate), + "--candidate-sha256", + hashlib.sha256(candidate.read_bytes()).hexdigest(), + "--candidate-name", + "microcosm_uk_2024_25", + "--spine-h5", + str(spine), + "--spine-sha256", + hashlib.sha256(spine.read_bytes()).hexdigest(), + "--diagnostics-json", + str(diagnostics), + "--build-record-json", + str(build_record), + "--seam-gate-report", + str(tmp_path / "terminal_gates.json"), + "--ledger-facts", + str(ledger), + "--ledger-facts-sha256", + "1" * 64, + "--ledger-manifest-sha256", + "2" * 64, + "--input-mass-reference", + str(tmp_path / "input_mass_reference.json"), + "--score-receipt", + str(tmp_path / "score.json"), + "--release-id", + "microcosm-uk-2024-25-national", + "--release-cut-gate-json", + str(tmp_path / "release_cut_gates.json"), + "--certification-json", + str(tmp_path / "release_certification.json"), + ] + ) diff --git a/packages/microcosm-build/tests/test_uk_release_assembler.py b/packages/microcosm-build/tests/test_uk_release_assembler.py index 428e70062..c20a96df0 100644 --- a/packages/microcosm-build/tests/test_uk_release_assembler.py +++ b/packages/microcosm-build/tests/test_uk_release_assembler.py @@ -242,7 +242,13 @@ def _build_assembler_inputs( build_record = green_certification_inputs["build_record"] build_record.update( build_id=_ATTEMPT_ID, + # The calibration's recorded parent is the spine this fixture writes, + # in both places the seam records it; the certifier binds the + # supplied spine to it. input_posture={"sha256": sha256(spine)}, + source_pins={ + "input_h5": {"sha256": sha256(spine), "size_bytes": spine.stat().st_size} + }, gate_summary={"uk_target_fit": "passed"}, ) build_record["spine_provenance"]["rules_engine"] = { @@ -275,6 +281,7 @@ def _build_assembler_inputs( "release_id": UK_NATIONAL_RELEASE_ID, "candidate_name": candidate.stem, "candidate_sha256": candidate_sha, + "spine_sha256": sha256(spine), "certification_path": certification_path, } compose_uk_release_certification(**compose_inputs) @@ -409,9 +416,7 @@ def test_assemble_preserves_successful_version_2_delivery( capsys.readouterr() manifest = json.loads( ( - assembler_inputs["out_dir"] - / UK_NATIONAL_RELEASE_ID - / "build_manifest.json" + assembler_inputs["out_dir"] / UK_NATIONAL_RELEASE_ID / "build_manifest.json" ).read_text() ) diff --git a/packages/microcosm-build/tests/test_uk_release_certification.py b/packages/microcosm-build/tests/test_uk_release_certification.py index f0ba4bdf3..8b835c6d3 100644 --- a/packages/microcosm-build/tests/test_uk_release_certification.py +++ b/packages/microcosm-build/tests/test_uk_release_certification.py @@ -343,18 +343,52 @@ def test_compose_refuses_a_score_receipt_pruned_through_an_unlisted_measure( from microcosm.build.uk_runtime.weighted_integrity import UKReviewedExclusion - listed = { - "benunit.not_reviewed": UKReviewedExclusion( - reason="the incumbent never carried it", - approved_by="juaristi22", - adjudication="microcosm#823 (test)", - approved_on="2026-09-01", - expires_on="2027-03-01", - ) - } + def _listed(approved_on: str, expires_on: str): + return { + "benunit.not_reviewed": UKReviewedExclusion( + reason="the incumbent never carried it", + approved_by="juaristi22", + adjudication="microcosm#823 (test)", + approved_on=approved_on, + expires_on=expires_on, + ) + } + + # In force on the certification's evaluation date (2026-08-27): accepted. compose_uk_release_certification( - **green_certification_inputs, reviewed_unresolvable_measures=listed + **green_certification_inputs, + reviewed_unresolvable_measures=_listed("2026-08-01", "2027-03-01"), ) + # The same receipt does not outlive the entry: expired or not yet in + # force on the evaluation date refuses (review finding A2). + with pytest.raises(UKReleaseCertificationError, match="expired 2026-08-20"): + compose_uk_release_certification( + **green_certification_inputs, + reviewed_unresolvable_measures=_listed("2026-08-01", "2026-08-20"), + ) + with pytest.raises(UKReleaseCertificationError, match="takes force 2026-09-01"): + compose_uk_release_certification( + **green_certification_inputs, + reviewed_unresolvable_measures=_listed("2026-09-01", "2027-03-01"), + ) + + +def test_compose_refuses_a_spine_that_is_not_the_recorded_parent( + green_certification_inputs, +): + """The spine is stage evidence; it must be the parent the calibration + consumed, not whatever spine pins correctly at the path (review C1).""" + green_certification_inputs["spine_sha256"] = "b" * 64 + with pytest.raises(UKReleaseCertificationError, match="not the parent"): + compose_uk_release_certification(**green_certification_inputs) + + +def test_compose_refuses_a_build_record_without_a_recorded_parent( + green_certification_inputs, +): + del green_certification_inputs["build_record"]["input_posture"] + with pytest.raises(UKReleaseCertificationError, match="not the parent"): + compose_uk_release_certification(**green_certification_inputs) def test_certification_summarises_the_score_receipt_verdict( @@ -412,3 +446,15 @@ def test_release_cut_battery_runs_and_signs(tmp_path: Path, monkeypatch): ) - set(UK_NATIONAL_GATE_SCOPE) on_disk = json.loads(report_path.read_text(encoding="utf-8")) assert on_disk["attestation"]["signature"] == payload["attestation"]["signature"] + + +def test_certification_fields_mirror_the_data_contract(green_certification_inputs): + """The composed certification carries exactly the field set the data + contract pins (lockstep: a field added on one side moves the other).""" + from microcosm.data import contract as data_contract + + certification = compose_uk_release_certification(**green_certification_inputs) + assert set(certification) == set(data_contract._UK_CERTIFICATION_REQUIRED_FIELDS) + assert certification["parent_spine"] == { + "sha256": green_certification_inputs["spine_sha256"] + } diff --git a/packages/microcosm-build/tests/uk_certification_fixtures.py b/packages/microcosm-build/tests/uk_certification_fixtures.py index 92d31f6c6..d3d47c232 100644 --- a/packages/microcosm-build/tests/uk_certification_fixtures.py +++ b/packages/microcosm-build/tests/uk_certification_fixtures.py @@ -106,6 +106,10 @@ def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() +#: The parent spine the green fixture's calibration recorded. +GREEN_PARENT_SPINE_SHA256 = "a" * 64 + + def green_score_receipt(candidate_sha: str) -> dict: """A score receipt the certifier accepts: measured on the candidate's bytes, scored on a closed common surface, rule 1 passed.""" @@ -212,6 +216,16 @@ def green_certification_inputs(tmp_path: Path): "spine_provenance": { "spine_gate_report": {"sha256": sha256(spine_report)}, }, + # The parent the calibration consumed, recorded twice as the seam + # writes it; the certifier binds the supplied spine to it. + "input_posture": { + "tier": "staging_candidate", + "sha256": GREEN_PARENT_SPINE_SHA256, + "size_bytes": 1, + }, + "source_pins": { + "input_h5": {"sha256": GREEN_PARENT_SPINE_SHA256, "size_bytes": 1} + }, "artifacts": { "staging_h5": {"sha256": candidate_sha}, "diagnostics_json": {"sha256": diagnostics_sha}, @@ -227,6 +241,7 @@ def green_certification_inputs(tmp_path: Path): "candidate_name": "microcosm_uk_2024", "candidate_path": candidate, "candidate_sha256": candidate_sha, + "spine_sha256": GREEN_PARENT_SPINE_SHA256, "spine_report_path": spine_report, "seam_report_path": seam_report, "release_cut_report_path": release_cut_report, diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index 01c43c525..7487d9912 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -793,6 +793,9 @@ def compatibility_claim_declarer_error(declared_by: object) -> str | None: "country", "release_id", "candidate", + # The spine whose stage receipts the release cut measured, bound by + # the certifier to the parent the calibration recorded. + "parent_spine", "parts", "spec", "doctrine", @@ -3096,6 +3099,15 @@ def _check_uk_release_certification( f"{file} release_id {certification.get('release_id')!r} does not " f"match the release under validation ({release_id!r})." ) + parent_spine = certification.get("parent_spine") + parent_sha = ( + parent_spine.get("sha256") if isinstance(parent_spine, Mapping) else None + ) + if not isinstance(parent_sha, str) or not _SHA256_RE.match(parent_sha): + failures.append( + f"{file} parent_spine.sha256 must be the sha256 hex digest of the " + "spine the calibration recorded as its parent." + ) parts = certification.get("parts") if not isinstance(parts, Mapping) or set(parts) != set( diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index 48eb5223f..75034ae10 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -5568,6 +5568,7 @@ def _green_uk_certification( "sha256": "b" * 64, "size_bytes": 1, }, + "parent_spine": {"sha256": "a" * 64}, "parts": parts, "spec": { "gates_manifest_sha256": contract._UK_GATE_BATTERY_GATES_MANIFEST_SHA256, @@ -5661,6 +5662,20 @@ def test_uk_release_certification_refusals(monkeypatch) -> None: for line in _certification_failures(certification, monkeypatch, key) ) + # The parent spine the release cut measured is named by digest. + certification = _green_uk_certification(key) + certification["parent_spine"] = {"sha256": "not-a-digest"} + assert any( + "parent_spine.sha256" in line + for line in _certification_failures(certification, monkeypatch, key) + ) + certification = _green_uk_certification(key) + del certification["parent_spine"] + assert any( + "must carry exactly the certification fields" in line + for line in _certification_failures(certification, monkeypatch, key) + ) + # A tampered field breaks the signature: the flag flip is caught both as # a verdict refusal and as a signature failure. certification = _green_uk_certification(key) diff --git a/tools/certify_uk_release_cut.py b/tools/certify_uk_release_cut.py index 11b186631..38dad3725 100644 --- a/tools/certify_uk_release_cut.py +++ b/tools/certify_uk_release_cut.py @@ -170,6 +170,17 @@ def main(argv: list[str] | None = None) -> int: return 0 +def _nested(payload: object, *keys: str) -> object: + """Walk mapping keys, returning None where the path is absent.""" + + current = payload + for key in keys: + if not isinstance(current, dict): + return None + current = current.get(key) + return current + + def _run(args: argparse.Namespace, state: AttemptState) -> dict[str, object]: measured = hashlib.sha256(args.candidate_h5.read_bytes()).hexdigest() if measured != args.candidate_sha256: @@ -201,6 +212,34 @@ def _run(args: argparse.Namespace, state: AttemptState) -> dict[str, object]: "error: --diagnostics-json bytes do not match the build record's " f"binding ({diagnostics_sha} != {recorded_diagnostics})" ) + # The spine is stage evidence for the family build-state gates, so it + # must be the parent the calibration actually consumed: the build record + # and the signed diagnostics both carry that parent's digest. A spine + # regenerated at the same path after the solve pins fine on its own hash + # and would lend another build's receipts to this candidate. + recorded_parents = { + "build_record.input_posture.sha256": _nested( + build_record, "input_posture", "sha256" + ), + "build_record.source_pins.input_h5.sha256": _nested( + build_record, "source_pins", "input_h5", "sha256" + ), + "diagnostics.build.input_posture.sha256": _nested( + diagnostics, "build", "input_posture", "sha256" + ), + } + mismatched = { + key: value + for key, value in recorded_parents.items() + if value != args.spine_sha256 + } + if mismatched: + raise SystemExit( + "error: --spine-h5 is not the candidate's recorded parent: the " + f"calibration recorded {mismatched}, the supplied spine measures " + f"{args.spine_sha256}; the family build-state evidence must come " + "from the spine this candidate was calibrated from." + ) append_phase(state, "inputs_bound") artifact = load_ledger_consumer_artifact( @@ -270,6 +309,7 @@ def _run(args: argparse.Namespace, state: AttemptState) -> dict[str, object]: candidate_name=args.candidate_name, candidate_path=args.candidate_h5, candidate_sha256=args.candidate_sha256, + spine_sha256=args.spine_sha256, spine_report_path=spine_report_path, seam_report_path=args.seam_gate_report, release_cut_report_path=args.release_cut_gate_json, From 981dd312ea506b9e156af31b83783c1bccb0ae23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:19:56 +0100 Subject: [PATCH 21/22] Relabel the take-up span and the child-poverty comparison in the #968 memo Adversarial review on #967 (A3, A4). The take-up probe holds the number of drawn benefit-unit records fixed, not the weighted caseload: the bounding arms represent 6.28m and 6.95m UC families against the baseline draw's 6.37m (outputs/takeup_probe.json), so the 2.6-point span is a joint allocation-and-caseload sensitivity, not an allocation-only bound at a fixed claimant count; the memo says so where it claimed otherwise, and the entitlement-ordered arm (6.57m) carries the same caveat. The child comparison counts age < 18 where HBAI's dependent child is every under-16 plus dependent 16-19-year-olds; the memo labels the mismatch and no longer reads 18.8 vs 23 as model error. A weighted-caseload rerun and a rerun on HBAI's child definition need the evaluation stack. Co-Authored-By: Claude Fable 5.1 --- docs/evidence/uk-poverty-968/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/evidence/uk-poverty-968/README.md b/docs/evidence/uk-poverty-968/README.md index 4cbe5646d..e6b5f3e7c 100644 --- a/docs/evidence/uk-poverty-968/README.md +++ b/docs/evidence/uk-poverty-968/README.md @@ -63,11 +63,11 @@ Basis swap (replace the engine's simulated means-tested benefits with the FRS-re So the FRS under-reporting that the DWP itself documents (FRS 2023/24 captures 67 % of UC families, 74 % of Housing Benefit, 71 % of Pension Credit, 61 % of ESA, table M.6a) costs the calibrated line about 0.6 points of measured poverty and nothing at design weights. -Take-up allocation (`measure_968_takeup_probe.py`, v20 at 2024, calibrated weights, claimant count held near the file's): +Take-up allocation (`measure_968_takeup_probe.py`, v20 at 2024, calibrated weights). **Caveat (adversarial review on PR #967, 2026-09-22): the probe holds the number of sampled benefit-unit records fixed, not the weighted caseload.** With unequal calibrated weights the arms represent different numbers of UC families (`outputs/takeup_probe.json`): baseline 6.37m, the three re-draws 6.57–6.66m, poorest-first 6.28m, richest-first 6.95m, so the poorest-to-richest arms differ by 0.67m families (10.7 %). The spans below are therefore a joint allocation-and-caseload sensitivity, not an allocation-only effect at a fixed caseload; a weighted-caseload-preserving rerun is the controlled version and has not been run. - Baseline draw: 15.30 % (6.37m UC families, £76.8bn). - Three alternative random draws of the same residual count: 15.40–15.59 %. -- Residual given to the poorest entitled units: 14.05 %. To the richest: 16.66 %. **The allocation alone spans 2.6 points at a fixed claimant count.** +- Residual given to the poorest entitled units: 14.05 % (6.28m families). To the richest: 16.66 % (6.95m families). **The two arms span 2.6 points, with the represented caseload moving by 10.7 % between them: a joint allocation-and-caseload sensitivity, not an allocation-only bound.** - Reported claimants only, no residual: 16.12 % (4.67m families, £63.6bn). Everyone entitled claims: 14.85 % (8.61m). - Child relative rate across the same runs: 17.6 % to 21.0 %. @@ -79,7 +79,7 @@ Reading: the residual gap to HBAI (about 2 points at 2024) is entirely inside th - The zero-income spike: HBAI has 0.66m individuals (1.0 %) at £0–10/week. spine-s design 0.34 %, v20 0.16 %, eFRS 0.62 %. Half of v20's deep-tail deficit is this spike; the engine's imputation fills incomes that the FRS leaves at zero. - Quantile ratios to the median: HBAI p10 0.48, p30 0.765, p70 1.31, p90 1.96, mean/median 1.22. v20 calibrated: 0.52, 0.765, 1.27, 1.75, 1.13. spine-s design: 0.51, 0.77, 1.29, 1.85, 1.14. eFRS: 0.46, 0.73, 1.32, 1.87, 1.13. The engine files are compressed at both ends; calibration compresses the top further (p90 ratio 1.85 → 1.75). That is the scorecard's open Gini item, not a poverty item, but it shares a cause candidate (weights on the SPI-synthetic and top households). - Income sources by quintile (share of gross, state support): HBAI 47 / 36 / 22 / 10 / 3. v20 2024: 41 / 31 / 20 / 13 / 4.5. The bottom two quintiles carry less state support and more market income than HBAI's; the top two carry more. Not a benefits-too-high-at-the-bottom signature. -- Groups, relative BHC, v20 2024 versus HBAI FYE 2024: children 18.8 vs 23 (the largest miss); pensioners 17.2 vs 19; working-age 13.6 vs 15. Children by family size: 1 child 9.9 vs 15, 2 children 15.3 vs 18, 3+ 32.3 vs 35. Tenure (household basis): social renters 20–22 vs 29, private renters 15.8 vs 22, owned outright 15.0 vs 18, mortgagors 12.5 vs 8. v20 has too few poor renters and too many poor mortgagors; the eFRS has the opposite renter error (social 31–44 %). Family typing here is household-level while HBAI's is benefit-unit-level, so single-adult categories are not comparable and are omitted. +- Groups, relative BHC, v20 2024 versus HBAI FYE 2024: children 18.8 vs 23 (the largest group gap, but not on one definition: the probes count `age < 18`, while HBAI's dependent child is every under-16 plus 16–19-year-olds in full-time non-advanced education or training who are unmarried and living with their parents, so dependent 18–19-year-olds are missing from the file's child population and independent 16–17-year-olds are in it; the difference is not read as model error here, and a rerun on HBAI's definition is the fix, adversarial review on PR #967); pensioners 17.2 vs 19; working-age 13.6 vs 15. Children by family size: 1 child 9.9 vs 15, 2 children 15.3 vs 18, 3+ 32.3 vs 35. Tenure (household basis): social renters 20–22 vs 29, private renters 15.8 vs 22, owned outright 15.0 vs 18, mortgagors 12.5 vs 8. v20 has too few poor renters and too many poor mortgagors; the eFRS has the opposite renter error (social 31–44 %). Family typing here is household-level while HBAI's is benefit-unit-level, so single-adult categories are not comparable and are omitted. ## 5. Attribution @@ -89,7 +89,7 @@ Against HBAI FYE 2024, relative BHC 17.3 %, v20 at 2024 measures 15.3 %: - Family-wide residual at design weights: about 1.1 points (16.2 vs 17.3), of which roughly half is the missing zero-income spike and the rest the year basis and the compressed bottom tail. - Calibration: −0.9 points (reweighting at a fixed line −1.2, line shift +0.3). - Simulated-versus-reported benefit basis: −0.6 points on the calibrated weights, ~0 at design weights. -- Take-up allocation: ±1.3 points around the draw at a fixed claimant count, which brackets the whole residual. +- Take-up allocation: ±1.3 points around the draw across the poorest-first and richest-first arms, which also move the represented caseload by 10.7 % between them (see the caveat in §3); the span brackets the whole residual but is not a controlled allocation-only bound. ## 6. Fix path as offered (rulings recorded below) @@ -101,7 +101,7 @@ Against HBAI FYE 2024, relative BHC 17.3 %, v20 at 2024 measures 15.3 %: ## 7. After the review (2026-09-21) -- **Direction of a realistic take-up allocation** (review point 3; `outputs/takeup_direction.json`). Ranking the drawn Universal Credit residual by entitlement size, largest first, which is DWP's finding that non-take-up concentrates in small entitlements, gives 14.9 % relative BHC at 2024 against the file's 15.3 % (children 18.7 % against 18.8 %; UC spend £84.3bn against £76.8bn at the same claimant count). Smallest first gives 15.9 %. So the correction the evidence favours moves measured poverty away from HBAI, not towards it, and the residual gap should be read as at least what the memo states. +- **Direction of a realistic take-up allocation** (review point 3; `outputs/takeup_direction.json`). Ranking the drawn Universal Credit residual by entitlement size, largest first, which is DWP's finding that non-take-up concentrates in small entitlements, gives 14.9 % relative BHC at 2024 against the file's 15.3 % (children 18.7 % against 18.8 %; UC spend £84.3bn against £76.8bn at the same number of drawn records). Smallest first gives 15.9 %. So the correction the evidence favours moves measured poverty away from HBAI, not towards it, and the residual gap should be read as at least what the memo states. Same caveat: this arm represents 6.57m UC families against the baseline draw's 6.37m (+3.2 %, `outputs/takeup_direction.json`), so the 0.4-point move is allocation and caseload together, not allocation at a fixed claimant count. - **Year basis** (review point 4). The scorecard probe now carries an `hbai_basis` block measured at 2024, the closest engine year to HBAI FYE 2024, with the same pair plus the p10 and p90 ratios to the median and the under-£10 share; the evaluation reports show that block in its own table beside HBAI FYE 2024 and keep the 2026 rows file against file only. - **Position taken (2026-09-21).** The scorecard is made accurate on concept and year, and the evidence is filed here. Poverty is not chased as a benchmark: HBAI is a survey statistic from another producer, not an administrative total, and a poverty rate is not a calibration target. The income-level and tenure work the decomposition points at is already under way on its own merits and is not tied to moving a poverty rate; the residual 2-point relative gap and the children and tenure rows stay on record here rather than in a follow-up issue. From c4d705614b93c5b9a6af72d52a06c891dae488f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 22 Sep 2026 18:12:22 +0100 Subject: [PATCH 22/22] Fold in Vahid's third pass: register digest at certification, public scorer name, gate note and A16 support caveat Vahid's third pass on #967 (2026-09-22) and his notes on #965. A2, second half: against the committed register of record the receipt's reviewed_register.sha256 must be the committed register's digest, so a receipt scored under another register cannot certify (checked after the membership refusal; tested both ways inside the entries' window). #965 note 1: the driver imported the scorer's private _default_measure_resolver_factory and every test stubbed the module. uk_default_measure_resolver_factory is the public name, the driver imports it, and a national-role test loads the real module. #965 note 2: the telemetry summary is tested with a populated pruned_targets mapping and asserted free of record arrays. Nits: the uk_input_mass_parity note recorded the retired charitable_investment_gifts exclusion as the sole breach; it now records the 2026-09-22 retirements and the two genuine breaches. The note rides the policy digest, so the UK gate-battery policy, manifest and fingerprint pins are re-minted in the data contract and its test (the scoped part digests are unchanged). The A16 evidence README says which three retirements were support-signed and that the thin-cell concern stays a diagnostic, not an error claim. Co-Authored-By: Claude Fable 5.1 --- .../uk-certifier-register-window-967.fixed.md | 2 +- .../uk-input-mass-gate-note-967.changed.md | 1 + .../uk-scorer-public-factory-967.changed.md | 1 + docs/evidence/uk-a16-v20/README.md | 4 ++ .../src/microcosm/build/uk/gates.json | 2 +- .../build/uk_runtime/candidate_score.py | 7 ++ .../build/uk_runtime/release_certification.py | 18 +++++ .../tests/test_uk_release_certification.py | 34 ++++++++++ .../tests/test_uk_rowwise_national_role.py | 66 ++++++++++++++++++- .../src/microcosm/data/contract.py | 6 +- .../microcosm-data/tests/test_contract.py | 4 +- tools/build_uk_rowwise_candidate.py | 2 +- 12 files changed, 138 insertions(+), 9 deletions(-) create mode 100644 changelog.d/uk-input-mass-gate-note-967.changed.md create mode 100644 changelog.d/uk-scorer-public-factory-967.changed.md diff --git a/changelog.d/uk-certifier-register-window-967.fixed.md b/changelog.d/uk-certifier-register-window-967.fixed.md index 2a4a6997e..d0dae87ae 100644 --- a/changelog.d/uk-certifier-register-window-967.fixed.md +++ b/changelog.d/uk-certifier-register-window-967.fixed.md @@ -1 +1 @@ -The release-cut certifier re-checks each pruned measure's reviewed incumbent-unresolvable entry against its window at the certification's own evaluation date, so a score receipt does not outlive the entry it was pruned under. +The release-cut certifier re-checks each pruned measure's reviewed incumbent-unresolvable entry against its window at the certification's own evaluation date, so a score receipt does not outlive the entry it was pruned under. Against the committed register of record, the receipt's `reviewed_register.sha256` must be the committed register's digest. diff --git a/changelog.d/uk-input-mass-gate-note-967.changed.md b/changelog.d/uk-input-mass-gate-note-967.changed.md new file mode 100644 index 000000000..44cb97dd3 --- /dev/null +++ b/changelog.d/uk-input-mass-gate-note-967.changed.md @@ -0,0 +1 @@ +The `uk_input_mass_parity` gate note no longer describes the retired `charitable_investment_gifts` exclusion as the sole breach; it records the 2026-09-22 retirements and the two genuine breaches the gate names. The note rides the policy digest, so the UK gate-battery policy, manifest and fingerprint pins move with it (data contract and its test, lockstep-checked). diff --git a/changelog.d/uk-scorer-public-factory-967.changed.md b/changelog.d/uk-scorer-public-factory-967.changed.md new file mode 100644 index 000000000..93b5adf69 --- /dev/null +++ b/changelog.d/uk-scorer-public-factory-967.changed.md @@ -0,0 +1 @@ +`candidate_score.uk_default_measure_resolver_factory` is the public name of the production measure-resolver factory; the rowwise driver's national role imports it (the private name stays as an alias), and a national-role test loads the real scorer module to prove the name exists. diff --git a/docs/evidence/uk-a16-v20/README.md b/docs/evidence/uk-a16-v20/README.md index 5b89ef66d..6ec5d6f8a 100644 --- a/docs/evidence/uk-a16-v20/README.md +++ b/docs/evidence/uk-a16-v20/README.md @@ -16,6 +16,10 @@ Result (`outputs/a16_rows_on_v20.json`): 67 excluded entries, 62 measured, 5 unr - `dwp.benefit_cap.capped_households_200_01_to_300` (microcosm#882): -10.1 % unbound - `dwp.benefit_cap.capped_households_400_01_to_500` (microcosm#882): -3.8 % unbound +### What the retirements do and do not say + +Three of the five rows were signed for *support*, not error size: `hmrc/employment_income_count_income_band_1_000_000_to_inf` (zero in-band carrier records on total income), `hmrc/self_employment_income_count_income_band_500_000_to_1_000_000` (four carriers under the SPI concept, sixteen under the proxy) and the two benefit-cap bands (a capped population resting on a handful of source households cloned many times; the two count as one concern). A +0.01 % or −3.8 % measured on a cell that thin does not speak to the concern those reasons named, that the solver distorts neighbouring cells to hit a published value with too few carriers. The retirement rests on the register's fence discipline (the row measures inside the fence at weights that never solved toward it), and the support concern stays live as a diagnostic: the cells are measured on every evaluation and the next build's target-fit diagnostics say whether binding them moved their neighbours (review note on PR #967, third pass). The other two rows (`dwp/uc_payment_dist/LONE_PARENT_annual_payment_10_800_to_12_000`, signed as a one-sided undershoot; the employment 1m+ row's sibling concern is above) were signed for error, which the measurement answers directly. + ## The row measured inside the fence but kept - `dwp.uc.households_housing_element` (microcosm#882): -2.8 % unbound. This is not noise inside the fence: DWP's any-tenure count includes an "other or unknown" tenure the engine's housing element cannot pay, 112,518 of 4,037,650 in calendar 2025 (2.79 %), so the model's any-tenure count is identically social plus private and the measured gap is the structural gap the entry records. María signed that reading in review of microcosm#921 on 2026-09-15; `4173ab38` had retired the entry on the measurement, and this PR restores it (the entry stays a measured-not-fit diagnostic to its 2026-10-15 expiry). diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index d3e48e3f6..aba2d591b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -801,7 +801,7 @@ "minimum_reference_total": 0.0, "reviewed_exclusions_resource": "input_mass_reviewed_exclusions.json" }, - "notes": "Weighted input-mass parity against the active named reference in the registry, with reviewed exclusions scoped per reference so a channel-blind reference exclusion does not suppress comparison against a future channel-aware reference. The tolerance is the worst surviving measured drift at the reviewed floor, exact, with no headroom; the sole breach, charitable_investment_gifts at +15,168.8% on the post-#710 surface, is the per-reference exclusion. The 0.0 floor excludes only arithmetically meaningless zero-mass denominators. Totals stay uncommitted under the UKDS licence, supplied per-run and pinned by totals_sha256; missing sidecar evidence records evidence_absent. Totals keys are flat frame column names since the #611 A4 dedup; that digest re-freeze was keys-only against the #610 values (identity unchanged, values byte-identical). Sensitivity floor, recorded rather than silent (microcosm#706 review): the tolerance is the worst surviving measured drift, and that edge is currently set by a column whose deviation is explained (vintage plus SPI composition, microcosm#703's head-to-head receipt), so at this value the gate is a gross mass-loss/explosion fence rather than a regression detector — the compared population sits at median 0.24 and p90 0.63. Tightening runs through this register: each explained top drift that earns a receipt lets the tolerance re-mint at the worst unexplained survivor, with the full distribution and the exclusions-versus-tolerance tradeoff in experiments/630-uk-gate-adjudication-receipts.md." + "notes": "Weighted input-mass parity against the active named reference in the registry, with reviewed exclusions scoped per reference so a channel-blind reference exclusion does not suppress comparison against a future channel-aware reference. The tolerance is the worst surviving measured drift at the reviewed floor, exact, with no headroom; the reviewed per-reference exclusions (charitable_investment_gifts, signed at +15,168.8% on the post-#710 surface; owned_land, signed for seed variance) were retired on 2026-09-22 when the first release-cut certification measured both inside the fence (-99.0% and -73.5% against the enhanced FRS reference, microcosm#967); the register is empty and the gate's current failures are the two genuine breaches it names. The 0.0 floor excludes only arithmetically meaningless zero-mass denominators. Totals stay uncommitted under the UKDS licence, supplied per-run and pinned by totals_sha256; missing sidecar evidence records evidence_absent. Totals keys are flat frame column names since the #611 A4 dedup; that digest re-freeze was keys-only against the #610 values (identity unchanged, values byte-identical). Sensitivity floor, recorded rather than silent (microcosm#706 review): the tolerance is the worst surviving measured drift, and that edge is currently set by a column whose deviation is explained (vintage plus SPI composition, microcosm#703's head-to-head receipt), so at this value the gate is a gross mass-loss/explosion fence rather than a regression detector — the compared population sits at median 0.24 and p90 0.63. Tightening runs through this register: each explained top drift that earns a receipt lets the tolerance re-mint at the worst unexplained survivor, with the full distribution and the exclusions-versus-tolerance tradeoff in experiments/630-uk-gate-adjudication-receipts.md." }, { "id": "uk_qrf_tail_concentration", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py index e4d891357..b69f8c0b2 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py @@ -273,6 +273,13 @@ def _scored_frame( ) +def uk_default_measure_resolver_factory(scratch_dir: Path, year: int): + """The production measure-resolver factory (public name; the rowwise + driver's national role imports it, so a rename here fails loudly).""" + + return _default_measure_resolver_factory(scratch_dir, year) + + def _default_measure_resolver_factory(scratch_dir: Path, year: int): def build(h5_path: Path, frame: Any): from microcosm.build.uk_runtime.measure_simulation import UKMeasureResolver diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py index 75c6a202e..6f2576458 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py @@ -832,6 +832,7 @@ def _verify_score_receipt( if measures: from microcosm.build.uk_runtime.candidate_score import ( load_uk_incumbent_unresolvable_measures, + uk_incumbent_unresolvable_register_digest, ) reviewed = ( @@ -847,6 +848,23 @@ def _verify_score_receipt( "from both arms stands only on a signed entry, so the cut " "cannot be certified on this receipt." ) + # The receipt names the register it was scored under; against the + # committed register of record that digest must be the committed + # one, so a receipt scored under another register cannot certify. + if reviewed_unresolvable_measures is None: + named = ( + pruned.get("reviewed_register", {}).get("sha256") + if isinstance(pruned.get("reviewed_register"), Mapping) + else None + ) + committed = uk_incumbent_unresolvable_register_digest()["sha256"] + if named != committed: + raise UKReleaseCertificationError( + "the score receipt was scored under incumbent-unresolvable " + f"register {named!r}, not the committed register " + f"({committed}); re-score the candidate on the register of " + "record." + ) # An entry suppresses only inside its window, at the certification's # own evaluation date: a receipt scored while an entry was in force # does not outlive the entry. diff --git a/packages/microcosm-build/tests/test_uk_release_certification.py b/packages/microcosm-build/tests/test_uk_release_certification.py index 8b835c6d3..6746d76e1 100644 --- a/packages/microcosm-build/tests/test_uk_release_certification.py +++ b/packages/microcosm-build/tests/test_uk_release_certification.py @@ -458,3 +458,37 @@ def test_certification_fields_mirror_the_data_contract(green_certification_input assert certification["parent_spine"] == { "sha256": green_certification_inputs["spine_sha256"] } + + +def test_compose_refuses_a_receipt_scored_under_another_register( + green_certification_inputs, +): + """Against the committed register of record, the receipt's register + digest must be the committed one (review A2, second half).""" + from datetime import date + + from microcosm.build.uk_runtime.candidate_score import ( + load_uk_incumbent_unresolvable_measures, + uk_incumbent_unresolvable_register_digest, + ) + + measure = sorted(load_uk_incumbent_unresolvable_measures())[0] + receipt = green_score_receipt(green_certification_inputs["candidate_sha256"]) + receipt["incumbent_unresolvable_pruned"].update( + {"n_pruned": 1, "n_scored": 306, "measures": [measure]} + ) + receipt["incumbent_unresolvable_pruned"]["reviewed_register"]["sha256"] = "0" * 64 + receipt["evaluation"]["scored_surface"].update({"n_pruned": 1, "n_scored": 306}) + path = green_certification_inputs["score_receipt_path"] + path.write_text(json.dumps(receipt), encoding="utf-8") + # Inside the committed entries' window (they take force 2026-09-22). + green_certification_inputs["exclusions_evaluated_on"] = date(2026, 10, 1) + with pytest.raises(UKReleaseCertificationError, match="not the committed register"): + compose_uk_release_certification(**green_certification_inputs) + + receipt["incumbent_unresolvable_pruned"]["reviewed_register"]["sha256"] = ( + uk_incumbent_unresolvable_register_digest()["sha256"] + ) + path.write_text(json.dumps(receipt), encoding="utf-8") + certification = compose_uk_release_certification(**green_certification_inputs) + assert certification["score_receipt"]["evaluation"]["n_pruned"] == 1 diff --git a/packages/microcosm-build/tests/test_uk_rowwise_national_role.py b/packages/microcosm-build/tests/test_uk_rowwise_national_role.py index be74ec157..a607a7a8a 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_national_role.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_national_role.py @@ -676,7 +676,7 @@ def evaluate_uk_candidate_against_incumbent(**kwargs): return receipt_of(kwargs) @staticmethod - def _default_measure_resolver_factory(scratch_dir, year): + def uk_default_measure_resolver_factory(scratch_dir, year): return lambda path, frame: None @staticmethod @@ -1001,3 +1001,67 @@ def test_uk_national_dry_run_records_the_incumbent(monkeypatch, tmp_path, capsys assert plan["incumbent"]["label"] == "efrs_fixture" assert plan["incumbent"]["sha256"] == _sha(tmp_path / "incumbent.h5") assert not (tmp_path / "plan").exists() + + +def test_uk_national_role_loads_the_real_scorer_by_its_public_name() -> None: + """The driver calls the scorer's public factory name; loading the real + module (no stand-in) proves the name exists (Vahid's #965 note 1).""" + builder = _CANDIDATE._load_builder_module() + module = builder._load_candidate_evaluator() + assert module.__name__ == "microcosm.build.uk_runtime.candidate_score" + assert callable(module.uk_default_measure_resolver_factory) + assert callable(module.evaluate_uk_candidate_against_incumbent) + + +def test_uk_score_receipt_telemetry_summary_keeps_the_pruned_block() -> None: + """The staged copy drops the three per-target arrays and keeps the + pruned block whole, including a populated pruned_targets mapping (the + block a real run fills with 120 rows; Vahid's #965 note 2).""" + builder = _CANDIDATE._load_builder_module() + pruned_targets = { + f"dwp/uc/family_{i}": { + "name": f"dwp/uc/family_{i}", + "family": "dwp_universal_credit", + "unresolvable_measure": "benunit.uc_calibration_child_count", + "reason": "provider failed computing benunit.uc_calibration_child_count", + "adjudication": "microcosm#823", + } + for i in range(3) + } + score = { + "artifacts": {"candidate": {"sha256": "1" * 64}}, + "target_drift": [{"target": "a@0", "error": 0.1}], + "signed_asymmetries": [{"id": "x"}], + "measure_resolution": {"candidate": {"rounds": []}}, + "incumbent_unresolvable_pruned": { + "n_pruned": 3, + "n_scored": 1, + "n_surface": 4, + "pruned_targets": pruned_targets, + "measures": ["benunit.uc_calibration_child_count"], + "families": {"dwp_universal_credit": 3}, + "reviewed_register": {"resource": "r.json", "sha256": "2" * 64}, + "note": "pruned", + }, + "evaluation": {"verdict": "passed"}, + } + summary = builder._score_receipt_telemetry_summary(score) + assert set(summary) == { + "artifacts", + "incumbent_unresolvable_pruned", + "evaluation", + } + assert summary["incumbent_unresolvable_pruned"]["pruned_targets"] == pruned_targets + + # No list of mappings survives (the staging content policy's record + # array refusal); the pruned rows are a mapping keyed by target name. + def _record_arrays(value): + if isinstance(value, list): + return any(isinstance(item, dict) for item in value) or any( + _record_arrays(item) for item in value + ) + if isinstance(value, dict): + return any(_record_arrays(item) for item in value.values()) + return False + + assert not _record_arrays(summary) diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index 7487d9912..0ba6e98d4 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -419,10 +419,10 @@ def compatibility_claim_declarer_error(declared_by: object) -> str | None: "211abff22b4eedf9cf69f4b43a6f77ca8966d61a386c1804c3fdb093b0e27aa0" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "462271cdc72631e4b6780be52b53d7ea7572ad97e9b91a00a1f3da199f56858c" + "ee6b1eb9451866654bfeb0d2090a8412e08b8c2caa8bfb16fb9aa3c5b5002d84" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "8baa7f5c0db3f64c5e00859ff7fd2bd4cf2daf611367ec36f519d1e428af1ebd" + "55cbe8b9f9dab5817f02c2f690965dfae6406c16facecd9f7ab3b08ba06c272e" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate @@ -779,7 +779,7 @@ def compatibility_claim_declarer_error(declared_by: object) -> str | None: }, "release_cut": { "gates_manifest_sha256": ( - "4becf8d00a08d319a26d33681b4bed159711a9873f6cae221af3d161929c5943" + "c65d20de918abd8dd2c31c89a49ed51f0ca75a7c5f31ef00e61089406c4db401" ), "policy_sha256": ( "a7250c519e79e22d366316cd4943f5f4bd2cc86a0e76b919ee2ff2eacb2335f3" diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index 75034ae10..e61245fd3 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -119,10 +119,10 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: "211abff22b4eedf9cf69f4b43a6f77ca8966d61a386c1804c3fdb093b0e27aa0" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "462271cdc72631e4b6780be52b53d7ea7572ad97e9b91a00a1f3da199f56858c" + "ee6b1eb9451866654bfeb0d2090a8412e08b8c2caa8bfb16fb9aa3c5b5002d84" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "8baa7f5c0db3f64c5e00859ff7fd2bd4cf2daf611367ec36f519d1e428af1ebd" + "55cbe8b9f9dab5817f02c2f690965dfae6406c16facecd9f7ab3b08ba06c272e" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "6f0243bcda09dad26945376230c44ec3cf55d4e417c3a25e29bae8c59bc1a69d" diff --git a/tools/build_uk_rowwise_candidate.py b/tools/build_uk_rowwise_candidate.py index ed49cbd1f..1bdf91f06 100644 --- a/tools/build_uk_rowwise_candidate.py +++ b/tools/build_uk_rowwise_candidate.py @@ -4706,7 +4706,7 @@ def _evaluate_against_incumbent( incumbent_sha256=incumbent["sha256"], target_registry=inputs["national_registry"], calibration_year=calibration_year, - measure_resolver_factory=module._default_measure_resolver_factory( + measure_resolver_factory=module.uk_default_measure_resolver_factory( out_dir, calibration_year ), candidate_label=output_paths["dataset"].stem,