From 5b4f904d725757f77232774621b829d6563b3960 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 07:02:22 -0400 Subject: [PATCH 1/9] Calibrate pension-age Housing Benefit to DWP's GB figures Replace the OBR table 4.9 Housing Benefit target, which is GB DWP-funded spending but was compared with UK-wide modelled Housing Benefit, with DWP Spring 2026 targets for Housing Benefit spending and claims over Pension Credit qualifying age in Great Britain. Targets can now name the countries they cover. The figures under Pension Credit qualifying age are kept but not calibrated: the model pays working-age Housing Benefit to too few records for a target to be met without concentrating it on a handful of them. Stop drawing would_claim_uc for benefit units whose adults have all reached State Pension age, keeping every other unit's draw unchanged. Require policyengine-uk 2.102.5, which lets pension-age families make new Housing Benefit claims (PolicyEngine/policyengine-uk#1901). Co-Authored-By: Claude Opus 5.5 --- .gitignore | 2 + changelog.d/hb-dwp-age-targets.fixed.md | 1 + policyengine_uk_data/datasets/frs.py | 42 +++ .../targets/build_loss_matrix.py | 14 +- policyengine_uk_data/targets/schema.py | 9 + .../targets/sources/dwp_housing_benefit.py | 190 ++++++++++++ policyengine_uk_data/targets/sources/obr.py | 7 +- .../tests/test_dwp_housing_benefit_targets.py | 277 ++++++++++++++++++ .../tests/test_would_claim_uc_pension_age.py | 56 ++++ pyproject.toml | 3 +- uv.lock | 87 +++++- 11 files changed, 674 insertions(+), 14 deletions(-) create mode 100644 changelog.d/hb-dwp-age-targets.fixed.md create mode 100644 policyengine_uk_data/targets/sources/dwp_housing_benefit.py create mode 100644 policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py create mode 100644 policyengine_uk_data/tests/test_would_claim_uc_pension_age.py diff --git a/.gitignore b/.gitignore index 9a741bc49..04136e33e 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,5 @@ **/_build !policyengine_uk_data/storage/*.csv **/version.json +# Build output: household weights by local area, derived from FRS microdata +policyengine_uk_data/storage/local_geography_weights.csv.gz diff --git a/changelog.d/hb-dwp-age-targets.fixed.md b/changelog.d/hb-dwp-age-targets.fixed.md new file mode 100644 index 000000000..2c4c56708 --- /dev/null +++ b/changelog.d/hb-dwp-age-targets.fixed.md @@ -0,0 +1 @@ +Calibrate pension-age Housing Benefit to DWP's Great Britain spending and claims over Pension Credit qualifying age, replacing the OBR Housing Benefit target that the model compared with UK-wide Housing Benefit. Stop setting `would_claim_uc` for benefit units whose adults have all reached State Pension age, and build with policyengine-uk 2.102.5 so pension-age families can make new Housing Benefit claims. diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index e440d4869..4a95842bb 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -392,6 +392,39 @@ def derive_is_parent_from_frs_microdata( return is_adult_record & has_dependent_children +def derive_every_adult_over_state_pension_age( + person_benunit_ids, + is_adult, + is_over_state_pension_age, + benunit_ids, +) -> np.ndarray: + """Identify benefit units with an adult, all of whose adults have reached + State Pension age. + + Such a unit cannot claim Universal Credit (Welfare Reform Act 2012 + s.4(1)(b)). The inputs are policyengine-uk's ``is_adult`` and + ``is_SP_age``, as in the pension-age route of its + ``housing_benefit_eligible``, so an 18 or 19 year old dependant makes the + unit working-age here as it does there. + """ + + is_adult = np.asarray(is_adult, dtype=bool) + over = is_adult & np.asarray(is_over_state_pension_age, dtype=bool) + counts = ( + pd.DataFrame( + { + "benunit": np.asarray(person_benunit_ids), + "adults": is_adult.astype(int), + "over": over.astype(int), + } + ) + .groupby("benunit")[["adults", "over"]] + .sum() + .reindex(np.asarray(benunit_ids), fill_value=0) + ) + return ((counts.adults > 0) & (counts.over == counts.adults)).to_numpy() + + def _as_non_negative_array(values) -> np.ndarray: values = np.asarray(values, dtype=float) return np.maximum(np.nan_to_num(values, nan=0.0), 0.0) @@ -1537,10 +1570,19 @@ def _reported_benunit_mask(person_column: str) -> np.ndarray: pension_credit_rate, reported_mask=_reported_benunit_mask("pension_credit_reported"), ) + # A benefit unit whose adults have all reached State Pension age cannot + # claim Universal Credit, so it never gets would_claim_uc, even if it + # reports UC. The draw still covers every unit, so the random stream and + # every other unit's value are unchanged. pe_benunit["would_claim_uc"] = assign_takeup_with_reported_anchors( generator.random(len(pe_benunit)), universal_credit_rate, reported_mask=_reported_benunit_mask("universal_credit_reported"), + ) & ~derive_every_adult_over_state_pension_age( + person_benunit_ids=sim.calculate("person_benunit_id", year).values, + is_adult=sim.calculate("is_adult", year).values, + is_over_state_pension_age=sim.calculate("is_SP_age", year).values, + benunit_ids=pe_benunit.benunit_id, ) pe_benunit["would_claim_tfc"] = generator.random(len(pe_benunit)) < tfc_rate diff --git a/policyengine_uk_data/targets/build_loss_matrix.py b/policyengine_uk_data/targets/build_loss_matrix.py index 9552aeea2..44b34272c 100644 --- a/policyengine_uk_data/targets/build_loss_matrix.py +++ b/policyengine_uk_data/targets/build_loss_matrix.py @@ -113,7 +113,7 @@ def create_target_matrix( col = _compute_column(target, ctx, year) if col is None: continue - df[target.name] = col + df[target.name] = restrict_to_countries(col, ctx.country, target.countries) target_names.append(target.name) target_values.append(val) except Exception as e: @@ -122,6 +122,18 @@ def create_target_matrix( return df, pd.Series(target_values, index=target_names) +def restrict_to_countries(column, household_country, countries): + """Zero a household column outside the countries a target covers. + + ``countries`` of None means the target covers the whole UK, and the + column is returned unchanged. + """ + if countries is None: + return column + in_scope = np.isin(np.asarray(household_country), countries) + return np.asarray(column, dtype=float) * in_scope + + def _resolve_value(target: Target, year: int) -> float | None: """Get the target value for a year, falling back to nearest year. diff --git a/policyengine_uk_data/targets/schema.py b/policyengine_uk_data/targets/schema.py index 97b814678..0a678a228 100644 --- a/policyengine_uk_data/targets/schema.py +++ b/policyengine_uk_data/targets/schema.py @@ -19,6 +19,11 @@ class Unit(str, Enum): RATE = "rate" +# DWP statistics cover Great Britain: benefits for Northern Ireland residents +# are the Northern Ireland Executive's responsibility. +GREAT_BRITAIN = ("ENGLAND", "SCOTLAND", "WALES") + + class Target(BaseModel): """A single calibration target from an official statistical source. @@ -41,6 +46,10 @@ class Target(BaseModel): is_count: bool = False reference_url: str | None = None forecast_vintage: str | None = None + # Countries a national source covers when that is less than the UK, as + # values of the model's `country` variable. The loss matrix column only + # counts households in these countries. None means the whole UK. + countries: tuple[str, ...] | None = None # For targets needing custom simulation logic (UC splits, # counterfactuals). Excluded from serialisation. diff --git a/policyengine_uk_data/targets/sources/dwp_housing_benefit.py b/policyengine_uk_data/targets/sources/dwp_housing_benefit.py new file mode 100644 index 000000000..69d17f9b6 --- /dev/null +++ b/policyengine_uk_data/targets/sources/dwp_housing_benefit.py @@ -0,0 +1,190 @@ +"""DWP Housing Benefit targets by age group. + +Housing Benefit spending and caseload over and under Pension Credit +qualifying age, from DWP's benefit expenditure and caseload tables for the +Spring Forecast 2026 (Housing benefits sheet, nominal £ million and +thousands of claims, 2022-23 to 2024-25 outturn, forecast after). Financial +year 2025-26 is stored as 2025. + +Coverage is Great Britain: Housing Benefit for Northern Ireland residents is +paid under Northern Ireland legislation and sits outside DWP's figures, so +the model columns count GB households only. + +These targets replace OBR EFO table 4.9 "Housing benefit (not on JSA)". +That line is DWP-funded spending only, while DWP's tables count all +Housing Benefit paid, including the part local authorities fund (£0.79bn +in 2025-26). The age split sums to that full amount, so targeting both +would ask for two different GB totals. + +DWP splits claims by benefit rules rather than by age alone (Notes, note +5): the two lines equal its Pension Credit plus State Pension benefit +groups and its ESA plus other working-age groups. Under regulation 5 of +both Housing Benefit Regulations 2006 (SI 2006/213 and 2006/214), the +pension-age rules apply when the claimant or partner has reached the +qualifying age for Pension Credit, unless either is on Universal Credit, +Income Support, income-based JSA or income-related ESA. A benefit unit is +therefore over Pension Credit qualifying age here when one of its adults +has reached State Pension age and it gets none of those benefits. Mixed-age +couples who kept pension-age Housing Benefit after May 2019 fall in the +older group, as they do in DWP's benefit groups. + +Only the older group is calibrated. policyengine-uk pays working-age +Housing Benefit only as a continuing award to families that report it and +do not claim Universal Credit: 267 of the 770 working-age records that +report it, about 24,000 weighted claims in 2025-26 against DWP's 460,000. +DWP's working-age figure also includes temporary and supported +accommodation (together £5.2bn of Housing Benefit in 2025-26, not split by +age), which the FRS barely samples. A test build on 2026-09-30 that also +targeted the younger group reached DWP's £5.8bn by loading it onto about +three effective records, and fitted the other targets no better. The +younger group's figures stay here for diagnostics and tests. + +Source: https://www.gov.uk/government/publications/benefit-expenditure-and-caseload-tables-2026 +""" + +import numpy as np + +from policyengine_uk_data.targets.schema import GREAT_BRITAIN, Target, Unit + +_REFERENCE_URL = ( + "https://www.gov.uk/government/publications/" + "benefit-expenditure-and-caseload-tables-2026" +) +_VINTAGE = "spring_2026" + +# Benefits that keep a claim under the working-age Housing Benefit rules +# when the claimant or partner has reached Pension Credit qualifying age. +_WORKING_AGE_BENEFITS = ( + "universal_credit", + "income_support", + "jsa_income", + "esa_income", +) + +# Housing benefits sheet rows "Housing Benefit over/under Pension Credit +# qualifying age": expenditure in £ million (nominal) and caseload in +# thousands (annual average, rounded to the nearest thousand by DWP). +_SPENDING_GBP_M = { + "over": { + 2022: 5_890.1, + 2023: 6_248.1, + 2024: 6_851.0, + 2025: 7_114.7, + 2026: 7_268.9, + 2027: 7_340.4, + 2028: 7_464.4, + 2029: 7_747.9, + 2030: 7_941.3, + }, + "under": { + 2022: 9_689.1, + 2023: 9_524.5, + 2024: 8_603.5, + 2025: 5_778.1, + 2026: 5_205.3, + 2027: 5_493.8, + 2028: 5_788.5, + 2029: 6_151.5, + 2030: 6_405.4, + }, +} +_CASELOAD_THOUSANDS = { + "over": { + 2022: 1_121, + 2023: 1_108, + 2024: 1_104, + 2025: 1_109, + 2026: 1_082, + 2027: 1_057, + 2028: 1_040, + 2029: 1_036, + 2030: 1_042, + }, + "under": { + 2022: 1_388, + 2023: 1_243, + 2024: 968, + 2025: 460, + 2026: 328, + 2027: 339, + 2028: 349, + 2029: 361, + 2030: 372, + }, +} + + +def _over_pension_credit_age(ctx) -> np.ndarray: + """Benefit units assessed under the pension-age Housing Benefit rules.""" + adult = np.asarray(ctx.pe_person("is_adult"), dtype=bool) + over = adult & np.asarray(ctx.pe_person("is_SP_age"), dtype=bool) + any_over = ( + np.asarray(ctx.sim.map_result(over.astype(float), "person", "benunit")) > 0 + ) + on_working_age_benefit = np.zeros_like(any_over) + for benefit in _WORKING_AGE_BENEFITS: + on_working_age_benefit |= np.asarray(ctx.sim.calculate(benefit).values) > 0 + return any_over & ~on_working_age_benefit + + +# Age groups the calibration targets; see the module docstring. +_CALIBRATED_AGE_GROUPS = ("over",) + + +def _make_compute(age_group: str, count: bool): + def compute(ctx, target: Target, year: int) -> np.ndarray: + housing_benefit = np.asarray( + ctx.sim.calculate("housing_benefit").values, dtype=float + ) + in_group = _over_pension_credit_age(ctx) + if age_group == "under": + in_group = ~in_group + value = (housing_benefit > 0) if count else housing_benefit + return np.asarray(ctx.household_from_family(value * in_group), dtype=float) + + return compute + + +def get_targets() -> list[Target]: + return build_targets(_CALIBRATED_AGE_GROUPS) + + +def build_targets(age_groups=("over", "under")) -> list[Target]: + """Housing Benefit spending and claims targets for these age groups.""" + targets = [] + for age_group in age_groups: + name = f"dwp/housing_benefit/{age_group}_pension_credit_age" + targets.append( + Target( + name=name, + variable="housing_benefit", + source="dwp", + unit=Unit.GBP, + values={ + year: value * 1e6 + for year, value in _SPENDING_GBP_M[age_group].items() + }, + reference_url=_REFERENCE_URL, + forecast_vintage=_VINTAGE, + countries=GREAT_BRITAIN, + custom_compute=_make_compute(age_group, count=False), + ) + ) + targets.append( + Target( + name=f"{name}_claims", + variable="housing_benefit", + source="dwp", + unit=Unit.COUNT, + values={ + year: value * 1e3 + for year, value in _CASELOAD_THOUSANDS[age_group].items() + }, + is_count=True, + reference_url=_REFERENCE_URL, + forecast_vintage=_VINTAGE, + countries=GREAT_BRITAIN, + custom_compute=_make_compute(age_group, count=True), + ) + ) + return targets diff --git a/policyengine_uk_data/targets/sources/obr.py b/policyengine_uk_data/targets/sources/obr.py index ad4e3cf17..bb4fc0ce6 100644 --- a/policyengine_uk_data/targets/sources/obr.py +++ b/policyengine_uk_data/targets/sources/obr.py @@ -482,11 +482,10 @@ def read_49(row_num: int) -> dict[int, float]: result[fy[col]] = float(val) * 1e9 return result + # "Housing benefit (not on JSA)" is not targeted: it is DWP-funded GB + # spending only, and dwp_housing_benefit.py targets total GB Housing + # Benefit split by age instead. benefit_rows = { - "housing_benefit": ( - "Housing benefit (not on JSA)", - "housing_benefit", - ), "pip": ( "Disability living allowance and personal independence p", "pip", diff --git a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py new file mode 100644 index 000000000..f92c2976c --- /dev/null +++ b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py @@ -0,0 +1,277 @@ +"""Tests for the DWP Housing Benefit targets by age group and GB scope.""" + +from types import SimpleNamespace + +import numpy as np +import openpyxl +import pandas as pd +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk_data.targets import get_all_targets +from policyengine_uk_data.targets.build_loss_matrix import restrict_to_countries +from policyengine_uk_data.targets.schema import GREAT_BRITAIN, Unit +from policyengine_uk_data.targets.sources import dwp_housing_benefit, obr + +# Other rows of the same DWP Housing benefits sheet, transcribed separately: +# total Housing Benefit is AME within the welfare cap + AME outside it + LA +# funded (£ million), and the total caseload (thousands). +_DWP_TOTAL_GBP_M = { + 2022: 14_839.6 + 160.8 + 578.7, + 2023: 14_946.4 + 111.0 + 715.2, + 2024: 14_593.4 + 57.2 + 804.0, + 2025: 12_103.3 + 0.0 + 789.6, + 2026: 11_643.4 + 0.0 + 830.8, + 2027: 11_969.5 + 0.0 + 864.6, + 2028: 12_352.1 + 0.0 + 900.8, + 2029: 12_950.1 + 0.0 + 949.2, + 2030: 13_368.2 + 0.0 + 978.5, +} +_DWP_TOTAL_CASELOAD_K = { + 2022: 2_509, + 2023: 2_351, + 2024: 2_072, + 2025: 1_569, + 2026: 1_410, + 2027: 1_396, + 2028: 1_390, + 2029: 1_397, + 2030: 1_414, +} + +_CALIBRATED = { + "dwp/housing_benefit/over_pension_credit_age", + "dwp/housing_benefit/over_pension_credit_age_claims", +} +_ALL = _CALIBRATED | { + "dwp/housing_benefit/under_pension_credit_age", + "dwp/housing_benefit/under_pension_credit_age_claims", +} + + +def _targets(): + """Both age groups, including the one calibration leaves out.""" + return {t.name: t for t in dwp_housing_benefit.build_targets()} + + +def test_targets_cover_great_britain_in_pounds_and_claims(): + targets = _targets() + assert set(targets) == _ALL + for name, target in targets.items(): + assert target.countries == GREAT_BRITAIN + assert target.variable == "housing_benefit" + assert target.is_count == name.endswith("_claims") + assert target.unit == (Unit.COUNT if target.is_count else Unit.GBP) + over = targets["dwp/housing_benefit/over_pension_credit_age"] + under = targets["dwp/housing_benefit/under_pension_credit_age_claims"] + assert over.values[2025] == 7_114.7e6 + assert under.values[2025] == 460e3 + + +def test_age_split_reconciles_with_dwp_totals(): + """Over + under matches DWP's total, including LA-funded spending.""" + targets = _targets() + over = targets["dwp/housing_benefit/over_pension_credit_age"].values + under = targets["dwp/housing_benefit/under_pension_credit_age"].values + over_k = targets["dwp/housing_benefit/over_pension_credit_age_claims"].values + under_k = targets["dwp/housing_benefit/under_pension_credit_age_claims"].values + for year, total in _DWP_TOTAL_GBP_M.items(): + assert abs((over[year] + under[year]) / 1e6 - total) < 0.25, year + for year, total in _DWP_TOTAL_CASELOAD_K.items(): + # DWP rounds each caseload to the nearest thousand. + assert abs((over_k[year] + under_k[year]) / 1e3 - total) <= 1, year + + +def test_only_pension_age_housing_benefit_is_calibrated(): + housing_benefit_targets = { + t.name for t in get_all_targets() if t.variable == "housing_benefit" + } + assert housing_benefit_targets == _CALIBRATED + + +def test_obr_housing_benefit_row_is_not_parsed(): + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "4.9" + ws["B2"] = "Housing benefit (not on JSA)1" + ws["B3"] = "Pension credit" + for row in (2, 3): + for col in "CDEFGHI": + ws[f"{col}{row}"] = 6.0 + names = {t.name for t in obr._parse_welfare(wb)} + assert "obr/pension_credit" in names + assert not any("housing_benefit" in name for name in names) + + +def _ctx( + benunit_of_person, + is_adult, + is_sp_age, + benunit_hb, + household_of_benunit, + benunit_benefits=None, +): + """Minimal loss-matrix context over explicit entity mappings. + + ``benunit_benefits`` maps benefit-unit variables such as + ``universal_credit`` to amounts; any other benefit is zero. + """ + benunit_of_person = np.asarray(benunit_of_person) + household_of_benunit = np.asarray(household_of_benunit) + n_benunits = len(benunit_hb) + n_households = household_of_benunit.max() + 1 if n_benunits else 0 + + def map_result(values, source, target): + values = np.asarray(values, dtype=float) + if (source, target) == ("person", "benunit"): + return np.bincount(benunit_of_person, values, minlength=n_benunits) + if (source, target) == ("benunit", "household"): + return np.bincount(household_of_benunit, values, minlength=n_households) + raise AssertionError((source, target)) + + person = {"is_adult": np.asarray(is_adult), "is_SP_age": np.asarray(is_sp_age)} + benunit_values = {"housing_benefit": benunit_hb, **(benunit_benefits or {})} + sim = SimpleNamespace( + map_result=map_result, + calculate=lambda variable: SimpleNamespace( + values=np.asarray( + benunit_values.get(variable, np.zeros(n_benunits)), dtype=float + ) + ), + ) + return SimpleNamespace( + sim=sim, + pe_person=lambda variable: person[variable], + household_from_family=lambda values: map_result(values, "benunit", "household"), + ) + + +def _column(ctx, name): + target = _targets()[name] + return target.custom_compute(ctx, target, 2025) + + +def test_benefit_rules_assign_mixed_age_couples(): + # Benefit units, one household each: pensioner couple; mixed-age couple + # on pension-age rules; working-age single; pensioner with an + # 18-year-old dependant; mixed-age couple whose younger partner gets + # income-related ESA; mixed-age couple on Universal Credit. + ctx = _ctx( + benunit_of_person=[0, 0, 1, 1, 2, 3, 3, 4, 4, 5, 5], + is_adult=[1] * 11, + is_sp_age=[1, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0], + benunit_hb=[5_000, 4_000, 3_000, 2_000, 1_000, 500], + household_of_benunit=[0, 1, 2, 3, 4, 5], + benunit_benefits={ + "esa_income": [0, 0, 0, 0, 4_000, 0], + "universal_credit": [0, 0, 0, 0, 0, 6_000], + }, + ) + over = _column(ctx, "dwp/housing_benefit/over_pension_credit_age") + under = _column(ctx, "dwp/housing_benefit/under_pension_credit_age") + np.testing.assert_array_equal(over, [5_000, 4_000, 0, 2_000, 0, 0]) + np.testing.assert_array_equal(under, [0, 0, 3_000, 0, 1_000, 500]) + + +@st.composite +def _population(draw): + n_benunits = draw(st.integers(1, 12)) + n_households = draw(st.integers(1, n_benunits)) + household_of_benunit = draw( + st.lists( + st.integers(0, n_households - 1), + min_size=n_benunits, + max_size=n_benunits, + ) + ) + n_people = draw(st.integers(n_benunits, 4 * n_benunits)) + # Every benefit unit has at least one member. + benunit_of_person = list(range(n_benunits)) + draw( + st.lists( + st.integers(0, n_benunits - 1), + min_size=n_people - n_benunits, + max_size=n_people - n_benunits, + ) + ) + flags = st.lists(st.booleans(), min_size=n_people, max_size=n_people) + benunit_hb = draw( + st.lists( + st.one_of(st.just(0.0), st.floats(1, 50_000)), + min_size=n_benunits, + max_size=n_benunits, + ) + ) + benefit = st.lists( + st.sampled_from([0.0, 100.0]), min_size=n_benunits, max_size=n_benunits + ) + return dict( + benunit_of_person=benunit_of_person, + is_adult=draw(flags), + is_sp_age=draw(flags), + benunit_hb=benunit_hb, + household_of_benunit=household_of_benunit, + benunit_benefits={ + name: draw(benefit) for name in dwp_housing_benefit._WORKING_AGE_BENEFITS + }, + ) + + +@settings(max_examples=200, deadline=None) +@given(_population()) +def test_age_split_partitions_housing_benefit(population): + """Over + under is all Housing Benefit, in pounds and in claims, and + the older group is exactly the units with an adult over SPA and no + working-age income benefit.""" + ctx = _ctx(**population) + household = np.asarray(population["household_of_benunit"]) + hb = np.asarray(population["benunit_hb"]) + n_households = household.max() + 1 + over = _column(ctx, "dwp/housing_benefit/over_pension_credit_age") + under = _column(ctx, "dwp/housing_benefit/under_pension_credit_age") + over_k = _column(ctx, "dwp/housing_benefit/over_pension_credit_age_claims") + under_k = _column(ctx, "dwp/housing_benefit/under_pension_credit_age_claims") + np.testing.assert_allclose( + over + under, np.bincount(household, hb, minlength=n_households) + ) + np.testing.assert_array_equal( + over_k + under_k, np.bincount(household, hb > 0, minlength=n_households) + ) + people = pd.DataFrame( + { + "benunit": population["benunit_of_person"], + "older": np.asarray(population["is_adult"]) + & np.asarray(population["is_sp_age"]), + } + ) + older_unit = ( + people.groupby("benunit").older.any().reindex(range(len(hb)), fill_value=False) + ).to_numpy() + on_working_age_benefit = np.any( + [np.asarray(v) > 0 for v in population["benunit_benefits"].values()], axis=0 + ) + expected_over = np.bincount( + household, hb * (older_unit & ~on_working_age_benefit), minlength=n_households + ) + np.testing.assert_allclose(over, expected_over) + + +_COUNTRIES = ["ENGLAND", "SCOTLAND", "WALES", "NORTHERN_IRELAND"] + + +@settings(max_examples=200, deadline=None) +@given( + st.lists( + st.tuples(st.floats(-1e9, 1e9), st.sampled_from(_COUNTRIES)), + min_size=1, + max_size=50, + ) +) +def test_country_restriction_partitions_the_uk(rows): + column = np.array([value for value, _ in rows]) + country = np.array([c for _, c in rows]) + gb = restrict_to_countries(column, country, GREAT_BRITAIN) + ni = restrict_to_countries(column, country, ("NORTHERN_IRELAND",)) + np.testing.assert_allclose(gb + ni, column) + np.testing.assert_array_equal(gb[country == "NORTHERN_IRELAND"], 0) + np.testing.assert_array_equal(restrict_to_countries(gb, country, GREAT_BRITAIN), gb) + assert restrict_to_countries(column, country, None) is column diff --git a/policyengine_uk_data/tests/test_would_claim_uc_pension_age.py b/policyengine_uk_data/tests/test_would_claim_uc_pension_age.py new file mode 100644 index 000000000..251848335 --- /dev/null +++ b/policyengine_uk_data/tests/test_would_claim_uc_pension_age.py @@ -0,0 +1,56 @@ +"""Wholly pension-age benefit units never get would_claim_uc.""" + +import numpy as np +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk_data.datasets.frs import ( + derive_every_adult_over_state_pension_age, +) + + +def test_examples(): + # Units: pensioner couple, mixed-age couple, working-age single, + # pensioner with an 18-year-old dependant, a unit with no adult, and a + # unit with no members at all. + result = derive_every_adult_over_state_pension_age( + person_benunit_ids=[10, 10, 20, 20, 30, 40, 40, 50], + is_adult=[1, 1, 1, 1, 1, 1, 1, 0], + is_over_state_pension_age=[1, 1, 1, 0, 0, 1, 0, 0], + benunit_ids=[10, 20, 30, 40, 50, 60], + ) + np.testing.assert_array_equal(result, [True, False, False, False, False, False]) + + +@settings(max_examples=300, deadline=None) +@given( + st.lists( + st.tuples(st.integers(0, 6), st.booleans(), st.booleans()), + max_size=30, + ), + st.permutations(list(range(8))), +) +def test_matches_a_direct_definition(people, benunit_order): + """Differential check against a per-unit loop, for any unit order.""" + benunit_ids = np.array(benunit_order) + result = derive_every_adult_over_state_pension_age( + person_benunit_ids=[p[0] for p in people], + is_adult=[p[1] for p in people], + is_over_state_pension_age=[p[2] for p in people], + benunit_ids=benunit_ids, + ) + for i, unit in enumerate(benunit_ids): + adults = [over for b, adult, over in people if b == unit and adult] + assert result[i] == (len(adults) > 0 and all(adults)) + + +def test_built_dataset_has_no_pension_age_uc_claimants(baseline): + year = 2025 + adult = baseline.calculate("is_adult", year).values.astype(bool) + over = adult & baseline.calculate("is_SP_age", year).values.astype(bool) + adults = baseline.map_result(adult.astype(float), "person", "benunit") + adults_over = baseline.map_result(over.astype(float), "person", "benunit") + wholly_pension_age = (adults > 0) & (adults_over == adults) + would_claim_uc = baseline.calculate("would_claim_uc", year).values.astype(bool) + assert wholly_pension_age.any() + assert not (would_claim_uc & wholly_pension_age).any() diff --git a/pyproject.toml b/pyproject.toml index beff7f1a4..e46cfa9a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "policyengine", "google-cloud-storage", "google-auth", - "policyengine-uk>=2.93.0", + "policyengine-uk>=2.102.5", "microcalibrate>=0.18.0", "microimpute>=1.0.1", "ruff>=0.9.0", @@ -38,6 +38,7 @@ dependencies = [ dev = [ "ruff>=0.9.0", "pytest", + "hypothesis", "torch", "l0-python>=0.4.0", "tables", diff --git a/uv.lock b/uv.lock index a3e9f44d9..370de706e 100644 --- a/uv.lock +++ b/uv.lock @@ -577,6 +577,75 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/f4/124858007ddf3c61e9b144107304c9152fa80b5b6c168da07d86fe583cc1/huggingface_hub-1.1.5-py3-none-any.whl", hash = "sha256:e88ecc129011f37b868586bbcfae6c56868cae80cd56a79d61575426a3aa0d7d", size = 516000, upload-time = "2025-11-20T15:49:30.926Z" }, ] +[[package]] +name = "hypothesis" +version = "6.168.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/b7/13118bbc45d6d8b9d04e2de779e2a4ff23145ea39efa692b994fb874ca72/hypothesis-6.168.3.tar.gz", hash = "sha256:a43388f9067678fef6e13bdff325b6cfa6961a590498bb37f7ff31589c83bc75", size = 511022, upload-time = "2026-09-28T05:20:58.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/cd/746a2fc1e5e5ba34f07ea6ee1ad07525a3dde5ff4836db3a7980af49a891/hypothesis-6.168.3-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d20972ca134e652e9928ecd200966a8a2857adc2812c1d74b12a872e5cd503eb", size = 791536, upload-time = "2026-09-28T05:18:57.162Z" }, + { url = "https://files.pythonhosted.org/packages/c3/0f/7a158e377b69556c8e12c25c8fd811d0103e54c24a12dab1f3b9819f202e/hypothesis-6.168.3-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:eafbec09d3e87d13d8242411d1f5868b5e879f1bd6d95e233e9ff80c527bea1c", size = 787313, upload-time = "2026-09-28T05:20:18.122Z" }, + { url = "https://files.pythonhosted.org/packages/45/c6/7df9104ee359e8fcd77791781c47e70794964ca69673665e0de2a6590f4c/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73c5627497968cc62d140e9fde1ea21e12cac7b64dc419fea8286cd34ff1ad4e", size = 1124036, upload-time = "2026-09-28T05:20:04.372Z" }, + { url = "https://files.pythonhosted.org/packages/92/0f/8a61715404a73b9a82cb1f976803428d603cdea976a9c30266c72ad6a7ce/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dc56e6dc6eeb0aeb08ac463f847279bde2336c4786faf7ee0af4b93cd031a7", size = 1147875, upload-time = "2026-09-28T05:19:15.495Z" }, + { url = "https://files.pythonhosted.org/packages/e4/3f/2b16e95cc3b9a069afa23830a012aefe377ce4a457a227e5a5026eb827ff/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:753bb501f8560d2e321ed3b62596b4496c56f15668b0a0669231ccc3e6c4e80d", size = 1149489, upload-time = "2026-09-28T05:19:18.886Z" }, + { url = "https://files.pythonhosted.org/packages/f6/e7/d7cf6dc068bb02732b2a6e38a35a0dcb7f2137608198fc79d740f69d197c/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71ab606c472449cb872ef2a7acaec679ee4f651b7f0a477a04ebc85d8933ef8c", size = 1191992, upload-time = "2026-09-28T05:19:21.83Z" }, + { url = "https://files.pythonhosted.org/packages/5f/26/f1e5b25dec15998e8375d722825ec3dae1075cbe7cb1f74221b2312a2e33/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26928956c54748e4dfa587333741ab750246123b93484955646ff316cca27eef", size = 1169911, upload-time = "2026-09-28T05:19:27.996Z" }, + { url = "https://files.pythonhosted.org/packages/03/36/901234a49147e6bf8a5b9e54b775706a223c2b52befea1106ea2aaf15d48/hypothesis-6.168.3-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:0bdfc53c041b61c854fa3761735736b991bc2bddafee45a361cfe4d43027c1ef", size = 1129406, upload-time = "2026-09-28T05:18:42.632Z" }, + { url = "https://files.pythonhosted.org/packages/55/c1/5fc9ff91ec9fba6bb527833386c66812524b5cc75fe7d4a4f9122ee1c420/hypothesis-6.168.3-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:650528e2b1e2a45e95c2df624d4b9364b8ac5a027ba84e1eea2bc5398dd01dcd", size = 1160399, upload-time = "2026-09-28T05:20:23.911Z" }, + { url = "https://files.pythonhosted.org/packages/9b/d2/49ad1ef5c55547c8abfd0fc571b3493806dda530fd5cb3d8263dbad86a72/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:209dc54cdb1b4d6d7020ad8d09e44c49756361b7183adacea4d6f5705085b595", size = 1299906, upload-time = "2026-09-28T05:18:58.417Z" }, + { url = "https://files.pythonhosted.org/packages/0d/5f/f98a26094c4f462c4215807bed0f9a6b5508b27a3c329bcba1bc8adc7ac7/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:af8ca98cd11dc7f9427bb90483abcd9688d4df2e8e63893a30a2423b027ebb11", size = 1425538, upload-time = "2026-09-28T05:20:49.074Z" }, + { url = "https://files.pythonhosted.org/packages/d0/b8/c5b4406e3a6591e51a42f85469351d1d824e0a54b84167aec9b2353759fe/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:f8122cfdd0bc0ba843063effa22ac310bdebdb3a1319bf1e63f14baa119c72ab", size = 1377084, upload-time = "2026-09-28T05:20:06.618Z" }, + { url = "https://files.pythonhosted.org/packages/e0/01/0d84a8ea469d024c15f602799c0b8410fbe03d99fdb0370449b33c3c916f/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4bedbb379eab34f792af7ee9a05aae04e9c08bbb52e0f90f5a2110d4fe4b2fbd", size = 1281162, upload-time = "2026-09-28T05:19:42.768Z" }, + { url = "https://files.pythonhosted.org/packages/22/b5/ea6435038de1a795f005cb603531e8ccc053febc50180f3ab55583c9c60e/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:5a953115b9f5c95133ab2d04efffeec96e5658c3207923dca7285f7db3e6bbef", size = 1300433, upload-time = "2026-09-28T05:18:55.634Z" }, + { url = "https://files.pythonhosted.org/packages/6c/78/fa6c77d9f64b6d592e69e582d83cbce7d7e56897faa4762885a2123b0166/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6173558e676ad25ed1e20507fa4024a0d816dd90f715f77006e5a28f19109026", size = 1336273, upload-time = "2026-09-28T05:19:08.273Z" }, + { url = "https://files.pythonhosted.org/packages/f1/56/4acd0d778bab818c9ea336fcba768bf43bc70db7f84cc373579298df4641/hypothesis-6.168.3-cp310-abi3-win32.whl", hash = "sha256:dc66390fb12d80585aa9222bf538ce8b7aa22cf5d118250647355a1c9e8f62f4", size = 678211, upload-time = "2026-09-28T05:19:23.355Z" }, + { url = "https://files.pythonhosted.org/packages/b0/db/0a02b146ad1c30f9716362f2e1a379dfd6b0f7be03ac1e22c9da24c9e2a9/hypothesis-6.168.3-cp310-abi3-win_amd64.whl", hash = "sha256:92325b276360fe86c5bf71a568c0d53a6d140b0de36dcf029f9164a17803bb24", size = 684906, upload-time = "2026-09-28T05:18:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/a3/c0/958deaf726848f96f52250740bf39f13f476b068e22f73e2b358eaa07532/hypothesis-6.168.3-cp310-abi3-win_arm64.whl", hash = "sha256:3cf6f1eeaf41cd8d60cf1f88fde905ca1dd77c906929a507d6ac7f66f2ccba2a", size = 683292, upload-time = "2026-09-28T05:19:57.404Z" }, + { url = "https://files.pythonhosted.org/packages/0c/a2/6787da846d929e52fc3344d803299c45782dbeac287528af08380e984bf8/hypothesis-6.168.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1b230a850de63334c16654a34a2d547e0179d36b9071d4439b3e7237f6d077e7", size = 793254, upload-time = "2026-09-28T05:19:36.294Z" }, + { url = "https://files.pythonhosted.org/packages/89/ba/2893f5ca42501f4d562cba3229c8994c3a3e5fac66ef60c1e0e0b5220a5d/hypothesis-6.168.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d63b0226cd3e0d8bdd97c3384b22a21934ed4d53246c1c93575dada616672499", size = 784699, upload-time = "2026-09-28T05:18:54.194Z" }, + { url = "https://files.pythonhosted.org/packages/b7/aa/7d7349daf75b71f6f35876f8de115e974c5c04d600e86df1b2779b0883ab/hypothesis-6.168.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0369f5df055f96e117ab12e5f249668ff144731ab5280bc7a205fdf81f990b89", size = 1123016, upload-time = "2026-09-28T05:20:36.372Z" }, + { url = "https://files.pythonhosted.org/packages/00/f0/7774e1ea072708ea46cb25c4aeee5f9978b6c271764809069e8a6789858e/hypothesis-6.168.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f076bcd0f77fdcdb7797826c879099573d03a02e228ebe649ea917081b962ac0", size = 1168989, upload-time = "2026-09-28T05:20:08.44Z" }, + { url = "https://files.pythonhosted.org/packages/a7/7d/113992abed9efbd7944e3a496a382da58152dce126fddf6419ff31a0a57d/hypothesis-6.168.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b823ba1fcec8da730f29316d010b06d3f7e0c3828dcf91020e24c55e7d24652a", size = 1298626, upload-time = "2026-09-28T05:19:40.986Z" }, + { url = "https://files.pythonhosted.org/packages/d5/65/3659fa5e733027e5b37a27486e57f4053535e40853d23501c422bb275043/hypothesis-6.168.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dd2849c269d674e4618590f3b48d443bd4c06b5aef3d8d869086c2e6d213d248", size = 1335184, upload-time = "2026-09-28T05:20:20.087Z" }, + { url = "https://files.pythonhosted.org/packages/b4/6c/35aab2221b5ea65125340f236e1a765a9e9f3b28d89a389ea0033fed29f7/hypothesis-6.168.3-cp313-cp313-win_amd64.whl", hash = "sha256:3ef7d26f5789e691401d5f87eafed9bd2763f0dbe47af6d6b66012a509404766", size = 682173, upload-time = "2026-09-28T05:19:12.762Z" }, + { url = "https://files.pythonhosted.org/packages/6e/79/27b0cb56ff5d2bc92458bf6fcebdb1f0dc01f57f043a178e9f9d74498169/hypothesis-6.168.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:e2d4c68729a13df9af4998d2652cfb5d541c5609b88c306880a5dfb284938ec2", size = 793287, upload-time = "2026-09-28T05:20:38.522Z" }, + { url = "https://files.pythonhosted.org/packages/e8/bd/5342f95c3bc36586777ef8cb8eba87ac7acf0184210327fb0e6d8654e295/hypothesis-6.168.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b345f818083ec99966a43ca4f7b38feb62c6920ce28572bc2bde4948a8b7eaba", size = 784857, upload-time = "2026-09-28T05:20:42.609Z" }, + { url = "https://files.pythonhosted.org/packages/4a/5f/fc774241518a0588d362680a3084275058aab86d6bc02a2c61e1073142d0/hypothesis-6.168.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0608c610fc002978fc5de0f471770d8817e9455cb8649a47983c9f8bccfc1897", size = 1123330, upload-time = "2026-09-28T05:19:29.735Z" }, + { url = "https://files.pythonhosted.org/packages/b0/e6/131f16775a3dca5f4fe27f0d6ad9a9851600098a54a872d163f6ebf0b664/hypothesis-6.168.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9dcf6448b1ecc37f2b23f2d1b3ddfc9ff6b6910614c15a642dc82f419b96037b", size = 1169178, upload-time = "2026-09-28T05:19:53.436Z" }, + { url = "https://files.pythonhosted.org/packages/06/61/c9f5bff8b73321c9fd12f2d69666c6e4861b97b60f5f24fdd6ac293007fd/hypothesis-6.168.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f9f094041dcce5119ebd7bab71062743ab02b6e654d056b370beda78c19e2", size = 1299127, upload-time = "2026-09-28T05:20:14.486Z" }, + { url = "https://files.pythonhosted.org/packages/e4/6d/d4618f8ab12dd76c4d58ea052252759aa2bf0c4e71f4b502ba5d577e0b9f/hypothesis-6.168.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d479985fe73af97badfb72dc6d20c6a353e486a36f6d065f600026e0cf954a86", size = 1335390, upload-time = "2026-09-28T05:20:10.326Z" }, + { url = "https://files.pythonhosted.org/packages/95/b6/727161e17cc297df1aeea945de05c532033b66751625bec192bc5b5ff707/hypothesis-6.168.3-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:785e2c45f8c08e274b4bf1ccae97f1a4e09407e9a68e80790cfab1d17a4a45fa", size = 624291, upload-time = "2026-09-28T05:18:36.376Z" }, + { url = "https://files.pythonhosted.org/packages/77/23/2f9b506392a0b8712440bcb781abc5713be9ffac1303b6d10f3669050882/hypothesis-6.168.3-cp314-cp314-win_amd64.whl", hash = "sha256:320920b1e3dae8611eee8a03d063cf2187446f2a17c38cfb8a7fc1466f71eee2", size = 682064, upload-time = "2026-09-28T05:20:16.336Z" }, + { url = "https://files.pythonhosted.org/packages/50/93/efabfd95eb2b69c0c9fa1e1c83c8290aa2715d46baeb5a7764faf09c133a/hypothesis-6.168.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:35380baa981108a7f60c4eab71e46acd8d8f58440520346a1a6aba06dca7e074", size = 791875, upload-time = "2026-09-28T05:19:37.789Z" }, + { url = "https://files.pythonhosted.org/packages/1b/fc/2a0ada1623a9a33048bbf9f00a7c92513398c6b8865cba5efb854262ce30/hypothesis-6.168.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f0aaaed00438fa6673d856aaee12a4a8afbde82a9f3c5ff487cacfb064239afe", size = 783412, upload-time = "2026-09-28T05:18:48.266Z" }, + { url = "https://files.pythonhosted.org/packages/41/31/73c615d37eeb12da0209555612c32ced628d860267c23cfd2691c6207aea/hypothesis-6.168.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27e6df1576bf838e7f484d4ae5cba92114497ca30afb917056bf1ea33e95675", size = 1121662, upload-time = "2026-09-28T05:19:26.225Z" }, + { url = "https://files.pythonhosted.org/packages/92/54/1893344a3b8bbf83fa7c9bb7e96f6836580213854f46671cb749f7b4706c/hypothesis-6.168.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bc85014577982ec6d2e266edc5cd6e7a7d3674c648791ca983c67a52f89a0c4", size = 1167761, upload-time = "2026-09-28T05:18:49.85Z" }, + { url = "https://files.pythonhosted.org/packages/26/97/a61d82968febd25a0f08ffbbfabfaeead66f68fc5fa73f3c9ece1c36fc8f/hypothesis-6.168.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ccfc29505aa1cdcc254cb9cd701fd0811fef5480addd97d4127a00df023410f7", size = 1297309, upload-time = "2026-09-28T05:19:51.711Z" }, + { url = "https://files.pythonhosted.org/packages/bc/01/fe8e4cf6d39d02f4efadaa351427fa13722086cfc54a2da3579741c5177a/hypothesis-6.168.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32d0699566aaa93f9e97a44705d7164386f1e91de78d277f53bada35e78bcc96", size = 1334260, upload-time = "2026-09-28T05:19:00.121Z" }, + { url = "https://files.pythonhosted.org/packages/39/6a/09177ebc62f4778f94379ecade6a51299d3d8c4f4b75ba636bf3a0202364/hypothesis-6.168.3-cp314-cp314t-win_amd64.whl", hash = "sha256:28d88fa174ecbd4ecbd7bb290f06d0db084a3971c3f511ae2830b65b5e25500f", size = 681992, upload-time = "2026-09-28T05:19:48.043Z" }, + { url = "https://files.pythonhosted.org/packages/f9/f6/890bf33d608cd63348d3146a3ca83362fbf09e589c5e5230a00c403070f4/hypothesis-6.168.3-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:61f5782d807b1e6aa5c9beef1054cb2037e7d1add48a64972cd6ee447778781f", size = 791231, upload-time = "2026-09-28T05:18:43.998Z" }, + { url = "https://files.pythonhosted.org/packages/03/dd/fc36b204f8aa7437c676576d9437f0422de46aa8a729e6cd5bbb0224e759/hypothesis-6.168.3-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:7aacf3cf40c7ce8f9e4924d5b57bc0b068beafdfed2347bcf160a28b4cfbba7b", size = 783171, upload-time = "2026-09-28T05:19:20.353Z" }, + { url = "https://files.pythonhosted.org/packages/5c/d2/3cd087d577db67c404991d7723e64d7cd75570cdc82a0f7329aed6f2086c/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01768a03a30dc54df7fe457c0b34016c84d598fa00d5965ededab93ba4eb3408", size = 1121152, upload-time = "2026-09-28T05:18:39.095Z" }, + { url = "https://files.pythonhosted.org/packages/42/6f/a05a68cc66e7a22701c50bf94fb9ddeb36f4f5d0de3cc130066709eda621/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1a8a4ffc6c6e6e577f2bfbcfebf7cffbb310283ba2532c729570a0a752cebaaa", size = 1144069, upload-time = "2026-09-28T05:20:44.705Z" }, + { url = "https://files.pythonhosted.org/packages/4a/f4/fdd7a093fb2fb3a0ce24256c92ad5bf67936f4669bd06d5ceae4298150ec/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b987d73eba95183a7e59cca6d1925c588aa4307922d9852cc2b4d282a2ae4128", size = 1146692, upload-time = "2026-09-28T05:20:25.83Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c8/6dbd4377e935505ae4fc8ee4c7b18c69994c4bbaca015447c470218bdbee/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4569c39bd97d9573e946429ed676f3b55a7c8ed80920addd67d384a47d59b38", size = 1189480, upload-time = "2026-09-28T05:20:46.734Z" }, + { url = "https://files.pythonhosted.org/packages/ba/bf/ff288b496b690000d2686dcfa7f67855e4c5a6dddb464ed8e4880be83bb2/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6042b8707a4b25bbbfe20b110258fb7549a68951e5525608a8ce06091039e9fc", size = 1167109, upload-time = "2026-09-28T05:19:24.758Z" }, + { url = "https://files.pythonhosted.org/packages/d9/d1/3fee2bc29fc747fabbb27e174515592e390809ca465d3bbbabbcfa3235a8/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:f413629de94d38a7a2ad259697a6752526143cb49c2e2d7bdba19381c80693f6", size = 1126823, upload-time = "2026-09-28T05:18:52.636Z" }, + { url = "https://files.pythonhosted.org/packages/58/6a/585294392fa6a6d9719446a042a777ba98ecca265d9e48cbedc10c88df0f/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f071737e4e775bebba07e319eb645a880d1e0186b4d24bad23255e849d86d483", size = 1155773, upload-time = "2026-09-28T05:20:51.268Z" }, + { url = "https://files.pythonhosted.org/packages/cd/4d/fa283ff79996debf1ff08593e01f8b773eb8211b19b3d6d5d491317a65bf/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:f59a3912858d0609c26054aa1c474847c797937f5e0e1c77f0d3f08032e50f09", size = 1296671, upload-time = "2026-09-28T05:18:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/42/05/98f9c2f628da5afabb6c3e8df5d2d299d2d50bf306b14a6e26d3954c4951/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:f09c05a23a8025dd5cad07c2ed299a46778e72d49ff0dd5bf353bcc0f7dc1580", size = 1422001, upload-time = "2026-09-28T05:19:04.188Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c9/f2109ade29a7ec284d55bca328d784743e8b13e321576fd34ee7e65f99af/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:44ace770bda3a0301739fc5a413c764de790c739df1f1a7218dd049f8594d9f5", size = 1374182, upload-time = "2026-09-28T05:20:27.764Z" }, + { url = "https://files.pythonhosted.org/packages/ce/6d/e05d5f72441564a3bebc71fa155deafd0fd3b015d6014ec8a00edf6e42bb/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:03b131043608f94a2578a2a896a1a72092acb7079815eef5b8513b08447e0e62", size = 1278402, upload-time = "2026-09-28T05:20:56.332Z" }, + { url = "https://files.pythonhosted.org/packages/96/d7/6988a7f1f69c5c530687dce03a32c157e2d878e3a2e32af9269ce016b78a/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:e9784aca26eddfe99b03a0292320db742cc8a74200ef864fdce949f526f973cd", size = 1297770, upload-time = "2026-09-28T05:19:11.387Z" }, + { url = "https://files.pythonhosted.org/packages/a2/28/5bb82b60b836bd2329e1fe01ad94efc2b14dfa806f8e50ed14b795d78770/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:2b52ac363096232bebc2add117e9178d91f4248f4cdb919fd1026b5f86a4bb16", size = 1333977, upload-time = "2026-09-28T05:20:02.483Z" }, + { url = "https://files.pythonhosted.org/packages/8e/7d/d419841b8f65481ea1a50c4ba36f2670d48b8c7a51e4e8586089564cf7f6/hypothesis-6.168.3-cp315-abi3.abi3t-win32.whl", hash = "sha256:d28e3a6b511a74ce37df5274b51f36c2b274fea7365e00b16f0c81e22acd5957", size = 675394, upload-time = "2026-09-28T05:19:55.237Z" }, + { url = "https://files.pythonhosted.org/packages/d8/d2/1de6a2ad100e44817f2e9a8e8ba3eeaf4769621f4f87f2d4966c4971fcc4/hypothesis-6.168.3-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:7b9638789548361a57d984f56619ac694a914c328181d911271618409261ff4a", size = 681699, upload-time = "2026-09-28T05:19:09.957Z" }, + { url = "https://files.pythonhosted.org/packages/c0/79/be3370fd02734d6b1d950183ae9224580d19fa8d356b95348d60f1e3eeb7/hypothesis-6.168.3-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:65d78e4357ec48ed2c67825f06740ee3599be4cfe770a6092bed07108d679ac5", size = 679849, upload-time = "2026-09-28T05:19:06.884Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -1322,7 +1391,7 @@ wheels = [ [[package]] name = "policyengine-core" -version = "3.31.1" +version = "3.32.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dpath" }, @@ -1342,14 +1411,14 @@ dependencies = [ { name = "standard-imghdr" }, { name = "wheel" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9f/1e/cd58cb6947720a5b7850c06a988abbc0eef53e2be192ed49a17ed62d78d4/policyengine_core-3.31.1.tar.gz", hash = "sha256:3ca8f666c3e67b0c0ce992fa3dc9d1ff2c095a3860f3d719883588b2f9e44ec3", size = 500450, upload-time = "2026-08-28T05:06:43.16Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/9f/0789273f03bad9d7274e487c97f212559fbbe1977a0320b38eeff411bbe2/policyengine_core-3.32.11.tar.gz", hash = "sha256:e32f5e89d0a3abbf4d5e7c5233168c26033b45171373947f604ef18920445b5d", size = 430637, upload-time = "2026-09-30T02:10:11.155Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/74/ce/995992049664b6b4cecd1e4db76b143e07aaee8239043057436f48ccbe75/policyengine_core-3.31.1-py3-none-any.whl", hash = "sha256:308200b28aa26bb79d7f0badebac391c0a81704d45ba4ef48ac79e88f0b21e6b", size = 245719, upload-time = "2026-08-28T05:06:41.769Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8b/f6b21e7872ce4681edf3b1794549fcaed19f4d3efc1dfafdb80044952e28/policyengine_core-3.32.11-py3-none-any.whl", hash = "sha256:591acd6161b992f09070c582e4f830f636a8be863b226d2de546fffc5bbe602a", size = 247934, upload-time = "2026-09-30T02:10:09.831Z" }, ] [[package]] name = "policyengine-uk" -version = "2.93.0" +version = "2.102.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -1357,14 +1426,14 @@ dependencies = [ { name = "pydantic" }, { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/af/04/ab8fbb526c22d5c422eed374244d977b123241baa6f96a604c600876647a/policyengine_uk-2.93.0.tar.gz", hash = "sha256:e3b8eab824e7dad3a1fd61fa8503442a8b1b2285e05fff7ab60f93b6a9b22578", size = 1246786, upload-time = "2026-08-28T14:55:16.266Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5d/a1/4700bcb06a11541236f1e77c44d68842731a3854e15ebf7117da0480d59e/policyengine_uk-2.102.5.tar.gz", hash = "sha256:1526264cd114e40c99bebbc9e2603ec6004c5f2a91bbf5d1a89018be4d05a29b", size = 1319280, upload-time = "2026-09-30T07:53:44.344Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6f/74/5e626f9f80b9485b10bcccbef8d93a635abe309bfdfad437b5de38a80fae/policyengine_uk-2.93.0-py3-none-any.whl", hash = "sha256:4f95feffa3b27fab927f9b903f8a9f1f0d63b2fba209d6b8ac010f0f07a5ec7c", size = 2046525, upload-time = "2026-08-28T14:55:14.114Z" }, + { url = "https://files.pythonhosted.org/packages/11/a0/da29962570fc4091227cf9264b56e2ba66f4c268b391108d1f643e38c649/policyengine_uk-2.102.5-py3-none-any.whl", hash = "sha256:c72dc93cfda8d267f24bbf6675bd27f9c50604844bd6dbefaa3cab6d74f371a6", size = 2195332, upload-time = "2026-09-30T07:53:42.803Z" }, ] [[package]] name = "policyengine-uk-data" -version = "1.56.16" +version = "1.57.4" source = { editable = "." } dependencies = [ { name = "google-auth" }, @@ -1392,6 +1461,7 @@ dependencies = [ dev = [ { name = "build" }, { name = "furo" }, + { name = "hypothesis" }, { name = "itables" }, { name = "l0-python" }, { name = "pytest" }, @@ -1410,6 +1480,7 @@ requires-dist = [ { name = "google-auth" }, { name = "google-cloud-storage" }, { name = "huggingface-hub" }, + { name = "hypothesis", marker = "extra == 'dev'" }, { name = "itables", marker = "extra == 'dev'" }, { name = "l0-python", marker = "extra == 'dev'", specifier = ">=0.4.0" }, { name = "microcalibrate", specifier = ">=0.18.0" }, @@ -1419,7 +1490,7 @@ requires-dist = [ { name = "pandas" }, { name = "policyengine" }, { name = "policyengine-core", specifier = ">=3.19.4" }, - { name = "policyengine-uk", specifier = ">=2.93.0" }, + { name = "policyengine-uk", specifier = ">=2.102.5" }, { name = "pydantic", specifier = ">=2.0" }, { name = "pytest", marker = "extra == 'dev'" }, { name = "pyyaml" }, From 25971022f8f7384f8c9436e2e5734fb2fb40d0f6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 08:05:20 -0400 Subject: [PATCH 2/9] Address review: release note, docstring precision, loss-matrix test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit State in the release note that modelled GB Housing Benefit falls to about £8.0bn. Mark the mixed-age classification as an assumption, date the benefit-group equality from 2024-25, and quote the calibrated build's working-age claims. Add a test that the calibrated columns build on the dataset, since the loss matrix skips a target whose column raises, and note that would_claim_uc uses survey-year State Pension age. Co-Authored-By: Claude Opus 5.5 --- changelog.d/hb-dwp-age-targets.changed.md | 1 + changelog.d/hb-dwp-age-targets.fixed.md | 1 - policyengine_uk_data/datasets/frs.py | 4 +++- .../targets/sources/dwp_housing_benefit.py | 9 +++++---- .../tests/test_dwp_housing_benefit_targets.py | 18 ++++++++++++++++++ 5 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 changelog.d/hb-dwp-age-targets.changed.md delete mode 100644 changelog.d/hb-dwp-age-targets.fixed.md diff --git a/changelog.d/hb-dwp-age-targets.changed.md b/changelog.d/hb-dwp-age-targets.changed.md new file mode 100644 index 000000000..759b2a2ff --- /dev/null +++ b/changelog.d/hb-dwp-age-targets.changed.md @@ -0,0 +1 @@ +Calibrate pension-age Housing Benefit to DWP's Great Britain spending and claims over Pension Credit qualifying age, replacing the OBR Housing Benefit target that the model compared with UK-wide Housing Benefit. Modelled GB Housing Benefit for 2025-26 falls from about £11.7bn to £8.0bn (DWP: £12.9bn), because pension-age Housing Benefit was about 1.6 times DWP's figure while working-age, temporary and supported-accommodation Housing Benefit remain largely unmodelled. Stop setting `would_claim_uc` for benefit units whose adults have all reached State Pension age, and build with policyengine-uk 2.102.5 so pension-age families can make new Housing Benefit claims. diff --git a/changelog.d/hb-dwp-age-targets.fixed.md b/changelog.d/hb-dwp-age-targets.fixed.md deleted file mode 100644 index 2c4c56708..000000000 --- a/changelog.d/hb-dwp-age-targets.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Calibrate pension-age Housing Benefit to DWP's Great Britain spending and claims over Pension Credit qualifying age, replacing the OBR Housing Benefit target that the model compared with UK-wide Housing Benefit. Stop setting `would_claim_uc` for benefit units whose adults have all reached State Pension age, and build with policyengine-uk 2.102.5 so pension-age families can make new Housing Benefit claims. diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index 4a95842bb..94e40c2a6 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -1573,7 +1573,9 @@ def _reported_benunit_mask(person_column: str) -> np.ndarray: # A benefit unit whose adults have all reached State Pension age cannot # claim Universal Credit, so it never gets would_claim_uc, even if it # reports UC. The draw still covers every unit, so the random stream and - # every other unit's value are unchanged. + # every other unit's value are unchanged. Ages are not rolled forward, so + # if State Pension age rises above an adult's survey age in a later year, + # that unit stays without would_claim_uc there. pe_benunit["would_claim_uc"] = assign_takeup_with_reported_anchors( generator.random(len(pe_benunit)), universal_credit_rate, diff --git a/policyengine_uk_data/targets/sources/dwp_housing_benefit.py b/policyengine_uk_data/targets/sources/dwp_housing_benefit.py index 69d17f9b6..79870d13d 100644 --- a/policyengine_uk_data/targets/sources/dwp_housing_benefit.py +++ b/policyengine_uk_data/targets/sources/dwp_housing_benefit.py @@ -17,8 +17,8 @@ would ask for two different GB totals. DWP splits claims by benefit rules rather than by age alone (Notes, note -5): the two lines equal its Pension Credit plus State Pension benefit -groups and its ESA plus other working-age groups. Under regulation 5 of +5): from 2024-25 the two lines equal its Pension Credit plus State Pension +benefit groups and its ESA plus other working-age groups. Under regulation 5 of both Housing Benefit Regulations 2006 (SI 2006/213 and 2006/214), the pension-age rules apply when the claimant or partner has reached the qualifying age for Pension Credit, unless either is on Universal Credit, @@ -26,12 +26,13 @@ therefore over Pension Credit qualifying age here when one of its adults has reached State Pension age and it gets none of those benefits. Mixed-age couples who kept pension-age Housing Benefit after May 2019 fall in the -older group, as they do in DWP's benefit groups. +older group; DWP does not publish its rule for them, and this assumes they +sit in its Pension Credit and State Pension groups. Only the older group is calibrated. policyengine-uk pays working-age Housing Benefit only as a continuing award to families that report it and do not claim Universal Credit: 267 of the 770 working-age records that -report it, about 24,000 weighted claims in 2025-26 against DWP's 460,000. +report it, about 17,000 weighted claims in 2025-26 against DWP's 460,000. DWP's working-age figure also includes temporary and supported accommodation (together £5.2bn of Housing Benefit in 2025-26, not split by age), which the FRS barely samples. A test build on 2026-09-30 that also diff --git a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py index f92c2976c..549006b42 100644 --- a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py +++ b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py @@ -275,3 +275,21 @@ def test_country_restriction_partitions_the_uk(rows): np.testing.assert_array_equal(gb[country == "NORTHERN_IRELAND"], 0) np.testing.assert_array_equal(restrict_to_countries(gb, country, GREAT_BRITAIN), gb) assert restrict_to_countries(column, country, None) is column + + +def test_calibrated_columns_build_on_the_dataset(baseline, enhanced_frs): + """The loss matrix skips a target whose column raises, so check that the + calibrated columns build on a real simulation and land near DWP.""" + from policyengine_uk_data.targets.build_loss_matrix import _SimContext + + year = 2025 + baseline.default_calculation_period = str(year) + ctx = _SimContext(baseline, str(year), enhanced_frs, None) + weight = baseline.calculate("household_weight", year).values + for target in dwp_housing_benefit.get_targets(): + column = restrict_to_countries( + target.custom_compute(ctx, target, year), ctx.country, target.countries + ) + assert np.isfinite(column).all(), target.name + ratio = (column * weight).sum() / target.values[year] + assert 0.5 < ratio < 2, (target.name, ratio) From e2e37ba72166d23e0de1a3ade1fb12990182bd55 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 1 Oct 2026 09:57:29 -0400 Subject: [PATCH 3/9] Fix create_frs smoke test; use policyengine-uk's claimant-or-partner flag The create_frs smoke test's fake simulation did not answer the variables the would_claim_uc rule now reads (person_benunit_id, is_adult, is_SP_age), which failed CI with KeyError: 'person_benunit_id'. Pick the claimant-and-partner flag the way the locked policyengine-uk does for Housing Benefit's pension-age route: is_claimant_or_partner where the release defines it, otherwise is_adult as in 2.102.5. Both the would_claim_uc rule and the age-group targets use it. Under 2.102.5 the base FRS is identical to the previous commit's in every column. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk_data/datasets/frs.py | 42 ++++++++++--------- .../targets/sources/dwp_housing_benefit.py | 14 +++++-- .../tests/test_dwp_housing_benefit_targets.py | 1 + .../tests/test_legacy_benefit_proxies.py | 9 +++- .../tests/test_would_claim_uc_pension_age.py | 28 +++++++++---- policyengine_uk_data/utils/benefit_units.py | 15 +++++++ 6 files changed, 75 insertions(+), 34 deletions(-) create mode 100644 policyengine_uk_data/utils/benefit_units.py diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index 94e40c2a6..6b5b13b27 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -24,6 +24,7 @@ add_disability_benefit_flags_from_reported_amounts, drop_internal_disability_reported_amounts, ) +from policyengine_uk_data.utils.benefit_units import claimant_or_partner_variable from policyengine_uk_data.utils.datasets import ( sum_to_entity, categorical, @@ -392,37 +393,36 @@ def derive_is_parent_from_frs_microdata( return is_adult_record & has_dependent_children -def derive_every_adult_over_state_pension_age( +def derive_all_claimants_over_state_pension_age( person_benunit_ids, - is_adult, + is_claimant_or_partner, is_over_state_pension_age, benunit_ids, ) -> np.ndarray: - """Identify benefit units with an adult, all of whose adults have reached + """Identify benefit units whose claimant and any partner have all reached State Pension age. Such a unit cannot claim Universal Credit (Welfare Reform Act 2012 - s.4(1)(b)). The inputs are policyengine-uk's ``is_adult`` and - ``is_SP_age``, as in the pension-age route of its - ``housing_benefit_eligible``, so an 18 or 19 year old dependant makes the - unit working-age here as it does there. + s.4(1)(b)). ``is_claimant_or_partner`` should be the variable that + ``claimant_or_partner_variable`` names, so the rule matches the + pension-age route of policyengine-uk's ``housing_benefit_eligible``. """ - is_adult = np.asarray(is_adult, dtype=bool) - over = is_adult & np.asarray(is_over_state_pension_age, dtype=bool) + claimant = np.asarray(is_claimant_or_partner, dtype=bool) + over = claimant & np.asarray(is_over_state_pension_age, dtype=bool) counts = ( pd.DataFrame( { "benunit": np.asarray(person_benunit_ids), - "adults": is_adult.astype(int), + "claimants": claimant.astype(int), "over": over.astype(int), } ) - .groupby("benunit")[["adults", "over"]] + .groupby("benunit")[["claimants", "over"]] .sum() .reindex(np.asarray(benunit_ids), fill_value=0) ) - return ((counts.adults > 0) & (counts.over == counts.adults)).to_numpy() + return ((counts.claimants > 0) & (counts.over == counts.claimants)).to_numpy() def _as_non_negative_array(values) -> np.ndarray: @@ -1570,19 +1570,21 @@ def _reported_benunit_mask(person_column: str) -> np.ndarray: pension_credit_rate, reported_mask=_reported_benunit_mask("pension_credit_reported"), ) - # A benefit unit whose adults have all reached State Pension age cannot - # claim Universal Credit, so it never gets would_claim_uc, even if it - # reports UC. The draw still covers every unit, so the random stream and - # every other unit's value are unchanged. Ages are not rolled forward, so - # if State Pension age rises above an adult's survey age in a later year, - # that unit stays without would_claim_uc there. + # A benefit unit whose claimant and any partner have all reached State + # Pension age cannot claim Universal Credit, so it never gets + # would_claim_uc, even if it reports UC. The draw still covers every unit, + # so the random stream and every other unit's value are unchanged. Ages + # are not rolled forward, so if State Pension age rises above a claimant's + # survey age in a later year, that unit stays without would_claim_uc there. pe_benunit["would_claim_uc"] = assign_takeup_with_reported_anchors( generator.random(len(pe_benunit)), universal_credit_rate, reported_mask=_reported_benunit_mask("universal_credit_reported"), - ) & ~derive_every_adult_over_state_pension_age( + ) & ~derive_all_claimants_over_state_pension_age( person_benunit_ids=sim.calculate("person_benunit_id", year).values, - is_adult=sim.calculate("is_adult", year).values, + is_claimant_or_partner=sim.calculate( + claimant_or_partner_variable(sim.tax_benefit_system.variables), year + ).values, is_over_state_pension_age=sim.calculate("is_SP_age", year).values, benunit_ids=pe_benunit.benunit_id, ) diff --git a/policyengine_uk_data/targets/sources/dwp_housing_benefit.py b/policyengine_uk_data/targets/sources/dwp_housing_benefit.py index 79870d13d..eff5309f8 100644 --- a/policyengine_uk_data/targets/sources/dwp_housing_benefit.py +++ b/policyengine_uk_data/targets/sources/dwp_housing_benefit.py @@ -23,8 +23,8 @@ pension-age rules apply when the claimant or partner has reached the qualifying age for Pension Credit, unless either is on Universal Credit, Income Support, income-based JSA or income-related ESA. A benefit unit is -therefore over Pension Credit qualifying age here when one of its adults -has reached State Pension age and it gets none of those benefits. Mixed-age +therefore over Pension Credit qualifying age here when its claimant or +partner has reached State Pension age and it gets none of those benefits. Mixed-age couples who kept pension-age Housing Benefit after May 2019 fall in the older group; DWP does not publish its rule for them, and this assumes they sit in its Pension Credit and State Pension groups. @@ -46,6 +46,7 @@ import numpy as np from policyengine_uk_data.targets.schema import GREAT_BRITAIN, Target, Unit +from policyengine_uk_data.utils.benefit_units import claimant_or_partner_variable _REFERENCE_URL = ( "https://www.gov.uk/government/publications/" @@ -117,8 +118,13 @@ def _over_pension_credit_age(ctx) -> np.ndarray: """Benefit units assessed under the pension-age Housing Benefit rules.""" - adult = np.asarray(ctx.pe_person("is_adult"), dtype=bool) - over = adult & np.asarray(ctx.pe_person("is_SP_age"), dtype=bool) + claimant = np.asarray( + ctx.pe_person( + claimant_or_partner_variable(ctx.sim.tax_benefit_system.variables) + ), + dtype=bool, + ) + over = claimant & np.asarray(ctx.pe_person("is_SP_age"), dtype=bool) any_over = ( np.asarray(ctx.sim.map_result(over.astype(float), "person", "benunit")) > 0 ) diff --git a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py index 549006b42..d9a805f14 100644 --- a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py +++ b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py @@ -132,6 +132,7 @@ def map_result(values, source, target): person = {"is_adult": np.asarray(is_adult), "is_SP_age": np.asarray(is_sp_age)} benunit_values = {"housing_benefit": benunit_hb, **(benunit_benefits or {})} sim = SimpleNamespace( + tax_benefit_system=SimpleNamespace(variables={"is_adult": None}), map_result=map_result, calculate=lambda variable: SimpleNamespace( values=np.asarray( diff --git a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py index 5f1acd85f..a6af94e3f 100644 --- a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py +++ b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py @@ -299,6 +299,7 @@ def __init__(self, dataset): "FakeTaxBenefitSystem", (), { + "variables": {"is_adult": None}, "parameters": lambda self, year: type( "FakeParametersRoot", (), @@ -349,7 +350,7 @@ def __init__(self, dataset): }, )() }, - )() + )(), }, )() @@ -360,6 +361,12 @@ def calculate(self, variable, year=None): return np.array([100]) if variable == "state_pension_age": return pd.Series([66]) + if variable == "person_benunit_id": + return pd.Series(self.dataset.person.person_benunit_id.values) + if variable == "is_adult": + return pd.Series(self.dataset.person.age.values >= 18) + if variable == "is_SP_age": + return pd.Series(self.dataset.person.age.values >= 66) if variable in ( "childcare_grant", "parents_learning_allowance", diff --git a/policyengine_uk_data/tests/test_would_claim_uc_pension_age.py b/policyengine_uk_data/tests/test_would_claim_uc_pension_age.py index 251848335..b6a4f634d 100644 --- a/policyengine_uk_data/tests/test_would_claim_uc_pension_age.py +++ b/policyengine_uk_data/tests/test_would_claim_uc_pension_age.py @@ -5,17 +5,18 @@ from hypothesis import strategies as st from policyengine_uk_data.datasets.frs import ( - derive_every_adult_over_state_pension_age, + derive_all_claimants_over_state_pension_age, ) +from policyengine_uk_data.utils.benefit_units import claimant_or_partner_variable def test_examples(): - # Units: pensioner couple, mixed-age couple, working-age single, - # pensioner with an 18-year-old dependant, a unit with no adult, and a - # unit with no members at all. - result = derive_every_adult_over_state_pension_age( + # Units: pensioner couple, mixed-age couple, working-age single, a + # pensioner and a younger person flagged as claimant or partner, a unit + # with no claimant, and a unit with no members at all. + result = derive_all_claimants_over_state_pension_age( person_benunit_ids=[10, 10, 20, 20, 30, 40, 40, 50], - is_adult=[1, 1, 1, 1, 1, 1, 1, 0], + is_claimant_or_partner=[1, 1, 1, 1, 1, 1, 1, 0], is_over_state_pension_age=[1, 1, 1, 0, 0, 1, 0, 0], benunit_ids=[10, 20, 30, 40, 50, 60], ) @@ -33,9 +34,9 @@ def test_examples(): def test_matches_a_direct_definition(people, benunit_order): """Differential check against a per-unit loop, for any unit order.""" benunit_ids = np.array(benunit_order) - result = derive_every_adult_over_state_pension_age( + result = derive_all_claimants_over_state_pension_age( person_benunit_ids=[p[0] for p in people], - is_adult=[p[1] for p in people], + is_claimant_or_partner=[p[1] for p in people], is_over_state_pension_age=[p[2] for p in people], benunit_ids=benunit_ids, ) @@ -44,9 +45,18 @@ def test_matches_a_direct_definition(people, benunit_order): assert result[i] == (len(adults) > 0 and all(adults)) +def test_claimant_or_partner_variable_prefers_the_legal_flag(): + assert claimant_or_partner_variable({"is_adult": 0}) == "is_adult" + assert ( + claimant_or_partner_variable({"is_adult": 0, "is_claimant_or_partner": 0}) + == "is_claimant_or_partner" + ) + + def test_built_dataset_has_no_pension_age_uc_claimants(baseline): year = 2025 - adult = baseline.calculate("is_adult", year).values.astype(bool) + claimant = claimant_or_partner_variable(baseline.tax_benefit_system.variables) + adult = baseline.calculate(claimant, year).values.astype(bool) over = adult & baseline.calculate("is_SP_age", year).values.astype(bool) adults = baseline.map_result(adult.astype(float), "person", "benunit") adults_over = baseline.map_result(over.astype(float), "person", "benunit") diff --git a/policyengine_uk_data/utils/benefit_units.py b/policyengine_uk_data/utils/benefit_units.py new file mode 100644 index 000000000..fd098770b --- /dev/null +++ b/policyengine_uk_data/utils/benefit_units.py @@ -0,0 +1,15 @@ +"""Benefit-unit membership helpers shared by the dataset build and targets.""" + + +def claimant_or_partner_variable(variables) -> str: + """Name of the policyengine-uk variable marking a benefit unit's claimant + and any partner. + + Releases that define ``is_claimant_or_partner`` use it for Housing + Benefit's pension-age route. Earlier releases, such as 2.102.5, use + ``is_adult`` (age 18 or over) there, which also counts an 18 or 19 year + old dependant. + """ + if "is_claimant_or_partner" in variables: + return "is_claimant_or_partner" + return "is_adult" From 304baad04de821caa82a605df52951506b598ab2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 4 Oct 2026 13:17:07 -0400 Subject: [PATCH 4/9] Calibrate working-age Housing Benefit net of supported and temporary accommodation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Max ruled on d703 that working-age Housing Benefit should be calibrated too. policyengine-uk pays no Housing Benefit for specified (supported) or temporary accommodation, so the working-age awards it can pay are general-needs ones. DWP's working-age line includes both kinds, and DWP splits accommodation type only across all ages. The calibrated working-age target is therefore DWP's working-age line less all of its supported and temporary accommodation: £584.8m and 107k claims in 2025-26, a lower bound on working-age general-needs Housing Benefit. It exists for 2024-25 and 2025-26 only; from 2026-27 the all-age accommodation lines exceed the whole working-age line. Seeded calibrations of one saved 2025 input compared the options. Against the full working-age line, uncapped calibration loaded it onto about three effective records. Capping household weights at 20-40 times their prior reached 54-60% of DWP's spending and raised income-related ESA claimants to 1.6-2.0 times DWP's count. Against the net figure, the calibration meets it (1.04x) and no other national target crosses the 10% line compared with calibrating pension-age figures only. Tests: DWP's accommodation lines reconcile with its age split in every year; the net figure equals all-age general needs less pension-age Housing Benefit, the same quantity computed the other way; a property test of the netting; the calibrated working-age column equals the full line's. Co-Authored-By: Claude Opus 5.5 --- .../targets/sources/dwp_housing_benefit.py | 115 +++++++++++++++--- .../tests/test_dwp_housing_benefit_targets.py | 106 +++++++++++++++- 2 files changed, 203 insertions(+), 18 deletions(-) diff --git a/policyengine_uk_data/targets/sources/dwp_housing_benefit.py b/policyengine_uk_data/targets/sources/dwp_housing_benefit.py index eff5309f8..4ed28076e 100644 --- a/policyengine_uk_data/targets/sources/dwp_housing_benefit.py +++ b/policyengine_uk_data/targets/sources/dwp_housing_benefit.py @@ -29,16 +29,30 @@ older group; DWP does not publish its rule for them, and this assumes they sit in its Pension Credit and State Pension groups. -Only the older group is calibrated. policyengine-uk pays working-age -Housing Benefit only as a continuing award to families that report it and -do not claim Universal Credit: 267 of the 770 working-age records that -report it, about 17,000 weighted claims in 2025-26 against DWP's 460,000. -DWP's working-age figure also includes temporary and supported -accommodation (together £5.2bn of Housing Benefit in 2025-26, not split by -age), which the FRS barely samples. A test build on 2026-09-30 that also -targeted the younger group reached DWP's £5.8bn by loading it onto about -three effective records, and fitted the other targets no better. The -younger group's figures stay here for diagnostics and tests. +Working-age Housing Benefit is calibrated net of supported and temporary +accommodation. policyengine-uk pays no Housing Benefit for specified +(supported) or temporary accommodation (``housing_benefit_eligible``: such +claims "are not modelled"; policyengine-uk#1911). So every working-age award +it pays is general-needs Housing Benefit: a continuing award to a family that +reports it and does not claim Universal Credit. DWP's working-age line +includes both kinds, but DWP splits accommodation type only across all ages +(Housing Benefit by Accomodation Type). The calibrated working-age figure is +therefore DWP's working-age line less all of its supported and temporary +accommodation: £584.8m and 107k claims in 2025-26, against the full line's +£5,778.1m and 460k. Part of that accommodation is pension-age, so the figure +is a lower bound on working-age general-needs Housing Benefit. It exists only +for 2024-25 and 2025-26. From 2026-27, DWP's all-age supported and temporary +accommodation exceeds its whole working-age line. + +Seeded test calibrations on 2026-10-04 tried the full working-age line too. +With only the pension-age figures calibrated, the model pays about 17,000 +working-age claims in 2025-26. Calibrated to the full line, it came within +12% of DWP's spending by loading it onto about three effective records. With +household weights capped at 20-40 times their prior, it reached only 54-60% +of that spending, and income-related ESA claimants rose to 1.6-2.0 times +DWP's count. Against the net figure, no other national target crosses the +10% line compared with calibrating the pension-age figures only. The full +working-age lines stay here (group "under") for tests and diagnostics. Source: https://www.gov.uk/government/publications/benefit-expenditure-and-caseload-tables-2026 """ @@ -116,6 +130,75 @@ } +# Housing benefits sheet, "Housing Benefit by Accomodation Type", rows "of +# which Supported Accomodation" and "of which Temporary Accomodation": all +# ages, same units, first published for 2024-25. +_SUPPORTED_GBP_M = { + 2024: 3_262.8, + 2025: 3_631.4, + 2026: 3_936.0, + 2027: 4_164.1, + 2028: 4_399.2, + 2029: 4_687.1, + 2030: 4_892.2, +} +_TEMPORARY_GBP_M = { + 2024: 1_431.1, + 2025: 1_561.9, + 2026: 1_643.7, + 2027: 1_732.1, + 2028: 1_819.1, + 2029: 1_929.8, + 2030: 2_006.6, +} +_SUPPORTED_THOUSANDS = { + 2024: 235, + 2025: 242, + 2026: 246, + 2027: 251, + 2028: 256, + 2029: 262, + 2030: 267, +} +_TEMPORARY_THOUSANDS = { + 2024: 104, + 2025: 111, + 2026: 117, + 2027: 123, + 2028: 129, + 2029: 136, + 2030: 143, +} + + +def net_of_supported_and_temporary( + working_age: dict, supported: dict, temporary: dict +) -> dict: + """Working-age figures less all-age supported and temporary + accommodation, for the years with all three where the result is + positive.""" + net = {} + for year in sorted(working_age.keys() & supported.keys() & temporary.keys()): + value = round(working_age[year] - supported[year] - temporary[year], 1) + if value > 0: + net[year] = value + return net + + +_SPENDING_GBP_M["under_general_needs"] = net_of_supported_and_temporary( + _SPENDING_GBP_M["under"], _SUPPORTED_GBP_M, _TEMPORARY_GBP_M +) +_CASELOAD_THOUSANDS["under_general_needs"] = net_of_supported_and_temporary( + _CASELOAD_THOUSANDS["under"], _SUPPORTED_THOUSANDS, _TEMPORARY_THOUSANDS +) + +_NAMES = { + "over": "dwp/housing_benefit/over_pension_credit_age", + "under": "dwp/housing_benefit/under_pension_credit_age", + "under_general_needs": "dwp/housing_benefit/under_pension_credit_age_general_needs", +} + + def _over_pension_credit_age(ctx) -> np.ndarray: """Benefit units assessed under the pension-age Housing Benefit rules.""" claimant = np.asarray( @@ -135,7 +218,7 @@ def _over_pension_credit_age(ctx) -> np.ndarray: # Age groups the calibration targets; see the module docstring. -_CALIBRATED_AGE_GROUPS = ("over",) +_CALIBRATED_AGE_GROUPS = ("over", "under_general_needs") def _make_compute(age_group: str, count: bool): @@ -144,7 +227,9 @@ def compute(ctx, target: Target, year: int) -> np.ndarray: ctx.sim.calculate("housing_benefit").values, dtype=float ) in_group = _over_pension_credit_age(ctx) - if age_group == "under": + if age_group != "over": + # The model pays no supported or temporary accommodation Housing + # Benefit, so both working-age groups share one column. in_group = ~in_group value = (housing_benefit > 0) if count else housing_benefit return np.asarray(ctx.household_from_family(value * in_group), dtype=float) @@ -156,11 +241,13 @@ def get_targets() -> list[Target]: return build_targets(_CALIBRATED_AGE_GROUPS) -def build_targets(age_groups=("over", "under")) -> list[Target]: +def build_targets( + age_groups=("over", "under", "under_general_needs"), +) -> list[Target]: """Housing Benefit spending and claims targets for these age groups.""" targets = [] for age_group in age_groups: - name = f"dwp/housing_benefit/{age_group}_pension_credit_age" + name = _NAMES[age_group] targets.append( Target( name=name, diff --git a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py index d9a805f14..bc140334c 100644 --- a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py +++ b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py @@ -39,9 +39,32 @@ 2030: 1_414, } +# Same sheet, "Housing Benefit by Accomodation Type: of which General +# Needs" (£ million and thousands, all ages), transcribed separately. +_DWP_GENERAL_NEEDS_GBP_M = { + 2024: 10_760.6, + 2025: 7_699.5, + 2026: 6_894.5, + 2027: 6_938.0, + 2028: 7_034.6, + 2029: 7_282.5, + 2030: 7_448.0, +} +_DWP_GENERAL_NEEDS_K = { + 2024: 1_734, + 2025: 1_216, + 2026: 1_047, + 2027: 1_021, + 2028: 1_004, + 2029: 1_000, + 2030: 1_004, +} + _CALIBRATED = { "dwp/housing_benefit/over_pension_credit_age", "dwp/housing_benefit/over_pension_credit_age_claims", + "dwp/housing_benefit/under_pension_credit_age_general_needs", + "dwp/housing_benefit/under_pension_credit_age_general_needs_claims", } _ALL = _CALIBRATED | { "dwp/housing_benefit/under_pension_credit_age", @@ -50,7 +73,8 @@ def _targets(): - """Both age groups, including the one calibration leaves out.""" + """Every group's targets, including the full working-age line that + calibration leaves out.""" return {t.name: t for t in dwp_housing_benefit.build_targets()} @@ -82,13 +106,72 @@ def test_age_split_reconciles_with_dwp_totals(): assert abs((over_k[year] + under_k[year]) / 1e3 - total) <= 1, year -def test_only_pension_age_housing_benefit_is_calibrated(): +def test_calibration_uses_working_age_net_of_supported_and_temporary(): housing_benefit_targets = { t.name for t in get_all_targets() if t.variable == "housing_benefit" } assert housing_benefit_targets == _CALIBRATED +def test_accommodation_types_reconcile_with_the_age_split(): + """General needs + supported + temporary is all Housing Benefit, the + same total as over + under, in every year DWP splits it.""" + h = dwp_housing_benefit + for year, general_needs in _DWP_GENERAL_NEEDS_GBP_M.items(): + by_type = general_needs + h._SUPPORTED_GBP_M[year] + h._TEMPORARY_GBP_M[year] + by_age = h._SPENDING_GBP_M["over"][year] + h._SPENDING_GBP_M["under"][year] + assert abs(by_type - by_age) <= 0.2, year + by_type_k = ( + _DWP_GENERAL_NEEDS_K[year] + + h._SUPPORTED_THOUSANDS[year] + + h._TEMPORARY_THOUSANDS[year] + ) + by_age_k = ( + h._CASELOAD_THOUSANDS["over"][year] + h._CASELOAD_THOUSANDS["under"][year] + ) + assert abs(by_type_k - by_age_k) <= 1, year + + +def test_working_age_general_needs_two_ways(): + """Working-age less supported and temporary accommodation equals all-age + general needs less pension-age Housing Benefit, since the two splits + share one total; it is defined only where positive (2024-25, 2025-26).""" + h = dwp_housing_benefit + spending = h._SPENDING_GBP_M["under_general_needs"] + caseload = h._CASELOAD_THOUSANDS["under_general_needs"] + assert set(spending) == set(caseload) == {2024, 2025} + assert spending[2025] == 584.8 and caseload[2025] == 107 + for year in spending: + other_way = _DWP_GENERAL_NEEDS_GBP_M[year] - h._SPENDING_GBP_M["over"][year] + assert abs(spending[year] - other_way) <= 0.2, year + other_way_k = _DWP_GENERAL_NEEDS_K[year] - h._CASELOAD_THOUSANDS["over"][year] + assert abs(caseload[year] - other_way_k) <= 1, year + assert spending[year] < h._SPENDING_GBP_M["under"][year] + assert caseload[year] < h._CASELOAD_THOUSANDS["under"][year] + + + +@settings(max_examples=200, deadline=None) +@given( + st.dictionaries(st.integers(2020, 2035), st.floats(0, 1e4), max_size=8), + st.dictionaries(st.integers(2020, 2035), st.floats(0, 1e4), max_size=8), + st.dictionaries(st.integers(2020, 2035), st.floats(0, 1e4), max_size=8), +) +def test_net_of_supported_and_temporary(working_age, supported, temporary): + net = dwp_housing_benefit.net_of_supported_and_temporary( + working_age, supported, temporary + ) + common = working_age.keys() & supported.keys() & temporary.keys() + assert set(net) <= common + for year in common: + difference = working_age[year] - supported[year] - temporary[year] + if year in net: + assert net[year] > 0 + assert abs(net[year] - difference) <= 0.05 + 1e-9 + else: + assert round(difference, 1) <= 0 + + def test_obr_housing_benefit_row_is_not_parsed(): wb = openpyxl.Workbook() ws = wb.active @@ -231,6 +314,18 @@ def test_age_split_partitions_housing_benefit(population): under = _column(ctx, "dwp/housing_benefit/under_pension_credit_age") over_k = _column(ctx, "dwp/housing_benefit/over_pension_credit_age_claims") under_k = _column(ctx, "dwp/housing_benefit/under_pension_credit_age_claims") + # The model pays no supported or temporary accommodation HB, so the + # calibrated working-age group shares the full line's column. + np.testing.assert_array_equal( + _column(ctx, "dwp/housing_benefit/under_pension_credit_age_general_needs"), + under, + ) + np.testing.assert_array_equal( + _column( + ctx, "dwp/housing_benefit/under_pension_credit_age_general_needs_claims" + ), + under_k, + ) np.testing.assert_allclose( over + under, np.bincount(household, hb, minlength=n_households) ) @@ -280,7 +375,9 @@ def test_country_restriction_partitions_the_uk(rows): def test_calibrated_columns_build_on_the_dataset(baseline, enhanced_frs): """The loss matrix skips a target whose column raises, so check that the - calibrated columns build on a real simulation and land near DWP.""" + calibrated columns build on a real simulation and land in range. The + working-age range is wider: a dataset calibrated without its target + carries about a sixth of it (17k of 107k claims in 2025-26).""" from policyengine_uk_data.targets.build_loss_matrix import _SimContext year = 2025 @@ -293,4 +390,5 @@ def test_calibrated_columns_build_on_the_dataset(baseline, enhanced_frs): ) assert np.isfinite(column).all(), target.name ratio = (column * weight).sum() / target.values[year] - assert 0.5 < ratio < 2, (target.name, ratio) + low = 0.5 if "over_pension_credit_age" in target.name else 0.1 + assert low < ratio < 2, (target.name, ratio) From d156176f8a0285f609a253a7bdd1d7cc91036b98 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 4 Oct 2026 14:37:31 -0400 Subject: [PATCH 5/9] Update the release note for the working-age Housing Benefit target Co-Authored-By: Claude Opus 5.5 --- changelog.d/hb-dwp-age-targets.changed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/hb-dwp-age-targets.changed.md b/changelog.d/hb-dwp-age-targets.changed.md index 759b2a2ff..c98ce26b4 100644 --- a/changelog.d/hb-dwp-age-targets.changed.md +++ b/changelog.d/hb-dwp-age-targets.changed.md @@ -1 +1 @@ -Calibrate pension-age Housing Benefit to DWP's Great Britain spending and claims over Pension Credit qualifying age, replacing the OBR Housing Benefit target that the model compared with UK-wide Housing Benefit. Modelled GB Housing Benefit for 2025-26 falls from about £11.7bn to £8.0bn (DWP: £12.9bn), because pension-age Housing Benefit was about 1.6 times DWP's figure while working-age, temporary and supported-accommodation Housing Benefit remain largely unmodelled. Stop setting `would_claim_uc` for benefit units whose adults have all reached State Pension age, and build with policyengine-uk 2.102.5 so pension-age families can make new Housing Benefit claims. +Calibrate Housing Benefit to DWP's Great Britain spending and claims by age group, replacing the OBR Housing Benefit target that the model compared with UK-wide Housing Benefit. Pension-age Housing Benefit is calibrated to DWP's figures over Pension Credit qualifying age. Working-age Housing Benefit is calibrated to DWP's figures under that age less all supported and temporary accommodation Housing Benefit, which policyengine-uk does not model (£585m and 107k claims in 2025-26, against £5.78bn and 460k in total). Modelled GB Housing Benefit for 2025-26 falls from about £11.7bn to £8.5bn (DWP: £12.9bn), because pension-age Housing Benefit was about 1.6 times DWP's figure while about £5.2bn of supported and temporary accommodation Housing Benefit remains unmodelled. Stop setting `would_claim_uc` for benefit units whose adults have all reached State Pension age, and build with policyengine-uk 2.102.5 so pension-age families can make new Housing Benefit claims. From 33092bfa07f8c5fbe58600db56dc44ffeb2a4ae9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 4 Oct 2026 14:40:47 -0400 Subject: [PATCH 6/9] Format the Housing Benefit target tests Co-Authored-By: Claude Opus 5.5 --- policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py | 1 - 1 file changed, 1 deletion(-) diff --git a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py index bc140334c..5267cd241 100644 --- a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py +++ b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py @@ -150,7 +150,6 @@ def test_working_age_general_needs_two_ways(): assert caseload[year] < h._CASELOAD_THOUSANDS["under"][year] - @settings(max_examples=200, deadline=None) @given( st.dictionaries(st.integers(2020, 2035), st.floats(0, 1e4), max_size=8), From 9f18a3d7c6530697e7a0797878b76b1d0e7e49e4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 5 Oct 2026 12:10:49 -0400 Subject: [PATCH 7/9] Don't carry the net working-age HB targets past 2025-26 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The loss matrix fills a year a target doesn't list with the latest earlier value within three years (_resolve_value). That carried the 2025-26 net working-age Housing Benefit targets (£584.8m, 107k claims) into 2026-27 to 2028-29, where DWP's lines give a negative net figure. Targets gain a carry_forward flag (default True, so nothing else changes), and the two net targets set it to False. The build calibrates 2025 only, so the dataset is unchanged; any loss matrix built for 2026-28 now leaves the net targets out. Also reword the docstring: policyengine-uk has no supported or temporary accommodation rules, but the FRS can't identify those residents, so a reported award there is modelled under the general-needs rules and treating the model's working-age HB as general needs is an approximation. The unmodelled supported and temporary accommodation HB (£5.2bn, all ages) is recorded as a limitation (d946). "Within 12%" becomes "about 12% below". Tests: the net targets resolve in 2024-25 and 2025-26 only while the other HB targets still carry forward; a property test of _resolve_value for any values, year and flag. Co-Authored-By: Claude Opus 5.5 --- .../targets/build_loss_matrix.py | 7 ++- policyengine_uk_data/targets/schema.py | 4 ++ .../targets/sources/dwp_housing_benefit.py | 44 ++++++++------ .../tests/test_dwp_housing_benefit_targets.py | 57 ++++++++++++++++++- 4 files changed, 91 insertions(+), 21 deletions(-) diff --git a/policyengine_uk_data/targets/build_loss_matrix.py b/policyengine_uk_data/targets/build_loss_matrix.py index 44b34272c..7ac3c997d 100644 --- a/policyengine_uk_data/targets/build_loss_matrix.py +++ b/policyengine_uk_data/targets/build_loss_matrix.py @@ -137,11 +137,14 @@ def restrict_to_countries(column, household_country, countries): def _resolve_value(target: Target, year: int) -> float | None: """Get the target value for a year, falling back to nearest year. - VOA council tax targets are population-uprated when extrapolating - from their base year (2024). + A missing year takes the latest earlier value within three years, unless + the target sets ``carry_forward=False``. VOA council tax targets are + population-uprated when extrapolating from their base year (2024). """ if year in target.values: return target.values[year] + if not target.carry_forward: + return None available = sorted(target.values.keys()) if not available: return None diff --git a/policyengine_uk_data/targets/schema.py b/policyengine_uk_data/targets/schema.py index 0a678a228..512ee284e 100644 --- a/policyengine_uk_data/targets/schema.py +++ b/policyengine_uk_data/targets/schema.py @@ -50,6 +50,10 @@ class Target(BaseModel): # values of the model's `country` variable. The loss matrix column only # counts households in these countries. None means the whole UK. countries: tuple[str, ...] | None = None + # Whether the loss matrix may carry the latest earlier value forward to a + # later year the target does not list (see _resolve_value). False for a + # figure that exists only in the years it lists. + carry_forward: bool = True # For targets needing custom simulation logic (UC splits, # counterfactuals). Excluded from serialisation. diff --git a/policyengine_uk_data/targets/sources/dwp_housing_benefit.py b/policyengine_uk_data/targets/sources/dwp_housing_benefit.py index 4ed28076e..d822251d6 100644 --- a/policyengine_uk_data/targets/sources/dwp_housing_benefit.py +++ b/policyengine_uk_data/targets/sources/dwp_housing_benefit.py @@ -30,29 +30,36 @@ sit in its Pension Credit and State Pension groups. Working-age Housing Benefit is calibrated net of supported and temporary -accommodation. policyengine-uk pays no Housing Benefit for specified +accommodation. policyengine-uk has no rules or input for specified (supported) or temporary accommodation (``housing_benefit_eligible``: such -claims "are not modelled"; policyengine-uk#1911). So every working-age award -it pays is general-needs Housing Benefit: a continuing award to a family that -reports it and does not claim Universal Credit. DWP's working-age line -includes both kinds, but DWP splits accommodation type only across all ages +claims "are not modelled"; policyengine-uk#1911). It pays working-age +Housing Benefit only as a continuing award to a family that reports it and +does not claim Universal Credit, computed under the general-needs rules. The +FRS does not identify those accommodation types, so a respondent in them who +reports Housing Benefit is modelled the same way; treating the model's +working-age Housing Benefit as general needs is an approximation. DWP's +Spring 2026 forecast tables split accommodation type only across all ages (Housing Benefit by Accomodation Type). The calibrated working-age figure is therefore DWP's working-age line less all of its supported and temporary accommodation: £584.8m and 107k claims in 2025-26, against the full line's £5,778.1m and 460k. Part of that accommodation is pension-age, so the figure -is a lower bound on working-age general-needs Housing Benefit. It exists only -for 2024-25 and 2025-26. From 2026-27, DWP's all-age supported and temporary -accommodation exceeds its whole working-age line. +is a lower bound on working-age general-needs Housing Benefit. It exists +only for 2024-25 and 2025-26, and the targets do not carry it forward +(``carry_forward=False``): from 2026-27, DWP's all-age supported and +temporary accommodation exceeds its whole working-age line, so these lines +give no positive lower bound. That leaves DWP's supported and temporary +accommodation Housing Benefit, £5.2bn across all ages in 2025-26, unmodelled +as such: a known limitation. Seeded test calibrations on 2026-10-04 tried the full working-age line too. With only the pension-age figures calibrated, the model pays about 17,000 -working-age claims in 2025-26. Calibrated to the full line, it came within -12% of DWP's spending by loading it onto about three effective records. With -household weights capped at 20-40 times their prior, it reached only 54-60% -of that spending, and income-related ESA claimants rose to 1.6-2.0 times -DWP's count. Against the net figure, no other national target crosses the -10% line compared with calibrating the pension-age figures only. The full -working-age lines stay here (group "under") for tests and diagnostics. +working-age claims in 2025-26. Calibrated to the full line, it came about +12% below DWP's spending by loading it onto about three effective records. +With household weights capped at 20-40 times their prior, it reached only +54-60% of that spending, and income-related ESA claimants rose to 1.6-2.0 +times DWP's count. Against the net figure, no other national target crosses +the 10% line compared with calibrating the pension-age figures only. The +full working-age lines stay here (group "under") for tests and diagnostics. Source: https://www.gov.uk/government/publications/benefit-expenditure-and-caseload-tables-2026 """ @@ -228,8 +235,9 @@ def compute(ctx, target: Target, year: int) -> np.ndarray: ) in_group = _over_pension_credit_age(ctx) if age_group != "over": - # The model pays no supported or temporary accommodation Housing - # Benefit, so both working-age groups share one column. + # The model has no supported or temporary accommodation rules, + # so both working-age groups share one column (see the module + # docstring). in_group = ~in_group value = (housing_benefit > 0) if count else housing_benefit return np.asarray(ctx.household_from_family(value * in_group), dtype=float) @@ -261,6 +269,7 @@ def build_targets( reference_url=_REFERENCE_URL, forecast_vintage=_VINTAGE, countries=GREAT_BRITAIN, + carry_forward=age_group != "under_general_needs", custom_compute=_make_compute(age_group, count=False), ) ) @@ -278,6 +287,7 @@ def build_targets( reference_url=_REFERENCE_URL, forecast_vintage=_VINTAGE, countries=GREAT_BRITAIN, + carry_forward=age_group != "under_general_needs", custom_compute=_make_compute(age_group, count=True), ) ) diff --git a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py index 5267cd241..bc7b739b0 100644 --- a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py +++ b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py @@ -9,8 +9,11 @@ from hypothesis import strategies as st from policyengine_uk_data.targets import get_all_targets -from policyengine_uk_data.targets.build_loss_matrix import restrict_to_countries -from policyengine_uk_data.targets.schema import GREAT_BRITAIN, Unit +from policyengine_uk_data.targets.build_loss_matrix import ( + _resolve_value, + restrict_to_countries, +) +from policyengine_uk_data.targets.schema import GREAT_BRITAIN, Target, Unit from policyengine_uk_data.targets.sources import dwp_housing_benefit, obr # Other rows of the same DWP Housing benefits sheet, transcribed separately: @@ -150,6 +153,56 @@ def test_working_age_general_needs_two_ways(): assert caseload[year] < h._CASELOAD_THOUSANDS["under"][year] +def test_net_working_age_targets_are_not_carried_forward(): + """The net figure is negative from 2026-27, so the loss matrix must not + reuse 2025-26's value for later years. Other HB targets still resolve.""" + targets = _targets() + for suffix in ("", "_claims"): + net = targets[ + f"dwp/housing_benefit/under_pension_credit_age_general_needs{suffix}" + ] + assert not net.carry_forward + assert _resolve_value(net, 2024) == net.values[2024] + assert _resolve_value(net, 2025) == net.values[2025] + for year in (2023, 2026, 2027, 2028): + assert _resolve_value(net, year) is None, year + for group in ("over", "under"): + target = targets[f"dwp/housing_benefit/{group}_pension_credit_age{suffix}"] + assert target.carry_forward + for year in (2026, 2027, 2028): + assert _resolve_value(target, year) == target.values[year] + assert _resolve_value(target, 2031) == target.values[2030] + + +@settings(max_examples=300, deadline=None) +@given( + st.dictionaries( + st.integers(2015, 2035), st.floats(-1e9, 1e9), min_size=1, max_size=8 + ), + st.integers(2010, 2040), + st.booleans(), +) +def test_resolve_value_carries_forward_only_when_allowed(values, year, carry): + target = Target( + name="test/target", + variable="housing_benefit", + source="test", + unit=Unit.GBP, + values=values, + carry_forward=carry, + ) + resolved = _resolve_value(target, year) + if year in values: + assert resolved == values[year] + return + # Nearest listed year; an earlier year wins a tie. + closest = min(sorted(values), key=lambda y: abs(y - year)) + if carry and closest < year and year - closest <= 3: + assert resolved == values[closest] + else: + assert resolved is None + + @settings(max_examples=200, deadline=None) @given( st.dictionaries(st.integers(2020, 2035), st.floats(0, 1e4), max_size=8), From 954c22508761b9f218e6c26f0c2e835e0534559a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 5 Oct 2026 13:36:44 -0400 Subject: [PATCH 8/9] Say what the model lacks for supported and temporary accommodation; describe the year resolver exactly The changelog and a test comment still said the model pays no supported or temporary accommodation Housing Benefit. It has no rules for it, and the FRS can't identify those residents, so treating its working-age HB as general needs is an approximation. The resolver docstring and the carry_forward comment now describe _resolve_value as it works: the nearest listed year, the earlier on a tie, used only if it is earlier and at most three years away. Co-Authored-By: Claude Opus 5.5 --- changelog.d/hb-dwp-age-targets.changed.md | 2 +- policyengine_uk_data/targets/build_loss_matrix.py | 5 +++-- policyengine_uk_data/targets/schema.py | 6 +++--- .../tests/test_dwp_housing_benefit_targets.py | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/changelog.d/hb-dwp-age-targets.changed.md b/changelog.d/hb-dwp-age-targets.changed.md index c98ce26b4..c36ead1fc 100644 --- a/changelog.d/hb-dwp-age-targets.changed.md +++ b/changelog.d/hb-dwp-age-targets.changed.md @@ -1 +1 @@ -Calibrate Housing Benefit to DWP's Great Britain spending and claims by age group, replacing the OBR Housing Benefit target that the model compared with UK-wide Housing Benefit. Pension-age Housing Benefit is calibrated to DWP's figures over Pension Credit qualifying age. Working-age Housing Benefit is calibrated to DWP's figures under that age less all supported and temporary accommodation Housing Benefit, which policyengine-uk does not model (£585m and 107k claims in 2025-26, against £5.78bn and 460k in total). Modelled GB Housing Benefit for 2025-26 falls from about £11.7bn to £8.5bn (DWP: £12.9bn), because pension-age Housing Benefit was about 1.6 times DWP's figure while about £5.2bn of supported and temporary accommodation Housing Benefit remains unmodelled. Stop setting `would_claim_uc` for benefit units whose adults have all reached State Pension age, and build with policyengine-uk 2.102.5 so pension-age families can make new Housing Benefit claims. +Calibrate Housing Benefit to DWP's Great Britain spending and claims by age group, replacing the OBR Housing Benefit target that the model compared with UK-wide Housing Benefit. Pension-age Housing Benefit is calibrated to DWP's figures over Pension Credit qualifying age. Working-age Housing Benefit is calibrated to DWP's figures under that age less all supported and temporary accommodation Housing Benefit, for which policyengine-uk has no rules (£585m and 107k claims in 2025-26, against £5.78bn and 460k in total); treating the model's working-age Housing Benefit as general needs is an approximation. Modelled GB Housing Benefit for 2025-26 falls from about £11.7bn to £8.5bn (DWP: £12.9bn), because pension-age Housing Benefit was about 1.6 times DWP's figure while about £5.2bn of supported and temporary accommodation Housing Benefit remains unmodelled as such. Stop setting `would_claim_uc` for benefit units whose adults have all reached State Pension age, and build with policyengine-uk 2.102.5 so pension-age families can make new Housing Benefit claims. diff --git a/policyengine_uk_data/targets/build_loss_matrix.py b/policyengine_uk_data/targets/build_loss_matrix.py index 7ac3c997d..3942df2df 100644 --- a/policyengine_uk_data/targets/build_loss_matrix.py +++ b/policyengine_uk_data/targets/build_loss_matrix.py @@ -137,8 +137,9 @@ def restrict_to_countries(column, household_country, countries): def _resolve_value(target: Target, year: int) -> float | None: """Get the target value for a year, falling back to nearest year. - A missing year takes the latest earlier value within three years, unless - the target sets ``carry_forward=False``. VOA council tax targets are + A missing year takes the value of the nearest listed year (the earlier + on a tie) if that year is earlier and at most three years away, unless + the target sets ``carry_forward=False``; otherwise there is no value. VOA council tax targets are population-uprated when extrapolating from their base year (2024). """ if year in target.values: diff --git a/policyengine_uk_data/targets/schema.py b/policyengine_uk_data/targets/schema.py index 512ee284e..73c584e2d 100644 --- a/policyengine_uk_data/targets/schema.py +++ b/policyengine_uk_data/targets/schema.py @@ -50,9 +50,9 @@ class Target(BaseModel): # values of the model's `country` variable. The loss matrix column only # counts households in these countries. None means the whole UK. countries: tuple[str, ...] | None = None - # Whether the loss matrix may carry the latest earlier value forward to a - # later year the target does not list (see _resolve_value). False for a - # figure that exists only in the years it lists. + # Whether the loss matrix may fill a year the target does not list with + # the nearest earlier listed year's value (see _resolve_value). False for + # a figure that exists only in the years it lists. carry_forward: bool = True # For targets needing custom simulation logic (UC splits, diff --git a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py index bc7b739b0..4dccafd55 100644 --- a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py +++ b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py @@ -366,7 +366,7 @@ def test_age_split_partitions_housing_benefit(population): under = _column(ctx, "dwp/housing_benefit/under_pension_credit_age") over_k = _column(ctx, "dwp/housing_benefit/over_pension_credit_age_claims") under_k = _column(ctx, "dwp/housing_benefit/under_pension_credit_age_claims") - # The model pays no supported or temporary accommodation HB, so the + # The model has no supported or temporary accommodation rules, so the # calibrated working-age group shares the full line's column. np.testing.assert_array_equal( _column(ctx, "dwp/housing_benefit/under_pension_credit_age_general_needs"), From f4d8f95c512ad47b902c2a2bb9c12f8853cce7b8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 5 Oct 2026 13:42:15 -0400 Subject: [PATCH 9/9] Describe the year resolver exactly in the carry_forward comment Co-Authored-By: Claude Opus 5.5 --- policyengine_uk_data/targets/schema.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/policyengine_uk_data/targets/schema.py b/policyengine_uk_data/targets/schema.py index 73c584e2d..ebdaba9d0 100644 --- a/policyengine_uk_data/targets/schema.py +++ b/policyengine_uk_data/targets/schema.py @@ -51,8 +51,9 @@ class Target(BaseModel): # counts households in these countries. None means the whole UK. countries: tuple[str, ...] | None = None # Whether the loss matrix may fill a year the target does not list with - # the nearest earlier listed year's value (see _resolve_value). False for - # a figure that exists only in the years it lists. + # the nearest listed year's value (the earlier on a tie), used only when + # that year is earlier and at most three years away (see _resolve_value). + # False for a figure that exists only in the years it lists. carry_forward: bool = True # For targets needing custom simulation logic (UC splits,