From a9e3371bcb3f3eef76b6ad6aca4a76b10025780f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 1 Oct 2026 11:58:51 -0400 Subject: [PATCH 1/3] Mark the benefit units that share their household's rent In FRS shared households (HHSTAT 2), later benefit units report the rent they pay (SRENTAMT) and their housing benefit (HBOTHAMT); HHRENT is the rent of the whole dwelling. Store liable_for_share_of_household_rent on those units so policyengine-uk can split the household's rent among the people liable for it. The FRS's own BURENT split is not used: in shared households it sums to about two and a half times HHRENT. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk_data/datasets/frs.py | 34 +++++++++ .../tests/test_frs_shared_household_rent.py | 76 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 policyengine_uk_data/tests/test_frs_shared_household_rent.py diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index 7e33f440..3309d807 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -275,6 +275,37 @@ def frs_boarder_and_lodger_rent(person: pd.DataFrame) -> tuple[np.ndarray, np.nd return rent_paid * is_boarder, rent_paid * ~is_boarder +def frs_liable_for_share_of_household_rent( + benunit: pd.DataFrame, person: pd.DataFrame, household: pd.DataFrame +) -> np.ndarray: + """Whether each benefit unit shares liability for its household's rent. + + In a shared household (HHSTAT 2: "shared on an equal basis", the head of + household unclear or arbitrary), each benefit unit after the first is + asked how much rent it pays (SRENTAMT, on its adults' records) and how + much housing benefit it gets (HBOTHAMT). HHRENT is the rent of the whole + dwelling, built from benefit unit 1's rent plus the others' SRENTAMT and + HBOTHAMT, so a later unit with either amount positive is one of the + people liable for HHRENT. policyengine-uk splits ``rent`` among them. + + The FRS's own split (BURENT) is not used: in shared households of the + 2023-24 and 2024-25 releases it gives the later units all of HHRENT + between them and benefit unit 1 a further amount, so the units' BURENT + sum to about two and a half times HHRENT. + """ + unit_number = benunit.benunit_id.values % 100 + shared = household.hhstat.reindex(benunit.household_id.values).values == 2 + srent = ( + person.srentamt.where(person.srentamt > 0, 0) + .groupby(person.benunit_id.values) + .sum() + .reindex(benunit.benunit_id.values, fill_value=0) + .values + ) + hb = np.maximum(0, benunit.hbothamt.fillna(0).values) + return (unit_number > 1) & shared & ((srent > 0) | (hb > 0)) + + def derive_is_in_non_advanced_education( current_education, is_apprentice=None, @@ -1320,6 +1351,9 @@ def determine_education_level(fted_val, typeed2_val, age_val): * WEEKS_IN_YEAR ) pe_household["rent"] = household.hhrent.fillna(0).values * WEEKS_IN_YEAR + pe_benunit["liable_for_share_of_household_rent"] = ( + frs_liable_for_share_of_household_rent(benunit, person, household) + ) pe_household["mortgage_interest_repayment"] = ( household.mortint.fillna(0).values * WEEKS_IN_YEAR ) diff --git a/policyengine_uk_data/tests/test_frs_shared_household_rent.py b/policyengine_uk_data/tests/test_frs_shared_household_rent.py new file mode 100644 index 00000000..6d557d72 --- /dev/null +++ b/policyengine_uk_data/tests/test_frs_shared_household_rent.py @@ -0,0 +1,76 @@ +import numpy as np +import pandas as pd +import pytest + +from policyengine_uk_data.datasets.frs import frs_liable_for_share_of_household_rent + +CONVENTIONAL, SHARED = 1, 2 + + +def frames(units, adults, hhstat): + benunit = pd.DataFrame(units, columns=["benunit_id", "household_id", "hbothamt"]) + person = pd.DataFrame(adults, columns=["person_id", "benunit_id", "srentamt"]) + household = pd.DataFrame( + {"hhstat": list(hhstat.values())}, index=list(hhstat.keys()) + ) + return benunit, person, household + + +def test_later_units_of_a_shared_household_that_pay_rent_are_liable(): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0), (1_03, 1, 0)], + [(1_001, 1_01, 0), (1_002, 1_02, 120), (1_003, 1_03, 110)], + {1: SHARED}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + # Benefit unit 1 is the household head's, liable for the rent anyway. + assert liable.tolist() == [False, True, True] + + +def test_housing_benefit_alone_marks_a_unit_liable(): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 85)], + [(1_001, 1_01, 0), (1_002, 1_02, 0)], + {1: SHARED}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert liable.tolist() == [False, True] + + +@pytest.mark.parametrize("srentamt", [0, -1, np.nan]) +def test_a_later_unit_paying_nothing_is_not_liable(srentamt): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, np.nan)], + [(1_001, 1_01, 0), (1_002, 1_02, srentamt)], + {1: SHARED}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert liable.tolist() == [False, False] + + +def test_units_of_conventional_households_are_never_sharers(): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 50), (2_01, 2, 0), (2_02, 2, 0)], + [(1_001, 1_01, 0), (1_002, 1_02, 0), (2_001, 2_01, 0), (2_002, 2_02, 90)], + {1: CONVENTIONAL, 2: CONVENTIONAL}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert not liable.any() + + +def test_a_couple_counts_once_and_households_do_not_leak(): + # A couple in unit 2 of a shared household reports SRENTAMT on one + # partner's record; the next household is conventional. + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0), (2_01, 2, 0), (2_02, 2, 0)], + [ + (1_001, 1_01, 0), + (1_002, 1_02, 200), + (1_003, 1_02, 0), + (2_001, 2_01, 0), + (2_002, 2_02, 0), + ], + {1: SHARED, 2: CONVENTIONAL}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert liable.tolist() == [False, True, False, False] From 479dd3eaaeb22aeee897891a2953f000ee0e219f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 1 Oct 2026 12:02:44 -0400 Subject: [PATCH 2/3] Changelog for the shared-household rent flag Co-Authored-By: Claude Opus 5.5 --- changelog.d/frs-shared-household-rent.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/frs-shared-household-rent.added.md diff --git a/changelog.d/frs-shared-household-rent.added.md b/changelog.d/frs-shared-household-rent.added.md new file mode 100644 index 00000000..998a5ef7 --- /dev/null +++ b/changelog.d/frs-shared-household-rent.added.md @@ -0,0 +1 @@ +Mark the benefit units of FRS shared households that pay part of the rent (liable_for_share_of_household_rent), so policyengine-uk can split the household's rent among them. From 10ce111df0070824809391281e46f6232f8ef7e8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 3 Oct 2026 14:47:33 -0400 Subject: [PATCH 3/3] Count a Universal Credit housing element as a share of the rent; test each signal A later unit of a shared household whose share of the rent is wholly met by Universal Credit reports no SRENTAMT and no HBOTHAMT, since the questionnaire takes the rent after state help. It is still liable, so a housing element on a UC record linked to DWP data (UCHOUSEL, BENEFIT 95) now marks it. Total UC alone does not. Tests added for each mutation the independent review found surviving: the first-unit exclusion, pooling rent within or across households, rent on the second adult, negative codes, dtype and length, table order, and the UC records that must not count. The docstring states the HHSTAT 1 scope limit and separates the observed BURENT failure from the 2003 derivation of HHRENT. Co-Authored-By: Claude Opus 5.5 --- .../frs-shared-household-rent.added.md | 2 +- policyengine_uk_data/datasets/frs.py | 59 +++++-- .../tests/test_frs_shared_household_rent.py | 158 ++++++++++++++++++ 3 files changed, 204 insertions(+), 15 deletions(-) diff --git a/changelog.d/frs-shared-household-rent.added.md b/changelog.d/frs-shared-household-rent.added.md index 998a5ef7..887843bd 100644 --- a/changelog.d/frs-shared-household-rent.added.md +++ b/changelog.d/frs-shared-household-rent.added.md @@ -1 +1 @@ -Mark the benefit units of FRS shared households that pay part of the rent (liable_for_share_of_household_rent), so policyengine-uk can split the household's rent among them. +Mark the benefit units of FRS shared households that are liable for part of the rent (liable_for_share_of_household_rent), so policyengine-uk can split the household's rent among them. A unit counts where it reports rent (SRENTAMT), housing benefit (HBOTHAMT) or a Universal Credit housing element on a linked benefit record (UCHOUSEL). diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index 3309d807..fa8ca742 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -275,35 +275,66 @@ def frs_boarder_and_lodger_rent(person: pd.DataFrame) -> tuple[np.ndarray, np.nd return rent_paid * is_boarder, rent_paid * ~is_boarder +UNIVERSAL_CREDIT_BENEFIT_CODE = 95 + + def frs_liable_for_share_of_household_rent( - benunit: pd.DataFrame, person: pd.DataFrame, household: pd.DataFrame + benunit: pd.DataFrame, + person: pd.DataFrame, + household: pd.DataFrame, + benefits: pd.DataFrame | None = None, ) -> np.ndarray: """Whether each benefit unit shares liability for its household's rent. In a shared household (HHSTAT 2: "shared on an equal basis", the head of household unclear or arbitrary), each benefit unit after the first is asked how much rent it pays (SRENTAMT, on its adults' records) and how - much housing benefit it gets (HBOTHAMT). HHRENT is the rent of the whole - dwelling, built from benefit unit 1's rent plus the others' SRENTAMT and - HBOTHAMT, so a later unit with either amount positive is one of the - people liable for HHRENT. policyengine-uk splits ``rent`` among them. - - The FRS's own split (BURENT) is not used: in shared households of the - 2023-24 and 2024-25 releases it gives the later units all of HHRENT - between them and benefit unit 1 a further amount, so the units' BURENT - sum to about two and a half times HHRENT. + much housing benefit it gets (HBOTHAMT). The questionnaire takes SRENTAMT + after state help with the rent, so a unit whose share is wholly met by + Universal Credit can report zero in both. Such a unit is still liable: + it counts where a Universal Credit record linked to DWP administrative + data carries a housing element (UCHOUSEL on a BENEFIT 95 record). A later + unit with any of the three is one of the people liable for HHRENT, the + rent of the whole dwelling, and policyengine-uk splits ``rent`` among + them. Total Universal Credit alone is not evidence of a rent liability. + + ``benefits`` is the FRS benefits table. Without it, or in a release + whose benefits table has no UCHOUSEL column, only SRENTAMT and HBOTHAMT + are used. + + Scope: conventional households (HHSTAT 1) are left out. Their later + units can also report HBOTHAMT or a Universal Credit housing element, + but the data do not say whether they owe a share of HHRENT to the + landlord or rent to the householder, which policyengine-uk treats + differently (the boarder and lodger inputs). + + The FRS's own split (BURENT) is not used. In shared households of the + 2023-24 and 2024-25 releases the later units between them get all of + HHRENT and benefit unit 1 a further amount; in 2024-25 the units' BURENT + sum to about two and a half times HHRENT. The published derivation of + HHRENT from benefit unit 1's rent plus the later units' SRENTAMT and + HBOTHAMT is the May 2003 derived-variable specification; this function + does not rely on a later one. """ - unit_number = benunit.benunit_id.values % 100 + benunit_id = benunit.benunit_id.values + unit_number = benunit_id % 100 shared = household.hhstat.reindex(benunit.household_id.values).values == 2 srent = ( person.srentamt.where(person.srentamt > 0, 0) .groupby(person.benunit_id.values) .sum() - .reindex(benunit.benunit_id.values, fill_value=0) + .reindex(benunit_id, fill_value=0) .values ) hb = np.maximum(0, benunit.hbothamt.fillna(0).values) - return (unit_number > 1) & shared & ((srent > 0) | (hb > 0)) + uc_housing = np.zeros(len(benunit), dtype=bool) + if benefits is not None and "uchousel" in benefits.columns: + with_housing_element = benefits.benunit_id[ + (benefits.benefit == UNIVERSAL_CREDIT_BENEFIT_CODE) + & (benefits.uchousel > 0) + ] + uc_housing = np.isin(benunit_id, with_housing_element.values) + return (unit_number > 1) & shared & ((srent > 0) | (hb > 0) | uc_housing) def derive_is_in_non_advanced_education( @@ -1352,7 +1383,7 @@ def determine_education_level(fted_val, typeed2_val, age_val): ) pe_household["rent"] = household.hhrent.fillna(0).values * WEEKS_IN_YEAR pe_benunit["liable_for_share_of_household_rent"] = ( - frs_liable_for_share_of_household_rent(benunit, person, household) + frs_liable_for_share_of_household_rent(benunit, person, household, benefits) ) pe_household["mortgage_interest_repayment"] = ( household.mortint.fillna(0).values * WEEKS_IN_YEAR diff --git a/policyengine_uk_data/tests/test_frs_shared_household_rent.py b/policyengine_uk_data/tests/test_frs_shared_household_rent.py index 6d557d72..df97152a 100644 --- a/policyengine_uk_data/tests/test_frs_shared_household_rent.py +++ b/policyengine_uk_data/tests/test_frs_shared_household_rent.py @@ -5,6 +5,7 @@ from policyengine_uk_data.datasets.frs import frs_liable_for_share_of_household_rent CONVENTIONAL, SHARED = 1, 2 +UNIVERSAL_CREDIT, HOUSING_BENEFIT = 95, 94 def frames(units, adults, hhstat): @@ -74,3 +75,160 @@ def test_a_couple_counts_once_and_households_do_not_leak(): ) liable = frs_liable_for_share_of_household_rent(benunit, person, household) assert liable.tolist() == [False, True, False, False] + + +def uc_records(rows): + """FRS benefits rows: (benefit unit, benefit code, weekly housing element).""" + return pd.DataFrame(rows, columns=["benunit_id", "benefit", "uchousel"]) + + +def test_the_result_is_one_boolean_per_benefit_unit(): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0), (1_03, 1, 40)], + [(1_001, 1_01, 0), (1_002, 1_02, 120), (1_003, 1_03, 0)], + {1: SHARED}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert isinstance(liable, np.ndarray) + assert liable.dtype == bool + assert liable.shape == (len(benunit),) + + +def test_the_first_unit_is_never_marked_whatever_it_reports(): + # Benefit unit 1 is liable as the household head's family; the flag is + # only for the later units. + benunit, person, household = frames( + [(1_01, 1, 60), (1_02, 1, 0)], + [(1_001, 1_01, 150), (1_002, 1_02, 0)], + {1: SHARED}, + ) + benefits = uc_records([(1_01, UNIVERSAL_CREDIT, 100)]) + liable = frs_liable_for_share_of_household_rent( + benunit, person, household, benefits + ) + assert liable.tolist() == [False, False] + + +def test_a_paying_unit_does_not_mark_a_non_paying_unit_of_its_household(): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0), (1_03, 1, 0)], + [(1_001, 1_01, 0), (1_002, 1_02, 130), (1_003, 1_03, 0)], + {1: SHARED}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert liable.tolist() == [False, True, False] + + +def test_a_paying_unit_does_not_mark_the_same_numbered_unit_elsewhere(): + # Two shared households: unit 2 pays in the first, not in the second. + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0), (2_01, 2, 0), (2_02, 2, 0)], + [(1_001, 1_01, 0), (1_002, 1_02, 130), (2_001, 2_01, 0), (2_002, 2_02, 0)], + {1: SHARED, 2: SHARED}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert liable.tolist() == [False, True, False, False] + + +def test_rent_on_the_second_adult_of_a_couple_counts(): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0)], + [(1_001, 1_01, 0), (1_002, 1_02, 0), (1_003, 1_02, 140)], + {1: SHARED}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert liable.tolist() == [False, True] + + +def test_a_missing_code_on_one_partner_does_not_cancel_the_other_partners_rent(): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0)], + [(1_001, 1_01, 0), (1_002, 1_02, -140), (1_003, 1_02, 140)], + {1: SHARED}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert liable.tolist() == [False, True] + + +@pytest.mark.parametrize("hbothamt", [-1, -9, 0, np.nan]) +def test_a_missing_or_zero_housing_benefit_amount_does_not_mark_a_unit(hbothamt): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, hbothamt)], + [(1_001, 1_01, 0), (1_002, 1_02, 0)], + {1: SHARED}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert liable.tolist() == [False, False] + + +def test_the_order_of_the_tables_does_not_matter(): + units = [(1_01, 1, 0), (1_02, 1, 0), (2_01, 2, 0), (2_02, 2, 70), (2_03, 2, 0)] + adults = [ + (1_001, 1_01, 0), + (1_002, 1_02, 120), + (2_001, 2_01, 0), + (2_002, 2_02, 0), + (2_003, 2_03, 0), + ] + benunit, person, household = frames(units, adults, {1: SHARED, 2: SHARED}) + expected = dict(zip(benunit.benunit_id, [False, True, False, True, False])) + shuffled_units = benunit.iloc[[3, 0, 4, 2, 1]].reset_index(drop=True) + shuffled_people = person.iloc[[4, 2, 0, 3, 1]].reset_index(drop=True) + shuffled_households = household.iloc[[1, 0]] + liable = frs_liable_for_share_of_household_rent( + shuffled_units, shuffled_people, shuffled_households + ) + assert liable.tolist() == [expected[i] for i in shuffled_units.benunit_id] + + +def test_a_universal_credit_housing_element_marks_a_unit_that_pays_nothing_itself(): + # The questionnaire takes SRENTAMT after state help with the rent, so a + # joint tenant whose share is wholly met by Universal Credit reports no + # rent and no housing benefit. + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0), (1_03, 1, 0)], + [(1_001, 1_01, 0), (1_002, 1_02, 0), (1_003, 1_03, 0)], + {1: SHARED}, + ) + benefits = uc_records([(1_02, UNIVERSAL_CREDIT, 100)]) + liable = frs_liable_for_share_of_household_rent( + benunit, person, household, benefits + ) + assert liable.tolist() == [False, True, False] + + +@pytest.mark.parametrize( + "record", + [ + (1_02, UNIVERSAL_CREDIT, 0), + (1_02, UNIVERSAL_CREDIT, -1), + (1_02, UNIVERSAL_CREDIT, np.nan), + # A housing element amount on a record of another benefit. + (1_02, HOUSING_BENEFIT, 100), + # Another unit's Universal Credit. + (2_02, UNIVERSAL_CREDIT, 100), + ], +) +def test_other_benefit_records_do_not_mark_a_unit(record): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0), (2_01, 2, 0), (2_02, 2, 0)], + [(1_001, 1_01, 0), (1_002, 1_02, 0), (2_001, 2_01, 0), (2_002, 2_02, 0)], + {1: SHARED, 2: CONVENTIONAL}, + ) + liable = frs_liable_for_share_of_household_rent( + benunit, person, household, uc_records([record]) + ) + assert not liable.any() + + +def test_a_benefits_table_without_the_housing_element_uses_the_other_two_signals(): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0), (1_03, 1, 55)], + [(1_001, 1_01, 0), (1_002, 1_02, 0), (1_003, 1_03, 0)], + {1: SHARED}, + ) + benefits = uc_records([(1_02, UNIVERSAL_CREDIT, 100)]).drop(columns="uchousel") + liable = frs_liable_for_share_of_household_rent( + benunit, person, household, benefits + ) + assert liable.tolist() == [False, False, True]