Mark the benefit units that share their household's rent

changelog.d/frs-shared-household-rent.added.md

@@ -0,0 +1 @@
+Mark the benefit units of FRS shared households that are liable for part of the rent (liable_for_share_of_household_rent), so policyengine-uk can split the household's rent among them. A unit counts where it reports rent (SRENTAMT), housing benefit (HBOTHAMT) or a Universal Credit housing element on a linked benefit record (UCHOUSEL).

policyengine_uk_data/datasets/frs.py

@@ -275,6 +275,68 @@ def frs_boarder_and_lodger_rent(person: pd.DataFrame) -> tuple[np.ndarray, np.nd
     return rent_paid * is_boarder, rent_paid * ~is_boarder
 
 
+UNIVERSAL_CREDIT_BENEFIT_CODE = 95
+
+
+def frs_liable_for_share_of_household_rent(
+    benunit: pd.DataFrame,
+    person: pd.DataFrame,
+    household: pd.DataFrame,
+    benefits: pd.DataFrame | None = None,
+) -> np.ndarray:
+    """Whether each benefit unit shares liability for its household's rent.
+
+    In a shared household (HHSTAT 2: "shared on an equal basis", the head of
+    household unclear or arbitrary), each benefit unit after the first is
+    asked how much rent it pays (SRENTAMT, on its adults' records) and how
+    much housing benefit it gets (HBOTHAMT). The questionnaire takes SRENTAMT
+    after state help with the rent, so a unit whose share is wholly met by
+    Universal Credit can report zero in both. Such a unit is still liable:
+    it counts where a Universal Credit record linked to DWP administrative
+    data carries a housing element (UCHOUSEL on a BENEFIT 95 record). A later
+    unit with any of the three is one of the people liable for HHRENT, the
+    rent of the whole dwelling, and policyengine-uk splits ``rent`` among
+    them. Total Universal Credit alone is not evidence of a rent liability.
+
+    ``benefits`` is the FRS benefits table. Without it, or in a release
+    whose benefits table has no UCHOUSEL column, only SRENTAMT and HBOTHAMT
+    are used.
+
+    Scope: conventional households (HHSTAT 1) are left out. Their later
+    units can also report HBOTHAMT or a Universal Credit housing element,
+    but the data do not say whether they owe a share of HHRENT to the
+    landlord or rent to the householder, which policyengine-uk treats
+    differently (the boarder and lodger inputs).
+
+    The FRS's own split (BURENT) is not used. In shared households of the
+    2023-24 and 2024-25 releases the later units between them get all of
+    HHRENT and benefit unit 1 a further amount; in 2024-25 the units' BURENT
+    sum to about two and a half times HHRENT. The published derivation of
+    HHRENT from benefit unit 1's rent plus the later units' SRENTAMT and
+    HBOTHAMT is the May 2003 derived-variable specification; this function
+    does not rely on a later one.
+    """
+    benunit_id = benunit.benunit_id.values
+    unit_number = benunit_id % 100
+    shared = household.hhstat.reindex(benunit.household_id.values).values == 2
+    srent = (
+        person.srentamt.where(person.srentamt > 0, 0)
+        .groupby(person.benunit_id.values)
+        .sum()
+        .reindex(benunit_id, fill_value=0)
+        .values
+    )
+    hb = np.maximum(0, benunit.hbothamt.fillna(0).values)
+    uc_housing = np.zeros(len(benunit), dtype=bool)
+    if benefits is not None and "uchousel" in benefits.columns:
+        with_housing_element = benefits.benunit_id[
+            (benefits.benefit == UNIVERSAL_CREDIT_BENEFIT_CODE)
+            & (benefits.uchousel > 0)
+        ]
+        uc_housing = np.isin(benunit_id, with_housing_element.values)
+    return (unit_number > 1) & shared & ((srent > 0) | (hb > 0) | uc_housing)
+
+
 def derive_is_in_non_advanced_education(
     current_education,
     is_apprentice=None,
@@ -1320,6 +1382,9 @@ def determine_education_level(fted_val, typeed2_val, age_val):
         * WEEKS_IN_YEAR
     )
     pe_household["rent"] = household.hhrent.fillna(0).values * WEEKS_IN_YEAR
+    pe_benunit["liable_for_share_of_household_rent"] = (
+        frs_liable_for_share_of_household_rent(benunit, person, household, benefits)
+    )
     pe_household["mortgage_interest_repayment"] = (
         household.mortint.fillna(0).values * WEEKS_IN_YEAR
     )

policyengine_uk_data/tests/test_frs_shared_household_rent.py

@@ -0,0 +1,234 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+from policyengine_uk_data.datasets.frs import frs_liable_for_share_of_household_rent
+
+CONVENTIONAL, SHARED = 1, 2
+UNIVERSAL_CREDIT, HOUSING_BENEFIT = 95, 94
+
+
+def frames(units, adults, hhstat):
+    benunit = pd.DataFrame(units, columns=["benunit_id", "household_id", "hbothamt"])
+    person = pd.DataFrame(adults, columns=["person_id", "benunit_id", "srentamt"])
+    household = pd.DataFrame(
+        {"hhstat": list(hhstat.values())}, index=list(hhstat.keys())
+    )
+    return benunit, person, household
+
+
+def test_later_units_of_a_shared_household_that_pay_rent_are_liable():
+    benunit, person, household = frames(
+        [(1_01, 1, 0), (1_02, 1, 0), (1_03, 1, 0)],
+        [(1_001, 1_01, 0), (1_002, 1_02, 120), (1_003, 1_03, 110)],
+        {1: SHARED},
+    )
+    liable = frs_liable_for_share_of_household_rent(benunit, person, household)
+    # Benefit unit 1 is the household head's, liable for the rent anyway.
+    assert liable.tolist() == [False, True, True]
+
+
+def test_housing_benefit_alone_marks_a_unit_liable():
+    benunit, person, household = frames(
+        [(1_01, 1, 0), (1_02, 1, 85)],
+        [(1_001, 1_01, 0), (1_002, 1_02, 0)],
+        {1: SHARED},
+    )
+    liable = frs_liable_for_share_of_household_rent(benunit, person, household)
+    assert liable.tolist() == [False, True]
+
+
+@pytest.mark.parametrize("srentamt", [0, -1, np.nan])
+def test_a_later_unit_paying_nothing_is_not_liable(srentamt):
+    benunit, person, household = frames(
+        [(1_01, 1, 0), (1_02, 1, np.nan)],
+        [(1_001, 1_01, 0), (1_002, 1_02, srentamt)],
+        {1: SHARED},
+    )
+    liable = frs_liable_for_share_of_household_rent(benunit, person, household)
+    assert liable.tolist() == [False, False]
+
+
+def test_units_of_conventional_households_are_never_sharers():
+    benunit, person, household = frames(
+        [(1_01, 1, 0), (1_02, 1, 50), (2_01, 2, 0), (2_02, 2, 0)],
+        [(1_001, 1_01, 0), (1_002, 1_02, 0), (2_001, 2_01, 0), (2_002, 2_02, 90)],
+        {1: CONVENTIONAL, 2: CONVENTIONAL},
+    )
+    liable = frs_liable_for_share_of_household_rent(benunit, person, household)
+    assert not liable.any()
+
+
+def test_a_couple_counts_once_and_households_do_not_leak():
+    # A couple in unit 2 of a shared household reports SRENTAMT on one
+    # partner's record; the next household is conventional.
+    benunit, person, household = frames(
+        [(1_01, 1, 0), (1_02, 1, 0), (2_01, 2, 0), (2_02, 2, 0)],
+        [
+            (1_001, 1_01, 0),
+            (1_002, 1_02, 200),
+            (1_003, 1_02, 0),
+            (2_001, 2_01, 0),
+            (2_002, 2_02, 0),
+        ],
+        {1: SHARED, 2: CONVENTIONAL},
+    )
+    liable = frs_liable_for_share_of_household_rent(benunit, person, household)
+    assert liable.tolist() == [False, True, False, False]
+
+
+def uc_records(rows):
+    """FRS benefits rows: (benefit unit, benefit code, weekly housing element)."""
+    return pd.DataFrame(rows, columns=["benunit_id", "benefit", "uchousel"])
+
+
+def test_the_result_is_one_boolean_per_benefit_unit():
+    benunit, person, household = frames(
+        [(1_01, 1, 0), (1_02, 1, 0), (1_03, 1, 40)],
+        [(1_001, 1_01, 0), (1_002, 1_02, 120), (1_003, 1_03, 0)],
+        {1: SHARED},
+    )
+    liable = frs_liable_for_share_of_household_rent(benunit, person, household)
+    assert isinstance(liable, np.ndarray)
+    assert liable.dtype == bool
+    assert liable.shape == (len(benunit),)
+
+
+def test_the_first_unit_is_never_marked_whatever_it_reports():
+    # Benefit unit 1 is liable as the household head's family; the flag is
+    # only for the later units.
+    benunit, person, household = frames(
+        [(1_01, 1, 60), (1_02, 1, 0)],
+        [(1_001, 1_01, 150), (1_002, 1_02, 0)],
+        {1: SHARED},
+    )
+    benefits = uc_records([(1_01, UNIVERSAL_CREDIT, 100)])
+    liable = frs_liable_for_share_of_household_rent(
+        benunit, person, household, benefits
+    )
+    assert liable.tolist() == [False, False]
+
+
+def test_a_paying_unit_does_not_mark_a_non_paying_unit_of_its_household():
+    benunit, person, household = frames(
+        [(1_01, 1, 0), (1_02, 1, 0), (1_03, 1, 0)],
+        [(1_001, 1_01, 0), (1_002, 1_02, 130), (1_003, 1_03, 0)],
+        {1: SHARED},
+    )
+    liable = frs_liable_for_share_of_household_rent(benunit, person, household)
+    assert liable.tolist() == [False, True, False]
+
+
+def test_a_paying_unit_does_not_mark_the_same_numbered_unit_elsewhere():
+    # Two shared households: unit 2 pays in the first, not in the second.
+    benunit, person, household = frames(
+        [(1_01, 1, 0), (1_02, 1, 0), (2_01, 2, 0), (2_02, 2, 0)],
+        [(1_001, 1_01, 0), (1_002, 1_02, 130), (2_001, 2_01, 0), (2_002, 2_02, 0)],
+        {1: SHARED, 2: SHARED},
+    )
+    liable = frs_liable_for_share_of_household_rent(benunit, person, household)
+    assert liable.tolist() == [False, True, False, False]
+
+
+def test_rent_on_the_second_adult_of_a_couple_counts():
+    benunit, person, household = frames(
+        [(1_01, 1, 0), (1_02, 1, 0)],
+        [(1_001, 1_01, 0), (1_002, 1_02, 0), (1_003, 1_02, 140)],
+        {1: SHARED},
+    )
+    liable = frs_liable_for_share_of_household_rent(benunit, person, household)
+    assert liable.tolist() == [False, True]
+
+
+def test_a_missing_code_on_one_partner_does_not_cancel_the_other_partners_rent():
+    benunit, person, household = frames(
+        [(1_01, 1, 0), (1_02, 1, 0)],
+        [(1_001, 1_01, 0), (1_002, 1_02, -140), (1_003, 1_02, 140)],
+        {1: SHARED},
+    )
+    liable = frs_liable_for_share_of_household_rent(benunit, person, household)
+    assert liable.tolist() == [False, True]
+
+
+@pytest.mark.parametrize("hbothamt", [-1, -9, 0, np.nan])
+def test_a_missing_or_zero_housing_benefit_amount_does_not_mark_a_unit(hbothamt):
+    benunit, person, household = frames(
+        [(1_01, 1, 0), (1_02, 1, hbothamt)],
+        [(1_001, 1_01, 0), (1_002, 1_02, 0)],
+        {1: SHARED},
+    )
+    liable = frs_liable_for_share_of_household_rent(benunit, person, household)
+    assert liable.tolist() == [False, False]
+
+
+def test_the_order_of_the_tables_does_not_matter():
+    units = [(1_01, 1, 0), (1_02, 1, 0), (2_01, 2, 0), (2_02, 2, 70), (2_03, 2, 0)]
+    adults = [
+        (1_001, 1_01, 0),
+        (1_002, 1_02, 120),
+        (2_001, 2_01, 0),
+        (2_002, 2_02, 0),
+        (2_003, 2_03, 0),
+    ]
+    benunit, person, household = frames(units, adults, {1: SHARED, 2: SHARED})
+    expected = dict(zip(benunit.benunit_id, [False, True, False, True, False]))
+    shuffled_units = benunit.iloc[[3, 0, 4, 2, 1]].reset_index(drop=True)
+    shuffled_people = person.iloc[[4, 2, 0, 3, 1]].reset_index(drop=True)
+    shuffled_households = household.iloc[[1, 0]]
+    liable = frs_liable_for_share_of_household_rent(
+        shuffled_units, shuffled_people, shuffled_households
+    )
+    assert liable.tolist() == [expected[i] for i in shuffled_units.benunit_id]
+
+
+def test_a_universal_credit_housing_element_marks_a_unit_that_pays_nothing_itself():
+    # The questionnaire takes SRENTAMT after state help with the rent, so a
+    # joint tenant whose share is wholly met by Universal Credit reports no
+    # rent and no housing benefit.
+    benunit, person, household = frames(
+        [(1_01, 1, 0), (1_02, 1, 0), (1_03, 1, 0)],
+        [(1_001, 1_01, 0), (1_002, 1_02, 0), (1_003, 1_03, 0)],
+        {1: SHARED},
+    )
+    benefits = uc_records([(1_02, UNIVERSAL_CREDIT, 100)])
+    liable = frs_liable_for_share_of_household_rent(
+        benunit, person, household, benefits
+    )
+    assert liable.tolist() == [False, True, False]
+
+
+@pytest.mark.parametrize(
+    "record",
+    [
+        (1_02, UNIVERSAL_CREDIT, 0),
+        (1_02, UNIVERSAL_CREDIT, -1),
+        (1_02, UNIVERSAL_CREDIT, np.nan),
+        # A housing element amount on a record of another benefit.
+        (1_02, HOUSING_BENEFIT, 100),
+        # Another unit's Universal Credit.
+        (2_02, UNIVERSAL_CREDIT, 100),
+    ],
+)
+def test_other_benefit_records_do_not_mark_a_unit(record):
+    benunit, person, household = frames(
+        [(1_01, 1, 0), (1_02, 1, 0), (2_01, 2, 0), (2_02, 2, 0)],
+        [(1_001, 1_01, 0), (1_002, 1_02, 0), (2_001, 2_01, 0), (2_002, 2_02, 0)],
+        {1: SHARED, 2: CONVENTIONAL},
+    )
+    liable = frs_liable_for_share_of_household_rent(
+        benunit, person, household, uc_records([record])
+    )
+    assert not liable.any()
+
+
+def test_a_benefits_table_without_the_housing_element_uses_the_other_two_signals():
+    benunit, person, household = frames(
+        [(1_01, 1, 0), (1_02, 1, 0), (1_03, 1, 55)],
+        [(1_001, 1_01, 0), (1_002, 1_02, 0), (1_003, 1_03, 0)],
+        {1: SHARED},
+    )
+    benefits = uc_records([(1_02, UNIVERSAL_CREDIT, 100)]).drop(columns="uchousel")
+    liable = frs_liable_for_share_of_household_rent(
+        benunit, person, household, benefits
+    )
+    assert liable.tolist() == [False, False, True]