From db089cef73ea130ae678dd4355d1e8aad25cceea Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 23:10:22 -0400 Subject: [PATCH 1/2] Stop counting rent paid by boarders and lodgers as their property income FRS CVPAY is the rent a boarder or lodger pays the householder, after deducting any state benefits to help with rent (FRS question CvPay, asked about each person not related to the HRP in the second and later benefit units of a conventional household). frs.py added it to the payer's own property_income. Move the property income calculation into a tested helper, frs_property_income, that keeps SUBRENT and ROYYR1 and drops CVPAY. Co-Authored-By: Claude Opus 5.5 --- .../frs-cvpay-not-property-income.fixed.md | 1 + policyengine_uk_data/datasets/frs.py | 48 ++++---- .../tests/test_frs_property_income.py | 106 ++++++++++++++++++ 3 files changed, 136 insertions(+), 19 deletions(-) create mode 100644 changelog.d/frs-cvpay-not-property-income.fixed.md create mode 100644 policyengine_uk_data/tests/test_frs_property_income.py diff --git a/changelog.d/frs-cvpay-not-property-income.fixed.md b/changelog.d/frs-cvpay-not-property-income.fixed.md new file mode 100644 index 00000000..55546420 --- /dev/null +++ b/changelog.d/frs-cvpay-not-property-income.fixed.md @@ -0,0 +1 @@ +Stop counting the rent a boarder or lodger pays (FRS CVPAY) as that boarder's or lodger's own property income. diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index e440d486..6ae20a90 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -220,6 +220,34 @@ def derive_receives_benefits_in_own_right(pe_person: pd.DataFrame) -> pd.Series: ) +def frs_property_income(person: pd.DataFrame, household: pd.DataFrame) -> np.ndarray: + """Annual property income each person reports in the FRS. + + Two FRS amounts, both weekly in the released data: + + - SUBRENT, rent the household received for letting part of its home to + someone outside the household. It goes to the household reference + person, and only in owner-occupied households (TENTYP2 5 or 6). + ``household`` must be indexed by ``household_id``. + - ROYYR1, the person's rent from other property, before tax and after + allowable expenses. + + CVPAY is not included. It is the rent that a boarder or lodger pays the + householder, and it sits on the boarder's or lodger's own adult record. + The FRS question (CvPay) asks how much rent [name] paid for board and + lodging, after deducting any state benefits to help with rent. + """ + is_head = person.hrpid == 1 + household_property_income = household.tentyp2.isin((5, 6)) * household.subrent + persons_household_property_income = ( + household_property_income.reindex(person.household_id).fillna(0).values + ) + return ( + np.maximum(0, is_head * persons_household_property_income + person.royyr1) + * WEEKS_IN_YEAR + ).values + + def derive_is_in_non_advanced_education( current_education, is_apprentice=None, @@ -1083,25 +1111,7 @@ def determine_education_level(fted_val, typeed2_val, age_val): ) * 52, ) - is_head = person.hrpid == 1 - household_property_income = ( - household.tentyp2.isin((5, 6)) * household.subrent - ) # Owned and subletting - persons_household_property_income = ( - pd.Series( - household_property_income[person.household_id].values, - index=person.person_id, - ) - .fillna(0) - .values - ) - pe_person["property_income"] = ( - np.maximum( - 0, - is_head * persons_household_property_income + person.cvpay + person.royyr1, - ) - * WEEKS_IN_YEAR - ) + pe_person["property_income"] = frs_property_income(person, household) maintenance_to_self = np.maximum( pd.Series(np.where(person.mntus1 == 2, person.mntusam1, person.mntamt1)).fillna( 0 diff --git a/policyengine_uk_data/tests/test_frs_property_income.py b/policyengine_uk_data/tests/test_frs_property_income.py new file mode 100644 index 00000000..f7e5e7b5 --- /dev/null +++ b/policyengine_uk_data/tests/test_frs_property_income.py @@ -0,0 +1,106 @@ +import numpy as np +import pandas as pd +import pytest + +from policyengine_uk_data.datasets.frs import WEEKS_IN_YEAR, frs_property_income + +HRP, NOT_HRP = 1, 2 +OWNED_WITH_MORTGAGE, OWNED_OUTRIGHT = 5, 6 +COUNCIL_RENTED, PRIVATE_RENTED_FURNISHED = 1, 4 + + +def make_tables(people, households): + person = pd.DataFrame( + people, columns=["household_id", "person_id", "hrpid", "royyr1", "cvpay"] + ) + household = pd.DataFrame( + households, columns=["household_id", "tentyp2", "subrent"] + ).set_index("household_id") + return person, household + + +def test_rent_paid_by_a_lodger_is_not_their_property_income(): + # Owner-occupier household with a lodger who pays £100 a week (CVPAY). + person, household = make_tables( + [(1, 1_001, HRP, 0, 0), (1, 1_002, NOT_HRP, 0, 100)], + [(1, OWNED_OUTRIGHT, 0)], + ) + assert frs_property_income(person, household).tolist() == [0, 0] + + +def test_rent_from_other_property_counts_for_any_adult(): + person, household = make_tables( + [(1, 1_001, HRP, 0, 0), (1, 1_002, NOT_HRP, 50, 0)], + [(1, COUNCIL_RENTED, 0)], + ) + np.testing.assert_allclose( + frs_property_income(person, household), [0, 50 * WEEKS_IN_YEAR] + ) + + +def test_subletting_rent_goes_to_the_owner_household_reference_person(): + person, household = make_tables( + [(1, 1_001, NOT_HRP, 0, 0), (1, 1_002, HRP, 0, 0)], + [(1, OWNED_WITH_MORTGAGE, 80)], + ) + np.testing.assert_allclose( + frs_property_income(person, household), [0, 80 * WEEKS_IN_YEAR] + ) + + +def random_tables(seed: int): + """Random households of one to four adults; the first is the HRP.""" + rng = np.random.default_rng(seed) + people, households = [], [] + for household_id in range(1, rng.integers(1, 30) + 1): + tenure = int(rng.integers(1, 9)) + subrent = float(rng.choice([0, rng.uniform(0, 500)])) + households.append((household_id, tenure, subrent)) + for person in range(1, rng.integers(1, 5) + 1): + people.append( + ( + household_id, + household_id * 1_000 + person, + HRP if person == 1 else NOT_HRP, + float(rng.choice([0, rng.uniform(0, 2_000)])), + float(rng.choice([0, rng.uniform(0, 400)])), + ) + ) + return make_tables(people, households) + + +SEEDS = range(200) + + +@pytest.mark.parametrize("seed", SEEDS) +def test_property_income_does_not_depend_on_cvpay(seed): + person, household = random_tables(seed) + without_cvpay = person.assign(cvpay=0.0) + np.testing.assert_array_equal( + frs_property_income(person, household), + frs_property_income(without_cvpay, household), + ) + + +@pytest.mark.parametrize("seed", SEEDS) +def test_property_income_conserves_reported_rent(seed): + # Each owner household's SUBRENT is counted once (on its HRP) and every + # ROYYR1 is counted on its own record, so the totals match. + person, household = random_tables(seed) + result = frs_property_income(person, household) + assert (result >= 0).all() + owner = household.tentyp2.isin((OWNED_WITH_MORTGAGE, OWNED_OUTRIGHT)) + expected = (household.subrent[owner].sum() + person.royyr1.sum()) * WEEKS_IN_YEAR + assert result.sum() == pytest.approx(expected) + + +@pytest.mark.parametrize("seed", SEEDS) +def test_extra_rent_from_other_property_moves_only_that_person(seed): + person, household = random_tables(seed) + before = frs_property_income(person, household) + row = seed % len(person) + person.loc[row, "royyr1"] += 10 + after = frs_property_income(person, household) + change = np.zeros(len(person)) + change[row] = 10 * WEEKS_IN_YEAR + np.testing.assert_allclose(after - before, change) From 0812cd1b3c5952a2b4b259ae046baf44dfced8a3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 23:21:36 -0400 Subject: [PATCH 2/2] Test the non-negative floor and the stacked adult and child index Review of #503: the seeded cases only draw non-negative amounts, so the floor at zero was never active, and every fixture had a unique index while create_frs stacks the adult and child tables. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_frs_property_income.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/policyengine_uk_data/tests/test_frs_property_income.py b/policyengine_uk_data/tests/test_frs_property_income.py index f7e5e7b5..4aacfab7 100644 --- a/policyengine_uk_data/tests/test_frs_property_income.py +++ b/policyengine_uk_data/tests/test_frs_property_income.py @@ -48,6 +48,29 @@ def test_subletting_rent_goes_to_the_owner_household_reference_person(): ) +def test_negative_amounts_do_not_become_negative_income(): + person, household = make_tables( + [(1, 1_001, HRP, -30, 0), (1, 1_002, NOT_HRP, -5, 0)], + [(1, OWNED_OUTRIGHT, 10)], + ) + assert frs_property_income(person, household).tolist() == [0, 0] + + +def test_adult_and_child_rows_sharing_index_labels(): + # create_frs stacks the adult and child tables, so index labels repeat. + adults, household = make_tables( + [(1, 1_001, HRP, 20, 0), (2, 2_001, HRP, 0, 60)], + [(1, OWNED_OUTRIGHT, 80), (2, PRIVATE_RENTED_FURNISHED, 0)], + ) + children, _ = make_tables([(1, 1_002, 0, 0, 0)], []) + person = pd.concat([adults, children]).sort_index(kind="stable") + assert person.index.tolist() == [0, 0, 1] + np.testing.assert_allclose( + frs_property_income(person, household), + [100 * WEEKS_IN_YEAR, 0, 0], + ) + + def random_tables(seed: int): """Random households of one to four adults; the first is the HRP.""" rng = np.random.default_rng(seed)