From db089cef73ea130ae678dd4355d1e8aad25cceea Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 23:10:22 -0400 Subject: [PATCH 1/4] Stop counting rent paid by boarders and lodgers as their property income FRS CVPAY is the rent a boarder or lodger pays the householder, after deducting any state benefits to help with rent (FRS question CvPay, asked about each person not related to the HRP in the second and later benefit units of a conventional household). frs.py added it to the payer's own property_income. Move the property income calculation into a tested helper, frs_property_income, that keeps SUBRENT and ROYYR1 and drops CVPAY. Co-Authored-By: Claude Opus 5.5 --- .../frs-cvpay-not-property-income.fixed.md | 1 + policyengine_uk_data/datasets/frs.py | 48 ++++---- .../tests/test_frs_property_income.py | 106 ++++++++++++++++++ 3 files changed, 136 insertions(+), 19 deletions(-) create mode 100644 changelog.d/frs-cvpay-not-property-income.fixed.md create mode 100644 policyengine_uk_data/tests/test_frs_property_income.py diff --git a/changelog.d/frs-cvpay-not-property-income.fixed.md b/changelog.d/frs-cvpay-not-property-income.fixed.md new file mode 100644 index 00000000..55546420 --- /dev/null +++ b/changelog.d/frs-cvpay-not-property-income.fixed.md @@ -0,0 +1 @@ +Stop counting the rent a boarder or lodger pays (FRS CVPAY) as that boarder's or lodger's own property income. diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index e440d486..6ae20a90 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -220,6 +220,34 @@ def derive_receives_benefits_in_own_right(pe_person: pd.DataFrame) -> pd.Series: ) +def frs_property_income(person: pd.DataFrame, household: pd.DataFrame) -> np.ndarray: + """Annual property income each person reports in the FRS. + + Two FRS amounts, both weekly in the released data: + + - SUBRENT, rent the household received for letting part of its home to + someone outside the household. It goes to the household reference + person, and only in owner-occupied households (TENTYP2 5 or 6). + ``household`` must be indexed by ``household_id``. + - ROYYR1, the person's rent from other property, before tax and after + allowable expenses. + + CVPAY is not included. It is the rent that a boarder or lodger pays the + householder, and it sits on the boarder's or lodger's own adult record. + The FRS question (CvPay) asks how much rent [name] paid for board and + lodging, after deducting any state benefits to help with rent. + """ + is_head = person.hrpid == 1 + household_property_income = household.tentyp2.isin((5, 6)) * household.subrent + persons_household_property_income = ( + household_property_income.reindex(person.household_id).fillna(0).values + ) + return ( + np.maximum(0, is_head * persons_household_property_income + person.royyr1) + * WEEKS_IN_YEAR + ).values + + def derive_is_in_non_advanced_education( current_education, is_apprentice=None, @@ -1083,25 +1111,7 @@ def determine_education_level(fted_val, typeed2_val, age_val): ) * 52, ) - is_head = person.hrpid == 1 - household_property_income = ( - household.tentyp2.isin((5, 6)) * household.subrent - ) # Owned and subletting - persons_household_property_income = ( - pd.Series( - household_property_income[person.household_id].values, - index=person.person_id, - ) - .fillna(0) - .values - ) - pe_person["property_income"] = ( - np.maximum( - 0, - is_head * persons_household_property_income + person.cvpay + person.royyr1, - ) - * WEEKS_IN_YEAR - ) + pe_person["property_income"] = frs_property_income(person, household) maintenance_to_self = np.maximum( pd.Series(np.where(person.mntus1 == 2, person.mntusam1, person.mntamt1)).fillna( 0 diff --git a/policyengine_uk_data/tests/test_frs_property_income.py b/policyengine_uk_data/tests/test_frs_property_income.py new file mode 100644 index 00000000..f7e5e7b5 --- /dev/null +++ b/policyengine_uk_data/tests/test_frs_property_income.py @@ -0,0 +1,106 @@ +import numpy as np +import pandas as pd +import pytest + +from policyengine_uk_data.datasets.frs import WEEKS_IN_YEAR, frs_property_income + +HRP, NOT_HRP = 1, 2 +OWNED_WITH_MORTGAGE, OWNED_OUTRIGHT = 5, 6 +COUNCIL_RENTED, PRIVATE_RENTED_FURNISHED = 1, 4 + + +def make_tables(people, households): + person = pd.DataFrame( + people, columns=["household_id", "person_id", "hrpid", "royyr1", "cvpay"] + ) + household = pd.DataFrame( + households, columns=["household_id", "tentyp2", "subrent"] + ).set_index("household_id") + return person, household + + +def test_rent_paid_by_a_lodger_is_not_their_property_income(): + # Owner-occupier household with a lodger who pays £100 a week (CVPAY). + person, household = make_tables( + [(1, 1_001, HRP, 0, 0), (1, 1_002, NOT_HRP, 0, 100)], + [(1, OWNED_OUTRIGHT, 0)], + ) + assert frs_property_income(person, household).tolist() == [0, 0] + + +def test_rent_from_other_property_counts_for_any_adult(): + person, household = make_tables( + [(1, 1_001, HRP, 0, 0), (1, 1_002, NOT_HRP, 50, 0)], + [(1, COUNCIL_RENTED, 0)], + ) + np.testing.assert_allclose( + frs_property_income(person, household), [0, 50 * WEEKS_IN_YEAR] + ) + + +def test_subletting_rent_goes_to_the_owner_household_reference_person(): + person, household = make_tables( + [(1, 1_001, NOT_HRP, 0, 0), (1, 1_002, HRP, 0, 0)], + [(1, OWNED_WITH_MORTGAGE, 80)], + ) + np.testing.assert_allclose( + frs_property_income(person, household), [0, 80 * WEEKS_IN_YEAR] + ) + + +def random_tables(seed: int): + """Random households of one to four adults; the first is the HRP.""" + rng = np.random.default_rng(seed) + people, households = [], [] + for household_id in range(1, rng.integers(1, 30) + 1): + tenure = int(rng.integers(1, 9)) + subrent = float(rng.choice([0, rng.uniform(0, 500)])) + households.append((household_id, tenure, subrent)) + for person in range(1, rng.integers(1, 5) + 1): + people.append( + ( + household_id, + household_id * 1_000 + person, + HRP if person == 1 else NOT_HRP, + float(rng.choice([0, rng.uniform(0, 2_000)])), + float(rng.choice([0, rng.uniform(0, 400)])), + ) + ) + return make_tables(people, households) + + +SEEDS = range(200) + + +@pytest.mark.parametrize("seed", SEEDS) +def test_property_income_does_not_depend_on_cvpay(seed): + person, household = random_tables(seed) + without_cvpay = person.assign(cvpay=0.0) + np.testing.assert_array_equal( + frs_property_income(person, household), + frs_property_income(without_cvpay, household), + ) + + +@pytest.mark.parametrize("seed", SEEDS) +def test_property_income_conserves_reported_rent(seed): + # Each owner household's SUBRENT is counted once (on its HRP) and every + # ROYYR1 is counted on its own record, so the totals match. + person, household = random_tables(seed) + result = frs_property_income(person, household) + assert (result >= 0).all() + owner = household.tentyp2.isin((OWNED_WITH_MORTGAGE, OWNED_OUTRIGHT)) + expected = (household.subrent[owner].sum() + person.royyr1.sum()) * WEEKS_IN_YEAR + assert result.sum() == pytest.approx(expected) + + +@pytest.mark.parametrize("seed", SEEDS) +def test_extra_rent_from_other_property_moves_only_that_person(seed): + person, household = random_tables(seed) + before = frs_property_income(person, household) + row = seed % len(person) + person.loc[row, "royyr1"] += 10 + after = frs_property_income(person, household) + change = np.zeros(len(person)) + change[row] = 10 * WEEKS_IN_YEAR + np.testing.assert_allclose(after - before, change) From 0812cd1b3c5952a2b4b259ae046baf44dfced8a3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 23:21:36 -0400 Subject: [PATCH 2/4] Test the non-negative floor and the stacked adult and child index Review of #503: the seeded cases only draw non-negative amounts, so the floor at zero was never active, and every fixture had a unique index while create_frs stacks the adult and child tables. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_frs_property_income.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/policyengine_uk_data/tests/test_frs_property_income.py b/policyengine_uk_data/tests/test_frs_property_income.py index f7e5e7b5..4aacfab7 100644 --- a/policyengine_uk_data/tests/test_frs_property_income.py +++ b/policyengine_uk_data/tests/test_frs_property_income.py @@ -48,6 +48,29 @@ def test_subletting_rent_goes_to_the_owner_household_reference_person(): ) +def test_negative_amounts_do_not_become_negative_income(): + person, household = make_tables( + [(1, 1_001, HRP, -30, 0), (1, 1_002, NOT_HRP, -5, 0)], + [(1, OWNED_OUTRIGHT, 10)], + ) + assert frs_property_income(person, household).tolist() == [0, 0] + + +def test_adult_and_child_rows_sharing_index_labels(): + # create_frs stacks the adult and child tables, so index labels repeat. + adults, household = make_tables( + [(1, 1_001, HRP, 20, 0), (2, 2_001, HRP, 0, 60)], + [(1, OWNED_OUTRIGHT, 80), (2, PRIVATE_RENTED_FURNISHED, 0)], + ) + children, _ = make_tables([(1, 1_002, 0, 0, 0)], []) + person = pd.concat([adults, children]).sort_index(kind="stable") + assert person.index.tolist() == [0, 0, 1] + np.testing.assert_allclose( + frs_property_income(person, household), + [100 * WEEKS_IN_YEAR, 0, 0], + ) + + def random_tables(seed: int): """Random households of one to four adults; the first is the HRP.""" rng = np.random.default_rng(seed) From be4d9356c80369029c8882bcb738105a4b4b0fb5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 1 Oct 2026 09:24:00 -0400 Subject: [PATCH 3/4] Stop counting property losses as income; count sub-let rent for every tenure ROYYR1 holds a property loss as a positive amount flagged by RENTPROF = 2, and the build counted it as income. A loss now counts as zero. SUBRENT was counted only for owner-occupiers, although the FRS asks every household about sub-letting. The tenure restriction is removed. SUBRENT stays as reported whether SUBALLOW says it is before or after allowable expenses; the FRS collects no expense amount. Co-Authored-By: Claude Opus 5.5 --- .../frs-property-losses-subrent.fixed.md | 1 + policyengine_uk_data/datasets/frs.py | 29 ++- .../tests/test_frs_property_income.py | 209 +++++++++++++++--- .../tests/test_legacy_benefit_proxies.py | 1 + 4 files changed, 205 insertions(+), 35 deletions(-) create mode 100644 changelog.d/frs-property-losses-subrent.fixed.md diff --git a/changelog.d/frs-property-losses-subrent.fixed.md b/changelog.d/frs-property-losses-subrent.fixed.md new file mode 100644 index 00000000..097b15f2 --- /dev/null +++ b/changelog.d/frs-property-losses-subrent.fixed.md @@ -0,0 +1 @@ +Stop counting FRS property losses (ROYYR1 with RENTPROF = 2) as property income, and count rent from sub-letting part of the home (SUBRENT) for every tenure, not only owner-occupiers. diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index 6ae20a90..abd52140 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -84,6 +84,8 @@ # FRS government-training question variants use 10 or 13 for "None of these". FRS_APPROVED_TRAINING_CODES = tuple(range(1, 10)) UNKNOWN_QUALIFYING_EDUCATION_OR_TRAINING_ENTRY_AGE = 1000 +# FRS RENTPROF: whether ROYYR1 is a profit (1) or a loss (2). +FRS_RENTPROF_LOSS = 2 @lru_cache(maxsize=None) @@ -226,11 +228,20 @@ def frs_property_income(person: pd.DataFrame, household: pd.DataFrame) -> np.nda Two FRS amounts, both weekly in the released data: - SUBRENT, rent the household received for letting part of its home to - someone outside the household. It goes to the household reference - person, and only in owner-occupied households (TENTYP2 5 or 6). - ``household`` must be indexed by ``household_id``. + someone outside the household. The FRS asks every household (SubLet), + whatever its tenure, so renting and rent-free households count too. It + goes to the household reference person. ``household`` must be indexed + by ``household_id``. - ROYYR1, the person's rent from other property, before tax and after - allowable expenses. + allowable expenses. The questionnaire cannot take a negative amount, + so a loss is entered as a positive amount with RENTPROF = 2 (question + RentProf, "Is that a profit or a loss from the property?"). A loss + counts as zero: it is not income, policyengine-uk has no property loss + input, and it is not set against the household's SUBRENT. + + SUBRENT is used as reported. SUBALLOW records whether it is before (1) + or after (2) allowable expenses, but the FRS collects no expense amount + to take off the before-expenses answers. CVPAY is not included. It is the rent that a boarder or lodger pays the householder, and it sits on the boarder's or lodger's own adult record. @@ -238,12 +249,14 @@ def frs_property_income(person: pd.DataFrame, household: pd.DataFrame) -> np.nda lodging, after deducting any state benefits to help with rent. """ is_head = person.hrpid == 1 - household_property_income = household.tentyp2.isin((5, 6)) * household.subrent - persons_household_property_income = ( - household_property_income.reindex(person.household_id).fillna(0).values + persons_household_subrent = ( + household.subrent.reindex(person.household_id).fillna(0).values + ) + rent_from_other_property = person.royyr1.where( + person.rentprof != FRS_RENTPROF_LOSS, 0 ) return ( - np.maximum(0, is_head * persons_household_property_income + person.royyr1) + np.maximum(0, is_head * persons_household_subrent + rent_from_other_property) * WEEKS_IN_YEAR ).values diff --git a/policyengine_uk_data/tests/test_frs_property_income.py b/policyengine_uk_data/tests/test_frs_property_income.py index 4aacfab7..1abcf931 100644 --- a/policyengine_uk_data/tests/test_frs_property_income.py +++ b/policyengine_uk_data/tests/test_frs_property_income.py @@ -2,19 +2,29 @@ import pandas as pd import pytest -from policyengine_uk_data.datasets.frs import WEEKS_IN_YEAR, frs_property_income +from policyengine_uk_data.datasets.frs import ( + FRS_RENTPROF_LOSS, + WEEKS_IN_YEAR, + frs_property_income, +) +from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE +from policyengine_uk_data.storage import STORAGE_FOLDER HRP, NOT_HRP = 1, 2 +NOT_ASKED, PROFIT, LOSS = 0, 1, FRS_RENTPROF_LOSS OWNED_WITH_MORTGAGE, OWNED_OUTRIGHT = 5, 6 COUNCIL_RENTED, PRIVATE_RENTED_FURNISHED = 1, 4 +TENURES = range(1, 9) +BEFORE_EXPENSES, AFTER_EXPENSES = 1, 2 def make_tables(people, households): person = pd.DataFrame( - people, columns=["household_id", "person_id", "hrpid", "royyr1", "cvpay"] + people, + columns=["household_id", "person_id", "hrpid", "royyr1", "rentprof", "cvpay"], ) household = pd.DataFrame( - households, columns=["household_id", "tentyp2", "subrent"] + households, columns=["household_id", "tentyp2", "subrent", "suballow"] ).set_index("household_id") return person, household @@ -22,36 +32,80 @@ def make_tables(people, households): def test_rent_paid_by_a_lodger_is_not_their_property_income(): # Owner-occupier household with a lodger who pays £100 a week (CVPAY). person, household = make_tables( - [(1, 1_001, HRP, 0, 0), (1, 1_002, NOT_HRP, 0, 100)], - [(1, OWNED_OUTRIGHT, 0)], + [(1, 1_001, HRP, 0, NOT_ASKED, 0), (1, 1_002, NOT_HRP, 0, NOT_ASKED, 100)], + [(1, OWNED_OUTRIGHT, 0, 0)], ) assert frs_property_income(person, household).tolist() == [0, 0] def test_rent_from_other_property_counts_for_any_adult(): person, household = make_tables( - [(1, 1_001, HRP, 0, 0), (1, 1_002, NOT_HRP, 50, 0)], - [(1, COUNCIL_RENTED, 0)], + [(1, 1_001, HRP, 0, NOT_ASKED, 0), (1, 1_002, NOT_HRP, 50, PROFIT, 0)], + [(1, COUNCIL_RENTED, 0, 0)], ) np.testing.assert_allclose( frs_property_income(person, household), [0, 50 * WEEKS_IN_YEAR] ) -def test_subletting_rent_goes_to_the_owner_household_reference_person(): +def test_a_loss_on_other_property_is_not_income(): + # ROYYR1 holds the size of the loss as a positive amount; RENTPROF flags it. person, household = make_tables( - [(1, 1_001, NOT_HRP, 0, 0), (1, 1_002, HRP, 0, 0)], - [(1, OWNED_WITH_MORTGAGE, 80)], + [(1, 1_001, HRP, 90, LOSS, 0), (1, 1_002, NOT_HRP, 50, PROFIT, 0)], + [(1, OWNED_WITH_MORTGAGE, 0, 0)], + ) + np.testing.assert_allclose( + frs_property_income(person, household), [0, 50 * WEEKS_IN_YEAR] + ) + + +def test_a_loss_on_other_property_does_not_reduce_subletting_rent(): + person, household = make_tables( + [(1, 1_001, HRP, 30, LOSS, 0)], + [(1, OWNED_OUTRIGHT, 80, AFTER_EXPENSES)], + ) + np.testing.assert_allclose( + frs_property_income(person, household), [80 * WEEKS_IN_YEAR] + ) + + +def test_rent_with_no_profit_or_loss_answer_still_counts(): + # Only an explicit loss removes the amount. + person, household = make_tables( + [(1, 1_001, HRP, 50, NOT_ASKED, 0)], [(1, OWNED_OUTRIGHT, 0, 0)] + ) + np.testing.assert_allclose( + frs_property_income(person, household), [50 * WEEKS_IN_YEAR] + ) + + +@pytest.mark.parametrize("tenure", TENURES) +def test_subletting_rent_goes_to_the_household_reference_person(tenure): + # Every tenure is asked SubLet, so renting households count too. + person, household = make_tables( + [(1, 1_001, NOT_HRP, 0, NOT_ASKED, 0), (1, 1_002, HRP, 0, NOT_ASKED, 0)], + [(1, tenure, 80, BEFORE_EXPENSES)], ) np.testing.assert_allclose( frs_property_income(person, household), [0, 80 * WEEKS_IN_YEAR] ) +@pytest.mark.parametrize("suballow", [0, BEFORE_EXPENSES, AFTER_EXPENSES]) +def test_subletting_rent_is_used_as_reported_whatever_its_expenses_basis(suballow): + person, household = make_tables( + [(1, 1_001, HRP, 0, NOT_ASKED, 0)], + [(1, PRIVATE_RENTED_FURNISHED, 80, suballow)], + ) + np.testing.assert_allclose( + frs_property_income(person, household), [80 * WEEKS_IN_YEAR] + ) + + def test_negative_amounts_do_not_become_negative_income(): person, household = make_tables( - [(1, 1_001, HRP, -30, 0), (1, 1_002, NOT_HRP, -5, 0)], - [(1, OWNED_OUTRIGHT, 10)], + [(1, 1_001, HRP, -30, PROFIT, 0), (1, 1_002, NOT_HRP, -5, PROFIT, 0)], + [(1, OWNED_OUTRIGHT, 10, AFTER_EXPENSES)], ) assert frs_property_income(person, household).tolist() == [0, 0] @@ -59,15 +113,18 @@ def test_negative_amounts_do_not_become_negative_income(): def test_adult_and_child_rows_sharing_index_labels(): # create_frs stacks the adult and child tables, so index labels repeat. adults, household = make_tables( - [(1, 1_001, HRP, 20, 0), (2, 2_001, HRP, 0, 60)], - [(1, OWNED_OUTRIGHT, 80), (2, PRIVATE_RENTED_FURNISHED, 0)], + [(1, 1_001, HRP, 20, PROFIT, 0), (2, 2_001, HRP, 40, LOSS, 60)], + [ + (1, OWNED_OUTRIGHT, 80, BEFORE_EXPENSES), + (2, PRIVATE_RENTED_FURNISHED, 30, AFTER_EXPENSES), + ], ) - children, _ = make_tables([(1, 1_002, 0, 0, 0)], []) + children, _ = make_tables([(1, 1_002, 0, 0, NOT_ASKED, 0)], []) person = pd.concat([adults, children]).sort_index(kind="stable") assert person.index.tolist() == [0, 0, 1] np.testing.assert_allclose( frs_property_income(person, household), - [100 * WEEKS_IN_YEAR, 0, 0], + [100 * WEEKS_IN_YEAR, 0, 30 * WEEKS_IN_YEAR], ) @@ -78,45 +135,88 @@ def random_tables(seed: int): for household_id in range(1, rng.integers(1, 30) + 1): tenure = int(rng.integers(1, 9)) subrent = float(rng.choice([0, rng.uniform(0, 500)])) - households.append((household_id, tenure, subrent)) + suballow = int(rng.integers(1, 3)) if subrent else 0 + households.append((household_id, tenure, subrent, suballow)) for person in range(1, rng.integers(1, 5) + 1): + royyr1 = float(rng.choice([0, rng.uniform(0, 2_000)])) people.append( ( household_id, household_id * 1_000 + person, HRP if person == 1 else NOT_HRP, - float(rng.choice([0, rng.uniform(0, 2_000)])), + royyr1, + int(rng.choice([PROFIT, PROFIT, LOSS])) if royyr1 else NOT_ASKED, float(rng.choice([0, rng.uniform(0, 400)])), ) ) return make_tables(people, households) +def property_income_one_person_at_a_time(person, household): + """The same rules written as a loop, to check the vectorised helper.""" + result = [] + for row in person.itertuples(): + weekly = 0.0 + if row.rentprof != LOSS: + weekly += row.royyr1 + if row.hrpid == HRP and row.household_id in household.index: + weekly += household.subrent[row.household_id] + result.append(max(0.0, weekly) * WEEKS_IN_YEAR) + return np.array(result) + + SEEDS = range(200) @pytest.mark.parametrize("seed", SEEDS) -def test_property_income_does_not_depend_on_cvpay(seed): +def test_property_income_matches_the_loop_version(seed): person, household = random_tables(seed) - without_cvpay = person.assign(cvpay=0.0) + np.testing.assert_allclose( + frs_property_income(person, household), + property_income_one_person_at_a_time(person, household), + ) + + +@pytest.mark.parametrize("seed", SEEDS) +def test_property_income_does_not_depend_on_cvpay_tenure_or_expenses_basis(seed): + person, household = random_tables(seed) + rng = np.random.default_rng(seed) + other_person = person.assign(cvpay=rng.uniform(0, 400, len(person))) + other_household = household.assign( + tentyp2=rng.integers(1, 9, len(household)), + suballow=rng.integers(1, 3, len(household)), + ) np.testing.assert_array_equal( frs_property_income(person, household), - frs_property_income(without_cvpay, household), + frs_property_income(other_person, other_household), ) @pytest.mark.parametrize("seed", SEEDS) def test_property_income_conserves_reported_rent(seed): - # Each owner household's SUBRENT is counted once (on its HRP) and every - # ROYYR1 is counted on its own record, so the totals match. + # Each household's SUBRENT is counted once (on its HRP) and every ROYYR1 + # that is not a loss is counted on its own record, so the totals match. person, household = random_tables(seed) result = frs_property_income(person, household) assert (result >= 0).all() - owner = household.tentyp2.isin((OWNED_WITH_MORTGAGE, OWNED_OUTRIGHT)) - expected = (household.subrent[owner].sum() + person.royyr1.sum()) * WEEKS_IN_YEAR + profits = person.royyr1[person.rentprof != LOSS] + expected = (household.subrent.sum() + profits.sum()) * WEEKS_IN_YEAR assert result.sum() == pytest.approx(expected) +@pytest.mark.parametrize("seed", SEEDS) +def test_marking_rent_as_a_loss_removes_it_from_that_person_only(seed): + person, household = random_tables(seed) + before = frs_property_income(person, household) + row = seed % len(person) + was_counted = person.loc[row, "rentprof"] != LOSS + person.loc[row, "rentprof"] = LOSS + after = frs_property_income(person, household) + change = np.zeros(len(person)) + change[row] = -person.loc[row, "royyr1"] * WEEKS_IN_YEAR * was_counted + np.testing.assert_allclose(after - before, change, atol=1e-6) + + @pytest.mark.parametrize("seed", SEEDS) def test_extra_rent_from_other_property_moves_only_that_person(seed): person, household = random_tables(seed) @@ -125,5 +225,60 @@ def test_extra_rent_from_other_property_moves_only_that_person(seed): person.loc[row, "royyr1"] += 10 after = frs_property_income(person, household) change = np.zeros(len(person)) - change[row] = 10 * WEEKS_IN_YEAR - np.testing.assert_allclose(after - before, change) + change[row] = 10 * WEEKS_IN_YEAR * (person.loc[row, "rentprof"] != LOSS) + np.testing.assert_allclose(after - before, change, atol=1e-6) + + +@pytest.mark.parametrize("seed", SEEDS) +def test_extra_subletting_rent_moves_only_that_households_reference_person(seed): + person, household = random_tables(seed) + before = frs_property_income(person, household) + household_id = household.index[seed % len(household)] + household.loc[household_id, "subrent"] += 10 + after = frs_property_income(person, household) + is_its_hrp = (person.household_id == household_id) & (person.hrpid == HRP) + np.testing.assert_allclose( + after - before, is_its_hrp * 10 * WEEKS_IN_YEAR, atol=1e-6 + ) + + +def test_built_frs_matches_the_raw_tables(frs): + """The built base FRS against a merge-based reading of the raw tables. + + Covers the call site in ``create_frs`` and the raw column names and codes. + The assertion carries no values, because the dataset is licensed microdata. + """ + raw_folder = STORAGE_FOLDER / CURRENT_FRS_RELEASE.name + if not (raw_folder / "adult.tab").exists(): + pytest.skip("Raw FRS tables not available") + adult = pd.read_csv( + raw_folder / "adult.tab", + sep="\t", + usecols=lambda c: ( + c.upper() in ("SERNUM", "PERSON", "HRPID", "ROYYR1", "RENTPROF") + ), + ) + househol = pd.read_csv( + raw_folder / "househol.tab", + sep="\t", + usecols=lambda c: c.upper() in ("SERNUM", "SUBRENT"), + ) + adult.columns = adult.columns.str.upper() + househol.columns = househol.columns.str.upper() + raw = adult.merge(househol, on="SERNUM", how="left").apply( + pd.to_numeric, errors="coerce" + ) + profit = raw.ROYYR1.where(raw.RENTPROF != 2, 0).fillna(0) + subrent = raw.SUBRENT.where(raw.HRPID == 1, 0).fillna(0) + expected = pd.Series( + ((profit + subrent).clip(lower=0) * WEEKS_IN_YEAR).values, + index=(raw.SERNUM * 1_000 + raw.PERSON).astype(int), + ) + built = pd.Series( + frs.person.property_income.values, index=frs.person.person_id.values + ) + adults_match = np.allclose(built.reindex(expected.index), expected, atol=0.01) + children_have_none = (built.drop(expected.index) == 0).all() + assert adults_match and children_have_none, ( + "property_income in the built FRS differs from the raw FRS tables" + ) diff --git a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py index 5f1acd85..5beb95ab 100644 --- a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py +++ b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py @@ -442,6 +442,7 @@ def fake_read_csv(path, *args, **kwargs): "mntus1": 0, "mntusam1": 0, "redamt": 0, + "rentprof": 0, "royyr1": 0, "seincam2": 0, "sex": 1, From cc937e0555d7ead58fc980b41b1959510709b1db Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 1 Oct 2026 09:54:34 -0400 Subject: [PATCH 4/4] Floor each property amount at zero before adding them Negative FRS values are missing-value codes (-1 to -9), not amounts. Flooring the sum let a missing ROYYR1 code cancel the household reference person's SUBRENT, which now matters for renting households too. Floor each amount instead. FRS 2024-25 has no negative codes in either field, so the build is unchanged. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk_data/datasets/frs.py | 10 ++-- .../tests/test_frs_property_income.py | 55 ++++++++++++------- 2 files changed, 42 insertions(+), 23 deletions(-) diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index abd52140..c7f8e179 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -243,6 +243,9 @@ def frs_property_income(person: pd.DataFrame, household: pd.DataFrame) -> np.nda or after (2) allowable expenses, but the FRS collects no expense amount to take off the before-expenses answers. + Negative values are FRS missing-value codes (-1 to -9), not amounts, so + each amount is floored at zero before the two are added. + CVPAY is not included. It is the rent that a boarder or lodger pays the householder, and it sits on the boarder's or lodger's own adult record. The FRS question (CvPay) asks how much rent [name] paid for board and @@ -250,14 +253,13 @@ def frs_property_income(person: pd.DataFrame, household: pd.DataFrame) -> np.nda """ is_head = person.hrpid == 1 persons_household_subrent = ( - household.subrent.reindex(person.household_id).fillna(0).values + household.subrent.clip(lower=0).reindex(person.household_id).fillna(0).values ) - rent_from_other_property = person.royyr1.where( + rent_from_other_property = person.royyr1.clip(lower=0).where( person.rentprof != FRS_RENTPROF_LOSS, 0 ) return ( - np.maximum(0, is_head * persons_household_subrent + rent_from_other_property) - * WEEKS_IN_YEAR + (is_head * persons_household_subrent + rent_from_other_property) * WEEKS_IN_YEAR ).values diff --git a/policyengine_uk_data/tests/test_frs_property_income.py b/policyengine_uk_data/tests/test_frs_property_income.py index 1abcf931..5fb2982c 100644 --- a/policyengine_uk_data/tests/test_frs_property_income.py +++ b/policyengine_uk_data/tests/test_frs_property_income.py @@ -12,6 +12,8 @@ HRP, NOT_HRP = 1, 2 NOT_ASKED, PROFIT, LOSS = 0, 1, FRS_RENTPROF_LOSS +# FRS missing-value codes run from -1 to -9. +MISSING, REFUSED = -1, -9 OWNED_WITH_MORTGAGE, OWNED_OUTRIGHT = 5, 6 COUNCIL_RENTED, PRIVATE_RENTED_FURNISHED = 1, 4 TENURES = range(1, 9) @@ -102,12 +104,20 @@ def test_subletting_rent_is_used_as_reported_whatever_its_expenses_basis(suballo ) -def test_negative_amounts_do_not_become_negative_income(): +def test_missing_value_codes_count_as_zero(): + # A missing code in one amount neither becomes income nor cancels the other. person, household = make_tables( - [(1, 1_001, HRP, -30, PROFIT, 0), (1, 1_002, NOT_HRP, -5, PROFIT, 0)], - [(1, OWNED_OUTRIGHT, 10, AFTER_EXPENSES)], + [ + (1, 1_001, HRP, MISSING, MISSING, 0), + (1, 1_002, NOT_HRP, REFUSED, MISSING, 0), + (2, 2_001, HRP, 50, PROFIT, 0), + ], + [(1, COUNCIL_RENTED, 10, AFTER_EXPENSES), (2, OWNED_OUTRIGHT, MISSING, 0)], + ) + np.testing.assert_allclose( + frs_property_income(person, household), + [10 * WEEKS_IN_YEAR, 0, 50 * WEEKS_IN_YEAR], ) - assert frs_property_income(person, household).tolist() == [0, 0] def test_adult_and_child_rows_sharing_index_labels(): @@ -134,18 +144,22 @@ def random_tables(seed: int): people, households = [], [] for household_id in range(1, rng.integers(1, 30) + 1): tenure = int(rng.integers(1, 9)) - subrent = float(rng.choice([0, rng.uniform(0, 500)])) - suballow = int(rng.integers(1, 3)) if subrent else 0 + subrent = float(rng.choice([0, MISSING, rng.uniform(0, 500)])) + suballow = int(rng.integers(1, 3)) if subrent > 0 else 0 households.append((household_id, tenure, subrent, suballow)) for person in range(1, rng.integers(1, 5) + 1): - royyr1 = float(rng.choice([0, rng.uniform(0, 2_000)])) + royyr1 = float(rng.choice([0, MISSING, rng.uniform(0, 2_000)])) + if royyr1 > 0: + rentprof = int(rng.choice([PROFIT, PROFIT, LOSS])) + else: + rentprof = MISSING if royyr1 < 0 else NOT_ASKED people.append( ( household_id, household_id * 1_000 + person, HRP if person == 1 else NOT_HRP, royyr1, - int(rng.choice([PROFIT, PROFIT, LOSS])) if royyr1 else NOT_ASKED, + rentprof, float(rng.choice([0, rng.uniform(0, 400)])), ) ) @@ -158,10 +172,10 @@ def property_income_one_person_at_a_time(person, household): for row in person.itertuples(): weekly = 0.0 if row.rentprof != LOSS: - weekly += row.royyr1 + weekly += max(0.0, row.royyr1) if row.hrpid == HRP and row.household_id in household.index: - weekly += household.subrent[row.household_id] - result.append(max(0.0, weekly) * WEEKS_IN_YEAR) + weekly += max(0.0, household.subrent[row.household_id]) + result.append(weekly * WEEKS_IN_YEAR) return np.array(result) @@ -199,8 +213,9 @@ def test_property_income_conserves_reported_rent(seed): person, household = random_tables(seed) result = frs_property_income(person, household) assert (result >= 0).all() - profits = person.royyr1[person.rentprof != LOSS] - expected = (household.subrent.sum() + profits.sum()) * WEEKS_IN_YEAR + profits = person.royyr1.clip(lower=0)[person.rentprof != LOSS] + subrent = household.subrent.clip(lower=0) + expected = (subrent.sum() + profits.sum()) * WEEKS_IN_YEAR assert result.sum() == pytest.approx(expected) @@ -213,7 +228,7 @@ def test_marking_rent_as_a_loss_removes_it_from_that_person_only(seed): person.loc[row, "rentprof"] = LOSS after = frs_property_income(person, household) change = np.zeros(len(person)) - change[row] = -person.loc[row, "royyr1"] * WEEKS_IN_YEAR * was_counted + change[row] = -max(0, person.loc[row, "royyr1"]) * WEEKS_IN_YEAR * was_counted np.testing.assert_allclose(after - before, change, atol=1e-6) @@ -222,7 +237,7 @@ def test_extra_rent_from_other_property_moves_only_that_person(seed): person, household = random_tables(seed) before = frs_property_income(person, household) row = seed % len(person) - person.loc[row, "royyr1"] += 10 + person.loc[row, "royyr1"] = max(0, person.loc[row, "royyr1"]) + 10 after = frs_property_income(person, household) change = np.zeros(len(person)) change[row] = 10 * WEEKS_IN_YEAR * (person.loc[row, "rentprof"] != LOSS) @@ -234,7 +249,9 @@ def test_extra_subletting_rent_moves_only_that_households_reference_person(seed) person, household = random_tables(seed) before = frs_property_income(person, household) household_id = household.index[seed % len(household)] - household.loc[household_id, "subrent"] += 10 + household.loc[household_id, "subrent"] = ( + max(0, household.loc[household_id, "subrent"]) + 10 + ) after = frs_property_income(person, household) is_its_hrp = (person.household_id == household_id) & (person.hrpid == HRP) np.testing.assert_allclose( @@ -268,10 +285,10 @@ def test_built_frs_matches_the_raw_tables(frs): raw = adult.merge(househol, on="SERNUM", how="left").apply( pd.to_numeric, errors="coerce" ) - profit = raw.ROYYR1.where(raw.RENTPROF != 2, 0).fillna(0) - subrent = raw.SUBRENT.where(raw.HRPID == 1, 0).fillna(0) + profit = raw.ROYYR1.clip(lower=0).where(raw.RENTPROF != 2, 0).fillna(0) + subrent = raw.SUBRENT.clip(lower=0).where(raw.HRPID == 1, 0).fillna(0) expected = pd.Series( - ((profit + subrent).clip(lower=0) * WEEKS_IN_YEAR).values, + ((profit + subrent) * WEEKS_IN_YEAR).values, index=(raw.SERNUM * 1_000 + raw.PERSON).astype(int), ) built = pd.Series(