diff --git a/changelog.d/resolved-household-head.fixed.md b/changelog.d/resolved-household-head.fixed.md new file mode 100644 index 0000000000..eedbdab011 --- /dev/null +++ b/changelog.d/resolved-household-head.fixed.md @@ -0,0 +1 @@ +Read one household head per household in every programme, so an `is_household_head` input that flags several members, or none, can no longer leave rent, non-dependant and Council Tax Reduction rules disagreeing about whose household it is. diff --git a/policyengine_uk/dynamics/participation.py b/policyengine_uk/dynamics/participation.py index 78349cfaeb..2b85e3f60b 100644 --- a/policyengine_uk/dynamics/participation.py +++ b/policyengine_uk/dynamics/participation.py @@ -45,7 +45,6 @@ def calculate_participation_elasticities( is_single = ~is_married # Get partner employment status for married individuals - is_household_head = sim.calculate("is_household_head", map_to="person") benunit_count_adults = composition["count_aged_18_or_over"].values employment_income = sim.calculate("employment_income") benunit_id = sim.calculate("benunit_id", map_to="person") diff --git a/policyengine_uk/tests/policy/baseline/household/demographic/resolved_household_head.yaml b/policyengine_uk/tests/policy/baseline/household/demographic/resolved_household_head.yaml new file mode 100644 index 0000000000..0b06e36f56 --- /dev/null +++ b/policyengine_uk/tests/policy/baseline/household/demographic/resolved_household_head.yaml @@ -0,0 +1,386 @@ +# One household head per household, whatever is_household_head flags. With +# several members flagged, the eldest flagged member is the head; with none, +# the eldest member is; a tie in age goes to the member listed first among +# the people (is_resolved_household_head). +# Rent liability (UC Regs 2013 Sch 4 para 24; HB Regs 2006 reg 12B), +# non-dependants and the size criteria (UC Regs 2013 Sch 4 paras 9-10; HB Regs +# 2006 regs 3, 13D and 74), boarders and lodgers, Council Tax Reduction +# claimants and non-dependants (SI 2012/2885 reg 9 and Sch 1 paras 2-3; SI +# 2013/3029 reg 9; SSI 2021/249 reg 8) and the maintenance loan's +# living-arrangement proxy all read the same head, so they agree on whose +# household it is. + +- name: Two members of separate families flagged as head - only the eldest flagged member's family has the rent, and the other family is its non-dependant + period: 2026 + absolute_error_margin: 0.01 + input: + people: + younger_flagged: + age: 40 + is_household_head: true + uc_individual_non_dep_deduction: 1_000 + household_benefits_individual_non_dep_deduction: 800 + council_tax_reduction_individual_non_dep_deduction: 200 + elder_flagged: + age: 60 + is_household_head: true + benunits: + younger_family: + members: [younger_flagged] + elder_family: + members: [elder_flagged] + households: + home: + members: [younger_flagged, elder_flagged] + rent: 12_000 + tenure_type: RENT_PRIVATELY + country: WALES + local_authority: CARDIFF + council_tax: 1_800 + output: + is_resolved_household_head: [false, true] + benunit_contains_household_head: [false, true] + # Each flagged family used to have half the rent (6,000), and the younger + # one was then neither a Council Tax Reduction claimant nor a Universal + # Credit or Housing Benefit non-dependant. + is_liable_for_household_rent: [false, true] + share_of_household_rent: [0, 1] + personal_rent: [0, 12_000] + benunit_rent: [0, 12_000] + is_non_dependant_of_household_head: [true, false] + uc_non_dep_deductions: [0, 1_000] + housing_benefit_non_dep_deductions: [0, 800] + LHA_allowed_bedrooms: [1, 2] + housing_benefit_LHA_allowed_bedrooms: [1, 2] + lha_renter_has_non_dependant: [false, true] + council_tax_reduction_claimant_benunit: [false, true] + council_tax_reduction_individual_non_dep_deduction_eligible: [true, false] + council_tax_reduction_non_dep_deductions: [0, 200] + +- name: A couple both flagged as head carry the household's rent once + period: 2026 + absolute_error_margin: 0.01 + input: + people: + head: + age: 40 + is_household_head: true + partner: + age: 38 + is_household_head: true + benunits: + family: + members: [head, partner] + households: + home: + members: [head, partner] + rent: 12_000 + tenure_type: RENT_PRIVATELY + output: + is_resolved_household_head: [true, false] + is_liable_for_household_rent: [true, true] + # The family's share sits on one person, not on each flagged partner. + personal_rent: [12_000, 0] + benunit_rent: 12_000 + +- name: No member flagged as head - the eldest member's family has the rent, and the other family is its non-dependant + period: 2026 + absolute_error_margin: 0.01 + input: + people: + younger: + age: 40 + is_household_head: false + uc_individual_non_dep_deduction: 1_000 + household_benefits_individual_non_dep_deduction: 800 + council_tax_reduction_individual_non_dep_deduction: 200 + elder: + age: 60 + is_household_head: false + benunits: + younger_family: + members: [younger] + elder_family: + members: [elder] + households: + home: + members: [younger, elder] + rent: 12_000 + tenure_type: RENT_PRIVATELY + country: SCOTLAND + local_authority: CITY_OF_EDINBURGH + council_tax: 1_800 + output: + is_resolved_household_head: [false, true] + benunit_contains_household_head: [false, true] + # No family used to be liable for the rent at all. + is_liable_for_household_rent: [false, true] + share_of_household_rent: [0, 1] + personal_rent: [0, 12_000] + is_non_dependant_of_household_head: [true, false] + uc_non_dep_deductions: [0, 1_000] + housing_benefit_non_dep_deductions: [0, 800] + LHA_allowed_bedrooms: [1, 2] + housing_benefit_LHA_allowed_bedrooms: [1, 2] + lha_renter_has_non_dependant: [false, true] + council_tax_reduction_claimant_benunit: [false, true] + council_tax_reduction_individual_non_dep_deduction_eligible: [true, false] + council_tax_reduction_non_dep_deductions: [0, 200] + +- name: No member flagged as head and the eldest tied - the first eldest member's family is the head's + period: 2026 + absolute_error_margin: 0.01 + input: + people: + first: + age: 50 + is_household_head: false + second: + age: 50 + is_household_head: false + uc_individual_non_dep_deduction: 1_000 + household_benefits_individual_non_dep_deduction: 800 + benunits: + first_family: + members: [first] + second_family: + members: [second] + households: + home: + members: [first, second] + rent: 12_000 + tenure_type: RENT_PRIVATELY + country: ENGLAND + local_authority: MAIDSTONE + council_tax: 1_800 + output: + is_resolved_household_head: [true, false] + personal_rent: [12_000, 0] + is_non_dependant_of_household_head: [false, true] + uc_non_dep_deductions: [1_000, 0] + housing_benefit_non_dep_deductions: [800, 0] + council_tax_reduction_claimant_benunit: [true, false] + council_tax_reduction_individual_non_dep_deduction_eligible: [false, true] + +- name: Two members flagged as head beside a sharer - the eldest flagged member's family and the sharer split the rent and claim, and the other flagged family is a non-dependant + period: 2026 + absolute_error_margin: 0.01 + input: + people: + younger_flagged: + age: 30 + is_household_head: true + uc_individual_non_dep_deduction: 1_000 + household_benefits_individual_non_dep_deduction: 800 + council_tax_reduction_individual_non_dep_deduction: 200 + elder_flagged: + age: 70 + is_household_head: true + sharer: + age: 45 + is_household_head: false + benunits: + younger_family: + members: [younger_flagged] + elder_family: + members: [elder_flagged] + sharer_family: + members: [sharer] + liable_for_share_of_household_rent: true + households: + home: + members: [younger_flagged, elder_flagged, sharer] + rent: 20_000 + tenure_type: RENT_PRIVATELY + country: WALES + local_authority: CARDIFF + council_tax: 1_800 + output: + is_resolved_household_head: [false, true, false] + benunit_contains_household_head: [false, true, false] + # Two people liable (UC Sch 4 para 24(4)): 10,000 each. + is_liable_for_household_rent: [false, true, true] + share_of_household_rent: [0, 0.5, 0.5] + benunit_rent: [0, 10_000, 10_000] + is_non_dependant_of_household_head: [true, false, false] + # UC counts the non-dependant in one claim by a liable renter only (UC + # Regs 2013 Sch 4 para 9(2)(f); see uc_non_dependants_counted): the + # head's family's if it claims, else the first liable family that does. + # The head is 70, over State Pension age, so not eligible for UC; the + # sharer's claim counts the non-dependant. HB apportions it between the + # joint occupiers by their shares of the rent (reg 74(5)). + is_uc_eligible: [true, false, true] + uc_non_dependants_counted: [false, false, true] + uc_non_dep_deductions: [0, 0, 1_000] + housing_benefit_non_dep_deductions: [0, 400, 400] + council_tax_reduction_claimant_benunit: [false, true, true] + council_tax_reduction_individual_non_dep_deduction_eligible: [true, false, false] + +- name: The head and a sharer both flagged beside a non-dependant - only the head's family's Universal Credit claim counts the non-dependant + period: 2026 + absolute_error_margin: 0.01 + input: + people: + head: + age: 50 + is_household_head: true + sharer: + age: 40 + is_household_head: true + non_dependant: + age: 30 + is_household_head: false + uc_individual_non_dep_deduction: 1_000 + household_benefits_individual_non_dep_deduction: 800 + council_tax_reduction_individual_non_dep_deduction: 200 + benunits: + head_family: + members: [head] + sharer_family: + members: [sharer] + liable_for_share_of_household_rent: true + non_dependant_family: + members: [non_dependant] + households: + home: + members: [head, sharer, non_dependant] + rent: 20_000 + tenure_type: RENT_PRIVATELY + country: WALES + local_authority: CARDIFF + council_tax: 1_800 + output: + is_resolved_household_head: [true, false, false] + benunit_contains_household_head: [true, false, false] + # Two people liable (UC Sch 4 para 24(4)): 10,000 each. + is_liable_for_household_rent: [true, true, false] + share_of_household_rent: [0.5, 0.5, 0] + benunit_rent: [10_000, 10_000, 0] + is_non_dependant_of_household_head: [false, false, true] + # Both renters claim Universal Credit. UC Regs 2013 Sch 4 para 9(2)(f) + # lets only one of their claims count the non-dependant; the model gives + # them to the head's family's claim, so they are not one in the sharer's: + # one deduction and one extra bedroom (para 10), on the head's family. + # Read from the raw flag, both claims counted them. + is_uc_eligible: [true, true, true] + uc_non_dependants_counted: [true, false, false] + uc_non_dep_deductions: [1_000, 0, 0] + LHA_allowed_bedrooms: [2, 1, 1] + # HB apportions the deduction between the joint occupiers by their shares + # of the rent (reg 74(5)). + housing_benefit_non_dep_deductions: [400, 400, 0] + council_tax_reduction_claimant_benunit: [true, true, false] + council_tax_reduction_individual_non_dep_deduction_eligible: [false, false, true] + +- name: A lodger flagged as head beside the householder pays the householder and has no share of the rent + period: 2026 + absolute_error_margin: 0.01 + input: + people: + householder: + age: 50 + is_household_head: true + lodger: + age: 30 + is_household_head: true + rent_paid_as_lodger: 5_200 + benunits: + householder_family: + members: [householder] + lodger_family: + members: [lodger] + households: + home: + members: [householder, lodger] + rent: 12_000 + tenure_type: RENT_FROM_COUNCIL + country: ENGLAND + local_authority: MAIDSTONE + council_tax: 1_800 + output: + is_resolved_household_head: [true, false] + pays_rent_to_householder: [false, true] + share_of_household_rent: [1, 0] + personal_rent: [12_000, 5_200] + # The lodger rents privately from the householder, not from the council. + benunit_tenure_type: [RENT_FROM_COUNCIL, RENT_PRIVATELY] + is_non_dependant_of_household_head: [false, false] + council_tax_reduction_claimant_benunit: [true, false] + council_tax_reduction_individual_non_dep_deduction_eligible: [false, false] + +- name: A student flagged as head beside their parent lives with their parents for the maintenance loan + period: 2025 + input: + people: + parent: + age: 52 + current_education: NOT_IN_EDUCATION + adjusted_net_income: 30_000 + is_household_head: true + student: + age: 20 + current_education: TERTIARY + adjusted_net_income: 2_000 + is_household_head: true + benunits: + parent_benunit: + members: [parent] + student_benunit: + members: [student] + households: + household: + members: [parent, student] + country: ENGLAND + region: NORTH_WEST + output: + is_resolved_household_head: [true, false] + maintenance_loan_living_arrangement: [AWAY_OUTSIDE_LONDON, LIVING_WITH_PARENTS] + maintenance_loan_household_income: [30_000, 32_000] + +- name: Where only one household flags its head, every other household's eldest member is its head + period: 2026 + absolute_error_margin: 0.01 + input: + people: + first_head: + age: 40 + is_household_head: true + first_parent: + age: 70 + is_household_head: false + second_younger: + age: 30 + second_elder: + age: 65 + benunits: + first_head_family: + members: [first_head] + first_parent_family: + members: [first_parent] + second_younger_family: + members: [second_younger] + second_elder_family: + members: [second_elder] + households: + first_home: + members: [first_head, first_parent] + rent: 12_000 + tenure_type: RENT_PRIVATELY + country: ENGLAND + local_authority: MAIDSTONE + council_tax: 1_800 + second_home: + members: [second_younger, second_elder] + rent: 9_000 + tenure_type: RENT_PRIVATELY + country: ENGLAND + local_authority: MAIDSTONE + council_tax: 1_500 + output: + # The second household's members default to false once anyone has the + # input, so no one there is flagged. + is_household_head: [true, false, false, false] + is_resolved_household_head: [true, false, false, true] + personal_rent: [12_000, 0, 0, 9_000] + is_non_dependant_of_household_head: [false, true, true, false] + council_tax_reduction_claimant_benunit: [true, false, false, true] diff --git a/policyengine_uk/tests/test_council_tax_reduction_claimant_properties.py b/policyengine_uk/tests/test_council_tax_reduction_claimant_properties.py index 09af6f3d4f..6424555e09 100644 --- a/policyengine_uk/tests/test_council_tax_reduction_claimant_properties.py +++ b/policyengine_uk/tests/test_council_tax_reduction_claimant_properties.py @@ -37,9 +37,12 @@ non-dependant (the applicant's family, SI 2012/2885 reg 9(2)(a)). Pin: no member of a sharer family is one, and every adult outside the claimant families, the sharer families and boarders' and lodgers' families is one. -5. Differential: where the input flags at most one head, the family holding - the head is the one holding the person-level household head that Housing - Benefit and Universal Credit use. +5. Differential: where the input flags exactly one member of a household, + or no household in the simulation has the input (so the flag's formula + picks the eldest), the family holding the head is the one holding the + person-level is_household_head flag. + (Every programme reads the same head; see + test_household_head_agreement_properties.py.) 6. Not age: with one head flagged, raising every other member's age above the head's does not change who claims. 7. No-op: where the flagged head is strictly the eldest member and the rent @@ -270,7 +273,7 @@ def test_claimant_invariants(population): np.logical_or.at(has_adult, facts["benunit"], age >= 18) assert not np.any(claimant & ~has_adult) liable = calc(sim, "council_tax_reduction_liable_person") - head_person = calc(sim, "council_tax_reduction_household_head") + head_person = calc(sim, "is_resolved_household_head") in_head_or_sharer = (head | sharer)[facts["benunit"]] assert np.all(age[liable] >= 18) assert np.all(in_head_or_sharer[liable]) @@ -315,7 +318,7 @@ def test_claimant_invariants(population): assert not np.any(non_dep & person_sharer) assert np.all(non_dep[adult & ~person_claimant & ~person_sharer & ~lodger]) - # 5. Differential with the person-level household head. Once anyone in + # 5. Differential with the person-level household head flag. Once anyone in # the simulation has the input, everyone else's defaults to false, so an # "unset" household is well formed only if every household is. all_unset = all(house["head_flags"] == "unset" for house in population) diff --git a/policyengine_uk/tests/test_household_head_agreement_properties.py b/policyengine_uk/tests/test_household_head_agreement_properties.py new file mode 100644 index 0000000000..4862578ad5 --- /dev/null +++ b/policyengine_uk/tests/test_household_head_agreement_properties.py @@ -0,0 +1,480 @@ +"""Property-based tests that every programme reads one household head. + +The input is_household_head can flag several members of a household, or none: +in user situations, and in any household left without the input once another +household has it (it then defaults to false). is_resolved_household_head +settles on exactly one head per household, and every programme reads it or +benunit_contains_household_head. + +Invariants, for any generated population of households and any flag input +(one member, several, both members of a couple, everyone, none, or no input +at all), with families of non-dependants, sharers, boarders and lodgers, +families of a child alone, and ties in age. Invariants marked "pin" restate +the formula, so they guard against regressions rather than check it +independently. + +1. One head: exactly one member of each household is the head: the eldest + flagged member where the input flags any member of the household, else the + eldest member, a tie in age going to the member listed first. Checked + against an independent implementation. +2. One family: benunit_contains_household_head holds for the head's family + and no other. +3. Agreement: every programme treats that family as the household head's. + - Rent: only the head's family and sharers have a share of the household's + rent; the head's family's share sits on the head alone; and the shares + sum to the household's rent. Beyond that, a person's rent is only what + they pay the householder as a boarder or lodger. + - Non-dependants (pin): a person is a non-dependant of the household head + if and only if their family is neither the head's nor liable for rent. + - Universal Credit non-dependants: at most one family in a household + counts them (uc_non_dependants_counted; UC Regs 2013 Sch 4 para + 9(2)(f)). Pin: deductions fall only on that family's claim, and where + the head's family is liable for the rent and claims Universal Credit, + it is the head's. Only families with a share of the rent have Housing + Benefit non-dependant deductions. + - Council Tax Reduction: only the head's family and sharers claim; where + the head is 18 or over, the head's family claims and no one in it is a + non-dependant. + - A family paying the householder as boarders or lodgers is never the + head's, and the maintenance loan never has the head living with parents. +4. Metamorphic: every value the simulation computes, other than the + is_household_head input itself, is identical to the simulation whose input + flags exactly the head from invariant 1. That input is well formed, so all + programmes agree on the head there; no flag input can make them disagree. +5. Structural: no module outside is_resolved_household_head reads + is_household_head, so a new formula cannot bypass the resolved head. +""" + +import re +from pathlib import Path + +import numpy as np +from hypothesis import HealthCheck, example, given, settings +from hypothesis import strategies as st + +import policyengine_uk +from policyengine_uk import Simulation + +YEAR = 2026 +PROPERTY_SETTINGS = settings( + max_examples=15, + deadline=None, + derandomize=True, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large], +) +SCHEMES = [ + ("ENGLAND", "MAIDSTONE"), + ("ENGLAND", "MERTON"), + ("ENGLAND", "OXFORD"), + ("SCOTLAND", "CITY_OF_EDINBURGH"), + ("WALES", "CARDIFF"), +] +TENURES = [ + "RENT_PRIVATELY", + "RENT_FROM_COUNCIL", + "RENT_FROM_HA", + "OWNED_OUTRIGHT", + "OWNED_WITH_MORTGAGE", +] +# Roles of the families after the first. +ROLES = ["non_dependant", "sharer", "boarder", "lodger"] +HEAD_FLAGS = ["one", "several", "couple", "everyone", "none", "unset"] +# Ages around 16, 18 and State Pension age, and repeated values, so ties and +# young adults are common. +adult_age = st.one_of( + st.sampled_from([16, 17, 18, 19, 20, 40, 66, 67, 80]), st.integers(16, 100) +) +money = st.floats(0, 40_000, allow_nan=False, allow_infinity=False) +OUTPUTS = [ + "household_net_income", + "hbai_household_net_income_ahc", + "maintenance_loan", + "is_resolved_household_head", + "benunit_contains_household_head", + "is_claimant_or_partner", + "share_of_household_rent", + "personal_rent", + "benunit_is_rent_liable", + "is_non_dependant_of_household_head", + "uc_non_dep_deductions", + "uc_non_dependants_counted", + "housing_benefit_non_dep_deductions", + "LHA_allowed_bedrooms", + "housing_benefit_LHA_allowed_bedrooms", + "lha_renter_has_non_dependant", + "council_tax_reduction_claimant_benunit", + "council_tax_reduction_individual_non_dep_deduction_eligible", + "pays_rent_to_householder", + "maintenance_loan_living_arrangement", + "maintenance_loan_household_income", +] + + +@st.composite +def family(draw): + # Sometimes a family of one child under 16 and no adult, so that the + # head's family can have no claimant and no one is liable for the rent. + children_only = draw(st.sampled_from([False, False, False, True])) + return dict( + ages=[] if children_only else draw(st.lists(adult_age, min_size=1, max_size=2)), + child_age=draw( + st.integers(0, 15) + if children_only + else st.one_of(st.none(), st.integers(0, 17)) + ), + earnings=draw(st.lists(money, min_size=2, max_size=2)), + role=draw(st.sampled_from(ROLES)), + payment=draw(st.floats(1, 12_000, allow_nan=False)), + student=draw(st.booleans()), + ) + + +def size(fam): + return len(fam["ages"]) + (fam["child_age"] is not None) + + +@st.composite +def households(draw): + families = draw(st.lists(family(), min_size=1, max_size=4)) + members = sum(size(fam) for fam in families) + return dict( + families=families, + scheme=draw(st.sampled_from(SCHEMES)), + tenure=draw(st.sampled_from(TENURES)), + rent=draw(money), + council_tax=draw(st.floats(0, 4_000, allow_nan=False)), + head_flags=draw(st.sampled_from(HEAD_FLAGS)), + # Which member is flagged ("one"), or which are ("several"). + head_choice=draw(st.lists(st.integers(0, members - 1), min_size=1)), + ) + + +population = st.lists(households(), min_size=1, max_size=5) + + +def _family(ages, child_age=None, role="non_dependant", student=False, earnings=0.0): + return dict( + ages=ages, + child_age=child_age, + earnings=[earnings, 0.0], + role=role, + payment=1.0, + student=student, + ) + + +def _household( + families, head_flags, head_choice, rent=9_000.0, tenure="RENT_PRIVATELY" +): + return dict( + families=families, + scheme=("ENGLAND", "MAIDSTONE"), + tenure=tenure, + rent=rent, + council_tax=1_500.0, + head_flags=head_flags, + head_choice=head_choice, + ) + + +# Households where no one is liable for the rent, so share_of_household_rent +# falls back to the head's family, and the raw flags disagree with the head. +# Random generation rarely builds them. +NO_ONE_LIABLE = [ + # A flagged household beside one of two children with no member flagged: + # the elder child's family has the rent. + _household([_family([40])], "one", [0], rent=6_000.0), + _household([_family([], 12), _family([], 9)], "none", [0]), + # Two flagged children: the elder, alone in its family, is the head, and + # the younger's parent is neither liable nor a sharer. + _household([_family([], 12), _family([40], 8)], "several", [0, 2]), +] +# A student in higher education flagged as head beside their parent, in an +# owner-occupied home: the only case where the maintenance loan's +# living-arrangement and household-income proxies turn on the head. The parent +# earns, so the sponsor's income counts. Random generation rarely builds it. +STUDENT_FLAGGED_BESIDE_PARENT = [ + _household( + [_family([52], earnings=30_000.0), _family([20], student=True)], + "several", + [0, 1], + rent=0.0, + tenure="OWNED_OUTRIGHT", + ), +] +# The head and a sharer both flagged beside a non-dependant, all three adults +# claiming Universal Credit: read from the raw flag, both renters' claims +# count the non-dependant. Random generation rarely builds a flagged sharer +# beside a non-dependant. +HEAD_AND_SHARER_FLAGGED = [ + _household( + [_family([50]), _family([40], role="sharer"), _family([30])], + "several", + [0, 1], + ), +] +# No one flagged, so the eldest member is the head, and the head's family is +# listed after a sharer's, beside a non-dependant, all claiming Universal +# Credit. The head's family's claim counts the non-dependant because the head +# comes first in the order, not because its family is listed first. Read from +# the raw flag, or without the head-first order, the sharer's claim would. +HEAD_LISTED_AFTER_SHARER = [ + _household( + [_family([30]), _family([40], role="sharer"), _family([50])], + "none", + [0], + ), +] + + +def flagged_members(house): + """Indices of the members the input flags, in household order.""" + families = house["families"] + members = sum(size(fam) for fam in families) + mode = house["head_flags"] + if mode == "one": + return {house["head_choice"][0]} + if mode == "several": + return set(house["head_choice"]) + if mode == "everyone": + return set(range(members)) + if mode == "couple": + start = 0 + for fam in families: + if len(fam["ages"]) == 2: + return {start, start + 1} + start += size(fam) + return set(house["head_choice"]) + return set() + + +def build(population, head_input=None): + """One situation for the whole population, plus per-person facts. + + head_input, if given, is the is_household_head input for every person, + in order; otherwise each household's head_flags decides it. + """ + people, benunits, homes = {}, {}, {} + facts = dict(flagged=[], age=[], household=[], benunit=[], sharer=[]) + benunit_index = 0 + for h, house in enumerate(population): + country, local_authority = house["scheme"] + flagged_set = flagged_members(house) + members, member_index = [], 0 + for f, fam in enumerate(house["families"]): + role = fam["role"] if f > 0 else "head_candidate" + ids = [] + ages = list(fam["ages"]) + if fam["child_age"] is not None: + ages.append(fam["child_age"]) + for i, age in enumerate(ages): + pid = f"h{h}_f{f}_m{i}" + adult = i < len(fam["ages"]) + person = {"age": age} + if adult: + person["employment_income"] = fam["earnings"][i] + if fam["student"] and i == 0 and 18 <= age < 25: + person["current_education"] = "TERTIARY" + if i == 0 and role in ("boarder", "lodger"): + person[f"rent_paid_as_{role}"] = fam["payment"] + flagged = member_index in flagged_set + if head_input is not None: + person["is_household_head"] = bool(head_input[len(facts["age"])]) + elif house["head_flags"] != "unset": + person["is_household_head"] = flagged + people[pid] = person + ids.append(pid) + facts["flagged"].append(flagged) + facts["age"].append(age) + facts["household"].append(h) + facts["benunit"].append(benunit_index) + member_index += 1 + facts["sharer"].append(role == "sharer") + benunits[f"h{h}_f{f}"] = { + "members": ids, + "liable_for_share_of_household_rent": role == "sharer", + "claims_all_entitled_benefits": True, + "would_claim_uc": True, + } + members.extend(ids) + benunit_index += 1 + homes[f"h{h}"] = { + "members": members, + "country": country, + "local_authority": local_authority, + "council_tax": house["council_tax"], + "rent": house["rent"], + "tenure_type": house["tenure"], + } + situation = { + group: { + name: { + key: value if key == "members" else {YEAR: value} + for key, value in entity.items() + } + for name, entity in entities.items() + } + for group, entities in ( + ("people", people), + ("benunits", benunits), + ("households", homes), + ) + } + return situation, {k: np.array(v) for k, v in facts.items()} + + +def reference_head(facts): + """The household head, implemented independently of the model.""" + head = np.zeros(facts["age"].size, dtype=bool) + for h in np.unique(facts["household"]): + members = np.flatnonzero(facts["household"] == h) # in order + flagged = members[facts["flagged"][members]] + candidates = flagged if flagged.size else members + # argmax takes the first of tied maxima: the member listed first. + head[candidates[np.argmax(facts["age"][candidates])]] = True + return head + + +def calc(simulation, variable): + return np.asarray(simulation.calculate(variable, YEAR)) + + +def any_in_benunit(values, facts): + result = np.zeros(facts["sharer"].size, dtype=bool) + np.logical_or.at(result, facts["benunit"], values) + return result + + +def sum_in_household(values, household): + return np.bincount(household, weights=values.astype(float)) + + +@PROPERTY_SETTINGS +@given(population) +@example(NO_ONE_LIABLE) +@example(STUDENT_FLAGGED_BESIDE_PARENT) +@example(HEAD_AND_SHARER_FLAGGED) +@example(HEAD_LISTED_AFTER_SHARER) +def test_every_programme_reads_one_head(population): + situation, facts = build(population) + sim = Simulation(situation=situation) + benunit_of = facts["benunit"] + age = facts["age"] + + # 1. One head, as the independent implementation finds it. + head = calc(sim, "is_resolved_household_head") + expected = reference_head(facts) + assert np.array_equal(head, expected) + assert np.all(sum_in_household(head, facts["household"]) == 1) + + # 2. One family. + head_family = calc(sim, "benunit_contains_household_head") + assert np.array_equal(head_family, any_in_benunit(head, facts)) + person_head_family = head_family[benunit_of] + + # 3. Agreement. Rent: + sharer = facts["sharer"] + share = calc(sim, "share_of_household_rent") + assert np.all(share[~head_family & ~sharer] == 0) + rent = np.array([house["rent"] for house in population]) + personal_rent = calc(sim, "personal_rent") + paid_to_householder = calc(sim, "rent_paid_as_boarder") + calc( + sim, "rent_paid_as_lodger" + ) + share_of_rent = personal_rent - paid_to_householder + is_benunit_head = calc(sim, "is_benunit_head") + carries_share = head | (sharer[benunit_of] & ~person_head_family & is_benunit_head) + assert np.all(np.abs(share_of_rent[~carries_share]) < 0.01) + rent_charged = sum_in_household(share_of_rent, facts["household"]) + assert np.allclose(rent_charged, rent, atol=0.01) + # Non-dependants (pin). + non_dependant = calc(sim, "is_non_dependant_of_household_head") + rent_liable = calc(sim, "benunit_is_rent_liable") + assert np.array_equal(non_dependant, ~person_head_family & ~rent_liable[benunit_of]) + # Universal Credit non-dependant deductions fall on the one claim that + # counts the household's non-dependants: the head's family's wherever it + # is liable for the rent and claims. + counted = calc(sim, "uc_non_dependants_counted") + assert np.all(counted[calc(sim, "uc_non_dep_deductions") > 0]) + household_of_benunit = np.zeros(head_family.size, dtype=int) + household_of_benunit[benunit_of] = facts["household"] + assert np.all(sum_in_household(counted, household_of_benunit) <= 1) + head_family_claims = ( + head_family + & any_in_benunit(calc(sim, "is_liable_for_household_rent"), facts) + & calc(sim, "is_uc_eligible") + & calc(sim, "would_claim_uc") + ) + head_claims_uc = sum_in_household(head_family_claims, household_of_benunit) > 0 + in_household = head_claims_uc[household_of_benunit] + assert np.array_equal(counted[in_household], head_family[in_household]) + # Housing Benefit non-dependant deductions. + assert np.all(calc(sim, "housing_benefit_non_dep_deductions")[share == 0] == 0) + # Council Tax Reduction. + claimant = calc(sim, "council_tax_reduction_claimant_benunit") + assert not np.any(claimant & ~head_family & ~sharer) + adult_head_family = any_in_benunit(head & (age >= 18), facts) + assert np.all(claimant[adult_head_family]) + ctr_non_dependant = calc( + sim, "council_tax_reduction_individual_non_dep_deduction_eligible" + ) + assert not np.any(ctr_non_dependant & adult_head_family[benunit_of]) + # Boarders and lodgers; the maintenance loan. + assert not np.any(calc(sim, "pays_rent_to_householder") & person_head_family) + arrangement = calc(sim, "maintenance_loan_living_arrangement") + assert not np.any(arrangement[head] == "LIVING_WITH_PARENTS") + + +def computed_values(simulation): + """Every (variable, period) array the simulation holds.""" + values = {} + for name in simulation.tax_benefit_system.variables: + holder = simulation.get_holder(name) + for period in holder.get_known_periods(): + values[(name, str(period))] = holder.get_array(period) + return values + + +def identical(a, b): + a, b = np.asarray(a), np.asarray(b) + if a.dtype.kind == "f": + return np.array_equal(a, b, equal_nan=True) + return np.array_equal(a, b) + + +@PROPERTY_SETTINGS +@given(population) +@example(NO_ONE_LIABLE) +@example(STUDENT_FLAGGED_BESIDE_PARENT) +@example(HEAD_AND_SHARER_FLAGGED) +@example(HEAD_LISTED_AFTER_SHARER) +def test_flags_matter_only_through_the_head(population): + situation, facts = build(population) + well_formed, _ = build(population, head_input=reference_head(facts)) + simulations = Simulation(situation=situation), Simulation(situation=well_formed) + for simulation in simulations: + for variable in OUTPUTS: + simulation.calculate(variable, YEAR) + # 4. Metamorphic: everything but the input itself is identical. + flagged, settled = (computed_values(simulation) for simulation in simulations) + flagged = {k: v for k, v in flagged.items() if k[0] != "is_household_head"} + settled = {k: v for k, v in settled.items() if k[0] != "is_household_head"} + assert flagged.keys() == settled.keys() + different = [key for key in flagged if not identical(flagged[key], settled[key])] + assert different == [] + + +def test_only_the_resolved_head_reads_the_flag(): + # 5. Structural: is_household_head is read in one place. + package = Path(policyengine_uk.__file__).parent + reader = re.compile(r"""["']is_household_head["']""") + allowed = { + Path("variables/household/demographic/is_resolved_household_head.py"), + } + readers = { + path.relative_to(package) + for path in package.rglob("*.py") + if "tests" not in path.relative_to(package).parts + and reader.search(path.read_text()) + } + assert readers == allowed diff --git a/policyengine_uk/variables/gov/dfe/maintenance_loans/maintenance_loan_household_income.py b/policyengine_uk/variables/gov/dfe/maintenance_loans/maintenance_loan_household_income.py index 3da851c3ec..4aaf08b03e 100644 --- a/policyengine_uk/variables/gov/dfe/maintenance_loans/maintenance_loan_household_income.py +++ b/policyengine_uk/variables/gov/dfe/maintenance_loans/maintenance_loan_household_income.py @@ -29,7 +29,7 @@ def formula(person, period, parameters): # academic year is independent. under_independent_age = person("age", period) < p.independent_student_age is_couple = person.benunit("is_couple", period) - is_household_head = person("is_household_head", period) + is_household_head = person("is_resolved_household_head", period) is_parent = person("is_parent", period) tenure_holder = person.household.get_holder("tenure_type") has_explicit_tenure = ( diff --git a/policyengine_uk/variables/gov/dfe/maintenance_loans/maintenance_loan_living_arrangement.py b/policyengine_uk/variables/gov/dfe/maintenance_loans/maintenance_loan_living_arrangement.py index 4a7429fd07..616f8ed97a 100644 --- a/policyengine_uk/variables/gov/dfe/maintenance_loans/maintenance_loan_living_arrangement.py +++ b/policyengine_uk/variables/gov/dfe/maintenance_loans/maintenance_loan_living_arrangement.py @@ -25,7 +25,7 @@ class maintenance_loan_living_arrangement(Variable): def formula(person, period, parameters): in_higher_education = person("maintenance_loan_in_higher_education", period) has_sponsor = person("maintenance_loan_has_sponsor", period) - is_household_head = person("is_household_head", period) + is_household_head = person("is_resolved_household_head", period) is_couple = person.benunit("is_couple", period) is_parent = person("is_parent", period) tenure_holder = person.household.get_holder("tenure_type") diff --git a/policyengine_uk/variables/gov/dwp/is_non_dependant_of_household_head.py b/policyengine_uk/variables/gov/dwp/is_non_dependant_of_household_head.py index a2e8556fe3..109ede8fe2 100644 --- a/policyengine_uk/variables/gov/dwp/is_non_dependant_of_household_head.py +++ b/policyengine_uk/variables/gov/dwp/is_non_dependant_of_household_head.py @@ -28,6 +28,6 @@ class is_non_dependant_of_household_head(Variable): def formula(person, period, parameters): # UC Regs 2013 Sch 4 para 9(2)(d)-(f); HB Regs 2006 reg 3(2)(d)-(e) # and 3(4); HB (SPC) Regs 2006 reg 3. - head_family = person.benunit.any(person("is_household_head", period)) + head_family = person.benunit("benunit_contains_household_head", period) liable_for_rent = person.benunit("benunit_is_rent_liable", period) return ~head_family & ~liable_for_rent diff --git a/policyengine_uk/variables/gov/dwp/universal_credit/housing_costs_element/non_dep_deduction/uc_non_dependants_counted.py b/policyengine_uk/variables/gov/dwp/universal_credit/housing_costs_element/non_dep_deduction/uc_non_dependants_counted.py index af1c22f5b9..f217751f58 100644 --- a/policyengine_uk/variables/gov/dwp/universal_credit/housing_costs_element/non_dep_deduction/uc_non_dependants_counted.py +++ b/policyengine_uk/variables/gov/dwp/universal_credit/housing_costs_element/non_dep_deduction/uc_non_dependants_counted.py @@ -30,7 +30,7 @@ def formula(benunit, period, parameters): & benunit("is_uc_eligible", period) & benunit("would_claim_uc", period) ) - head_family = benunit.any(person("is_household_head", period)) + head_family = benunit("benunit_contains_household_head", period) # Order the claims: the household head's family first, then the # others in the order the families were given. position = np.arange(len(head_family)) diff --git a/policyengine_uk/variables/gov/local_authorities/council_tax_reduction/benunit_contains_household_head.py b/policyengine_uk/variables/gov/local_authorities/council_tax_reduction/benunit_contains_household_head.py deleted file mode 100644 index 195f8b4ea7..0000000000 --- a/policyengine_uk/variables/gov/local_authorities/council_tax_reduction/benunit_contains_household_head.py +++ /dev/null @@ -1,30 +0,0 @@ -from policyengine_uk.model_api import * - - -class benunit_contains_household_head(Variable): - value_type = bool - entity = BenUnit - label = "Benefit unit contains the household head" - documentation = ( - "Whether the household head is in this family. The head is the " - "household reference person: a householder, in whose name the " - "accommodation is owned or rented. Council Tax Reduction takes the " - "head as the resident liable for the council tax under the Local " - "Government Finance Act 1992 (s.6 in England and Wales, s.75 in " - "Scotland), which every class of applicant requires. Each household " - "has exactly one head. Where the input flags more than one member, " - "the eldest flagged member is the head; where it flags none, the " - "eldest member is. Rent and non-dependant rules for Universal Credit " - "and Housing Benefit read is_household_head directly, so they agree " - "with this only where exactly one member is flagged." - ) - definition_period = YEAR - reference = ( - "https://www.legislation.gov.uk/ukpga/1992/14/section/6", - "https://www.legislation.gov.uk/ukpga/1992/14/section/75", - ) - - def formula(benunit, period, parameters): - return benunit.any( - benunit.members("council_tax_reduction_household_head", period) - ) diff --git a/policyengine_uk/variables/gov/local_authorities/council_tax_reduction/council_tax_reduction_household_head.py b/policyengine_uk/variables/gov/local_authorities/council_tax_reduction/council_tax_reduction_household_head.py deleted file mode 100644 index 1f0229bf12..0000000000 --- a/policyengine_uk/variables/gov/local_authorities/council_tax_reduction/council_tax_reduction_household_head.py +++ /dev/null @@ -1,27 +0,0 @@ -from policyengine_uk.model_api import * - - -class council_tax_reduction_household_head(Variable): - value_type = bool - entity = Person - label = "Household head for Council Tax Reduction" - documentation = ( - "The member Council Tax Reduction treats as the household head: the " - "household reference person, exactly one per household. Where the " - "input flags more than one member, the eldest flagged member is the " - "head; where it flags none, the eldest member is. Ties go by order." - ) - definition_period = YEAR - - def formula(person, period, parameters): - household = person.household - age = person("age", period) - flagged = person("is_household_head", period) - # get_rank breaks ties in age by order within the household, so - # exactly one member has rank 0. - rank = where( - household.any(flagged), - person.get_rank(household, -age, condition=flagged), - person.get_rank(household, -age), - ) - return rank == 0 diff --git a/policyengine_uk/variables/gov/local_authorities/council_tax_reduction/council_tax_reduction_liable_person.py b/policyengine_uk/variables/gov/local_authorities/council_tax_reduction/council_tax_reduction_liable_person.py index cdc697fb4a..f00dac1484 100644 --- a/policyengine_uk/variables/gov/local_authorities/council_tax_reduction/council_tax_reduction_liable_person.py +++ b/policyengine_uk/variables/gov/local_authorities/council_tax_reduction/council_tax_reduction_liable_person.py @@ -32,7 +32,7 @@ class council_tax_reduction_liable_person(Variable): def formula(person, period, parameters): adult = person("age", period) >= 18 # s.6(5), s.99(1) - head = person("council_tax_reduction_household_head", period) + head = person("is_resolved_household_head", period) claimant_or_partner = person("is_claimant_or_partner", period) head_is_claimant_or_partner = person.benunit.any(head & claimant_or_partner) head_family = person.benunit.any(head) diff --git a/policyengine_uk/variables/household/consumption/personal_rent.py b/policyengine_uk/variables/household/consumption/personal_rent.py index 8166a552a3..c00db1ee85 100644 --- a/policyengine_uk/variables/household/consumption/personal_rent.py +++ b/policyengine_uk/variables/household/consumption/personal_rent.py @@ -25,8 +25,8 @@ class personal_rent(Variable): def formula(person, period, parameters): rent = person.household("rent", period) share = person.benunit("share_of_household_rent", period) - household_head = person("is_household_head", period) - head_family = person.benunit.any(household_head) + household_head = person("is_resolved_household_head", period) + head_family = person.benunit("benunit_contains_household_head", period) holder = where(head_family, household_head, person("is_benunit_head", period)) # A licence or other permission to occupy is a rent payment (UC Regs # 2013 Sch 1 para 2(b); HB Regs 2006 reg 12(1)): boarders and lodgers diff --git a/policyengine_uk/variables/household/consumption/rent/is_liable_for_household_rent.py b/policyengine_uk/variables/household/consumption/rent/is_liable_for_household_rent.py index 9ddfc87b32..6d3838bd37 100644 --- a/policyengine_uk/variables/household/consumption/rent/is_liable_for_household_rent.py +++ b/policyengine_uk/variables/household/consumption/rent/is_liable_for_household_rent.py @@ -17,6 +17,6 @@ class is_liable_for_household_rent(Variable): ) def formula(person, period, parameters): - head_family = person.benunit.any(person("is_household_head", period)) + head_family = person.benunit("benunit_contains_household_head", period) sharer = person.benunit("liable_for_share_of_household_rent", period) return person("is_claimant_or_partner", period) & (head_family | sharer) diff --git a/policyengine_uk/variables/household/consumption/rent/share_of_household_rent.py b/policyengine_uk/variables/household/consumption/rent/share_of_household_rent.py index 4b63374370..0d3e883f5b 100644 --- a/policyengine_uk/variables/household/consumption/rent/share_of_household_rent.py +++ b/policyengine_uk/variables/household/consumption/rent/share_of_household_rent.py @@ -23,7 +23,7 @@ def formula(benunit, period, parameters): person = benunit.members liable = person("is_liable_for_household_rent", period) liable_people = benunit.max(person.household.sum(liable)) - head_family = benunit.any(person("is_household_head", period)) + head_family = benunit("benunit_contains_household_head", period) return where( liable_people > 0, benunit.sum(liable) / max_(liable_people, 1), diff --git a/policyengine_uk/variables/household/demographic/benunit_contains_household_head.py b/policyengine_uk/variables/household/demographic/benunit_contains_household_head.py new file mode 100644 index 0000000000..7f8a351a27 --- /dev/null +++ b/policyengine_uk/variables/household/demographic/benunit_contains_household_head.py @@ -0,0 +1,19 @@ +from policyengine_uk.model_api import * + + +class benunit_contains_household_head(Variable): + value_type = bool + entity = BenUnit + label = "Benefit unit contains the household head" + documentation = ( + "Whether the household head (see is_resolved_household_head) is in " + "this family, so exactly one family in each household contains it " + "whatever is_household_head flags. Rent liability, Universal Credit " + "and Housing Benefit non-dependants and size criteria, boarders and " + "lodgers, and the Council Tax Reduction claimant all read this " + "family as the household head's." + ) + definition_period = YEAR + + def formula(benunit, period, parameters): + return benunit.any(benunit.members("is_resolved_household_head", period)) diff --git a/policyengine_uk/variables/household/demographic/is_household_head.py b/policyengine_uk/variables/household/demographic/is_household_head.py index febf1489bf..bd9e9d8022 100644 --- a/policyengine_uk/variables/household/demographic/is_household_head.py +++ b/policyengine_uk/variables/household/demographic/is_household_head.py @@ -6,6 +6,15 @@ class is_household_head(Variable): value_type = bool entity = Person label = "Whether this person is the head-of-household" + documentation = ( + "Input flag for the household reference person; in the Family " + "Resources Survey data, the HRP. Where no one in the simulation " + "has the input, the eldest member of each household is the head; " + "once anyone has it, everyone else's defaults to false. Input can " + "flag several members, or none. " + "Programmes read is_resolved_household_head instead, which settles " + "on exactly one head per household." + ) definition_period = YEAR def formula(person, period, parameters): diff --git a/policyengine_uk/variables/household/demographic/is_resolved_household_head.py b/policyengine_uk/variables/household/demographic/is_resolved_household_head.py new file mode 100644 index 0000000000..e8abc82359 --- /dev/null +++ b/policyengine_uk/variables/household/demographic/is_resolved_household_head.py @@ -0,0 +1,34 @@ +from policyengine_uk.model_api import * + + +class is_resolved_household_head(Variable): + value_type = bool + entity = Person + label = "Household head, exactly one per household" + documentation = ( + "The household head every programme uses. It stands for the " + "household reference person, a householder in whose name the " + "accommodation is owned or rented, but is chosen from the " + "is_household_head input and age alone. Exactly one member of each " + "household is the head: the member the input flags; where it flags " + "more than one member, the eldest flagged member; where it flags " + "none, the eldest member. A tie in age goes to the member listed " + "first among the people. " + "Read this, or benunit_contains_household_head, rather than " + "is_household_head, so that rent, non-dependant and Council Tax " + "Reduction rules agree on whose household it is." + ) + definition_period = YEAR + + def formula(person, period, parameters): + household = person.household + age = person("age", period) + flagged = person("is_household_head", period) + # get_rank sorts stably, so a tie in age goes to the member listed + # first and exactly one member has rank 0. + rank = where( + household.any(flagged), + person.get_rank(household, -age, condition=flagged), + person.get_rank(household, -age), + ) + return rank == 0 diff --git a/policyengine_uk/variables/household/income/pays_rent_to_householder.py b/policyengine_uk/variables/household/income/pays_rent_to_householder.py index 2cc695e85e..f21363c374 100644 --- a/policyengine_uk/variables/household/income/pays_rent_to_householder.py +++ b/policyengine_uk/variables/household/income/pays_rent_to_householder.py @@ -21,5 +21,5 @@ class pays_rent_to_householder(Variable): def formula(person, period, parameters): paid = add(person, period, ["rent_paid_as_boarder", "rent_paid_as_lodger"]) - in_head_benunit = person.benunit.any(person("is_household_head", period)) + in_head_benunit = person.benunit("benunit_contains_household_head", period) return person.benunit.any(paid > 0) & ~in_head_benunit