From e77f3c7ce0d9f7e876607d5c1d182d0747f99c7a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 1 Oct 2026 15:54:52 -0400 Subject: [PATCH 1/6] Default the BRMA from the household's region brma had no formula and defaulted to Maidstone, so a household calculation that set a region but no BRMA got Maidstone's LHA rates everywhere. It now gets its region's BRMA with the most entries in the list of rents (the pool policyengine-uk-data samples BRMAs from), and Maidstone only if the region is unknown. A formula stops policyengine-core carrying an earlier year's input forward, so the formula does that itself: the latest earlier BRMA applies while the household's region is the same as in that year. Fixes #2020 Co-Authored-By: Claude Opus 5.5 --- changelog.d/2020.fixed.md | 1 + .../baseline/finance/benefit/family/LHA.yaml | 7 +- .../tests/policy/baseline/household/brma.yaml | 143 ++++++++++++++ .../integration/entitledto_scenarios.yaml | 10 +- .../tests/test_brma_region_default.py | 178 ++++++++++++++++++ policyengine_uk/variables/household/BRMA.py | 54 +++++- 6 files changed, 387 insertions(+), 6 deletions(-) create mode 100644 changelog.d/2020.fixed.md create mode 100644 policyengine_uk/tests/policy/baseline/household/brma.yaml create mode 100644 policyengine_uk/tests/test_brma_region_default.py diff --git a/changelog.d/2020.fixed.md b/changelog.d/2020.fixed.md new file mode 100644 index 0000000000..ced7896fbd --- /dev/null +++ b/changelog.d/2020.fixed.md @@ -0,0 +1 @@ +- Place a household with no BRMA input in a BRMA in its region (the region's BRMA with the most entries in the LHA list of rents, Maidstone if the region is unknown) instead of Maidstone in every region. A household that sets no region is in London, the region default, so it now gets Inner South East London's LHA rates. A BRMA input still carries forward to later years while the household stays in the same region. diff --git a/policyengine_uk/tests/policy/baseline/finance/benefit/family/LHA.yaml b/policyengine_uk/tests/policy/baseline/finance/benefit/family/LHA.yaml index e3141fde06..dfaf68e31b 100644 --- a/policyengine_uk/tests/policy/baseline/finance/benefit/family/LHA.yaml +++ b/policyengine_uk/tests/policy/baseline/finance/benefit/family/LHA.yaml @@ -1,3 +1,6 @@ +# With no region input the household is in London (region's default), so +# brma defaults to London's BRMA rather than Maidstone, which it used for +# every region before brma had a formula. Maidstone's rate here was 9,771. - name: BRMA default value period: 2020 absolute_error_margin: 20 @@ -5,8 +8,8 @@ age: 18 LHA_category: C output: - brma: MAIDSTONE - BRMA_LHA_rate: 9_771 + brma: INNER_SOUTH_EAST_LONDON + BRMA_LHA_rate: 16_155 - name: BRMA inputs period: 2020 absolute_error_margin: 0 diff --git a/policyengine_uk/tests/policy/baseline/household/brma.yaml b/policyengine_uk/tests/policy/baseline/household/brma.yaml new file mode 100644 index 0000000000..e54b4cb553 --- /dev/null +++ b/policyengine_uk/tests/policy/baseline/household/brma.yaml @@ -0,0 +1,143 @@ +# A household with no BRMA input is placed in its region's BRMA with the most +# entries in the LHA list of rents (REGION_DEFAULT_BRMA in +# variables/household/BRMA.py). Each BRMA_LHA_rate below is the model's rate +# for the same household with that BRMA input explicitly, so the default and +# the explicit input must agree. A single 40-year-old private renter is in LHA +# category B (one bedroom). + +- name: London household without a BRMA gets Inner South East London + period: 2025 + absolute_error_margin: 0.01 + input: + age: 40 + region: LONDON + tenure_type: RENT_PRIVATELY + rent: 12_000 + output: + brma: INNER_SOUTH_EAST_LONDON + LHA_category: B + BRMA_LHA_rate: 16_750.24 + +- name: Scottish household without a BRMA gets Aberdeen and Shire + period: 2025 + absolute_error_margin: 0.01 + input: + age: 40 + region: SCOTLAND + tenure_type: RENT_PRIVATELY + rent: 12_000 + output: + brma: ABERDEEN_AND_SHIRE + LHA_category: B + BRMA_LHA_rate: 6_408.48 + +- name: Welsh household without a BRMA gets Cardiff + period: 2025 + absolute_error_margin: 0.01 + input: + age: 40 + region: WALES + tenure_type: RENT_PRIVATELY + rent: 12_000 + output: + brma: CARDIFF + LHA_category: B + BRMA_LHA_rate: 6_918.60 + +- name: Northern Ireland household without a BRMA gets Belfast + period: 2025 + absolute_error_margin: 0.01 + input: + age: 40 + region: NORTHERN_IRELAND + tenure_type: RENT_PRIVATELY + rent: 12_000 + output: + brma: BELFAST + LHA_category: B + BRMA_LHA_rate: 5_702.32 + +- name: An input BRMA overrides the region default + period: 2025 + absolute_error_margin: 0.01 + input: + age: 40 + region: SCOTLAND + brma: LOTHIAN + tenure_type: RENT_PRIVATELY + rent: 12_000 + output: + brma: LOTHIAN + BRMA_LHA_rate: 8_375.12 + +- name: An input BRMA outside the household's region is still used + period: 2025 + absolute_error_margin: 0.01 + input: + age: 40 + region: LONDON + brma: MAIDSTONE + tenure_type: RENT_PRIVATELY + rent: 12_000 + output: + brma: MAIDSTONE + BRMA_LHA_rate: 9_467.64 + +- name: Unknown region keeps Maidstone + period: 2025 + absolute_error_margin: 0.01 + input: + age: 40 + region: UNKNOWN + tenure_type: RENT_PRIVATELY + rent: 12_000 + output: + brma: MAIDSTONE + BRMA_LHA_rate: 9_467.64 + +- name: Every region without a BRMA gets its default + period: 2025 + input: + people: + p1: {age: 40} + p2: {age: 40} + p3: {age: 40} + p4: {age: 40} + p5: {age: 40} + p6: {age: 40} + p7: {age: 40} + p8: {age: 40} + p9: {age: 40} + p10: {age: 40} + p11: {age: 40} + p12: {age: 40} + p13: {age: 40} + households: + h1: {members: [p1], region: NORTH_EAST} + h2: {members: [p2], region: NORTH_WEST} + h3: {members: [p3], region: YORKSHIRE} + h4: {members: [p4], region: EAST_MIDLANDS} + h5: {members: [p5], region: WEST_MIDLANDS} + h6: {members: [p6], region: EAST_OF_ENGLAND} + h7: {members: [p7], region: LONDON} + h8: {members: [p8], region: SOUTH_EAST} + h9: {members: [p9], region: SOUTH_WEST} + h10: {members: [p10], region: WALES} + h11: {members: [p11], region: SCOTLAND} + h12: {members: [p12], region: NORTHERN_IRELAND} + h13: {members: [p13], region: UNKNOWN} + output: + brma: + - TYNESIDE + - CENTRAL_GREATER_MANCHESTER + - SHEFFIELD + - LEICESTER + - BIRMINGHAM + - PETERBOROUGH + - INNER_SOUTH_EAST_LONDON + - SOUTHAMPTON + - BRISTOL + - CARDIFF + - ABERDEEN_AND_SHIRE + - BELFAST + - MAIDSTONE diff --git a/policyengine_uk/tests/policy/integration/entitledto_scenarios.yaml b/policyengine_uk/tests/policy/integration/entitledto_scenarios.yaml index d6c407cf8d..292c9599c9 100644 --- a/policyengine_uk/tests/policy/integration/entitledto_scenarios.yaml +++ b/policyengine_uk/tests/policy/integration/entitledto_scenarios.yaml @@ -27,13 +27,19 @@ region: LONDON country: ENGLAND tenure_type: RENT_PRIVATELY + # No BRMA is input, so the household gets London's default BRMA (Inner + # South East London) rather than Maidstone, which every household without + # a BRMA used before brma had a formula. The shared accommodation rate + # rises from £5,623.28 to £7,523.36 a year, so the Universal Credit + # housing element and net income rise by £1,900.08 (from £3,824.96 and + # £14,450.42). output: income_tax: 0 national_insurance: 0 - universal_credit: 3_824.96 + universal_credit: 5_725.04 housing_benefit: 0 child_benefit: 0 - household_net_income: 14_450.42 + household_net_income: 16_350.49 - name: "EntitledTo #2 - couple with 2 children, one earner at £22k, social housing in North West" period: 2025 diff --git a/policyengine_uk/tests/test_brma_region_default.py b/policyengine_uk/tests/test_brma_region_default.py new file mode 100644 index 0000000000..c920637ca6 --- /dev/null +++ b/policyengine_uk/tests/test_brma_region_default.py @@ -0,0 +1,178 @@ +"""Tests for the BRMA a household gets when none is input. + +brma (variables/household/BRMA.py) places a household with no BRMA input in +its region's entry in REGION_DEFAULT_BRMA, unless an earlier year's BRMA +carries forward. These properties hold for any households, regions by year, +BRMA inputs and order in which years are calculated: + +1. The table is the list of rents' most common BRMA in each region, every + BRMA in it lies in its own region, and it covers every region but UNKNOWN. +2. Inputs win: in a year with a BRMA input, brma is that input. +3. Defaults: in a year with no BRMA input in it or before it, brma is the + default for that year's region. +4. Membership: brma is either the default for that year's region or an + input from that year or earlier made while the household was in the + same region. +5. Carry forward: if the household's region has not changed since the + latest input at or before a year, brma is that input. Where each region + occupies one unbroken run of years, 2, 3 and 5 fix brma exactly. + +Regions are input for every year so that region itself does not depend on +the order of calculation. +""" + +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from policyengine_uk import Simulation +from policyengine_uk.parameters.gov.dwp.LHA import lha_list_of_rents +from policyengine_uk.variables.household.BRMA import REGION_DEFAULT_BRMA +from policyengine_uk.variables.household.demographic.geography import Region +from policyengine_uk.variables.household.demographic.locations import BRMAName + +YEARS = list(range(2024, 2031)) +REGIONS = [region.name for region in Region] +BRMAS = [brma.name for brma in BRMAName] +PROPERTY_SETTINGS = settings( + max_examples=40, + deadline=None, + derandomize=True, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large], +) + + +def default_brma(region: str) -> str: + defaults = {key.name: value.name for key, value in REGION_DEFAULT_BRMA.items()} + return defaults.get(region, BRMAName.MAIDSTONE.name) + + +def simulate(regions_by_household, brma_inputs, years_to_calculate): + """Return {year: [brma per household]} calculated in the given order.""" + people, benunits, households = {}, {}, {} + for index, regions in enumerate(regions_by_household): + person, benunit, household = f"p{index}", f"b{index}", f"h{index}" + people[person] = {"age": {YEARS[0]: 40}} + benunits[benunit] = {"members": [person]} + households[household] = { + "members": [person], + "region": dict(zip(YEARS, regions)), + } + inputs = {year: values[index] for year, values in brma_inputs.items()} + if inputs: + households[household]["brma"] = inputs + simulation = Simulation( + situation={"people": people, "benunits": benunits, "households": households} + ) + return { + year: [str(value) for value in simulation.calculate("brma", year)] + for year in years_to_calculate + } + + +def brma_for(regions, brma, year): + situation = { + "people": {"adult": {"age": {2025: 40}}}, + "benunits": {"benunit": {"members": ["adult"]}}, + "households": { + "household": {"members": ["adult"], "region": regions, "brma": brma} + }, + } + return str(Simulation(situation=situation).calculate("brma", year)[0]) + + +def test_region_defaults_are_the_most_common_brma_in_the_list_of_rents(): + counts = lha_list_of_rents.groupby(["region", "brma"]).size() + for region, brma in REGION_DEFAULT_BRMA.items(): + region_counts = counts.loc[region.name].sort_values(ascending=False) + assert region_counts.index[0] == brma.name, region + # A strict maximum, so the table does not depend on tie-breaking. + assert region_counts.iloc[0] > region_counts.iloc[1], region + + +def test_region_defaults_cover_every_region_but_unknown(): + assert set(REGION_DEFAULT_BRMA) == set(Region) - {Region.UNKNOWN} + assert set(lha_list_of_rents.region.unique()) == { + region.name for region in REGION_DEFAULT_BRMA + } + + +def test_no_brma_spans_two_regions(): + regions_per_brma = lha_list_of_rents.groupby("brma").region.nunique() + assert regions_per_brma.max() == 1 + region_of = lha_list_of_rents.groupby("brma").region.first() + for region, brma in REGION_DEFAULT_BRMA.items(): + assert region_of[brma.name] == region.name + + +def test_input_carries_forward_to_later_years(): + # Region is input for 2025 only and carries forward like any input. + regions = {2025: "SCOTLAND"} + assert brma_for(regions, {2025: "LOTHIAN"}, 2026) == "LOTHIAN" + assert brma_for(regions, {2025: "LOTHIAN"}, 2040) == "LOTHIAN" + + +def test_input_does_not_apply_to_earlier_years(): + regions = {2024: "SCOTLAND", 2025: "SCOTLAND"} + assert brma_for(regions, {2025: "LOTHIAN"}, 2024) == "ABERDEEN_AND_SHIRE" + + +def test_household_that_changes_region_gets_the_new_regions_default(): + regions = {2025: "SCOTLAND", 2026: "WALES"} + assert brma_for(regions, {2025: "LOTHIAN"}, 2026) == "CARDIFF" + + +@st.composite +def scenarios(draw): + household_count = draw(st.integers(1, 4)) + regions_by_household = [] + for _ in range(household_count): + # Runs of years in one region; a region may recur after a move. + regions = [] + while len(regions) < len(YEARS): + region = draw(st.sampled_from(REGIONS)) + regions += [region] * draw(st.integers(1, len(YEARS))) + regions_by_household.append(regions[: len(YEARS)]) + input_years = draw(st.sets(st.sampled_from(YEARS), max_size=3)) + brma_inputs = { + year: [draw(st.sampled_from(BRMAS)) for _ in range(household_count)] + for year in sorted(input_years) + } + order = draw(st.permutations(YEARS)) + return regions_by_household, brma_inputs, order + + +@PROPERTY_SETTINGS +@given(scenarios()) +def test_brma_properties(scenario): + regions_by_household, brma_inputs, order = scenario + result = simulate(regions_by_household, brma_inputs, order) + for index, regions in enumerate(regions_by_household): + region_in = dict(zip(YEARS, regions)) + inputs = {year: values[index] for year, values in brma_inputs.items()} + unbroken_runs = all( + regions.index(region) + regions.count(region) - 1 + == len(regions) - 1 - regions[::-1].index(region) + for region in set(regions) + ) + for year in YEARS: + brma = result[year][index] + default = default_brma(region_in[year]) + earlier_inputs = [past for past in inputs if past <= year] + if year in inputs: + assert brma == inputs[year] + if not earlier_inputs: + assert brma == default + assert brma == default or any( + brma == inputs[past] and region_in[past] == region_in[year] + for past in earlier_inputs + ) + if earlier_inputs: + latest = max(earlier_inputs) + unchanged = all( + region_in[between] == region_in[year] + for between in range(latest, year + 1) + ) + if unchanged: + assert brma == inputs[latest] + elif unbroken_runs: + assert brma == default diff --git a/policyengine_uk/variables/household/BRMA.py b/policyengine_uk/variables/household/BRMA.py index ec0b8b422b..ed025d5007 100644 --- a/policyengine_uk/variables/household/BRMA.py +++ b/policyengine_uk/variables/household/BRMA.py @@ -1,8 +1,28 @@ from policyengine_uk.model_api import * from policyengine_uk.variables.household.demographic.locations import BRMAName from policyengine_uk.variables.household.demographic.geography import Region -import pandas as pd -import numpy as np + +# The BRMA a household is placed in when none is input: the BRMA with the +# most entries for the region in the list of rents +# (parameters/gov/dwp/LHA/lha_list_of_rents.csv.gz, all years together). +# policyengine-uk-data imputes survey households' BRMAs by sampling those +# entries within region and LHA category, so microsimulation datasets set +# brma for every household and do not use this table. +# test_brma_region_default.py checks it against the CSV. +REGION_DEFAULT_BRMA = { + Region.NORTH_EAST: BRMAName.TYNESIDE, + Region.NORTH_WEST: BRMAName.CENTRAL_GREATER_MANCHESTER, + Region.YORKSHIRE: BRMAName.SHEFFIELD, + Region.EAST_MIDLANDS: BRMAName.LEICESTER, + Region.WEST_MIDLANDS: BRMAName.BIRMINGHAM, + Region.EAST_OF_ENGLAND: BRMAName.PETERBOROUGH, + Region.LONDON: BRMAName.INNER_SOUTH_EAST_LONDON, + Region.SOUTH_EAST: BRMAName.SOUTHAMPTON, + Region.SOUTH_WEST: BRMAName.BRISTOL, + Region.WALES: BRMAName.CARDIFF, + Region.SCOTLAND: BRMAName.ABERDEEN_AND_SHIRE, + Region.NORTHERN_IRELAND: BRMAName.BELFAST, +} class brma(Variable): @@ -11,4 +31,34 @@ class brma(Variable): default_value = BRMAName.MAIDSTONE entity = Household label = "Broad Rental Market Area" + documentation = ( + "The Broad Rental Market Area whose Local Housing Allowance rates " + "apply to the household. If it is not provided, the latest BRMA from " + "an earlier year applies when the household's region is the same as " + "in that year; otherwise the household is placed in its region's BRMA " + "with the most entries in the list of rents (Maidstone if the region " + "is unknown)." + ) definition_period = YEAR + + def formula(household, period, parameters): + region = household("region", period) + region_default = select( + [region == region_value for region_value in REGION_DEFAULT_BRMA], + list(REGION_DEFAULT_BRMA.values()), + default=BRMAName.MAIDSTONE, + ) + # An input keeps applying in later years, as policyengine-core's + # auto_carry_over_input_variables does for input-only variables. A + # household whose region differs from that year's gets its new + # region's default: no BRMA spans two regions. + earlier_periods = [ + known_period + for known_period in household.get_holder("brma").get_known_periods() + if known_period.start < period.start + ] + if not earlier_periods: + return region_default + latest = max(earlier_periods, key=lambda known_period: known_period.start) + same_region = household("region", latest) == region + return where(same_region, household("brma", latest).decode(), region_default) From 5a64494ec1ae9972ccabdc695977b05bf9e4982e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 2 Oct 2026 02:39:17 -0400 Subject: [PATCH 2/6] Take the default BRMA from census private renters The list of rents' Scottish, Welsh and Northern Ireland lists are copies of English lists (policyengine-uk-data#515), so their entry counts say nothing about where people rent. Each region's default is now its BRMA with the most private-rented households in the 2021 (England, Wales, Northern Ireland) and 2022 (Scotland) censuses, mapped to BRMAs. Scotland moves to Lothian, Yorkshire to Leeds and the East of England to Central Norfolk & Norwich. The new YAML tests no longer pin LHA rates, which #2022 changes; a Python test checks instead that each default gives the same rates as inputting it. Co-Authored-By: Claude Opus 5.5 --- changelog.d/2020.fixed.md | 2 +- .../tests/policy/baseline/household/brma.yaml | 40 +++-------- .../tests/test_brma_region_default.py | 69 +++++++++++++------ policyengine_uk/variables/household/BRMA.py | 23 +++---- 4 files changed, 71 insertions(+), 63 deletions(-) diff --git a/changelog.d/2020.fixed.md b/changelog.d/2020.fixed.md index ced7896fbd..64f3c590da 100644 --- a/changelog.d/2020.fixed.md +++ b/changelog.d/2020.fixed.md @@ -1 +1 @@ -- Place a household with no BRMA input in a BRMA in its region (the region's BRMA with the most entries in the LHA list of rents, Maidstone if the region is unknown) instead of Maidstone in every region. A household that sets no region is in London, the region default, so it now gets Inner South East London's LHA rates. A BRMA input still carries forward to later years while the household stays in the same region. +- Place a household with no BRMA input in its region's BRMA with the most private-rented households, from the 2021 and 2022 censuses (Maidstone if the region is unknown), instead of Maidstone in every region. A household that sets no region is in London, the region default, so it now gets Inner South East London's LHA rates. A BRMA input still carries forward to later years while the household stays in the same region. diff --git a/policyengine_uk/tests/policy/baseline/household/brma.yaml b/policyengine_uk/tests/policy/baseline/household/brma.yaml index e54b4cb553..f38bfa17eb 100644 --- a/policyengine_uk/tests/policy/baseline/household/brma.yaml +++ b/policyengine_uk/tests/policy/baseline/household/brma.yaml @@ -1,13 +1,10 @@ # A household with no BRMA input is placed in its region's BRMA with the most -# entries in the LHA list of rents (REGION_DEFAULT_BRMA in -# variables/household/BRMA.py). Each BRMA_LHA_rate below is the model's rate -# for the same household with that BRMA input explicitly, so the default and -# the explicit input must agree. A single 40-year-old private renter is in LHA -# category B (one bedroom). +# private-rented households (REGION_DEFAULT_BRMA in +# variables/household/BRMA.py). test_brma_region_default.py checks that the +# default gives the same LHA rates as inputting that BRMA. - name: London household without a BRMA gets Inner South East London period: 2025 - absolute_error_margin: 0.01 input: age: 40 region: LONDON @@ -15,25 +12,19 @@ rent: 12_000 output: brma: INNER_SOUTH_EAST_LONDON - LHA_category: B - BRMA_LHA_rate: 16_750.24 -- name: Scottish household without a BRMA gets Aberdeen and Shire +- name: Scottish household without a BRMA gets Lothian period: 2025 - absolute_error_margin: 0.01 input: age: 40 region: SCOTLAND tenure_type: RENT_PRIVATELY rent: 12_000 output: - brma: ABERDEEN_AND_SHIRE - LHA_category: B - BRMA_LHA_rate: 6_408.48 + brma: LOTHIAN - name: Welsh household without a BRMA gets Cardiff period: 2025 - absolute_error_margin: 0.01 input: age: 40 region: WALES @@ -41,12 +32,9 @@ rent: 12_000 output: brma: CARDIFF - LHA_category: B - BRMA_LHA_rate: 6_918.60 - name: Northern Ireland household without a BRMA gets Belfast period: 2025 - absolute_error_margin: 0.01 input: age: 40 region: NORTHERN_IRELAND @@ -54,25 +42,20 @@ rent: 12_000 output: brma: BELFAST - LHA_category: B - BRMA_LHA_rate: 5_702.32 - name: An input BRMA overrides the region default period: 2025 - absolute_error_margin: 0.01 input: age: 40 region: SCOTLAND - brma: LOTHIAN + brma: GREATER_GLASGOW tenure_type: RENT_PRIVATELY rent: 12_000 output: - brma: LOTHIAN - BRMA_LHA_rate: 8_375.12 + brma: GREATER_GLASGOW - name: An input BRMA outside the household's region is still used period: 2025 - absolute_error_margin: 0.01 input: age: 40 region: LONDON @@ -81,11 +64,9 @@ rent: 12_000 output: brma: MAIDSTONE - BRMA_LHA_rate: 9_467.64 - name: Unknown region keeps Maidstone period: 2025 - absolute_error_margin: 0.01 input: age: 40 region: UNKNOWN @@ -93,7 +74,6 @@ rent: 12_000 output: brma: MAIDSTONE - BRMA_LHA_rate: 9_467.64 - name: Every region without a BRMA gets its default period: 2025 @@ -130,14 +110,14 @@ brma: - TYNESIDE - CENTRAL_GREATER_MANCHESTER - - SHEFFIELD + - LEEDS - LEICESTER - BIRMINGHAM - - PETERBOROUGH + - CENTRAL_NORFOLK_NORWICH - INNER_SOUTH_EAST_LONDON - SOUTHAMPTON - BRISTOL - CARDIFF - - ABERDEEN_AND_SHIRE + - LOTHIAN - BELFAST - MAIDSTONE diff --git a/policyengine_uk/tests/test_brma_region_default.py b/policyengine_uk/tests/test_brma_region_default.py index c920637ca6..1c073b624c 100644 --- a/policyengine_uk/tests/test_brma_region_default.py +++ b/policyengine_uk/tests/test_brma_region_default.py @@ -5,8 +5,9 @@ carries forward. These properties hold for any households, regions by year, BRMA inputs and order in which years are calculated: -1. The table is the list of rents' most common BRMA in each region, every - BRMA in it lies in its own region, and it covers every region but UNKNOWN. +1. The table is each region's BRMA with the most private-rented households in + brma_private_rented_households.csv, a strict maximum, and it covers every + region but UNKNOWN. A default gives the same LHA rates as inputting it. 2. Inputs win: in a year with a BRMA input, brma is that input. 3. Defaults: in a year with no BRMA input in it or before it, brma is the default for that year's region. @@ -21,15 +22,23 @@ the order of calculation. """ +from pathlib import Path + +import pandas as pd +import pytest from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st +import policyengine_uk from policyengine_uk import Simulation -from policyengine_uk.parameters.gov.dwp.LHA import lha_list_of_rents from policyengine_uk.variables.household.BRMA import REGION_DEFAULT_BRMA from policyengine_uk.variables.household.demographic.geography import Region from policyengine_uk.variables.household.demographic.locations import BRMAName +PRIVATE_RENTED_HOUSEHOLDS = pd.read_csv( + Path(policyengine_uk.__file__).parent + / "parameters/gov/dwp/LHA/brma_private_rented_households.csv" +) YEARS = list(range(2024, 2031)) REGIONS = [region.name for region in Region] BRMAS = [brma.name for brma in BRMAName] @@ -80,45 +89,65 @@ def brma_for(regions, brma, year): return str(Simulation(situation=situation).calculate("brma", year)[0]) -def test_region_defaults_are_the_most_common_brma_in_the_list_of_rents(): - counts = lha_list_of_rents.groupby(["region", "brma"]).size() +def test_region_defaults_have_the_most_private_rented_households(): + households = PRIVATE_RENTED_HOUSEHOLDS.groupby(["region", "brma"]).households.sum() for region, brma in REGION_DEFAULT_BRMA.items(): - region_counts = counts.loc[region.name].sort_values(ascending=False) - assert region_counts.index[0] == brma.name, region + region_households = households.loc[region.name].sort_values(ascending=False) + assert region_households.index[0] == brma.name, region # A strict maximum, so the table does not depend on tie-breaking. - assert region_counts.iloc[0] > region_counts.iloc[1], region + assert region_households.iloc[0] > region_households.iloc[1], region def test_region_defaults_cover_every_region_but_unknown(): assert set(REGION_DEFAULT_BRMA) == set(Region) - {Region.UNKNOWN} - assert set(lha_list_of_rents.region.unique()) == { + assert set(PRIVATE_RENTED_HOUSEHOLDS.region) == { region.name for region in REGION_DEFAULT_BRMA } + assert set(PRIVATE_RENTED_HOUSEHOLDS.brma) == set(BRMAS) + + +@pytest.mark.parametrize("region", REGIONS) +def test_default_gives_the_same_lha_rates_as_inputting_it(region): + def calculate(household_inputs): + situation = { + "people": {"adult": {"age": {2025: 40}}}, + "benunits": {"benunit": {"members": ["adult"]}}, + "households": { + "household": { + "members": ["adult"], + "region": {2025: region}, + "tenure_type": {2025: "RENT_PRIVATELY"}, + "rent": {2025: 12_000}, + **household_inputs, + } + }, + } + simulation = Simulation(situation=situation) + return [ + float(simulation.calculate(variable, 2025)[0]) + for variable in ("BRMA_LHA_rate", "uc_LHA_cap") + ] - -def test_no_brma_spans_two_regions(): - regions_per_brma = lha_list_of_rents.groupby("brma").region.nunique() - assert regions_per_brma.max() == 1 - region_of = lha_list_of_rents.groupby("brma").region.first() - for region, brma in REGION_DEFAULT_BRMA.items(): - assert region_of[brma.name] == region.name + defaulted = calculate({}) + assert defaulted[0] > 0 + assert defaulted == calculate({"brma": {2025: default_brma(region)}}) def test_input_carries_forward_to_later_years(): # Region is input for 2025 only and carries forward like any input. regions = {2025: "SCOTLAND"} - assert brma_for(regions, {2025: "LOTHIAN"}, 2026) == "LOTHIAN" - assert brma_for(regions, {2025: "LOTHIAN"}, 2040) == "LOTHIAN" + assert brma_for(regions, {2025: "GREATER_GLASGOW"}, 2026) == "GREATER_GLASGOW" + assert brma_for(regions, {2025: "GREATER_GLASGOW"}, 2040) == "GREATER_GLASGOW" def test_input_does_not_apply_to_earlier_years(): regions = {2024: "SCOTLAND", 2025: "SCOTLAND"} - assert brma_for(regions, {2025: "LOTHIAN"}, 2024) == "ABERDEEN_AND_SHIRE" + assert brma_for(regions, {2025: "GREATER_GLASGOW"}, 2024) == "LOTHIAN" def test_household_that_changes_region_gets_the_new_regions_default(): regions = {2025: "SCOTLAND", 2026: "WALES"} - assert brma_for(regions, {2025: "LOTHIAN"}, 2026) == "CARDIFF" + assert brma_for(regions, {2025: "GREATER_GLASGOW"}, 2026) == "CARDIFF" @st.composite diff --git a/policyengine_uk/variables/household/BRMA.py b/policyengine_uk/variables/household/BRMA.py index ed025d5007..7cec94d778 100644 --- a/policyengine_uk/variables/household/BRMA.py +++ b/policyengine_uk/variables/household/BRMA.py @@ -3,24 +3,23 @@ from policyengine_uk.variables.household.demographic.geography import Region # The BRMA a household is placed in when none is input: the BRMA with the -# most entries for the region in the list of rents -# (parameters/gov/dwp/LHA/lha_list_of_rents.csv.gz, all years together). -# policyengine-uk-data imputes survey households' BRMAs by sampling those -# entries within region and LHA category, so microsimulation datasets set -# brma for every household and do not use this table. -# test_brma_region_default.py checks it against the CSV. +# most private-rented households in the region, from the 2021 (England, Wales, +# Northern Ireland) and 2022 (Scotland) censuses mapped to BRMAs +# (parameters/gov/dwp/LHA/brma_private_rented_households.csv). +# Microsimulation datasets set brma for every household and do not use this +# table. test_brma_region_default.py checks it against the CSV. REGION_DEFAULT_BRMA = { Region.NORTH_EAST: BRMAName.TYNESIDE, Region.NORTH_WEST: BRMAName.CENTRAL_GREATER_MANCHESTER, - Region.YORKSHIRE: BRMAName.SHEFFIELD, + Region.YORKSHIRE: BRMAName.LEEDS, Region.EAST_MIDLANDS: BRMAName.LEICESTER, Region.WEST_MIDLANDS: BRMAName.BIRMINGHAM, - Region.EAST_OF_ENGLAND: BRMAName.PETERBOROUGH, + Region.EAST_OF_ENGLAND: BRMAName.CENTRAL_NORFOLK_NORWICH, Region.LONDON: BRMAName.INNER_SOUTH_EAST_LONDON, Region.SOUTH_EAST: BRMAName.SOUTHAMPTON, Region.SOUTH_WEST: BRMAName.BRISTOL, Region.WALES: BRMAName.CARDIFF, - Region.SCOTLAND: BRMAName.ABERDEEN_AND_SHIRE, + Region.SCOTLAND: BRMAName.LOTHIAN, Region.NORTHERN_IRELAND: BRMAName.BELFAST, } @@ -36,8 +35,8 @@ class brma(Variable): "apply to the household. If it is not provided, the latest BRMA from " "an earlier year applies when the household's region is the same as " "in that year; otherwise the household is placed in its region's BRMA " - "with the most entries in the list of rents (Maidstone if the region " - "is unknown)." + "with the most private-rented households (Maidstone if the region is " + "unknown)." ) definition_period = YEAR @@ -51,7 +50,7 @@ def formula(household, period, parameters): # An input keeps applying in later years, as policyengine-core's # auto_carry_over_input_variables does for input-only variables. A # household whose region differs from that year's gets its new - # region's default: no BRMA spans two regions. + # region's default instead. earlier_periods = [ known_period for known_period in household.get_holder("brma").get_known_periods() From 15eeb50760fc81ed21424682b286bddb55067863 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 2 Oct 2026 02:39:47 -0400 Subject: [PATCH 3/6] Ship the census BRMA table gzipped The repo ignores *.csv, so the previous commit left the table out. Store it as .csv.gz beside lha_list_of_rents.csv.gz (gzip -n, so the archive is reproducible); the uncompressed content is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../LHA/brma_private_rented_households.csv.gz | Bin 0 -> 6494 bytes .../tests/test_brma_region_default.py | 4 ++-- policyengine_uk/variables/household/BRMA.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 policyengine_uk/parameters/gov/dwp/LHA/brma_private_rented_households.csv.gz diff --git a/policyengine_uk/parameters/gov/dwp/LHA/brma_private_rented_households.csv.gz b/policyengine_uk/parameters/gov/dwp/LHA/brma_private_rented_households.csv.gz new file mode 100644 index 0000000000000000000000000000000000000000..f4537ee35fa0d63b55e98c8bc19660f9e83f3366 GIT binary patch literal 6494 zcmV-k8KLGMiwFP!000021C?D%Z}T{o-Pd0kFZg~el~_rPD0UzxpDQb)8(o+lpnD4F z#{B=e=Sr4EiKN_{s(d;}5=CC#7cc35{qf)b^XLEW{`#+f{$=;qKmPWwKmYvazjuHC z^Z)++$KU__$KU?_Ticw!t{;be*Btlf-Sb=9U)rv_{%X3e{n^Q#wr+!?cEN`Yj@|{S zHaOHO99FNe0G(Ie{*xnXK& zVhPhbTYM(HQRxM^Z~G^_%?jbNklIP3t<7S3Cq)n;i{WKKtnNDi6fjL*L@7y?=vYRJ~Io#)qhAX!lT^;FSci`@%ZZl-h?-zaw~KhlN1D}Z4t-I zm0V7M?r?lQb@*oGg(`qFH-wb3fa+Zg-WEWpK5um%WtqIbw8t|mKw&p&&_nA!SkY+o zrP!vip#3)A9P={v2C0%+(1Bs=Ladky&+K%AkgXWpj_G9<$_=Xw(Q}l&ugj zLVBgz=L{MpDPx7eMrzg0fpPpgU!S4Tn+}FktaSRCrgqWTZ5s5slYWy2-R@UPf_c~D z>FbLo8GQ)EIB2`YLT^PUw`kNh9lYM6p=rvB6>ZiiDU9=F0_l2hg((wAf>e-|S3rQw zs;QrC^M1bWn~vpJDp4a)BVx1_0yV+NbXpP#?U=AHUaxB$9>!VQsS zm|nip12Z`1h@9Wrmz$HRR-W1&RXgv?sNOm4izpQ7%0D`vzQ5k!g1q)^L*sM(XpW;h z%1T842Yk{dD-%=WlL1(NOofjRSP8{f^U zZw@2KajoFw$5>y%q3dcm8lYGBQ@e+k3rlHlZfOQ4Ay=8!-(VUQD@_q*O@C9vbDY{m zj`p?Pe|)3QBs8Vpq^X_rDs0j;O-!pdX|UfS({sNA8P5LY)V<^Xm%}qmLQuj#Ooz7>g?gAz4JNy29_B-P`;2^%qUb)( zHxcu>ZoD(pF2sx&%@`zTO1T;XvL<8ZHK*>bL`hmzje!BB`B0;fU#N2Zb7w%Xlv`8Y zn!Y<-NCe3t8aqb2D7JYd_bRAu9tBe-Jn&wq$>`P->5c3n`A2(ppk=&y9(e|-rI_i5 z83XFks2YQ7&!;Bm=2DT*hF7n@vHKpP zG4F&dfaIxYrwgFo1utCzM6cP5=3K?2K5lTJB<&W7q)7(5LE`XzEpfT1P{{{q6u-L% zXb`anXl(NGyemfb+MSNW?04}2+RXQ^@;GLFfCnkc=uMI^ay|z-V^b?b#&apLF~;HX zhhwy@eV7h2sOnFjTCFqYI71|@pAqg1k@MJ(5IK*x)&h%&((&xO330FKRW0^oR!_{5@C1Nt2!h~UnDF*sFvlkQ1Hjh+oG+~=Z z5tqnJ#ubknm@#4lk*804KDWQaSO#aed7R8C_34~Q$~UK(S3GW^iZ$Y`3&|d5Y~_lm zkWjY|Jdv8{q}k>n&Q~(?z_0dKdu(3YaiUB^EtP(dOp)T0VcjIdLv!juG8%#d5mJ`1fFeUCNZsX8UKCQLd!fr5#qHxi562@Zl2q}kh{&l(ptlE_c4Y$6GJ-my zNI+`LlJF&jr5^QKs|tfWRIO}{fndjZTVWu04$DuS&C#hbMwqC#Xt3r8O*d#*pAz0| z(I6i(h-~9hYc8$UdW%MbrZk1u8#LDI%52dP)<*K26I$9TrQsGA2?|=p#WI0o>*d%i z+7QU--v~KB2UAFv3DgGbg(wrK4HS;6G64bdooT-yF0rN~qS0+G~z0tq1U?FjS=-V+L z+Pxqb{Fi_HgRC~inrli2fB1TAn^*MqrR|w4ts?_gC4YZ@j%1}t1lHtn{q4XUil$?U z!k;qoBneCA#WT^o5hE6d3dHGQF8UH@{?_Pa`0cJEcKfF94#(Fahae#K zpUXy30D0P*^w#fwTp5)j2aud-?^Fgs%!qu|8sWe@Cf~+?et$f*J$DAoK-!K4LG6OE zH3D)rX%8(3_*R=?T%W&@jBXHYl|hB7r6P*;qcRHjI~jrV_i;acIImCU!6Cz^3#i7* zN(tm(*CY@AYAIkj>{||UgUmu$&GhL&Oc<<)^dgI4gq==$=nEO4;dMdVn1$e@datq= zmq7@d#h_}0xIGhdX!q3g{poT&9Ir3mJ<$-ykjT_2Fr9vbg9Rq3>Kv|<@(Jg}c`+=6 zjmum{gersWff6;!FvCKP!WDETq1@t|?7FI=5KBGCCjUJIs~==z3M94mz z1p0XhOHBpGWTAkkOkjsus4{^Q;>yhltn1Onu^f*qk}BieLB5P@B)>u#=K`ZBR)kvz zD4+KJ41`)n5o8{P6A@hj-*7PwsfSt=603D`H24G)F z&H;?Yt?pEz$}EdvVcxksrcpX)^B9|hn1G{=y2~~N$)ZBww5QS)21%JpRnLOJ8Ezpn zjL8Jc#V0JS@XXW6;3y!KmFIHgi0ilaESPDI^I}$0`MF2}wISjGBu>NJl%x zDVR{{Y)WyI(3r~ec?Q_b3nn!d(vM6~RAyUMgfaB73?tIldIf*B$K&DH4u#R72&-KT z&KF@4fZ)q_bGVq!(#zqs;Utw6etjK2nO*cW15i0S$Th%vad}5SEEyZ=T?XJ{=8(IE z6aA?@84-3oNKc)gPhUg97=?-ZwjDg_n-m2UhSn&Gb4IbsL7$T4tx*OyNY6O^>rdB> zUi}x3Dg}aNT*1!}WgMl+LY8qZ1PwF%r{|WP*)6a(htMR_`Fm&vJ1KJrQn6yzUHk8k zm;TTW1%t0m*PSj6?<}%PY_Xt5z1?DIlqqjMi`!-9EDEGs<}}CG?=x@25xzoz5E|<$ z3@m;bSs`$O!pz#rK047T-lQ0B+7T3EodTDfqLeB}qn5BvL38{}CSMLOgO@lT8oo$M z#m*(1FOt}!I4QlzYkKpxz3|MO_SlgR?+PHfeo=-3h*8$Q0CJ9f){>h`^V&7%*VAQ4 z{QFQQAhz(P@&vLM#;7vRzJr_*-VU#C-7xa|>)||<&cbyHW`l%0n2Q@E>}Ha7gEVA_ zW&+urzTOVauq1=^w-o@xCFY-T19(x)WbRBNIv%#J7@<7(9PyT`cjF~v*?;E%7v`R)2d&S=}St4YefLLi}`$g@J=0`A%q z&}Z}Zj%zZB43S4AYYc_;^jKqP%sy3PpdIrsVb^G-p5t8Ha}-o}C|l={dY*S`j=~AE zu;9$5Fv6WYNF!oaPrpxPCFf300*-0U`!dL`IlqmyywajIYXlo0waOvLSA^_6oCg!> zi{={!SWV_InxY=6B8Tw+>paF0CI;mWJ~d}lsD7Q^S-uozR~|)uGUPLlk}lTedxyMC z(mhXI^Zb5c`!U%?i-PJZhitHrq0T`un6z~cFV%#k>z_`@a^EU%Z8Rkij)%1+6qVz} zTRHVA60|3ybxUo(oqi3WO^6#L%r2)?ousko9_k!uKAC>&b8|%T-XMfp=}-V65Xu)o zILAU3z+p>zB1@?8|7wr7je#W$HAkX=(VX_afU)Gq1-Ll*Kw;&r!k}4aWtBm54`qrK z#;_$U?e4e3J^e*PQ3C1p*g=&CS;@!(IJ|Vxu0w~7@Vwq2oXBG^x2wjE7&a3dm&aJl zWg?x=eRH7>xb`i@zZ~ktphb;=H9EDB%mszRYgeT}moG+sPjcT=NZNi5Q5gBOs*EdW z6H&&wA7mM4$55CKT~*cnLgv}wo9K;E!ml98cUq(vp@1=@G04fkanXPQ`D#YVY%tV_;ivjH=IOcWf|9yF;&KgQzDWvJGPh6 zFiY=Z4uRWivIqjAkVTLeN^Re)R_MJwezBDhvBn@}weZZA49uEz_AeQ1s8SnVuF!Lj z&=8hA!o!m3BRqE5OT>E(n%91Vgm^*N4H5!v?>0yrA(`U)7S*_^dr>ahGieM_F56qU z?3KzrWfC25`hC>@mt4Vnj znPs1L$qa>I3u%nkx6PiyA9iL{I)*ohi;&Vue1q_XGL12A5SKuvKKZ%n4(CZNs63>g zX!8)Vq%IG!F-vd5pVI*>U53>ivQBw#|W@nmktGe<1D7pX~;GMak@%6`DY+> zOPZ{hfkWCpeJ-rYFz13va}g6nCfTL)YJzZs)QT`i|IR0He<1Ix8*0rbeV9Plb|k&X zLA+DKWFI(`zD*JAP?$E4`mM390KyxeT-qgwLqkj`fShVjNrT$o+d(i*FgXC{8>SyV z0!UYi+#Q(9Q6wGm+MT|qBh0LJB~WpvxB^NZqL?P*5@f;g3Hnaq_vteHjqSZsmQclo z@~#AHJ`2j1VD?O7^6-!LxIcW}DxB8{gJFpHw?ynHRhh--KHZX zlKNpfmf56#n9dTCcJ9Ex$*9tV`>d`R`!-0>WiH**;F6r{>KaXn)!+{Yn9s)&a_ZyoD{(IphYzqdsgsc}<$^spN{ zZSv#+8o7Auan{I=&Z>iykedz_^;;sh#UaV{qyOgz3CWiaNp#??~Q(&hm&^v z1Mg+eCsp*924?~di-kj=2Wi`2AaB1xL0W5r0zH&aa8AkbdcJ-%g90?E+DUQb)lw~& z;(U@#<@p>Jq{f0f>vHJ6rbYgR(OeLaP!BZ%N*YpcU_lrLB$YF_&JIr!ZMiI1=eJ{btpM)~ZRSHw9_@FM{suXoxUW5q|a(NZ{*@)#;xX2aU)-0~#Xl~~g7da4^ zx{9un#N+6UX+sDCnmjmWvhpB;##vva;IJq+HKpy_{o(a6ZUdBYeI1+E@85I34)GFV zeVyaqso^i(;W$lHtkbP7V?QrVq1APGiPJdGt`9qQ$Lk-cg8AvM&p*e(I>kJKnAh7Z zgxWdbvk*H-xA9pze)le3I(;PnHVdHuVw%S*Z_mryUB{MU)GKmb*RQ5~I`Kvs>5C}7 zPPrnCbcobNn3G%gyDwv{y~OD-=|zPTn9Sm0(to#$ECk~AsiP~`U4HmvHsy2%fEAsF zXby0dL&yNPioZGXdAj`Y$yCZ`uJZxOIs;(;VXzqhonupAFb=nqY;mqb0mV6PMwL+H z!;C4RYz?Mq`q_Rw^{4myRxt`Avk=#R$1H>z6|J+-aEieQ4VLNk@^ftW!$EpBi(sKe zWf9(?l0`Uro^HA{$8*~Z6)<6Ok&JdQ%Zq$csaRa(Y-zYrv6tc6znzZzJ#YCDld4D% z)Z6LoJ#dag_8!)plX3Vu^)F;nVVbpHT;sI1SzLsO8MnANl*Eh|e>KC3;%nD@ocf{m zUFu1rtzm@s!WChRxGTdjnd;V?%S{flfS}_-7Qr;NS%hD7L;CjjvED>Ys`Yje3ElhC z$BXa~C%L^{Gzlv;Jg{WnkeS5*oqYR_;0LIFlD$(LdFG|jSk zrQZdSdNuZ?9|kePKK4`+6>H@~RLt$wm{U)-^ERru8nl#xU*uSgSxSLlwDFW-#bY_A zkS63db2$QbRTptS63)9aflos;p-dQ71DAyKJbhC^$&%J!l=4(y4t*DRa)7@Ck;h6qR6^+nc)OOm85n- ztW^7w#2W>}28k=p$Qb{eZrc)`n*MkmY^22UTn52bJyPr$gb!ynIb=9q&&?rn79d$; z5z9@lMM*YogN1lo$xRw2y7yv>#@U51rzIyP=>mfCbd!hhVw;EfB)>T{`+YF&l95P$ z9SlFPlf_UmLvCvp8|K0Mb@$)3lBi=|1H++;82}NU&fURIaUv*hO?P-{e&@bR`r!sB zY=t!$0GU!LG63g&PV?J(7$)>a1^&uK8NtR`!w}45EhekDGe1Kl z8OQknBGTs(R4C*h#@l}8NZanmpWyO-bzQL#+0}JC-y}jbTxW4~YCd}X;rZ?3gq;}8 z$5Ypi6;&dc*clE&ZKdlRghO$I!%2kX3;GjI!x%%MU`!U`Z67iVY5b0e&q91c=Dc^$ z&&}*OYK_&yJg87UVGAI>)4KvVWJ!F2&VF!6Dj)LLs)v)`8nQvgJjO2KyldvaG8AzZ zLk>b6V5+j`KQJT>3hQ$i>;sC+?%3?;vqYKC&OM`!&H*^Z%H{yJkMmnD&3ilja8eX9 z3-LDbL}A`BEWIXQdxwS)XWmQriz2Of0* E0EP>ZbN~PV literal 0 HcmV?d00001 diff --git a/policyengine_uk/tests/test_brma_region_default.py b/policyengine_uk/tests/test_brma_region_default.py index 1c073b624c..49ad7cd250 100644 --- a/policyengine_uk/tests/test_brma_region_default.py +++ b/policyengine_uk/tests/test_brma_region_default.py @@ -6,7 +6,7 @@ BRMA inputs and order in which years are calculated: 1. The table is each region's BRMA with the most private-rented households in - brma_private_rented_households.csv, a strict maximum, and it covers every + brma_private_rented_households.csv.gz, a strict maximum, and it covers every region but UNKNOWN. A default gives the same LHA rates as inputting it. 2. Inputs win: in a year with a BRMA input, brma is that input. 3. Defaults: in a year with no BRMA input in it or before it, brma is the @@ -37,7 +37,7 @@ PRIVATE_RENTED_HOUSEHOLDS = pd.read_csv( Path(policyengine_uk.__file__).parent - / "parameters/gov/dwp/LHA/brma_private_rented_households.csv" + / "parameters/gov/dwp/LHA/brma_private_rented_households.csv.gz" ) YEARS = list(range(2024, 2031)) REGIONS = [region.name for region in Region] diff --git a/policyengine_uk/variables/household/BRMA.py b/policyengine_uk/variables/household/BRMA.py index 7cec94d778..62f323bbee 100644 --- a/policyengine_uk/variables/household/BRMA.py +++ b/policyengine_uk/variables/household/BRMA.py @@ -5,7 +5,7 @@ # The BRMA a household is placed in when none is input: the BRMA with the # most private-rented households in the region, from the 2021 (England, Wales, # Northern Ireland) and 2022 (Scotland) censuses mapped to BRMAs -# (parameters/gov/dwp/LHA/brma_private_rented_households.csv). +# (parameters/gov/dwp/LHA/brma_private_rented_households.csv.gz). # Microsimulation datasets set brma for every household and do not use this # table. test_brma_region_default.py checks it against the CSV. REGION_DEFAULT_BRMA = { From d1b0ade6dd77871317d94abe31c8f783cf51f77e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 2 Oct 2026 03:07:29 -0400 Subject: [PATCH 4/6] Compare only known regions when carrying a BRMA forward Reading region for an earlier year from inside brma could cache the wrong region for that year: core's carry-over returns region's default (London) for a year once a later year is known. That dropped carried BRMA inputs and changed other results for the earlier year, such as income tax. The formula now never calculates region for an earlier year. It takes the region in effect in the BRMA's year as the last region known at or before it (or region's default if none is), and carries the BRMA forward unless a region known after it, up to this year, differs. A household that leaves a region and returns now gets the region default in any calculation order. Tests: the property test now asserts the exact rule in any order; a second property test inputs region in only some years and checks the rule and that calculating brma leaves every year's region unchanged; the review's repros are regression tests. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_brma_region_default.py | 189 ++++++++++++++---- policyengine_uk/variables/household/BRMA.py | 70 +++++-- 2 files changed, 203 insertions(+), 56 deletions(-) diff --git a/policyengine_uk/tests/test_brma_region_default.py b/policyengine_uk/tests/test_brma_region_default.py index 49ad7cd250..cf5d5fcf7c 100644 --- a/policyengine_uk/tests/test_brma_region_default.py +++ b/policyengine_uk/tests/test_brma_region_default.py @@ -2,24 +2,22 @@ brma (variables/household/BRMA.py) places a household with no BRMA input in its region's entry in REGION_DEFAULT_BRMA, unless an earlier year's BRMA -carries forward. These properties hold for any households, regions by year, -BRMA inputs and order in which years are calculated: +carries forward. 1. The table is each region's BRMA with the most private-rented households in brma_private_rented_households.csv.gz, a strict maximum, and it covers every region but UNKNOWN. A default gives the same LHA rates as inputting it. -2. Inputs win: in a year with a BRMA input, brma is that input. -3. Defaults: in a year with no BRMA input in it or before it, brma is the - default for that year's region. -4. Membership: brma is either the default for that year's region or an - input from that year or earlier made while the household was in the - same region. -5. Carry forward: if the household's region has not changed since the - latest input at or before a year, brma is that input. Where each region - occupies one unbroken run of years, 2, 3 and 5 fix brma exactly. - -Regions are input for every year so that region itself does not depend on -the order of calculation. +2. The rule: in a year with a BRMA input, brma is that input. Otherwise, if + there is an earlier input and the household's region has been the same in + every year from the latest one to this year, brma is that input; if not, + brma is the default for this year's region. + +With region input for every year, the rule holds exactly for any households +(each with or without BRMA inputs in the same years), regions by year and order +in which years are calculated. With region input in only some years, it holds +for years calculated in order, taking each year's region as the latest region +input at or before it (or London, region's default), and calculating brma never +changes any year's region. """ from pathlib import Path @@ -140,6 +138,12 @@ def test_input_carries_forward_to_later_years(): assert brma_for(regions, {2025: "GREATER_GLASGOW"}, 2040) == "GREATER_GLASGOW" +def test_input_carries_forward_when_region_was_input_earlier(): + regions = {2024: "SCOTLAND"} + assert brma_for(regions, {2025: "GREATER_GLASGOW"}, 2026) == "GREATER_GLASGOW" + assert brma_for(regions, {2025: "GREATER_GLASGOW"}, 2040) == "GREATER_GLASGOW" + + def test_input_does_not_apply_to_earlier_years(): regions = {2024: "SCOTLAND", 2025: "SCOTLAND"} assert brma_for(regions, {2025: "GREATER_GLASGOW"}, 2024) == "LOTHIAN" @@ -148,6 +152,70 @@ def test_input_does_not_apply_to_earlier_years(): def test_household_that_changes_region_gets_the_new_regions_default(): regions = {2025: "SCOTLAND", 2026: "WALES"} assert brma_for(regions, {2025: "GREATER_GLASGOW"}, 2026) == "CARDIFF" + # The move counts even when the BRMA's own year has no region known. + regions = {2024: "SCOTLAND", 2026: "WALES"} + assert brma_for(regions, {2025: "GREATER_GLASGOW"}, 2026) == "CARDIFF" + + +def test_household_that_returns_to_a_region_gets_its_default_in_any_order(): + regions = {2026: "SCOTLAND", 2027: "WALES", 2028: "SCOTLAND"} + brma = {2026: "GREATER_GLASGOW"} + assert brma_for(regions, brma, 2028) == "LOTHIAN" + situation = { + "people": {"adult": {"age": {2026: 40}}}, + "benunits": {"benunit": {"members": ["adult"]}}, + "households": { + "household": {"members": ["adult"], "region": regions, "brma": brma} + }, + } + simulation = Simulation(situation=situation) + assert [str(simulation.calculate("brma", year)[0]) for year in (2027, 2028)] == [ + "CARDIFF", + "LOTHIAN", + ] + + +def test_calculating_brma_does_not_change_region_or_other_results(): + # Region input in 2024 only and a BRMA input in 2025: brma for later years + # must not calculate 2025's region after a later year's, which core would + # answer with region's default (London). + def simulation(): + return Simulation( + situation={ + "people": {"adult": {"age": {2024: 40}, "employment_income": 50_000}}, + "benunits": {"benunit": {"members": ["adult"]}}, + "households": { + "household": { + "members": ["adult"], + "region": {2024: "SCOTLAND"}, + "brma": {2025: "GREATER_GLASGOW", 2026: "LOTHIAN"}, + "tenure_type": {2024: "RENT_PRIVATELY"}, + "rent": {2024: 12_000}, + } + }, + } + ) + + fresh = simulation() + expected_tax = float(fresh.calculate("income_tax", 2025)[0]) + tested = simulation() + tested.calculate("BRMA_LHA_rate", 2026) + assert str(tested.calculate("brma", 2027)[0]) == "LOTHIAN" + assert str(tested.calculate("region", 2025)[0]) == "SCOTLAND" + assert str(tested.calculate("region", 2026)[0]) == "SCOTLAND" + assert float(tested.calculate("income_tax", 2025)[0]) == expected_tax + + +def expected_brma(regions, inputs, year): + """The rule, given the region each year resolves to and the BRMA inputs.""" + if year in inputs: + return inputs[year] + earlier = [past for past in inputs if past < year] + if earlier: + latest = max(earlier) + if all(regions[between] == regions[year] for between in range(latest, year)): + return inputs[latest] + return default_brma(regions[year]) @st.composite @@ -172,36 +240,75 @@ def scenarios(draw): @PROPERTY_SETTINGS @given(scenarios()) -def test_brma_properties(scenario): +def test_brma_follows_the_rule_in_any_order(scenario): regions_by_household, brma_inputs, order = scenario result = simulate(regions_by_household, brma_inputs, order) for index, regions in enumerate(regions_by_household): region_in = dict(zip(YEARS, regions)) inputs = {year: values[index] for year, values in brma_inputs.items()} - unbroken_runs = all( - regions.index(region) + regions.count(region) - 1 - == len(regions) - 1 - regions[::-1].index(region) - for region in set(regions) - ) for year in YEARS: - brma = result[year][index] - default = default_brma(region_in[year]) - earlier_inputs = [past for past in inputs if past <= year] - if year in inputs: - assert brma == inputs[year] - if not earlier_inputs: - assert brma == default - assert brma == default or any( - brma == inputs[past] and region_in[past] == region_in[year] - for past in earlier_inputs - ) - if earlier_inputs: - latest = max(earlier_inputs) - unchanged = all( - region_in[between] == region_in[year] - for between in range(latest, year + 1) - ) - if unchanged: - assert brma == inputs[latest] - elif unbroken_runs: - assert brma == default + assert result[year][index] == expected_brma(region_in, inputs, year) + + +def build(region_inputs_by_household, brma_inputs): + people, benunits, households = {}, {}, {} + for index, region_inputs in enumerate(region_inputs_by_household): + person, benunit, household = f"p{index}", f"b{index}", f"h{index}" + people[person] = {"age": {YEARS[0]: 40}} + benunits[benunit] = {"members": [person]} + households[household] = {"members": [person]} + if region_inputs: + households[household]["region"] = region_inputs + inputs = {year: values[index] for year, values in brma_inputs.items()} + if inputs: + households[household]["brma"] = inputs + return Simulation( + situation={"people": people, "benunits": benunits, "households": households} + ) + + +@st.composite +def sparse_scenarios(draw): + household_count = draw(st.integers(1, 4)) + region_years = sorted(draw(st.sets(st.sampled_from(YEARS), max_size=3))) + region_inputs_by_household = [ + {year: draw(st.sampled_from(REGIONS)) for year in region_years} + for _ in range(household_count) + ] + input_years = draw(st.sets(st.sampled_from(YEARS), max_size=3)) + brma_inputs = { + year: [draw(st.sampled_from(BRMAS)) for _ in range(household_count)] + for year in sorted(input_years) + } + return region_inputs_by_household, brma_inputs + + +@PROPERTY_SETTINGS +@given(sparse_scenarios()) +def test_brma_with_sparse_regions_follows_the_rule_and_keeps_regions(scenario): + region_inputs_by_household, brma_inputs = scenario + fresh = build(region_inputs_by_household, brma_inputs) + fresh_regions = { + year: [str(value) for value in fresh.calculate("region", year)] + for year in YEARS + } + tested = build(region_inputs_by_household, brma_inputs) + result = { + year: [str(value) for value in tested.calculate("brma", year)] for year in YEARS + } + for year in YEARS: + regions_after = [str(value) for value in tested.calculate("region", year)] + assert regions_after == fresh_regions[year] + for index, region_inputs in enumerate(region_inputs_by_household): + # The region in effect each year: the latest input at or before it, or + # region's default (London). Core's carry-over can instead give the + # default for a year that has a later input; brma follows the inputs. + regions = { + year: region_inputs[max(past for past in region_inputs if past <= year)] + if any(past <= year for past in region_inputs) + else "LONDON" + for year in YEARS + } + inputs = {year: values[index] for year, values in brma_inputs.items()} + for year in YEARS: + assert result[year][index] == expected_brma(regions, inputs, year) diff --git a/policyengine_uk/variables/household/BRMA.py b/policyengine_uk/variables/household/BRMA.py index 62f323bbee..89b684184a 100644 --- a/policyengine_uk/variables/household/BRMA.py +++ b/policyengine_uk/variables/household/BRMA.py @@ -33,31 +33,71 @@ class brma(Variable): documentation = ( "The Broad Rental Market Area whose Local Housing Allowance rates " "apply to the household. If it is not provided, the latest BRMA from " - "an earlier year applies when the household's region is the same as " - "in that year; otherwise the household is placed in its region's BRMA " - "with the most private-rented households (Maidstone if the region is " + "an earlier year applies unless the household has changed region " + "since; otherwise the household is placed in its region's BRMA with " + "the most private-rented households (Maidstone if the region is " "unknown)." ) definition_period = YEAR def formula(household, period, parameters): - region = household("region", period) - region_default = select( - [region == region_value for region_value in REGION_DEFAULT_BRMA], - list(REGION_DEFAULT_BRMA.values()), - default=BRMAName.MAIDSTONE, - ) # An input keeps applying in later years, as policyengine-core's - # auto_carry_over_input_variables does for input-only variables. A - # household whose region differs from that year's gets its new - # region's default instead. + # auto_carry_over_input_variables does for input-only variables, + # unless the household has changed region since. Only regions already + # known are compared: calculating region for an earlier year here + # could cache the wrong value, because core returns region's default + # for a year once a later year is known. earlier_periods = [ known_period for known_period in household.get_holder("brma").get_known_periods() if known_period.start < period.start ] + moved = np.zeros(household.count, dtype=bool) + if earlier_periods: + latest = max(earlier_periods, key=lambda known_period: known_period.start) + # The region in effect in the latest year is the last one known at + # or before it, or region's default if none is (core gives the + # default then, without storing it). Every region known after it, + # up to this year, must match it. + region_holder = household.get_holder("region") + branch_name = household.simulation.branch_name + region_periods = [ + known_period + for known_period in region_holder.get_known_periods() + if known_period.start <= period.start + ] + up_to_latest = [ + known_period + for known_period in region_periods + if known_period.start <= latest.start + ] + if up_to_latest: + start_region = region_holder.get_array( + max(up_to_latest, key=lambda known_period: known_period.start), + branch_name, + ) + else: + start_region = region_holder.default_array() + later_regions = [ + region_holder.get_array(known_period, branch_name) + for known_period in region_periods + if known_period.start > latest.start + ] + known_regions = [ + np.asarray(known_region) + for known_region in [start_region, *later_regions] + if known_region is not None + ] + for known_region in known_regions[1:]: + moved |= known_region != known_regions[0] + if not moved.any(): + return household("brma", latest) + region = household("region", period) + region_default = select( + [region == region_value for region_value in REGION_DEFAULT_BRMA], + list(REGION_DEFAULT_BRMA.values()), + default=BRMAName.MAIDSTONE, + ) if not earlier_periods: return region_default - latest = max(earlier_periods, key=lambda known_period: known_period.start) - same_region = household("region", latest) == region - return where(same_region, household("brma", latest).decode(), region_default) + return where(moved, region_default, household("brma", latest).decode()) From a312679d9119e03f5a16a985febb68e9de007188 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 2 Oct 2026 03:42:08 -0400 Subject: [PATCH 5/6] Use OGL-only Northern Ireland rows in the census BRMA table The Northern Ireland private-rented split came from ONS Postcode Directory records under the LPS end user licence, which does not clearly allow publishing derived figures. The rows now use only OGL sources: NISRA households by postcode district, mapped to NIHE's postcode-district BRMAs, times Northern Ireland's private-rented share (policyengine-uk-data#516, d3b5ba0). England, Wales and Scotland are byte-identical, and every default is unchanged; Belfast now leads Lough Neagh Upper by 79 households. Co-Authored-By: Claude Opus 5.5 --- .../LHA/brma_private_rented_households.csv.gz | Bin 6494 -> 6499 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/policyengine_uk/parameters/gov/dwp/LHA/brma_private_rented_households.csv.gz b/policyengine_uk/parameters/gov/dwp/LHA/brma_private_rented_households.csv.gz index f4537ee35fa0d63b55e98c8bc19660f9e83f3366..9a299df40eaf1027234399d236d38ed9202f007f 100644 GIT binary patch literal 6499 zcmV-p8Jy-HiwFP!000021C?D%Z}T{o-Pd0kFZd?4RAMDDqF8~Pe6FmFZggRKfbJ=v z8}t9?o~wsNiKN_{s(d;}5=CC#7cc35{qf)b^XLB_{`#+f{^ju3KmPWwKmYvazYl-^ z^Z)++$KU__$KU?_TT@@YZXc&(Tc3}Y!}D9yUz@hQ{i@rx`8mjgwr+=`4#9^Vjy?pb zb~xt04q8`R(&ek^UrtSXq}E7pY>A-`HYinM=z|f?lo-?wuQrVCS9AXA(IeXrnJ{%Q zRSwe!n|&s|QRx+UZ2Bj>%?ja?kUB`CtxaP3AVm-%iQ#2IZ0_sM&o%mhUg<;@q?f8H zi|T_mDvQ#K&H5`FhR(OkSN-zR^~cNGsc-Pr-ao`t)N$@Hsy--DRS!|o(rW!7uYG;~ zdaFNf=jK}X?+^qh^$v;TQEWPFO(K!>mE0jwo1}7X!#aIjSWdjmK-2`AyN!gh%3*k6 zScC7U^Xui-{+!SB$|?`4gARFAZv|n2(u>zmZky9{GfJm4Musd-FBY=O<0J*aX`99I zawXRjpgo9^og0~qEs?S@UMp-6rFU|SF3Q*Wx8uZY%4>mL! zeJS>7EO~xCBp;{tt2^J`o1b%ohPg69L3I}yG%}mciy73Aux{?T>oH3{iAIeuM%e-Z zBcwN~eaWCvlCmlg*hp>KIWW#&m)kQmdfmcsij7X+($t|ccAo})?xf%4LAU#jl3?EL z+?F$vqmiO$V>{XlRws$GT-XmP(We)QGCG1p+m~=n?_$#b$DTHl%!>y1x5GgIqnJ>@ZWb?ZI6E1)*pm0NE z9;TOX^uPj6IU<+0=4Em+)y7l1qw3&&9@Ph@eHMix-S|hB?)&QvF34-&)HFV~kNP~i zqpU>qf50bgvN16wJ{f@R#}xSZfQ?Xm)gR5}bZl;~ZFjkJpXk8N4B3#_k<@!E>d%eV z+mN_h*R8sC?eTJ>KX1?7AkIqIutOqOiKhP!iJV*)_MQYc!jD7Uz1*7fD|msX4;Sg{8EUTbh7L$W@~C6HKFGrOCpq=_fV3#Hn56 zXy2OS$2a;+LR0!(nmRbI!Y)nI#I$;s2Ky}%J@;IHJoTsJD>RRHwki-1cu7%Wz+Brz zW9|%Uh7)cp0HYx%!`Z)d?K}Q|Jw3xD1SR~#ba-2pP!IE|!DJWB!+dCOpO7z76z#`l z5;33Z#ydkDLX{As1%m`lDOX}Z)?}4+2 zf&uktREfb>#BGK7^m!R&-RhvTFpU<4BO;4q@)@nNIQz&6-5Z6rD%c`}G^7r)#2|g7 z6G||SbII4-_5%$YiB$LUHJISrPu+jOC}T>AM8mUu_$vR&^QpIz;nmx3Y`=$S z%sU}7AbBd<=?th3!Aq9`(Q7uLIXCgBuXZ?4l6H?o(j zG>F&(G&Xr@-VGyrYrFF>`(5<_ZQ*+tc^tDoz=ISe^d?CdIiCZavALBY;ko447~}Bx z!!g>{K1_!hROP47t=0*1oFJ0cPY8E{$a(BXh@8jUYJpY6X_TqT9HawO;+3noQwotV z5bMxW%Y^6-p~o2}(x<$O8CN_ht2}{bhAX`QO2i~Og$ctDQw;QVVK1gK`#e&$l?nSi zinv5#GH!U>z=RPSh&+9o%cc1p#xgj&&*NlHu1}XlQobq8yy0;RRaGKRT}bvgVJkO8 zg@n3&;EB{kC(S+&alVp?2Yxlbnsfcyj1y%ZYN_;tWQr8$4C^i#9-31RlG&&l`P(>q zClCCtM@U͛{ZLFz7z@}iK+-3wjjC~hDBc{-m-k;IBuRfwFj1bTa*X_qG;EhDHS zvIL~YED2vkSnE-*wJI>kL)FTb7zlQpw*>}*=dk|N#T=a*V}z;n9u3wUq3I3{>r=v; zJsRXg29a!BY|W+BTJOF=ZF@!N@=*oS%QLAaaEqcvGsav zR&5Am^d~~f&%qRuc>=Y;dLi-zY6FGiDo=m_Rd#N}V3lLhkpbRumVkbs03uJIHaP8U zp1>P@f+eoG)JPc2-z~k>0zV^WRA0ZYbE^Hd&APuvF_W``E|%4NXSYWq)9xJP258(0>0HI7`NwdB%^BtTX|5SYAK6i{V0#Z{fiE! z=?to|vXTQi*fr6EzZwcyPRE9W+#r(>Rx^D(5HkiVBE3jr7-6TQ9{NH?Xn0-Fwn{?q zQN345jLRT|O=3_rLQK!Z96CJJecxSgr}ORQyC)jr7!sK>1*X&QaInB6Rhh$eQa<8z zoEO7F*tpDPL?|-Y9!wf6O&MlbC{ehA&P0?czDcgDDhjdGgKYBOL$LZmHdfw+v=6c& zuR?_6!%3i@hp^NXa7-2oc=7~xn1#v{I3cdwlEAtiZ5+$-q;)Efa|ii6u95r-d7KN3 zDzPC<9iV*N`wI|i8AXt36i!6tG9ahlP4rnWKkc!Z7mS1UA&2tvbv(*Tm?K@Ch4E_V zqK8ki(Vl)a{b$#;?7G~wV1|TB5baHwL?Y?P$uERfRHwD75z#czhSt~un(_9R^Ls55cFv~&l+#UzvXw132 zHSJg(D`b_%Xb^mCX73o1@?NDe-V6{)V4~hh`dCjVs-1kLWRXIWAazxwpe-S4Cx=l> zaR~8f$2bKODxOU-juIMEcs|bnn|Q&j#zOp&5sJ!etFkbLzRJUh^tIl=pUwGvIyXaM zbjZT$PzC3+FbP2L`MWt>jA!Zf^jdS0$_l@|4xh}fdYS>K93A8qV7<7$qaT)x@7aI| z;9};Gx`h+{u{{|Pc05Q=U7owIp%P#B|Sers<66y3kG=rU#DFmrl zvFNVj_s2_rYKDTr*Sc-HYt1{0kjmL%L5+I5$I>WM-hCFg%Pd(GNH@%>&#&JX-iRZ7 zfdC;i))g37{4%mY-~@$Pw3U5yqEWm_G2Xl*sH(~oxa1V26ge8Tgk=hv;}-@kL(KTej_mXXd=ej(m8R0m=2NWXOORW$iN{=hzo5nOvIJ zw!XY}*CFxmL!N-x!kfYq$X*zu@;LhrazS`Iy}q@>$RBT~%TPKC*D07C67pcK+96># zle9aeAw#qf$hP}>JJrLI4A$Q^01TH{e#QjwqL@kC!Fpw>xW+|?`ds(#v#l0OiP$pW z^y{j`&`2Y>5(8Fq*=?T<80e1o)4A_BTcnVpEHI#ORPh-ag@v}n;C3wD(l_UqzPS!= z9gohL;?U>zXH3(H2f3IXm3)k=QP=MtpXP0y|-DfGVY4nC84MgBY*x97!R;cV;o^(Q10MUeL;olSNG2Hr7*kFDC(0TpJ|kI zvCiK+dl)T@Zlo{ZL%+J5VP4WUh_c1W0APN_0UW6?d7InaDE{?_OEjN-jP2({87 z141B_&wy}_h0K7%mh?!LP~-pAoTrU}ISe&NB7@PK_CABLM|ui-sZx((AE8f3)78~Jty+JsU#$awYjU6#; zCN?gOv6#z9I-mRcN*!?P8;XB9)T@FPB?i{$)Jif}6b`Rlkpf-58u=s1eNQ22^EpIe z)J|K}oQQ4V9iwQ#hRl7`|5)2hVqB*mqP8PO zxLuo?H-1`ng_9rxkod6+kltzIj$44dVZlTrL$OGMO3*8`?@$}=^El2lu{w`q$Biu9 z&pUmiv6wb!#KvISnPIB!RY=$ND)01LUjC?0N3!rN$XZI1#$YI{NMkq}Li;qvNwRp{ zKbj*=s@vr{tYJ7I@-VNI7H-Efa~1n}5zgtV5Ou?P(~0r%BwKGdsf?9*TtmiG9v@DL zh{o*PTt~w!y{l3P++LGJ5DyK5eQbq(5*O(N7*wx2&2uo|1$*ZqSY%&)h zb`D`RDUBkt?Bgz3pfGG9j`8}YK2rF@&a8^ZaDuo9DW1d=gfEnFj4?r60vY?{m%2S& zX0@Qwkb3L~c_6y($hfCjiPljuo(tui+1qarim{ z(6BBAU~5=BQAS{>dyIyUEZZ=0&dL*vSI-aY5VxOuqMNt3ntD*%n+Gm7tgC1!dV`1xG_V#m3lT^V{sfS6z!|+ zWpd!u?D5|(liyZZv$>2ZR)zR_l!~DKQz8fRl9;kNSo4Zn7Ut;R`3UY$k(c_`^p4=}M8h19Q0&QHQ*?-S>Qinbj@_D()1QLCHfD^JH9uEI2+w-zog=uEXEh z-YaDeRa_|Vav1SaZ>aM@A!q(FI5(B#jSKq#3Al`DS#9-ez#FXyL z(|BGTka?68U*TSy-c-058a`4`i!j8cu9K+>sq5GYle~fVoJ7D-(AM>5vNG*37|Yd| zdxm$Pj*v*|hv`^mlm1~kOGwPW?mnQiCW-Khd92{S>hCjh06h0E-A6O&7;JFg<-zZ* z{5}sS?f3`Y%briH=&v=-1R53#hd>X~w!=W)eusjz)(!=FD5Bt;lJo6y`=|#6XkxV! z zaq-JXK#fH`8vwN{l}P|Y)o%6Ml@Bm{YU}1)f8SoOVM^bi}QVc4}MbNNknfTtMD-r zc5Zc%1A)1#=qgD(j?S1jgdm_vgJUKu4I*e<^hFE~i*j>Qn!Y)nUQgpTKv`|CWApm` zd+ygEUP5fIbNo9u{G~me=ZT7Sy3J+m=cOsMxehOJ9_PjNVaM)x{R34nKmG0b=R8=a zDvcoK^)?Bic24*t#17IlK5NJC-oLs@tcI zH_Aw#Me%jYWnrX4q|U;e+`8X=9c%3+PKSvvDxAP%Ru`lGn=Y~ti0M;DSFXMO@X2h- z=>z~PIuFqj;3|iZ0H%t+CGy!_fB0l7UaF1c1)5F)$d1=_Fg6>ySZl zj+;?A6!|b?awuDad76GUA5VSve%~raVPq2G`tK?Up+-gPBs832FhYZ6dcFRfo8xeh zo=qZHXi-Uocc>&0j-Kb6uJ!rS)I$YKSY0He9nAV7pHwPV7dcxRCMxza-1@ifd_3}& zA2F+n2tmD_PTm9OI3(|3%{d!~udaU~lM2(U{puR0waw}xM9jF=#i1l-y!fjgRuo^` z`lIWI+IOjEjkbjm-V2w7G2$){!(^(dH`kjSWClUUg(QM$YLf`RXhQnt_p#nY&8l^} zh=lI_>ElIsh_l>I7rCgK-qtM7hkZKrANRwzl?x(I;6gQwbB=(-p%+!2z#*u|Jd7DA zd!n%Qd;TE{EklvRnLR6t91JB-k^%?T%DPRbB)mH*cS-!)0(&GB!0eK^E@j196&{H^ z3`b4nVU!;g;K3K!;_IJZkvkMV3vGv^50%-WAe7p4#r-!=_*YdjC~D7e073>vF3IO% z_B749dAZ*Ok$N-sxgQ2G!anv?5*2IZLsZP|&6rb9Hp@1uYBOjt1;5C#8MBxIzi8(v z!-~gxP9aUmZRTUXruT9`>^S~%Uj=l&RbTnJu8c26O}MJPvDbhyw4MO&TcgCrE5=b z-|et(+lU+rhk0XUW)wMB%TR6Edlov1eiBKZ=#iD&C@2!96nM1+OdWe$wD)K+Y zJ`V@#ZYFLrb!EJJ;CVcMHb&yw^}~wJ+=trAJrYeNyOG8<2@6)$-jjx+8C_$U6i0&cR=1~@`giWT zs2?UkVJoaj0LYX|kpMU+95wh2r{;_$&dPe5g!qRi(vaq3?vfB!^~wk%?L4ej_NTGj zh!4~(FcglRRI)_DDxI~WMByE8)pF$DPyKt_3`cPrcsyH8U;0b$_@r8NnNSMbfU zHOn(ZqH$avAR>JlL4`v4VZ7~Ui8Re|{0T1aH`f&lk=HO99FNe0G(Ie{*xnXK& zVhPhbTYM(HQRxM^Z~G^_%?jbNklIP3t<7S3Cq)n;i{WKKtnNDi6fjL*L@7y?=vYRJ~Io#)qhAX!lT^;FSci`@%ZZl-h?-zaw~KhlN1D}Z4t-I zm0V7M?r?lQb@*oGg(`qFH-wb3fa+Zg-WEWpK5um%WtqIbw8t|mKw&p&&_nA!SkY+o zrP!vip#3)A9P={v2C0%+(1Bs=Ladky&+K%AkgXWpj_G9<$_=Xw(Q}l&ugj zLVBgz=L{MpDPx7eMrzg0fpPpgU!S4Tn+}FktaSRCrgqWTZ5s5slYWy2-R@UPf_c~D z>FbLo8GQ)EIB2`YLT^PUw`kNh9lYM6p=rvB6>ZiiDU9=F0_l2hg((wAf>e-|S3rQw zs;QrC^M1bWn~vpJDp4a)BVx1_0yV+NbXpP#?U=AHUaxB$9>!VQsS zm|nip12Z`1h@9Wrmz$HRR-W1&RXgv?sNOm4izpQ7%0D`vzQ5k!g1q)^L*sM(XpW;h z%1T842Yk{dD-%=WlL1(NOofjRSP8{f^U zZw@2KajoFw$5>y%q3dcm8lYGBQ@e+k3rlHlZfOQ4Ay=8!-(VUQD@_q*O@C9vbDY{m zj`p?Pe|)3QBs8Vpq^X_rDs0j;O-!pdX|UfS({sNA8P5LY)V<^Xm%}qmLQuj#Ooz7>g?gAz4JNy29_B-P`;2^%qUb)( zHxcu>ZoD(pF2sx&%@`zTO1T;XvL<8ZHK*>bL`hmzje!BB`B0;fU#N2Zb7w%Xlv`8Y zn!Y<-NCe3t8aqb2D7JYd_bRAu9tBe-Jn&wq$>`P->5c3n`A2(ppk=&y9(e|-rI_i5 z83XFks2YQ7&!;Bm=2DT*hF7n@vHKpP zG4F&dfaIxYrwgFo1utCzM6cP5=3K?2K5lTJB<&W7q)7(5LE`XzEpfT1P{{{q6u-L% zXb`anXl(NGyemfb+MSNW?04}2+RXQ^@;GLFfCnkc=uMI^ay|z-V^b?b#&apLF~;HX zhhwy@eV7h2sOnFjTCFqYI71|@pAqg1k@MJ(5IK*x)&h%&((&xO330FKRW0^oR!_{5@C1Nt2!h~UnDF*sFvlkQ1Hjh+oG+~=Z z5tqnJ#ubknm@#4lk*804KDWQaSO#aed7R8C_34~Q$~UK(S3GW^iZ$Y`3&|d5Y~_lm zkWjY|Jdv8{q}k>n&Q~(?z_0dKdu(3YaiUB^EtP(dOp)T0VcjIdLv!juG8%#d5mJ`1fFeUCNZsX8UKCQLd!fr5#qHxi562@Zl2q}kh{&l(ptlE_c4Y$6GJ-my zNI+`LlJF&jr5^QKs|tfWRIO}{fndjZTVWu04$DuS&C#hbMwqC#Xt3r8O*d#*pAz0| z(I6i(h-~9hYc8$UdW%MbrZk1u8#LDI%52dP)<*K26I$9TrQsGA2?|=p#WI0o>*d%i z+7QU--v~KB2UAFv3DgGbg(wrK4HS;6G64bdooT-yF0rN~qS0+G~z0tq1U?FjS=-V+L z+Pxqb{Fi_HgRC~inrli2fB1TAn^*MqrR|w4ts?_gC4YZ@j%1}t1lHtn{q4XUil$?U z!k;qoBneCA#WT^o5hE6d3dHGQF8UH@{?_Pa`0cJEcKfF94#(Fahae#K zpUXy30D0P*^w#fwTp5)j2aud-?^Fgs%!qu|8sWe@Cf~+?et$f*J$DAoK-!K4LG6OE zH3D)rX%8(3_*R=?T%W&@jBXHYl|hB7r6P*;qcRHjI~jrV_i;acIImCU!6Cz^3#i7* zN(tm(*CY@AYAIkj>{||UgUmu$&GhL&Oc<<)^dgI4gq==$=nEO4;dMdVn1$e@datq= zmq7@d#h_}0xIGhdX!q3g{poT&9Ir3mJ<$-ykjT_2Fr9vbg9Rq3>Kv|<@(Jg}c`+=6 zjmum{gersWff6;!FvCKP!WDETq1@t|?7FI=5KBGCCjUJIs~==z3M94mz z1p0XhOHBpGWTAkkOkjsus4{^Q;>yhltn1Onu^f*qk}BieLB5P@B)>u#=K`ZBR)kvz zD4+KJ41`)n5o8{P6A@hj-*7PwsfSt=603D`H24G)F z&H;?Yt?pEz$}EdvVcxksrcpX)^B9|hn1G{=y2~~N$)ZBww5QS)21%JpRnLOJ8Ezpn zjL8Jc#V0JS@XXW6;3y!KmFIHgi0ilaESPDI^I}$0`MF2}wISjGBu>NJl%x zDVR{{Y)WyI(3r~ec?Q_b3nn!d(vM6~RAyUMgfaB73?tIldIf*B$K&DH4u#R72&-KT z&KF@4fZ)q_bGVq!(#zqs;Utw6etjK2nO*cW15i0S$Th%vad}5SEEyZ=T?XJ{=8(IE z6aA?@84-3oNKc)gPhUg97=?-ZwjDg_n-m2UhSn&Gb4IbsL7$T4tx*OyNY6O^>rdB> zUi}x3Dg}aNT*1!}WgMl+LY8qZ1PwF%r{|WP*)6a(htMR_`Fm&vJ1KJrQn6yzUHk8k zm;TTW1%t0m*PSj6?<}%PY_Xt5z1?DIlqqjMi`!-9EDEGs<}}CG?=x@25xzoz5E|<$ z3@m;bSs`$O!pz#rK047T-lQ0B+7T3EodTDfqLeB}qn5BvL38{}CSMLOgO@lT8oo$M z#m*(1FOt}!I4QlzYkKpxz3|MO_SlgR?+PHfeo=-3h*8$Q0CJ9f){>h`^V&7%*VAQ4 z{QFQQAhz(P@&vLM#;7vRzJr_*-VU#C-7xa|>)||<&cbyHW`l%0n2Q@E>}Ha7gEVA_ zW&+urzTOVauq1=^w-o@xCFY-T19(x)WbRBNIv%#J7@<7(9PyT`cjF~v*?;E%7v`R)2d&S=}St4YefLLi}`$g@J=0`A%q z&}Z}Zj%zZB43S4AYYc_;^jKqP%sy3PpdIrsVb^G-p5t8Ha}-o}C|l={dY*S`j=~AE zu;9$5Fv6WYNF!oaPrpxPCFf300*-0U`!dL`IlqmyywajIYXlo0waOvLSA^_6oCg!> zi{={!SWV_InxY=6B8Tw+>paF0CI;mWJ~d}lsD7Q^S-uozR~|)uGUPLlk}lTedxyMC z(mhXI^Zb5c`!U%?i-PJZhitHrq0T`un6z~cFV%#k>z_`@a^EU%Z8Rkij)%1+6qVz} zTRHVA60|3ybxUo(oqi3WO^6#L%r2)?ousko9_k!uKAC>&b8|%T-XMfp=}-V65Xu)o zILAU3z+p>zB1@?8|7wr7je#W$HAkX=(VX_afU)Gq1-Ll*Kw;&r!k}4aWtBm54`qrK z#;_$U?e4e3J^e*PQ3C1p*g=&CS;@!(IJ|Vxu0w~7@Vwq2oXBG^x2wjE7&a3dm&aJl zWg?x=eRH7>xb`i@zZ~ktphb;=H9EDB%mszRYgeT}moG+sPjcT=NZNi5Q5gBOs*EdW z6H&&wA7mM4$55CKT~*cnLgv}wo9K;E!ml98cUq(vp@1=@G04fkanXPQ`D#YVY%tV_;ivjH=IOcWf|9yF;&KgQzDWvJGPh6 zFiY=Z4uRWivIqjAkVTLeN^Re)R_MJwezBDhvBn@}weZZA49uEz_AeQ1s8SnVuF!Lj z&=8hA!o!m3BRqE5OT>E(n%91Vgm^*N4H5!v?>0yrA(`U)7S*_^dr>ahGieM_F56qU z?3KzrWfC25`hC>@mt4Vnj znPs1L$qa>I3u%nkx6PiyA9iL{I)*ohi;&Vue1q_XGL12A5SKuvKKZ%n4(CZNs63>g zX!8)Vq%IG!F-vd5pVI*>U53>ivQBw#|W@nmktGe<1D7pX~;GMak@%6`DY+> zOPZ{hfkWCpeJ-rYFz13va}g6nCfTL)YJzZs)QT`i|IR0He<1Ix8*0rbeV9Plb|k&X zLA+DKWFI(`zD*JAP?$E4`mM390KyxeT-qgwLqkj`fShVjNrT$o+d(i*FgXC{8>SyV z0!UYi+#Q(9Q6wGm+MT|qBh0LJB~WpvxB^NZqL?P*5@f;g3Hnaq_vteHjqSZsmQclo z@~#AHJ`2j1VD?O7^6-!LxIcW}DxB8{gJFpHw?ynHRhh--KHZX zlKNpfmf56#n9dTCcJ9Ex$*9tV`>d`R`!-0>WiH**;F6r{>KaXn)!+{Yn9s)&a_ZyoD{(IphYzqdsgsc}<$^spN{ zZSv#+8o7Auan{I=&Z>iykedz_^;;sh#UaV{qyOgz3CWiaNp#??~Q(&hm&^v z1Mg+eCsp*924?~di-kj=2Wi`2AaB1xL0W5r0zH&aa8AkbdcJ-%g90?E+DUQb)lw~& z;(U@#<@p>Jq{f0f>vHJ6rbYgR(OeLaP!BZ%N*YpcU_lrLB$YF_&JIr!ZMiI1=eJ{btpM)~ZRSHw9_@FM{suXoxUW5q|a(NZ{*@)#;xX2aU)-0~#Xl~~g7da4^ zx{9un#N+6UX+sDCnmjmWvhpB;##vva;IJq+HKpy_{o(a6ZUdBYeI1+E@85I34)GFV zeVyaqso^i(;W$lHtkbP7V?QrVq1APGiPJdGt`9qQ$Lk-cg8AvM&p*e(I>kJKnAh7Z zgxWdbvk*H-xA9pze)le3I(;PnHVdHuVw%S*Z_mryUB{MU)GKmb*RQ5~I`Kvs>5C}7 zPPrnCbcobNn3G%gyDwv{y~OD-=|zPTn9Sm0(to#$ECk~AsiP~`U4HmvHsy2%fEAsF zXby0dL&yNPioZGXdAj`Y$yCZ`uJZxOIs;(;VXzqhonupAFb=nqY;mqb0mV6PMwL+H z!;C4RYz?Mq`q_Rw^{4myRxt`Avk=#R$1H>z6|J+-aEieQ4VLNk@^ftW!$EpBi(sKe zWf9(?l0`Uro^HA{$8*~Z6)<6Ok&JdQ%Zq$csaRa(Y-zYrv6tc6znzZzJ#YCDld4D% z)Z6LoJ#dag_8!)plX3Vu^)F;nVVbpHT;sI1SzLsO8MnANl*Eh|e>KC3;%nD@ocf{m zUFu1rtzm@s!WChRxGTdjnd;V?%S{flfS}_-7Qr;NS%hD7L;CjjvED>Ys`Yje3ElhC z$BXa~C%L^{Gzlv;Jg{WnkeS5*oqYR_;0LIFlD$(LdFG|jSk zrQZdSdNuZ?9|kePKK4`+6>H@~RLt$wm{U)-^ERru8nl#xU*uSgSxSLlwDFW-#bY_A zkS63db2$QbRTptS63)9aflos;p-dQ71DAyKJbhC^$&%J!l=4(y4t*DRa)7@Ck;h6qR6^+nc)OOm85n- ztW^7w#2W>}28k=p$Qb{eZrc)`n*MkmY^22UTn52bJyPr$gb!ynIb=9q&&?rn79d$; z5z9@lMM*YogN1lo$xRw2y7yv>#@U51rzIyP=>mfCbd!hhVw;EfB)>T{`+YF&l95P$ z9SlFPlf_UmLvCvp8|K0Mb@$)3lBi=|1H++;82}NU&fURIaUv*hO?P-{e&@bR`r!sB zY=t!$0GU!LG63g&PV?J(7$)>a1^&uK8NtR`!w}45EhekDGe1Kl z8OQknBGTs(R4C*h#@l}8NZanmpWyO-bzQL#+0}JC-y}jbTxW4~YCd}X;rZ?3gq;}8 z$5Ypi6;&dc*clE&ZKdlRghO$I!%2kX3;GjI!x%%MU`!U`Z67iVY5b0e&q91c=Dc^$ z&&}*OYK_&yJg87UVGAI>)4KvVWJ!F2&VF!6Dj)LLs)v)`8nQvgJjO2KyldvaG8AzZ zLk>b6V5+j`KQJT>3hQ$i>;sC+?%3?;vqYKC&OM`!&H*^Z%H{yJkMmnD&3ilja8eX9 z3-LDbL}A`BEWIXQdxwS)XWmQriz2Of0* E0EP>ZbN~PV From 14eb69d8864481936fbc2d65f2db480ecea68aa4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 2 Oct 2026 06:22:42 -0400 Subject: [PATCH 6/6] Read regions and earlier BRMAs without calculating them Calculating this year's region from brma could still change earlier years' region, and so their income tax, through core's carry-over, which gives region's default for a year once a later year is known. And a BRMA stored only on another branch was recalculated year by year until core's spiral limit gave Maidstone. brma now reads only stored values. Each year's region is the last region known at or before it that this branch can read, or region's default; the latest earlier BRMA is the latest one this branch can read. brma never calculates region or an earlier year's BRMA. Tests: the review's three repros as regression tests, and the sparse-region property test now calculates brma in a random order and checks that every year's region is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_brma_region_default.py | 80 +++++++++++-- policyengine_uk/variables/household/BRMA.py | 105 +++++++++--------- 2 files changed, 120 insertions(+), 65 deletions(-) diff --git a/policyengine_uk/tests/test_brma_region_default.py b/policyengine_uk/tests/test_brma_region_default.py index cf5d5fcf7c..45766d17cc 100644 --- a/policyengine_uk/tests/test_brma_region_default.py +++ b/policyengine_uk/tests/test_brma_region_default.py @@ -12,12 +12,12 @@ every year from the latest one to this year, brma is that input; if not, brma is the default for this year's region. -With region input for every year, the rule holds exactly for any households -(each with or without BRMA inputs in the same years), regions by year and order -in which years are calculated. With region input in only some years, it holds -for years calculated in order, taking each year's region as the latest region -input at or before it (or London, region's default), and calculating brma never -changes any year's region. +Each year's region is the latest region input at or before it, or London +(region's default) if there is none. brma reads stored values only, so the +rule holds in any order of calculation, and calculating brma never changes any +year's region. Like core's own input handling, it assumes every household has +BRMA inputs in the same years (or none) and region inputs in the same years: +core fills a household missing from a year's input with the default. """ from pathlib import Path @@ -206,6 +206,62 @@ def simulation(): assert float(tested.calculate("income_tax", 2025)[0]) == expected_tax +def test_calculating_brma_without_inputs_does_not_change_region_or_tax(): + def simulation(): + return Simulation( + situation={ + "people": { + "p": {"age": {2024: 40}, "employment_income": {2025: 50_000}} + }, + "benunits": {"b": {"members": ["p"]}}, + "households": {"h": {"members": ["p"], "region": {2024: "SCOTLAND"}}}, + } + ) + + expected_tax = float(simulation().calculate("income_tax", 2025)[0]) + tested = simulation() + assert str(tested.calculate("brma", 2026)[0]) == "LOTHIAN" + assert str(tested.calculate("region", 2025)[0]) == "SCOTLAND" + assert float(tested.calculate("income_tax", 2025)[0]) == expected_tax + + +def test_region_carries_forward_until_its_next_input(): + regions = {2024: "SCOTLAND", 2026: "WALES"} + situation = { + "people": {"adult": {"age": {2024: 40}}}, + "benunits": {"benunit": {"members": ["adult"]}}, + "households": {"household": {"members": ["adult"], "region": regions}}, + } + simulation = Simulation(situation=situation) + assert [str(simulation.calculate("brma", year)[0]) for year in (2025, 2026)] == [ + "LOTHIAN", + "CARDIFF", + ] + + +def test_brma_stored_only_on_another_branch_is_not_carried(): + simulation = Simulation( + situation={ + "people": {"adult": {"age": {2024: 40}}}, + "benunits": {"benunit": {"members": ["adult"]}}, + "households": { + "household": { + "members": ["adult"], + "region": {2024: "SCOTLAND"}, + "brma": {2024: "GREATER_GLASGOW"}, + } + }, + } + ) + child = simulation.get_branch("child") + # More years than core's spiral limit, all stored only on the child. + for year in range(2025, 2036): + child.calculate("brma", year) + sibling = child.get_branch("default") + assert str(sibling.calculate("brma", 2036)[0]) == "GREATER_GLASGOW" + assert str(simulation.calculate("brma", 2036)[0]) == "GREATER_GLASGOW" + + def expected_brma(regions, inputs, year): """The rule, given the region each year resolves to and the BRMA inputs.""" if year in inputs: @@ -280,13 +336,14 @@ def sparse_scenarios(draw): year: [draw(st.sampled_from(BRMAS)) for _ in range(household_count)] for year in sorted(input_years) } - return region_inputs_by_household, brma_inputs + order = draw(st.permutations(YEARS)) + return region_inputs_by_household, brma_inputs, order @PROPERTY_SETTINGS @given(sparse_scenarios()) def test_brma_with_sparse_regions_follows_the_rule_and_keeps_regions(scenario): - region_inputs_by_household, brma_inputs = scenario + region_inputs_by_household, brma_inputs, order = scenario fresh = build(region_inputs_by_household, brma_inputs) fresh_regions = { year: [str(value) for value in fresh.calculate("region", year)] @@ -294,15 +351,16 @@ def test_brma_with_sparse_regions_follows_the_rule_and_keeps_regions(scenario): } tested = build(region_inputs_by_household, brma_inputs) result = { - year: [str(value) for value in tested.calculate("brma", year)] for year in YEARS + year: [str(value) for value in tested.calculate("brma", year)] for year in order } for year in YEARS: regions_after = [str(value) for value in tested.calculate("region", year)] assert regions_after == fresh_regions[year] for index, region_inputs in enumerate(region_inputs_by_household): # The region in effect each year: the latest input at or before it, or - # region's default (London). Core's carry-over can instead give the - # default for a year that has a later input; brma follows the inputs. + # region's default (London). Core's own carry-over can instead give + # the default for a year that has a later input + # (policyengine-core#562); brma follows the inputs. regions = { year: region_inputs[max(past for past in region_inputs if past <= year)] if any(past <= year for past in region_inputs) diff --git a/policyengine_uk/variables/household/BRMA.py b/policyengine_uk/variables/household/BRMA.py index 89b684184a..c21e567de5 100644 --- a/policyengine_uk/variables/household/BRMA.py +++ b/policyengine_uk/variables/household/BRMA.py @@ -41,63 +41,60 @@ class brma(Variable): definition_period = YEAR def formula(household, period, parameters): - # An input keeps applying in later years, as policyengine-core's - # auto_carry_over_input_variables does for input-only variables, - # unless the household has changed region since. Only regions already - # known are compared: calculating region for an earlier year here - # could cache the wrong value, because core returns region's default - # for a year once a later year is known. - earlier_periods = [ - known_period - for known_period in household.get_holder("brma").get_known_periods() - if known_period.start < period.start - ] - moved = np.zeros(household.count, dtype=bool) - if earlier_periods: - latest = max(earlier_periods, key=lambda known_period: known_period.start) - # The region in effect in the latest year is the last one known at - # or before it, or region's default if none is (core gives the - # default then, without storing it). Every region known after it, - # up to this year, must match it. - region_holder = household.get_holder("region") - branch_name = household.simulation.branch_name - region_periods = [ - known_period - for known_period in region_holder.get_known_periods() - if known_period.start <= period.start - ] - up_to_latest = [ - known_period - for known_period in region_periods - if known_period.start <= latest.start - ] - if up_to_latest: - start_region = region_holder.get_array( - max(up_to_latest, key=lambda known_period: known_period.start), - branch_name, - ) - else: - start_region = region_holder.default_array() - later_regions = [ - region_holder.get_array(known_period, branch_name) - for known_period in region_periods - if known_period.start > latest.start - ] - known_regions = [ - np.asarray(known_region) - for known_region in [start_region, *later_regions] - if known_region is not None - ] - for known_region in known_regions[1:]: - moved |= known_region != known_regions[0] - if not moved.any(): - return household("brma", latest) - region = household("region", period) + # Everything here is read from values already stored, without + # calculating region or an earlier year's BRMA. Calculating region + # for a year can cache the wrong value for an earlier one (core's + # carry-over gives region's default for a year once a later year is + # known), and a BRMA stored only on another branch can't be read here. + branch_name = household.simulation.branch_name + region_holder = household.get_holder("region") + brma_holder = household.get_holder("brma") + + def latest_readable(holder, condition): + known_periods = sorted( + ( + known_period + for known_period in holder.get_known_periods() + if condition(known_period) + ), + key=lambda known_period: known_period.start, + reverse=True, + ) + for known_period in known_periods: + values = holder.get_array(known_period, branch_name) + if values is not None: + return known_period, values + return None, None + + def region_in(year): + # The last region known at or before the year, as region's own + # inputs carry forward, or region's default if none is. + _, values = latest_readable( + region_holder, lambda known: known.start <= year.start + ) + return region_holder.default_array() if values is None else values + + region = region_in(period) region_default = select( [region == region_value for region_value in REGION_DEFAULT_BRMA], list(REGION_DEFAULT_BRMA.values()), default=BRMAName.MAIDSTONE, ) - if not earlier_periods: + # An input keeps applying in later years, as policyengine-core's + # auto_carry_over_input_variables does for input-only variables, + # unless the household has changed region since. + latest, latest_brma = latest_readable( + brma_holder, lambda known: known.start < period.start + ) + if latest is None: return region_default - return where(moved, region_default, household("brma", latest).decode()) + start_region = np.asarray(region_in(latest)) + moved = np.zeros(household.count, dtype=bool) + for known_period in region_holder.get_known_periods(): + if latest.start < known_period.start <= period.start: + values = region_holder.get_array(known_period, branch_name) + if values is not None: + moved |= np.asarray(values) != start_region + if not moved.any(): + return latest_brma + return where(moved, region_default, latest_brma.decode())