From 67274942eff2775184704e9025494bc3b900f5bf Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Thu, 3 Sep 2026 12:18:08 +0100 Subject: [PATCH 01/25] Warn instead of silently mislabelling a dataset's fiscal year MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `UKSingleYearDataset` took `fiscal_year=2025` as a default but used it only when building from DataFrames: with a file path the period is read from the file and the argument was ignored without comment. Both halves failed quietly. Passing a conflicting year alongside a file had no effect, and omitting it when assembling frames by hand labelled the data 2025 — already a year ahead of the newest release, and wrong differently every year. The default now warns on use and the conflicting-argument case warns too, so neither can mislabel a dataset unnoticed. Removing the default outright would break callers, so it is deprecated rather than dropped. Closes #1845 Co-Authored-By: Claude Opus 5 (1M context) --- changelog.d/dataset-fiscal-year-default.md | 1 + policyengine_uk/data/dataset_schema.py | 26 +++++++- .../tests/test_dataset_fiscal_year.py | 66 +++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 changelog.d/dataset-fiscal-year-default.md create mode 100644 policyengine_uk/tests/test_dataset_fiscal_year.py diff --git a/changelog.d/dataset-fiscal-year-default.md b/changelog.d/dataset-fiscal-year-default.md new file mode 100644 index 0000000000..e36744fc77 --- /dev/null +++ b/changelog.d/dataset-fiscal-year-default.md @@ -0,0 +1 @@ +- Warned when `UKSingleYearDataset` is given a `fiscal_year` alongside a file path, where it cannot take effect, and when it is built from DataFrames without one. diff --git a/policyengine_uk/data/dataset_schema.py b/policyengine_uk/data/dataset_schema.py index 6f294022b6..dd5e429ffc 100644 --- a/policyengine_uk/data/dataset_schema.py +++ b/policyengine_uk/data/dataset_schema.py @@ -1,3 +1,4 @@ +import warnings import pandas as pd import numpy as np from typing import TYPE_CHECKING @@ -40,16 +41,28 @@ def validate_file_path(file_path: str, raise_exception: bool = True): return True + # Used when a dataset is built from DataFrames without a fiscal year. + # Deprecated: pass fiscal_year explicitly, as this will become required. + DEFAULT_FISCAL_YEAR = 2025 + def __init__( self, file_path: str = None, person: pd.DataFrame = None, benunit: pd.DataFrame = None, household: pd.DataFrame = None, - fiscal_year: int = 2025, + fiscal_year: int = None, ): file_path = str(file_path) if file_path else None if file_path is not None: + if fiscal_year is not None: + warnings.warn( + "fiscal_year is ignored when loading from a file: the " + "time period is read from the file itself. Drop the " + "argument, or use uprate_dataset to change the period.", + UserWarning, + stacklevel=2, + ) self.validate_file_path(file_path) # Read-only: the file may be a cached download the cache manager # has made unwritable, and the default append mode would refuse it. @@ -63,6 +76,17 @@ def __init__( raise ValueError( "Must provide either a file path or all three DataFrames (person, benunit, household)." ) + if fiscal_year is None: + warnings.warn( + "fiscal_year was not given, so this dataset is labelled " + f"{self.DEFAULT_FISCAL_YEAR}. The label is the year the " + "financial year starts, so FRS 2024/25 data is 2024. " + "Pass fiscal_year explicitly; the default will be " + "removed.", + DeprecationWarning, + stacklevel=2, + ) + fiscal_year = self.DEFAULT_FISCAL_YEAR self._person = person self._benunit = benunit self._household = household diff --git a/policyengine_uk/tests/test_dataset_fiscal_year.py b/policyengine_uk/tests/test_dataset_fiscal_year.py new file mode 100644 index 0000000000..20761b913a --- /dev/null +++ b/policyengine_uk/tests/test_dataset_fiscal_year.py @@ -0,0 +1,66 @@ +"""`fiscal_year` labelling on UKSingleYearDataset. + +The year label is the year the financial year starts, so FRS 2024/25 data +is labelled 2024. A dataset loaded from a file takes its period from the +file; one built from DataFrames takes it from the argument. +""" + +import warnings + +import pandas as pd +import pytest + +from policyengine_uk.data.dataset_schema import UKSingleYearDataset + + +@pytest.fixture +def frames(): + person = pd.DataFrame( + {"person_id": [1], "person_benunit_id": [1], "person_household_id": [1]} + ) + benunit = pd.DataFrame({"benunit_id": [1]}) + household = pd.DataFrame({"household_id": [1]}) + return person, benunit, household + + +def test_fiscal_year_sets_the_time_period(frames): + person, benunit, household = frames + dataset = UKSingleYearDataset( + person=person, benunit=benunit, household=household, fiscal_year=2024 + ) + assert dataset.time_period == "2024" + + +def test_omitting_fiscal_year_warns(frames): + person, benunit, household = frames + with pytest.warns(DeprecationWarning, match="fiscal_year was not given"): + dataset = UKSingleYearDataset( + person=person, benunit=benunit, household=household + ) + assert dataset.time_period == str(UKSingleYearDataset.DEFAULT_FISCAL_YEAR) + + +def test_fiscal_year_alongside_a_file_path_warns(tmp_path, frames): + person, benunit, household = frames + file_path = tmp_path / "dataset.h5" + UKSingleYearDataset( + person=person, benunit=benunit, household=household, fiscal_year=2024 + ).save(file_path) + + with pytest.warns(UserWarning, match="ignored when loading from a file"): + dataset = UKSingleYearDataset(file_path=file_path, fiscal_year=2030) + + # The file's own period wins, so the ignored argument cannot mislabel it. + assert dataset.time_period == "2024" + + +def test_loading_without_fiscal_year_does_not_warn(tmp_path, frames): + person, benunit, household = frames + file_path = tmp_path / "dataset.h5" + UKSingleYearDataset( + person=person, benunit=benunit, household=household, fiscal_year=2024 + ).save(file_path) + + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert UKSingleYearDataset(file_path=file_path).time_period == "2024" From 00c07d6a8c33a73b05bf3ab26e6681e4ea0da071 Mon Sep 17 00:00:00 2001 From: vahid-ahmadi Date: Wed, 7 Oct 2026 14:28:37 +0100 Subject: [PATCH 02/25] Stop silencing warnings process-wide on import Three LHA variable modules called warnings.filterwarnings("ignore") at import, left over from a 2023 module that time-shifted BRMA rent tables with pandas, and bi_household_phaseout did the same inside its formula. Importing policyengine_uk therefore hid every warning in the process, including the two fiscal_year warnings this PR adds. - Remove the four blanket filters. The guarded divisions they were hiding (basic, new and additional State Pension, and the basic income household phase-out) now use np.divide(..., where=), so no 0/0 RuntimeWarning reaches callers. - Fix the three parameter files whose unknown keys now warn at import (notes -> documentation, top-level label -> metadata.label). - Raise the missing-fiscal_year deprecation as FutureWarning, which Python shows to end users by default. - Add a fresh-interpreter test under default filters for both ambiguous paths; pytest.warns overrides filters and could not catch this. - Give the changelog fragment its .fixed type. Co-Authored-By: Claude Opus 5.5 --- .../dataset-fiscal-year-default.fixed.md | 1 + changelog.d/dataset-fiscal-year-default.md | 1 - policyengine_uk/data/dataset_schema.py | 2 +- .../gov/dwp/JSA/eligibility/min_age.yaml | 2 +- .../student_loans/interest_rates/index.yaml | 3 +- .../interest_rates/plan_2/index.yaml | 3 +- .../tests/test_dataset_fiscal_year.py | 48 ++++++++++++++++++- .../basic_income/bi_household_phaseout.py | 10 ++-- .../variables/gov/dwp/LHA_allowed_bedrooms.py | 5 -- policyengine_uk/variables/gov/dwp/LHA_cap.py | 5 -- .../variables/gov/dwp/LHA_eligible.py | 5 -- .../gov/dwp/additional_state_pension.py | 9 ++-- .../variables/gov/dwp/basic_state_pension.py | 9 ++-- .../variables/gov/dwp/new_state_pension.py | 9 ++-- 14 files changed, 75 insertions(+), 37 deletions(-) create mode 100644 changelog.d/dataset-fiscal-year-default.fixed.md delete mode 100644 changelog.d/dataset-fiscal-year-default.md diff --git a/changelog.d/dataset-fiscal-year-default.fixed.md b/changelog.d/dataset-fiscal-year-default.fixed.md new file mode 100644 index 0000000000..9f1275ad38 --- /dev/null +++ b/changelog.d/dataset-fiscal-year-default.fixed.md @@ -0,0 +1 @@ +Warned when `UKSingleYearDataset` is given a `fiscal_year` alongside a file path, where it cannot take effect, and when it is built from DataFrames without one. Removed the process-wide `warnings.filterwarnings("ignore")` that importing the package installed, which hid these and every other warning from callers. diff --git a/changelog.d/dataset-fiscal-year-default.md b/changelog.d/dataset-fiscal-year-default.md deleted file mode 100644 index e36744fc77..0000000000 --- a/changelog.d/dataset-fiscal-year-default.md +++ /dev/null @@ -1 +0,0 @@ -- Warned when `UKSingleYearDataset` is given a `fiscal_year` alongside a file path, where it cannot take effect, and when it is built from DataFrames without one. diff --git a/policyengine_uk/data/dataset_schema.py b/policyengine_uk/data/dataset_schema.py index dd5e429ffc..37f0956250 100644 --- a/policyengine_uk/data/dataset_schema.py +++ b/policyengine_uk/data/dataset_schema.py @@ -83,7 +83,7 @@ def __init__( "financial year starts, so FRS 2024/25 data is 2024. " "Pass fiscal_year explicitly; the default will be " "removed.", - DeprecationWarning, + FutureWarning, stacklevel=2, ) fiscal_year = self.DEFAULT_FISCAL_YEAR diff --git a/policyengine_uk/parameters/gov/dwp/JSA/eligibility/min_age.yaml b/policyengine_uk/parameters/gov/dwp/JSA/eligibility/min_age.yaml index 7729fa752f..c387d1daa0 100644 --- a/policyengine_uk/parameters/gov/dwp/JSA/eligibility/min_age.yaml +++ b/policyengine_uk/parameters/gov/dwp/JSA/eligibility/min_age.yaml @@ -9,7 +9,7 @@ metadata: href: https://www.legislation.gov.uk/uksi/1996/207/regulation/57 - title: The Jobseeker's Allowance Regulations 1996, Schedule 1 href: https://www.legislation.gov.uk/uksi/1996/207/schedule/1 -notes: > +documentation: > This is the standard age threshold used for legacy JSA modelling and proxying. Some 16 and 17 year olds can qualify under special cases in regulation 57. values: diff --git a/policyengine_uk/parameters/gov/hmrc/student_loans/interest_rates/index.yaml b/policyengine_uk/parameters/gov/hmrc/student_loans/interest_rates/index.yaml index 0d122a1e7c..50bf8f1a10 100644 --- a/policyengine_uk/parameters/gov/hmrc/student_loans/interest_rates/index.yaml +++ b/policyengine_uk/parameters/gov/hmrc/student_loans/interest_rates/index.yaml @@ -1,5 +1,6 @@ description: Student loan interest rates. -label: Interest rates +metadata: + label: Interest rates documentation: | Interest rates applied to student loan balances vary by plan type. Plan 2 has income-contingent rates that vary between RPI and RPI+3%. diff --git a/policyengine_uk/parameters/gov/hmrc/student_loans/interest_rates/plan_2/index.yaml b/policyengine_uk/parameters/gov/hmrc/student_loans/interest_rates/plan_2/index.yaml index 0ff4c1ab69..aae6ba7a54 100644 --- a/policyengine_uk/parameters/gov/hmrc/student_loans/interest_rates/plan_2/index.yaml +++ b/policyengine_uk/parameters/gov/hmrc/student_loans/interest_rates/plan_2/index.yaml @@ -1,5 +1,6 @@ description: Plan 2 student loan interest rates. -label: Plan 2 interest rates +metadata: + label: Plan 2 interest rates documentation: | Plan 2 student loan interest rates are income-contingent after graduation. diff --git a/policyengine_uk/tests/test_dataset_fiscal_year.py b/policyengine_uk/tests/test_dataset_fiscal_year.py index 20761b913a..abdbafa419 100644 --- a/policyengine_uk/tests/test_dataset_fiscal_year.py +++ b/policyengine_uk/tests/test_dataset_fiscal_year.py @@ -5,6 +5,10 @@ file; one built from DataFrames takes it from the argument. """ +import os +import subprocess +import sys +import textwrap import warnings import pandas as pd @@ -33,7 +37,7 @@ def test_fiscal_year_sets_the_time_period(frames): def test_omitting_fiscal_year_warns(frames): person, benunit, household = frames - with pytest.warns(DeprecationWarning, match="fiscal_year was not given"): + with pytest.warns(FutureWarning, match="fiscal_year was not given"): dataset = UKSingleYearDataset( person=person, benunit=benunit, household=household ) @@ -64,3 +68,45 @@ def test_loading_without_fiscal_year_does_not_warn(tmp_path, frames): with warnings.catch_warnings(): warnings.simplefilter("error") assert UKSingleYearDataset(file_path=file_path).time_period == "2024" + + +# pytest.warns overrides the active filters, so the tests above would pass +# even if importing the package silenced warnings process-wide. Run a fresh +# interpreter under Python's default filters, as an ordinary caller would. +FRESH_PROCESS_SCRIPT = textwrap.dedent( + """ + import sys + import tempfile + from pathlib import Path + + import pandas as pd + + import policyengine_uk # noqa: F401 (import side effects are under test) + from policyengine_uk.data.dataset_schema import UKSingleYearDataset + + frames = dict( + person=pd.DataFrame( + {"person_id": [1], "person_benunit_id": [1], "person_household_id": [1]} + ), + benunit=pd.DataFrame({"benunit_id": [1]}), + household=pd.DataFrame({"household_id": [1]}), + ) + UKSingleYearDataset(**frames) + with tempfile.TemporaryDirectory() as directory: + file_path = Path(directory) / "dataset.h5" + UKSingleYearDataset(**frames, fiscal_year=2024).save(file_path) + UKSingleYearDataset(file_path=file_path, fiscal_year=2030) + """ +) + + +def test_warnings_reach_a_fresh_process_under_default_filters(): + result = subprocess.run( + [sys.executable, "-c", FRESH_PROCESS_SCRIPT], + capture_output=True, + text=True, + env={k: v for k, v in os.environ.items() if k != "PYTHONWARNINGS"}, + ) + assert result.returncode == 0, result.stderr + assert "FutureWarning: fiscal_year was not given" in result.stderr + assert "UserWarning: fiscal_year is ignored when loading" in result.stderr diff --git a/policyengine_uk/variables/contrib/ubi_center/basic_income/bi_household_phaseout.py b/policyengine_uk/variables/contrib/ubi_center/basic_income/bi_household_phaseout.py index 1edff68552..798d2d59fe 100644 --- a/policyengine_uk/variables/contrib/ubi_center/basic_income/bi_household_phaseout.py +++ b/policyengine_uk/variables/contrib/ubi_center/basic_income/bi_household_phaseout.py @@ -1,5 +1,4 @@ from policyengine_uk.model_api import * -import warnings class bi_household_phaseout(Variable): @@ -27,7 +26,10 @@ def formula(person, period, parameters): uncapped_deduction = bi.phase_out.household.rate * income_over_threshold capped_deduction = min_(household_bi, uncapped_deduction) - warnings.filterwarnings("ignore") - - percent_reduction = where(household_bi > 0, capped_deduction / household_bi, 0) + percent_reduction = np.divide( + capped_deduction, + household_bi, + out=np.zeros_like(household_bi, dtype=float), + where=household_bi > 0, + ) return percent_reduction * remaining_bi diff --git a/policyengine_uk/variables/gov/dwp/LHA_allowed_bedrooms.py b/policyengine_uk/variables/gov/dwp/LHA_allowed_bedrooms.py index ce8e5543af..4c87a0df20 100644 --- a/policyengine_uk/variables/gov/dwp/LHA_allowed_bedrooms.py +++ b/policyengine_uk/variables/gov/dwp/LHA_allowed_bedrooms.py @@ -1,9 +1,4 @@ from policyengine_uk.model_api import * -import pandas as pd -import warnings -from policyengine_core.model_api import * - -warnings.filterwarnings("ignore") def child_bedrooms(boys_under_10, boys_10_to_15, girls_under_10, girls_10_to_15): diff --git a/policyengine_uk/variables/gov/dwp/LHA_cap.py b/policyengine_uk/variables/gov/dwp/LHA_cap.py index af611c409d..3b26f2ba9e 100644 --- a/policyengine_uk/variables/gov/dwp/LHA_cap.py +++ b/policyengine_uk/variables/gov/dwp/LHA_cap.py @@ -1,9 +1,4 @@ from policyengine_uk.model_api import * -import pandas as pd -import warnings -from policyengine_core.model_api import * - -warnings.filterwarnings("ignore") class LHA_cap(Variable): diff --git a/policyengine_uk/variables/gov/dwp/LHA_eligible.py b/policyengine_uk/variables/gov/dwp/LHA_eligible.py index 008e3f822b..662b36e484 100644 --- a/policyengine_uk/variables/gov/dwp/LHA_eligible.py +++ b/policyengine_uk/variables/gov/dwp/LHA_eligible.py @@ -1,9 +1,4 @@ from policyengine_uk.model_api import * -import pandas as pd -import warnings -from policyengine_core.model_api import * - -warnings.filterwarnings("ignore") class LHA_eligible(Variable): diff --git a/policyengine_uk/variables/gov/dwp/additional_state_pension.py b/policyengine_uk/variables/gov/dwp/additional_state_pension.py index 932e7ff563..a476661b78 100644 --- a/policyengine_uk/variables/gov/dwp/additional_state_pension.py +++ b/policyengine_uk/variables/gov/dwp/additional_state_pension.py @@ -69,10 +69,11 @@ def formula(person, period, parameters): max_(reported - max_for_type_data, 0), 0, ) - uprating = where( - max_for_type_data > 0, - max_for_type_period / max_for_type_data, - 1, + uprating = np.divide( + max_for_type_period, + max_for_type_data, + out=np.ones_like(max_for_type_data, dtype=float), + where=max_for_type_data > 0, ) # No State Pension is paid before State Pension age. A computed type # is already NONE there; this also holds if the type is an input. diff --git a/policyengine_uk/variables/gov/dwp/basic_state_pension.py b/policyengine_uk/variables/gov/dwp/basic_state_pension.py index 23d4a7a883..48ac56fb13 100644 --- a/policyengine_uk/variables/gov/dwp/basic_state_pension.py +++ b/policyengine_uk/variables/gov/dwp/basic_state_pension.py @@ -43,10 +43,11 @@ def formula(person, period, parameters): # Compute the person's share of the data-year maximum so reforms can # scale the current-period amount while preserving the original cap. - share = where( - max_sp_data_year > 0, - min_(reported, max_sp_data_year) / max_sp_data_year, - 0, + share = np.divide( + min_(reported, max_sp_data_year), + max_sp_data_year, + out=np.zeros_like(reported, dtype=float), + where=max_sp_data_year > 0, ) return ( where( diff --git a/policyengine_uk/variables/gov/dwp/new_state_pension.py b/policyengine_uk/variables/gov/dwp/new_state_pension.py index f26e4a9cfd..69c9a3bfce 100644 --- a/policyengine_uk/variables/gov/dwp/new_state_pension.py +++ b/policyengine_uk/variables/gov/dwp/new_state_pension.py @@ -48,9 +48,10 @@ def formula(person, period, parameters): # flows through ``additional_state_pension`` — mirrors the # BASIC / ASP split so ``new_state_pension`` is always the headline # flat-rate component. - share = where( - max_new_data_year > 0, - min_(reported_weekly, max_new_data_year) / max_new_data_year, - 0, + share = np.divide( + min_(reported_weekly, max_new_data_year), + max_new_data_year, + out=np.zeros_like(reported_weekly, dtype=float), + where=max_new_data_year > 0, ) return eligible * share * max_new_period * WEEKS_IN_YEAR From 583b3d20ff7d7beadc1305b10b8ff0302e0ca7f7 Mon Sep 17 00:00:00 2001 From: vahid-ahmadi Date: Thu, 8 Oct 2026 12:05:19 +0100 Subject: [PATCH 03/25] Put road-fuel litre divisors on DESNZ pump prices and re-derive the litre proxies (#2169 part A) petrol_litres and diesel_litres divided spending by RAC prices dated 1 January 2024, held flat, while Microcosm UK prices household fuel spending on DESNZ calendar-year averages. The divisors now hold the DESNZ QEP table 4.1.2 ULSP and ULSD annual averages for 2010-2025, a provisional January-September 2026 average from table 4.1.1, and are flat in cash terms afterwards. The petrol and diesel spending litre proxies are re-derived from the new prices with the existing formula, (1 + volume) x price_t / price_t-1 / (1 + population) - 1, so weighted litres still follow the HMRC/OBR road-fuel volume path. A new test fails if the proxy and the prices drift apart. On the certified Microcosm UK 2024-25 file, 2025 litres are 23.699bn (the joint sensitivity in the issue), year-on-year litres growth is unchanged, and household fuel duty rises by about 0.25bn a year. Co-Authored-By: Claude Opus 5.5 --- changelog.d/2169.fixed.md | 1 + docs/book/assumptions/growthfactors.md | 2 + .../gov/economic_assumptions/yoy_growth.yaml | 36 ++++++------ .../consumption/fuel/prices/diesel.yaml | 40 +++++++++---- .../consumption/fuel/prices/petrol.yaml | 41 +++++++++---- .../consumption/road_fuel_litres.yaml | 46 +++++++++++++++ .../tests/test_road_fuel_prices.py | 58 +++++++++++++++++++ 7 files changed, 183 insertions(+), 41 deletions(-) create mode 100644 changelog.d/2169.fixed.md create mode 100644 policyengine_uk/tests/policy/baseline/consumption/road_fuel_litres.yaml create mode 100644 policyengine_uk/tests/test_road_fuel_prices.py diff --git a/changelog.d/2169.fixed.md b/changelog.d/2169.fixed.md new file mode 100644 index 0000000000..b8ff4b838d --- /dev/null +++ b/changelog.d/2169.fixed.md @@ -0,0 +1 @@ +Put the petrol and diesel pump prices that convert spending into litres on the DESNZ annual ULSP and ULSD series (the basis Microcosm UK prices fuel spending on), replacing the January 2024 RAC prices, and re-derived the petrol and diesel spending litre proxies so weighted litres still follow the HMRC/OBR road-fuel volume path (part A of #2169). diff --git a/docs/book/assumptions/growthfactors.md b/docs/book/assumptions/growthfactors.md index 1a1d52e1ee..5871432b7b 100644 --- a/docs/book/assumptions/growthfactors.md +++ b/docs/book/assumptions/growthfactors.md @@ -32,6 +32,8 @@ We use HMRC road-fuel clearances and OBR-implied forecast volumes for petrol and We apply this to: `diesel_spending`, `petrol_spending` +The pump prices that turn spending into litres (`household.consumption.fuel.prices.petrol` and `.diesel`) are DESNZ calendar-year averages (Quarterly Energy Prices tables 4.1.1 and 4.1.2), the same basis Microcosm UK uses to price household fuel spending. The spending growth rates (`petrol_spending_litre_proxy`, `diesel_spending_litre_proxy`) are derived from those prices, so that weighted litres follow the volume path above: (1 + growth) = (1 + volume growth) × pricet / pricet−1 ÷ (1 + population growth). Whenever the prices change, re-derive the growth rates; a test checks that they match. + ## Average earnings ```{note} diff --git a/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml b/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml index 01fa695891..e21a645890 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml +++ b/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml @@ -345,14 +345,14 @@ obr: href: https://obr.uk/docs/dlm_uploads/Fuel-duty-supplementary-release_receipts-by-vehicle-type.pdf petrol_spending_litre_proxy: - description: Unleaded-petrol spending proxy growth that preserves weighted road-fuel litres after dividing by model pump prices. + description: Unleaded-petrol spending proxy growth that preserves weighted road-fuel litres after dividing by model pump prices. Derived as (1 + road_fuel_volume) x (price_t / price_t-1) / (1 + population) - 1, using household.consumption.fuel.prices; re-derive whenever those prices change. values: - 2021-01-01: 0.09966723889828133 - 2022-01-01: 0.47125942029399615 - 2023-01-01: 0.09943596417596279 - 2024-01-01: -0.14810324567590538 - 2025-01-01: -0.0269132439679719 - 2026-01-01: -0.018120813771079436 + 2021-01-01: 0.4276130160591334 + 2022-01-01: 0.32114330849740713 + 2023-01-01: -0.11978022401038202 + 2024-01-01: -0.05378435951558702 + 2025-01-01: -0.07097011631498962 + 2026-01-01: 0.09631434800374827 2027-01-01: -0.027765858532455168 2028-01-01: -0.030878786058031737 2029-01-01: -0.04149818363243518 @@ -408,18 +408,18 @@ obr: href: https://www.gov.uk/government/statistics/hydrocarbon-oils-bulletin - title: OBR EFO March 2026 (fuel-duty receipts forecast) href: https://obr.uk/efo/economic-and-fiscal-outlook-march-2026/ - - title: RAC latest average petrol price - href: https://www.rac.co.uk/drive/advice/fuel-watch/ + - title: DESNZ annual average ULSP pump price (household.consumption.fuel.prices.petrol) + href: https://www.gov.uk/government/statistical-data-sets/oil-and-petroleum-products-monthly-statistics diesel_spending_litre_proxy: - description: Diesel spending proxy growth that preserves weighted road-fuel litres after dividing by model pump prices. + description: Diesel spending proxy growth that preserves weighted road-fuel litres after dividing by model pump prices. Derived as (1 + road_fuel_volume) x (price_t / price_t-1) / (1 + population) - 1, using household.consumption.fuel.prices; re-derive whenever those prices change. values: - 2021-01-01: 0.13390420538515913 - 2022-01-01: 0.3702255435452164 - 2023-01-01: -0.01858407316867594 - 2024-01-01: 0.03164670292538285 - 2025-01-01: -0.0269132439679719 - 2026-01-01: -0.018120813771079436 + 2021-01-01: 0.4035933750476013 + 2022-01-01: 0.38608334996374616 + 2023-01-01: -0.12613670608493466 + 2024-01-01: -0.07344928074891444 + 2025-01-01: -0.06483399761456943 + 2026-01-01: 0.1748436772712496 2027-01-01: -0.027765858532455168 2028-01-01: -0.030878786058031737 2029-01-01: -0.04149818363243518 @@ -475,8 +475,8 @@ obr: href: https://www.gov.uk/government/statistics/hydrocarbon-oils-bulletin - title: OBR EFO March 2026 (fuel-duty receipts forecast) href: https://obr.uk/efo/economic-and-fiscal-outlook-march-2026/ - - title: RAC latest average diesel price - href: https://www.rac.co.uk/drive/advice/fuel-watch/ + - title: DESNZ annual average ULSD pump price (household.consumption.fuel.prices.diesel) + href: https://www.gov.uk/government/statistical-data-sets/oil-and-petroleum-products-monthly-statistics consumer_price_index_ahc: description: Consumer price index year-on-year growth, modified to remove housing costs. diff --git a/policyengine_uk/parameters/household/consumption/fuel/prices/diesel.yaml b/policyengine_uk/parameters/household/consumption/fuel/prices/diesel.yaml index 992f1f1e40..16141abd1f 100644 --- a/policyengine_uk/parameters/household/consumption/fuel/prices/diesel.yaml +++ b/policyengine_uk/parameters/household/consumption/fuel/prices/diesel.yaml @@ -1,17 +1,35 @@ -description: Average price of diesel per litre. +description: Average pump price of diesel (ULSD) per litre, including fuel duty and VAT. Calendar-year average of DESNZ monthly typical retail prices; the same basis Microcosm UK uses to price household diesel spending. values: - 2012-04-01: 1.4052 - 2014-09-15: 1.3339 - 2017-05-01: 1.2225 - 2020-05-14: 1.1186 - 2022-01-01: 1.4559 - 2024-01-01: 1.52 + 2010-01-01: 1.19258627 + 2011-01-01: 1.38716127 + 2012-01-01: 1.41828260 + 2013-01-01: 1.40405189 + 2014-01-01: 1.33457999 + 2015-01-01: 1.14898456 + 2016-01-01: 1.10128630 + 2017-01-01: 1.20149237 + 2018-01-01: 1.29982166 + 2019-01-01: 1.31475463 + 2020-01-01: 1.19135223 + 2021-01-01: 1.34937007 + 2022-01-01: 1.77658132 + 2023-01-01: 1.58188711 + 2024-01-01: 1.48328885 + 2025-01-01: 1.42548575 + # Provisional: mean of the January to September 2026 monthly prices + # (DESNZ table 4.1.1, published 29 September 2026). Replace with the + # calendar-year average from table 4.1.2 when DESNZ publishes it, and + # re-derive the diesel_spending_litre_proxy growth series. + 2026-01-01: 1.70563033 metadata: unit: currency-GBP label: Price of diesel per litre economy: false + # Held flat in cash terms after the last DESNZ year. Litres and fuel duty + # do not depend on this projection: the spending litre proxy is derived + # from these prices, so later price changes cancel out of spending / price. reference: - - title: RAC - href: https://www.rac.co.uk/drive/advice/fuel-watch/ - - title: RAC Foundation historical averages - href: https://www.racfoundation.org/data/uk-pump-prices-over-time + - title: DESNZ, Quarterly Energy Prices table 4.1.2, annual typical retail prices of petroleum products (ULSD) + href: https://www.gov.uk/government/statistical-data-sets/oil-and-petroleum-products-monthly-statistics + - title: DESNZ, Quarterly Energy Prices table 4.1.1, monthly typical retail prices (ULSD), January to September 2026 + href: https://www.gov.uk/government/statistical-data-sets/oil-and-petroleum-products-monthly-statistics diff --git a/policyengine_uk/parameters/household/consumption/fuel/prices/petrol.yaml b/policyengine_uk/parameters/household/consumption/fuel/prices/petrol.yaml index 6ff14ca293..f701f127f8 100644 --- a/policyengine_uk/parameters/household/consumption/fuel/prices/petrol.yaml +++ b/policyengine_uk/parameters/household/consumption/fuel/prices/petrol.yaml @@ -1,18 +1,35 @@ -description: Average price of unleaded petrol per litre, including fuel duty. +description: Average pump price of unleaded petrol (ULSP) per litre, including fuel duty and VAT. Calendar-year average of DESNZ monthly typical retail prices; the same basis Microcosm UK uses to price household petrol spending. values: - 2012-04-01: 1.3492 - 2014-09-15: 1.2918 - 2017-05-01: 1.2023 - 2020-05-14: 1.0669 - 2022-01-01: 1.491 - 2022-03-22: 1.6703 - 2024-01-01: 1.44 + 2010-01-01: 1.16902571 + 2011-01-01: 1.33268790 + 2012-01-01: 1.35390547 + 2013-01-01: 1.34145278 + 2014-01-01: 1.27495853 + 2015-01-01: 1.11130760 + 2016-01-01: 1.08845640 + 2017-01-01: 1.17588883 + 2018-01-01: 1.25195971 + 2019-01-01: 1.24877998 + 2020-01-01: 1.13947293 + 2021-01-01: 1.31269579 + 2022-01-01: 1.64732274 + 2023-01-01: 1.47746334 + 2024-01-01: 1.41477665 + 2025-01-01: 1.35072210 + # Provisional: mean of the January to September 2026 monthly prices + # (DESNZ table 4.1.1, published 29 September 2026). Replace with the + # calendar-year average from table 4.1.2 when DESNZ publishes it, and + # re-derive the petrol_spending_litre_proxy growth series. + 2026-01-01: 1.50814483 metadata: unit: currency-GBP label: Price of unleaded petrol per litre economy: false + # Held flat in cash terms after the last DESNZ year. Litres and fuel duty + # do not depend on this projection: the spending litre proxy is derived + # from these prices, so later price changes cancel out of spending / price. reference: - - title: RAC latest average - href: https://www.rac.co.uk/drive/advice/fuel-watch/ - - title: RAC Foundation historical averages - href: https://www.racfoundation.org/data/uk-pump-prices-over-time + - title: DESNZ, Quarterly Energy Prices table 4.1.2, annual typical retail prices of petroleum products (ULSP) + href: https://www.gov.uk/government/statistical-data-sets/oil-and-petroleum-products-monthly-statistics + - title: DESNZ, Quarterly Energy Prices table 4.1.1, monthly typical retail prices (ULSP), January to September 2026 + href: https://www.gov.uk/government/statistical-data-sets/oil-and-petroleum-products-monthly-statistics diff --git a/policyengine_uk/tests/policy/baseline/consumption/road_fuel_litres.yaml b/policyengine_uk/tests/policy/baseline/consumption/road_fuel_litres.yaml new file mode 100644 index 0000000000..5ef8ff3170 --- /dev/null +++ b/policyengine_uk/tests/policy/baseline/consumption/road_fuel_litres.yaml @@ -0,0 +1,46 @@ +# Litres divide spending by the DESNZ calendar-year average pump price +# (QEP table 4.1.2): 141.477665p ULSP and 148.328885p diesel in 2024, +# 135.072210p and 142.548575p in 2025. Microcosm UK prices household +# petrol and diesel spending on the same basis (PolicyEngine/policyengine-uk#2169). +- name: 2024 petrol spending at the DESNZ ULSP price buys 1,000 litres + period: 2024 + absolute_error_margin: 0.01 + input: + people: + adult: + age: 40 + benunits: + benunit: + members: [adult] + households: + household: + members: [adult] + petrol_spending: 1_414.77665 + diesel_spending: 1_483.28885 + output: + petrol_price: 1.41477665 + diesel_price: 1.48328885 + petrol_litres: 1_000 + diesel_litres: 1_000 + +- name: 2025 litres use the 2025 DESNZ prices, not the January 2024 RAC prices + period: 2025 + absolute_error_margin: 0.01 + input: + people: + adult: + age: 40 + benunits: + benunit: + members: [adult] + households: + household: + members: [adult] + petrol_spending: 1_350.7221 + diesel_spending: 1_425.48575 + output: + petrol_price: 1.3507221 + diesel_price: 1.42548575 + # Under the old £1.44 / £1.52 divisors these would be 938.0 and 937.8. + petrol_litres: 1_000 + diesel_litres: 1_000 diff --git a/policyengine_uk/tests/test_road_fuel_prices.py b/policyengine_uk/tests/test_road_fuel_prices.py new file mode 100644 index 0000000000..ab0dd05262 --- /dev/null +++ b/policyengine_uk/tests/test_road_fuel_prices.py @@ -0,0 +1,58 @@ +from pathlib import Path + +import pytest +import yaml + +import policyengine_uk +from policyengine_uk.system import system + +PRICES = system.parameters.household.consumption.fuel.prices +YOY = system.parameters.gov.economic_assumptions.yoy_growth + + +@pytest.mark.parametrize( + "year, petrol_pence, diesel_pence", + [ + # DESNZ Quarterly Energy Prices table 4.1.2, annual typical retail + # prices (ULSP, ULSD), the series Microcosm UK uses to price spending. + (2023, 147.746334, 158.188711), + (2024, 141.477665, 148.328885), + (2025, 135.072210, 142.548575), + ], +) +def test_pump_prices_are_desnz_annual_averages(year, petrol_pence, diesel_pence): + assert PRICES.petrol(year) * 100 == pytest.approx(petrol_pence, abs=1e-5) + assert PRICES.diesel(year) * 100 == pytest.approx(diesel_pence, abs=1e-5) + + +@pytest.mark.parametrize("fuel", ["petrol", "diesel"]) +def test_spending_litre_proxy_is_derived_from_the_pump_prices(fuel): + """The proxy must be re-derived whenever the pump prices change. + + Spending grows by the proxy, household weights by population, and + litres are spending divided by the price, so weighted litres follow + the HMRC/OBR road-fuel volume path only if + (1 + proxy) = (1 + volume) x price_t / price_t-1 / (1 + population). + """ + proxy = getattr(YOY.obr, f"{fuel}_spending_litre_proxy") + price = getattr(PRICES, fuel) + volume = YOY.obr.road_fuel_volume + population = YOY.ons.population + # The years the YAML itself declares; the loaded parameter also carries + # values the system adds before the first declared date. + source = Path(policyengine_uk.__file__).parent / ( + "parameters/gov/economic_assumptions/yoy_growth.yaml" + ) + declared = yaml.safe_load(source.read_text())["obr"][ + f"{fuel}_spending_litre_proxy" + ]["values"] + years = sorted(int(str(date)[:4]) for date in declared) + assert years[0] == 2021 + for year in years: + expected = (1 + volume(year)) * price(year) / price(year - 1) / ( + 1 + population(year) + ) - 1 + assert proxy(year) == pytest.approx(expected, abs=1e-12), ( + f"{fuel}_spending_litre_proxy for {year} no longer matches the " + f"pump prices; re-derive it from household.consumption.fuel.prices." + ) From c53eb90582d1bf76acb44ac7f6e1b1516e19200e Mon Sep 17 00:00:00 2001 From: vahid-ahmadi Date: Thu, 8 Oct 2026 15:46:09 +0100 Subject: [PATCH 04/25] Pick up new documentation pages by folder Add pattern entries to the myst.yml table of contents for the assumptions, validation and policy folders and a Programmes section for docs/book/programs/gov//*.md, so a new Markdown page appears without editing myst.yml. Explicit entries now carry the .md extension so the patterns don't add them twice, and the Policy section's two book pages come from the pattern (same order). Notebooks are not pattern-matched. Co-Authored-By: Claude Opus 5.5 --- changelog.d/docs-toc-patterns.changed.md | 1 + myst.yml | 25 +++++++++++++++--------- 2 files changed, 17 insertions(+), 9 deletions(-) create mode 100644 changelog.d/docs-toc-patterns.changed.md diff --git a/changelog.d/docs-toc-patterns.changed.md b/changelog.d/docs-toc-patterns.changed.md new file mode 100644 index 0000000000..9b7931a53e --- /dev/null +++ b/changelog.d/docs-toc-patterns.changed.md @@ -0,0 +1 @@ +Documentation pages in docs/book/assumptions, validation, policy and programs/gov/ are now picked up by folder, so a new page no longer needs a myst.yml entry. This also publishes eight existing pages that were missing from the contents. diff --git a/myst.yml b/myst.yml index 43cc808e87..b5b3053685 100644 --- a/myst.yml +++ b/myst.yml @@ -13,23 +13,30 @@ project: - file: docs/book/index.md - title: Usage children: - - file: docs/book/usage/getting-started - - file: docs/book/usage/simulations - - file: docs/book/usage/scenarios - - file: docs/book/usage/dynamics + - file: docs/book/usage/getting-started.md + - file: docs/book/usage/simulations.md + - file: docs/book/usage/scenarios.md + - file: docs/book/usage/dynamics.md + # Pages listed by `file:` keep their place at the top of a section. Any other + # Markdown page in a section's folder is added after them by `pattern:`, so a + # new page needs no change here. Notebooks are not pattern-matched. - title: Assumptions children: - - file: docs/book/assumptions/growthfactors - - file: docs/book/assumptions/water-bills + - file: docs/book/assumptions/growthfactors.md + - file: docs/book/assumptions/water-bills.md + - pattern: docs/book/assumptions/*.md - title: Validation children: - - file: docs/book/validation/hbai + - file: docs/book/validation/hbai.md + - pattern: docs/book/validation/*.md - title: Policy children: - file: docs/engineering/excise-duties.md - file: docs/engineering/bus-fares.md - - file: docs/book/policy/model-baseline - - file: docs/book/policy/uc-rebalancing + - pattern: docs/book/policy/*.md + - title: Programmes + children: + - pattern: docs/book/programs/gov/*/*.md site: options: logo: docs/book/logo.png From 77dae439e7e9b9d4ac47efedee32b5dbbcde799f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 16:59:48 -0400 Subject: [PATCH 05/25] Split the Test job into a YAML job and three shards of the Python tests The Test job took 59 minutes (run 38075384588): 2.6 minutes of YAML tests, then 55.8 minutes of pytest on four balanced xdist workers, 220 worker-minutes in all. No test dominates: 51 property-test files take 74% of it and the slowest test takes 3.5%. So the job is split rather than one test fixed. - `--shard K/N` (tests/conftest.py) keeps the tests whose node id hashes to shard K of N. The shard is a function of the node id alone, so every collected test is in exactly one shard, including tests added later. - The workflows run the YAML tests in one job and the Python tests as a matrix of three shards, N taken from the size of the matrix. - A final `Test` job runs .github/check_test_shards.py over the shards' JUnit files and an unsharded `pytest --collect-only -q` listing. It fails unless every collected test ran in exactly one shard and none failed. - Coverage is measured with sys.monitoring (COVERAGE_CORE=sysmon) instead of the C trace function. On one property-test file the two report the same 12,262 of 18,401 lines, and sys.monitoring adds 6% to the CPU time where the trace function adds 33%. Refs #2099 Co-Authored-By: Claude Opus 5.5 --- .github/check_test_shards.py | 195 ++++++++++ .github/workflows/code_changes.yaml | 114 +++++- .github/workflows/pr_code_changes.yaml | 114 +++++- Makefile | 6 +- changelog.d/ci-test-runtime-2099.fixed.md | 1 + docs/engineering/skills/testing.md | 21 ++ .../tests/code_health/test_ci_test_shards.py | 332 ++++++++++++++++++ policyengine_uk/tests/conftest.py | 57 +++ 8 files changed, 817 insertions(+), 23 deletions(-) create mode 100644 .github/check_test_shards.py create mode 100644 changelog.d/ci-test-runtime-2099.fixed.md create mode 100644 policyengine_uk/tests/code_health/test_ci_test_shards.py diff --git a/.github/check_test_shards.py b/.github/check_test_shards.py new file mode 100644 index 0000000000..69f40fa83f --- /dev/null +++ b/.github/check_test_shards.py @@ -0,0 +1,195 @@ +"""Check that the shards of the Test job ran every test exactly once. + +CI splits ``pytest policyengine_uk/tests`` over several runners with +``--shard K/N`` (see policyengine_uk/tests/conftest.py). This script compares +what the shards ran with what an unsharded ``pytest --collect-only -q`` lists: + + python .github/check_test_shards.py \ + --collected collected.txt --junit junit-1.xml junit-2.xml junit-3.xml + +It fails if a collected test is missing from every shard, ran in more than one, +ran without being collected, or failed. So splitting the suite cannot drop a +test without the Test job going red. + +Standard library only: the job that runs it does not install the package. +""" + +import argparse +import re +import sys +import xml.etree.ElementTree as ElementTree +from collections import Counter +from pathlib import Path + +_COLLECTED_SUMMARY = re.compile(r"^(\d+) tests? collected\b(.*)$") +_NOTHING_COLLECTED = re.compile(r"^no tests collected\b") + + +class ShardCheckError(Exception): + """An input that cannot be checked, as opposed to a check that failed.""" + + +def junit_key(nodeid: str) -> tuple[str, str]: + """The ``(classname, name)`` pytest gives ``nodeid`` in JUnit XML. + + Follows _pytest.junitxml.mangle_test_address; a test compares the two. + pytest also rewrites characters that XML does not allow, such as control + characters. Its node ids do not contain them, because it escapes them in + parameter ids. If one ever did, the names would differ and the check + would fail, not pass. + """ + path, bracket, params = nodeid.partition("[") + names = path.split("::") + names[0] = re.sub(r"\.py$", "", names[0].replace("/", ".")) + names[-1] += bracket + params + return ".".join(names[:-1]), names[-1] + + +def read_collected(text: str) -> list[str]: + """Node ids from the output of ``pytest --collect-only -q``. + + The listing is one node id per line, a blank line, then any warnings and + ``N tests collected``. The ids found must number N, so a listing this + cannot parse is an error, never a shorter list. + """ + ids = [] + total = None + listing_ended = False + for line in text.splitlines(): + summary = _COLLECTED_SUMMARY.match(line) + if summary: + if "error" in summary.group(2): + raise ShardCheckError(f"collection reported errors: {line!r}") + total = int(summary.group(1)) + elif _NOTHING_COLLECTED.match(line): + total = 0 + elif not line.strip(): + listing_ended = listing_ended or bool(ids) + elif "::" in line and not listing_ended: + ids.append(line) + if total is None: + raise ShardCheckError("no 'N tests collected' line in the collection listing") + if len(ids) != total: + raise ShardCheckError( + f"the collection listing says {total} tests but lists {len(ids)}" + ) + if len(set(ids)) != len(ids): + raise ShardCheckError("the collection listing repeats a node id") + return ids + + +def read_junit(path: Path) -> list[tuple[tuple[str, str], str]]: + """``((classname, name), outcome)`` for each test case in a JUnit file. + + The outcome is ``failed`` for a failure or an error, else ``skipped`` for a + skip or an expected failure, else ``passed``. + """ + cases = [] + for case in ElementTree.parse(path).getroot().iter("testcase"): + children = {child.tag for child in case} + if children & {"failure", "error"}: + outcome = "failed" + elif "skipped" in children: + outcome = "skipped" + else: + outcome = "passed" + cases.append(((case.get("classname", ""), case.get("name", "")), outcome)) + return cases + + +def check(collected: list[str], shards: list[list[tuple[tuple[str, str], str]]]): + """Compare the collected node ids with what each shard ran. + + Returns ``(problems, summary)``: ``problems`` is empty when every collected + test ran in exactly one shard and none failed. + """ + expected = {} + for nodeid in collected: + key = junit_key(nodeid) + if key in expected: + raise ShardCheckError( + f"{nodeid!r} and {expected[key]!r} have the same JUnit name" + ) + expected[key] = nodeid + + runs = Counter() + outcomes = Counter() + failed = [] + for number, cases in enumerate(shards, start=1): + for key, outcome in cases: + runs[key] += 1 + outcomes[outcome] += 1 + if outcome == "failed": + failed.append((number, key)) + + def name(key): + return expected.get(key) or f"{key[0]}::{key[1]}" + + problems = [] + missing = sorted(expected[key] for key in expected if key not in runs) + if missing: + problems.append((f"{len(missing)} collected tests ran in no shard", missing)) + repeated = sorted(name(key) for key, count in runs.items() if count > 1) + if repeated: + problems.append((f"{len(repeated)} tests ran more than once", repeated)) + extra = sorted(name(key) for key in runs if key not in expected) + if extra: + problems.append((f"{len(extra)} tests ran but were not collected", extra)) + if failed: + problems.append( + ( + f"{len(failed)} tests failed", + [f"shard {number}: {name(key)}" for number, key in failed], + ) + ) + + sizes = "/".join(str(len(cases)) for cases in shards) + summary = ( + f"{len(collected)} tests collected; {len(shards)} shards ran " + f"{sizes} = {sum(len(cases) for cases in shards)} " + f"({outcomes['passed']} passed, {outcomes['skipped']} skipped or " + f"expected failures, {outcomes['failed']} failed)" + ) + return problems, summary + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--collected", + required=True, + type=Path, + help="output of an unsharded `pytest --collect-only -q`", + ) + parser.add_argument( + "--junit", + required=True, + nargs="+", + type=Path, + help="the JUnit XML file of each shard", + ) + args = parser.parse_args(argv) + + try: + collected = read_collected(args.collected.read_text(encoding="utf-8")) + shards = [read_junit(path) for path in args.junit] + problems, summary = check(collected, shards) + except (ShardCheckError, OSError, ElementTree.ParseError) as error: + print(f"Cannot check the shards: {error}", file=sys.stderr) + return 2 + + print(summary) + for title, names in problems: + print(f"\n{title}:") + for line in names[:50]: + print(f" {line}") + if len(names) > 50: + print(f" ... and {len(names) - 50} more") + if problems: + return 1 + print("Every collected test ran in exactly one shard.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/code_changes.yaml b/.github/workflows/code_changes.yaml index 5ba890a038..bd414330c1 100644 --- a/.github/workflows/code_changes.yaml +++ b/.github/workflows/code_changes.yaml @@ -31,19 +31,79 @@ jobs: uses: astral-sh/setup-uv@v8.1.0 - name: Check formatting run: uvx ruff format --check . - Test: + # The tests run as four jobs: the YAML policy tests, and the Python tests + # split over three runners. As one job they took 59 minutes on 10 October + # 2026 (run 38075384588: 2.6 minutes of YAML tests, then 55.8 minutes of + # pytest on four workers), and runs were cancelled at a 60-minute limit + # (issues #2099 and #2257). + Test-YAML: + name: Test (YAML) # Linux arm64 keeps the CPU architecture of the macOS runners this job # used to run on. GitHub-hosted macOS runners are scarce, so PR runs # queued for hours; the parameter tree and every test result are # identical on both. runs-on: ubuntu-24.04-arm - # The whole job took about 18 minutes when this limit was set; by - # October 2026 it takes about an hour (57 to 60 minutes on 9 October, - # and every run on 10 October was cancelled at a 60-minute limit, one - # at 99% with every test passing; issue #2257). 120 minutes lets a - # slow run finish and still stops a hung one long before the 6-hour - # GitHub limit. The suite's runtime needs its own attention. - timeout-minutes: 120 + timeout-minutes: 30 + permissions: + contents: "read" + id-token: "write" + steps: + - name: Checkout repo + uses: actions/checkout@v6 + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + - uses: "google-github-actions/auth@v3" + with: + workload_identity_provider: "projects/322898545428/locations/global/workloadIdentityPools/policyengine-research-id-pool/providers/prod-github-provider" + service_account: "policyengine-research@policyengine-research.iam.gserviceaccount.com" + + - name: Install package + run: uv pip install -e .[dev] --system + - name: Install policyengine + run: uv pip install policyengine --system + - name: Install JB + run: uv pip install "jupyter-book>=2.0.0a0" --system + - name: UV sync + run: uv sync --extra dev + - name: Run YAML tests + run: make test-yaml + - name: List the Python tests + # Every Python test, collected without --shard. The Test job + # checks that the shards below ran each of these exactly once. + run: | + # Only export token if non-empty (defensive against missing secrets) + if [ -n "$HF_TOKEN_VALUE" ]; then + export HUGGING_FACE_TOKEN="$HF_TOKEN_VALUE" + fi + pytest policyengine_uk/tests --collect-only -q -p no:cacheprovider > collected.txt + tail -n 1 collected.txt + env: + HF_TOKEN_VALUE: ${{ secrets.HUGGING_FACE_TOKEN }} + - name: Upload the list of Python tests + uses: actions/upload-artifact@v7 + with: + name: test-collected + path: collected.txt + if-no-files-found: error + overwrite: true + retention-days: 7 + Test-Python: + # Shard K of N runs the tests whose node id hashes to K (--shard in + # policyengine_uk/tests/conftest.py), so each test is in exactly one + # shard. N is the number of jobs this matrix makes: to add a shard, add + # a number to the list. + name: Test (Python ${{ matrix.shard }}/${{ strategy.job-total }}) + runs-on: ubuntu-24.04-arm + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] permissions: contents: "read" id-token: "write" @@ -61,7 +121,7 @@ jobs: with: workload_identity_provider: "projects/322898545428/locations/global/workloadIdentityPools/policyengine-research-id-pool/providers/prod-github-provider" service_account: "policyengine-research@policyengine-research.iam.gserviceaccount.com" - + - name: Install package run: uv pip install -e .[dev] --system - name: Install policyengine @@ -78,12 +138,46 @@ jobs: fi # Four pytest-xdist workers, one per core; together they peak at # about 9 GB of the runner's 16 GB. - make test PYTEST_ARGS="-n 4 --dist worksteal" + make test-python PYTEST_ARGS="-n 4 --dist worksteal --shard ${{ matrix.shard }}/${{ strategy.job-total }} --junitxml=junit-${{ matrix.shard }}.xml" env: HF_TOKEN_VALUE: ${{ secrets.HUGGING_FACE_TOKEN }} + # Measure coverage with sys.monitoring instead of the C trace + # function: the same lines are reported, at less cost per line. + COVERAGE_CORE: sysmon + - name: Upload the test results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: test-junit-${{ matrix.shard }} + path: junit-${{ matrix.shard }}.xml + if-no-files-found: error + overwrite: true + retention-days: 7 - name: Upload coverage to Codecov uses: codecov/codecov-action@v6 with: files: ./coverage.xml fail_ci_if_error: false verbose: true + Test: + # One result for the whole suite. It fails if any test job failed, or + # if the shards did not run every collected Python test exactly once. + needs: [Test-YAML, Test-Python] + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout repo + uses: actions/checkout@v6 + - name: Download the test lists and results + uses: actions/download-artifact@v8 + with: + pattern: test-* + merge-multiple: true + - name: Check that every Python test ran in exactly one shard + run: python3 .github/check_test_shards.py --collected collected.txt --junit junit-*.xml + - name: Check that every test job passed + run: | + echo "YAML: ${{ needs.Test-YAML.result }}; Python: ${{ needs.Test-Python.result }}" + test "${{ needs.Test-YAML.result }}" = success + test "${{ needs.Test-Python.result }}" = success diff --git a/.github/workflows/pr_code_changes.yaml b/.github/workflows/pr_code_changes.yaml index 055741c182..26d410db9d 100644 --- a/.github/workflows/pr_code_changes.yaml +++ b/.github/workflows/pr_code_changes.yaml @@ -51,19 +51,79 @@ jobs: uses: astral-sh/setup-uv@v8.1.0 - name: Check formatting run: uvx ruff format --check . - Test: + # The tests run as four jobs: the YAML policy tests, and the Python tests + # split over three runners. As one job they took 59 minutes on 10 October + # 2026 (run 38075384588: 2.6 minutes of YAML tests, then 55.8 minutes of + # pytest on four workers), and runs were cancelled at a 60-minute limit + # (issues #2099 and #2257). + Test-YAML: + name: Test (YAML) # Linux arm64 keeps the CPU architecture of the macOS runners this job # used to run on. GitHub-hosted macOS runners are scarce, so PR runs # queued for hours; the parameter tree and every test result are # identical on both. runs-on: ubuntu-24.04-arm - # The whole job took about 18 minutes when this limit was set; by - # October 2026 it takes about an hour (57 to 60 minutes on 9 October, - # and every run on 10 October was cancelled at a 60-minute limit, one - # at 99% with every test passing; issue #2257). 120 minutes lets a - # slow run finish and still stops a hung one long before the 6-hour - # GitHub limit. The suite's runtime needs its own attention. - timeout-minutes: 120 + timeout-minutes: 30 + permissions: + contents: "read" + id-token: "write" + steps: + - name: Checkout repo + uses: actions/checkout@v6 + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + - uses: "google-github-actions/auth@v3" + with: + workload_identity_provider: "projects/322898545428/locations/global/workloadIdentityPools/policyengine-research-id-pool/providers/prod-github-provider" + service_account: "policyengine-research@policyengine-research.iam.gserviceaccount.com" + + - name: Install package + run: uv pip install -e .[dev] --system + - name: Install policyengine + run: uv pip install policyengine --system + - name: UV sync + run: uv sync --extra dev + - name: Run YAML tests + run: make test-yaml + - name: List the Python tests + # Every Python test, collected without --shard. The Test job + # checks that the shards below ran each of these exactly once. + run: | + # Only export token if non-empty (avoids 'Bearer ' error for Dependabot PRs) + if [ -n "$HF_TOKEN_VALUE" ]; then + export HUGGING_FACE_TOKEN="$HF_TOKEN_VALUE" + fi + pytest policyengine_uk/tests --collect-only -q -p no:cacheprovider > collected.txt + tail -n 1 collected.txt + env: + HF_TOKEN_VALUE: ${{ secrets.HUGGING_FACE_TOKEN }} + - name: Upload the list of Python tests + uses: actions/upload-artifact@v7 + with: + name: test-collected + path: collected.txt + if-no-files-found: error + overwrite: true + retention-days: 7 + - name: Test documentation builds + run: make documentation + Test-Python: + # Shard K of N runs the tests whose node id hashes to K (--shard in + # policyengine_uk/tests/conftest.py), so each test is in exactly one + # shard. N is the number of jobs this matrix makes: to add a shard, add + # a number to the list. + name: Test (Python ${{ matrix.shard }}/${{ strategy.job-total }}) + runs-on: ubuntu-24.04-arm + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] permissions: contents: "read" id-token: "write" @@ -96,14 +156,46 @@ jobs: fi # Four pytest-xdist workers, one per core; together they peak at # about 9 GB of the runner's 16 GB. - make test PYTEST_ARGS="-n 4 --dist worksteal" + make test-python PYTEST_ARGS="-n 4 --dist worksteal --shard ${{ matrix.shard }}/${{ strategy.job-total }} --junitxml=junit-${{ matrix.shard }}.xml" env: HF_TOKEN_VALUE: ${{ secrets.HUGGING_FACE_TOKEN }} + # Measure coverage with sys.monitoring instead of the C trace + # function: the same lines are reported, at less cost per line. + COVERAGE_CORE: sysmon + - name: Upload the test results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: test-junit-${{ matrix.shard }} + path: junit-${{ matrix.shard }}.xml + if-no-files-found: error + overwrite: true + retention-days: 7 - name: Upload coverage to Codecov uses: codecov/codecov-action@v6 with: files: ./coverage.xml fail_ci_if_error: false verbose: true - - name: Test documentation builds - run: make documentation + Test: + # One result for the whole suite. It fails if any test job failed, or + # if the shards did not run every collected Python test exactly once. + needs: [Test-YAML, Test-Python] + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout repo + uses: actions/checkout@v6 + - name: Download the test lists and results + uses: actions/download-artifact@v8 + with: + pattern: test-* + merge-multiple: true + - name: Check that every Python test ran in exactly one shard + run: python3 .github/check_test_shards.py --collected collected.txt --junit junit-*.xml + - name: Check that every test job passed + run: | + echo "YAML: ${{ needs.Test-YAML.result }}; Python: ${{ needs.Test-Python.result }}" + test "${{ needs.Test-YAML.result }}" = success + test "${{ needs.Test-Python.result }}" = success diff --git a/Makefile b/Makefile index 5723221068..528f7b9be3 100644 --- a/Makefile +++ b/Makefile @@ -19,8 +19,10 @@ test: test-yaml test-python test-yaml: policyengine-core test policyengine_uk/tests/policy -c policyengine_uk -# CI passes PYTEST_ARGS="-n 4 --dist worksteal" (pytest-xdist) to spread the -# tests over the runner's four cores. Without it this runs serially, as before. +# CI passes PYTEST_ARGS to spread the tests over a runner's four cores +# ("-n 4 --dist worksteal", pytest-xdist) and over several runners +# ("--shard K/N", policyengine_uk/tests/conftest.py). Without it this runs +# every test serially. test-python: pytest policyengine_uk/tests/ --cov=policyengine_uk --cov-report=xml --maxfail=0 -v $(PYTEST_ARGS) diff --git a/changelog.d/ci-test-runtime-2099.fixed.md b/changelog.d/ci-test-runtime-2099.fixed.md new file mode 100644 index 0000000000..413d63714e --- /dev/null +++ b/changelog.d/ci-test-runtime-2099.fixed.md @@ -0,0 +1 @@ +- Split the CI `Test` job into a YAML job and three parallel shards of the Python tests, with a check that every collected test ran in exactly one shard, and measured coverage with `sys.monitoring` (#2099). diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index 66cb54d3c4..6894b55b4a 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -58,3 +58,24 @@ make test-microsimulation ``` When a command is not run, record why. + +## How CI Runs The Tests + +The test jobs in `.github/workflows/pr_code_changes.yaml` and +`code_changes.yaml` run the YAML policy tests in one job and split the Python +tests over several runners. Each of those runs one shard: + +```bash +make test-python PYTEST_ARGS="-n 4 --dist worksteal --shard K/N" +``` + +`--shard K/N` is defined in `policyengine_uk/tests/conftest.py`. It keeps the +tests whose node id hashes to shard K of N, so every test is in exactly one +shard, including a test added later. The final `Test` job runs +`.github/check_test_shards.py`, which fails unless every collected test ran in +exactly one shard. + +- To rerun what one CI shard ran, pass the same `--shard K/N` locally. +- To make CI faster, add a shard to the `shard` matrix in both workflows. +- Do not split tests by file or directory in a workflow: a test outside the + listed paths would silently not run. diff --git a/policyengine_uk/tests/code_health/test_ci_test_shards.py b/policyengine_uk/tests/code_health/test_ci_test_shards.py new file mode 100644 index 0000000000..17e3d71586 --- /dev/null +++ b/policyengine_uk/tests/code_health/test_ci_test_shards.py @@ -0,0 +1,332 @@ +"""The Test job's shards run every test exactly once. + +CI runs ``pytest policyengine_uk/tests`` as several jobs, each with +``--shard K/N`` (conftest.py), and then runs ``.github/check_test_shards.py`` +over their JUnit files and an unsharded ``pytest --collect-only -q`` listing. +These tests hold the two halves to the invariants the split relies on: + +1. ``shard_of(nodeid, N)`` is between 1 and N and depends only on its + arguments, so it is the same in every process and on every runner. +2. For any set of node ids and any N, the N shards are disjoint and their + union is the whole set. +3. ``parse_shard`` accepts exactly ``K/N`` with 1 <= K <= N. +4. ``junit_key`` gives a node id the name pytest gives it in JUnit XML. +5. The check passes when every collected test ran in exactly one shard and + none failed, and reports a test that is missing, repeated, not collected + or failed. +6. With real pytest runs, the collection listing and the shards' JUnit files + pass the check, with and without pytest-xdist, and fail it if a shard's + file is left out. +""" + +import argparse +import importlib.util +import os +import shutil +import subprocess +import sys +import xml.etree.ElementTree as ElementTree +from pathlib import Path + +import pytest +from hypothesis import HealthCheck, example, given, settings +from hypothesis import strategies as st + +TESTS_DIR = Path(__file__).resolve().parents[1] +REPO_ROOT = TESTS_DIR.parents[1] +CHECKER = REPO_ROOT / ".github" / "check_test_shards.py" + +if not CHECKER.exists(): + pytest.skip( + "the .github folder is not part of an installed package", + allow_module_level=True, + ) + +_spec = importlib.util.spec_from_file_location("check_test_shards", CHECKER) +checker = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(checker) + +PROPERTY_SETTINGS = settings( + max_examples=200, + deadline=None, + derandomize=True, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.function_scoped_fixture], +) + + +@pytest.fixture +def conftest(request): + path = TESTS_DIR / "conftest.py" + for plugin in request.config.pluginmanager.get_plugins(): + if getattr(plugin, "__file__", None) and Path(plugin.__file__) == path: + return plugin + raise AssertionError(f"{path} is not loaded as a plugin") + + +# Node ids as pytest writes them: a path, optional classes, a test name and +# optional parameters. pytest escapes non-ASCII and control characters in +# parameter ids, so the parameters here are any printable ASCII text, +# including "[", "]" and "::". +_identifier = st.from_regex(r"[A-Za-z_][A-Za-z0-9_]{0,12}", fullmatch=True) +_path = st.lists(_identifier, min_size=1, max_size=4).map( + lambda parts: "/".join(parts) + ".py" +) +_printable = st.characters(min_codepoint=0x20, max_codepoint=0x7E) +_params = st.one_of( + st.just(""), + st.text(alphabet=_printable, max_size=20).map(lambda text: f"[{text}]"), +) +nodeids = st.builds( + lambda path, classes, name, params: "::".join([path, *classes, name + params]), + _path, + st.lists(_identifier, max_size=2), + _identifier, + _params, +) +# Node ids that could come from one collection: distinct, and with distinct +# JUnit names, which pytest's own naming guarantees for real suites. +collections = st.lists(nodeids, max_size=30, unique_by=checker.junit_key) + + +def junit_cases(ids, outcome="passed"): + return [(checker.junit_key(nodeid), outcome) for nodeid in ids] + + +@PROPERTY_SETTINGS +@given(nodeid=nodeids, shards=st.integers(1, 64)) +@example(nodeid="policyengine_uk/tests/test_x.py::test_y", shards=3) +def test_shard_is_in_range_and_repeatable(conftest, nodeid, shards): + shard = conftest.shard_of(nodeid, shards) + assert 1 <= shard <= shards + assert shard == conftest.shard_of(nodeid, shards) + + +def test_shard_does_not_depend_on_the_process(conftest): + # Pinned values: every runner and pytest-xdist worker must agree, which + # Python's own hash, salted per process, would not give. + assert [ + conftest.shard_of("policyengine_uk/tests/test_x.py::test_y", shards) + for shards in (1, 2, 3, 4, 7) + ] == [1, 2, 1, 4, 7] + assert conftest.shard_of("policyengine_uk/tests/test_x.py::test_y[2026-a]", 3) == 2 + + +@PROPERTY_SETTINGS +@given(ids=st.lists(nodeids, max_size=40, unique=True), shards=st.integers(1, 8)) +def test_shards_partition_any_collection(conftest, ids, shards): + members = { + shard: [nodeid for nodeid in ids if conftest.shard_of(nodeid, shards) == shard] + for shard in range(1, shards + 1) + } + assert sorted(sum(members.values(), [])) == sorted(ids) + assert sum(len(group) for group in members.values()) == len(ids) + + +@PROPERTY_SETTINGS +@given(shards=st.integers(1, 200), data=st.data()) +def test_parse_shard_accepts_k_of_n(conftest, shards, data): + shard = data.draw(st.integers(1, shards)) + assert conftest.parse_shard(f"{shard}/{shards}") == (shard, shards) + + +@pytest.mark.parametrize( + "value", + [ + "0/3", + "4/3", + "1/0", + "0/0", + "3", + "1/3/5", + "a/b", + "1-3", + "", + " 1/3", + "1/3 ", + "-1/3", + ], +) +def test_parse_shard_rejects_anything_else(conftest, value): + with pytest.raises(argparse.ArgumentTypeError): + conftest.parse_shard(value) + + +@PROPERTY_SETTINGS +@given(nodeid=nodeids) +@example(nodeid="policyengine_uk/tests/test_x.py::TestA::test_y[a/b.yaml-1]") +@example(nodeid="policyengine_uk/tests/test_x.py::test_y[a::b[c]-d e]") +def test_junit_key_matches_pytest(nodeid): + # Differential test against pytest's own naming, written out to XML and + # read back as the check reads it. + junitxml = pytest.importorskip("_pytest.junitxml") + names = junitxml.mangle_test_address(nodeid) + element = ElementTree.Element( + "testcase", + classname=".".join(names[:-1]), + name=junitxml.bin_xml_escape(names[-1]), + ) + read_back = ElementTree.fromstring(ElementTree.tostring(element)) + assert checker.junit_key(nodeid) == ( + read_back.get("classname"), + read_back.get("name"), + ) + + +@PROPERTY_SETTINGS +@given(ids=collections, shards=st.integers(1, 5), data=st.data()) +def test_check_passes_exactly_when_every_test_ran_once(conftest, ids, shards, data): + ran = [ + junit_cases(i for i in ids if conftest.shard_of(i, shards) == shard) + for shard in range(1, shards + 1) + ] + problems, summary = checker.check(ids, ran) + assert problems == [] + assert summary.startswith(f"{len(ids)} tests collected") + + if not ids: + return + victim = data.draw(st.sampled_from(ids)) + key = checker.junit_key(victim) + home = conftest.shard_of(victim, shards) - 1 + + missing = [list(cases) for cases in ran] + missing[home].remove((key, "passed")) + problems, _ = checker.check(ids, missing) + assert [names for title, names in problems if "ran in no shard" in title] == [ + [victim] + ] + + repeated = [list(cases) for cases in ran] + repeated[data.draw(st.integers(0, shards - 1))].append((key, "passed")) + problems, _ = checker.check(ids, repeated) + assert [names for title, names in problems if "more than once" in title] == [ + [victim] + ] + + failed = [list(cases) for cases in ran] + failed[home][failed[home].index((key, "passed"))] = (key, "failed") + problems, _ = checker.check(ids, failed) + assert [names for title, names in problems if "failed" in title] == [ + [f"shard {home + 1}: {victim}"] + ] + + uncollected = [nodeid for nodeid in ids if nodeid != victim] + problems, _ = checker.check(uncollected, ran) + assert [names for title, names in problems if "not collected" in title] == [ + [f"{key[0]}::{key[1]}"] + ] + + +def test_read_collected_requires_the_listed_count(): + listing = "a/test_x.py::test_one\na/test_x.py::test_two[p]\n\n2 tests collected in 0.01s\n" + assert checker.read_collected(listing) == [ + "a/test_x.py::test_one", + "a/test_x.py::test_two[p]", + ] + assert checker.read_collected("\nno tests collected in 0.01s\n") == [] + # Warnings follow the listing and may mention a node id. + with_warnings = listing.replace( + "\n\n2 tests", + "\n\n=== warnings summary ===\na/test_x.py::test_one\n x\n2 tests", + ) + assert checker.read_collected(with_warnings) == checker.read_collected(listing) + for bad in ( + "a/test_x.py::test_one\n", + "a/test_x.py::test_one\n\n2 tests collected in 0.01s\n", + "a/test_x.py::test_one\na/test_x.py::test_one\n\n2 tests collected in 0.01s\n", + "a/test_x.py::test_one\n\n1 test collected, 1 error in 0.01s\n", + ): + with pytest.raises(checker.ShardCheckError): + checker.read_collected(bad) + + +SAMPLE_SUITE = """ +import pytest + + +@pytest.mark.parametrize("value", range(40)) +def test_number(value): + assert value >= 0 + + +@pytest.mark.parametrize( + "name", ["parameters/gov/a.yaml", "b c", "d[e]", "f::g", "h-1", "caf\\u00e9", "tab\\there"] +) +def test_name(name): + assert name + + +class TestGroup: + @pytest.mark.parametrize("value", ["x", "y"]) + def test_member(self, value): + assert value + + @pytest.mark.skip(reason="skipped tests still count as run") + def test_skipped(self): + raise AssertionError + + @pytest.mark.xfail(strict=True) + def test_expected_failure(self): + raise AssertionError +""" + + +def _pytest(cwd, *args): + # A clean run of pytest: no options, plugins or coverage settings carried + # over from the session that runs this test. + env = { + name: value + for name, value in os.environ.items() + if not name.startswith(("PYTEST_", "COV_CORE_", "COVERAGE_")) + } + return subprocess.run( + [sys.executable, "-m", "pytest", "-p", "no:cacheprovider", *args], + cwd=cwd, + env=env, + capture_output=True, + text=True, + timeout=600, + ) + + +@pytest.mark.parametrize("xdist", [(), ("-n", "2", "--dist", "worksteal")]) +def test_real_pytest_shards_pass_the_check(tmp_path, xdist): + suite = tmp_path / "suite" + suite.mkdir() + # The suite's real conftest.py, so this runs the --shard CI uses. + shutil.copy(TESTS_DIR / "conftest.py", suite / "conftest.py") + (suite / "test_sample.py").write_text(SAMPLE_SUITE, encoding="utf-8") + + listing = _pytest(tmp_path, "suite", "--collect-only", "-q") + assert listing.returncode == 0, listing.stdout + listing.stderr + collected = tmp_path / "collected.txt" + collected.write_text(listing.stdout, encoding="utf-8") + assert len(checker.read_collected(listing.stdout)) == 51 + + shards = 3 + junit = [] + for shard in range(1, shards + 1): + junit.append(tmp_path / f"junit-{shard}.xml") + run = _pytest( + tmp_path, + "suite", + "--shard", + f"{shard}/{shards}", + f"--junitxml={junit[-1]}", + *xdist, + ) + assert run.returncode == 0, run.stdout + run.stderr + assert f"shard: {shard} of {shards}" in run.stdout + + sizes = [len(checker.read_junit(path)) for path in junit] + assert sum(sizes) == 51 and all(sizes) + + def check(*paths): + return checker.main( + ["--collected", str(collected), "--junit", *map(str, paths)] + ) + + assert check(*junit) == 0 + assert check(*junit[:-1]) == 1 + assert check(*junit, junit[0]) == 1 + assert check(tmp_path / "absent.xml") == 2 diff --git a/policyengine_uk/tests/conftest.py b/policyengine_uk/tests/conftest.py index d611a0c1c5..ca783dd9ab 100644 --- a/policyengine_uk/tests/conftest.py +++ b/policyengine_uk/tests/conftest.py @@ -1,4 +1,6 @@ +import argparse import gc +import hashlib import os import re import sys @@ -29,7 +31,62 @@ def cloned_uk_tax_benefit_system(): yield +# CI splits the suite over several runners with ``--shard K/N``. The shard of a +# test is a function of its node id alone, so the shards partition whatever is +# collected: every test runs in exactly one of them, including a test added +# later. The Test job checks that on each run (.github/check_test_shards.py). +def shard_of(nodeid: str, shards: int) -> int: + """The shard, from 1 to ``shards``, that runs the test ``nodeid``. + + Uses SHA-256 rather than ``hash``, which Python salts per process: every + runner and every pytest-xdist worker must give the same answer. + """ + digest = hashlib.sha256(nodeid.encode("utf-8")).digest() + return int.from_bytes(digest[:8], "big") % shards + 1 + + +def parse_shard(value: str) -> tuple[int, int]: + """Parse ``K/N`` into ``(K, N)``, requiring 1 <= K <= N.""" + match = re.fullmatch(r"([0-9]+)/([0-9]+)", value) + if match: + shard, shards = int(match.group(1)), int(match.group(2)) + if 1 <= shard <= shards: + return shard, shards + raise argparse.ArgumentTypeError( + f"expected K/N with 1 <= K <= N, such as 1/3; got {value!r}" + ) + + +def pytest_addoption(parser): + parser.addoption( + "--shard", + type=parse_shard, + default=None, + metavar="K/N", + help="Run only shard K of N. Every test is in exactly one shard.", + ) + + +def pytest_report_header(config): + shard = config.getoption("shard", default=None) + if shard is not None: + return f"shard: {shard[0]} of {shard[1]}, chosen by node id" + + def pytest_collection_modifyitems(config, items): + shard = config.getoption("shard", default=None) + if shard is not None: + this_shard, shards = shard + selected, deselected = [], [] + for item in items: + if shard_of(item.nodeid, shards) == this_shard: + selected.append(item) + else: + deselected.append(item) + items[:] = selected + if deselected: + config.hook.pytest_deselected(items=deselected) + has_default_dataset = bool(os.environ.get("POLICYENGINE_UK_DEFAULT_DATASET")) if has_default_dataset: return From 841d2a8899000ea54773c878f3a83fbf810ea9b4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:00:16 -0400 Subject: [PATCH 06/25] Run the final Test job on the shards' runners and report failed jobs first On 10 October the 5-second final Test job waited 32 minutes for an ubuntu-latest runner after the shards finished (run 38085920460). It now runs on ubuntu-24.04-arm like the shards. The job-result check runs first, and the shard check still runs after a failure so its report names the failed tests. Record the measured job times by the timeouts. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/code_changes.yaml | 24 ++++++++++++++++++------ .github/workflows/pr_code_changes.yaml | 24 ++++++++++++++++++------ 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/.github/workflows/code_changes.yaml b/.github/workflows/code_changes.yaml index bd414330c1..6d30dcf3b9 100644 --- a/.github/workflows/code_changes.yaml +++ b/.github/workflows/code_changes.yaml @@ -43,6 +43,7 @@ jobs: # queued for hours; the parameter tree and every test result are # identical on both. runs-on: ubuntu-24.04-arm + # About 4 minutes on 10 October (run 38085920460). timeout-minutes: 30 permissions: contents: "read" @@ -99,6 +100,9 @@ jobs: # a number to the list. name: Test (Python ${{ matrix.shard }}/${{ strategy.job-total }}) runs-on: ubuntu-24.04-arm + # The shards took 15, 20 and 15 minutes on 10 October (run + # 38085920460). 60 minutes leaves room for a slow runner and still + # stops a hung shard. timeout-minutes: 60 strategy: fail-fast: false @@ -164,20 +168,28 @@ jobs: # if the shards did not run every collected Python test exactly once. needs: [Test-YAML, Test-Python] if: ${{ !cancelled() }} - runs-on: ubuntu-latest + # The shards' runners, not ubuntu-latest: on 10 October this 5-second + # job waited 32 minutes for an ubuntu-latest runner after the shards + # had finished (run 38085920460). + runs-on: ubuntu-24.04-arm timeout-minutes: 10 steps: + - name: Check that every test job passed + run: | + echo "YAML: ${{ needs.Test-YAML.result }}; Python: ${{ needs.Test-Python.result }}" + test "${{ needs.Test-YAML.result }}" = success + test "${{ needs.Test-Python.result }}" = success - name: Checkout repo + if: ${{ !cancelled() }} uses: actions/checkout@v6 - name: Download the test lists and results + if: ${{ !cancelled() }} uses: actions/download-artifact@v8 with: pattern: test-* merge-multiple: true - name: Check that every Python test ran in exactly one shard + # Runs even when a test job failed, so its report names the + # failed tests. + if: ${{ !cancelled() }} run: python3 .github/check_test_shards.py --collected collected.txt --junit junit-*.xml - - name: Check that every test job passed - run: | - echo "YAML: ${{ needs.Test-YAML.result }}; Python: ${{ needs.Test-Python.result }}" - test "${{ needs.Test-YAML.result }}" = success - test "${{ needs.Test-Python.result }}" = success diff --git a/.github/workflows/pr_code_changes.yaml b/.github/workflows/pr_code_changes.yaml index 26d410db9d..2c8f588ba6 100644 --- a/.github/workflows/pr_code_changes.yaml +++ b/.github/workflows/pr_code_changes.yaml @@ -63,6 +63,7 @@ jobs: # queued for hours; the parameter tree and every test result are # identical on both. runs-on: ubuntu-24.04-arm + # About 4 minutes on 10 October (run 38085920460). timeout-minutes: 30 permissions: contents: "read" @@ -119,6 +120,9 @@ jobs: # a number to the list. name: Test (Python ${{ matrix.shard }}/${{ strategy.job-total }}) runs-on: ubuntu-24.04-arm + # The shards took 15, 20 and 15 minutes on 10 October (run + # 38085920460). 60 minutes leaves room for a slow runner and still + # stops a hung shard. timeout-minutes: 60 strategy: fail-fast: false @@ -182,20 +186,28 @@ jobs: # if the shards did not run every collected Python test exactly once. needs: [Test-YAML, Test-Python] if: ${{ !cancelled() }} - runs-on: ubuntu-latest + # The shards' runners, not ubuntu-latest: on 10 October this 5-second + # job waited 32 minutes for an ubuntu-latest runner after the shards + # had finished (run 38085920460). + runs-on: ubuntu-24.04-arm timeout-minutes: 10 steps: + - name: Check that every test job passed + run: | + echo "YAML: ${{ needs.Test-YAML.result }}; Python: ${{ needs.Test-Python.result }}" + test "${{ needs.Test-YAML.result }}" = success + test "${{ needs.Test-Python.result }}" = success - name: Checkout repo + if: ${{ !cancelled() }} uses: actions/checkout@v6 - name: Download the test lists and results + if: ${{ !cancelled() }} uses: actions/download-artifact@v8 with: pattern: test-* merge-multiple: true - name: Check that every Python test ran in exactly one shard + # Runs even when a test job failed, so its report names the + # failed tests. + if: ${{ !cancelled() }} run: python3 .github/check_test_shards.py --collected collected.txt --junit junit-*.xml - - name: Check that every test job passed - run: | - echo "YAML: ${{ needs.Test-YAML.result }}; Python: ${{ needs.Test-Python.result }}" - test "${{ needs.Test-YAML.result }}" = success - test "${{ needs.Test-Python.result }}" = success From a310fa22cc17842e8b2ec88e2b033da4cb5641e2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:07:49 -0400 Subject: [PATCH 07/25] Assert the 2023-24 lower profits limit alignment instead of expecting it to fail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #1968 (the model's 2023-24 Class 4 LPL was £11,908, not the £12,570 NICA 2022 s2 sets from 6 April 2023) was fixed by #2058 on 10 October. The non-strict xfail kept for it now passes silently as an XPASS, so the alignment test protected nothing for that year. Remove the marker so a regression to £11,908 fails the suite. Co-Authored-By: Claude Opus 5.5 --- .../remove-stale-xfail-1968.changed.md | 1 + .../tests/test_threshold_indexation.py | 19 ++++--------------- 2 files changed, 5 insertions(+), 15 deletions(-) create mode 100644 changelog.d/remove-stale-xfail-1968.changed.md diff --git a/changelog.d/remove-stale-xfail-1968.changed.md b/changelog.d/remove-stale-xfail-1968.changed.md new file mode 100644 index 0000000000..8c89b101f7 --- /dev/null +++ b/changelog.d/remove-stale-xfail-1968.changed.md @@ -0,0 +1 @@ +- Removed the stale non-strict expected failure for the 2023-24 Class 4 lower profits limit (#1968, fixed by #2058), so the alignment test now asserts that year. diff --git a/policyengine_uk/tests/test_threshold_indexation.py b/policyengine_uk/tests/test_threshold_indexation.py index 01d5396e02..23211c9960 100644 --- a/policyengine_uk/tests/test_threshold_indexation.py +++ b/policyengine_uk/tests/test_threshold_indexation.py @@ -35,8 +35,7 @@ £0.26 of the UPL. From 2023: LPL = PA and 52 x PT is within £0.26 of the LPL. Intended exceptions, as the law stood: in 2021-22 the PT and LPL were £9,568 a year, below the PA; in 2022-23 the PT rose to the PA only - from 6 July 2022 and the LPL was £11,908 for the whole year. The model's - 2023-24 LPL (£11,908, not £12,570) is #1968. + from 6 July 2022 and the LPL was £11,908 for the whole year. 6. Secondary threshold: from 2031, ST(Y) = ST(Y-1) x (1 + g_Y). 7. For any freeze end and September CPI path (function level): values up to each parameter's last stated year are unchanged; after it, each NICs @@ -322,19 +321,9 @@ def test_upper_thresholds_align_with_the_higher_rate_threshold(system, year): ) -@pytest.mark.parametrize( - "year", - [ - pytest.param( - 2023, - marks=pytest.mark.xfail( - reason="#1968: the model's 2023-24 LPL is £11,908, not £12,570", - strict=False, - ), - ), - *range(2024, 2041), - ], -) +# 2023-24 is the first aligned year: NICA 2022 s2 set the LPL to £12,570 +# from 6 April 2023 (#1968, fixed by #2058). +@pytest.mark.parametrize("year", range(2023, 2041)) def test_lower_thresholds_align_with_the_personal_allowance(system, year): values = thresholds(system, year) assert values["lower_profits_limit"] == pytest.approx( From 7f7fc5d1911c6b751c6a957a339f93186edb235e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:54:08 -0400 Subject: [PATCH 08/25] Drop the #1968 exception from the NI threshold freeze test's invariants too The sibling module repeated the 2023-24 lower profits limit as an open defect (#1968); #2058 fixed it. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/tests/test_ni_threshold_freeze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/policyengine_uk/tests/test_ni_threshold_freeze.py b/policyengine_uk/tests/test_ni_threshold_freeze.py index 0403c92137..43cceb3712 100644 --- a/policyengine_uk/tests/test_ni_threshold_freeze.py +++ b/policyengine_uk/tests/test_ni_threshold_freeze.py @@ -26,7 +26,7 @@ 4. Alignment, in test_threshold_indexation.py: the UEL and UPL with the higher rate threshold in every year 2021-2040, and the PT and LPL with the personal allowance from 2023. The 2021-22 and 2022-23 lower thresholds - are intended exceptions, and the 2023-24 LPL is #1968. + are intended exceptions. 5. No cash cuts: no threshold falls from one year to the next, 2026-2040. 6. Differential: for any earnings or profits >= 0 in 2026-2030, primary and additional Class 1, secondary Class 1 and Class 4 equal an exact-rational From d639a0334844b256e9a9cc65bf35ee4ede011145 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:59:59 -0400 Subject: [PATCH 09/25] Apply the Carer's Allowance earnings limit (SSCBA 1992 s. 70(1)(b), SI 1976/409 reg. 8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #2309. Carer's Allowance was paid whatever the carer earned. A person whose net weekly earnings exceed the limit (£204 from 6 April 2026) is gainfully employed and not entitled. Net earnings are pay and trading profit less the income tax and NI deducted from them and half of pension contributions (Computation of Earnings Regulations 1996 regs. 10(4), 13). Co-Authored-By: Claude Opus 5.5 --- changelog.d/2309.fixed.md | 1 + .../dwp/carers_allowance/earnings_limit.yaml | 62 ++++++++ ..._childcare_entitlement_work_condition.yaml | 8 +- .../dwp/carers_allowance_earnings_limit.yaml | 142 ++++++++++++++++++ ...ers_allowance_earnings_limit_properties.py | 120 +++++++++++++++ .../gov/dwp/carers_allowance_earnings.py | 62 ++++++++ .../gov/dwp/carers_allowance_pre_overlap.py | 7 + ...gainfully_employed_for_carers_allowance.py | 29 ++++ 8 files changed, 429 insertions(+), 2 deletions(-) create mode 100644 changelog.d/2309.fixed.md create mode 100644 policyengine_uk/parameters/gov/dwp/carers_allowance/earnings_limit.yaml create mode 100644 policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml create mode 100644 policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py create mode 100644 policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py create mode 100644 policyengine_uk/variables/gov/dwp/is_gainfully_employed_for_carers_allowance.py diff --git a/changelog.d/2309.fixed.md b/changelog.d/2309.fixed.md new file mode 100644 index 0000000000..db731a062c --- /dev/null +++ b/changelog.d/2309.fixed.md @@ -0,0 +1 @@ +- Stop paying Carer's Allowance to a carer whose net earnings exceed the earnings limit (£204 a week from April 2026). diff --git a/policyengine_uk/parameters/gov/dwp/carers_allowance/earnings_limit.yaml b/policyengine_uk/parameters/gov/dwp/carers_allowance/earnings_limit.yaml new file mode 100644 index 0000000000..76beb1c9fc --- /dev/null +++ b/policyengine_uk/parameters/gov/dwp/carers_allowance/earnings_limit.yaml @@ -0,0 +1,62 @@ +description: Weekly earnings above which a person is treated as gainfully employed and so is not entitled to Carer's Allowance. The test uses net earnings in the preceding week. +metadata: + economy: false + period: week + unit: currency-GBP + label: Carer's Allowance earnings limit + reference: + - title: The Social Security (Invalid Care Allowance) Regulations 1976 reg. 8(1) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8 + - title: Social Security Contributions and Benefits Act 1992 s. 70(1)(b) + href: https://www.legislation.gov.uk/ukpga/1992/4/section/70 +values: + 2015-04-06: + value: 110 + reference: + - title: The Social Security (Invalid Care Allowance) (Amendment) Regulations 2015 (SI 2015/162) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2015-04-06 + 2017-04-10: + value: 116 + reference: + - title: The Social Security (Invalid Care Allowance) (Amendment) Regulations 2017 (SI 2017/386) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2017-04-10 + 2018-04-09: + value: 120 + reference: + - title: The Social Security (Invalid Care Allowance) (Amendment) Regulations 2018 (SI 2018/280) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2018-04-09 + 2019-04-08: + value: 123 + reference: + - title: The Social Security Benefits Up-rating Regulations 2019 (SI 2019/552) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2019-04-08 + 2020-04-06: + value: 128 + reference: + - title: The Social Security Benefits Up-rating Regulations 2020 (SI 2020/266) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2021-04-12 + 2022-04-11: + value: 132 + reference: + - title: The Social Security Benefits Up-rating Regulations 2022 (SI 2022/342) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2022-04-11 + 2023-04-10: + value: 139 + reference: + - title: The Social Security Benefits Up-rating Regulations 2023 (SI 2023/340) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2023-04-10 + 2024-04-08: + value: 151 + reference: + - title: The Social Security Benefits Up-Rating Regulations 2024 (SI 2024/386) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2024-04-08 + 2025-04-07: + value: 196 + reference: + - title: The Social Security Benefits Up-rating Regulations 2025 (SI 2025/352) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2025-04-07 + 2026-04-06: + value: 204 + reference: + - title: The Social Security Benefits Up-rating Regulations 2026 (SI 2026/218) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8 diff --git a/policyengine_uk/tests/policy/baseline/gov/dfe/extended_childcare_entitlement/extended_childcare_entitlement_work_condition.yaml b/policyengine_uk/tests/policy/baseline/gov/dfe/extended_childcare_entitlement/extended_childcare_entitlement_work_condition.yaml index ef56a091ad..39860f1648 100644 --- a/policyengine_uk/tests/policy/baseline/gov/dfe/extended_childcare_entitlement/extended_childcare_entitlement_work_condition.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/dfe/extended_childcare_entitlement/extended_childcare_entitlement_work_condition.yaml @@ -506,14 +506,18 @@ # overlapped to nil by maternity allowance: entitled but not paid, so still # excluded. - name: 2022, working partner entitled to Carer's Allowance overlapped to nil - the partner on contributory ESA does not meet the condition + # Carer's Allowance entitlement needs net earnings within the earnings + # limit (£132 a week in 2022-23): pay of £8,400 less half of £3,100 of + # pension contributions is £6,850, or £131.73 a week. period: 2022 input: people: parent1: age: 40 is_parent: true - employment_income: 20_000 - adjusted_net_income: 20_000 + employment_income: 8_400 + employee_pension_contributions: 3_100 + adjusted_net_income: 5_300 care_hours: 35 maternity_allowance_reported: 5_000 minimum_wage: 10 diff --git a/policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml b/policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml new file mode 100644 index 0000000000..fec4df3ccb --- /dev/null +++ b/policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml @@ -0,0 +1,142 @@ +# Carer's Allowance earnings limit. SSCBA 1992 s. 70(1)(b) bars anyone +# gainfully employed; reg. 8(1) of the Social Security (Invalid Care +# Allowance) Regulations 1976 treats a person as gainfully employed when net +# earnings exceed £204 a week from 6 April 2026 (SI 2026/218). Net earnings +# are gross pay less income tax, primary Class 1 contributions and half of +# pension contributions (Computation of Earnings Regulations 1996 reg. +# 10(4)), and trading profit less tax, Class 2 and Class 4 (reg. 13). +# 2026-27: personal allowance and NI primary threshold £12,570, employee +# Class 1 main rate 8%, Class 4 lower profits limit £12,570 at 6%, basic rate +# 20%. Carer's Allowance £86.45 a week. + +- name: Case 1, pay of £8,000 (£153.85 a week, no tax or NI) is under the limit. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 8_000 + care_hours: 40 + country: ENGLAND + output: + carers_allowance_earnings: 8_000 + is_gainfully_employed_for_carers_allowance: false + carers_allowance: 86.45 * 52 + +- name: Case 2, pay of £15,000 is £275.38 a week net, over the limit. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 15_000 + care_hours: 40 + country: ENGLAND + output: + # Tax 20% x 2,430 = 486; NI 8% x 2,430 = 194.40. + carers_allowance_earnings: 15_000 - 486 - 194.40 + is_gainfully_employed_for_carers_allowance: true + carers_allowance_pre_overlap: 0 + carers_allowance: 0 + +- name: Case 3, earnings of exactly £204 a week have not exceeded the limit. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 204 * 52 + care_hours: 40 + country: ENGLAND + output: + is_gainfully_employed_for_carers_allowance: false + carers_allowance: 86.45 * 52 + +- name: Case 4, one pound a year more is over the limit. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 204 * 52 + 1 + care_hours: 40 + country: ENGLAND + output: + is_gainfully_employed_for_carers_allowance: true + carers_allowance: 0 + +- name: Case 5, half of pension contributions comes off earnings. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 11_000 + employee_pension_contributions: 1_000 + care_hours: 40 + country: ENGLAND + output: + # 11,000 - 500 = 10,500, or £201.92 a week; without the deduction it + # would be £211.54 and over the limit. + carers_allowance_earnings: 10_500 + is_gainfully_employed_for_carers_allowance: false + carers_allowance: 86.45 * 52 + +- name: Case 6, trading profit of £14,000 is £262.08 a week net, over the limit. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + self_employment_income: 14_000 + care_hours: 40 + country: ENGLAND + output: + # Tax 20% x 1,430 = 286; Class 4 6% x 1,430 = 85.80. + carers_allowance_earnings: 14_000 - 286 - 85.80 + is_gainfully_employed_for_carers_allowance: true + carers_allowance: 0 + +- name: Case 7, a trading loss counts as nil earnings. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + self_employment_income: -5_000 + care_hours: 40 + country: ENGLAND + output: + carers_allowance_earnings: 0 + carers_allowance: 86.45 * 52 + +- name: Case 8, a reported recipient over the limit is not entitled either. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 15_000 + carers_allowance_reported: 4_000 + country: ENGLAND + output: + is_gainfully_employed_for_carers_allowance: true + carers_allowance: 0 + +- name: Case 9, savings and pension income are not earnings. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + savings_interest_income: 20_000 + private_pension_income: 20_000 + care_hours: 40 + country: ENGLAND + output: + carers_allowance_earnings: 0 + is_gainfully_employed_for_carers_allowance: false + carers_allowance: 86.45 * 52 + +- name: Case 10, the 2025-26 limit was £196. + period: 2025 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 200 * 52 + care_hours: 40 + country: ENGLAND + output: + is_gainfully_employed_for_carers_allowance: true + carers_allowance: 0 diff --git a/policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py b/policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py new file mode 100644 index 0000000000..d59d49f038 --- /dev/null +++ b/policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py @@ -0,0 +1,120 @@ +"""Carer's Allowance earnings limit. + +SSCBA 1992 s. 70(1)(b) bars a person who is gainfully employed, and reg. 8(1) +of the Social Security (Invalid Care Allowance) Regulations 1976 treats a +person as gainfully employed when their earnings exceed the limit (£204 a +week from 6 April 2026). Earnings are net of income tax, primary Class 1 (and +Class 2 and 4) contributions and half of pension contributions (Computation of +Earnings Regulations 1996 regs. 10(4) and 13). + +Invariants, for a carer in England in 2026 caring 35 hours or more, with pay, +trading profit, employee pension contributions and unearned income drawn +freely: + +1. Bounds: 0 <= carers_allowance_earnings <= pay + max(0, profit). +2. Differential: the test is passed exactly when + carers_allowance_earnings / 52 <= 204, and the allowance is the full rate + then and nil otherwise. +3. Monotonic: raising pay never lowers net earnings, so a person over the + limit stays over it. +4. Unearned income is not earnings: savings interest and pension income + leave carers_allowance_earnings unchanged. +""" + +import numpy as np +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from policyengine_uk import Simulation + +LIMIT = 204 +RATE = 86.45 +WEEKS = 52 + + +def _person(pay, profit, pension, savings=0.0, private_pension=0.0): + situation = { + "people": { + "carer": { + "age": {"2026": 45}, + "employment_income": {"2026": pay}, + "self_employment_income": {"2026": profit}, + "employee_pension_contributions": {"2026": pension}, + "savings_interest_income": {"2026": savings}, + "private_pension_income": {"2026": private_pension}, + "care_hours": {"2026": 40}, + } + }, + "benunits": {"benunit": {"members": ["carer"]}}, + "households": { + "household": {"members": ["carer"], "country": {"2026": "ENGLAND"}} + }, + } + sim = Simulation(situation=situation) + return { + name: float(np.asarray(sim.calculate(name, 2026))[0]) + for name in ( + "carers_allowance_earnings", + "is_gainfully_employed_for_carers_allowance", + "carers_allowance", + ) + } + + +money = st.floats(min_value=0, max_value=60_000, allow_nan=False) +profit_money = st.floats(min_value=-20_000, max_value=60_000, allow_nan=False) + + +@settings( + max_examples=12, + deadline=None, + suppress_health_check=[HealthCheck.too_slow], +) +@given(pay=money, profit=profit_money, pension_share=st.floats(0, 0.3)) +def test_bounds_and_differential(pay, profit, pension_share): + pension = pay * pension_share + result = _person(pay, profit, pension) + earnings = result["carers_allowance_earnings"] + assert -0.01 <= earnings <= pay + max(0, profit) + 0.01 + over = earnings / WEEKS > LIMIT + assert bool(result["is_gainfully_employed_for_carers_allowance"]) == over + expected = 0 if over else RATE * WEEKS + assert abs(result["carers_allowance"] - expected) < 0.01 + + +@settings( + max_examples=8, + deadline=None, + suppress_health_check=[HealthCheck.too_slow], +) +@given(pay=money, rise=st.floats(min_value=1, max_value=20_000)) +def test_net_earnings_never_fall_as_pay_rises(pay, rise): + lower = _person(pay, 0, 0) + higher = _person(pay + rise, 0, 0) + assert ( + higher["carers_allowance_earnings"] >= lower["carers_allowance_earnings"] - 0.01 + ) + if lower["is_gainfully_employed_for_carers_allowance"]: + assert higher["is_gainfully_employed_for_carers_allowance"] + + +@settings( + max_examples=6, + deadline=None, + suppress_health_check=[HealthCheck.too_slow], +) +@given( + pay=st.floats(min_value=0, max_value=15_000), + savings=money, + private_pension=money, +) +def test_unearned_income_is_not_earnings(pay, savings, private_pension): + alone = _person(pay, 0, 0) + with_unearned = _person(pay, 0, 0, savings=savings, private_pension=private_pension) + assert ( + abs( + alone["carers_allowance_earnings"] + - with_unearned["carers_allowance_earnings"] + ) + < 0.01 + ) diff --git a/policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py b/policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py new file mode 100644 index 0000000000..f853f306ed --- /dev/null +++ b/policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py @@ -0,0 +1,62 @@ +from policyengine_uk.model_api import * + + +class carers_allowance_earnings(Variable): + value_type = float + entity = Person + label = "earnings for the Carer's Allowance earnings limit" + documentation = ( + "Net earnings from employment and self-employment, as the Carer's " + "Allowance gainful-employment test counts them: gross pay and trading " + "profit less the income tax and National Insurance deducted from them " + "and half of the person's pension contributions. A trading loss counts " + "as nil. The income tax is the tax PAYE would deduct if the earnings " + "were the person's only income: earnings less net-pay pension " + "contributions and the standard personal allowance, at the person's " + "rest-of-UK or Scottish rates. That is the tax deducted from pay, and " + "it avoids a circularity, since Carer's Allowance is itself taxable. " + "The deduction for care charges (Computation of Earnings Regulations " + "1996 Schedule 3, up to half of net earnings) and the Schedule 1 " + "disregards are not modelled, so this can overstate earnings for a " + "carer who pays for care while at work." + ) + definition_period = YEAR + unit = GBP + reference = [ + dict( + title="Social Security Benefit (Computation of Earnings) Regulations 1996 reg. 10(4)", + href="https://www.legislation.gov.uk/uksi/1996/2745/regulation/10", + ), + dict( + title="Social Security Benefit (Computation of Earnings) Regulations 1996 reg. 13", + href="https://www.legislation.gov.uk/uksi/1996/2745/regulation/13", + ), + ] + + def formula(person, period, parameters): + income_tax = parameters(period).gov.hmrc.income_tax + employment = person("employment_income", period) + trading = max_(0, person("self_employment_income", period)) + net_pay_pension = person("employee_pension_contributions", period) + taxable = max_( + 0, + employment + + trading + - net_pay_pension + - income_tax.allowances.personal_allowance.amount, + ) + tax = where( + person("pays_scottish_income_tax", period), + income_tax.rates.scotland.rates.calc(taxable), + income_tax.rates.uk.calc(taxable), + ) + national_insurance = person("uc_national_insurance_on_earnings", period) + half_pension_contributions = 0.5 * person("pension_contributions", period) + return max_( + 0, + employment + + trading + - tax + - national_insurance + - half_pension_contributions, + ) diff --git a/policyengine_uk/variables/gov/dwp/carers_allowance_pre_overlap.py b/policyengine_uk/variables/gov/dwp/carers_allowance_pre_overlap.py index ca06fafbb2..6bca6f6b93 100644 --- a/policyengine_uk/variables/gov/dwp/carers_allowance_pre_overlap.py +++ b/policyengine_uk/variables/gov/dwp/carers_allowance_pre_overlap.py @@ -25,9 +25,16 @@ def formula(person, period, parameters): weekly_care_hours = person("care_hours", period) meets_work_condition = weekly_care_hours >= ca.min_hours would_claim = person("would_claim_carers_allowance", period) + # SSCBA 1992 s. 70(1)(b): no allowance for anyone gainfully employed, + # which reg. 8 of the 1976 Regulations defines by an earnings limit. + # It applies to reported recipients too. + gainfully_employed = person( + "is_gainfully_employed_for_carers_allowance", period + ) eligible = ( ~(in_scotland & csp_replaces_ca) & (meets_work_condition | receives_ca) + & ~gainfully_employed & would_claim ) return eligible * ca.rate * WEEKS_IN_YEAR diff --git a/policyengine_uk/variables/gov/dwp/is_gainfully_employed_for_carers_allowance.py b/policyengine_uk/variables/gov/dwp/is_gainfully_employed_for_carers_allowance.py new file mode 100644 index 0000000000..f3befdd60b --- /dev/null +++ b/policyengine_uk/variables/gov/dwp/is_gainfully_employed_for_carers_allowance.py @@ -0,0 +1,29 @@ +from policyengine_uk.model_api import * + + +class is_gainfully_employed_for_carers_allowance(Variable): + value_type = bool + entity = Person + label = "gainfully employed for Carer's Allowance" + documentation = ( + "Whether the person's weekly net earnings exceed the Carer's Allowance " + "earnings limit, so that they are treated as gainfully employed and " + "are not entitled to the allowance. The regulation tests the earnings " + "of the preceding week; annual earnings are spread evenly over the year." + ) + definition_period = YEAR + reference = [ + dict( + title="Social Security Contributions and Benefits Act 1992 s. 70(1)(b)", + href="https://www.legislation.gov.uk/ukpga/1992/4/section/70", + ), + dict( + title="The Social Security (Invalid Care Allowance) Regulations 1976 reg. 8(1)", + href="https://www.legislation.gov.uk/uksi/1976/409/regulation/8", + ), + ] + + def formula(person, period, parameters): + limit = parameters(period).gov.dwp.carers_allowance.earnings_limit + weekly_earnings = person("carers_allowance_earnings", period) / WEEKS_IN_YEAR + return weekly_earnings > limit From 9b589b311a12d6a5c96f76985efe4bba49be45b3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 12:47:01 -0400 Subject: [PATCH 10/25] Round the High Income Child Benefit Charge as ITEPA 2003 s. 681C(3) requires Fixes #2321. The charge used the unrounded share of the phase-out band. The percentage is rounded down to a whole number, and the Child Benefit total and the charge down to whole pounds. Three tests that encoded unrounded charges now expect the statutory ones; the softer-taper opt-out case moves its income so the rounded charge stays positive. Co-Authored-By: Claude Opus 5.5 --- changelog.d/2321.fixed.md | 1 + .../income/uc_earnings_deductions.yaml | 8 +- .../gov/hmrc/child_benefit/opt_out.yaml | 10 +- .../charges/child_benefit_hitc.yaml | 93 ++++++++++++++++++- ...hild_benefit_charge_rounding_properties.py | 90 ++++++++++++++++++ policyengine_uk/utils/child_benefit.py | 23 +++++ .../income_tax/charges/child_benefit_hitc.py | 28 +++++- 7 files changed, 238 insertions(+), 15 deletions(-) create mode 100644 changelog.d/2321.fixed.md create mode 100644 policyengine_uk/tests/test_child_benefit_charge_rounding_properties.py diff --git a/changelog.d/2321.fixed.md b/changelog.d/2321.fixed.md new file mode 100644 index 0000000000..4c44b28086 --- /dev/null +++ b/changelog.d/2321.fixed.md @@ -0,0 +1 @@ +- Round the High Income Child Benefit Charge as ITEPA 2003 s. 681C(3) requires: the percentage down to a whole number, and the Child Benefit and the charge down to whole pounds. diff --git a/policyengine_uk/tests/policy/baseline/finance/benefit/family/universal_credit/income/uc_earnings_deductions.yaml b/policyengine_uk/tests/policy/baseline/finance/benefit/family/universal_credit/income/uc_earnings_deductions.yaml index d5e2aef012..99132fcbd3 100644 --- a/policyengine_uk/tests/policy/baseline/finance/benefit/family/universal_credit/income/uc_earnings_deductions.yaml +++ b/policyengine_uk/tests/policy/baseline/finance/benefit/family/universal_credit/income/uc_earnings_deductions.yaml @@ -272,10 +272,10 @@ household: members: [parent, child] output: - # Child Benefit 27.05 x 52 = 1,406.60; the charge takes - # (70,000 - 60,000) / 200 = 50% of it, 703.30. - CB_HITC: [703.30, 0] - income_tax: [16_135.30, 0] + # Child Benefit 27.05 x 52 = 1,406.60, rounded down to £1,406; the charge + # takes (70,000 - 60,000) / 200 = 50% of it, £703 (s. 681C(3)). + CB_HITC: [703, 0] + income_tax: [16_135, 0] # 37,700 x 20% + (57,430 - 37,700) x 40%. uc_income_tax_on_earnings: [15_432, 0] diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/child_benefit/opt_out.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/child_benefit/opt_out.yaml index 248ef3aa10..3f0d456896 100644 --- a/policyengine_uk/tests/policy/baseline/gov/hmrc/child_benefit/opt_out.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/child_benefit/opt_out.yaml @@ -87,7 +87,7 @@ absolute_error_margin: 0.01 input: people: - person1: {age: 40, adjusted_net_income: 100_000} + person1: {age: 40, adjusted_net_income: 200_000} person2: {age: 8} person3: {age: 5} benunits: @@ -98,8 +98,9 @@ output: child_benefit_entitlement: 2_337.40 child_benefit: 2_337.40 - # (100,000 - 60,000) / (10,000,000 - 60,000) * 2,337.40. - CB_HITC: [9.406036, 0, 0] + # (200,000 - 60,000) / (10,000,000 - 60,000) = 1.41%, rounded down to 1% + # (ITEPA 2003 s. 681C(3)); 1% of £2,337 is £23.37, rounded down to £23. + CB_HITC: [23, 0, 0] - name: Case 9, the configurable opt-out threshold includes its exact boundary. period: 2026 @@ -111,7 +112,8 @@ gov.hmrc.child_benefit.opt_out_charge_share: 0.5 output: child_benefit: [1_000, 0, 0] - CB_HITC: [499.95, 0, 0] + # 49.995% is rounded down to 49% (s. 681C(3)): £490. + CB_HITC: [490, 0, 0] - name: Case 10, a zero behavioural threshold still requires a positive charge. period: 2026 diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/charges/child_benefit_hitc.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/charges/child_benefit_hitc.yaml index c20990a727..860e9ce911 100644 --- a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/charges/child_benefit_hitc.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/charges/child_benefit_hitc.yaml @@ -47,6 +47,95 @@ members: [person1, person2, person3, person4] output: child_benefit: 2_337.40 - CB_HITC: [2_337.40, 0, 0, 0] + # s. 681C(3): the Child Benefit total is rounded down to £2,337. + CB_HITC: [2_337, 0, 0, 0] income_tax_pre_charges: [35_742, 4_486, 0, 0] - income_tax: [38_079.40, 4_486, 0, 0] + income_tax: [38_079, 4_486, 0, 0] + +# ITEPA 2003 s. 681C(3): the Child Benefit total and the charge are rounded +# down to whole pounds, and the appropriate percentage ((ANI - £60,000) / £200 +# from 2024-25) to a whole number. +- name: Case 6, the percentage, the Child Benefit and the charge are rounded down. + period: 2026 + absolute_error_margin: 0 + input: + people: + parent: + age: 40 + employment_income: 71_336 + child: + age: 5 + benunit: + members: [parent, child] + output: + # (71,336 - 60,000) / 200 = 56.68, so 56%; £1,406.60 becomes £1,406; + # 56% of £1,406 is £787.36, so £787. + child_benefit: 1_406.60 + CB_HITC: [787, 0] + +- name: Case 7, a whole-number percentage is not rounded below itself. + period: 2026 + absolute_error_margin: 0 + input: + people: + parent: + age: 40 + employment_income: 70_000 + child: + age: 5 + benunit: + members: [parent, child] + output: + # (70,000 - 60,000) / 200 = 50%; 50% of £1,406 is £703. + CB_HITC: [703, 0] + +- name: Case 8, just under the next £200 step keeps the lower percentage. + period: 2026 + absolute_error_margin: 0 + input: + people: + parent: + age: 40 + employment_income: 70_199 + child: + age: 5 + benunit: + members: [parent, child] + output: + # 50.995% rounds down to 50%: £703. + CB_HITC: [703, 0] + +- name: Case 9, at or above £80,000 the whole Child Benefit, in whole pounds. + period: 2026 + absolute_error_margin: 0 + input: + people: + parent: + age: 40 + employment_income: 90_000 + child1: + age: 5 + child2: + age: 3 + benunit: + members: [parent, child1, child2] + output: + child_benefit: 2_337.40 + CB_HITC: [2_337, 0, 0] + +- name: Case 10, the 2023-24 rules, 1% per £100 above £50,000. + period: 2023 + absolute_error_margin: 0 + input: + people: + parent: + age: 40 + employment_income: 55_550 + child: + age: 5 + benunit: + members: [parent, child] + child_benefit: 1_000 + output: + # (55,550 - 50,000) / 100 = 55.5%, so 55%; 55% of £1,000 is £550. + CB_HITC: [550, 0] diff --git a/policyengine_uk/tests/test_child_benefit_charge_rounding_properties.py b/policyengine_uk/tests/test_child_benefit_charge_rounding_properties.py new file mode 100644 index 0000000000..7744da65b6 --- /dev/null +++ b/policyengine_uk/tests/test_child_benefit_charge_rounding_properties.py @@ -0,0 +1,90 @@ +"""High Income Child Benefit Charge rounding (ITEPA 2003 s. 681C(3)). + +The charge is the appropriate percentage of the Child Benefit concerned: +(ANI - L) / X percent, capped at 100% (s. 681C(2); L = £60,000 and X = £200 +from 2024-25). Subsection (3) rounds the Child Benefit total and the charge +down to whole pounds, and the percentage down to a whole number. + +Invariants, for any adjusted net income and Child Benefit: + +1. The charge is a whole number of pounds. +2. Bounds: 0 <= charge <= Child Benefit rounded down to pounds. +3. Differential: the charge equals floor(floor((ANI - L) / X) / 100 x floor(CB)) + with the percentage capped at 100, computed here from the statute in + integer arithmetic. +4. Monotonic: a higher income never lowers the charge. +5. The model's charge from a full simulation matches the statute. +""" + +import numpy as np +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk import Simulation +from policyengine_uk.utils.child_benefit import ( + child_benefit_charge_percentage, + round_down_to_pounds, +) + +L = 60_000 +X = 200 +END = L + 100 * X + + +def statutory_charge(ani_pounds: int, child_benefit_pence: int) -> int: + """s. 681C in integer arithmetic: whole pounds, whole percentage points.""" + if ani_pounds <= L: + return 0 + points = min(100, (ani_pounds - L) // X) + pounds = child_benefit_pence // 100 + return (points * pounds) // 100 + + +def model_charge(ani: float, child_benefit: float) -> float: + percentage = child_benefit_charge_percentage(np.array([ani]), L, END) + return float( + round_down_to_pounds(percentage * round_down_to_pounds(child_benefit))[0] + ) + + +incomes = st.integers(min_value=0, max_value=200_000) +benefit_pence = st.integers(min_value=0, max_value=1_000_000) + + +@settings(max_examples=500, deadline=None) +@given(ani=incomes, cb_pence=benefit_pence) +def test_charge_matches_the_statute(ani, cb_pence): + child_benefit = cb_pence / 100 + charge = model_charge(ani, child_benefit) + assert charge == int(charge) + assert 0 <= charge <= cb_pence // 100 + assert charge == statutory_charge(ani, cb_pence) + + +@settings(max_examples=300, deadline=None) +@given( + ani=incomes, rise=st.integers(min_value=0, max_value=50_000), cb_pence=benefit_pence +) +def test_a_higher_income_never_lowers_the_charge(ani, rise, cb_pence): + child_benefit = cb_pence / 100 + assert model_charge(ani + rise, child_benefit) >= model_charge(ani, child_benefit) + + +@settings(max_examples=8, deadline=None) +@given(pay=st.integers(min_value=55_000, max_value=90_000), children=st.integers(1, 3)) +def test_simulated_charge_matches_the_statute(pay, children): + people = {"parent": {"age": {"2026": 40}, "employment_income": {"2026": pay}}} + for i in range(children): + people[f"child{i}"] = {"age": {"2026": 3 + i}} + members = list(people) + sim = Simulation( + situation={ + "people": people, + "benunits": {"b": {"members": members}}, + "households": {"h": {"members": members, "country": {"2026": "ENGLAND"}}}, + } + ) + ani = float(sim.calculate("adjusted_net_income", 2026)[0]) + child_benefit = float(sim.calculate("child_benefit", 2026)[0]) + charge = float(sim.calculate("CB_HITC", 2026)[0]) + assert charge == statutory_charge(int(ani), round(child_benefit * 100)) diff --git a/policyengine_uk/utils/child_benefit.py b/policyengine_uk/utils/child_benefit.py index e8d1092c38..b2b093a778 100644 --- a/policyengine_uk/utils/child_benefit.py +++ b/policyengine_uk/utils/child_benefit.py @@ -1,5 +1,9 @@ import numpy as np +# Floating-point slack, so an exact whole number computed as 55.999999... +# is not rounded down to the number below. +ROUNDING_TOLERANCE = 1e-9 + def child_benefit_charge_share( income: np.ndarray, phase_out_start: float, phase_out_end: float @@ -14,3 +18,22 @@ def child_benefit_charge_share( where=width > 0, ) return np.clip(share, 0, 1) + + +def child_benefit_charge_percentage( + income: np.ndarray, phase_out_start: float, phase_out_end: float +) -> np.ndarray: + """The appropriate percentage of ITEPA 2003 s. 681C(2), as a fraction. + + The share of the phase-out band crossed is a percentage of one point per + hundredth of the band (one point per £200 from 2024-25), which s. 681C(3)(b) + rounds down to a whole number. + """ + share = child_benefit_charge_share(income, phase_out_start, phase_out_end) + return np.floor(share * 100 + ROUNDING_TOLERANCE) / 100 + + +def round_down_to_pounds(amount: np.ndarray) -> np.ndarray: + """ITEPA 2003 s. 681C(3)(a): amounts not a whole number of pounds are + rounded down.""" + return np.floor(np.asarray(amount) + ROUNDING_TOLERANCE) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/charges/child_benefit_hitc.py b/policyengine_uk/variables/gov/hmrc/income_tax/charges/child_benefit_hitc.py index b5add0442d..a18a822f9e 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/charges/child_benefit_hitc.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/charges/child_benefit_hitc.py @@ -1,21 +1,39 @@ from policyengine_uk.model_api import * -from policyengine_uk.utils.child_benefit import child_benefit_charge_share +from policyengine_uk.utils.child_benefit import ( + child_benefit_charge_percentage, + round_down_to_pounds, +) class CB_HITC(Variable): value_type = float entity = Person label = "Child Benefit High-Income Tax Charge" + documentation = ( + "The appropriate percentage of the Child Benefit received, charged on " + "the partner with the higher adjusted net income. Section 681C(3) " + "rounds down the Child Benefit total and the charge to whole pounds, " + "and the percentage to a whole number." + ) definition_period = YEAR - reference = "https://www.legislation.gov.uk/ukpga/2003/1/part/10/chapter/8" + reference = [ + dict( + title="Income Tax (Earnings and Pensions) Act 2003 s. 681C", + href="https://www.legislation.gov.uk/ukpga/2003/1/section/681C", + ), + dict( + title="Income Tax (Earnings and Pensions) Act 2003 Part 10 Chapter 8", + href="https://www.legislation.gov.uk/ukpga/2003/1/part/10/chapter/8", + ), + ] unit = GBP defined_for = "is_higher_earner" def formula(person, period, parameters): - CB_received = person.benunit("child_benefit", period) + CB_received = round_down_to_pounds(person.benunit("child_benefit", period)) hitc = parameters(period).gov.hmrc.income_tax.charges.CB_HITC income = person("adjusted_net_income", period) - percentage = child_benefit_charge_share( + percentage = child_benefit_charge_percentage( income, hitc.phase_out_start, hitc.phase_out_end ) - return percentage * CB_received + return round_down_to_pounds(percentage * CB_received) From d5e40764042b4bd31ad3616337cb3982f8d483ff Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 12:54:54 -0400 Subject: [PATCH 11/25] Relieve a miscellaneous loss only against miscellaneous income (ITA 2007 s. 152(4)) Fixes #2326. taxable_miscellaneous_income added miscellaneous_income as it stood, so a loss reduced total and adjusted net income and the tax on pay, pensions, savings and dividends. A net loss now counts as nil; carrying it forward is not modelled. Co-Authored-By: Claude Opus 5.5 --- changelog.d/2326.fixed.md | 1 + .../bases/taxable_miscellaneous_income.yaml | 38 ++++++++++ .../test_miscellaneous_loss_properties.py | 69 +++++++++++++++++++ .../bases/taxable_miscellaneous_income.py | 24 +++++-- 4 files changed, 127 insertions(+), 5 deletions(-) create mode 100644 changelog.d/2326.fixed.md create mode 100644 policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.yaml create mode 100644 policyengine_uk/tests/test_miscellaneous_loss_properties.py diff --git a/changelog.d/2326.fixed.md b/changelog.d/2326.fixed.md new file mode 100644 index 0000000000..8692add888 --- /dev/null +++ b/changelog.d/2326.fixed.md @@ -0,0 +1 @@ +- Stop relieving a miscellaneous loss against other income: ITA 2007 s. 152(4) allows it only against miscellaneous income. diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.yaml new file mode 100644 index 0000000000..42874616cd --- /dev/null +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.yaml @@ -0,0 +1,38 @@ +# A miscellaneous loss is relieved only against miscellaneous income +# (ITA 2007 s. 152(4)). 2026-27: personal allowance £12,570, basic rate 20% +# to £50,270, higher rate 40%. + +- name: Case 1, miscellaneous income is taxed with other income. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 30_000 + miscellaneous_income: 5_000 + output: + taxable_miscellaneous_income: 5_000 + # (35,000 - 12,570) x 20%. + income_tax: 4_486 + +- name: Case 2, a miscellaneous loss does not reduce tax on pay. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 50_000 + miscellaneous_income: -10_000 + output: + taxable_miscellaneous_income: 0 + # (50,000 - 12,570) x 20%, as with no loss. + income_tax: 7_486 + +- name: Case 3, nor adjusted net income. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 110_000 + miscellaneous_income: -20_000 + output: + taxable_miscellaneous_income: 0 + adjusted_net_income: 110_000 diff --git a/policyengine_uk/tests/test_miscellaneous_loss_properties.py b/policyengine_uk/tests/test_miscellaneous_loss_properties.py new file mode 100644 index 0000000000..4238e66f14 --- /dev/null +++ b/policyengine_uk/tests/test_miscellaneous_loss_properties.py @@ -0,0 +1,69 @@ +"""A miscellaneous loss is relieved only against miscellaneous income +(ITA 2007 s. 152(4)). + +Invariants, for any pay, savings, dividends and pension income: + +1. A miscellaneous loss leaves Income Tax and adjusted net income exactly as + they are with no miscellaneous income. +2. taxable_miscellaneous_income is never negative and equals the income when + it is positive. +""" + +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from policyengine_uk import Simulation + + +def _calc(misc, pay, savings, dividends, pension): + situation = { + "people": { + "p": { + "age": {"2026": 50}, + "employment_income": {"2026": pay}, + "savings_interest_income": {"2026": savings}, + "dividend_income": {"2026": dividends}, + "private_pension_income": {"2026": pension}, + "miscellaneous_income": {"2026": misc}, + } + }, + "benunits": {"b": {"members": ["p"]}}, + "households": {"h": {"members": ["p"], "country": {"2026": "ENGLAND"}}}, + } + sim = Simulation(situation=situation) + return { + name: float(sim.calculate(name, 2026)[0]) + for name in ( + "income_tax", + "adjusted_net_income", + "taxable_miscellaneous_income", + ) + } + + +amounts = st.floats(min_value=0, max_value=150_000, allow_nan=False) + + +@settings(max_examples=10, deadline=None, suppress_health_check=[HealthCheck.too_slow]) +@given( + loss=st.floats(min_value=1, max_value=500_000, allow_nan=False), + pay=amounts, + savings=st.floats(min_value=0, max_value=20_000), + dividends=st.floats(min_value=0, max_value=20_000), + pension=st.floats(min_value=0, max_value=40_000), +) +def test_a_miscellaneous_loss_changes_nothing_else( + loss, pay, savings, dividends, pension +): + with_loss = _calc(-loss, pay, savings, dividends, pension) + without = _calc(0, pay, savings, dividends, pension) + assert with_loss["taxable_miscellaneous_income"] == 0 + assert abs(with_loss["income_tax"] - without["income_tax"]) < 0.01 + assert abs(with_loss["adjusted_net_income"] - without["adjusted_net_income"]) < 0.01 + + +@settings(max_examples=6, deadline=None, suppress_health_check=[HealthCheck.too_slow]) +@given(income=st.floats(min_value=0, max_value=100_000, allow_nan=False), pay=amounts) +def test_positive_miscellaneous_income_is_taxable_in_full(income, pay): + result = _calc(income, pay, 0, 0, 0) + assert abs(result["taxable_miscellaneous_income"] - income) < 0.01 diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.py index 30f407bf45..563c959055 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.py @@ -5,11 +5,25 @@ class taxable_miscellaneous_income(Variable): value_type = float entity = Person label = "Amount of miscellaneous income that is taxable" - definition_period = YEAR - reference = dict( - title="Income Tax Act (Trading and Other Income) 2005, s. 574", - href="https://www.legislation.gov.uk/ukpga/2005/5/section/574", + documentation = ( + "Miscellaneous income charged to income tax. A miscellaneous loss can " + "be relieved only against miscellaneous income (ITA 2007 s. 152(4)), " + "never against other income, so a net loss counts as nil here. Carrying " + "the loss forward against later years' miscellaneous income is not " + "modelled." ) + definition_period = YEAR + reference = [ + dict( + title="Income Tax (Trading and Other Income) Act 2005 s. 574", + href="https://www.legislation.gov.uk/ukpga/2005/5/section/574", + ), + dict( + title="Income Tax Act 2007 s. 152", + href="https://www.legislation.gov.uk/ukpga/2007/3/section/152", + ), + ] unit = GBP - adds = ["miscellaneous_income"] + def formula(person, period, parameters): + return max_(0, person("miscellaneous_income", period)) From ba13d61f88c2f410a52177f6e68c1429f8f288e8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 12:56:11 -0400 Subject: [PATCH 12/25] Test that the Child Benefit total is rounded down before the percentage applies Co-Authored-By: Claude Opus 5.5 --- .../income_tax/charges/child_benefit_hitc.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/charges/child_benefit_hitc.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/charges/child_benefit_hitc.yaml index 860e9ce911..f67d208ec7 100644 --- a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/charges/child_benefit_hitc.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/charges/child_benefit_hitc.yaml @@ -139,3 +139,20 @@ output: # (55,550 - 50,000) / 100 = 55.5%, so 55%; 55% of £1,000 is £550. CB_HITC: [550, 0] + +- name: Case 11, the Child Benefit total is rounded down before the percentage applies. + period: 2026 + absolute_error_margin: 0 + input: + people: + parent: + age: 40 + employment_income: 79_800 + child: + age: 5 + benunit: + members: [parent, child] + output: + # (79,800 - 60,000) / 200 = 99%; 99% of £1,406 is £1,391.94, so £1,391. + # Rounding only the charge (99% of £1,406.60 = £1,392.53) would give £1,392. + CB_HITC: [1_391, 0] From ae1869cd71511bb4868c9ac26452caac8af93430 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 13:15:01 -0400 Subject: [PATCH 13/25] Update the opt-out Python tests to the statutory rounding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cliff case now expects the Child Benefit rounded down to pounds, and the dataset softer-taper case moves its taper end so the rounded charge stays positive (2%, £46). Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/tests/test_child_benefit_opt_out.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/policyengine_uk/tests/test_child_benefit_opt_out.py b/policyengine_uk/tests/test_child_benefit_opt_out.py index 012ef7ceca..54e235fea5 100644 --- a/policyengine_uk/tests/test_child_benefit_opt_out.py +++ b/policyengine_uk/tests/test_child_benefit_opt_out.py @@ -76,7 +76,9 @@ def test_cliff_charge_has_no_division_warnings(income, opts_out, taper_end): charge = simulation.calculate("CB_HITC", 2026).sum() entitlement = simulation.calculate("child_benefit_entitlement", 2026)[0] assert payment == (0 if opts_out and income > 60_000 else entitlement) - assert charge == (payment if income > 60_000 else 0) + # A cliff charges the whole Child Benefit, rounded down to whole pounds + # (ITEPA 2003 s. 681C(3)(a)). + assert charge == (np.floor(payment) if income > 60_000 else 0) @pytest.mark.usefixtures("cloned_uk_tax_benefit_system") @@ -106,9 +108,13 @@ def test_cliff_charge_has_no_division_warnings(income, opts_out, taper_end): 0, ), ( - {"gov.hmrc.income_tax.charges.CB_HITC.phase_out_end": {"2026": 10_000_000}}, + # A softer taper: (100,000 - 60,000) / (2,000,000 - 60,000) = 2.06%, + # rounded down to 2% (ITEPA 2003 s. 681C(3)(b)); 2% of £2,337 is + # £46.74, rounded down to £46. Payment resumes because the share is + # below the opt-out threshold. + {"gov.hmrc.income_tax.charges.CB_HITC.phase_out_end": {"2026": 2_000_000}}, 2_337.40, - 2_337.40 * 40_000 / 9_940_000, + 46, ), ], ) From 0d15fc733bd8a1db65c7f580e1878055f9613bc2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 14:43:25 -0400 Subject: [PATCH 14/25] Follow regs 13-14 and reg 5 in the Carer's Allowance earnings test; keep UC carer status Review changes: trading profit takes reg. 14's notional basic-rate tax and main-rate Class 4 rather than progressive PAYE, with one set of personal reliefs; a fraction of a penny of weekly earnings is disregarded (reg. 5); a reported award the person would claim still marks them as a carer for Universal Credit when the earnings limit stops payment (UC reg. 30 has no earnings test); the PAYE approximation is described as one. A property test that assumes an entitled carer now keeps the carer's pay under the limit. Co-Authored-By: Claude Opus 5.5 --- .../dwp/carers_allowance_earnings_limit.yaml | 80 +++++++++++++++- ...ers_allowance_earnings_limit_properties.py | 3 +- ...l_tax_reduction_carer_income_properties.py | 6 ++ .../gov/dwp/carers_allowance_earnings.py | 94 +++++++++++++------ ...gainfully_employed_for_carers_allowance.py | 16 +++- .../demographic/is_carer_for_benefits.py | 9 ++ 6 files changed, 171 insertions(+), 37 deletions(-) diff --git a/policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml b/policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml index fec4df3ccb..8e28e7a095 100644 --- a/policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml @@ -4,7 +4,8 @@ # earnings exceed £204 a week from 6 April 2026 (SI 2026/218). Net earnings # are gross pay less income tax, primary Class 1 contributions and half of # pension contributions (Computation of Earnings Regulations 1996 reg. -# 10(4)), and trading profit less tax, Class 2 and Class 4 (reg. 13). +# 10(4)), and trading profit less a notional basic-rate tax and main-rate +# Class 4 (regs. 13(4) and 14). A fraction of a penny is disregarded (reg. 5). # 2026-27: personal allowance and NI primary threshold £12,570, employee # Class 1 main rate 8%, Class 4 lower profits limit £12,570 at 6%, basic rate # 20%. Carer's Allowance £86.45 a week. @@ -103,7 +104,7 @@ carers_allowance_earnings: 0 carers_allowance: 86.45 * 52 -- name: Case 8, a reported recipient over the limit is not entitled either. +- name: Case 8, a reported recipient over the limit is not entitled either, but stays a carer for Universal Credit. period: 2026 absolute_error_margin: 0.01 input: @@ -114,6 +115,8 @@ output: is_gainfully_employed_for_carers_allowance: true carers_allowance: 0 + # UC regulation 30 does not apply the Carer's Allowance earnings limit. + is_carer_for_benefits: true - name: Case 9, savings and pension income are not earnings. period: 2026 @@ -140,3 +143,76 @@ output: is_gainfully_employed_for_carers_allowance: true carers_allowance: 0 + +- name: Case 11, trading profit takes notional basic-rate tax and main-rate Class 4 only. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + self_employment_income: 60_000 + care_hours: 40 + country: ENGLAND + output: + # Reg. 14: 20% x (60,000 - 12,570) = 9,486, not the progressive tax; + # 6% x (50,270 - 12,570) = 2,262, with no additional-rate Class 4. + carers_allowance_earnings: 60_000 - 9_486 - 2_262 + is_gainfully_employed_for_carers_allowance: true + carers_allowance: 0 + +- name: Case 12, half of personal pension contributions brings trading earnings under the limit. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + self_employment_income: 15_000 + personal_pension_contributions: 7_700 + care_hours: 40 + country: ENGLAND + output: + # 15,000 - 20% x 2,430 (486) - 6% x 2,430 (145.80) - 7,700 / 2 (3,850) + # = 10,518.20, or £202.27 a week. + carers_allowance_earnings: 10_518.20 + is_gainfully_employed_for_carers_allowance: false + carers_allowance: 86.45 * 52 + +- name: Case 13, a fraction of a penny over £204 is disregarded. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 10_608.26 + care_hours: 40 + country: ENGLAND + output: + # 10,608.26 / 52 = £204.005, which counts as £204.00 (reg. 5). + is_gainfully_employed_for_carers_allowance: false + carers_allowance: 86.45 * 52 + +- name: Case 14, a whole penny over £204 is over the limit. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 10_608.52 + care_hours: 40 + country: ENGLAND + output: + # 10,608.52 / 52 = £204.01. + is_gainfully_employed_for_carers_allowance: true + carers_allowance: 0 + +- name: Case 15, pay and trading profit share one personal allowance. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 10_000 + self_employment_income: 5_000 + care_hours: 40 + country: ENGLAND + output: + # Pay uses £10,000 of the allowance, leaving £2,570: trading tax + # 20% x (5,000 - 2,570) = 486. No Class 1 on £10,000; no Class 4 under + # £12,570. 15,000 - 486 = 14,514. + carers_allowance_earnings: 14_514 + is_gainfully_employed_for_carers_allowance: true diff --git a/policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py b/policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py index d59d49f038..e6cbb5901d 100644 --- a/policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py +++ b/policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py @@ -76,7 +76,8 @@ def test_bounds_and_differential(pay, profit, pension_share): result = _person(pay, profit, pension) earnings = result["carers_allowance_earnings"] assert -0.01 <= earnings <= pay + max(0, profit) + 0.01 - over = earnings / WEEKS > LIMIT + # Regulation 5: a fraction of a penny is disregarded. + over = np.floor(earnings / WEEKS * 100 + 1e-6) / 100 > LIMIT assert bool(result["is_gainfully_employed_for_carers_allowance"]) == over expected = 0 if over else RATE * WEEKS assert abs(result["carers_allowance"] - expected) < 0.01 diff --git a/policyengine_uk/tests/test_council_tax_reduction_carer_income_properties.py b/policyengine_uk/tests/test_council_tax_reduction_carer_income_properties.py index 8795cdd8c6..e317bf68e5 100644 --- a/policyengine_uk/tests/test_council_tax_reduction_carer_income_properties.py +++ b/policyengine_uk/tests/test_council_tax_reduction_carer_income_properties.py @@ -192,6 +192,12 @@ def situation(families): "private_pension_income": {YEAR: float(pension) * (j == 0)}, } if j == fam["carer"]: + # Keep the carer's pay within the Carer's Allowance + # earnings limit (£204 a week in 2026-27), so the + # carer stays entitled; £10,000 is £192 a week. + person["employment_income"] = { + YEAR: min(a.get("employment_income", 0.0), 10_000.0) + } if fam["by_hours"]: person["care_hours"] = {YEAR: 35} else: diff --git a/policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py b/policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py index f853f306ed..73cee0fbe6 100644 --- a/policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py +++ b/policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py @@ -7,18 +7,26 @@ class carers_allowance_earnings(Variable): label = "earnings for the Carer's Allowance earnings limit" documentation = ( "Net earnings from employment and self-employment, as the Carer's " - "Allowance gainful-employment test counts them: gross pay and trading " - "profit less the income tax and National Insurance deducted from them " - "and half of the person's pension contributions. A trading loss counts " - "as nil. The income tax is the tax PAYE would deduct if the earnings " - "were the person's only income: earnings less net-pay pension " - "contributions and the standard personal allowance, at the person's " - "rest-of-UK or Scottish rates. That is the tax deducted from pay, and " - "it avoids a circularity, since Carer's Allowance is itself taxable. " - "The deduction for care charges (Computation of Earnings Regulations " - "1996 Schedule 3, up to half of net earnings) and the Schedule 1 " - "disregards are not modelled, so this can overstate earnings for a " - "carer who pays for care while at work." + "Allowance gainful-employment test counts them (Computation of " + "Earnings Regulations 1996), less half of the person's pension " + "contributions. A trading loss counts as nil.\n\n" + "Employment (reg. 10(4)): gross pay less the income tax and primary " + "Class 1 contributions deducted from it. The law deducts the tax " + "actually withheld; the model approximates it as the tax PAYE would " + "take on a standard code if the pay were the person's only income " + "(pay less net-pay pension contributions and the personal allowance, " + "at rest-of-UK or Scottish rates). That also avoids a circularity, " + "since Carer's Allowance is itself taxable, and makes unearned income " + "irrelevant here by assumption.\n\n" + "Self-employment (regs. 13(4) and 14): trading profit less a notional " + "income tax at the basic rate (the Scottish basic rate for a Scottish " + "taxpayer) on the profit above whatever personal allowance the pay " + "leaves unused, and Class 4 contributions at the main rate on the " + "profit between the lower and upper profits limits. Regulation 14 " + "deducts no Class 2 contribution.\n\n" + "Not modelled: the deduction for care charges (Schedule 3, up to half " + "of net earnings) and the Schedule 1 disregards, so this can overstate " + "earnings for a carer who pays for care while at work." ) definition_period = YEAR unit = GBP @@ -31,32 +39,56 @@ class carers_allowance_earnings(Variable): title="Social Security Benefit (Computation of Earnings) Regulations 1996 reg. 13", href="https://www.legislation.gov.uk/uksi/1996/2745/regulation/13", ), + dict( + title="Social Security Benefit (Computation of Earnings) Regulations 1996 reg. 14", + href="https://www.legislation.gov.uk/uksi/1996/2745/regulation/14", + ), ] def formula(person, period, parameters): - income_tax = parameters(period).gov.hmrc.income_tax - employment = person("employment_income", period) - trading = max_(0, person("self_employment_income", period)) - net_pay_pension = person("employee_pension_contributions", period) - taxable = max_( - 0, - employment - + trading - - net_pay_pension - - income_tax.allowances.personal_allowance.amount, + p = parameters(period).gov.hmrc + income_tax = p.income_tax + class_4 = p.national_insurance.class_4 + scottish = person("pays_scottish_income_tax", period) + personal_allowance = income_tax.allowances.personal_allowance.amount + + # Employment: PAYE on the pay alone (reg. 10(4)(a)). + pay = person("employment_income", period) + taxable_pay = max_(0, pay - person("employee_pension_contributions", period)) + pay_tax = where( + scottish, + income_tax.rates.scotland.rates.calc( + max_(0, taxable_pay - personal_allowance) + ), + income_tax.rates.uk.calc(max_(0, taxable_pay - personal_allowance)), ) - tax = where( - person("pays_scottish_income_tax", period), - income_tax.rates.scotland.rates.calc(taxable), - income_tax.rates.uk.calc(taxable), + primary_class_1 = person("ni_class_1_employee", period) + + # Self-employment: notional basic-rate tax and main-rate Class 4 + # (reg. 14(1)-(2)), with one set of personal reliefs, used by the pay + # first. + profit = max_(0, person("self_employment_income", period)) + unused_allowance = max_(0, personal_allowance - taxable_pay) + basic_rate = where( + scottish, + income_tax.rates.scotland.rates.rates[1], + income_tax.rates.uk.rates[0], + ) + trading_tax = basic_rate * max_(0, profit - unused_allowance) + trading_class_4 = class_4.rates.main * max_( + 0, + min_(profit, class_4.thresholds.upper_profits_limit) + - class_4.thresholds.lower_profits_limit, ) - national_insurance = person("uc_national_insurance_on_earnings", period) + half_pension_contributions = 0.5 * person("pension_contributions", period) return max_( 0, - employment - + trading - - tax - - national_insurance + pay + - pay_tax + - primary_class_1 + + profit + - trading_tax + - trading_class_4 - half_pension_contributions, ) diff --git a/policyengine_uk/variables/gov/dwp/is_gainfully_employed_for_carers_allowance.py b/policyengine_uk/variables/gov/dwp/is_gainfully_employed_for_carers_allowance.py index f3befdd60b..b6e6b4b981 100644 --- a/policyengine_uk/variables/gov/dwp/is_gainfully_employed_for_carers_allowance.py +++ b/policyengine_uk/variables/gov/dwp/is_gainfully_employed_for_carers_allowance.py @@ -9,7 +9,10 @@ class is_gainfully_employed_for_carers_allowance(Variable): "Whether the person's weekly net earnings exceed the Carer's Allowance " "earnings limit, so that they are treated as gainfully employed and " "are not entitled to the allowance. The regulation tests the earnings " - "of the preceding week; annual earnings are spread evenly over the year." + "of the preceding week; annual earnings are spread evenly over 52 " + "weeks. A fraction of a penny is disregarded, as regulation 5 of the " + "Computation of Earnings Regulations requires when that favours the " + "claimant." ) definition_period = YEAR reference = [ @@ -21,9 +24,16 @@ class is_gainfully_employed_for_carers_allowance(Variable): title="The Social Security (Invalid Care Allowance) Regulations 1976 reg. 8(1)", href="https://www.legislation.gov.uk/uksi/1976/409/regulation/8", ), + dict( + title="Social Security Benefit (Computation of Earnings) Regulations 1996 reg. 5", + href="https://www.legislation.gov.uk/uksi/1996/2745/regulation/5", + ), ] def formula(person, period, parameters): limit = parameters(period).gov.dwp.carers_allowance.earnings_limit - weekly_earnings = person("carers_allowance_earnings", period) / WEEKS_IN_YEAR - return weekly_earnings > limit + weekly = person("carers_allowance_earnings", period) / WEEKS_IN_YEAR + # Disregard a fraction of a penny; the tolerance keeps an exact + # number of pence from being knocked down by floating-point error. + weekly_in_pence = np.floor(weekly * 100 + 1e-6) / 100 + return weekly_in_pence > limit diff --git a/policyengine_uk/variables/household/demographic/is_carer_for_benefits.py b/policyengine_uk/variables/household/demographic/is_carer_for_benefits.py index 4c5e80aa56..f3ce7c1750 100644 --- a/policyengine_uk/variables/household/demographic/is_carer_for_benefits.py +++ b/policyengine_uk/variables/household/demographic/is_carer_for_benefits.py @@ -27,8 +27,17 @@ class is_carer_for_benefits(Variable): def formula(person, period, parameters): min_hours = parameters(period).gov.dwp.carers_allowance.min_hours meets_hours = person("care_hours", period) >= min_hours + # A reported Carer's Allowance award the person would claim is + # evidence of the caring itself, where no hours are recorded. It still + # counts when the earnings limit (which the Universal Credit carer + # element does not apply) stops the allowance being paid. An award the + # person would not claim counts no more than it did before. + reports_carers_allowance = ( + person("carers_allowance_reported", period) > 0 + ) & person("would_claim_carers_allowance", period) return ( person("receives_carer_benefit", period) | person("is_entitled_to_carer_benefit", period) | meets_hours + | reports_carers_allowance ) From 7145d5b6085e0de403e4fca3df28f6ab5abbdb6f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 14:53:58 -0400 Subject: [PATCH 15/25] Keep a miscellaneous loss out of total income and UC earnings too Review changes: total_income (ITA 2007 s. 23 Step 1) and Universal Credit earned income now read taxable_miscellaneous_income, so a loss neither reduces total income (the Winter Fuel test, non-dependant deductions) nor widens the UC earnings error in #1961. The docstring states s. 152(5)'s same-provision limit and the input's wider scope. Tests add the personal allowance taper, total income and UC earnings; the property test covers total income, with 25 examples and a taper-region example. Co-Authored-By: Claude Opus 5.5 --- changelog.d/2326.fixed.md | 2 +- .../bases/taxable_miscellaneous_income.yaml | 25 +++++++++++++++++++ .../test_miscellaneous_loss_properties.py | 16 ++++++------ .../uc_individual_earned_income_before_mif.py | 5 +++- .../bases/taxable_miscellaneous_income.py | 16 ++++++++---- .../gov/hmrc/income_tax/total_income.py | 3 ++- 6 files changed, 52 insertions(+), 15 deletions(-) diff --git a/changelog.d/2326.fixed.md b/changelog.d/2326.fixed.md index 8692add888..d486a2a5a0 100644 --- a/changelog.d/2326.fixed.md +++ b/changelog.d/2326.fixed.md @@ -1 +1 @@ -- Stop relieving a miscellaneous loss against other income: ITA 2007 s. 152(4) allows it only against miscellaneous income. +- Stop relieving a miscellaneous loss against other income in Income Tax, adjusted net income, total income and Universal Credit earnings: ITA 2007 s. 152(4) allows it only against miscellaneous income. diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.yaml index 42874616cd..08d5fcf444 100644 --- a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.yaml @@ -36,3 +36,28 @@ output: taxable_miscellaneous_income: 0 adjusted_net_income: 110_000 + # Personal allowance 12,570 - (110,000 - 100,000) / 2 = 7,570; taxable + # 102,430: 37,700 x 20% + 64,730 x 40% = 33,432. + income_tax: 33_432 + +- name: Case 4, total income ignores a miscellaneous loss. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 70 + private_pension_income: 40_000 + miscellaneous_income: -10_000 + output: + total_income: 40_000 + +- name: Case 5, a miscellaneous loss does not reduce Universal Credit earnings. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 30 + employment_income: 30_000 + miscellaneous_income: -10_000 + output: + # 30,000 less income tax (30,000 - 12,570) x 20% = 3,486 and primary + # Class 1 (30,000 - 12,570) x 8% = 1,394.40. + uc_individual_earned_income_before_mif: 30_000 - 3_486 - 1_394.40 diff --git a/policyengine_uk/tests/test_miscellaneous_loss_properties.py b/policyengine_uk/tests/test_miscellaneous_loss_properties.py index 4238e66f14..79ff5589aa 100644 --- a/policyengine_uk/tests/test_miscellaneous_loss_properties.py +++ b/policyengine_uk/tests/test_miscellaneous_loss_properties.py @@ -1,15 +1,15 @@ """A miscellaneous loss is relieved only against miscellaneous income (ITA 2007 s. 152(4)). -Invariants, for any pay, savings, dividends and pension income: +Invariants: -1. A miscellaneous loss leaves Income Tax and adjusted net income exactly as - they are with no miscellaneous income. -2. taxable_miscellaneous_income is never negative and equals the income when - it is positive. +1. For any pay, savings, dividends and pension income, a miscellaneous loss + leaves Income Tax, adjusted net income and total income exactly as they are + with no miscellaneous income. +2. For any pay, positive miscellaneous income is taxable in full. """ -from hypothesis import HealthCheck, given, settings +from hypothesis import HealthCheck, example, given, settings from hypothesis import strategies as st from policyengine_uk import Simulation @@ -44,7 +44,8 @@ def _calc(misc, pay, savings, dividends, pension): amounts = st.floats(min_value=0, max_value=150_000, allow_nan=False) -@settings(max_examples=10, deadline=None, suppress_health_check=[HealthCheck.too_slow]) +@settings(max_examples=25, deadline=None, suppress_health_check=[HealthCheck.too_slow]) +@example(loss=20_000, pay=110_000, savings=0, dividends=0, pension=0) @given( loss=st.floats(min_value=1, max_value=500_000, allow_nan=False), pay=amounts, @@ -60,6 +61,7 @@ def test_a_miscellaneous_loss_changes_nothing_else( assert with_loss["taxable_miscellaneous_income"] == 0 assert abs(with_loss["income_tax"] - without["income_tax"]) < 0.01 assert abs(with_loss["adjusted_net_income"] - without["adjusted_net_income"]) < 0.01 + assert abs(with_loss["total_income"] - without["total_income"]) < 0.01 @settings(max_examples=6, deadline=None, suppress_health_check=[HealthCheck.too_slow]) diff --git a/policyengine_uk/variables/gov/dwp/universal_credit/income/uc_individual_earned_income_before_mif.py b/policyengine_uk/variables/gov/dwp/universal_credit/income/uc_individual_earned_income_before_mif.py index 73a93702a1..9bf59a96a5 100644 --- a/policyengine_uk/variables/gov/dwp/universal_credit/income/uc_individual_earned_income_before_mif.py +++ b/policyengine_uk/variables/gov/dwp/universal_credit/income/uc_individual_earned_income_before_mif.py @@ -35,7 +35,10 @@ def formula(person, period, parameters): # The deductions are the person's own and come off only their own # earnings, so one partner's tax, NI or pension contributions never # reduce the other partner's earned income. - earnings_components = ["employment_income", "miscellaneous_income"] + # Miscellaneous income as income tax counts it, so a loss does not + # reduce pay (and matches uc_income_tax_on_earnings). Whether it is + # earnings at all is issue #1961. + earnings_components = ["employment_income", "taxable_miscellaneous_income"] bi = parameters(period).gov.contrib.ubi_center.basic_income if bi.interactions.include_in_means_tests: earnings_components.append("basic_income") diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.py index 563c959055..ce8be05b12 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.py @@ -6,11 +6,17 @@ class taxable_miscellaneous_income(Variable): entity = Person label = "Amount of miscellaneous income that is taxable" documentation = ( - "Miscellaneous income charged to income tax. A miscellaneous loss can " - "be relieved only against miscellaneous income (ITA 2007 s. 152(4)), " - "never against other income, so a net loss counts as nil here. Carrying " - "the loss forward against later years' miscellaneous income is not " - "modelled." + "Miscellaneous income charged to income tax. On a claim, a loss on a " + "miscellaneous transaction is relieved only against relevant " + "miscellaneous income: income under the same section 1016 provision as " + "the loss (ITA 2007 s. 152(4)-(5)). It is never set against other " + "income, so a net loss counts as nil here. The model holds one net " + "figure per person, so it assumes the items netted inside " + "miscellaneous_income can be relieved against each other; that can " + "undertax a person with a gain under one provision and a loss under " + "another. The input also covers income from sources outside section " + "1016, some of which carry no loss relief at all. Carrying a loss " + "forward against later years' miscellaneous income is not modelled." ) definition_period = YEAR reference = [ diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/total_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/total_income.py index f13a71fb41..0503c8447c 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/total_income.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/total_income.py @@ -20,5 +20,6 @@ class total_income(Variable): "property_income", "savings_interest_income", "dividend_income", - "miscellaneous_income", + # A miscellaneous loss is never part of total income (ITA 2007 s. 152(4)). + "taxable_miscellaneous_income", ] From 9e2192d193b68d61d88519044e4126be9ba5de29 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 14:56:33 -0400 Subject: [PATCH 16/25] Return total income from the miscellaneous-loss property test helper Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/tests/test_miscellaneous_loss_properties.py | 1 + 1 file changed, 1 insertion(+) diff --git a/policyengine_uk/tests/test_miscellaneous_loss_properties.py b/policyengine_uk/tests/test_miscellaneous_loss_properties.py index 79ff5589aa..e3f08a5734 100644 --- a/policyengine_uk/tests/test_miscellaneous_loss_properties.py +++ b/policyengine_uk/tests/test_miscellaneous_loss_properties.py @@ -36,6 +36,7 @@ def _calc(misc, pay, savings, dividends, pension): for name in ( "income_tax", "adjusted_net_income", + "total_income", "taxable_miscellaneous_income", ) } From 14f3f88dab378ec4d7349b317799904db2ead3e6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 15:10:04 -0400 Subject: [PATCH 17/25] Round the charge exactly in float32 and in the household-based reform MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fixes for #2323: - Work out the appropriate percentage in float64, multiplying before dividing, so every exact £200 step gives exactly that many points when incomes are held as float32 (50 of 100 steps lost a point before). - Take an amount to the nearest penny before rounding down to pounds, so four children's £4,199 stored as 4,198.9995 stays £4,199. - Apply the s. 681C(3) rounding in the household-based charge reform too. - Describe the opt-out test as the continuous charge share, as distinct from the rounded statutory charge. - Regression tests: float32 steps, four children, single-parent reform equivalence and the couple's combined charge. Co-Authored-By: Claude Opus 5.5 --- .../gov/hmrc/child_benefit/README.md | 14 ++-- .../conservatives/household_based_hitc.py | 12 +++- .../gov/hmrc/child_benefit/opt_out.yaml | 2 +- .../charges/child_benefit_hitc.yaml | 38 ++++++++++ ...hild_benefit_charge_rounding_properties.py | 71 +++++++++++++++++-- policyengine_uk/utils/child_benefit.py | 31 +++++--- .../gov/hmrc/child_benefit_opts_out.py | 10 +-- 7 files changed, 148 insertions(+), 30 deletions(-) diff --git a/policyengine_uk/parameters/gov/hmrc/child_benefit/README.md b/policyengine_uk/parameters/gov/hmrc/child_benefit/README.md index a5190ffabe..3e5e568dd5 100644 --- a/policyengine_uk/parameters/gov/hmrc/child_benefit/README.md +++ b/policyengine_uk/parameters/gov/hmrc/child_benefit/README.md @@ -29,11 +29,15 @@ data in a release; updating model code alone does not repair existing data. `gov.hmrc.child_benefit.opt_out_charge_share` is a behavioural assumption, not a statutory eligibility rule. It defaults to one: a flagged family continues -to opt out only while the charge would recover the full benefit. The charge -share is calculated independently of payment and clipped to [0, 1]. A reform -that reduces the share below the threshold, removes the charge or neutralises -`CB_HITC` restores payment. A zero charge never sustains an opt-out, even if -the assumed threshold is zero. Genuine nonclaimants remain outside payment. +to opt out only while the continuous charge share is one, that is while the +higher adjusted net income in the family is at or above the end of the +phase-out band. The share is the unrounded fraction of the band crossed, calculated independently of +payment and clipped to [0, 1]. A reform that reduces the share below the +threshold, removes the charge or neutralises `CB_HITC` restores payment. A zero +continuous charge share never sustains an opt-out, even if the assumed +threshold is zero. The statutory charge (`CB_HITC`) is different: ITEPA 2003 +s. 681C(3) rounds its percentage and amount down, so it can be nil while the +share is small but positive. Genuine nonclaimants remain outside payment. Microcosm selects fully charged families first and can draw from partly charged families if needed to meet its target. The model's default threshold is an diff --git a/policyengine_uk/reforms/conservatives/household_based_hitc.py b/policyengine_uk/reforms/conservatives/household_based_hitc.py index 861c04d3d0..05295e581d 100644 --- a/policyengine_uk/reforms/conservatives/household_based_hitc.py +++ b/policyengine_uk/reforms/conservatives/household_based_hitc.py @@ -1,4 +1,8 @@ from policyengine_uk.model_api import * +from policyengine_uk.utils.child_benefit import ( + child_benefit_charge_percentage, + round_down_to_pounds, +) class CB_HITC(Variable): @@ -15,10 +19,12 @@ def formula(person, period, parameters): hitc = parameters(period).gov.hmrc.income_tax.charges.CB_HITC personal_income = person("adjusted_net_income", period) income = person.benunit.sum(personal_income) - percentage = max_(income - hitc.phase_out_start, 0) / ( - hitc.phase_out_end - hitc.phase_out_start + # The same s. 681C(3) rounding as the baseline charge, on the + # household's combined income. + percentage = child_benefit_charge_percentage( + income, hitc.phase_out_start, hitc.phase_out_end ) - return min_(percentage, 1) * CB_received + return round_down_to_pounds(percentage * round_down_to_pounds(CB_received)) class make_cb_hitc_household_based(Variable): diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/child_benefit/opt_out.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/child_benefit/opt_out.yaml index 3f0d456896..45d122fe22 100644 --- a/policyengine_uk/tests/policy/baseline/gov/hmrc/child_benefit/opt_out.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/child_benefit/opt_out.yaml @@ -115,7 +115,7 @@ # 49.995% is rounded down to 49% (s. 681C(3)): £490. CB_HITC: [490, 0, 0] -- name: Case 10, a zero behavioural threshold still requires a positive charge. +- name: Case 10, a zero behavioural threshold still requires a positive continuous charge share. period: 2026 input: adjusted_net_income: [59_999, 60_000, 60_001] diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/charges/child_benefit_hitc.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/charges/child_benefit_hitc.yaml index f67d208ec7..ecfd90acb5 100644 --- a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/charges/child_benefit_hitc.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/charges/child_benefit_hitc.yaml @@ -156,3 +156,41 @@ # (79,800 - 60,000) / 200 = 99%; 99% of £1,406 is £1,391.94, so £1,391. # Rounding only the charge (99% of £1,406.60 = £1,392.53) would give £1,392. CB_HITC: [1_391, 0] + +- name: Case 12, an income on an exact £200 step gets that many points. + period: 2026 + absolute_error_margin: 0 + input: + people: + parent: + age: 40 + employment_income: 60_200 + child: + age: 5 + benunit: + members: [parent, child] + output: + # 1% of £1,406 is £14.06, so £14. + CB_HITC: [14, 0] + +- name: Case 13, four children's £4,199 is not rounded down to £4,198. + period: 2026 + absolute_error_margin: 0 + input: + people: + parent: + age: 40 + employment_income: 90_000 + child1: + age: 2 + child2: + age: 4 + child3: + age: 6 + child4: + age: 8 + benunit: + members: [parent, child1, child2, child3, child4] + output: + # 52 x (27.05 + 3 x 17.90) = 4,199. + CB_HITC: [4_199, 0, 0, 0, 0] diff --git a/policyengine_uk/tests/test_child_benefit_charge_rounding_properties.py b/policyengine_uk/tests/test_child_benefit_charge_rounding_properties.py index 7744da65b6..0f271add41 100644 --- a/policyengine_uk/tests/test_child_benefit_charge_rounding_properties.py +++ b/policyengine_uk/tests/test_child_benefit_charge_rounding_properties.py @@ -5,15 +5,19 @@ from 2024-25). Subsection (3) rounds the Child Benefit total and the charge down to whole pounds, and the percentage down to a whole number. -Invariants, for any adjusted net income and Child Benefit: +Invariants, for any finite, nonnegative adjusted net income and Child Benefit: 1. The charge is a whole number of pounds. 2. Bounds: 0 <= charge <= Child Benefit rounded down to pounds. -3. Differential: the charge equals floor(floor((ANI - L) / X) / 100 x floor(CB)) - with the percentage capped at 100, computed here from the statute in - integer arithmetic. -4. Monotonic: a higher income never lowers the charge. -5. The model's charge from a full simulation matches the statute. +3. Differential: the charge equals floor(points / 100 x floor(CB)), where + points = min(100, max(0, floor((ANI - L) / X))), computed here from the + statute in integer arithmetic. +4. Monotonic: with the Child Benefit fixed, a higher income never lowers the + charge. +5. Precision: every exact £200 step, held as float32 as in a microsimulation, + gives exactly that many points. +6. The model's charge from a full simulation matches the statute, with one to + four children. """ import numpy as np @@ -21,6 +25,7 @@ from hypothesis import strategies as st from policyengine_uk import Simulation +from policyengine_uk.utils.scenario import Scenario from policyengine_uk.utils.child_benefit import ( child_benefit_charge_percentage, round_down_to_pounds, @@ -71,7 +76,7 @@ def test_a_higher_income_never_lowers_the_charge(ani, rise, cb_pence): @settings(max_examples=8, deadline=None) -@given(pay=st.integers(min_value=55_000, max_value=90_000), children=st.integers(1, 3)) +@given(pay=st.integers(min_value=55_000, max_value=90_000), children=st.integers(1, 4)) def test_simulated_charge_matches_the_statute(pay, children): people = {"parent": {"age": {"2026": 40}, "employment_income": {"2026": pay}}} for i in range(children): @@ -88,3 +93,55 @@ def test_simulated_charge_matches_the_statute(pay, children): child_benefit = float(sim.calculate("child_benefit", 2026)[0]) charge = float(sim.calculate("CB_HITC", 2026)[0]) assert charge == statutory_charge(int(ani), round(child_benefit * 100)) + + +def test_every_exact_step_gives_exactly_its_points_in_float32(): + for points in range(0, 101): + income = np.array([L + X * points], dtype=np.float32) + assert child_benefit_charge_percentage(income, L, END)[0] == points / 100 + + +def test_a_float32_annual_amount_is_taken_to_the_penny_first(): + # Four children's £4,199 accumulated by month and stored as float32. + assert round_down_to_pounds(np.float32(4198.99951171875)) == 4199 + assert round_down_to_pounds(np.float32(1406.5999755859375)) == 1406 + + +# Structural reforms are chosen when the simulation is built, so the switch has +# to be set before data load. +HOUSEHOLD_BASED = Scenario( + applied_before_data_load=True, + parameter_changes={"gov.contrib.conservatives.cb_hitc_household": {"2025": True}}, +) + + +def _charge(pay: list[int], scenario=None) -> float: + people = { + f"parent{i}": {"age": {2026: 40}, "employment_income": {2026: p}} + for i, p in enumerate(pay) + } + people["child"] = {"age": {2026: 5}} + situation = { + "people": people, + "benunits": {"family": {"members": list(people)}}, + "households": {"home": {"members": list(people), "country": {2026: "ENGLAND"}}}, + } + return float( + Simulation(situation=situation, scenario=scenario) + .calculate("CB_HITC", 2026) + .sum() + ) + + +def test_the_household_based_reform_rounds_as_the_baseline_does(): + """A single parent's household income is their own, so the reform charges + what the baseline does: 99% of £1,406 at £79,800 is £1,391, not £1,392.53.""" + assert _charge([79_800]) == 1_391 + assert _charge([79_800], HOUSEHOLD_BASED) == 1_391 + + +def test_the_household_based_reform_charges_on_combined_income(): + """A couple on £50,000 and £30,000 pay nothing in the baseline; on £80,000 + combined the reform takes the whole Child Benefit, in whole pounds.""" + assert _charge([50_000, 30_000]) == 0 + assert _charge([50_000, 30_000], HOUSEHOLD_BASED) == 1_406 diff --git a/policyengine_uk/utils/child_benefit.py b/policyengine_uk/utils/child_benefit.py index b2b093a778..84a29b85a6 100644 --- a/policyengine_uk/utils/child_benefit.py +++ b/policyengine_uk/utils/child_benefit.py @@ -1,8 +1,9 @@ import numpy as np -# Floating-point slack, so an exact whole number computed as 55.999999... -# is not rounded down to the number below. -ROUNDING_TOLERANCE = 1e-9 +# Slack for floating-point error, so an exact whole number held as, say, +# 55.999999 (float32 storage, or a fraction such as 0.01 that binary floats +# cannot hold exactly) is not rounded down to the number below. +ROUNDING_TOLERANCE = 1e-6 def child_benefit_charge_share( @@ -25,15 +26,25 @@ def child_benefit_charge_percentage( ) -> np.ndarray: """The appropriate percentage of ITEPA 2003 s. 681C(2), as a fraction. - The share of the phase-out band crossed is a percentage of one point per - hundredth of the band (one point per £200 from 2024-25), which s. 681C(3)(b) - rounds down to a whole number. + One percentage point per hundredth of the phase-out band crossed (per £200 + from 2024-25), capped at 100 and rounded down to a whole number + (s. 681C(3)(b)). Worked in float64, multiplying before dividing, so that + an income on an exact step gives exactly that many points; a nonpositive + band width is a cliff (100 points above the start). """ - share = child_benefit_charge_share(income, phase_out_start, phase_out_end) - return np.floor(share * 100 + ROUNDING_TOLERANCE) / 100 + income = np.asarray(income, dtype=np.float64) + width = float(phase_out_end) - float(phase_out_start) + excess = np.maximum(income - float(phase_out_start), 0) + if width > 0: + points = np.minimum(excess * 100 / width, 100) + else: + points = np.where(income > float(phase_out_start), 100.0, 0.0) + return np.floor(points + ROUNDING_TOLERANCE) / 100 def round_down_to_pounds(amount: np.ndarray) -> np.ndarray: """ITEPA 2003 s. 681C(3)(a): amounts not a whole number of pounds are - rounded down.""" - return np.floor(np.asarray(amount) + ROUNDING_TOLERANCE) + rounded down. The amount is first taken to the nearest penny in float64, + so that £4,199 stored as 4,198.9995 is not rounded down to £4,198.""" + pence = np.round(np.asarray(amount, dtype=np.float64) * 100) + return np.floor(pence / 100 + ROUNDING_TOLERANCE) diff --git a/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out.py b/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out.py index 758358a59c..03fb18e275 100644 --- a/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out.py +++ b/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out.py @@ -7,10 +7,12 @@ class child_benefit_opts_out(Variable): "Whether a family would opt out of Child Benefit payments because of the " "High Income Child Benefit Charge. This is independent of " "would_claim_child_benefit and never identifies a claim by itself. " - "For claimants, payments resume when the simulated charge share falls below " - "the assumed " - "opt_out_charge_share threshold (one by default), or the charge is zero " - "or neutralised. Entitlement and nonclaimant status are unchanged." + "For claimants, payments resume when the continuous charge share (the " + "unrounded share of the phase-out band crossed) falls below the assumed " + "opt_out_charge_share threshold (one by default), or is zero, or the " + "charge is neutralised. The statutory charge itself is rounded " + "(ITEPA 2003 s. 681C(3)) and can be nil where the share is small but " + "positive. Entitlement and nonclaimant status are unchanged." ) entity = BenUnit definition_period = YEAR From c4aa96c5ef9569e6b6428f9bd432df00c85ae721 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 15:15:07 -0400 Subject: [PATCH 18/25] Give trading profit the full personal allowance; read the Scottish basic rate robustly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second-round review fixes for #2312: - Reg. 14(1) deducts the personal reliefs the claimant is entitled to, with no reduction for allowance the pay uses, so trading profit now takes the full personal allowance. Sharing one allowance deducted too much tax and could make a carer over the limit eligible (Case 16). - Read the Scottish basic rate from the parameter tree's second bracket. The scale at an instant drops the null starter bracket of 2016-17 and 2017-18, so index 1 was the 40% higher rate in those years. - Case 12 carries £5,000 of capital allowances, which reg. 13(6)(a) does not deduct; Case 15 now expects the full allowance. - Wording: the UC carer status description names the reported-award evidence and calls would_claim a modelling convention; the property-test docstring drops Class 2; the 2020 limit cites the 2020-04-06 version. Co-Authored-By: Claude Opus 5.5 --- .../dwp/carers_allowance/earnings_limit.yaml | 2 +- .../dwp/carers_allowance_earnings_limit.yaml | 33 ++++++++++--- ...ers_allowance_earnings_limit_properties.py | 49 +++++++++++++++++-- .../gov/dwp/carers_allowance_earnings.py | 26 ++++++---- .../demographic/is_carer_for_benefits.py | 13 +++-- 5 files changed, 101 insertions(+), 22 deletions(-) diff --git a/policyengine_uk/parameters/gov/dwp/carers_allowance/earnings_limit.yaml b/policyengine_uk/parameters/gov/dwp/carers_allowance/earnings_limit.yaml index 76beb1c9fc..22c25451e7 100644 --- a/policyengine_uk/parameters/gov/dwp/carers_allowance/earnings_limit.yaml +++ b/policyengine_uk/parameters/gov/dwp/carers_allowance/earnings_limit.yaml @@ -34,7 +34,7 @@ values: value: 128 reference: - title: The Social Security Benefits Up-rating Regulations 2020 (SI 2020/266) - href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2021-04-12 + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2020-04-06 2022-04-11: value: 132 reference: diff --git a/policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml b/policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml index 8e28e7a095..cdb8ee2568 100644 --- a/policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml @@ -165,12 +165,14 @@ input: age: 45 self_employment_income: 15_000 + capital_allowances: 5_000 personal_pension_contributions: 7_700 care_hours: 40 country: ENGLAND output: # 15,000 - 20% x 2,430 (486) - 6% x 2,430 (145.80) - 7,700 / 2 (3,850) - # = 10,518.20, or £202.27 a week. + # = 10,518.20, or £202.27 a week. Capital allowances are not deducted + # (reg. 13(6)(a)), though they cut the actual Class 4 to nil. carers_allowance_earnings: 10_518.20 is_gainfully_employed_for_carers_allowance: false carers_allowance: 86.45 * 52 @@ -201,7 +203,7 @@ is_gainfully_employed_for_carers_allowance: true carers_allowance: 0 -- name: Case 15, pay and trading profit share one personal allowance. +- name: Case 15, pay and trading profit each take the full personal allowance. period: 2026 absolute_error_margin: 0.01 input: @@ -211,8 +213,27 @@ care_hours: 40 country: ENGLAND output: - # Pay uses £10,000 of the allowance, leaving £2,570: trading tax - # 20% x (5,000 - 2,570) = 486. No Class 1 on £10,000; no Class 4 under - # £12,570. 15,000 - 486 = 14,514. - carers_allowance_earnings: 14_514 + # No tax or Class 1 on £10,000 of pay; no notional tax or Class 4 on + # £5,000 of profit under the full £12,570 (reg. 14(1)). £15,000 is + # £288.46 a week. + carers_allowance_earnings: 15_000 is_gainfully_employed_for_carers_allowance: true + carers_allowance: 0 + +- name: Case 16, the pay's use of the allowance does not shrink the profit's. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 6_000 + self_employment_income: 8_000 + personal_pension_contributions: 6_600 + care_hours: 40 + country: ENGLAND + output: + # 6,000 + 8,000 - 6,600 / 2 = 10,700, or £205.76 a week: over the limit. + # Sharing one allowance would deduct 20% x (8,000 - 6,570) = 286 and + # give £200.27, under it. + carers_allowance_earnings: 10_700 + is_gainfully_employed_for_carers_allowance: true + carers_allowance: 0 diff --git a/policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py b/policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py index e6cbb5901d..2e6c29bb20 100644 --- a/policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py +++ b/policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py @@ -3,9 +3,10 @@ SSCBA 1992 s. 70(1)(b) bars a person who is gainfully employed, and reg. 8(1) of the Social Security (Invalid Care Allowance) Regulations 1976 treats a person as gainfully employed when their earnings exceed the limit (£204 a -week from 6 April 2026). Earnings are net of income tax, primary Class 1 (and -Class 2 and 4) contributions and half of pension contributions (Computation of -Earnings Regulations 1996 regs. 10(4) and 13). +week from 6 April 2026). Earnings are net of income tax, primary Class 1 (or, +for trading profit, Class 4) contributions and half of pension contributions +(Computation of Earnings Regulations 1996 regs. 10(4), 13 and 14; reg. 14(2)(a), +which deducted Class 2, was omitted from 6 April 2024). Invariants, for a carer in England in 2026 caring 35 hours or more, with pay, trading profit, employee pension contributions and unearned income drawn @@ -22,6 +23,7 @@ """ import numpy as np +import pytest from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st @@ -119,3 +121,44 @@ def test_unearned_income_is_not_earnings(pay, savings, private_pension): ) < 0.01 ) + + +@pytest.mark.parametrize("year", [2016, 2017, 2018, 2026]) +@pytest.mark.parametrize("country", ["ENGLAND", "SCOTLAND"]) +def test_trading_profit_takes_the_basic_rate_in_every_year(year, country): + """Reg. 14(1): the basic rate, or for a Scottish taxpayer the Scottish + basic rate, which were both 20% in each of these years. Before 2018-19 the + Scottish scale has no starter rate, so reading its second bracket by + position at an instant would pick the higher rate.""" + profit = 30_000 + simulation = Simulation( + situation={ + "people": { + "carer": { + "age": {year: 45}, + "self_employment_income": {year: profit}, + "care_hours": {year: 40}, + } + }, + "benunits": {"family": {"members": ["carer"]}}, + "households": {"home": {"members": ["carer"], "country": {year: country}}}, + } + ) + hmrc = simulation.tax_benefit_system.parameters(year).gov.hmrc + allowance = hmrc.income_tax.allowances.personal_allowance.amount + class_4 = hmrc.national_insurance.class_4 + expected = ( + profit + - 0.20 * (profit - allowance) + - class_4.rates.main + * ( + min(profit, class_4.thresholds.upper_profits_limit) + - class_4.thresholds.lower_profits_limit + ) + ) + assert bool(simulation.calculate("pays_scottish_income_tax", year)[0]) == ( + country == "SCOTLAND" + ) + assert simulation.calculate("carers_allowance_earnings", year)[0] == pytest.approx( + expected, abs=0.01 + ) diff --git a/policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py b/policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py index 73cee0fbe6..2b962edb28 100644 --- a/policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py +++ b/policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py @@ -20,10 +20,12 @@ class carers_allowance_earnings(Variable): "irrelevant here by assumption.\n\n" "Self-employment (regs. 13(4) and 14): trading profit less a notional " "income tax at the basic rate (the Scottish basic rate for a Scottish " - "taxpayer) on the profit above whatever personal allowance the pay " - "leaves unused, and Class 4 contributions at the main rate on the " - "profit between the lower and upper profits limits. Regulation 14 " - "deducts no Class 2 contribution.\n\n" + "taxpayer) on the profit above the full personal allowance, and Class 4 " + "contributions at the main rate on the profit between the lower and " + "upper profits limits. Regulation 14(1) deducts the personal reliefs " + "the claimant is entitled to, with no reduction for any allowance the " + "pay uses, and capital expenditure is not deducted (reg. 13(6)(a)). " + "Regulation 14 deducts no Class 2 contribution.\n\n" "Not modelled: the deduction for care charges (Schedule 3, up to half " "of net earnings) and the Schedule 1 disregards, so this can overstate " "earnings for a carer who pays for care while at work." @@ -65,16 +67,22 @@ def formula(person, period, parameters): primary_class_1 = person("ni_class_1_employee", period) # Self-employment: notional basic-rate tax and main-rate Class 4 - # (reg. 14(1)-(2)), with one set of personal reliefs, used by the pay - # first. + # (reg. 14(1)-(2)). The tax is on the profit as if it were the only + # income, less the full personal allowance. profit = max_(0, person("self_employment_income", period)) - unused_allowance = max_(0, personal_allowance - taxable_pay) + # The basic rate is the first band of the rest-of-UK scale (ITA 2007 + # s. 6(2)). The Scottish basic rate is the second bracket of the + # Scottish scale, after the starter rate; read it from the parameter + # tree, which keeps the null starter bracket of years before 2018-19 in + # place. The scale at an instant drops null brackets, so its index 1 + # was the higher rate in 2016-17 and 2017-18. + scottish_scale = person.simulation.tax_benefit_system.parameters.gov.hmrc.income_tax.rates.scotland.rates basic_rate = where( scottish, - income_tax.rates.scotland.rates.rates[1], + scottish_scale.brackets[1].rate(period.start), income_tax.rates.uk.rates[0], ) - trading_tax = basic_rate * max_(0, profit - unused_allowance) + trading_tax = basic_rate * max_(0, profit - personal_allowance) trading_class_4 = class_4.rates.main * max_( 0, min_(profit, class_4.thresholds.upper_profits_limit) diff --git a/policyengine_uk/variables/household/demographic/is_carer_for_benefits.py b/policyengine_uk/variables/household/demographic/is_carer_for_benefits.py index f3ce7c1750..f86e87520d 100644 --- a/policyengine_uk/variables/household/demographic/is_carer_for_benefits.py +++ b/policyengine_uk/variables/household/demographic/is_carer_for_benefits.py @@ -20,7 +20,11 @@ class is_carer_for_benefits(Variable): "person cared for receives a qualifying disability benefit (part of " "the Carer's Allowance conditions referenced by regulation 30), so a " "survey hours field counts caring for " - "anyone and a dataset's take-up input carries that simplification." + "anyone and a dataset's take-up input carries that simplification. " + "A reported Carer's Allowance award the person would claim also " + "counts, so a carer whose earnings put them over the Carer's Allowance " + "earnings limit keeps the Universal Credit carer element, which does " + "not apply that limit (regulation 30(1)(a)(i))." ) reference = "https://www.legislation.gov.uk/uksi/2013/376/regulation/30" @@ -30,8 +34,11 @@ def formula(person, period, parameters): # A reported Carer's Allowance award the person would claim is # evidence of the caring itself, where no hours are recorded. It still # counts when the earnings limit (which the Universal Credit carer - # element does not apply) stops the allowance being paid. An award the - # person would not claim counts no more than it did before. + # element does not apply) stops the allowance being paid. Requiring + # would_claim is a modelling convention, not the law: regulation 30(2) + # applies whether or not a claim is made. It keeps an award the person + # would not claim counting no more than it did before; in the data, + # would_claim follows reported receipt. reports_carers_allowance = ( person("carers_allowance_reported", period) > 0 ) & person("would_claim_carers_allowance", period) From 9a69394ff3092697a93404482eafdb31b37187cb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 15:40:49 -0400 Subject: [PATCH 19/25] Set the Personal Savings Allowance on rest-of-UK bands for Scottish taxpayers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ITA 2007 s. 12B(8)(a)(iv) and (b)(iv) count a Scottish or Welsh taxpayer's higher- and additional-rate income as what would be charged at those rates if they were not a Scottish or Welsh taxpayer. savings_allowance read tax_band, which follows the Scottish bands, so a Scottish taxpayer with taxable income from £62,430 to £125,140 got no allowance (law: £500) and one from £31,092 to £37,700 got £500 (law: £1,000). It now works out the band on the rest-of-UK thresholds itself; tax_band keeps the Scottish bands for the Marriage Allowance conditions. This also corrects 2016-17, where tax_band put the higher band at £150,000. Fixes #2347 Co-Authored-By: Claude Opus 5.5 --- changelog.d/2347.fixed.md | 1 + .../gov/hmrc/income_tax/allowances.yaml | 12 +- .../allowances/savings_allowance.yaml | 116 ++++++++++++++++++ .../test_savings_allowance_properties.py | 85 +++++++++++++ .../allowances/savings_allowance.py | 43 ++++--- 5 files changed, 237 insertions(+), 20 deletions(-) create mode 100644 changelog.d/2347.fixed.md create mode 100644 policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances/savings_allowance.yaml create mode 100644 policyengine_uk/tests/test_savings_allowance_properties.py diff --git a/changelog.d/2347.fixed.md b/changelog.d/2347.fixed.md new file mode 100644 index 0000000000..792fa497c4 --- /dev/null +++ b/changelog.d/2347.fixed.md @@ -0,0 +1 @@ +Set the Personal Savings Allowance on the rest-of-UK bands for Scottish taxpayers, as ITA 2007 s. 12B(8) requires, and on the right bands before 2017-18. diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances.yaml index 6f2f1f0f16..6b98ba9812 100644 --- a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances.yaml @@ -94,21 +94,27 @@ period: 2020 absolute_error_margin: 0 input: - tax_band: "BASIC" + # 2019-20 rules: taxable income £27,500, under the £37,500 basic rate limit. + adjusted_net_income: 40_000 + allowances: 12_500 output: savings_allowance: 1000 - name: Savings allowance at higher rate period: 2020 absolute_error_margin: 0 input: - tax_band: "HIGHER" + # Taxable income £47,500, between £37,500 and £150,000. + adjusted_net_income: 60_000 + allowances: 12_500 output: savings_allowance: 500 - name: No savings allowance at additional rate period: 2020 absolute_error_margin: 0 input: - tax_band: "ADDITIONAL" + # Taxable income £200,000, above the £150,000 higher rate limit. + adjusted_net_income: 200_000 + allowances: 0 output: savings_allowance: 0 - name: Allowances by default diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances/savings_allowance.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances/savings_allowance.yaml new file mode 100644 index 0000000000..9a8dd94af8 --- /dev/null +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances/savings_allowance.yaml @@ -0,0 +1,116 @@ +# Personal Savings Allowance, ITA 2007 s. 12B. A Scottish taxpayer's higher- +# and additional-rate income is what would be charged at those rates if they +# were not a Scottish taxpayer (s. 12B(8)(a)(iv) and (b)(iv)), so the +# allowance follows the rest-of-UK bands: in 2026-27, £37,700 and £125,140 of +# taxable income, with a £12,570 personal allowance tapered above £100,000. +# Savings income is taxed at the UK savings rates. + +- name: Case 1, a Scottish advanced-rate taxpayer below £125,140 gets £500. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 80_000 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: SCOTLAND + output: + # Taxable income 81,000 - 12,570 = 68,430: Scottish advanced rate, but + # UK higher rate. (1,000 - 500) x 40% = 200. + pays_scottish_income_tax: true + tax_band: ADDITIONAL + savings_allowance: 500 + savings_income_tax: 200 + +- name: Case 2, a Scottish higher-rate taxpayer below £37,700 gets £1,000. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 45_000 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: SCOTLAND + output: + # Taxable income 46,000 - 12,570 = 33,430: Scottish higher rate, but UK + # basic rate, so the whole £1,000 of interest is covered. + tax_band: HIGHER + savings_allowance: 1_000 + savings_income_tax: 0 + +- name: Case 3, an English taxpayer with the same income is unchanged. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 80_000 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: ENGLAND + output: + tax_band: HIGHER + savings_allowance: 500 + savings_income_tax: 200 + +- name: Case 4, a Scottish taxpayer above £125,140 gets nothing. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 130_000 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: SCOTLAND + output: + # The personal allowance is tapered to nil, so taxable income is + # 131,000, above £125,140. + savings_allowance: 0 + +- name: Case 5, 2016-17 bands for a higher-rate taxpayer. + period: 2016 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 60_000 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: ENGLAND + output: + # Taxable income above the £32,000 basic rate limit but below £150,000. + savings_allowance: 500 + +- name: Case 6, 2016-17 bands for an additional-rate taxpayer. + period: 2016 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 200_000 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: ENGLAND + output: + savings_allowance: 0 diff --git a/policyengine_uk/tests/test_savings_allowance_properties.py b/policyengine_uk/tests/test_savings_allowance_properties.py new file mode 100644 index 0000000000..e0771a772b --- /dev/null +++ b/policyengine_uk/tests/test_savings_allowance_properties.py @@ -0,0 +1,85 @@ +"""Personal Savings Allowance, ITA 2007 s. 12B. + +A Scottish taxpayer's higher- and additional-rate income is what would be +charged at those rates if they were not a Scottish taxpayer (s. 12B(8)(a)(iv) +and (b)(iv)), so the allowance follows the rest-of-UK bands for everyone. + +Invariants, for single adults in 2026 with pay, savings interest and +dividends drawn freely: + +1. Country: the allowance is the same for a Scottish and an English taxpayer + with the same income. +2. Differential: it equals the statute computed independently here: £1,000 + when taxable income is under the £37,700 basic rate limit, £500 under the + £125,140 higher rate limit, and nil above, with taxable income as total + income less the personal allowance, tapered by £1 for every £2 of income + over £100,000. +3. Monotonic: a higher pay never raises the allowance. +""" + +import numpy as np +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from policyengine_uk import Simulation + +YEAR = 2026 +PEOPLE = 12 +money = st.integers(min_value=0, max_value=300_000) +small = st.integers(min_value=0, max_value=20_000) +incomes = st.lists(st.tuples(money, small, small), min_size=PEOPLE, max_size=PEOPLE) + + +def _allowances(rows, country): + """One single-adult household per row, all in one simulation.""" + people, benunits, households = {}, {}, {} + for i, (pay, savings, dividends) in enumerate(rows): + name = f"p{i}" + people[name] = { + "age": {YEAR: 40}, + "employment_income": {YEAR: pay}, + "savings_interest_income": {YEAR: savings}, + "dividend_income": {YEAR: dividends}, + } + benunits[f"b{i}"] = {"members": [name]} + households[f"h{i}"] = {"members": [name], "country": {YEAR: country}} + simulation = Simulation( + situation={"people": people, "benunits": benunits, "households": households} + ) + return np.asarray(simulation.calculate("savings_allowance", YEAR)) + + +def _statute(pay, savings, dividends): + income = pay + savings + dividends + allowance = max(0, 12_570 - max(0, income - 100_000) / 2) + taxable = income - allowance + if taxable >= 125_140: + return 0 + if taxable >= 37_700: + return 500 + return 1_000 + + +@settings( + max_examples=10, + deadline=None, + suppress_health_check=[HealthCheck.too_slow], +) +@given(rows=incomes) +def test_scottish_and_english_taxpayers_get_the_same_allowance(rows): + scottish = _allowances(rows, "SCOTLAND") + english = _allowances(rows, "ENGLAND") + assert np.array_equal(scottish, english) + assert list(scottish) == [_statute(*row) for row in rows] + + +@settings( + max_examples=10, + deadline=None, + suppress_health_check=[HealthCheck.too_slow], +) +@given(rows=incomes, rise=st.integers(min_value=1, max_value=100_000)) +def test_a_higher_pay_never_raises_the_allowance(rows, rise): + before = _allowances(rows, "SCOTLAND") + after = _allowances([(pay + rise, s, d) for pay, s, d in rows], "SCOTLAND") + assert np.all(after <= before) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/allowances/savings_allowance.py b/policyengine_uk/variables/gov/hmrc/income_tax/allowances/savings_allowance.py index 6c046ff0c0..306f7854e7 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/allowances/savings_allowance.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/allowances/savings_allowance.py @@ -5,26 +5,35 @@ class savings_allowance(Variable): value_type = float entity = Person label = "Savings Allowance for the year" + documentation = ( + "The Personal Savings Allowance (ITA 2007 s. 12B): the basic amount " + "with no higher-rate income, the higher amount with higher-rate but " + "no additional-rate income, and the additional amount (nil) with any " + "additional-rate income. For a Scottish or Welsh taxpayer, s. " + "12B(8)(a)(iv) and (b)(iv) count as higher- or additional-rate income " + "what would be charged at those rates if they were not a Scottish or " + "Welsh taxpayer, so the bands are the rest-of-UK ones for everyone. " + "tax_band follows the Scottish bands for a Scottish taxpayer, as the " + "Marriage Allowance conditions need, so it is not read here. Taxable " + "income is taken, as in tax_band, as adjusted net income less " + "allowances." + ) definition_period = YEAR - reference = "Income Tax Act 2007 s. 12B" + reference = "https://www.legislation.gov.uk/ukpga/2007/3/section/12B" unit = GBP def formula(person, period, parameters): - tax_band = person("tax_band", period) - tax_bands = tax_band.possible_values - amounts = parameters( - period - ).gov.hmrc.income_tax.allowances.personal_savings_allowance + income_tax = parameters(period).gov.hmrc.income_tax + amounts = income_tax.allowances.personal_savings_allowance + thresholds = income_tax.rates.uk.thresholds + taxable_income = person("adjusted_net_income", period) - person( + "allowances", period + ) + # thresholds[1] is the basic rate limit and thresholds[2] the higher + # rate limit. Income at or above a limit counts as reaching that band, + # as in tax_band. return select( - [ - tax_band == tax_bands.ADDITIONAL, - tax_band == tax_bands.HIGHER, - ( - (tax_band == tax_bands.STARTER) - | (tax_band == tax_bands.BASIC) - | (tax_band == tax_bands.INTERMEDIATE) - ), - tax_band == tax_bands.NONE, - ], - [amounts.additional, amounts.higher, amounts.basic, amounts.basic], + [taxable_income >= thresholds[2], taxable_income >= thresholds[1]], + [amounts.additional, amounts.higher], + default=amounts.basic, ) From bdca3141a82948f1db6693837c54dd0d27119487 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 16:25:22 -0400 Subject: [PATCH 20/25] Band savings and dividends above property income MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ITA 2007 s. 16 treats savings and dividend income as the highest part of total income, so property income, which is non-savings income, fills the bands below them. Since #1408, earned_taxable_income leaves property income out (property_income_tax taxes it separately), and the savings bands, the savings starting-rate taper and dividend_income_tax all stacked on earned_taxable_income, so they were banded as if property income were not there. £40,000 of property income and £20,000 of dividends paid £2,096.25 of dividend tax instead of £4,528.75. New non_savings_taxable_income (earned_taxable_income plus property income after allowances, from the new taxable_property_income_after_allowances, which property_income_tax now reads too) is the base for those variables. Fixes #2351 Co-Authored-By: Claude Opus 5.5 --- changelog.d/2351.fixed.md | 1 + .../income_tax/property_income_band_base.yaml | 99 +++++++++++++++ ...st_property_income_band_base_properties.py | 115 ++++++++++++++++++ .../non_savings_taxable_income.py | 26 ++++ ...axable_property_income_after_allowances.py | 31 +++++ .../add_rate_savings_income.py | 4 +- .../basic_rate_savings_income.py | 4 +- .../basic_rate_savings_income_pre_starter.py | 4 +- .../higher_rate_savings_income.py | 4 +- .../savings_starter_rate_income.py | 4 +- .../liability/dividend_income_tax.py | 4 +- .../liability/property_income_tax.py | 5 +- 12 files changed, 293 insertions(+), 8 deletions(-) create mode 100644 changelog.d/2351.fixed.md create mode 100644 policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/property_income_band_base.yaml create mode 100644 policyengine_uk/tests/test_property_income_band_base_properties.py create mode 100644 policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/non_savings_taxable_income.py create mode 100644 policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/taxable_property_income_after_allowances.py diff --git a/changelog.d/2351.fixed.md b/changelog.d/2351.fixed.md new file mode 100644 index 0000000000..b40f9582f0 --- /dev/null +++ b/changelog.d/2351.fixed.md @@ -0,0 +1 @@ +Band savings and dividend income above property income, which is non-savings income (ITA 2007 s. 16). diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/property_income_band_base.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/property_income_band_base.yaml new file mode 100644 index 0000000000..49d781b6e2 --- /dev/null +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/property_income_band_base.yaml @@ -0,0 +1,99 @@ +# Savings and dividend income are the highest part of total income (ITA 2007 +# s. 16), so property income, like pay, fills the bands below them. 2026-27, +# single adult: personal allowance £12,570, basic rate limit £37,700, savings +# allowance £500 for a higher-rate taxpayer, dividend allowance £500 at +# 10.75% / 35.75%. + +- name: Case 1, dividends sit above property income. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 45 + property_income: 40_000 + dividend_income: 20_000 + households: + household: + members: [person] + country: ENGLAND + output: + # Non-savings taxable income 40,000 - 12,570 = 27,430. The £500 dividend + # allowance and 9,770 fill the basic-rate band (9,770 x 10.75% = + # 1,050.275); 9,730 is at 35.75% (3,478.475). Total 4,528.75, as for pay. + non_savings_taxable_income: 27_430 + dividend_income_tax: 4_528.75 + income_tax: 5_486 + 4_528.75 + +- name: Case 2, savings sit above property income. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 45 + property_income: 40_000 + savings_interest_income: 20_000 + households: + household: + members: [person] + country: ENGLAND + output: + # £500 savings allowance, then 9,770 at 20% (1,954) to the basic rate + # limit and 9,730 at 40% (3,892). + savings_income_tax: 5_846 + +- name: Case 3, pay and property income together form the band base. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 45 + employment_income: 10_000 + property_income: 30_000 + dividend_income: 20_000 + households: + household: + members: [person] + country: ENGLAND + output: + # The allowance covers the pay first: 0 earned and 27,430 property + # income taxable, so dividends sit on 27,430 as in Case 1. + taxable_property_income_after_allowances: 27_430 + non_savings_taxable_income: 27_430 + dividend_income_tax: 4_528.75 + +- name: Case 4, a Scottish taxpayer's dividends use UK bands above the property income. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 45 + property_income: 40_000 + dividend_income: 20_000 + households: + household: + members: [person] + country: SCOTLAND + output: + dividend_income_tax: 4_528.75 + +- name: Case 5, from 2027-28 property income has its own rates but still sits below dividends. + period: 2027 + absolute_error_margin: 0.01 + input: + people: + person: + age: 45 + property_income: 40_000 + dividend_income: 20_000 + households: + household: + members: [person] + country: ENGLAND + output: + # Property income at the 22% property basic rate: 27,430 x 22% = 6,034.60. + property_income_tax: 6_034.60 + dividend_income_tax: 4_528.75 diff --git a/policyengine_uk/tests/test_property_income_band_base_properties.py b/policyengine_uk/tests/test_property_income_band_base_properties.py new file mode 100644 index 0000000000..b51a4f409b --- /dev/null +++ b/policyengine_uk/tests/test_property_income_band_base_properties.py @@ -0,0 +1,115 @@ +"""Savings and dividends sit above all non-savings income (ITA 2007 s. 16). + +Property income is non-savings income, so it fills the bands before savings +and dividends, as pay does. + +Invariants, for single adults in 2026 with pay, property income, savings +interest and dividends drawn freely: + +1. Differential: replacing property income with pay equal to its taxable + amount leaves savings and dividend tax unchanged, in England and in + Scotland. (The taxable amount, because the £1,000 property allowance can + relieve small property income in full.) +2. Accounting: non_savings_taxable_income equals adjusted net income less + savings, dividends and allowances, floored at nil. +3. Monotonic: more property income never lowers savings or dividend tax. +""" + +import numpy as np +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from policyengine_uk import Simulation + +YEAR = 2026 +PEOPLE = 10 +amount = st.integers(min_value=0, max_value=200_000) +rows_strategy = st.lists( + st.tuples(amount, amount, st.integers(0, 60_000), st.integers(0, 60_000)), + min_size=PEOPLE, + max_size=PEOPLE, +) +VARIABLES = [ + "taxable_property_income", + "savings_income_tax", + "dividend_income_tax", + "non_savings_taxable_income", + "adjusted_net_income", + "taxable_savings_interest_income", + "taxable_dividend_income", + "allowances", +] + + +def _calculate(rows, country): + """rows: (pay, property, savings, dividends), one single adult each.""" + people, benunits, households = {}, {}, {} + for i, (pay, prop, savings, dividends) in enumerate(rows): + name = f"p{i}" + people[name] = { + "age": {YEAR: 45}, + "employment_income": {YEAR: pay}, + "property_income": {YEAR: prop}, + "savings_interest_income": {YEAR: savings}, + "dividend_income": {YEAR: dividends}, + } + benunits[f"b{i}"] = {"members": [name]} + households[f"h{i}"] = {"members": [name], "country": {YEAR: country}} + simulation = Simulation( + situation={"people": people, "benunits": benunits, "households": households} + ) + return {v: np.asarray(simulation.calculate(v, YEAR)) for v in VARIABLES} + + +SETTINGS = settings( + max_examples=8, + deadline=None, + suppress_health_check=[HealthCheck.too_slow], +) + + +@SETTINGS +@given(rows=rows_strategy, country=st.sampled_from(["ENGLAND", "SCOTLAND"])) +def test_moving_income_between_pay_and_property_leaves_savings_and_dividend_tax( + rows, country +): + as_property = _calculate(rows, country) + taxable_property = as_property["taxable_property_income"] + as_pay = _calculate( + [ + (pay + float(taxable), 0, s, d) + for (pay, _, s, d), taxable in zip(rows, taxable_property) + ], + country, + ) + for name in [ + "savings_income_tax", + "dividend_income_tax", + "non_savings_taxable_income", + ]: + np.testing.assert_allclose(as_property[name], as_pay[name], atol=0.01) + + +@SETTINGS +@given(rows=rows_strategy) +def test_non_savings_taxable_income_is_the_rest_after_allowances(rows): + v = _calculate(rows, "ENGLAND") + expected = np.maximum( + 0, + v["adjusted_net_income"] + - v["taxable_savings_interest_income"] + - v["taxable_dividend_income"] + - v["allowances"], + ) + np.testing.assert_allclose(v["non_savings_taxable_income"], expected, atol=0.01) + + +@SETTINGS +@given(rows=rows_strategy, rise=st.integers(min_value=1, max_value=100_000)) +def test_more_property_income_never_lowers_savings_or_dividend_tax(rows, rise): + before = _calculate(rows, "ENGLAND") + after = _calculate( + [(pay, prop + rise, s, d) for pay, prop, s, d in rows], "ENGLAND" + ) + for name in ["savings_income_tax", "dividend_income_tax"]: + assert np.all(after[name] >= before[name] - 0.01) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/non_savings_taxable_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/non_savings_taxable_income.py new file mode 100644 index 0000000000..0c6f878d0f --- /dev/null +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/non_savings_taxable_income.py @@ -0,0 +1,26 @@ +from policyengine_uk.model_api import * + + +class non_savings_taxable_income(Variable): + value_type = float + entity = Person + label = "Taxable income other than savings and dividend income" + documentation = ( + "The part of taxable income below savings and dividend income, which " + "ITA 2007 s. 16 treats as the highest part of total income: " + "earned_taxable_income plus property income after allowances. " + "earned_taxable_income leaves property income out because " + "property_income_tax taxes it separately, but property income still " + "fills the bands before savings and dividends do." + ) + definition_period = YEAR + reference = dict( + title="Income Tax Act 2007, s. 16", + href="https://www.legislation.gov.uk/ukpga/2007/3/section/16", + ) + unit = GBP + + def formula(person, period, parameters): + return person("earned_taxable_income", period) + person( + "taxable_property_income_after_allowances", period + ) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/taxable_property_income_after_allowances.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/taxable_property_income_after_allowances.py new file mode 100644 index 0000000000..cb27d2133b --- /dev/null +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/taxable_property_income_after_allowances.py @@ -0,0 +1,31 @@ +from policyengine_uk.model_api import * + + +class taxable_property_income_after_allowances(Variable): + value_type = float + entity = Person + label = "Property income for Income Tax after allowances" + documentation = ( + "Taxable property income less the part of the allowances it absorbs. " + "Allowances go to non-savings, non-dividend income first, and " + "property income is stacked above the rest of it, so it takes " + "whatever allowances the other non-savings income leaves." + ) + definition_period = YEAR + reference = dict( + title="Income Tax Act 2007, s. 25(2)", + href="https://www.legislation.gov.uk/ukpga/2007/3/section/25", + ) + unit = GBP + + def formula(person, period, parameters): + non_savings_after_allowances = max_( + 0, + person("adjusted_net_income", period) + - person("taxable_savings_interest_income", period) + - person("taxable_dividend_income", period) + - person("allowances", period), + ) + return min_( + person("taxable_property_income", period), non_savings_after_allowances + ) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/add_rate_savings_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/add_rate_savings_income.py index 610b9481b2..979d1c254d 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/add_rate_savings_income.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/add_rate_savings_income.py @@ -14,7 +14,9 @@ class add_rate_savings_income(Variable): def formula(person, period, parameters): thresholds = parameters(period).gov.hmrc.income_tax.rates.uk.thresholds - other_income = person("earned_taxable_income", period) + # Savings sit above all non-savings income, property income + # included (ITA 2007 s. 16). + other_income = person("non_savings_taxable_income", period) savings_after_allowances = max_( 0, person("taxable_savings_interest_income", period) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/basic_rate_savings_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/basic_rate_savings_income.py index cea212f46d..d4aa229abb 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/basic_rate_savings_income.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/basic_rate_savings_income.py @@ -14,7 +14,9 @@ class basic_rate_savings_income(Variable): def formula(person, period, parameters): thresholds = parameters(period).gov.hmrc.income_tax.rates.uk.thresholds - other_income = person("earned_taxable_income", period) + # Savings sit above all non-savings income, property income + # included (ITA 2007 s. 16). + other_income = person("non_savings_taxable_income", period) savings_after_allowances = max_( 0, person("taxable_savings_interest_income", period) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/basic_rate_savings_income_pre_starter.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/basic_rate_savings_income_pre_starter.py index a3f99e40ce..214b51cfcf 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/basic_rate_savings_income_pre_starter.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/basic_rate_savings_income_pre_starter.py @@ -19,7 +19,9 @@ def formula(person, period, parameters): person("taxable_savings_interest_income", period) - person("received_allowances_savings_income", period), ) - other_income = person("earned_taxable_income", period) + # Savings sit above all non-savings income, property income + # included (ITA 2007 s. 16). + other_income = person("non_savings_taxable_income", period) basic_rate_amount_with_savings = clip( savings_income + other_income, thresholds[0], diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/higher_rate_savings_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/higher_rate_savings_income.py index cfb7da1a31..96f845b5b3 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/higher_rate_savings_income.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/higher_rate_savings_income.py @@ -14,7 +14,9 @@ class higher_rate_savings_income(Variable): def formula(person, period, parameters): thresholds = parameters(period).gov.hmrc.income_tax.rates.uk.thresholds - other_income = person("earned_taxable_income", period) + # Savings sit above all non-savings income, property income + # included (ITA 2007 s. 16). + other_income = person("non_savings_taxable_income", period) savings_after_allowances = max_( 0, person("taxable_savings_interest_income", period) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/savings_starter_rate_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/savings_starter_rate_income.py index 354b3908c8..6cb516b928 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/savings_starter_rate_income.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/savings_starter_rate_income.py @@ -37,7 +37,9 @@ def formula(person, period, parameters): - person("received_allowances_savings_income", period), ), ) - earned_taxable_income = person("earned_taxable_income", period) + # The starting rate band is reduced by non-savings income, property + # income included (ITA 2007 ss. 12, 16). + earned_taxable_income = person("non_savings_taxable_income", period) removed_amount = max_( 0, diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/liability/dividend_income_tax.py b/policyengine_uk/variables/gov/hmrc/income_tax/liability/dividend_income_tax.py index e812a8f769..e9858f4184 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/liability/dividend_income_tax.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/liability/dividend_income_tax.py @@ -14,7 +14,9 @@ class dividend_income_tax(Variable): def formula(person, period, parameters): rates = parameters(period).gov.hmrc.income_tax.rates - other_income = person("earned_taxable_income", period) + max_( + # Dividends are the highest part of income, above savings and all + # non-savings income, property income included (ITA 2007 s. 16). + other_income = person("non_savings_taxable_income", period) + max_( 0, person("taxable_savings_interest_income", period) - person("received_allowances_savings_income", period), diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/liability/property_income_tax.py b/policyengine_uk/variables/gov/hmrc/income_tax/liability/property_income_tax.py index 0fff929eac..9fcafebf6f 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/liability/property_income_tax.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/liability/property_income_tax.py @@ -34,7 +34,6 @@ def formula(person, period, parameters): determining which tax band it falls into. """ rates = parameters(period).gov.hmrc.income_tax.rates - taxable_property = person("taxable_property_income", period) non_savings_after_allowances = max_( 0, person("adjusted_net_income", period) @@ -42,7 +41,9 @@ def formula(person, period, parameters): - person("taxable_dividend_income", period) - person("allowances", period), ) - property_after_allowances = min_(taxable_property, non_savings_after_allowances) + property_after_allowances = person( + "taxable_property_income_after_allowances", period + ) other_earned_taxable = max_( 0, non_savings_after_allowances - property_after_allowances ) From 1a80b21d1ae968c919dd35761e5906cc59ea493b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 16:31:27 -0400 Subject: [PATCH 21/25] Correct the property integration test's savings tax, which encoded the bug The case expected nil savings tax for two people with property income and savings interest. With savings above the property income (ITA 2007 s. 16), person 1 is a higher-rate taxpayer: (2,900 - 500) x 40% = 960; person 2 a basic-rate one: (1,525 - 1,000) x 20% = 105. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/tests/policy/integration/income.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/policyengine_uk/tests/policy/integration/income.yaml b/policyengine_uk/tests/policy/integration/income.yaml index dc8b7ea721..798953e25f 100644 --- a/policyengine_uk/tests/policy/integration/income.yaml +++ b/policyengine_uk/tests/policy/integration/income.yaml @@ -100,8 +100,13 @@ members: [person1, person2] output: property_income_tax: [17_792, 6_776.20] - savings_income_tax: [0, 0] - income_tax: [17_792, 6_776.20] + # Savings sit above the property income (ITA 2007 s. 16). Person 1: + # 75,900 - 12,570 = 63,330 of non-savings taxable income, so a higher- + # rate taxpayer with a £500 savings allowance and no starting rate: + # (2,900 - 500) x 40% = 960. Person 2: 46,451 - 12,570 = 33,881, a + # basic-rate taxpayer with £1,000: (1,525 - 1,000) x 20% = 105. + savings_income_tax: [960, 105] + income_tax: [18_752, 6_881.20] - name: Household net income can be negative period: 2020 From a0516c0d9f8001b5d8a7fb0950dc0e328cca60d6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 17:04:30 -0400 Subject: [PATCH 22/25] Drop the rounding slack: exact steps are exact without it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second-round review fix for #2323. With whole-pound band edges, an income on an exact step gives an integer-valued quotient when multiplied before dividing in float64, which IEEE division returns exactly, so the 1e-6 slack was not needed. Under a wide taper it rounded a penny income just below a step up to it: £79,399.99 under a £60,000-£2,000,000 band (as a reform or test can set) charged £14 instead of £0. round_down_to_pounds now floors whole pence by integer division. Regressions: every tested step and the float32 income just below it under the statutory, £2,000,000 and £10,000,000 band ends, and the £79,399.99 case. The property docstring states the whole-pound and whole-pence domain. Co-Authored-By: Claude Opus 5.5 --- ...hild_benefit_charge_rounding_properties.py | 41 +++++++++++++++++-- policyengine_uk/utils/child_benefit.py | 23 ++++++----- 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/policyengine_uk/tests/test_child_benefit_charge_rounding_properties.py b/policyengine_uk/tests/test_child_benefit_charge_rounding_properties.py index 0f271add41..7e448efab2 100644 --- a/policyengine_uk/tests/test_child_benefit_charge_rounding_properties.py +++ b/policyengine_uk/tests/test_child_benefit_charge_rounding_properties.py @@ -5,7 +5,8 @@ from 2024-25). Subsection (3) rounds the Child Benefit total and the charge down to whole pounds, and the percentage down to a whole number. -Invariants, for any finite, nonnegative adjusted net income and Child Benefit: +Invariants, for adjusted net income in whole pounds and Child Benefit totals in whole +pence (the amounts the model produces), including their float32 storage: 1. The charge is a whole number of pounds. 2. Bounds: 0 <= charge <= Child Benefit rounded down to pounds. @@ -14,8 +15,9 @@ statute in integer arithmetic. 4. Monotonic: with the Child Benefit fixed, a higher income never lowers the charge. -5. Precision: every exact £200 step, held as float32 as in a microsimulation, - gives exactly that many points. +5. Precision: every exact step, held as float32 as in a microsimulation, + gives exactly that many points, under the statutory band and under wider + tapers a reform can set; a penny below a step gives one point fewer. 6. The model's charge from a full simulation matches the statute, with one to four children. """ @@ -145,3 +147,36 @@ def test_the_household_based_reform_charges_on_combined_income(): combined the reform takes the whole Child Benefit, in whole pounds.""" assert _charge([50_000, 30_000]) == 0 assert _charge([50_000, 30_000], HOUSEHOLD_BASED) == 1_406 + + +WIDE_TAPER_ENDS = [END, 2_000_000, 10_000_000] + + +def test_exact_steps_and_the_income_just_below_under_wide_tapers(): + """A reform can widen the band (tests use £2,000,000 and £10,000,000). + Each step is width / 100. The largest float32 income below a step stays + one point lower: no slack rounds it up. (Above about £524,288, float32 + cannot hold a penny below a step, so the nearest float32 below it is + used.)""" + for end in WIDE_TAPER_ENDS: + step = (end - L) / 100 + for points in [1, 2, 37, 99, 100]: + on_step = np.float32(L + step * points) + below = np.nextafter(on_step, np.float32(0)) + assert ( + child_benefit_charge_percentage(np.array([on_step]), L, end)[0] + == points / 100 + ) + assert ( + child_benefit_charge_percentage(np.array([below]), L, end)[0] + == (points - 1) / 100 + ) + + +def test_a_penny_below_the_first_wide_taper_step_is_not_charged(): + # £79,399.99 is £79,399.9921875 in float32: 0 points under a £60,000 to + # £2,000,000 band, so no charge on one child's £1,406. + income = np.array([79_399.99], dtype=np.float32) + percentage = child_benefit_charge_percentage(income, L, 2_000_000) + assert percentage[0] == 0 + assert round_down_to_pounds(percentage * round_down_to_pounds(1_406.60))[0] == 0 diff --git a/policyengine_uk/utils/child_benefit.py b/policyengine_uk/utils/child_benefit.py index 84a29b85a6..f6c94270be 100644 --- a/policyengine_uk/utils/child_benefit.py +++ b/policyengine_uk/utils/child_benefit.py @@ -1,10 +1,5 @@ import numpy as np -# Slack for floating-point error, so an exact whole number held as, say, -# 55.999999 (float32 storage, or a fraction such as 0.01 that binary floats -# cannot hold exactly) is not rounded down to the number below. -ROUNDING_TOLERANCE = 1e-6 - def child_benefit_charge_share( income: np.ndarray, phase_out_start: float, phase_out_end: float @@ -28,9 +23,12 @@ def child_benefit_charge_percentage( One percentage point per hundredth of the phase-out band crossed (per £200 from 2024-25), capped at 100 and rounded down to a whole number - (s. 681C(3)(b)). Worked in float64, multiplying before dividing, so that - an income on an exact step gives exactly that many points; a nonpositive - band width is a cliff (100 points above the start). + (s. 681C(3)(b)). Worked in float64, multiplying before dividing. With + whole-pound band edges, an income on an exact step (exact in float32 too, + below £16.7m) gives an integer-valued quotient, which IEEE division returns + exactly, so no slack is added: slack would round a penny income just + below a step up to it. A nonpositive band width is a cliff (100 points + above the start). """ income = np.asarray(income, dtype=np.float64) width = float(phase_out_end) - float(phase_out_start) @@ -39,12 +37,15 @@ def child_benefit_charge_percentage( points = np.minimum(excess * 100 / width, 100) else: points = np.where(income > float(phase_out_start), 100.0, 0.0) - return np.floor(points + ROUNDING_TOLERANCE) / 100 + return np.floor(points) / 100 def round_down_to_pounds(amount: np.ndarray) -> np.ndarray: """ITEPA 2003 s. 681C(3)(a): amounts not a whole number of pounds are rounded down. The amount is first taken to the nearest penny in float64, - so that £4,199 stored as 4,198.9995 is not rounded down to £4,198.""" + so that £4,199 stored as 4,198.9995 is not rounded down to £4,198, then + to whole pounds by integer division of the pence. Child Benefit totals and + charges are whole pence (weekly rates in pence, and whole percentage + points of whole pounds), so the penny step loses nothing.""" pence = np.round(np.asarray(amount, dtype=np.float64) * 100) - return np.floor(pence / 100 + ROUNDING_TOLERANCE) + return np.floor_divide(pence, 100) From 49fad8443710497cc708daa2c8ad14b45c4ca470 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 17:07:15 -0400 Subject: [PATCH 23/25] Give the higher allowance at exactly a rate limit; test the boundaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fixes for #2348: - ITA 2007 s. 10(2)-(3) charges income up to a limit at the lower rate and only income above it at the higher one, so taxable income exactly at the basic rate limit has no higher-rate income (£1,000) and exactly at the higher rate limit no additional-rate income (£500). The test is now ">". - YAML Cases 8-11 at and one pound over each limit; Case 7 for period 2017, which the old tax_band formula also covered; the property tests always run the boundary rows and the statute oracle uses ">". - Changelog and comment wording. Co-Authored-By: Claude Opus 5.5 --- changelog.d/2347.fixed.md | 2 +- .../gov/hmrc/income_tax/allowances.yaml | 2 +- .../allowances/savings_allowance.yaml | 84 +++++++++++++++++++ .../test_savings_allowance_properties.py | 23 +++-- .../allowances/savings_allowance.py | 7 +- 5 files changed, 106 insertions(+), 12 deletions(-) diff --git a/changelog.d/2347.fixed.md b/changelog.d/2347.fixed.md index 792fa497c4..06c17f5efc 100644 --- a/changelog.d/2347.fixed.md +++ b/changelog.d/2347.fixed.md @@ -1 +1 @@ -Set the Personal Savings Allowance on the rest-of-UK bands for Scottish taxpayers, as ITA 2007 s. 12B(8) requires, and on the right bands before 2017-18. +Set the Personal Savings Allowance on the rest-of-UK bands for Scottish taxpayers, as ITA 2007 s. 12B(8) requires; give the higher amount at exactly the basic rate limit and the lower amount at exactly the higher rate limit (ITA 2007 s. 10); and use the right bands for periods 2016 and 2017. diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances.yaml index 6b98ba9812..e8e0e20c0e 100644 --- a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances.yaml @@ -94,7 +94,7 @@ period: 2020 absolute_error_margin: 0 input: - # 2019-20 rules: taxable income £27,500, under the £37,500 basic rate limit. + # 2020 parameters: taxable income £27,500, under the £37,500 basic rate limit. adjusted_net_income: 40_000 allowances: 12_500 output: diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances/savings_allowance.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances/savings_allowance.yaml index 9a8dd94af8..3ef8fabad3 100644 --- a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances/savings_allowance.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances/savings_allowance.yaml @@ -114,3 +114,87 @@ country: ENGLAND output: savings_allowance: 0 + +- name: Case 7, the 2016-17 rules also apply to period 2017. + period: 2017 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 60_000 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: ENGLAND + output: + savings_allowance: 500 + +- name: Case 8, taxable income exactly at the basic rate limit has no higher-rate income. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 49_270 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: ENGLAND + output: + # 50,270 - 12,570 = 37,700: nothing above the limit (ITA 2007 s. 10(2)-(3)). + savings_allowance: 1_000 + savings_income_tax: 0 + +- name: Case 9, a pound over the basic rate limit is higher-rate income. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 49_271 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: ENGLAND + output: + savings_allowance: 500 + +- name: Case 10, taxable income exactly at the higher rate limit has no additional-rate income. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 124_140 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: ENGLAND + output: + # Adjusted net income 125,140 tapers the personal allowance to nil, so + # taxable income is 125,140. + savings_allowance: 500 + +- name: Case 11, a pound over the higher rate limit is additional-rate income. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 124_141 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: ENGLAND + output: + savings_allowance: 0 diff --git a/policyengine_uk/tests/test_savings_allowance_properties.py b/policyengine_uk/tests/test_savings_allowance_properties.py index e0771a772b..1ed0d174f6 100644 --- a/policyengine_uk/tests/test_savings_allowance_properties.py +++ b/policyengine_uk/tests/test_savings_allowance_properties.py @@ -10,15 +10,15 @@ 1. Country: the allowance is the same for a Scottish and an English taxpayer with the same income. 2. Differential: it equals the statute computed independently here: £1,000 - when taxable income is under the £37,700 basic rate limit, £500 under the - £125,140 higher rate limit, and nil above, with taxable income as total - income less the personal allowance, tapered by £1 for every £2 of income - over £100,000. + when taxable income is at or under the £37,700 basic rate limit, £500 at + or under the £125,140 higher rate limit, and nil above (ITA 2007 s. 10(2)- + (3)), with taxable income as total income less the personal allowance, + tapered by £1 for every £2 of income over £100,000. 3. Monotonic: a higher pay never raises the allowance. """ import numpy as np -from hypothesis import HealthCheck, given, settings +from hypothesis import HealthCheck, example, given, settings from hypothesis import strategies as st from policyengine_uk import Simulation @@ -49,13 +49,21 @@ def _allowances(rows, country): return np.asarray(simulation.calculate("savings_allowance", YEAR)) +BOUNDARY_ROWS = [ + (49_270, 1_000, 0), # taxable 37,700: at the basic rate limit + (49_271, 1_000, 0), # taxable 37,701 + (124_140, 1_000, 0), # taxable 125,140 (allowance tapered to nil) + (124_141, 1_000, 0), # taxable 125,141 +] + [(0, 0, 0)] * (PEOPLE - 4) + + def _statute(pay, savings, dividends): income = pay + savings + dividends allowance = max(0, 12_570 - max(0, income - 100_000) / 2) taxable = income - allowance - if taxable >= 125_140: + if taxable > 125_140: return 0 - if taxable >= 37_700: + if taxable > 37_700: return 500 return 1_000 @@ -66,6 +74,7 @@ def _statute(pay, savings, dividends): suppress_health_check=[HealthCheck.too_slow], ) @given(rows=incomes) +@example(rows=BOUNDARY_ROWS) def test_scottish_and_english_taxpayers_get_the_same_allowance(rows): scottish = _allowances(rows, "SCOTLAND") english = _allowances(rows, "ENGLAND") diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/allowances/savings_allowance.py b/policyengine_uk/variables/gov/hmrc/income_tax/allowances/savings_allowance.py index 306f7854e7..56f2599a13 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/allowances/savings_allowance.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/allowances/savings_allowance.py @@ -30,10 +30,11 @@ def formula(person, period, parameters): "allowances", period ) # thresholds[1] is the basic rate limit and thresholds[2] the higher - # rate limit. Income at or above a limit counts as reaching that band, - # as in tax_band. + # rate limit. Income up to a limit is charged at the lower rate and + # only income above it at the higher one (ITA 2007 s. 10(2)-(3)), so + # taxable income exactly at a limit has none in the band above. return select( - [taxable_income >= thresholds[2], taxable_income >= thresholds[1]], + [taxable_income > thresholds[2], taxable_income > thresholds[1]], [amounts.additional, amounts.higher], default=amounts.basic, ) From 2de2210d64ea37dcca61ace8d6eb3790563e0308 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 17:19:48 -0400 Subject: [PATCH 24/25] Use the full band base for the annual allowance charge; tidy the base variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups for #2352: - taxed_income, which sets the annual allowance charge's marginal rate, stacked on earned_taxable_income too and so left property income out; it now adds non_savings_taxable_income (Case 6: £10,000 of excess contributions with £60,000 of property income pay 40%, not 20%). - One non_savings_income_after_allowances variable feeds both taxable_property_income_after_allowances and property_income_tax. - Cite ss. 16A and 25(3A) for 2027-28; say the identity holds without Marriage Allowance and pin that case (#1947); point the starting-rate comment at #1799; replace the monotonic check, which main also passed, with a higher-rate check main fails. Co-Authored-By: Claude Opus 5.5 --- changelog.d/2351.fixed.md | 2 +- .../income_tax/property_income_band_base.yaml | 42 +++++++++++++++++++ ...st_property_income_band_base_properties.py | 16 ++++++- .../non_savings_income_after_allowances.py | 29 +++++++++++++ .../non_savings_taxable_income.py | 20 ++++++--- ...axable_property_income_after_allowances.py | 27 +++++++----- .../savings_starter_rate_income.py | 6 ++- .../liability/property_income_tax.py | 8 +--- .../gov/hmrc/income_tax/taxed_income.py | 5 ++- 9 files changed, 127 insertions(+), 28 deletions(-) create mode 100644 policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/non_savings_income_after_allowances.py diff --git a/changelog.d/2351.fixed.md b/changelog.d/2351.fixed.md index b40f9582f0..429087f2a7 100644 --- a/changelog.d/2351.fixed.md +++ b/changelog.d/2351.fixed.md @@ -1 +1 @@ -Band savings and dividend income above property income, which is non-savings income (ITA 2007 s. 16). +Band savings and dividend income above property income, which is non-savings income (ITA 2007 ss. 16, 16A), including in the income the annual allowance charge's marginal rate is read from. diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/property_income_band_base.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/property_income_band_base.yaml index 49d781b6e2..7312c6d5bd 100644 --- a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/property_income_band_base.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/property_income_band_base.yaml @@ -97,3 +97,45 @@ # Property income at the 22% property basic rate: 27,430 x 22% = 6,034.60. property_income_tax: 6_034.60 dividend_income_tax: 4_528.75 + +- name: Case 6, the annual allowance charge reads the rate on property income too. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 45 + employment_income: 10_000 + property_income: 60_000 + employee_pension_contributions: 70_000 + households: + household: + members: [person] + country: ENGLAND + output: + # Relief is limited to the £10,000 of pay, so taxable income is + # 70,000 - 12,570 - 10,000 = 47,430, in the higher-rate band; the + # £10,000 over the £60,000 annual allowance is charged at 40%. + taxed_income: 47_430 + personal_pension_contributions_tax: 4_000 + +- name: Case 7, Marriage Allowance still comes off the band base (pinned; #1947). + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 45 + employment_income: 20_000 + property_income: 10_000 + marriage_allowance: 1_260 + households: + household: + members: [person] + country: ENGLAND + output: + # earned_taxable_income subtracts Marriage Allowance, which the law gives + # as a tax reduction (ITA 2007 s. 55B), so the base is 30,000 - 12,570 - + # 1,260 = 16,170 rather than 17,430. Pinned here until #1947 is fixed. + non_savings_taxable_income: 16_170 + non_savings_income_after_allowances: 17_430 diff --git a/policyengine_uk/tests/test_property_income_band_base_properties.py b/policyengine_uk/tests/test_property_income_band_base_properties.py index b51a4f409b..7d5730f867 100644 --- a/policyengine_uk/tests/test_property_income_band_base_properties.py +++ b/policyengine_uk/tests/test_property_income_band_base_properties.py @@ -10,9 +10,12 @@ amount leaves savings and dividend tax unchanged, in England and in Scotland. (The taxable amount, because the £1,000 property allowance can relieve small property income in full.) -2. Accounting: non_savings_taxable_income equals adjusted net income less - savings, dividends and allowances, floored at nil. +2. Accounting: with no Marriage Allowance, non_savings_taxable_income equals + adjusted net income less savings, dividends and allowances, floored at nil + (earned_taxable_income also subtracts Marriage Allowance; #1947). 3. Monotonic: more property income never lowers savings or dividend tax. + (Main passes this too; the differential and the higher-rate check below + are what catch the bug.) """ import numpy as np @@ -113,3 +116,12 @@ def test_more_property_income_never_lowers_savings_or_dividend_tax(rows, rise): ) for name in ["savings_income_tax", "dividend_income_tax"]: assert np.all(after[name] >= before[name] - 0.01) + + +def test_savings_above_property_income_past_the_basic_rate_limit_pay_the_higher_rate(): + """£40,000 of property income leaves £10,270 of the basic-rate band, so + some of £20,000 of savings must pay 40%: savings tax exceeds 20% of + everything above the £500 savings allowance (£3,900). Main banded the + savings as if the property income were not there and charged £2,900.""" + v = _calculate([(0, 40_000, 20_000, 0)] + [(0, 0, 0, 0)] * (PEOPLE - 1), "ENGLAND") + assert v["savings_income_tax"][0] > (20_000 - 500) * 0.20 diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/non_savings_income_after_allowances.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/non_savings_income_after_allowances.py new file mode 100644 index 0000000000..0bc9405495 --- /dev/null +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/non_savings_income_after_allowances.py @@ -0,0 +1,29 @@ +from policyengine_uk.model_api import * + + +class non_savings_income_after_allowances(Variable): + value_type = float + entity = Person + label = "Non-savings, non-dividend income for Income Tax after allowances" + documentation = ( + "Adjusted net income less savings and dividend income and allowances, " + "floored at nil: the income below savings and dividends (ITA 2007 s. " + "16) once the allowances, which go to it first, are taken off. " + "property_income_tax and taxable_property_income_after_allowances " + "both read it." + ) + definition_period = YEAR + reference = dict( + title="Income Tax Act 2007, s. 16", + href="https://www.legislation.gov.uk/ukpga/2007/3/section/16", + ) + unit = GBP + + def formula(person, period, parameters): + return max_( + 0, + person("adjusted_net_income", period) + - person("taxable_savings_interest_income", period) + - person("taxable_dividend_income", period) + - person("allowances", period), + ) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/non_savings_taxable_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/non_savings_taxable_income.py index 0c6f878d0f..8e1d2253a4 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/non_savings_taxable_income.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/non_savings_taxable_income.py @@ -11,13 +11,23 @@ class non_savings_taxable_income(Variable): "earned_taxable_income plus property income after allowances. " "earned_taxable_income leaves property income out because " "property_income_tax taxes it separately, but property income still " - "fills the bands before savings and dividends do." + "fills the bands before savings and dividends do (from 2027-28, " + "s. 16A places it immediately below savings income). With no Marriage " + "Allowance it equals non_savings_income_after_allowances; " + "earned_taxable_income also subtracts Marriage Allowance, which the " + "law gives as a tax reduction (#1947)." ) definition_period = YEAR - reference = dict( - title="Income Tax Act 2007, s. 16", - href="https://www.legislation.gov.uk/ukpga/2007/3/section/16", - ) + reference = [ + dict( + title="Income Tax Act 2007, s. 16", + href="https://www.legislation.gov.uk/ukpga/2007/3/section/16", + ), + dict( + title="Income Tax Act 2007, s. 16A (from 2027-28)", + href="https://www.legislation.gov.uk/ukpga/2007/3/section/16A", + ), + ] unit = GBP def formula(person, period, parameters): diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/taxable_property_income_after_allowances.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/taxable_property_income_after_allowances.py index cb27d2133b..b38aabcf36 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/taxable_property_income_after_allowances.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/taxable_property_income_after_allowances.py @@ -9,22 +9,27 @@ class taxable_property_income_after_allowances(Variable): "Taxable property income less the part of the allowances it absorbs. " "Allowances go to non-savings, non-dividend income first, and " "property income is stacked above the rest of it, so it takes " - "whatever allowances the other non-savings income leaves." + "whatever allowances the other non-savings income leaves. From " + "2027-28 that order is statutory: s. 25(3A) sets allowances against " + "income other than property, savings and dividend income first, and " + "s. 16A places property income immediately below savings income." ) definition_period = YEAR - reference = dict( - title="Income Tax Act 2007, s. 25(2)", - href="https://www.legislation.gov.uk/ukpga/2007/3/section/25", - ) + reference = [ + dict( + title="Income Tax Act 2007, s. 25(3A) (from 2027-28)", + href="https://www.legislation.gov.uk/ukpga/2007/3/section/25", + ), + dict( + title="Income Tax Act 2007, s. 16A (from 2027-28)", + href="https://www.legislation.gov.uk/ukpga/2007/3/section/16A", + ), + ] unit = GBP def formula(person, period, parameters): - non_savings_after_allowances = max_( - 0, - person("adjusted_net_income", period) - - person("taxable_savings_interest_income", period) - - person("taxable_dividend_income", period) - - person("allowances", period), + non_savings_after_allowances = person( + "non_savings_income_after_allowances", period ) return min_( person("taxable_property_income", period), non_savings_after_allowances diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/savings_starter_rate_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/savings_starter_rate_income.py index 6cb516b928..94e38d7101 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/savings_starter_rate_income.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/savings_starter_rate_income.py @@ -37,8 +37,10 @@ def formula(person, period, parameters): - person("received_allowances_savings_income", period), ), ) - # The starting rate band is reduced by non-savings income, property - # income included (ITA 2007 ss. 12, 16). + # The taper reads all non-savings income, property income included + # (ITA 2007 s. 16). Where the taper starts is a separate, known defect: + # s. 12 shrinks the band from the first pound of taxable non-savings + # income, not from the personal allowance plus the band (#1799). earned_taxable_income = person("non_savings_taxable_income", period) removed_amount = max_( diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/liability/property_income_tax.py b/policyengine_uk/variables/gov/hmrc/income_tax/liability/property_income_tax.py index 9fcafebf6f..fad31dd093 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/liability/property_income_tax.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/liability/property_income_tax.py @@ -34,12 +34,8 @@ def formula(person, period, parameters): determining which tax band it falls into. """ rates = parameters(period).gov.hmrc.income_tax.rates - non_savings_after_allowances = max_( - 0, - person("adjusted_net_income", period) - - person("taxable_savings_interest_income", period) - - person("taxable_dividend_income", period) - - person("allowances", period), + non_savings_after_allowances = person( + "non_savings_income_after_allowances", period ) property_after_allowances = person( "taxable_property_income_after_allowances", period diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/taxed_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/taxed_income.py index 3729ad0428..0a937cb36d 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/taxed_income.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/taxed_income.py @@ -12,8 +12,11 @@ class taxed_income(Variable): href="https://www.legislation.gov.uk/ukpga/2007/3/section/23", ) + # Property income is taxed income too: non_savings_taxable_income adds it + # to earned_taxable_income, which leaves it out because + # property_income_tax taxes it separately. adds = [ - "earned_taxable_income", + "non_savings_taxable_income", "taxed_savings_income", "taxed_dividend_income", ] From b934055b110f34f90848297903953bc04fd30bb2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 17:39:27 -0400 Subject: [PATCH 25/25] Deduct Class 2 from trading earnings before April 2024 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third-round review fix for #2312. Until it was omitted from 6 April 2024, reg. 14(2)(a) of the Computation of Earnings Regulations deducted the Class 2 contributions payable on the chargeable profit. The deduction now uses the engine's Class 2 rules (class_2_liable, the year's contribution weeks) on the profit itself, not ni_class_2's base after capital allowances. The flat rate is nil from 2024-25, so 2026 is unchanged. Case 17 (2018): £7,000 of profit and £1,400 of personal pension contributions give £118.20 a week, under the £120 limit; without Class 2 it was £121.15, over it. The year-by-country oracle deducts Class 2 for 2016-2018. Co-Authored-By: Claude Opus 5.5 --- .../dwp/carers_allowance_earnings_limit.yaml | 18 ++++++++++++++++++ ...rers_allowance_earnings_limit_properties.py | 8 +++++++- .../gov/dwp/carers_allowance_earnings.py | 17 ++++++++++++++++- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml b/policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml index cdb8ee2568..f7ad5724ac 100644 --- a/policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml @@ -237,3 +237,21 @@ carers_allowance_earnings: 10_700 is_gainfully_employed_for_carers_allowance: true carers_allowance: 0 + +- name: Case 17, before April 2024 trading earnings are also net of Class 2. + period: 2018 + absolute_error_margin: 0.01 + input: + age: 45 + self_employment_income: 7_000 + personal_pension_contributions: 1_400 + care_hours: 40 + country: ENGLAND + output: + # 2018-19: no tax or Class 4 under the £11,850 allowance and lower + # profits limit; Class 2 at £2.95 for 52 weeks (reg. 14(2)(a)) = 153.40; + # half the pension 700. 7,000 - 153.40 - 700 = 6,146.60, £118.20 a week, + # under the £120 limit. Without Class 2 it would be £121.15, over it. + carers_allowance_earnings: 6_146.60 + is_gainfully_employed_for_carers_allowance: false + carers_allowance: 64.60 * 52 diff --git a/policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py b/policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py index 2e6c29bb20..497d9ad7f9 100644 --- a/policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py +++ b/policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py @@ -28,6 +28,7 @@ from hypothesis import strategies as st from policyengine_uk import Simulation +from policyengine_uk.utils.class_2 import class_2_contribution_weeks LIMIT = 204 RATE = 86.45 @@ -129,7 +130,9 @@ def test_trading_profit_takes_the_basic_rate_in_every_year(year, country): """Reg. 14(1): the basic rate, or for a Scottish taxpayer the Scottish basic rate, which were both 20% in each of these years. Before 2018-19 the Scottish scale has no starter rate, so reading its second bracket by - position at an instant would pick the higher rate.""" + position at an instant would pick the higher rate. Before 2024-25 reg. + 14(2)(a) also deducted Class 2: £30,000 is above the small profits + threshold in each year, so a full year of the flat rate.""" profit = 30_000 simulation = Simulation( situation={ @@ -147,9 +150,12 @@ def test_trading_profit_takes_the_basic_rate_in_every_year(year, country): hmrc = simulation.tax_benefit_system.parameters(year).gov.hmrc allowance = hmrc.income_tax.allowances.personal_allowance.amount class_4 = hmrc.national_insurance.class_4 + class_2 = hmrc.national_insurance.class_2 + class_2_weeks = class_2_contribution_weeks(year) expected = ( profit - 0.20 * (profit - allowance) + - class_2.flat_rate * class_2_weeks - class_4.rates.main * ( min(profit, class_4.thresholds.upper_profits_limit) diff --git a/policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py b/policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py index 2b962edb28..d9d7e6f167 100644 --- a/policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py +++ b/policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py @@ -1,4 +1,5 @@ from policyengine_uk.model_api import * +from policyengine_uk.utils.class_2 import class_2_contribution_weeks, class_2_liable class carers_allowance_earnings(Variable): @@ -25,7 +26,10 @@ class carers_allowance_earnings(Variable): "upper profits limits. Regulation 14(1) deducts the personal reliefs " "the claimant is entitled to, with no reduction for any allowance the " "pay uses, and capital expenditure is not deducted (reg. 13(6)(a)). " - "Regulation 14 deducts no Class 2 contribution.\n\n" + "Until 6 April 2024 reg. 14(2)(a) also deducted the Class 2 " + "contributions payable on the chargeable profit; that paragraph was " + "omitted with compulsory Class 2, whose flat rate is nil from " + "2024-25.\n\n" "Not modelled: the deduction for care charges (Schedule 3, up to half " "of net earnings) and the Schedule 1 disregards, so this can overstate " "earnings for a carer who pays for care while at work." @@ -88,6 +92,16 @@ def formula(person, period, parameters): min_(profit, class_4.thresholds.upper_profits_limit) - class_4.thresholds.lower_profits_limit, ) + # Reg. 14(2)(a), omitted from 6 April 2024: the Class 2 contributions + # payable on the chargeable profit (SSCBA 1992 s. 11), on the + # profit itself rather than ni_class_2's base after capital + # allowances. The flat rate is nil from 2024-25. + class_2 = p.national_insurance.class_2 + trading_class_2 = ( + class_2_liable(profit, class_2) + * class_2.flat_rate + * class_2_contribution_weeks(period.start.year) + ) half_pension_contributions = 0.5 * person("pension_contributions", period) return max_( @@ -98,5 +112,6 @@ def formula(person, period, parameters): + profit - trading_tax - trading_class_4 + - trading_class_2 - half_pension_contributions, )