From 40ea9453385b2197060e70c592e1495c1ac989d3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 3 Oct 2026 16:26:47 -0400 Subject: [PATCH 1/5] Restore HMRC salary sacrifice relief calibration targets GOV.UK withdrew the July 2025 Tables 6.1/6.2 CSV (410 Gone) when HMRC published the July 2026 private pension statistics, and get_targets() logged the error and returned without the relief targets, so every build since has calibrated without hmrc/salary_sacrifice_it_relief_* and the HMRC NICs relief targets. - Point sources.yaml at the July 2026 CSV (tax year 2024-25) and commit a copy in storage; a failed download uses it, and a table without the expected rows raises instead of returning fewer targets. - Map the tax year from the table: PolicyEngine year N is tax year N to N+1, so 2024-25 is 2024 (the old comment put 2023-24 at 2024). - Split IT relief across the bands each person's relief straddles, as HMRC does, instead of classifying by adjusted net income against taxable-income thresholds (which put every basic-rate taxpayer with income between 37,700 and 50,270 in the higher-rate target). - Drop obr/salary_sacrifice_{employee,employer}_ni_relief: static copies of HMRC's 2023-24 Table 6.2 figures computed on the same column as the restored HMRC NICs targets. Co-Authored-By: Claude Opus 5.5 --- ...c-salary-sacrifice-relief-targets.fixed.md | 1 + ..._pension_relief_tables_6_1_6_2_2024_25.csv | 83 +++++++ .../targets/build_loss_matrix.py | 2 - .../targets/compute/income.py | 84 ++++--- policyengine_uk_data/targets/sources.yaml | 4 +- .../targets/sources/hmrc_salary_sacrifice.py | 215 ++++++++++-------- policyengine_uk_data/targets/sources/obr.py | 24 +- .../test_hmrc_salary_sacrifice_targets.py | 178 +++++++++++++++ .../tests/test_target_registry.py | 19 ++ 9 files changed, 468 insertions(+), 142 deletions(-) create mode 100644 changelog.d/hmrc-salary-sacrifice-relief-targets.fixed.md create mode 100644 policyengine_uk_data/storage/hmrc_pension_relief_tables_6_1_6_2_2024_25.csv create mode 100644 policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py diff --git a/changelog.d/hmrc-salary-sacrifice-relief-targets.fixed.md b/changelog.d/hmrc-salary-sacrifice-relief-targets.fixed.md new file mode 100644 index 000000000..46b3fb23a --- /dev/null +++ b/changelog.d/hmrc-salary-sacrifice-relief-targets.fixed.md @@ -0,0 +1 @@ +Restore the HMRC salary sacrifice income tax and NICs relief calibration targets, lost to a withdrawn (410 Gone) HMRC CSV since the July 2026 private pension statistics release. They now use the 2024-25 Tables 6.1 and 6.2, mapped to year 2024, with a committed copy used when the download fails; a changed table fails the build. The rate-band targets split each person's relief across the bands it is given in (they had been assigned by adjusted net income against taxable-income thresholds), and the OBR-labelled NICs relief targets, which repeated the 2023-24 HMRC figures, are removed. diff --git a/policyengine_uk_data/storage/hmrc_pension_relief_tables_6_1_6_2_2024_25.csv b/policyengine_uk_data/storage/hmrc_pension_relief_tables_6_1_6_2_2024_25.csv new file mode 100644 index 000000000..48b149176 --- /dev/null +++ b/policyengine_uk_data/storage/hmrc_pension_relief_tables_6_1_6_2_2024_25.csv @@ -0,0 +1,83 @@ +"tax_year","income_tax_nics","contribution_type","nics_relief_class","sector_scheme","scheme_type","tax_rate","value_of_relief" +"2024 to 2025","Income Tax","Individual contributions to net pay arrangements","Not applicable","Total","Total","Total","7000" +"2024 to 2025","Income Tax","Individual contributions to relief at source schemes","Not applicable","Total","Total","Total","5900" +"2024 to 2025","Income Tax","Salary sacrificed contributions","Not applicable","Total","Total","Total","8800" +"2024 to 2025","Income Tax","Employer contributions to net pay arrangements plus deficit reduction contributions","Not applicable","Total","Total","Total","24800" +"2024 to 2025","Income Tax","Employer contributions to relief at source schemes","Not applicable","Total","Total","Total","9200" +"2024 to 2025","Income Tax","Individual contributions to net pay arrangements","Not applicable","Public sector occupational scheme","Total","Total","4500" +"2024 to 2025","Income Tax","Individual contributions to relief at source schemes","Not applicable","Public sector occupational scheme","Total","Total","200" +"2024 to 2025","Income Tax","Salary sacrificed contributions","Not applicable","Public sector occupational scheme","Total","Total","600" +"2024 to 2025","Income Tax","Employer contributions to net pay arrangements plus deficit reduction contributions","Not applicable","Public sector occupational scheme","Total","Total","14000" +"2024 to 2025","Income Tax","Employer contributions to relief at source schemes","Not applicable","Public sector occupational scheme","Total","Total","400" +"2024 to 2025","Income Tax","Individual contributions to net pay arrangements","Not applicable","Personal or private sector occupational scheme","Total","Total","2500" +"2024 to 2025","Income Tax","Individual contributions to relief at source schemes","Not applicable","Personal or private sector occupational scheme","Total","Total","5700" +"2024 to 2025","Income Tax","Salary sacrificed contributions","Not applicable","Personal or private sector occupational scheme","Total","Total","8300" +"2024 to 2025","Income Tax","Employer contributions to net pay arrangements plus deficit reduction contributions","Not applicable","Personal or private sector occupational scheme","Total","Total","10800" +"2024 to 2025","Income Tax","Employer contributions to relief at source schemes","Not applicable","Personal or private sector occupational scheme","Total","Total","8800" +"2024 to 2025","Income Tax","Individual contributions to net pay arrangements","Not applicable","Total","Defined benefit","Total","5200" +"2024 to 2025","Income Tax","Individual contributions to relief at source schemes","Not applicable","Total","Defined benefit","Total","0" +"2024 to 2025","Income Tax","Salary sacrificed contributions","Not applicable","Total","Defined benefit","Total","1500" +"2024 to 2025","Income Tax","Employer contributions to net pay arrangements plus deficit reduction contributions","Not applicable","Total","Defined benefit","Total","16600" +"2024 to 2025","Income Tax","Employer contributions to relief at source schemes","Not applicable","Total","Defined benefit","Total","0" +"2024 to 2025","Income Tax","Individual contributions to net pay arrangements","Not applicable","Total","Defined contribution","Total","1800" +"2024 to 2025","Income Tax","Individual contributions to relief at source schemes","Not applicable","Total","Defined contribution","Total","5900" +"2024 to 2025","Income Tax","Salary sacrificed contributions","Not applicable","Total","Defined contribution","Total","7400" +"2024 to 2025","Income Tax","Employer contributions to net pay arrangements plus deficit reduction contributions","Not applicable","Total","Defined contribution","Total","8200" +"2024 to 2025","Income Tax","Employer contributions to relief at source schemes","Not applicable","Total","Defined contribution","Total","9200" +"2024 to 2025","Income Tax","Individual contributions to net pay arrangements","Not applicable","Total","Total","Basic Rate","2400" +"2024 to 2025","Income Tax","Individual contributions to relief at source schemes","Not applicable","Total","Total","Basic Rate","3200" +"2024 to 2025","Income Tax","Salary sacrificed contributions","Not applicable","Total","Total","Basic Rate","1600" +"2024 to 2025","Income Tax","Employer contributions to net pay arrangements plus deficit reduction contributions","Not applicable","Total","Total","Basic Rate","6700" +"2024 to 2025","Income Tax","Employer contributions to relief at source schemes","Not applicable","Total","Total","Basic Rate","2200" +"2024 to 2025","Income Tax","Individual contributions to net pay arrangements","Not applicable","Total","Total","Higher Rate","3800" +"2024 to 2025","Income Tax","Individual contributions to relief at source schemes","Not applicable","Total","Total","Higher Rate","1600" +"2024 to 2025","Income Tax","Salary sacrificed contributions","Not applicable","Total","Total","Higher Rate","5500" +"2024 to 2025","Income Tax","Employer contributions to net pay arrangements plus deficit reduction contributions","Not applicable","Total","Total","Higher Rate","15600" +"2024 to 2025","Income Tax","Employer contributions to relief at source schemes","Not applicable","Total","Total","Higher Rate","5200" +"2024 to 2025","Income Tax","Individual contributions to net pay arrangements","Not applicable","Total","Total","Additional Rate","900" +"2024 to 2025","Income Tax","Individual contributions to relief at source schemes","Not applicable","Total","Total","Additional Rate","1100" +"2024 to 2025","Income Tax","Salary sacrificed contributions","Not applicable","Total","Total","Additional Rate","1800" +"2024 to 2025","Income Tax","Employer contributions to net pay arrangements plus deficit reduction contributions","Not applicable","Total","Total","Additional Rate","2500" +"2024 to 2025","Income Tax","Employer contributions to relief at source schemes","Not applicable","Total","Total","Additional Rate","1700" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Primary (employee)","Total","Total","Total","3600" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Primary (employee)","Total","Total","Total","1200" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Primary (employee)","Total","Total","Total","1000" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Secondary (employer)","Total","Total","Total","10600" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Secondary (employer)","Total","Total","Total","3800" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Secondary (employer)","Total","Total","Total","3400" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Primary (employee)","Public sector occupational scheme","Total","Total","2300" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Primary (employee)","Public sector occupational scheme","Total","Total","900" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Primary (employee)","Public sector occupational scheme","Total","Total","100" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Secondary (employer)","Public sector occupational scheme","Total","Total","6300" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Secondary (employer)","Public sector occupational scheme","Total","Total","200" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Secondary (employer)","Public sector occupational scheme","Total","Total","200" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Primary (employee)","Personal or private sector occupational scheme","Total","Total","1300" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Primary (employee)","Personal or private sector occupational scheme","Total","Total","300" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Primary (employee)","Personal or private sector occupational scheme","Total","Total","900" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Secondary (employer)","Personal or private sector occupational scheme","Total","Total","4300" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Secondary (employer)","Personal or private sector occupational scheme","Total","Total","3600" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Secondary (employer)","Personal or private sector occupational scheme","Total","Total","3100" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Primary (employee)","Total","Defined benefit","Total","2500" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Primary (employee)","Total","Defined benefit","Total","0" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Primary (employee)","Total","Defined benefit","Total","200" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Secondary (employer)","Total","Defined benefit","Total","7200" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Secondary (employer)","Total","Defined benefit","Total","0" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Secondary (employer)","Total","Defined benefit","Total","600" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Primary (employee)","Total","Defined contribution","Total","1100" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Primary (employee)","Total","Defined contribution","Total","1200" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Primary (employee)","Total","Defined contribution","Total","800" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Secondary (employer)","Total","Defined contribution","Total","3300" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Secondary (employer)","Total","Defined contribution","Total","3800" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Secondary (employer)","Total","Defined contribution","Total","2800" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Primary (employee)","Total","Total","Main Rate","2800" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Primary (employee)","Total","Total","Main Rate","900" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Primary (employee)","Total","Total","Main Rate","600" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Secondary (employer)","Total","Total","Main Rate","10600" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Secondary (employer)","Total","Total","Main Rate","3800" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Secondary (employer)","Total","Total","Main Rate","3400" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Primary (employee)","Total","Total","Additional Rate","800" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Primary (employee)","Total","Total","Additional Rate","300" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Primary (employee)","Total","Total","Additional Rate","300" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Secondary (employer)","Total","Total","Additional Rate","[z]" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Secondary (employer)","Total","Total","Additional Rate","[z]" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Secondary (employer)","Total","Total","Additional Rate","[z]" diff --git a/policyengine_uk_data/targets/build_loss_matrix.py b/policyengine_uk_data/targets/build_loss_matrix.py index 9552aeea2..0c374dbf8 100644 --- a/policyengine_uk_data/targets/build_loss_matrix.py +++ b/policyengine_uk_data/targets/build_loss_matrix.py @@ -386,9 +386,7 @@ def _compute_column(target: Target, ctx: _SimContext, year: int) -> np.ndarray | # Salary sacrifice NI relief if name in ( "hmrc/salary_sacrifice_employee_nics_relief", - "obr/salary_sacrifice_employee_ni_relief", "hmrc/salary_sacrifice_employer_nics_relief", - "obr/salary_sacrifice_employer_ni_relief", ): return compute_ss_ni_relief(target, ctx) diff --git a/policyengine_uk_data/targets/compute/income.py b/policyengine_uk_data/targets/compute/income.py index f5be3e2bb..d96c9234e 100644 --- a/policyengine_uk_data/targets/compute/income.py +++ b/policyengine_uk_data/targets/compute/income.py @@ -20,34 +20,66 @@ def compute_income_band(target, ctx) -> np.ndarray: return ctx.household_from_person(income_df[variable] * in_band) -def compute_ss_it_relief(target, ctx) -> np.ndarray: - """Compute salary sacrifice IT relief by tax band.""" - it_base = ctx.sim.calculate("income_tax") - it_cf = ctx.counterfactual_sim.calculate("income_tax", ctx.time_period) - it_relief = it_cf - it_base - - adj_net_income_cf = ctx.counterfactual_sim.calculate( - "adjusted_net_income", ctx.time_period - ) +# Income tax on earned income in each rUK band (rates.uk thresholds on +# earned_taxable_income), keyed by the band word in the target name. Scottish +# taxpayers are split on the same rUK boundaries, an approximation: HMRC +# groups their relief by Scottish marginal rate (starter and intermediate +# with basic), and the Scottish band thresholds differ from the rUK ones. +_BAND_TAX_VARIABLES = { + "basic": "basic_rate_earned_income_tax", + "higher": "higher_rate_earned_income_tax", + "additional": "add_rate_earned_income_tax", +} + + +def split_relief_by_band( + relief: np.ndarray, band_tax_cf: dict, band_tax_base: dict +) -> dict: + """Share each person's income tax relief across the bands it is given in. + + HMRC's Table 6.1 models contributions that straddle bands ("not all be + relievable at the same rate"; private pension statistics, background and + methodology), so a person's relief is split in proportion to the fall in + their earned-income tax within each band. Relief with no fall in any + earned band (it comes through savings or dividend bands) counts as basic + rate. The shares sum to one, so the bands add up to ``relief``. + """ + drops = { + band: np.maximum(np.asarray(band_tax_cf[band]) - band_tax_base[band], 0) + for band in _BAND_TAX_VARIABLES + } + total = sum(drops.values()) + has_drop = total > 0 + safe_total = np.where(has_drop, total, 1) + relief = np.asarray(relief) + return { + band: relief + * np.where(has_drop, drop / safe_total, 1.0 if band == "basic" else 0.0) + for band, drop in drops.items() + } - params = ctx.sim.tax_benefit_system.parameters.gov.hmrc.income_tax.rates.uk - basic_thresh = params[0].threshold(ctx.time_period) - higher_thresh = params[1].threshold(ctx.time_period) - additional_thresh = params[2].threshold(ctx.time_period) - name = target.name - if "basic" in name: - mask = (adj_net_income_cf > basic_thresh) & (adj_net_income_cf <= higher_thresh) - elif "higher" in name: - mask = (adj_net_income_cf > higher_thresh) & ( - adj_net_income_cf <= additional_thresh - ) - elif "additional" in name: - mask = adj_net_income_cf > additional_thresh - else: - mask = np.ones_like(it_relief, dtype=bool) - - return ctx.household_from_person(it_relief * mask) +def compute_ss_it_relief(target, ctx) -> np.ndarray: + """Compute salary sacrifice IT relief, in total or for one rate band.""" + period = ctx.time_period + cf, base = ctx.counterfactual_sim, ctx.sim + relief = np.asarray(cf.calculate("income_tax", period)) - np.asarray( + base.calculate("income_tax", period) + ) + band = next((b for b in _BAND_TAX_VARIABLES if b in target.name), None) + if band is not None: + relief = split_relief_by_band( + relief, + { + b: np.asarray(cf.calculate(v, period)) + for b, v in _BAND_TAX_VARIABLES.items() + }, + { + b: np.asarray(base.calculate(v, period)) + for b, v in _BAND_TAX_VARIABLES.items() + }, + )[band] + return ctx.household_from_person(relief) def compute_ss_contributions(target, ctx) -> np.ndarray: diff --git a/policyengine_uk_data/targets/sources.yaml b/policyengine_uk_data/targets/sources.yaml index 5d1cd89ba..973e39567 100644 --- a/policyengine_uk_data/targets/sources.yaml +++ b/policyengine_uk_data/targets/sources.yaml @@ -11,7 +11,9 @@ hmrc: spi_geography: "https://assets.publishing.service.gov.uk/media/69f1f17cc42061e837e3ac3b/Collated_Tables_3_12_to_3_15a_2324.ods" income_tax_liabilities: "https://www.gov.uk/government/statistics/income-tax-liabilities-statistics-tax-year-2022-to-2023-to-tax-year-2025-to-2026" capital_gains_statistics: "https://www.gov.uk/government/statistics/capital-gains-tax-statistics" - salary_sacrifice_table_6: "https://assets.publishing.service.gov.uk/media/687a294e312ee8a5f0806b6d/Tables_6_1_and_6_2.csv" + # Private pension statistics, July 2026 (tax year 2024-25). Refresh with + # storage/hmrc_pension_relief_tables_6_1_6_2_*.csv (hmrc_salary_sacrifice.py). + salary_sacrifice_table_6: "https://assets.publishing.service.gov.uk/media/6a673e0e5e87122783093901/Tables_6_1_and_6_2.csv" dwp: stat_xplore_api: "https://stat-xplore.dwp.gov.uk/webapi/rest/v1" diff --git a/policyengine_uk_data/targets/sources/hmrc_salary_sacrifice.py b/policyengine_uk_data/targets/sources/hmrc_salary_sacrifice.py index f9865e6f7..828d48688 100644 --- a/policyengine_uk_data/targets/sources/hmrc_salary_sacrifice.py +++ b/policyengine_uk_data/targets/sources/hmrc_salary_sacrifice.py @@ -1,13 +1,23 @@ """HMRC salary sacrifice income tax and NICs relief targets. -Downloads Table 6.2 CSV from HMRC to get salary sacrifice IT relief -by tax rate band and NICs relief (employee + employer). - -Source: https://assets.publishing.service.gov.uk/media/687a294e312ee8a5f0806b6d/Tables_6_1_and_6_2.csv +Reads HMRC private pension statistics Tables 6.1 and 6.2 (tidy CSV): income +tax relief on salary-sacrificed pension contributions by marginal rate, and +the Class 1 primary (employee) and secondary (employer) NICs relief on them. + +GOV.UK withdraws a release's assets when HMRC publishes the next one: the +July 2025 CSV has returned 410 Gone since the July 2026 release, and the +broad ``except`` this module used to have turned that into calibration +builds that silently lacked these targets. A failed download now falls back +to the copy committed in storage (the same release as ``sources.yaml``), and +a table that no longer has the expected rows raises instead of returning +fewer targets. + +Source: https://www.gov.uk/government/statistics/personal-and-stakeholder-pensions-statistics """ import io import logging +import re import pandas as pd import requests @@ -15,110 +25,135 @@ from policyengine_uk_data.targets.schema import Target, Unit from policyengine_uk_data.targets.sources._common import ( HEADERS, + STORAGE, load_config, to_float, ) logger = logging.getLogger(__name__) +# Tables 6.1 and 6.2 from the July 2026 release (tax year 2024-25), as +# downloaded from the sources.yaml URL. Replace both together. +FALLBACK_CSV = STORAGE / "hmrc_pension_relief_tables_6_1_6_2_2024_25.csv" + # Uprate 3% pa for wage growth from the base year _GROWTH = 1.03 -_BASE_YEAR = 2024 # 2023-24 tax year → calendar 2024 +_LAST_YEAR = 2031 + +# Table 6.1 rate rows → target name suffix. HMRC counts Scottish starter +# and intermediate rate relief as basic rate. +_IT_RATES = { + "Basic Rate": "basic_rate", + "Higher Rate": "higher_rate", + "Additional Rate": "additional_rate", + "Total": "total", +} +# Table 6.2 NICs classes → (target name, PolicyEngine variable) +_NICS_CLASSES = { + "Class 1 Primary (employee)": ( + "hmrc/salary_sacrifice_employee_nics_relief", + "ni_employee", + ), + "Class 1 Secondary (employer)": ( + "hmrc/salary_sacrifice_employer_nics_relief", + "ni_employer", + ), +} + +# Total salary sacrifice contributions (SPP Review 2025: £24bn base) +_SS_CONTRIBUTIONS_BASE_YEAR = 2024 + + +def _read_table(url: str) -> pd.DataFrame: + try: + r = requests.get(url, headers=HEADERS, allow_redirects=True, timeout=30) + r.raise_for_status() + text = r.content.decode("utf-8-sig") + except requests.RequestException as e: + logger.warning( + "HMRC Tables 6.1 and 6.2 download failed (%s); using %s", + e, + FALLBACK_CSV.name, + ) + text = FALLBACK_CSV.read_text(encoding="utf-8-sig") + return pd.read_csv(io.StringIO(text), dtype=str) + + +def _base_year(df: pd.DataFrame) -> int: + """Map the table's tax year to a PolicyEngine year. + + PolicyEngine UK's year N is tax year N to N+1 (a year-N simulation reads + the parameter values in force from 6 April N), so "2024 to 2025" is 2024. + """ + years = df["tax_year"].unique() + match = re.fullmatch(r"(\d{4}) to (\d{4})", years[0]) if len(years) == 1 else None + if match is None or int(match[2]) != int(match[1]) + 1: + raise ValueError(f"Expected one tax year in Tables 6.1/6.2, got {years}") + return int(match[1]) + + +def _relief(rows: pd.DataFrame, column: str, label: str) -> float: + """The single positive £m value in ``rows`` whose ``column`` is ``label``.""" + values = rows.loc[rows[column] == label, "value_of_relief"].map(to_float) + if len(values) != 1 or values.iloc[0] <= 0: + raise ValueError( + f"HMRC salary sacrifice relief: expected one positive value for " + f"{label!r}, got {values.tolist()}" + ) + return values.iloc[0] * 1e6 + + +def _relief_targets(df: pd.DataFrame, reference_url: str) -> list[Target]: + base_year = _base_year(df) + ss = df[ + (df["contribution_type"] == "Salary sacrificed contributions") + & (df["sector_scheme"] == "Total") + & (df["scheme_type"] == "Total") + ] + income_tax = ss[ss["income_tax_nics"] == "Income Tax"] + nics = ss[(ss["income_tax_nics"] == "NICs") & (ss["tax_rate"] == "Total")] + + specs = [ + ( + f"hmrc/salary_sacrifice_it_relief_{suffix}", + "income_tax", + _relief(income_tax, "tax_rate", rate), + ) + for rate, suffix in _IT_RATES.items() + ] + [ + (name, variable, _relief(nics, "nics_relief_class", nics_class)) + for nics_class, (name, variable) in _NICS_CLASSES.items() + ] + return [ + Target( + name=name, + variable=variable, + source="hmrc", + unit=Unit.GBP, + values={ + y: base * _GROWTH ** (y - base_year) + for y in range(base_year, _LAST_YEAR + 1) + }, + reference_url=reference_url, + ) + for name, variable, base in specs + ] def get_targets() -> list[Target]: - config = load_config() - ref = config["hmrc"]["salary_sacrifice_table_6"] - targets = [] + ref = load_config()["hmrc"]["salary_sacrifice_table_6"] + targets = _relief_targets(_read_table(ref), ref) - try: - r = requests.get(ref, headers=HEADERS, allow_redirects=True, timeout=30) - r.raise_for_status() - df = pd.read_csv(io.StringIO(r.content.decode("utf-8-sig"))) - - ss = df[df["contribution_type"] == "Salary sacrificed contributions"] - - # IT relief by tax band - ss_it = ss[ - (ss["income_tax_nics"] == "Income Tax") - & (ss["sector_scheme"] == "Total") - & (ss["scheme_type"] == "Total") - ] - for _, row in ss_it.iterrows(): - rate = row["tax_rate"] - val = to_float(row["value_of_relief"]) - if val <= 0: - continue - rate_key = rate.lower().replace(" ", "_") - base = val * 1e6 - targets.append( - Target( - name=f"hmrc/salary_sacrifice_it_relief_{rate_key}", - variable="income_tax", - source="hmrc", - unit=Unit.GBP, - values={ - y: base * _GROWTH ** max(0, y - _BASE_YEAR) - for y in range(_BASE_YEAR, 2032) - }, - reference_url=ref, - ) - ) - - # NICs relief (employee + employer) - ss_nics = ss[ - (ss["income_tax_nics"] == "NICs") - & (ss["sector_scheme"] == "Total") - & (ss["scheme_type"] == "Total") - ] - for _, row in ss_nics.iterrows(): - nics_class = row["nics_relief_class"] - val = to_float(row["value_of_relief"]) - if val <= 0: - continue - if "employee" in str(nics_class).lower(): - name = "hmrc/salary_sacrifice_employee_nics_relief" - variable = "ni_employee" - elif "employer" in str(nics_class).lower(): - name = "hmrc/salary_sacrifice_employer_nics_relief" - variable = "ni_employer" - else: - continue - - # Only take the first (Total scheme) row for each class - existing = {t.name for t in targets} - if name in existing: - continue - - base = val * 1e6 - targets.append( - Target( - name=name, - variable=variable, - source="hmrc", - unit=Unit.GBP, - values={ - y: base * _GROWTH ** max(0, y - _BASE_YEAR) - for y in range(_BASE_YEAR, 2032) - }, - reference_url=ref, - ) - ) - - except Exception as e: - logger.error("Failed to download/parse HMRC salary sacrifice CSV: %s", e) - - # Total salary sacrifice contributions (SPP Review 2025: £24bn base) - _SS_CONTRIBUTIONS = { - y: 24e9 * _GROWTH ** max(0, y - _BASE_YEAR) for y in range(_BASE_YEAR, 2030) - } targets.append( Target( name="hmrc/salary_sacrifice_contributions", variable="pension_contributions_via_salary_sacrifice", source="hmrc", unit=Unit.GBP, - values=_SS_CONTRIBUTIONS, + values={ + y: 24e9 * _GROWTH ** (y - _SS_CONTRIBUTIONS_BASE_YEAR) + for y in range(_SS_CONTRIBUTIONS_BASE_YEAR, 2030) + }, reference_url=( "https://assets.publishing.service.gov.uk/media/" "67ce0e7c08e764d17a5d3c21/2025_SPP_Review.pdf" diff --git a/policyengine_uk_data/targets/sources/obr.py b/policyengine_uk_data/targets/sources/obr.py index ad4e3cf17..82909c0ca 100644 --- a/policyengine_uk_data/targets/sources/obr.py +++ b/policyengine_uk_data/targets/sources/obr.py @@ -617,9 +617,7 @@ def _parse_tv_licence(wb: openpyxl.Workbook) -> list[Target]: # ISC census: private school students (roughly constant at ~557k) _PRIVATE_SCHOOL = {y: 557_000 for y in range(2018, 2032)} -# SPP Review: salary sacrifice NI relief (uprated 3% pa from 2024 base) -_SS_EMPLOYEE_NI = {y: 1.2e9 * 1.03 ** max(0, y - 2024) for y in range(2024, 2032)} -_SS_EMPLOYER_NI = {y: 2.9e9 * 1.03 ** max(0, y - 2024) for y in range(2024, 2032)} +# Salary sacrifice NICs relief comes from HMRC Table 6.2 (hmrc_salary_sacrifice.py). # Salary sacrifice headcount: 7.7m total (3.3m above £2k, 4.3m below) # OBR para 1.7: SS population grows 0.9% faster than employees (~2.4%/yr) @@ -666,26 +664,6 @@ def get_targets() -> list[Target]: reference_url="https://www.isc.co.uk/research/annual-census/", ) ) - targets.append( - Target( - name="obr/salary_sacrifice_employee_ni_relief", - variable="ni_employee", - source="obr", - unit=Unit.GBP, - values=_SS_EMPLOYEE_NI, - reference_url="https://assets.publishing.service.gov.uk/media/67ce0e7c08e764d17a5d3c21/2025_SPP_Review.pdf", - ) - ) - targets.append( - Target( - name="obr/salary_sacrifice_employer_ni_relief", - variable="ni_employer", - source="obr", - unit=Unit.GBP, - values=_SS_EMPLOYER_NI, - reference_url="https://assets.publishing.service.gov.uk/media/67ce0e7c08e764d17a5d3c21/2025_SPP_Review.pdf", - ) - ) # Salary sacrifice headcount targets _SS_REF = ( diff --git a/policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py b/policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py new file mode 100644 index 000000000..801e86a9f --- /dev/null +++ b/policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py @@ -0,0 +1,178 @@ +"""Tests for the HMRC salary sacrifice relief targets (Tables 6.1 and 6.2). + +The July 2025 CSV returned 410 Gone once HMRC published the July 2026 +release, and the source module logged the error and returned without the +relief targets, so builds calibrated without them. These tests pin: the +committed table parses to every relief target, a failed download uses it, a +changed table raises, and the rate-band columns split relief the way HMRC +does (across the bands a contribution straddles). +""" + +from unittest.mock import patch + +import numpy as np +import pandas as pd +import pytest +import requests + +from policyengine_uk_data.targets.compute.income import ( + compute_ss_it_relief, + split_relief_by_band, +) +from policyengine_uk_data.targets.schema import Target, Unit +from policyengine_uk_data.targets.sources import hmrc_salary_sacrifice as hmrc_ss + +# Tables 6.1 and 6.2, tax year 2024-25 (£m), from the committed CSV. +RELIEF_2024_25 = { + "hmrc/salary_sacrifice_it_relief_total": 8_800, + "hmrc/salary_sacrifice_it_relief_basic_rate": 1_600, + "hmrc/salary_sacrifice_it_relief_higher_rate": 5_500, + "hmrc/salary_sacrifice_it_relief_additional_rate": 1_800, + "hmrc/salary_sacrifice_employee_nics_relief": 1_000, + "hmrc/salary_sacrifice_employer_nics_relief": 3_400, +} + + +def _committed_table() -> pd.DataFrame: + return pd.read_csv(hmrc_ss.FALLBACK_CSV, dtype=str, encoding="utf-8-sig") + + +def test_committed_table_gives_every_relief_target(): + targets = {t.name: t for t in hmrc_ss._relief_targets(_committed_table(), "x")} + assert set(targets) == set(RELIEF_2024_25) + for name, millions in RELIEF_2024_25.items(): + values = targets[name].values + # Tax year 2024-25 is PolicyEngine year 2024. + assert min(values) == 2024 + assert values[2024] == pytest.approx(millions * 1e6) + assert values[2025] == pytest.approx(millions * 1e6 * 1.03) + assert targets["hmrc/salary_sacrifice_employee_nics_relief"].variable == ( + "ni_employee" + ) + assert targets["hmrc/salary_sacrifice_employer_nics_relief"].variable == ( + "ni_employer" + ) + + +def _gone(*args, **kwargs): + response = requests.Response() + response.status_code = 410 + response.url = args[0] + return response + + +def _offline(*args, **kwargs): + raise requests.ConnectionError("offline") + + +@pytest.mark.parametrize("get", [_gone, _offline], ids=["410", "offline"]) +def test_failed_download_uses_committed_table(get): + with patch.object(hmrc_ss.requests, "get", side_effect=get): + names = {t.name for t in hmrc_ss.get_targets()} + assert set(RELIEF_2024_25) <= names + assert "hmrc/salary_sacrifice_contributions" in names + + +@pytest.mark.parametrize( + "change", + [ + lambda df: df[df["tax_rate"] != "Higher Rate"], + lambda df: df[df["nics_relief_class"] != "Class 1 Secondary (employer)"], + lambda df: pd.concat( + [df.iloc[:-1], df.iloc[-1:].assign(tax_year="2025 to 2026")] + ), + lambda df: df.assign(tax_year="2024 to 2026"), + lambda df: df.drop(columns="sector_scheme"), + ], + ids=["no-higher-rate", "no-employer-nics", "two-years", "bad-year", "no-column"], +) +def test_changed_table_raises(change): + with pytest.raises((ValueError, KeyError)): + hmrc_ss._relief_targets(change(_committed_table()), "x") + + +BANDS = ("basic", "higher", "additional") + + +def test_split_straddling_relief(): + # £4k sacrificed from £52k pay (2025-26 rUK): earned taxable income falls + # from £39,430 to £35,430, so £1,730 leaves the higher band and £2,270 + # the basic band. + split = split_relief_by_band( + np.array([1_146.0]), + {"basic": [7_540.0], "higher": [692.0], "additional": [0.0]}, + {"basic": [7_086.0], "higher": [0.0], "additional": [0.0]}, + ) + assert split["basic"] == pytest.approx([454.0]) + assert split["higher"] == pytest.approx([692.0]) + assert split["additional"] == pytest.approx([0.0]) + + +def test_split_invariants_on_random_inputs(): + """Bands add up to the relief, keep its sign, and default to basic rate.""" + rng = np.random.default_rng(0) + n = 5_000 + relief = rng.normal(500, 1_000, n) * rng.integers(0, 2, n) + base = {b: rng.uniform(0, 20_000, n) * rng.integers(0, 2, n) for b in BANDS} + # A smaller sacrifice never raises band tax; some people see no change. + cf = {b: base[b] + rng.uniform(0, 3_000, n) * rng.integers(0, 2, n) for b in BANDS} + split = split_relief_by_band(relief, cf, base) + + np.testing.assert_allclose(sum(split.values()), relief, rtol=1e-12, atol=1e-9) + for band in BANDS: + assert np.all(split[band] * np.sign(relief) >= -1e-9) + no_drop = sum(cf[b] - base[b] for b in BANDS) == 0 + assert no_drop.any() + np.testing.assert_array_equal(split["basic"][no_drop], relief[no_drop]) + + +class _Ctx: + """The parts of build_loss_matrix._SimContext the compute function reads.""" + + time_period = 2025 + + def __init__(self, base_pay, sacrifice): + from policyengine_uk import Simulation + + def sim(pay): + return Simulation( + situation={ + "people": { + "a": {"age": {2025: 40}, "employment_income": {2025: pay}} + }, + "benunits": {"b": {"members": ["a"]}}, + "households": {"h": {"members": ["a"]}}, + } + ) + + # The counterfactual adds the sacrifice back to pay, as + # _SimContext.counterfactual_sim does. + self.sim = sim(base_pay) + self.counterfactual_sim = sim(base_pay + sacrifice) + + @staticmethod + def household_from_person(values): + return np.asarray(values) + + +def _relief(ctx, suffix): + target = Target( + name=f"hmrc/salary_sacrifice_it_relief_{suffix}", + variable="income_tax", + source="hmrc", + unit=Unit.GBP, + values={2025: 1.0}, + ) + return float(compute_ss_it_relief(target, ctx)[0]) + + +def test_compute_splits_relief_for_a_basic_rate_taxpayer_near_the_threshold(): + """Pay of £48k after a £4k sacrifice: 40% relief only above £50,270.""" + ctx = _Ctx(base_pay=48_000, sacrifice=4_000) + by_band = { + s: _relief(ctx, s) for s in ("basic_rate", "higher_rate", "additional_rate") + } + assert _relief(ctx, "total") == pytest.approx(1_146) + assert by_band == pytest.approx( + {"basic_rate": 454, "higher_rate": 692, "additional_rate": 0} + ) diff --git a/policyengine_uk_data/tests/test_target_registry.py b/policyengine_uk_data/tests/test_target_registry.py index 22cf7b2d0..59d0b09f5 100644 --- a/policyengine_uk_data/tests/test_target_registry.py +++ b/policyengine_uk_data/tests/test_target_registry.py @@ -81,6 +81,25 @@ def pe_count(variable): assert _compute_column(target, DummyCtx(), 2025) == [1, 0, 1] +def test_hmrc_salary_sacrifice_relief_targets(): + """HMRC Table 6.1/6.2 relief targets reach the registry (uk-data: 410 Gone + on the old CSV dropped them from every build without failing it).""" + targets = {t.name: t for t in get_all_targets(year=2025)} + for name in ( + "hmrc/salary_sacrifice_it_relief_total", + "hmrc/salary_sacrifice_it_relief_basic_rate", + "hmrc/salary_sacrifice_it_relief_higher_rate", + "hmrc/salary_sacrifice_it_relief_additional_rate", + "hmrc/salary_sacrifice_employee_nics_relief", + "hmrc/salary_sacrifice_employer_nics_relief", + ): + assert name in targets, f"{name} missing from the target registry" + # One NICs relief target per class: the OBR-labelled copies of the + # 2023-24 figures are gone. + assert "obr/salary_sacrifice_employee_ni_relief" not in targets + assert "obr/salary_sacrifice_employer_ni_relief" not in targets + + def test_voa_council_tax_targets(): """VOA council tax band targets should exist.""" targets = get_all_targets(year=2025) From 138c72ac246315cfeaefcf0402b0be76f95c45f4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 4 Oct 2026 00:35:22 -0400 Subject: [PATCH 2/5] Address review: rate-adjust NICs targets, relieve pay at rUK or Scottish bands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Employer NICs relief targets follow the Class 1 rate in PolicyEngine's parameters (13.8% in 2024-25, 15% from April 2025), so the 2025 target is £3.81bn, not £3.50bn; employee targets weight the main and additional rates by HMRC's split (unchanged, 8% and 2%). - Income tax relief is the rise in tax on earned income, allocated to HMRC's basic/higher/additional categories under the person's rUK or Scottish rates (starter and intermediate with basic, advanced with higher, top with additional). HMRC applies income tax rates to ASHE pay; the previous measure (change in income_tax) included the High Income Child Benefit Charge and savings and dividend tax, and split Scottish relief on rUK boundaries. - Drop the IT relief total target: it is the sum of the bands, which HMRC rounds separately (£8.9bn vs £8.8bn in 2024-25). - The table's tax year must equal _TAX_YEAR, which names the committed copy, so pointing sources.yaml at a new release without refreshing the copy fails instead of giving online and offline builds different targets. - Test that every relief target produces a loss-matrix column on the built dataset (create_target_matrix skips with a warning otherwise). Co-Authored-By: Claude Opus 5.5 --- .../targets/compute/income.py | 103 +++++----- .../targets/sources/hmrc_salary_sacrifice.py | 104 ++++++++-- .../test_hmrc_salary_sacrifice_targets.py | 188 +++++++++++++----- .../tests/test_target_registry.py | 3 +- 4 files changed, 269 insertions(+), 129 deletions(-) diff --git a/policyengine_uk_data/targets/compute/income.py b/policyengine_uk_data/targets/compute/income.py index d96c9234e..19c49ac71 100644 --- a/policyengine_uk_data/targets/compute/income.py +++ b/policyengine_uk_data/targets/compute/income.py @@ -20,66 +20,59 @@ def compute_income_band(target, ctx) -> np.ndarray: return ctx.household_from_person(income_df[variable] * in_band) -# Income tax on earned income in each rUK band (rates.uk thresholds on -# earned_taxable_income), keyed by the band word in the target name. Scottish -# taxpayers are split on the same rUK boundaries, an approximation: HMRC -# groups their relief by Scottish marginal rate (starter and intermediate -# with basic), and the Scottish band thresholds differ from the rUK ones. -_BAND_TAX_VARIABLES = { - "basic": "basic_rate_earned_income_tax", - "higher": "higher_rate_earned_income_tax", - "additional": "add_rate_earned_income_tax", -} - - -def split_relief_by_band( - relief: np.ndarray, band_tax_cf: dict, band_tax_base: dict -) -> dict: - """Share each person's income tax relief across the bands it is given in. - - HMRC's Table 6.1 models contributions that straddle bands ("not all be - relievable at the same rate"; private pension statistics, background and - methodology), so a person's relief is split in proportion to the fall in - their earned-income tax within each band. Relief with no fall in any - earned band (it comes through savings or dividend bands) counts as basic - rate. The shares sum to one, so the bands add up to ``relief``. +def tax_by_band(income: np.ndarray, thresholds, rates) -> dict: + """Tax on ``income`` within each of HMRC's rate categories. + + HMRC's Table 6.1 groups relief by marginal rate: basic (with the Scottish + starter and intermediate rates), higher and additional. Brackets taxed + below 30% are basic, brackets at the scale's top rate are additional (the + rUK additional rate, the Scottish top rate) and the rest are higher (the + Scottish higher and advanced rates, which cover the income range of the + rUK higher rate). The categories sum to the scale's tax on ``income``. """ - drops = { - band: np.maximum(np.asarray(band_tax_cf[band]) - band_tax_base[band], 0) - for band in _BAND_TAX_VARIABLES - } - total = sum(drops.values()) - has_drop = total > 0 - safe_total = np.where(has_drop, total, 1) - relief = np.asarray(relief) - return { - band: relief - * np.where(has_drop, drop / safe_total, 1.0 if band == "basic" else 0.0) - for band, drop in drops.items() - } + income = np.asarray(income, dtype=float) + thresholds = [t for t, r in zip(thresholds, rates) if r is not None] + rates = [r for r in rates if r is not None] + uppers = thresholds[1:] + [np.inf] + bands = {"basic": 0.0, "higher": 0.0, "additional": 0.0} + for lower, upper, rate in zip(thresholds, uppers, rates): + band = ( + "basic" if rate < 0.3 else "additional" if rate == rates[-1] else "higher" + ) + bands[band] = bands[band] + rate * np.clip(income - lower, 0, upper - lower) + return bands + + +def ss_it_relief_by_band(ctx) -> dict: + """Person-level salary sacrifice income tax relief in each rate category. + + HMRC applies income tax rates to employees' pay (ASHE), so relief is the + rise in tax on earned income when the sacrifice is paid as salary, under + the person's rUK or Scottish rates. Contributions that straddle a band + boundary are relieved partly at each rate, as in HMRC's estimates. + """ + period = ctx.time_period + rates = ctx.sim.tax_benefit_system.parameters(period).gov.hmrc.income_tax.rates + scottish = np.asarray(ctx.sim.calculate("pays_scottish_income_tax", period)) + base = np.asarray(ctx.sim.calculate("earned_taxable_income", period)) + cf = np.asarray(ctx.counterfactual_sim.calculate("earned_taxable_income", period)) + relief = {} + for scale, in_scale in ((rates.uk, ~scottish), (rates.scotland.rates, scottish)): + band_cf = tax_by_band(cf, scale.thresholds, scale.rates) + band_base = tax_by_band(base, scale.thresholds, scale.rates) + for band in band_cf: + relief[band] = relief.get(band, 0) + in_scale * ( + band_cf[band] - band_base[band] + ) + return relief def compute_ss_it_relief(target, ctx) -> np.ndarray: - """Compute salary sacrifice IT relief, in total or for one rate band.""" - period = ctx.time_period - cf, base = ctx.counterfactual_sim, ctx.sim - relief = np.asarray(cf.calculate("income_tax", period)) - np.asarray( - base.calculate("income_tax", period) + """Compute salary sacrifice income tax relief at one rate.""" + band = target.name.removeprefix("hmrc/salary_sacrifice_it_relief_") + return ctx.household_from_person( + ss_it_relief_by_band(ctx)[band.removesuffix("_rate")] ) - band = next((b for b in _BAND_TAX_VARIABLES if b in target.name), None) - if band is not None: - relief = split_relief_by_band( - relief, - { - b: np.asarray(cf.calculate(v, period)) - for b, v in _BAND_TAX_VARIABLES.items() - }, - { - b: np.asarray(base.calculate(v, period)) - for b, v in _BAND_TAX_VARIABLES.items() - }, - )[band] - return ctx.household_from_person(relief) def compute_ss_contributions(target, ctx) -> np.ndarray: diff --git a/policyengine_uk_data/targets/sources/hmrc_salary_sacrifice.py b/policyengine_uk_data/targets/sources/hmrc_salary_sacrifice.py index 828d48688..f90754fe0 100644 --- a/policyengine_uk_data/targets/sources/hmrc_salary_sacrifice.py +++ b/policyengine_uk_data/targets/sources/hmrc_salary_sacrifice.py @@ -9,15 +9,19 @@ broad ``except`` this module used to have turned that into calibration builds that silently lacked these targets. A failed download now falls back to the copy committed in storage (the same release as ``sources.yaml``), and -a table that no longer has the expected rows raises instead of returning -fewer targets. +a table for another tax year, or without the expected rows, raises instead +of returning fewer targets. + +Targets grow 3% a year from the table's year. The NICs relief targets also +follow the Class 1 rates in PolicyEngine's parameters, so the April 2025 +rise in the employer rate from 13.8% to 15% raises the employer target in +the years the simulation charges 15%. Source: https://www.gov.uk/government/statistics/personal-and-stakeholder-pensions-statistics """ import io import logging -import re import pandas as pd import requests @@ -32,21 +36,27 @@ logger = logging.getLogger(__name__) -# Tables 6.1 and 6.2 from the July 2026 release (tax year 2024-25), as -# downloaded from the sources.yaml URL. Replace both together. -FALLBACK_CSV = STORAGE / "hmrc_pension_relief_tables_6_1_6_2_2024_25.csv" +# Tax year of the Tables 6.1 and 6.2 release in sources.yaml (July 2026, +# 2024-25) and of its committed copy. Refresh the URL, the copy and this +# together: a table for any other year raises. +_TAX_YEAR = 2024 +FALLBACK_CSV = ( + STORAGE + / f"hmrc_pension_relief_tables_6_1_6_2_{_TAX_YEAR}_{(_TAX_YEAR + 1) % 100:02d}.csv" +) # Uprate 3% pa for wage growth from the base year _GROWTH = 1.03 _LAST_YEAR = 2031 # Table 6.1 rate rows → target name suffix. HMRC counts Scottish starter -# and intermediate rate relief as basic rate. +# and intermediate rate relief as basic rate. The Total row is not a target: +# it is the sum of the bands, which HMRC rounds separately (2024-25: bands +# £8.9bn, total £8.8bn), so targeting both would ask for two values. _IT_RATES = { "Basic Rate": "basic_rate", "Higher Rate": "higher_rate", "Additional Rate": "additional_rate", - "Total": "total", } # Table 6.2 NICs classes → (target name, PolicyEngine variable) _NICS_CLASSES = { @@ -60,6 +70,13 @@ ), } +# Table 6.2 (class, rate row) → the PolicyEngine Class 1 rate it was relieved at +_NICS_RATE_PARAMETERS = { + ("Class 1 Primary (employee)", "Main Rate"): "employee.main", + ("Class 1 Primary (employee)", "Additional Rate"): "employee.additional", + ("Class 1 Secondary (employer)", "Main Rate"): "employer", +} + # Total salary sacrifice contributions (SPP Review 2025: £24bn base) _SS_CONTRIBUTIONS_BASE_YEAR = 2024 @@ -79,17 +96,54 @@ def _read_table(url: str) -> pd.DataFrame: return pd.read_csv(io.StringIO(text), dtype=str) -def _base_year(df: pd.DataFrame) -> int: - """Map the table's tax year to a PolicyEngine year. +def _check_tax_year(df: pd.DataFrame) -> int: + """The table's PolicyEngine year, which must be ``_TAX_YEAR``. PolicyEngine UK's year N is tax year N to N+1 (a year-N simulation reads the parameter values in force from 6 April N), so "2024 to 2025" is 2024. """ - years = df["tax_year"].unique() - match = re.fullmatch(r"(\d{4}) to (\d{4})", years[0]) if len(years) == 1 else None - if match is None or int(match[2]) != int(match[1]) + 1: - raise ValueError(f"Expected one tax year in Tables 6.1/6.2, got {years}") - return int(match[1]) + years = list(df["tax_year"].unique()) + expected = f"{_TAX_YEAR} to {_TAX_YEAR + 1}" + if years != [expected]: + raise ValueError( + f"HMRC Tables 6.1/6.2 cover {years}, but {FALLBACK_CSV.name} and " + f"_TAX_YEAR are {expected!r}: refresh the sources.yaml URL, the " + "committed copy and _TAX_YEAR together." + ) + return _TAX_YEAR + + +def _nics_rate_factors(rows: pd.DataFrame, nics_class: str, base_year: int) -> dict: + """Class 1 rate in each year relative to the table's year, by year. + + Weighted by HMRC's split of the class's relief between the main and + additional rates (rows suppressed as [z] weigh nothing). + """ + from policyengine_uk import CountryTaxBenefitSystem + + parameters = CountryTaxBenefitSystem().parameters + weights = {} + for rate_row in ("Main Rate", "Additional Rate"): + values = rows.loc[ + (rows["nics_relief_class"] == nics_class) & (rows["tax_rate"] == rate_row), + "value_of_relief", + ].map(to_float) + if values.sum() > 0: + path = _NICS_RATE_PARAMETERS[(nics_class, rate_row)] + weights[path] = values.sum() + if not weights: + raise ValueError(f"HMRC Table 6.2: no rate split for {nics_class!r}") + + def rate(path, year): + return parameters.get_child( + f"gov.hmrc.national_insurance.class_1.rates.{path}" + )(str(year)) + + return { + year: sum(w * rate(p, year) / rate(p, base_year) for p, w in weights.items()) + / sum(weights.values()) + for year in range(base_year, _LAST_YEAR + 1) + } def _relief(rows: pd.DataFrame, column: str, label: str) -> float: @@ -104,24 +158,32 @@ def _relief(rows: pd.DataFrame, column: str, label: str) -> float: def _relief_targets(df: pd.DataFrame, reference_url: str) -> list[Target]: - base_year = _base_year(df) + base_year = _check_tax_year(df) ss = df[ (df["contribution_type"] == "Salary sacrificed contributions") & (df["sector_scheme"] == "Total") & (df["scheme_type"] == "Total") ] income_tax = ss[ss["income_tax_nics"] == "Income Tax"] - nics = ss[(ss["income_tax_nics"] == "NICs") & (ss["tax_rate"] == "Total")] + nics = ss[ss["income_tax_nics"] == "NICs"] + nics_total = nics[nics["tax_rate"] == "Total"] + unchanged = {y: 1.0 for y in range(base_year, _LAST_YEAR + 1)} specs = [ ( f"hmrc/salary_sacrifice_it_relief_{suffix}", "income_tax", _relief(income_tax, "tax_rate", rate), + unchanged, ) for rate, suffix in _IT_RATES.items() ] + [ - (name, variable, _relief(nics, "nics_relief_class", nics_class)) + ( + name, + variable, + _relief(nics_total, "nics_relief_class", nics_class), + _nics_rate_factors(nics, nics_class, base_year), + ) for nics_class, (name, variable) in _NICS_CLASSES.items() ] return [ @@ -131,12 +193,12 @@ def _relief_targets(df: pd.DataFrame, reference_url: str) -> list[Target]: source="hmrc", unit=Unit.GBP, values={ - y: base * _GROWTH ** (y - base_year) - for y in range(base_year, _LAST_YEAR + 1) + y: base * _GROWTH ** (y - base_year) * factor + for y, factor in factors.items() }, reference_url=reference_url, ) - for name, variable, base in specs + for name, variable, base, factors in specs ] diff --git a/policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py b/policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py index 801e86a9f..2e6aafdac 100644 --- a/policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py +++ b/policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py @@ -4,8 +4,9 @@ release, and the source module logged the error and returned without the relief targets, so builds calibrated without them. These tests pin: the committed table parses to every relief target, a failed download uses it, a -changed table raises, and the rate-band columns split relief the way HMRC -does (across the bands a contribution straddles). +changed table raises, the rate-band columns measure relief the way HMRC does +(tax on pay, relieved at each rate a contribution straddles), and on a built +dataset every relief target produces a loss-matrix column. """ from unittest.mock import patch @@ -15,37 +16,57 @@ import pytest import requests +from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE +from policyengine_uk_data.targets.build_loss_matrix import ( + _compute_column, + _resolve_value, + _SimContext, +) from policyengine_uk_data.targets.compute.income import ( compute_ss_it_relief, - split_relief_by_band, + tax_by_band, ) from policyengine_uk_data.targets.schema import Target, Unit from policyengine_uk_data.targets.sources import hmrc_salary_sacrifice as hmrc_ss # Tables 6.1 and 6.2, tax year 2024-25 (£m), from the committed CSV. RELIEF_2024_25 = { - "hmrc/salary_sacrifice_it_relief_total": 8_800, "hmrc/salary_sacrifice_it_relief_basic_rate": 1_600, "hmrc/salary_sacrifice_it_relief_higher_rate": 5_500, "hmrc/salary_sacrifice_it_relief_additional_rate": 1_800, "hmrc/salary_sacrifice_employee_nics_relief": 1_000, "hmrc/salary_sacrifice_employer_nics_relief": 3_400, } +# Class 1 secondary (employer) rate: 13.8% in 2024-25, 15% from 6 April 2025 +# (National Insurance Contributions (Secondary Class 1 Contributions) Act +# 2025). The primary rates (8% main, 2% additional) did not change. +EMPLOYER_RATE_RISE = 0.15 / 0.138 def _committed_table() -> pd.DataFrame: return pd.read_csv(hmrc_ss.FALLBACK_CSV, dtype=str, encoding="utf-8-sig") +def _offline(*args, **kwargs): + raise requests.ConnectionError("offline") + + +def _offline_targets() -> list[Target]: + with patch.object(hmrc_ss.requests, "get", side_effect=_offline): + return hmrc_ss.get_targets() + + def test_committed_table_gives_every_relief_target(): targets = {t.name: t for t in hmrc_ss._relief_targets(_committed_table(), "x")} assert set(targets) == set(RELIEF_2024_25) for name, millions in RELIEF_2024_25.items(): values = targets[name].values + rate_rise = EMPLOYER_RATE_RISE if "employer" in name else 1 # Tax year 2024-25 is PolicyEngine year 2024. assert min(values) == 2024 assert values[2024] == pytest.approx(millions * 1e6) - assert values[2025] == pytest.approx(millions * 1e6 * 1.03) + assert values[2025] == pytest.approx(millions * 1e6 * 1.03 * rate_rise) + assert values[2026] == pytest.approx(millions * 1e6 * 1.03**2 * rate_rise) assert targets["hmrc/salary_sacrifice_employee_nics_relief"].variable == ( "ni_employee" ) @@ -54,6 +75,12 @@ def test_committed_table_gives_every_relief_target(): ) +def test_fallback_file_matches_the_tax_year_it_is_checked_against(): + assert list(_committed_table()["tax_year"].unique()) == [ + f"{hmrc_ss._TAX_YEAR} to {hmrc_ss._TAX_YEAR + 1}" + ] + + def _gone(*args, **kwargs): response = requests.Response() response.status_code = 410 @@ -61,10 +88,6 @@ def _gone(*args, **kwargs): return response -def _offline(*args, **kwargs): - raise requests.ConnectionError("offline") - - @pytest.mark.parametrize("get", [_gone, _offline], ids=["410", "offline"]) def test_failed_download_uses_committed_table(get): with patch.object(hmrc_ss.requests, "get", side_effect=get): @@ -78,52 +101,71 @@ def test_failed_download_uses_committed_table(get): [ lambda df: df[df["tax_rate"] != "Higher Rate"], lambda df: df[df["nics_relief_class"] != "Class 1 Secondary (employer)"], + lambda df: df[ + ~( + (df["nics_relief_class"] == "Class 1 Secondary (employer)") + & (df["tax_rate"] == "Main Rate") + ) + ], lambda df: pd.concat( [df.iloc[:-1], df.iloc[-1:].assign(tax_year="2025 to 2026")] ), - lambda df: df.assign(tax_year="2024 to 2026"), + lambda df: df.assign(tax_year="2025 to 2026"), lambda df: df.drop(columns="sector_scheme"), ], - ids=["no-higher-rate", "no-employer-nics", "two-years", "bad-year", "no-column"], + ids=[ + "no-higher-rate", + "no-employer-nics", + "no-employer-rate-split", + "two-years", + "next-release", + "no-column", + ], ) def test_changed_table_raises(change): with pytest.raises((ValueError, KeyError)): hmrc_ss._relief_targets(change(_committed_table()), "x") -BANDS = ("basic", "higher", "additional") +def _scales(year): + from policyengine_uk import CountryTaxBenefitSystem - -def test_split_straddling_relief(): - # £4k sacrificed from £52k pay (2025-26 rUK): earned taxable income falls - # from £39,430 to £35,430, so £1,730 leaves the higher band and £2,270 - # the basic band. - split = split_relief_by_band( - np.array([1_146.0]), - {"basic": [7_540.0], "higher": [692.0], "additional": [0.0]}, - {"basic": [7_086.0], "higher": [0.0], "additional": [0.0]}, - ) - assert split["basic"] == pytest.approx([454.0]) - assert split["higher"] == pytest.approx([692.0]) - assert split["additional"] == pytest.approx([0.0]) + rates = CountryTaxBenefitSystem().parameters(year).gov.hmrc.income_tax.rates + return {"uk": rates.uk, "scotland": rates.scotland.rates} -def test_split_invariants_on_random_inputs(): - """Bands add up to the relief, keep its sign, and default to basic rate.""" +@pytest.mark.parametrize("year", ["2023", "2024", "2025"]) +@pytest.mark.parametrize("schedule", ["uk", "scotland"]) +def test_tax_by_band_adds_up_to_the_schedule_and_rises_with_income(year, schedule): + """Differential against PolicyEngine's own scale, plus monotonicity.""" + scale = _scales(year)[schedule] rng = np.random.default_rng(0) - n = 5_000 - relief = rng.normal(500, 1_000, n) * rng.integers(0, 2, n) - base = {b: rng.uniform(0, 20_000, n) * rng.integers(0, 2, n) for b in BANDS} - # A smaller sacrifice never raises band tax; some people see no change. - cf = {b: base[b] + rng.uniform(0, 3_000, n) * rng.integers(0, 2, n) for b in BANDS} - split = split_relief_by_band(relief, cf, base) + income = np.concatenate( + [rng.uniform(-1_000, 200_000, 5_000), np.asarray(scale.thresholds[1:])] + ) + bands = tax_by_band(income, scale.thresholds, scale.rates) + np.testing.assert_allclose(sum(bands.values()), scale.calc(income), atol=1e-6) + more = tax_by_band( + income + rng.uniform(0, 20_000, income.size), scale.thresholds, scale.rates + ) + for band in bands: + assert np.all(more[band] >= bands[band] - 1e-9) - np.testing.assert_allclose(sum(split.values()), relief, rtol=1e-12, atol=1e-9) - for band in BANDS: - assert np.all(split[band] * np.sign(relief) >= -1e-9) - no_drop = sum(cf[b] - base[b] for b in BANDS) == 0 - assert no_drop.any() - np.testing.assert_array_equal(split["basic"][no_drop], relief[no_drop]) + +def test_tax_by_band_groups_scottish_rates_into_hmrc_categories(): + # 2025-26 Scottish taxable-income bands: intermediate (21%) up to £31,092, + # higher (42%) to £62,430, advanced (45%) to £112,570, then top (48%). + scale = _scales("2025")["scotland"] + bands = tax_by_band( + np.array([20_000, 50_000, 100_000, 150_000]), scale.thresholds, scale.rates + ) + assert bands["higher"][0] == 0 and bands["additional"][0] == 0 + assert bands["higher"][1] > 0 and bands["additional"][1] == 0 + # The advanced rate counts as higher rate. + assert bands["higher"][2] == pytest.approx( + 0.42 * (62_430 - 31_092) + 0.45 * (100_000 - 62_430) + ) + assert bands["additional"][3] == pytest.approx(0.48 * (150_000 - 112_570)) class _Ctx: @@ -131,7 +173,7 @@ class _Ctx: time_period = 2025 - def __init__(self, base_pay, sacrifice): + def __init__(self, base_pay, sacrifice, region="LONDON"): from policyengine_uk import Simulation def sim(pay): @@ -141,7 +183,7 @@ def sim(pay): "a": {"age": {2025: 40}, "employment_income": {2025: pay}} }, "benunits": {"b": {"members": ["a"]}}, - "households": {"h": {"members": ["a"]}}, + "households": {"h": {"members": ["a"], "region": {2025: region}}}, } ) @@ -155,9 +197,9 @@ def household_from_person(values): return np.asarray(values) -def _relief(ctx, suffix): +def _relief(ctx, band): target = Target( - name=f"hmrc/salary_sacrifice_it_relief_{suffix}", + name=f"hmrc/salary_sacrifice_it_relief_{band}_rate", variable="income_tax", source="hmrc", unit=Unit.GBP, @@ -166,13 +208,55 @@ def _relief(ctx, suffix): return float(compute_ss_it_relief(target, ctx)[0]) -def test_compute_splits_relief_for_a_basic_rate_taxpayer_near_the_threshold(): - """Pay of £48k after a £4k sacrifice: 40% relief only above £50,270.""" - ctx = _Ctx(base_pay=48_000, sacrifice=4_000) - by_band = { - s: _relief(ctx, s) for s in ("basic_rate", "higher_rate", "additional_rate") - } - assert _relief(ctx, "total") == pytest.approx(1_146) - assert by_band == pytest.approx( - {"basic_rate": 454, "higher_rate": 692, "additional_rate": 0} +@pytest.mark.parametrize( + "base_pay, sacrifice, region, expected", + [ + # rUK, £48k after a £4k sacrifice: taxable income £35,430 → £39,430, + # 40% only above £37,700. + (48_000, 4_000, "LONDON", {"basic": 454, "higher": 692, "additional": 0}), + # Scotland, £42k after £4k: taxable £29,430 → £33,430, 21% to £31,092 + # (basic category), 42% above. + ( + 42_000, + 4_000, + "SCOTLAND", + {"basic": 349.02, "higher": 981.96, "additional": 0}, + ), + # rUK, £100k after £10k: £5k of personal allowance is withdrawn, so + # relief is 60% of the sacrifice, all in the higher band. + (100_000, 10_000, "LONDON", {"basic": 0, "higher": 6_000, "additional": 0}), + ], + ids=["ruk-straddle", "scotland-straddle", "allowance-taper"], +) +def test_compute_relieves_each_slice_at_its_rate(base_pay, sacrifice, region, expected): + ctx = _Ctx(base_pay, sacrifice, region) + by_band = {band: _relief(ctx, band) for band in expected} + assert by_band == pytest.approx(expected, abs=0.01) + # With pay as the only income, the bands add up to the fall in income tax. + income_tax = [ + float(s.calculate("income_tax", 2025)[0]) + for s in (ctx.counterfactual_sim, ctx.sim) + ] + assert sum(by_band.values()) == pytest.approx( + income_tax[0] - income_tax[1], abs=0.01 ) + + +def test_relief_targets_produce_loss_matrix_columns(enhanced_frs): + """create_target_matrix skips a target, with only a warning, when + _resolve_value or _compute_column raises or returns None. On the built + dataset neither may happen for these targets.""" + from policyengine_uk import Microsimulation + + year = CURRENT_FRS_RELEASE.calibration_year + sim = Microsimulation(dataset=enhanced_frs) + sim.default_calculation_period = year + ctx = _SimContext(sim, year, enhanced_frs, None) + weights = np.asarray(sim.calculate("household_weight", year)) + targets = [t for t in _offline_targets() if t.name in RELIEF_2024_25] + assert len(targets) == len(RELIEF_2024_25) + for target in targets: + assert _resolve_value(target, year) is not None, target.name + column = np.asarray(_compute_column(target, ctx, year), dtype=float) + assert np.isfinite(column).all(), target.name + assert column @ weights > 0, target.name diff --git a/policyengine_uk_data/tests/test_target_registry.py b/policyengine_uk_data/tests/test_target_registry.py index 59d0b09f5..c2a1c1c37 100644 --- a/policyengine_uk_data/tests/test_target_registry.py +++ b/policyengine_uk_data/tests/test_target_registry.py @@ -86,7 +86,6 @@ def test_hmrc_salary_sacrifice_relief_targets(): on the old CSV dropped them from every build without failing it).""" targets = {t.name: t for t in get_all_targets(year=2025)} for name in ( - "hmrc/salary_sacrifice_it_relief_total", "hmrc/salary_sacrifice_it_relief_basic_rate", "hmrc/salary_sacrifice_it_relief_higher_rate", "hmrc/salary_sacrifice_it_relief_additional_rate", @@ -94,6 +93,8 @@ def test_hmrc_salary_sacrifice_relief_targets(): "hmrc/salary_sacrifice_employer_nics_relief", ): assert name in targets, f"{name} missing from the target registry" + # The bands, not HMRC's separately rounded total, are the targets. + assert "hmrc/salary_sacrifice_it_relief_total" not in targets # One NICs relief target per class: the OBR-labelled copies of the # 2023-24 figures are gone. assert "obr/salary_sacrifice_employee_ni_relief" not in targets From 585402d9d6784226faf862227e288908332ef819 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 4 Oct 2026 00:49:29 -0400 Subject: [PATCH 3/5] Update the changelog fragment for the review changes Co-Authored-By: Claude Opus 5.5 --- changelog.d/hmrc-salary-sacrifice-relief-targets.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/hmrc-salary-sacrifice-relief-targets.fixed.md b/changelog.d/hmrc-salary-sacrifice-relief-targets.fixed.md index 46b3fb23a..f7fb4a92f 100644 --- a/changelog.d/hmrc-salary-sacrifice-relief-targets.fixed.md +++ b/changelog.d/hmrc-salary-sacrifice-relief-targets.fixed.md @@ -1 +1 @@ -Restore the HMRC salary sacrifice income tax and NICs relief calibration targets, lost to a withdrawn (410 Gone) HMRC CSV since the July 2026 private pension statistics release. They now use the 2024-25 Tables 6.1 and 6.2, mapped to year 2024, with a committed copy used when the download fails; a changed table fails the build. The rate-band targets split each person's relief across the bands it is given in (they had been assigned by adjusted net income against taxable-income thresholds), and the OBR-labelled NICs relief targets, which repeated the 2023-24 HMRC figures, are removed. +Restore the HMRC salary sacrifice income tax and NICs relief calibration targets. They had been lost since the July 2026 private pension statistics release, when the old HMRC CSV was withdrawn (410 Gone). The targets now come from the 2024-25 Tables 6.1 and 6.2, mapped to year 2024. A committed copy is used when the download fails, and a table for another tax year or without the expected rows fails the build. Income tax relief is the rise in tax on pay under the person's rUK or Scottish rates, relieved at each rate the sacrifice straddles; it had been assigned whole to one band by comparing adjusted net income with taxable-income thresholds. The NICs targets follow the Class 1 rates, including the employer rate rise to 15% from April 2025. The OBR-labelled NICs relief targets, which repeated HMRC's 2023-24 figures, and the separately rounded income tax relief total are removed. From b22417123f45824a8cf4b72ab6fa95ab2488440a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 4 Oct 2026 01:58:03 -0400 Subject: [PATCH 4/5] Address review r2 nits: blank cells raise, pin relief to pay, unpin a PE threshold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _relief rejects a blank (NaN) cell instead of returning a NaN target. - Test that dividends pushed into a higher band by the sacrifice do not count as relief (a revert to the change in income_tax passed every test before). - The Scottish category test reads the thresholds from PolicyEngine's schedule instead of stating its top-rate threshold (£112,570) as law (policyengine-uk#2130: the statutory threshold is £125,140). Co-Authored-By: Claude Opus 5.5 --- .../targets/sources/hmrc_salary_sacrifice.py | 2 +- .../test_hmrc_salary_sacrifice_targets.py | 48 +++++++++++++++---- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/policyengine_uk_data/targets/sources/hmrc_salary_sacrifice.py b/policyengine_uk_data/targets/sources/hmrc_salary_sacrifice.py index f90754fe0..fd21f15c6 100644 --- a/policyengine_uk_data/targets/sources/hmrc_salary_sacrifice.py +++ b/policyengine_uk_data/targets/sources/hmrc_salary_sacrifice.py @@ -149,7 +149,7 @@ def rate(path, year): def _relief(rows: pd.DataFrame, column: str, label: str) -> float: """The single positive £m value in ``rows`` whose ``column`` is ``label``.""" values = rows.loc[rows[column] == label, "value_of_relief"].map(to_float) - if len(values) != 1 or values.iloc[0] <= 0: + if len(values) != 1 or not values.iloc[0] > 0: # also rejects a blank cell raise ValueError( f"HMRC salary sacrifice relief: expected one positive value for " f"{label!r}, got {values.tolist()}" diff --git a/policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py b/policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py index 2e6aafdac..d70f9384f 100644 --- a/policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py +++ b/policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py @@ -112,6 +112,9 @@ def test_failed_download_uses_committed_table(get): ), lambda df: df.assign(tax_year="2025 to 2026"), lambda df: df.drop(columns="sector_scheme"), + lambda df: df.assign( + value_of_relief=df["value_of_relief"].where(df["tax_rate"] != "Basic Rate") + ), ], ids=[ "no-higher-rate", @@ -120,6 +123,7 @@ def test_failed_download_uses_committed_table(get): "two-years", "next-release", "no-column", + "blank-cell", ], ) def test_changed_table_raises(change): @@ -153,19 +157,25 @@ def test_tax_by_band_adds_up_to_the_schedule_and_rises_with_income(year, schedul def test_tax_by_band_groups_scottish_rates_into_hmrc_categories(): - # 2025-26 Scottish taxable-income bands: intermediate (21%) up to £31,092, - # higher (42%) to £62,430, advanced (45%) to £112,570, then top (48%). + """Starter, basic and intermediate → basic; higher and advanced → higher; + top → additional. Thresholds come from PolicyEngine's 2025 schedule + (policyengine-uk#2130: its top-rate threshold is £112,570 rather than + the statutory £125,140).""" scale = _scales("2025")["scotland"] - bands = tax_by_band( - np.array([20_000, 50_000, 100_000, 150_000]), scale.thresholds, scale.rates + (_, _, _, higher, advanced, top) = scale.thresholds + assert list(scale.rates) == [0.19, 0.20, 0.21, 0.42, 0.45, 0.48] + income = np.array( + [higher - 1, (higher + advanced) / 2, (advanced + top) / 2, top + 10_000] ) + bands = tax_by_band(income, scale.thresholds, scale.rates) assert bands["higher"][0] == 0 and bands["additional"][0] == 0 - assert bands["higher"][1] > 0 and bands["additional"][1] == 0 + assert bands["higher"][1] == pytest.approx(0.42 * (income[1] - higher)) # The advanced rate counts as higher rate. assert bands["higher"][2] == pytest.approx( - 0.42 * (62_430 - 31_092) + 0.45 * (100_000 - 62_430) + 0.42 * (advanced - higher) + 0.45 * (income[2] - advanced) ) - assert bands["additional"][3] == pytest.approx(0.48 * (150_000 - 112_570)) + assert bands["additional"][2] == 0 + assert bands["additional"][3] == pytest.approx(0.48 * 10_000) class _Ctx: @@ -173,14 +183,18 @@ class _Ctx: time_period = 2025 - def __init__(self, base_pay, sacrifice, region="LONDON"): + def __init__(self, base_pay, sacrifice, region="LONDON", dividends=0): from policyengine_uk import Simulation def sim(pay): return Simulation( situation={ "people": { - "a": {"age": {2025: 40}, "employment_income": {2025: pay}} + "a": { + "age": {2025: 40}, + "employment_income": {2025: pay}, + "dividend_income": {2025: dividends}, + } }, "benunits": {"b": {"members": ["a"]}}, "households": {"h": {"members": ["a"], "region": {2025: region}}}, @@ -260,3 +274,19 @@ def test_relief_targets_produce_loss_matrix_columns(enhanced_frs): column = np.asarray(_compute_column(target, ctx, year), dtype=float) assert np.isfinite(column).all(), target.name assert column @ weights > 0, target.name + + +def test_relief_is_tax_on_pay_not_on_other_income(): + """HMRC applies income tax rates to pay. Paying the sacrifice as salary + also pushes £10k of dividends from the basic into the higher dividend + band; that extra dividend tax is not salary sacrifice relief.""" + ctx = _Ctx(48_000, 4_000, dividends=10_000) + by_band = {band: _relief(ctx, band) for band in ("basic", "higher", "additional")} + assert by_band == pytest.approx( + {"basic": 454, "higher": 692, "additional": 0}, abs=0.01 + ) + income_tax = [ + float(s.calculate("income_tax", 2025)[0]) + for s in (ctx.counterfactual_sim, ctx.sim) + ] + assert income_tax[0] - income_tax[1] > sum(by_band.values()) + 100 From 0863a6aa540be49dac65cfd73c17ebe82b17eecd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 4 Oct 2026 03:02:09 -0400 Subject: [PATCH 5/5] Correct the dividend test's docstring (review r3 T1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only the dividends taxed in the basic rate band move to the higher band, not the whole £10k. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_hmrc_salary_sacrifice_targets.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py b/policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py index d70f9384f..ec01d2801 100644 --- a/policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py +++ b/policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py @@ -278,8 +278,9 @@ def test_relief_targets_produce_loss_matrix_columns(enhanced_frs): def test_relief_is_tax_on_pay_not_on_other_income(): """HMRC applies income tax rates to pay. Paying the sacrifice as salary - also pushes £10k of dividends from the basic into the higher dividend - band; that extra dividend tax is not salary sacrifice relief.""" + also moves the dividends that were taxed in the basic rate band into the + higher dividend band; that extra dividend tax is not salary sacrifice + relief.""" ctx = _Ctx(48_000, 4_000, dividends=10_000) by_band = {band: _relief(ctx, band) for band in ("basic", "higher", "additional")} assert by_band == pytest.approx(