diff --git a/changelog.d/hmrc-salary-sacrifice-relief-targets.fixed.md b/changelog.d/hmrc-salary-sacrifice-relief-targets.fixed.md new file mode 100644 index 00000000..f7fb4a92 --- /dev/null +++ b/changelog.d/hmrc-salary-sacrifice-relief-targets.fixed.md @@ -0,0 +1 @@ +Restore the HMRC salary sacrifice income tax and NICs relief calibration targets. They had been lost since the July 2026 private pension statistics release, when the old HMRC CSV was withdrawn (410 Gone). The targets now come from the 2024-25 Tables 6.1 and 6.2, mapped to year 2024. A committed copy is used when the download fails, and a table for another tax year or without the expected rows fails the build. Income tax relief is the rise in tax on pay under the person's rUK or Scottish rates, relieved at each rate the sacrifice straddles; it had been assigned whole to one band by comparing adjusted net income with taxable-income thresholds. The NICs targets follow the Class 1 rates, including the employer rate rise to 15% from April 2025. The OBR-labelled NICs relief targets, which repeated HMRC's 2023-24 figures, and the separately rounded income tax relief total are removed. diff --git a/policyengine_uk_data/storage/hmrc_pension_relief_tables_6_1_6_2_2024_25.csv b/policyengine_uk_data/storage/hmrc_pension_relief_tables_6_1_6_2_2024_25.csv new file mode 100644 index 00000000..48b14917 --- /dev/null +++ b/policyengine_uk_data/storage/hmrc_pension_relief_tables_6_1_6_2_2024_25.csv @@ -0,0 +1,83 @@ +"tax_year","income_tax_nics","contribution_type","nics_relief_class","sector_scheme","scheme_type","tax_rate","value_of_relief" +"2024 to 2025","Income Tax","Individual contributions to net pay arrangements","Not applicable","Total","Total","Total","7000" +"2024 to 2025","Income Tax","Individual contributions to relief at source schemes","Not applicable","Total","Total","Total","5900" +"2024 to 2025","Income Tax","Salary sacrificed contributions","Not applicable","Total","Total","Total","8800" +"2024 to 2025","Income Tax","Employer contributions to net pay arrangements plus deficit reduction contributions","Not applicable","Total","Total","Total","24800" +"2024 to 2025","Income Tax","Employer contributions to relief at source schemes","Not applicable","Total","Total","Total","9200" +"2024 to 2025","Income Tax","Individual contributions to net pay arrangements","Not applicable","Public sector occupational scheme","Total","Total","4500" +"2024 to 2025","Income Tax","Individual contributions to relief at source schemes","Not applicable","Public sector occupational scheme","Total","Total","200" +"2024 to 2025","Income Tax","Salary sacrificed contributions","Not applicable","Public sector occupational scheme","Total","Total","600" +"2024 to 2025","Income Tax","Employer contributions to net pay arrangements plus deficit reduction contributions","Not applicable","Public sector occupational scheme","Total","Total","14000" +"2024 to 2025","Income Tax","Employer contributions to relief at source schemes","Not applicable","Public sector occupational scheme","Total","Total","400" +"2024 to 2025","Income Tax","Individual contributions to net pay arrangements","Not applicable","Personal or private sector occupational scheme","Total","Total","2500" +"2024 to 2025","Income Tax","Individual contributions to relief at source schemes","Not applicable","Personal or private sector occupational scheme","Total","Total","5700" +"2024 to 2025","Income Tax","Salary sacrificed contributions","Not applicable","Personal or private sector occupational scheme","Total","Total","8300" +"2024 to 2025","Income Tax","Employer contributions to net pay arrangements plus deficit reduction contributions","Not applicable","Personal or private sector occupational scheme","Total","Total","10800" +"2024 to 2025","Income Tax","Employer contributions to relief at source schemes","Not applicable","Personal or private sector occupational scheme","Total","Total","8800" +"2024 to 2025","Income Tax","Individual contributions to net pay arrangements","Not applicable","Total","Defined benefit","Total","5200" +"2024 to 2025","Income Tax","Individual contributions to relief at source schemes","Not applicable","Total","Defined benefit","Total","0" +"2024 to 2025","Income Tax","Salary sacrificed contributions","Not applicable","Total","Defined benefit","Total","1500" +"2024 to 2025","Income Tax","Employer contributions to net pay arrangements plus deficit reduction contributions","Not applicable","Total","Defined benefit","Total","16600" +"2024 to 2025","Income Tax","Employer contributions to relief at source schemes","Not applicable","Total","Defined benefit","Total","0" +"2024 to 2025","Income Tax","Individual contributions to net pay arrangements","Not applicable","Total","Defined contribution","Total","1800" +"2024 to 2025","Income Tax","Individual contributions to relief at source schemes","Not applicable","Total","Defined contribution","Total","5900" +"2024 to 2025","Income Tax","Salary sacrificed contributions","Not applicable","Total","Defined contribution","Total","7400" +"2024 to 2025","Income Tax","Employer contributions to net pay arrangements plus deficit reduction contributions","Not applicable","Total","Defined contribution","Total","8200" +"2024 to 2025","Income Tax","Employer contributions to relief at source schemes","Not applicable","Total","Defined contribution","Total","9200" +"2024 to 2025","Income Tax","Individual contributions to net pay arrangements","Not applicable","Total","Total","Basic Rate","2400" +"2024 to 2025","Income Tax","Individual contributions to relief at source schemes","Not applicable","Total","Total","Basic Rate","3200" +"2024 to 2025","Income Tax","Salary sacrificed contributions","Not applicable","Total","Total","Basic Rate","1600" +"2024 to 2025","Income Tax","Employer contributions to net pay arrangements plus deficit reduction contributions","Not applicable","Total","Total","Basic Rate","6700" +"2024 to 2025","Income Tax","Employer contributions to relief at source schemes","Not applicable","Total","Total","Basic Rate","2200" +"2024 to 2025","Income Tax","Individual contributions to net pay arrangements","Not applicable","Total","Total","Higher Rate","3800" +"2024 to 2025","Income Tax","Individual contributions to relief at source schemes","Not applicable","Total","Total","Higher Rate","1600" +"2024 to 2025","Income Tax","Salary sacrificed contributions","Not applicable","Total","Total","Higher Rate","5500" +"2024 to 2025","Income Tax","Employer contributions to net pay arrangements plus deficit reduction contributions","Not applicable","Total","Total","Higher Rate","15600" +"2024 to 2025","Income Tax","Employer contributions to relief at source schemes","Not applicable","Total","Total","Higher Rate","5200" +"2024 to 2025","Income Tax","Individual contributions to net pay arrangements","Not applicable","Total","Total","Additional Rate","900" +"2024 to 2025","Income Tax","Individual contributions to relief at source schemes","Not applicable","Total","Total","Additional Rate","1100" +"2024 to 2025","Income Tax","Salary sacrificed contributions","Not applicable","Total","Total","Additional Rate","1800" +"2024 to 2025","Income Tax","Employer contributions to net pay arrangements plus deficit reduction contributions","Not applicable","Total","Total","Additional Rate","2500" +"2024 to 2025","Income Tax","Employer contributions to relief at source schemes","Not applicable","Total","Total","Additional Rate","1700" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Primary (employee)","Total","Total","Total","3600" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Primary (employee)","Total","Total","Total","1200" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Primary (employee)","Total","Total","Total","1000" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Secondary (employer)","Total","Total","Total","10600" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Secondary (employer)","Total","Total","Total","3800" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Secondary (employer)","Total","Total","Total","3400" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Primary (employee)","Public sector occupational scheme","Total","Total","2300" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Primary (employee)","Public sector occupational scheme","Total","Total","900" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Primary (employee)","Public sector occupational scheme","Total","Total","100" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Secondary (employer)","Public sector occupational scheme","Total","Total","6300" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Secondary (employer)","Public sector occupational scheme","Total","Total","200" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Secondary (employer)","Public sector occupational scheme","Total","Total","200" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Primary (employee)","Personal or private sector occupational scheme","Total","Total","1300" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Primary (employee)","Personal or private sector occupational scheme","Total","Total","300" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Primary (employee)","Personal or private sector occupational scheme","Total","Total","900" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Secondary (employer)","Personal or private sector occupational scheme","Total","Total","4300" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Secondary (employer)","Personal or private sector occupational scheme","Total","Total","3600" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Secondary (employer)","Personal or private sector occupational scheme","Total","Total","3100" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Primary (employee)","Total","Defined benefit","Total","2500" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Primary (employee)","Total","Defined benefit","Total","0" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Primary (employee)","Total","Defined benefit","Total","200" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Secondary (employer)","Total","Defined benefit","Total","7200" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Secondary (employer)","Total","Defined benefit","Total","0" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Secondary (employer)","Total","Defined benefit","Total","600" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Primary (employee)","Total","Defined contribution","Total","1100" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Primary (employee)","Total","Defined contribution","Total","1200" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Primary (employee)","Total","Defined contribution","Total","800" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Secondary (employer)","Total","Defined contribution","Total","3300" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Secondary (employer)","Total","Defined contribution","Total","3800" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Secondary (employer)","Total","Defined contribution","Total","2800" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Primary (employee)","Total","Total","Main Rate","2800" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Primary (employee)","Total","Total","Main Rate","900" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Primary (employee)","Total","Total","Main Rate","600" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Secondary (employer)","Total","Total","Main Rate","10600" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Secondary (employer)","Total","Total","Main Rate","3800" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Secondary (employer)","Total","Total","Main Rate","3400" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Primary (employee)","Total","Total","Additional Rate","800" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Primary (employee)","Total","Total","Additional Rate","300" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Primary (employee)","Total","Total","Additional Rate","300" +"2024 to 2025","NICs","Employer contributions to net pay arrangements and on employer deficit reduction contributions","Class 1 Secondary (employer)","Total","Total","Additional Rate","[z]" +"2024 to 2025","NICs","Employer contributions to relief at source schemes","Class 1 Secondary (employer)","Total","Total","Additional Rate","[z]" +"2024 to 2025","NICs","Salary sacrificed contributions","Class 1 Secondary (employer)","Total","Total","Additional Rate","[z]" diff --git a/policyengine_uk_data/targets/build_loss_matrix.py b/policyengine_uk_data/targets/build_loss_matrix.py index 9552aeea..0c374dbf 100644 --- a/policyengine_uk_data/targets/build_loss_matrix.py +++ b/policyengine_uk_data/targets/build_loss_matrix.py @@ -386,9 +386,7 @@ def _compute_column(target: Target, ctx: _SimContext, year: int) -> np.ndarray | # Salary sacrifice NI relief if name in ( "hmrc/salary_sacrifice_employee_nics_relief", - "obr/salary_sacrifice_employee_ni_relief", "hmrc/salary_sacrifice_employer_nics_relief", - "obr/salary_sacrifice_employer_ni_relief", ): return compute_ss_ni_relief(target, ctx) diff --git a/policyengine_uk_data/targets/compute/income.py b/policyengine_uk_data/targets/compute/income.py index f5be3e2b..19c49ac7 100644 --- a/policyengine_uk_data/targets/compute/income.py +++ b/policyengine_uk_data/targets/compute/income.py @@ -20,34 +20,59 @@ def compute_income_band(target, ctx) -> np.ndarray: return ctx.household_from_person(income_df[variable] * in_band) -def compute_ss_it_relief(target, ctx) -> np.ndarray: - """Compute salary sacrifice IT relief by tax band.""" - it_base = ctx.sim.calculate("income_tax") - it_cf = ctx.counterfactual_sim.calculate("income_tax", ctx.time_period) - it_relief = it_cf - it_base +def tax_by_band(income: np.ndarray, thresholds, rates) -> dict: + """Tax on ``income`` within each of HMRC's rate categories. + + HMRC's Table 6.1 groups relief by marginal rate: basic (with the Scottish + starter and intermediate rates), higher and additional. Brackets taxed + below 30% are basic, brackets at the scale's top rate are additional (the + rUK additional rate, the Scottish top rate) and the rest are higher (the + Scottish higher and advanced rates, which cover the income range of the + rUK higher rate). The categories sum to the scale's tax on ``income``. + """ + income = np.asarray(income, dtype=float) + thresholds = [t for t, r in zip(thresholds, rates) if r is not None] + rates = [r for r in rates if r is not None] + uppers = thresholds[1:] + [np.inf] + bands = {"basic": 0.0, "higher": 0.0, "additional": 0.0} + for lower, upper, rate in zip(thresholds, uppers, rates): + band = ( + "basic" if rate < 0.3 else "additional" if rate == rates[-1] else "higher" + ) + bands[band] = bands[band] + rate * np.clip(income - lower, 0, upper - lower) + return bands - adj_net_income_cf = ctx.counterfactual_sim.calculate( - "adjusted_net_income", ctx.time_period - ) - params = ctx.sim.tax_benefit_system.parameters.gov.hmrc.income_tax.rates.uk - basic_thresh = params[0].threshold(ctx.time_period) - higher_thresh = params[1].threshold(ctx.time_period) - additional_thresh = params[2].threshold(ctx.time_period) +def ss_it_relief_by_band(ctx) -> dict: + """Person-level salary sacrifice income tax relief in each rate category. + + HMRC applies income tax rates to employees' pay (ASHE), so relief is the + rise in tax on earned income when the sacrifice is paid as salary, under + the person's rUK or Scottish rates. Contributions that straddle a band + boundary are relieved partly at each rate, as in HMRC's estimates. + """ + period = ctx.time_period + rates = ctx.sim.tax_benefit_system.parameters(period).gov.hmrc.income_tax.rates + scottish = np.asarray(ctx.sim.calculate("pays_scottish_income_tax", period)) + base = np.asarray(ctx.sim.calculate("earned_taxable_income", period)) + cf = np.asarray(ctx.counterfactual_sim.calculate("earned_taxable_income", period)) + relief = {} + for scale, in_scale in ((rates.uk, ~scottish), (rates.scotland.rates, scottish)): + band_cf = tax_by_band(cf, scale.thresholds, scale.rates) + band_base = tax_by_band(base, scale.thresholds, scale.rates) + for band in band_cf: + relief[band] = relief.get(band, 0) + in_scale * ( + band_cf[band] - band_base[band] + ) + return relief - name = target.name - if "basic" in name: - mask = (adj_net_income_cf > basic_thresh) & (adj_net_income_cf <= higher_thresh) - elif "higher" in name: - mask = (adj_net_income_cf > higher_thresh) & ( - adj_net_income_cf <= additional_thresh - ) - elif "additional" in name: - mask = adj_net_income_cf > additional_thresh - else: - mask = np.ones_like(it_relief, dtype=bool) - return ctx.household_from_person(it_relief * mask) +def compute_ss_it_relief(target, ctx) -> np.ndarray: + """Compute salary sacrifice income tax relief at one rate.""" + band = target.name.removeprefix("hmrc/salary_sacrifice_it_relief_") + return ctx.household_from_person( + ss_it_relief_by_band(ctx)[band.removesuffix("_rate")] + ) def compute_ss_contributions(target, ctx) -> np.ndarray: diff --git a/policyengine_uk_data/targets/sources.yaml b/policyengine_uk_data/targets/sources.yaml index 5d1cd89b..973e3956 100644 --- a/policyengine_uk_data/targets/sources.yaml +++ b/policyengine_uk_data/targets/sources.yaml @@ -11,7 +11,9 @@ hmrc: spi_geography: "https://assets.publishing.service.gov.uk/media/69f1f17cc42061e837e3ac3b/Collated_Tables_3_12_to_3_15a_2324.ods" income_tax_liabilities: "https://www.gov.uk/government/statistics/income-tax-liabilities-statistics-tax-year-2022-to-2023-to-tax-year-2025-to-2026" capital_gains_statistics: "https://www.gov.uk/government/statistics/capital-gains-tax-statistics" - salary_sacrifice_table_6: "https://assets.publishing.service.gov.uk/media/687a294e312ee8a5f0806b6d/Tables_6_1_and_6_2.csv" + # Private pension statistics, July 2026 (tax year 2024-25). Refresh with + # storage/hmrc_pension_relief_tables_6_1_6_2_*.csv (hmrc_salary_sacrifice.py). + salary_sacrifice_table_6: "https://assets.publishing.service.gov.uk/media/6a673e0e5e87122783093901/Tables_6_1_and_6_2.csv" dwp: stat_xplore_api: "https://stat-xplore.dwp.gov.uk/webapi/rest/v1" diff --git a/policyengine_uk_data/targets/sources/hmrc_salary_sacrifice.py b/policyengine_uk_data/targets/sources/hmrc_salary_sacrifice.py index f9865e6f..fd21f15c 100644 --- a/policyengine_uk_data/targets/sources/hmrc_salary_sacrifice.py +++ b/policyengine_uk_data/targets/sources/hmrc_salary_sacrifice.py @@ -1,9 +1,23 @@ """HMRC salary sacrifice income tax and NICs relief targets. -Downloads Table 6.2 CSV from HMRC to get salary sacrifice IT relief -by tax rate band and NICs relief (employee + employer). - -Source: https://assets.publishing.service.gov.uk/media/687a294e312ee8a5f0806b6d/Tables_6_1_and_6_2.csv +Reads HMRC private pension statistics Tables 6.1 and 6.2 (tidy CSV): income +tax relief on salary-sacrificed pension contributions by marginal rate, and +the Class 1 primary (employee) and secondary (employer) NICs relief on them. + +GOV.UK withdraws a release's assets when HMRC publishes the next one: the +July 2025 CSV has returned 410 Gone since the July 2026 release, and the +broad ``except`` this module used to have turned that into calibration +builds that silently lacked these targets. A failed download now falls back +to the copy committed in storage (the same release as ``sources.yaml``), and +a table for another tax year, or without the expected rows, raises instead +of returning fewer targets. + +Targets grow 3% a year from the table's year. The NICs relief targets also +follow the Class 1 rates in PolicyEngine's parameters, so the April 2025 +rise in the employer rate from 13.8% to 15% raises the employer target in +the years the simulation charges 15%. + +Source: https://www.gov.uk/government/statistics/personal-and-stakeholder-pensions-statistics """ import io @@ -15,110 +29,193 @@ from policyengine_uk_data.targets.schema import Target, Unit from policyengine_uk_data.targets.sources._common import ( HEADERS, + STORAGE, load_config, to_float, ) logger = logging.getLogger(__name__) +# Tax year of the Tables 6.1 and 6.2 release in sources.yaml (July 2026, +# 2024-25) and of its committed copy. Refresh the URL, the copy and this +# together: a table for any other year raises. +_TAX_YEAR = 2024 +FALLBACK_CSV = ( + STORAGE + / f"hmrc_pension_relief_tables_6_1_6_2_{_TAX_YEAR}_{(_TAX_YEAR + 1) % 100:02d}.csv" +) + # Uprate 3% pa for wage growth from the base year _GROWTH = 1.03 -_BASE_YEAR = 2024 # 2023-24 tax year → calendar 2024 +_LAST_YEAR = 2031 + +# Table 6.1 rate rows → target name suffix. HMRC counts Scottish starter +# and intermediate rate relief as basic rate. The Total row is not a target: +# it is the sum of the bands, which HMRC rounds separately (2024-25: bands +# £8.9bn, total £8.8bn), so targeting both would ask for two values. +_IT_RATES = { + "Basic Rate": "basic_rate", + "Higher Rate": "higher_rate", + "Additional Rate": "additional_rate", +} +# Table 6.2 NICs classes → (target name, PolicyEngine variable) +_NICS_CLASSES = { + "Class 1 Primary (employee)": ( + "hmrc/salary_sacrifice_employee_nics_relief", + "ni_employee", + ), + "Class 1 Secondary (employer)": ( + "hmrc/salary_sacrifice_employer_nics_relief", + "ni_employer", + ), +} + +# Table 6.2 (class, rate row) → the PolicyEngine Class 1 rate it was relieved at +_NICS_RATE_PARAMETERS = { + ("Class 1 Primary (employee)", "Main Rate"): "employee.main", + ("Class 1 Primary (employee)", "Additional Rate"): "employee.additional", + ("Class 1 Secondary (employer)", "Main Rate"): "employer", +} + +# Total salary sacrifice contributions (SPP Review 2025: £24bn base) +_SS_CONTRIBUTIONS_BASE_YEAR = 2024 + + +def _read_table(url: str) -> pd.DataFrame: + try: + r = requests.get(url, headers=HEADERS, allow_redirects=True, timeout=30) + r.raise_for_status() + text = r.content.decode("utf-8-sig") + except requests.RequestException as e: + logger.warning( + "HMRC Tables 6.1 and 6.2 download failed (%s); using %s", + e, + FALLBACK_CSV.name, + ) + text = FALLBACK_CSV.read_text(encoding="utf-8-sig") + return pd.read_csv(io.StringIO(text), dtype=str) + + +def _check_tax_year(df: pd.DataFrame) -> int: + """The table's PolicyEngine year, which must be ``_TAX_YEAR``. + + PolicyEngine UK's year N is tax year N to N+1 (a year-N simulation reads + the parameter values in force from 6 April N), so "2024 to 2025" is 2024. + """ + years = list(df["tax_year"].unique()) + expected = f"{_TAX_YEAR} to {_TAX_YEAR + 1}" + if years != [expected]: + raise ValueError( + f"HMRC Tables 6.1/6.2 cover {years}, but {FALLBACK_CSV.name} and " + f"_TAX_YEAR are {expected!r}: refresh the sources.yaml URL, the " + "committed copy and _TAX_YEAR together." + ) + return _TAX_YEAR + + +def _nics_rate_factors(rows: pd.DataFrame, nics_class: str, base_year: int) -> dict: + """Class 1 rate in each year relative to the table's year, by year. + + Weighted by HMRC's split of the class's relief between the main and + additional rates (rows suppressed as [z] weigh nothing). + """ + from policyengine_uk import CountryTaxBenefitSystem + + parameters = CountryTaxBenefitSystem().parameters + weights = {} + for rate_row in ("Main Rate", "Additional Rate"): + values = rows.loc[ + (rows["nics_relief_class"] == nics_class) & (rows["tax_rate"] == rate_row), + "value_of_relief", + ].map(to_float) + if values.sum() > 0: + path = _NICS_RATE_PARAMETERS[(nics_class, rate_row)] + weights[path] = values.sum() + if not weights: + raise ValueError(f"HMRC Table 6.2: no rate split for {nics_class!r}") + + def rate(path, year): + return parameters.get_child( + f"gov.hmrc.national_insurance.class_1.rates.{path}" + )(str(year)) + + return { + year: sum(w * rate(p, year) / rate(p, base_year) for p, w in weights.items()) + / sum(weights.values()) + for year in range(base_year, _LAST_YEAR + 1) + } + + +def _relief(rows: pd.DataFrame, column: str, label: str) -> float: + """The single positive £m value in ``rows`` whose ``column`` is ``label``.""" + values = rows.loc[rows[column] == label, "value_of_relief"].map(to_float) + if len(values) != 1 or not values.iloc[0] > 0: # also rejects a blank cell + raise ValueError( + f"HMRC salary sacrifice relief: expected one positive value for " + f"{label!r}, got {values.tolist()}" + ) + return values.iloc[0] * 1e6 + + +def _relief_targets(df: pd.DataFrame, reference_url: str) -> list[Target]: + base_year = _check_tax_year(df) + ss = df[ + (df["contribution_type"] == "Salary sacrificed contributions") + & (df["sector_scheme"] == "Total") + & (df["scheme_type"] == "Total") + ] + income_tax = ss[ss["income_tax_nics"] == "Income Tax"] + nics = ss[ss["income_tax_nics"] == "NICs"] + nics_total = nics[nics["tax_rate"] == "Total"] + unchanged = {y: 1.0 for y in range(base_year, _LAST_YEAR + 1)} + + specs = [ + ( + f"hmrc/salary_sacrifice_it_relief_{suffix}", + "income_tax", + _relief(income_tax, "tax_rate", rate), + unchanged, + ) + for rate, suffix in _IT_RATES.items() + ] + [ + ( + name, + variable, + _relief(nics_total, "nics_relief_class", nics_class), + _nics_rate_factors(nics, nics_class, base_year), + ) + for nics_class, (name, variable) in _NICS_CLASSES.items() + ] + return [ + Target( + name=name, + variable=variable, + source="hmrc", + unit=Unit.GBP, + values={ + y: base * _GROWTH ** (y - base_year) * factor + for y, factor in factors.items() + }, + reference_url=reference_url, + ) + for name, variable, base, factors in specs + ] def get_targets() -> list[Target]: - config = load_config() - ref = config["hmrc"]["salary_sacrifice_table_6"] - targets = [] + ref = load_config()["hmrc"]["salary_sacrifice_table_6"] + targets = _relief_targets(_read_table(ref), ref) - try: - r = requests.get(ref, headers=HEADERS, allow_redirects=True, timeout=30) - r.raise_for_status() - df = pd.read_csv(io.StringIO(r.content.decode("utf-8-sig"))) - - ss = df[df["contribution_type"] == "Salary sacrificed contributions"] - - # IT relief by tax band - ss_it = ss[ - (ss["income_tax_nics"] == "Income Tax") - & (ss["sector_scheme"] == "Total") - & (ss["scheme_type"] == "Total") - ] - for _, row in ss_it.iterrows(): - rate = row["tax_rate"] - val = to_float(row["value_of_relief"]) - if val <= 0: - continue - rate_key = rate.lower().replace(" ", "_") - base = val * 1e6 - targets.append( - Target( - name=f"hmrc/salary_sacrifice_it_relief_{rate_key}", - variable="income_tax", - source="hmrc", - unit=Unit.GBP, - values={ - y: base * _GROWTH ** max(0, y - _BASE_YEAR) - for y in range(_BASE_YEAR, 2032) - }, - reference_url=ref, - ) - ) - - # NICs relief (employee + employer) - ss_nics = ss[ - (ss["income_tax_nics"] == "NICs") - & (ss["sector_scheme"] == "Total") - & (ss["scheme_type"] == "Total") - ] - for _, row in ss_nics.iterrows(): - nics_class = row["nics_relief_class"] - val = to_float(row["value_of_relief"]) - if val <= 0: - continue - if "employee" in str(nics_class).lower(): - name = "hmrc/salary_sacrifice_employee_nics_relief" - variable = "ni_employee" - elif "employer" in str(nics_class).lower(): - name = "hmrc/salary_sacrifice_employer_nics_relief" - variable = "ni_employer" - else: - continue - - # Only take the first (Total scheme) row for each class - existing = {t.name for t in targets} - if name in existing: - continue - - base = val * 1e6 - targets.append( - Target( - name=name, - variable=variable, - source="hmrc", - unit=Unit.GBP, - values={ - y: base * _GROWTH ** max(0, y - _BASE_YEAR) - for y in range(_BASE_YEAR, 2032) - }, - reference_url=ref, - ) - ) - - except Exception as e: - logger.error("Failed to download/parse HMRC salary sacrifice CSV: %s", e) - - # Total salary sacrifice contributions (SPP Review 2025: £24bn base) - _SS_CONTRIBUTIONS = { - y: 24e9 * _GROWTH ** max(0, y - _BASE_YEAR) for y in range(_BASE_YEAR, 2030) - } targets.append( Target( name="hmrc/salary_sacrifice_contributions", variable="pension_contributions_via_salary_sacrifice", source="hmrc", unit=Unit.GBP, - values=_SS_CONTRIBUTIONS, + values={ + y: 24e9 * _GROWTH ** (y - _SS_CONTRIBUTIONS_BASE_YEAR) + for y in range(_SS_CONTRIBUTIONS_BASE_YEAR, 2030) + }, reference_url=( "https://assets.publishing.service.gov.uk/media/" "67ce0e7c08e764d17a5d3c21/2025_SPP_Review.pdf" diff --git a/policyengine_uk_data/targets/sources/obr.py b/policyengine_uk_data/targets/sources/obr.py index ad4e3cf1..82909c0c 100644 --- a/policyengine_uk_data/targets/sources/obr.py +++ b/policyengine_uk_data/targets/sources/obr.py @@ -617,9 +617,7 @@ def _parse_tv_licence(wb: openpyxl.Workbook) -> list[Target]: # ISC census: private school students (roughly constant at ~557k) _PRIVATE_SCHOOL = {y: 557_000 for y in range(2018, 2032)} -# SPP Review: salary sacrifice NI relief (uprated 3% pa from 2024 base) -_SS_EMPLOYEE_NI = {y: 1.2e9 * 1.03 ** max(0, y - 2024) for y in range(2024, 2032)} -_SS_EMPLOYER_NI = {y: 2.9e9 * 1.03 ** max(0, y - 2024) for y in range(2024, 2032)} +# Salary sacrifice NICs relief comes from HMRC Table 6.2 (hmrc_salary_sacrifice.py). # Salary sacrifice headcount: 7.7m total (3.3m above £2k, 4.3m below) # OBR para 1.7: SS population grows 0.9% faster than employees (~2.4%/yr) @@ -666,26 +664,6 @@ def get_targets() -> list[Target]: reference_url="https://www.isc.co.uk/research/annual-census/", ) ) - targets.append( - Target( - name="obr/salary_sacrifice_employee_ni_relief", - variable="ni_employee", - source="obr", - unit=Unit.GBP, - values=_SS_EMPLOYEE_NI, - reference_url="https://assets.publishing.service.gov.uk/media/67ce0e7c08e764d17a5d3c21/2025_SPP_Review.pdf", - ) - ) - targets.append( - Target( - name="obr/salary_sacrifice_employer_ni_relief", - variable="ni_employer", - source="obr", - unit=Unit.GBP, - values=_SS_EMPLOYER_NI, - reference_url="https://assets.publishing.service.gov.uk/media/67ce0e7c08e764d17a5d3c21/2025_SPP_Review.pdf", - ) - ) # Salary sacrifice headcount targets _SS_REF = ( diff --git a/policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py b/policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py new file mode 100644 index 00000000..ec01d280 --- /dev/null +++ b/policyengine_uk_data/tests/test_hmrc_salary_sacrifice_targets.py @@ -0,0 +1,293 @@ +"""Tests for the HMRC salary sacrifice relief targets (Tables 6.1 and 6.2). + +The July 2025 CSV returned 410 Gone once HMRC published the July 2026 +release, and the source module logged the error and returned without the +relief targets, so builds calibrated without them. These tests pin: the +committed table parses to every relief target, a failed download uses it, a +changed table raises, the rate-band columns measure relief the way HMRC does +(tax on pay, relieved at each rate a contribution straddles), and on a built +dataset every relief target produces a loss-matrix column. +""" + +from unittest.mock import patch + +import numpy as np +import pandas as pd +import pytest +import requests + +from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE +from policyengine_uk_data.targets.build_loss_matrix import ( + _compute_column, + _resolve_value, + _SimContext, +) +from policyengine_uk_data.targets.compute.income import ( + compute_ss_it_relief, + tax_by_band, +) +from policyengine_uk_data.targets.schema import Target, Unit +from policyengine_uk_data.targets.sources import hmrc_salary_sacrifice as hmrc_ss + +# Tables 6.1 and 6.2, tax year 2024-25 (£m), from the committed CSV. +RELIEF_2024_25 = { + "hmrc/salary_sacrifice_it_relief_basic_rate": 1_600, + "hmrc/salary_sacrifice_it_relief_higher_rate": 5_500, + "hmrc/salary_sacrifice_it_relief_additional_rate": 1_800, + "hmrc/salary_sacrifice_employee_nics_relief": 1_000, + "hmrc/salary_sacrifice_employer_nics_relief": 3_400, +} +# Class 1 secondary (employer) rate: 13.8% in 2024-25, 15% from 6 April 2025 +# (National Insurance Contributions (Secondary Class 1 Contributions) Act +# 2025). The primary rates (8% main, 2% additional) did not change. +EMPLOYER_RATE_RISE = 0.15 / 0.138 + + +def _committed_table() -> pd.DataFrame: + return pd.read_csv(hmrc_ss.FALLBACK_CSV, dtype=str, encoding="utf-8-sig") + + +def _offline(*args, **kwargs): + raise requests.ConnectionError("offline") + + +def _offline_targets() -> list[Target]: + with patch.object(hmrc_ss.requests, "get", side_effect=_offline): + return hmrc_ss.get_targets() + + +def test_committed_table_gives_every_relief_target(): + targets = {t.name: t for t in hmrc_ss._relief_targets(_committed_table(), "x")} + assert set(targets) == set(RELIEF_2024_25) + for name, millions in RELIEF_2024_25.items(): + values = targets[name].values + rate_rise = EMPLOYER_RATE_RISE if "employer" in name else 1 + # Tax year 2024-25 is PolicyEngine year 2024. + assert min(values) == 2024 + assert values[2024] == pytest.approx(millions * 1e6) + assert values[2025] == pytest.approx(millions * 1e6 * 1.03 * rate_rise) + assert values[2026] == pytest.approx(millions * 1e6 * 1.03**2 * rate_rise) + assert targets["hmrc/salary_sacrifice_employee_nics_relief"].variable == ( + "ni_employee" + ) + assert targets["hmrc/salary_sacrifice_employer_nics_relief"].variable == ( + "ni_employer" + ) + + +def test_fallback_file_matches_the_tax_year_it_is_checked_against(): + assert list(_committed_table()["tax_year"].unique()) == [ + f"{hmrc_ss._TAX_YEAR} to {hmrc_ss._TAX_YEAR + 1}" + ] + + +def _gone(*args, **kwargs): + response = requests.Response() + response.status_code = 410 + response.url = args[0] + return response + + +@pytest.mark.parametrize("get", [_gone, _offline], ids=["410", "offline"]) +def test_failed_download_uses_committed_table(get): + with patch.object(hmrc_ss.requests, "get", side_effect=get): + names = {t.name for t in hmrc_ss.get_targets()} + assert set(RELIEF_2024_25) <= names + assert "hmrc/salary_sacrifice_contributions" in names + + +@pytest.mark.parametrize( + "change", + [ + lambda df: df[df["tax_rate"] != "Higher Rate"], + lambda df: df[df["nics_relief_class"] != "Class 1 Secondary (employer)"], + lambda df: df[ + ~( + (df["nics_relief_class"] == "Class 1 Secondary (employer)") + & (df["tax_rate"] == "Main Rate") + ) + ], + lambda df: pd.concat( + [df.iloc[:-1], df.iloc[-1:].assign(tax_year="2025 to 2026")] + ), + lambda df: df.assign(tax_year="2025 to 2026"), + lambda df: df.drop(columns="sector_scheme"), + lambda df: df.assign( + value_of_relief=df["value_of_relief"].where(df["tax_rate"] != "Basic Rate") + ), + ], + ids=[ + "no-higher-rate", + "no-employer-nics", + "no-employer-rate-split", + "two-years", + "next-release", + "no-column", + "blank-cell", + ], +) +def test_changed_table_raises(change): + with pytest.raises((ValueError, KeyError)): + hmrc_ss._relief_targets(change(_committed_table()), "x") + + +def _scales(year): + from policyengine_uk import CountryTaxBenefitSystem + + rates = CountryTaxBenefitSystem().parameters(year).gov.hmrc.income_tax.rates + return {"uk": rates.uk, "scotland": rates.scotland.rates} + + +@pytest.mark.parametrize("year", ["2023", "2024", "2025"]) +@pytest.mark.parametrize("schedule", ["uk", "scotland"]) +def test_tax_by_band_adds_up_to_the_schedule_and_rises_with_income(year, schedule): + """Differential against PolicyEngine's own scale, plus monotonicity.""" + scale = _scales(year)[schedule] + rng = np.random.default_rng(0) + income = np.concatenate( + [rng.uniform(-1_000, 200_000, 5_000), np.asarray(scale.thresholds[1:])] + ) + bands = tax_by_band(income, scale.thresholds, scale.rates) + np.testing.assert_allclose(sum(bands.values()), scale.calc(income), atol=1e-6) + more = tax_by_band( + income + rng.uniform(0, 20_000, income.size), scale.thresholds, scale.rates + ) + for band in bands: + assert np.all(more[band] >= bands[band] - 1e-9) + + +def test_tax_by_band_groups_scottish_rates_into_hmrc_categories(): + """Starter, basic and intermediate → basic; higher and advanced → higher; + top → additional. Thresholds come from PolicyEngine's 2025 schedule + (policyengine-uk#2130: its top-rate threshold is £112,570 rather than + the statutory £125,140).""" + scale = _scales("2025")["scotland"] + (_, _, _, higher, advanced, top) = scale.thresholds + assert list(scale.rates) == [0.19, 0.20, 0.21, 0.42, 0.45, 0.48] + income = np.array( + [higher - 1, (higher + advanced) / 2, (advanced + top) / 2, top + 10_000] + ) + bands = tax_by_band(income, scale.thresholds, scale.rates) + assert bands["higher"][0] == 0 and bands["additional"][0] == 0 + assert bands["higher"][1] == pytest.approx(0.42 * (income[1] - higher)) + # The advanced rate counts as higher rate. + assert bands["higher"][2] == pytest.approx( + 0.42 * (advanced - higher) + 0.45 * (income[2] - advanced) + ) + assert bands["additional"][2] == 0 + assert bands["additional"][3] == pytest.approx(0.48 * 10_000) + + +class _Ctx: + """The parts of build_loss_matrix._SimContext the compute function reads.""" + + time_period = 2025 + + def __init__(self, base_pay, sacrifice, region="LONDON", dividends=0): + from policyengine_uk import Simulation + + def sim(pay): + return Simulation( + situation={ + "people": { + "a": { + "age": {2025: 40}, + "employment_income": {2025: pay}, + "dividend_income": {2025: dividends}, + } + }, + "benunits": {"b": {"members": ["a"]}}, + "households": {"h": {"members": ["a"], "region": {2025: region}}}, + } + ) + + # The counterfactual adds the sacrifice back to pay, as + # _SimContext.counterfactual_sim does. + self.sim = sim(base_pay) + self.counterfactual_sim = sim(base_pay + sacrifice) + + @staticmethod + def household_from_person(values): + return np.asarray(values) + + +def _relief(ctx, band): + target = Target( + name=f"hmrc/salary_sacrifice_it_relief_{band}_rate", + variable="income_tax", + source="hmrc", + unit=Unit.GBP, + values={2025: 1.0}, + ) + return float(compute_ss_it_relief(target, ctx)[0]) + + +@pytest.mark.parametrize( + "base_pay, sacrifice, region, expected", + [ + # rUK, £48k after a £4k sacrifice: taxable income £35,430 → £39,430, + # 40% only above £37,700. + (48_000, 4_000, "LONDON", {"basic": 454, "higher": 692, "additional": 0}), + # Scotland, £42k after £4k: taxable £29,430 → £33,430, 21% to £31,092 + # (basic category), 42% above. + ( + 42_000, + 4_000, + "SCOTLAND", + {"basic": 349.02, "higher": 981.96, "additional": 0}, + ), + # rUK, £100k after £10k: £5k of personal allowance is withdrawn, so + # relief is 60% of the sacrifice, all in the higher band. + (100_000, 10_000, "LONDON", {"basic": 0, "higher": 6_000, "additional": 0}), + ], + ids=["ruk-straddle", "scotland-straddle", "allowance-taper"], +) +def test_compute_relieves_each_slice_at_its_rate(base_pay, sacrifice, region, expected): + ctx = _Ctx(base_pay, sacrifice, region) + by_band = {band: _relief(ctx, band) for band in expected} + assert by_band == pytest.approx(expected, abs=0.01) + # With pay as the only income, the bands add up to the fall in income tax. + income_tax = [ + float(s.calculate("income_tax", 2025)[0]) + for s in (ctx.counterfactual_sim, ctx.sim) + ] + assert sum(by_band.values()) == pytest.approx( + income_tax[0] - income_tax[1], abs=0.01 + ) + + +def test_relief_targets_produce_loss_matrix_columns(enhanced_frs): + """create_target_matrix skips a target, with only a warning, when + _resolve_value or _compute_column raises or returns None. On the built + dataset neither may happen for these targets.""" + from policyengine_uk import Microsimulation + + year = CURRENT_FRS_RELEASE.calibration_year + sim = Microsimulation(dataset=enhanced_frs) + sim.default_calculation_period = year + ctx = _SimContext(sim, year, enhanced_frs, None) + weights = np.asarray(sim.calculate("household_weight", year)) + targets = [t for t in _offline_targets() if t.name in RELIEF_2024_25] + assert len(targets) == len(RELIEF_2024_25) + for target in targets: + assert _resolve_value(target, year) is not None, target.name + column = np.asarray(_compute_column(target, ctx, year), dtype=float) + assert np.isfinite(column).all(), target.name + assert column @ weights > 0, target.name + + +def test_relief_is_tax_on_pay_not_on_other_income(): + """HMRC applies income tax rates to pay. Paying the sacrifice as salary + also moves the dividends that were taxed in the basic rate band into the + higher dividend band; that extra dividend tax is not salary sacrifice + relief.""" + ctx = _Ctx(48_000, 4_000, dividends=10_000) + by_band = {band: _relief(ctx, band) for band in ("basic", "higher", "additional")} + assert by_band == pytest.approx( + {"basic": 454, "higher": 692, "additional": 0}, abs=0.01 + ) + income_tax = [ + float(s.calculate("income_tax", 2025)[0]) + for s in (ctx.counterfactual_sim, ctx.sim) + ] + assert income_tax[0] - income_tax[1] > sum(by_band.values()) + 100 diff --git a/policyengine_uk_data/tests/test_target_registry.py b/policyengine_uk_data/tests/test_target_registry.py index 22cf7b2d..c2a1c1c3 100644 --- a/policyengine_uk_data/tests/test_target_registry.py +++ b/policyengine_uk_data/tests/test_target_registry.py @@ -81,6 +81,26 @@ def pe_count(variable): assert _compute_column(target, DummyCtx(), 2025) == [1, 0, 1] +def test_hmrc_salary_sacrifice_relief_targets(): + """HMRC Table 6.1/6.2 relief targets reach the registry (uk-data: 410 Gone + on the old CSV dropped them from every build without failing it).""" + targets = {t.name: t for t in get_all_targets(year=2025)} + for name in ( + "hmrc/salary_sacrifice_it_relief_basic_rate", + "hmrc/salary_sacrifice_it_relief_higher_rate", + "hmrc/salary_sacrifice_it_relief_additional_rate", + "hmrc/salary_sacrifice_employee_nics_relief", + "hmrc/salary_sacrifice_employer_nics_relief", + ): + assert name in targets, f"{name} missing from the target registry" + # The bands, not HMRC's separately rounded total, are the targets. + assert "hmrc/salary_sacrifice_it_relief_total" not in targets + # One NICs relief target per class: the OBR-labelled copies of the + # 2023-24 figures are gone. + assert "obr/salary_sacrifice_employee_ni_relief" not in targets + assert "obr/salary_sacrifice_employer_ni_relief" not in targets + + def test_voa_council_tax_targets(): """VOA council tax band targets should exist.""" targets = get_all_targets(year=2025)