diff --git a/changelog.d/uprate-four-remaining-columns.fixed.md b/changelog.d/uprate-four-remaining-columns.fixed.md new file mode 100644 index 0000000000..d3a71d0683 --- /dev/null +++ b/changelog.d/uprate-four-remaining-columns.fixed.md @@ -0,0 +1 @@ +- Uprate `water_and_sewerage_charges`, `housing_service_charges` and `pension_contributions_via_salary_sacrifice`, which declared an uprating index but were absent from `uprating_indices.yaml` and so stayed flat through 2030. Also uprate eleven columns added since with the same gap, each by the index it declares: `cash_isa`, `stocks_and_shares_isa`, `mortgage_debt`, `consumer_debt`, `rent_paid_as_boarder`, `rent_paid_as_lodger`, `owned_company_capital`, `owned_company_holding_value` and `owned_company_trade_assets` (per-capita GDP) and `self_employment_gross_receipts` and `owned_company_income_share` (per-capita mixed income). Adds a code-health test asserting that every variable declaring `uprating` appears in that file under the matching index, so the gap fails at the point it is introduced. `rail_usage` is excluded because its declared index is a cumulative level rather than a year-on-year rate, and the three energy columns because the data build already prices them (#1868). Updates Ofwat water bill growth to the published 27% for 2025-26 (was the 26% forecast) and 5.4% for 2026-27 (was a 5.5% projection), and documents the sources and vintage of the water bill and population growth series. diff --git a/docs/book/assumptions/water-bills.md b/docs/book/assumptions/water-bills.md index 990944a734..bdc0027549 100644 --- a/docs/book/assumptions/water-bills.md +++ b/docs/book/assumptions/water-bills.md @@ -65,15 +65,19 @@ We store water bills year-on-year growth rates in: ### Growth rates -The model uses these resulting year-on-year growth rates: - -| Year | Growth rate | -|------|-------------| -| 2022 | 5.2% | -| 2023 | 9.2% | -| 2024 | 4.4% | -| 2025 | 6.1% | -| 2026 | 6.1% | -| 2027 | 5.1% | -| 2028 | 3.8% | -| 2029 | 4.3% | +The model uses these year-on-year growth rates. Each year is the charging year starting in April, so 2025 is 2025-26. + +| Year | Growth rate | Source | +|------|-------------|--------| +| 2022 | 5.2% | Ofwat average bills, real bills converted to nominal with CPIH | +| 2023 | 9.2% | As above | +| 2024 | 4.4% | As above | +| 2025 | 27% | [Ofwat statement, 29 January 2026](https://www.ofwat.gov.uk/ofwat-responds-to-water-sector-average-bill-figures/) (later figure; Ofwat's [30 January 2025 statement](https://www.ofwat.gov.uk/average-bills-2025-26-press-statement/) forecast 26%) | +| 2026 | 5.4% | [Water UK, 28 January 2026](https://www.water.org.uk/news-views-publications/news/record-investment-average-household-water-bills-rise-ps33-year-april): national average £639, up £33 | +| 2027 | 4.6% | Projection from `forecast_water_bills.py` | +| 2028 | 4.5% | As above | +| 2029 | 4.5% | As above | + +```{warning} +The 2027-2029 projections cannot be reproduced from the repository. They take the CCW table, which reflects Ofwat's PR24 *draft* determinations rather than the final ones, weight each company by customer numbers, and add a hardcoded CPIH path. The customer weights were in `ofwat_increases.csv`, which is gitignored and was never committed. +``` diff --git a/policyengine_uk/data/uprating_indices.yaml b/policyengine_uk/data/uprating_indices.yaml index 7ba6c9431d..ee511b60fb 100644 --- a/policyengine_uk/data/uprating_indices.yaml +++ b/policyengine_uk/data/uprating_indices.yaml @@ -32,6 +32,7 @@ gov.economic_assumptions.yoy_growth.obr.average_earnings: - employer_pension_contributions - employment_income - employment_income_before_lsr +- pension_contributions_via_salary_sacrifice - personal_pension_contributions - student_loan_repayments gov.economic_assumptions.yoy_growth.obr.consumer_price_index: @@ -120,13 +121,28 @@ gov.economic_assumptions.yoy_growth.obr.per_capita.gdp: - savings - savings_interest_income - sublet_income +- cash_isa +- consumer_debt +- mortgage_debt +- rent_paid_as_boarder +- rent_paid_as_lodger +- stocks_and_shares_isa +- owned_company_capital +- owned_company_holding_value +- owned_company_trade_assets # A formula of benunit_reported_capital; listed so a dataset that stores it # directly is uprated like the input. - uc_reported_capital gov.economic_assumptions.yoy_growth.obr.per_capita.mixed_income: - self_employment_income +- self_employment_gross_receipts - trading_loss +- owned_company_income_share gov.economic_assumptions.yoy_growth.obr.private_pension_index: - private_pension_income +gov.economic_assumptions.yoy_growth.obr.lagged_average_earnings: +- housing_service_charges +gov.economic_assumptions.yoy_growth.ofwat.water_bills: +- water_and_sewerage_charges gov.economic_assumptions.yoy_growth.ons.population: - household_weight diff --git a/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml b/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml index 0a437cc2ad..0be7333749 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml +++ b/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml @@ -655,7 +655,9 @@ obr: 2022-01-01: 0.1019 2023-01-01: 0.0532 2024-01-01: 0.0372 - # Forecast (2025-2030) from OBR EFO March 2026, Table A.1 nominal GDP minus population growth + # Forecast (2025-2030) from OBR EFO March 2026, Table A.1 nominal GDP minus population growth. + # Population growth here is ons.population (March 2025 vintage), not the + # lower growth implied by the March 2026 EFO; see ons.population. 2025-01-01: 0.0438 2026-01-01: 0.0292 2027-01-01: 0.0323 @@ -678,6 +680,7 @@ obr: 2023-01-01: -0.0060 2024-01-01: 0.0273 # Forecast (2025-2030) from OBR EFO March 2026, Table 1.12 minus population growth + # (ons.population, March 2025 vintage; see ons.population). 2025-01-01: 0.0069 2026-01-01: 0.0306 2027-01-01: 0.0371 @@ -700,6 +703,7 @@ obr: 2023-01-01: 0.1084 2024-01-01: 0.0385 # Forecast (2025-2030) from OBR EFO March 2026, Table 1.12 minus population growth + # (ons.population, March 2025 vintage; see ons.population). 2025-01-01: 0.0275 2026-01-01: 0.0358 2027-01-01: 0.0474 @@ -816,6 +820,19 @@ obr: ons: population: description: Population year-on-year growth. + # 2022-2073 is the population growth row of the OBR's long-term economic + # determinants for the March 2025 EFO, published 19 June 2025 (#1305). + # The fiscal year starting in year t is stored at t-01-01, rounded to + # four decimals (2025-26's 0.7230% is 2025-01-01: 0.0072). The OBR notes + # that readings are at the start of the fiscal year or mid-year. To + # 2029-30 the series is the OBR's March 2025 forecast; from 2030-31 it is + # the migration category variant of the ONS 2022-based national + # population projections. The 2021 value predates #1305 and has no + # recorded source. This series is a March 2025 vintage. Calendar-year + # population growth implied by the March 2026 EFO economy tables (real GDP + # in Table 1.1 over real GDP per person in Table 1.5) is lower: 0.37% in + # 2025, against 0.72% here, and 0.30% to 0.36% a year in 2026-2030. The + # ONS has since published 2024-based projections (28 April 2026). values: 2021-01-01: 0.004 2022-01-01: 0.0093 @@ -874,8 +891,10 @@ ons: unit: /1 label: Population growth reference: - - title: ONS Population Projections - href: https://www.ons.gov.uk/ + - title: OBR, "Long-term economic determinants - March 2025 Economic and fiscal outlook" (19 June 2025), population growth row + href: https://obr.uk/supplementary-forecast-information-on-long-term-economic-determinants-and-personal-independence-payment-policy-costing/ + - title: ONS, "National population projections - 2022-based" (migration category variant, used by the OBR from 2030-31) + href: https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/bulletins/nationalpopulationprojections/2022based household_interest_income: description: Household interest income year-on-year growth from ONS National Accounts D.41g. values: @@ -1191,12 +1210,27 @@ ons: ofwat: water_bills: description: Water and sewerage bills year-on-year growth. + # Growth in the average household water and sewerage bill in England and + # Wales. Each value is the charging year starting in April, so 2025-01-01 + # holds 2025-26. + # - 2022-2024: nominal growth in Ofwat's average bill, from the real + # bills and CPIH in policyengine_uk/utils/water/forecast_water_bills.py. + # - 2025 (2025-26): 27%, the increase Ofwat reports for 2025-26 in its + # 29 January 2026 statement. It replaces the 26% forecast in Ofwat's + # 30 January 2025 statement. + # - 2026 (2026-27): 5.4%, the national average from Water UK's 28 January + # 2026 company charges (GBP 639, up GBP 33), confirmed by Ofwat the next day. + # - 2027-2029: projections from forecast_water_bills.py (#1239). They are + # the customer-weighted mean of per-company bills before inflation, from + # the CCW table of Ofwat's PR24 draft determinations, plus a hardcoded + # CPIH path. The weights were in ofwat_increases.csv, which was never + # committed, so these three values cannot be reproduced from the repo. values: 2022-01-01: 0.052 2023-01-01: 0.092 2024-01-01: 0.044 - 2025-01-01: 0.26 - 2026-01-01: 0.055 + 2025-01-01: 0.27 + 2026-01-01: 0.054 2027-01-01: 0.046 2028-01-01: 0.045 2029-01-01: 0.045 @@ -1204,10 +1238,14 @@ ofwat: unit: /1 label: Water bills growth reference: - - title: Ofwat (and custom projections) - href: https://www.ofwat.gov.uk/price-review/ - - title: Ofwat 2025-26 media statement + - title: Ofwat, "Ofwat responds to water sector average bill figures" (29 January 2026) + href: https://www.ofwat.gov.uk/ofwat-responds-to-water-sector-average-bill-figures/ + - title: Water UK, "Record investment as average household water bills rise £33 a year from April" (28 January 2026) + href: https://www.water.org.uk/news-views-publications/news/record-investment-average-household-water-bills-rise-ps33-year-april + - title: Ofwat, "Average bills 2025/26 press statement" (30 January 2025) href: https://www.ofwat.gov.uk/average-bills-2025-26-press-statement/ + - title: CCW, "Breakdown of water companies' bill increases 2025-30" (Ofwat PR24 draft determinations) + href: https://www.ccw.org.uk/our-work/price-review/how-much-will-my-water-and-sewerage-bills-increase-by-2030/breakdown-of-water-companies-bill-increases-2025-30/ finance_ni: domestic_rates: description: Growth in the average Northern Ireland domestic rates bill (regional rate plus district rate), used to project the domestic_rates dataset input. diff --git a/policyengine_uk/tests/code_health/test_uprating_declarations.py b/policyengine_uk/tests/code_health/test_uprating_declarations.py new file mode 100644 index 0000000000..b4b2572614 --- /dev/null +++ b/policyengine_uk/tests/code_health/test_uprating_declarations.py @@ -0,0 +1,180 @@ +"""Every variable declaring `uprating` should appear in uprating_indices.yaml. + +Through 2030, `uprating_indices.yaml` is the only mechanism that applies +uprating, via `apply_single_year_uprating` and `reset_growthfactor_uprating` +in `data/economic_assumptions.py`. The class-level `uprating = "..."` +attribute on a Variable is read only by the fallback in +`_unmapped_uprating_indices`, which grows unlisted columns after +LEGACY_PROJECTION_END_YEAR (2030). + +A variable can therefore declare an index and silently stay flat from the +dataset year to 2030. That is what #1859 found for electricity and gas +consumption, and #1862 for three more columns. This guard makes the gap fail loudly at the point it is +introduced rather than surfacing in a downstream study. +""" + +from pathlib import Path + +import yaml + +from policyengine_uk import CountryTaxBenefitSystem + +UPRATING_INDICES = Path(__file__).parents[2] / "data" / "uprating_indices.yaml" + +# Variables that declare an index but are deliberately not in the YAML, with +# the reason. Anything else failing this test is a genuine gap. +KNOWN_UNUPRATED = { + # gov.dft.rail.ridership_index is a CUMULATIVE index (2020 = 1.0), not a + # year-on-year growth rate, so it cannot go in a yoy_growth list without + # compounding wrongly: doing so uprates rail_usage by ~684% over 2024-27. + # Uprating this needs a yoy series, not a YAML entry (#1862). + "rail_usage", + # Reported counterpart of employee_pension_contributions, which IS + # uprated; the reported variable feeds imputation rather than results. + "employee_pension_contributions_reported", + # policyengine-uk-data stores energy spend already priced at the dataset's + # target price level (Ofgem Q2 2026 unit rates), so CPI-uprating from the + # data year re-applies price changes the build has priced in (#1868). + "domestic_energy_consumption", + "electricity_consumption", + "gas_consumption", +} + +# Variables the YAML deliberately uprates by an index other than the one they +# declare, with the reason. +KNOWN_INDEX_OVERRIDES = { + # Declares ons.household_interest_income; the YAML applies per_capita.gdp. + # Flagged in #1862 for someone with the context to say which is intended. + "savings_interest_income", +} + + +def _listed_indices(): + listed = yaml.safe_load(UPRATING_INDICES.read_text()) + return { + variable: index for index, variables in listed.items() for variable in variables + } + + +def test_declared_uprating_variables_are_in_the_yaml(): + listed = _listed_indices() + missing = sorted( + name + for name, variable in CountryTaxBenefitSystem().variables.items() + if getattr(variable, "uprating", None) + and name not in listed + and name not in KNOWN_UNUPRATED + ) + assert not missing, ( + "these variables declare `uprating` but are absent from " + f"uprating_indices.yaml, so they are not uprated through 2030: " + f"{missing}. Add them to the matching index block, or to " + "KNOWN_UNUPRATED with a reason." + ) + + +def _uprated_by_another_index(declared, listed_index): + # The YAML keys are yoy_growth twins of the declared indices. + equivalent = declared.replace(".indices.", ".yoy_growth.") + return listed_index not in (declared, equivalent) + + +def test_the_yaml_uprates_by_the_declared_index(): + listed = _listed_indices() + mismatched = [] + for name, variable in CountryTaxBenefitSystem().variables.items(): + declared = getattr(variable, "uprating", None) + if not declared or name not in listed or name in KNOWN_INDEX_OVERRIDES: + continue + if _uprated_by_another_index(declared, listed[name]): + mismatched.append((name, declared, listed[name])) + assert not mismatched, ( + "these variables are uprated by a different index from the one they " + f"declare: {mismatched}" + ) + + +def _stale_exceptions(variables, listed, unuprated, index_overrides): + """Return a message for every exception whose reason no longer holds.""" + stale = [] + for name in sorted(unuprated | index_overrides): + if name not in variables: + stale.append(f"{name} no longer exists; drop the exception") + elif not getattr(variables[name], "uprating", None): + stale.append(f"{name} no longer declares uprating; drop the exception") + for name in sorted(unuprated & variables.keys()): + if name in listed: + stale.append( + f"{name} is now in uprating_indices.yaml under {listed[name]}; " + "remove it from KNOWN_UNUPRATED" + ) + for name in sorted(index_overrides & variables.keys()): + declared = getattr(variables[name], "uprating", None) + if not declared: + continue + if name not in listed: + stale.append( + f"{name} is no longer in uprating_indices.yaml, so there is no " + "index to override; remove it from KNOWN_INDEX_OVERRIDES" + ) + elif not _uprated_by_another_index(declared, listed[name]): + stale.append( + f"{name} is now uprated by its declared index ({declared}); " + "remove it from KNOWN_INDEX_OVERRIDES" + ) + return stale + + +def test_every_known_exception_still_holds(): + """Keeps the exception lists honest: an entry whose reason no longer + holds should be removed rather than left to hide a future gap.""" + stale = _stale_exceptions( + CountryTaxBenefitSystem().variables, + _listed_indices(), + KNOWN_UNUPRATED, + KNOWN_INDEX_OVERRIDES, + ) + assert not stale, "stale uprating exceptions:\n" + "\n".join(stale) + + +class _Declares: + def __init__(self, uprating=None): + self.uprating = uprating + + +CPI = "gov.economic_assumptions.indices.obr.consumer_price_index" +CPI_YOY = "gov.economic_assumptions.yoy_growth.obr.consumer_price_index" +GDP_YOY = "gov.economic_assumptions.yoy_growth.obr.per_capita.gdp" + + +def test_the_guard_passes_exceptions_that_still_hold(): + variables = {"a": _Declares(CPI), "b": _Declares(CPI)} + listed = {"b": GDP_YOY} + assert _stale_exceptions(variables, listed, {"a"}, {"b"}) == [] + + +def test_the_guard_flags_an_unuprated_exception_added_to_the_yaml(): + variables = {"a": _Declares(CPI)} + (stale,) = _stale_exceptions(variables, {"a": CPI_YOY}, {"a"}, set()) + assert "remove it from KNOWN_UNUPRATED" in stale + + +def test_the_guard_flags_an_override_that_now_matches(): + variables = {"b": _Declares(CPI)} + (stale,) = _stale_exceptions(variables, {"b": CPI_YOY}, set(), {"b"}) + assert "remove it from KNOWN_INDEX_OVERRIDES" in stale + + +def test_the_guard_flags_an_override_dropped_from_the_yaml(): + variables = {"b": _Declares(CPI)} + (stale,) = _stale_exceptions(variables, {}, set(), {"b"}) + assert "remove it from KNOWN_INDEX_OVERRIDES" in stale + + +def test_the_guard_flags_missing_or_undeclared_exceptions(): + variables = {"a": _Declares(None)} + stale = _stale_exceptions(variables, {}, {"a", "gone"}, set()) + assert stale == [ + "a no longer declares uprating; drop the exception", + "gone no longer exists; drop the exception", + ] diff --git a/policyengine_uk/tests/test_projection_horizon.py b/policyengine_uk/tests/test_projection_horizon.py index 7d6f77375e..a9837a99f9 100644 --- a/policyengine_uk/tests/test_projection_horizon.py +++ b/policyengine_uk/tests/test_projection_horizon.py @@ -177,20 +177,13 @@ def test_each_singly_indexed_input_grows_by_its_index_every_year(projected): def test_unmapped_inputs_preserve_legacy_years_and_grow_after_2030(projected): """Previously unmapped columns keep their old values through 2030, then grow by the same cumulative-index ratio Core used beyond that year. + Columns uprating_indices.yaml now lists (the debts, ISAs, lodger and + boarder rent, gross receipts, salary sacrifice, water bills and service + charges) grow every year instead, so they are no longer fallbacks. """ assert { - "mortgage_debt", - "consumer_debt", - "rent_paid_as_boarder", - "self_employment_gross_receipts", - "rent_paid_as_lodger", "rail_usage", - "stocks_and_shares_isa", - "cash_isa", - "water_and_sewerage_charges", "employee_pension_contributions_reported", - "pension_contributions_via_salary_sacrifice", - "housing_service_charges", } <= set(FALLBACK_INDEX_OF) for year in range(BASE_YEAR + 1, PROJECTION_END_YEAR + 1): for variable, index_name in FALLBACK_INDEX_OF.items(): diff --git a/policyengine_uk/tests/test_water_bills_growth.py b/policyengine_uk/tests/test_water_bills_growth.py new file mode 100644 index 0000000000..3cf6021b67 --- /dev/null +++ b/policyengine_uk/tests/test_water_bills_growth.py @@ -0,0 +1,25 @@ +"""The published Ofwat years of the water bills series match their sources. + +2025-26: Ofwat, "Ofwat responds to water sector average bill figures" +(29 January 2026) reports a 27% increase in 2025-26, replacing the 26% +forecast in its 30 January 2025 statement. +2026-27: Water UK (28 January 2026) puts the national average bill at GBP 639, +up GBP 33 or 5.4%. +""" + +import pytest + +from policyengine_uk.system import system + +WATER_BILLS = system.parameters.gov.economic_assumptions.yoy_growth.ofwat.water_bills +WATER_INDEX = system.parameters.gov.economic_assumptions.indices.ofwat.water_bills + + +@pytest.mark.parametrize("year, growth", [(2025, 0.27), (2026, 0.054)]) +def test_published_water_bill_growth(year, growth): + assert WATER_BILLS(f"{year}-01-01") == pytest.approx(growth) + + +def test_water_index_compounds_the_published_years(): + ratio = WATER_INDEX("2026-01-01") / WATER_INDEX("2024-01-01") + assert ratio == pytest.approx(1.27 * 1.054, rel=1e-3)