From fd13f1685b7f1583bae0a6ca16520a77fbd6003c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 15:13:20 -0400 Subject: [PATCH 1/2] Refresh the OBR receipts growth series from the EFO receipts tables import_obr_forecasts.py now rewrites yoy_growth.obr.stamp_duty_land_tax.* and business_rates.* from a new EFO's receipts tables, with the previous EFO's tables as the base for outturn growth. Tables are found by title and row label, years before the outturn year are kept, years after the OBR forecast follow average_earnings, and the rewrite is checked to change only those values and references. Run on the March 2026 tables, it reproduces the committed series exactly. Fixes #2341 Co-Authored-By: Claude Opus 5.5 --- changelog.d/obr-receipts-importer.added.md | 1 + .../gov/economic_assumptions/README.md | 33 + .../gov/economic_assumptions/yoy_growth.yaml | 25 +- .../tests/test_import_obr_forecasts.py | 782 +++++++++++++++++- policyengine_uk/utils/import_obr_forecasts.py | 618 +++++++++++++- 5 files changed, 1403 insertions(+), 56 deletions(-) create mode 100644 changelog.d/obr-receipts-importer.added.md diff --git a/changelog.d/obr-receipts-importer.added.md b/changelog.d/obr-receipts-importer.added.md new file mode 100644 index 0000000000..523018e9e5 --- /dev/null +++ b/changelog.d/obr-receipts-importer.added.md @@ -0,0 +1 @@ +- `import_obr_forecasts.py` now refreshes the OBR receipts growth series that project corporate SDLT and business rates (`yoy_growth.obr.stamp_duty_land_tax.*` and `.business_rates.*`) from a new EFO's receipts tables, with the previous EFO's tables (`--previous-receipts-file`) as the base for outturn growth, and refuses receipts tables from a different EFO. diff --git a/policyengine_uk/parameters/gov/economic_assumptions/README.md b/policyengine_uk/parameters/gov/economic_assumptions/README.md index d59e6c8878..7bc8f2a1bf 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/README.md +++ b/policyengine_uk/parameters/gov/economic_assumptions/README.md @@ -20,6 +20,19 @@ generated from it by | `petrol_spending_litre_proxy` | Spending growth that preserves road-fuel litres | HMRC + OBR | | `diesel_spending_litre_proxy` | Same, for diesel | HMRC + OBR | | `finance_ni.domestic_rates` | Average Northern Ireland domestic rates bill (regional plus district poundage) | DoF rate poundages, LPS rates paid, NI draft Budget 2026-29 | +| `stamp_duty_land_tax.residential` | SDLT receipts from residential property | EFO receipts Table 3.7 | +| `stamp_duty_land_tax.non_residential` | SDLT receipts from non-residential property | EFO receipts Table 3.7 | +| `business_rates.great_britain` | Business rates receipts, England, Scotland and Wales (cash) | EFO receipts, current receipts (cash basis): Table 3.8 in March 2026 | +| `business_rates.northern_ireland` | Northern Ireland business rates receipts | EFO receipts Table 3.2 | + +The four receipts series project the corporate SDLT and business rates +statistics (`gov.hmrc.stamp_duty.statistics`, +`gov.hmrc.business_rates.statistics`) from their latest outturn. Each is the +ratio of consecutive published receipts levels, to six decimals: outturn +growth for the EFO's outturn year (its outturn over the year before's outturn +in an earlier EFO), the EFO's forecast years, and after them the +`average_earnings` path. Years before the outturn year are kept, so each +index keeps its base year. ## Time horizons @@ -118,6 +131,26 @@ threshold by the same rises, up to the last year of `yoy_growth.yaml`; pass the receipts tables with `--receipts-file` or `--receipts-url` so September CPI uses the OBR's September forecast. If you edit `yoy_growth.yaml` by hand instead, rerun it with `--gaps-only`. +6. Refresh the receipts series (`stamp_duty_land_tax.*`, + `business_rates.*`). The importer does this in the same run when it also + gets the receipts tables of the EFO whose outturn year is one earlier + (March 2025 for March 2026), with `--previous-receipts-file` or + `--previous-receipts-url`; their outturn is the base for the outturn + year's growth. It finds each line by table title and row label, so table + renumbering does not matter, and it refuses receipts tables from another + EFO. Then update the published levels in + `policyengine_uk/tests/test_corporate_receipts_statistics.py` + (`OBR_RECEIPTS_BN`), which pin the series to the OBR's tables. Pass + `--skip-receipts-growth` only to leave them on the previous EFO. + +For example, for March 2026: + +```bash +uv run python -m policyengine_uk.utils.import_obr_forecasts \ + --file economy_tables_march_2026.xlsx \ + --receipts-file receipts_tables_march_2026.xlsx \ + --previous-receipts-file receipts_tables_march_2025.xlsx +``` The EFO refresh does not touch `finance_ni.domestic_rates`; OBR does not publish a Northern Ireland domestic rates bill series (see below). diff --git a/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml b/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml index 4492945ab4..665aab2e75 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml +++ b/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml @@ -647,19 +647,22 @@ obr: href: https://obr.uk/efo/economic-and-fiscal-outlook-march-2026/ # Receipts used to project the statistics behind corporate incidence. - # Each is the OBR's growth in a receipts line: 2024 is outturn growth - # (2024-25 outturn in the March 2026 EFO over 2023-24 outturn in the March - # 2025 EFO), 2025-2030 the March 2026 EFO forecast, and 2031 onwards the - # average_earnings long-run path (the OBR forecasts receipts only to - # 2030-31). Values are ratios of the published levels, to six decimals. - # The 2024 value never enters a projection: indices start at 1.0 in a - # series' first year, so it only anchors the index at 2024, where the + # Each is the OBR's growth in a receipts line, the ratio of consecutive + # published levels to six decimals. The latest EFO supplies outturn growth + # for its outturn year (its outturn over the previous year's outturn in an + # earlier EFO) and growth for each forecast year. After the OBR's receipts + # forecast, each series follows the average_earnings long-run path. The + # comments in each series name the EFOs and tables behind its values. + # policyengine_uk/utils/import_obr_forecasts.py rewrites these series from + # a new EFO's receipts tables and keeps the years before its outturn year. + # The first value, 2024, never enters a projection: indices start at 1.0 + # in a series' first year, so it only anchors the index at 2024, where the # statistics with 2024-25 as their latest outturn are uprated from. stamp_duty_land_tax: residential: description: Growth in stamp duty land tax receipts from residential property (England and Northern Ireland). values: - # Outturn (2024-25 over 2023-24), Table 3.7 "SDLT from residential property" + # Outturn (2024-25 over 2023-24) from OBR EFO March 2026 over March 2025, receipts Table 3.7 "SDLT from residential property" 2024-01-01: 0.211202 # Forecast (2025-2030) from OBR EFO March 2026, receipts Table 3.7 2025-01-01: 0.1302 @@ -686,7 +689,7 @@ obr: non_residential: description: Growth in stamp duty land tax receipts from non-residential property (England and Northern Ireland). values: - # Outturn (2024-25 over 2023-24), Table 3.7 "SDLT from non-residential property" + # Outturn (2024-25 over 2023-24) from OBR EFO March 2026 over March 2025, receipts Table 3.7 "SDLT from non-residential property" 2024-01-01: 0.151067 # Forecast (2025-2030) from OBR EFO March 2026, receipts Table 3.7 2025-01-01: 0.048515 @@ -714,7 +717,7 @@ obr: great_britain: description: Growth in business rates receipts in England, Scotland and Wales (the OBR "Business rates" line, cash basis). values: - # Outturn (2024-25 over 2023-24), Table 3.8 (March 2025: Table 3.9) "Business rates" + # Outturn (2024-25 over 2023-24) from OBR EFO March 2026 over March 2025, receipts Table 3.8 (March 2025: Table 3.9) "Business rates" 2024-01-01: 0.093818 # Forecast (2025-2030) from OBR EFO March 2026, receipts Table 3.8 2025-01-01: 0.051739 @@ -741,7 +744,7 @@ obr: northern_ireland: description: Growth in Northern Ireland business rates receipts (the OBR "Northern Ireland business rates" line in other taxes). values: - # Outturn (2024-25 over 2023-24), Table 3.2 "Northern Ireland business rates" + # Outturn (2024-25 over 2023-24) from OBR EFO March 2026 over March 2025, receipts Table 3.2 "Northern Ireland business rates" 2024-01-01: 0.032513 # Forecast (2025-2030) from OBR EFO March 2026, receipts Table 3.2 2025-01-01: 0.032388 diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index 9e14ed94b9..34abcd633b 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -6,16 +6,27 @@ import pytest import yaml +from hypothesis import given, settings +from hypothesis import strategies as st from policyengine_uk.utils import import_obr_forecasts from policyengine_uk.utils.import_obr_forecasts import ( + RECEIPTS_SERIES_SPECS, STATUTORY_GAP_SPECS, ForecastGap, + average_earnings_path, + compute_receipts_growth, + find_receipts_line, format_gap, + format_growth, + get_yoy_growth_path, + infer_workbook_release, main, read_calendar_values, + read_workbook_sheets, render_forecast_gap_yaml, + render_receipts_growth_yaml, build_efo_href, compute_statutory_forecast_gaps, extract_annual_series_from_xlsx, @@ -365,22 +376,110 @@ def test_update_yoy_growth_yaml_keeps_existing_values_when_obr_has_blank_years( assert "2027-01-01: 0.0553" in content -def make_receipts_xlsx() -> bytes: - workbook_xml = """ - - - - - - -""" - rels_xml = """ - - - - -""" +def make_workbook(sheets: dict[str, bytes]) -> bytes: + """An XLSX with the given sheets, in order.""" + names = list(sheets) + workbook_xml = ( + '' + '' + "" + + "".join( + f'' + for i, name in enumerate(names, start=1) + ) + + "" + ) + rels_xml = ( + '' + '' + + "".join( + f'' + for i in range(1, len(names) + 1) + ) + + "" + ) + buffer = BytesIO() + with ZipFile(buffer, "w") as archive: + archive.writestr("xl/workbook.xml", workbook_xml) + archive.writestr("xl/_rels/workbook.xml.rels", rels_xml) + for i, name in enumerate(names, start=1): + archive.writestr(f"xl/worksheets/sheet{i}.xml", sheets[name]) + return buffer.getvalue() + + +def column_letter(index: int) -> str: + return chr(ord("A") + index - 1) + + +def receipts_table( + title: str, + first_year: int, + rows: list[tuple[str, tuple]], + first_column: int = 3, + outturn_years: int = 1, +) -> bytes: + """A receipts table laid out as the OBR's: title in B2, the "Outturn" + and "Forecast" markers in row 4 (each only where it has years), fiscal + years in row 5, then one row per line (label in column B). Levels may be + numbers or published cell text.""" + years = max(len(levels) for _, levels in rows) + columns = [column_letter(first_column + i) for i in range(years)] + sheet = { + 1: [make_inline_cell("A1", "Back to contents")], + 2: [make_inline_cell("B2", title)], + 3: [make_inline_cell(f"{columns[0]}3", "£ billion")], + 4: [ + *([make_inline_cell(f"{columns[0]}4", "Outturn")] if outturn_years else []), + *( + [make_inline_cell(f"{columns[outturn_years]}4", "Forecast")] + if outturn_years < years + else [] + ), + ], + 5: [ + make_inline_cell( + f"{column}5", f"{first_year + i}-{str(first_year + i + 1)[2:]}" + ) + for i, column in enumerate(columns) + ], + } + for row, (label, levels) in enumerate(rows, start=6): + sheet[row] = [make_inline_cell(f"B{row}", label)] + [ + make_number_cell(f"{column}{row}", level) + for column, level in zip(columns, levels) + ] + return make_sheet(sheet) + + +# Synthetic receipts, £bn, by property type and nation, from each EFO's +# first (outturn) year. Growth is checked against these by hand. +RECEIPTS_LEVELS = { + "residential": (10.0, 11.0, 12.1), + "non_residential": (4.0, 5.0, 4.5), + "great_britain": (30.0, 31.5, 33.075), + "northern_ireland": (0.4, 0.41, 0.4182), +} +PREVIOUS_RECEIPTS_LEVELS = { + "residential": (8.0, 9.9, 10.5), + "non_residential": (3.2, 3.0, 3.1), + "great_britain": (29.0, 30.1, 31.0), + "northern_ireland": (0.39, 0.395, 0.4), +} + + +def make_receipts_xlsx( + release: str | None = "March 2026", + first_year: int = 2024, + levels: dict[str, tuple] | None = None, + cash_table: str = "3.8", + northern_ireland_label: str = "Northern Ireland business rates", + outturn_years: int = 1, + sdlt_rows: list[tuple[str, tuple]] | None = None, +) -> bytes: + levels = levels or RECEIPTS_LEVELS sheet_318 = make_sheet({2: [make_inline_cell("B2", "Other table")]}) sheet_319 = make_sheet( { @@ -400,13 +499,89 @@ def make_receipts_xlsx() -> bytes: ], } ) - buffer = BytesIO() - with ZipFile(buffer, "w") as archive: - archive.writestr("xl/workbook.xml", workbook_xml) - archive.writestr("xl/_rels/workbook.xml.rels", rels_xml) - archive.writestr("xl/worksheets/sheet1.xml", sheet_318) - archive.writestr("xl/worksheets/sheet2.xml", sheet_319) - return buffer.getvalue() + sheets = {} + if release is not None: + sheets["Contents"] = make_sheet( + { + 2: [ + make_inline_cell( + "B2", + f"{release} Economic and fiscal outlook – detailed " + "forecast tables: receipts", + ) + ], + # The contents list names tables but holds none of their rows. + 6: [make_inline_cell("B6", "3.2 Other taxes")], + } + ) + sheets["3.2"] = receipts_table( + "3.2 Other taxes", + first_year, + [ + ( + "Northern Ireland domestic rates", + (0.5,) * len(levels["northern_ireland"]), + ), + (northern_ireland_label, levels["northern_ireland"]), + ("Other taxes1", (12.0,) * len(levels["northern_ireland"])), + ], + outturn_years=outturn_years, + ) + sheets["3.7"] = receipts_table( + "3.7 Property transactions taxes: Receipts by sector", + first_year, + sdlt_rows + if sdlt_rows is not None + else [ + ( + "Total SDLT", + tuple( + r + n + for r, n in zip(levels["residential"], levels["non_residential"]) + ), + ), + ("SDLT from residential property", levels["residential"]), + ("SDLT from non-residential property", levels["non_residential"]), + ], + outturn_years=outturn_years, + ) + # An accrued table with a "Business rates" row of its own, which the + # cash-basis series must not read. + sheets["3.5"] = receipts_table( + "3.5 Current receipts (on an accrued basis)", + first_year, + [("Business rates", (99.0,) * len(levels["great_britain"]))], + outturn_years=outturn_years, + ) + sheets[cash_table] = receipts_table( + f"{cash_table} Current receipts (on a cash basis)", + first_year, + [ + ("Stamp duty land tax", (14.0,) * len(levels["great_britain"])), + ("Business rates", levels["great_britain"]), + ("Council tax", (46.0,) * len(levels["great_britain"])), + ], + first_column=4, + outturn_years=outturn_years, + ) + sheets["3.18"] = sheet_318 + sheets["3.19"] = sheet_319 + return make_workbook(sheets) + + +def make_previous_receipts_xlsx(**overrides) -> bytes: + """The EFO before: outturn a year earlier, current receipts in Table 3.9 + and the Northern Ireland label in title case, as in March 2025.""" + return make_receipts_xlsx( + **{ + "release": "March 2025", + "first_year": 2023, + "levels": PREVIOUS_RECEIPTS_LEVELS, + "cash_table": "3.9", + "northern_ireland_label": "Northern Ireland Business Rates", + **overrides, + } + ) def test_september_cpi_is_keyed_to_the_september_before_the_uprating_year(): @@ -534,8 +709,7 @@ def test_small_gaps_are_written_in_fixed_point(value, text): "mortgage_interest", "rent", ] -YOY_GROWTH_FIXTURE = "obr:\n" + "".join( - f""" {key}: +FLAT_SERIES_FIXTURE = """ {key}: values: 2024-01-01: 0.0200 2025-01-01: 0.0000 @@ -546,7 +720,42 @@ def test_small_gaps_are_written_in_fixed_point(value, text): - title: Old href: https://example.com/old """ - for key in SERIES_KEYS +RECEIPTS_SERIES_FIXTURE = """ {key}: + description: {key} receipts growth. + values: + # Outturn (2023-24 over 2022-23), from an older EFO + 2023-01-01: 0.0500 + # Older forecast + 2024-01-01: 0.0100 + 2025-01-01: 0.0100 + 2031-01-01: 0.0200 + metadata: + unit: /1 + label: {key} receipts growth + reference: + - title: Old + href: https://example.com/old +""" +RECEIPTS_FIXTURE = ( + "\n # Receipts growth; this comment and the blank lines around it stay.\n" + " stamp_duty_land_tax:\n" + + RECEIPTS_SERIES_FIXTURE.format(key="residential") + + RECEIPTS_SERIES_FIXTURE.format(key="non_residential") + + " business_rates:\n" + + RECEIPTS_SERIES_FIXTURE.format(key="great_britain") + + RECEIPTS_SERIES_FIXTURE.format(key="northern_ireland") + + "\n" +) +YOY_GROWTH_FIXTURE = ( + "obr:\n" + + "".join(FLAT_SERIES_FIXTURE.format(key=key) for key in SERIES_KEYS[:2]) + + RECEIPTS_FIXTURE + + "".join(FLAT_SERIES_FIXTURE.format(key=key) for key in SERIES_KEYS[2:]) + + """ons: + population: + values: + 2024-01-01: 0.0100 +""" ) GAP_FIXTURE = """description: Gap. values: @@ -572,11 +781,15 @@ def write_tree(tmp_path): return yoy, gap_dir -def run_main(tmp_path, workbook, *extra): +def run_main( + tmp_path, workbook, *extra, receipts=None, previous_receipts=None, previous=True +): economy = tmp_path / "economy_march_2026.xlsx" economy.write_bytes(workbook) - receipts = tmp_path / "receipts.xlsx" - receipts.write_bytes(make_receipts_xlsx()) + receipts_path = tmp_path / "receipts.xlsx" + receipts_path.write_bytes(receipts or make_receipts_xlsx()) + previous_path = tmp_path / "previous_receipts.xlsx" + previous_path.write_bytes(previous_receipts or make_previous_receipts_xlsx()) return main( [ "--file", @@ -590,7 +803,8 @@ def run_main(tmp_path, workbook, *extra): "--release-year", "2026", "--receipts-file", - str(receipts), + str(receipts_path), + *(["--previous-receipts-file", str(previous_path)] if previous else []), *extra, ] ) @@ -634,7 +848,14 @@ def test_gaps_only_leaves_growth_alone(tmp_path): yoy, gap_dir = write_tree(tmp_path) growth = yoy.read_text() assert ( - run_main(tmp_path, make_test_xlsx(), "--yaml-path", str(yoy), "--gaps-only") + run_main( + tmp_path, + make_test_xlsx(), + "--yaml-path", + str(yoy), + "--gaps-only", + previous=False, + ) == 0 ) @@ -721,3 +942,502 @@ def failing_replace(source, target): if path not in (unrestored, Path(calls[2])): assert path.read_text() == original, path assert list(tmp_path.rglob("*.staged")) == [] + + +# Receipts growth series: stamp_duty_land_tax and business_rates under obr. + +RECEIPTS_PATHS = {spec.path[-1]: spec.path for spec in RECEIPTS_SERIES_SPECS} + + +def _series(tree, path): + node = tree["obr"] + for key in path: + node = node[key] + return node + + +def _values(content, path): + return { + instant.year: value + for instant, value in _series(yaml.safe_load(content), path)["values"].items() + } + + +def _in_effect(values, year): + return values[max(y for y in values if y <= year)] + + +def render(content=YOY_GROWTH_FIXTURE, current=None, previous=None, **releases): + growth = compute_receipts_growth( + current or make_receipts_xlsx(), previous or make_previous_receipts_xlsx() + ) + return render_receipts_growth_yaml( + content, + growth, + releases.get("release", ("March", 2026)), + releases.get("previous_release", ("March", 2025)), + ) + + +def test_receipts_lines_are_found_by_title_and_label_not_table_number(): + current = read_workbook_sheets(make_receipts_xlsx()) + previous = read_workbook_sheets(make_previous_receipts_xlsx()) + tables = { + spec.path: ( + find_receipts_line(current, spec), + find_receipts_line(previous, spec), + ) + for spec in RECEIPTS_SERIES_SPECS + } + + great_britain, great_britain_before = tables[RECEIPTS_PATHS["great_britain"]] + # Current receipts on a cash basis move from 3.9 to 3.8; the accrued + # table's own "Business rates" row is ignored. + assert (great_britain.table, great_britain_before.table) == ("3.8", "3.9") + assert great_britain.levels == {2024: 30.0, 2025: 31.5, 2026: 33.075} + northern_ireland, northern_ireland_before = tables[ + RECEIPTS_PATHS["northern_ireland"] + ] + assert northern_ireland.label == "Northern Ireland business rates" + assert northern_ireland_before.label == "Northern Ireland Business Rates" + assert northern_ireland_before.levels[2023] == 0.39 + for line, line_before in tables.values(): + assert line.outturn_years == {2024} + assert line_before.outturn_years == {2023} + + +def test_receipts_growth_is_the_ratio_of_published_levels(): + growth = compute_receipts_growth( + make_receipts_xlsx(), make_previous_receipts_xlsx() + ) + # The first year is this EFO's outturn over the previous EFO's outturn, + # e.g. residential 10.0 / 8.0; later years are this EFO's own ratios. + assert {path[-1]: series.values for path, series in growth.items()} == { + "residential": {2024: 0.25, 2025: 0.1, 2026: 0.1}, + "non_residential": {2024: 0.25, 2025: 0.25, 2026: -0.1}, + "great_britain": {2024: 0.034483, 2025: 0.05, 2026: 0.05}, + "northern_ireland": {2024: 0.025641, 2025: 0.025, 2026: 0.02}, + } + + +def test_the_previous_efo_must_publish_outturn_for_the_base_year(): + # An EFO whose first outturn year is 2022-23 has 2023-24 as a forecast. + two_years_back = make_previous_receipts_xlsx(first_year=2022) + with pytest.raises(ValueError, match="no 2023-24 outturn"): + compute_receipts_growth(make_receipts_xlsx(), two_years_back) + # An EFO from the same outturn year has no 2023-24 at all. + with pytest.raises(ValueError, match="no 2023-24 outturn"): + compute_receipts_growth(make_receipts_xlsx(), make_receipts_xlsx()) + + +def test_the_first_year_must_be_outturn(): + no_outturn = make_receipts_xlsx(outturn_years=0) + with pytest.raises( + ValueError, + match="residential: 2024-25, the first year in Table 3.7, is not outturn", + ): + compute_receipts_growth(no_outturn, make_previous_receipts_xlsx()) + + +@pytest.mark.parametrize( + "sdlt_rows, message", + [ + ([("SDLT from non-residential property", (4.0, 5.0, 4.5))], "found none"), + ( + [ + ("SDLT from residential property", (10.0, 11.0, 12.1)), + ("SDLT from residential property", (10.0, 11.0, 12.1)), + ("SDLT from non-residential property", (4.0, 5.0, 4.5)), + ], + "found sheet 3.7, sheet 3.7", + ), + ], +) +def test_each_receipts_line_must_appear_exactly_once(sdlt_rows, message): + with pytest.raises(ValueError, match=message): + compute_receipts_growth( + make_receipts_xlsx(sdlt_rows=sdlt_rows), make_previous_receipts_xlsx() + ) + + +def test_receipts_growth_rejects_a_missing_or_non_positive_level(): + missing = make_receipts_xlsx( + sdlt_rows=[ + ("SDLT from residential property", (10.0, 11.0)), + ("SDLT from non-residential property", (4.0, 5.0, 4.5)), + ] + ) + with pytest.raises(ValueError, match="no 2026-27"): + compute_receipts_growth(missing, make_previous_receipts_xlsx()) + zero = make_receipts_xlsx( + sdlt_rows=[ + ("SDLT from residential property", (10.0, 0.0, 12.1)), + ("SDLT from non-residential property", (4.0, 5.0, 4.5)), + ] + ) + with pytest.raises(ValueError, match="cannot grow from 2025-26"): + compute_receipts_growth(zero, make_previous_receipts_xlsx()) + + +def test_render_rewrites_values_and_references(): + rendered = render() + start = rendered.index(" residential:") + end = rendered.index(" non_residential:") + assert rendered[start:end] == ( + """ residential: + description: residential receipts growth. + values: + # Outturn (2023-24 over 2022-23), from an older EFO + 2023-01-01: 0.0500 + # Outturn (2024-25 over 2023-24) from OBR EFO March 2026 over March 2025, receipts Table 3.7 "SDLT from residential property" + 2024-01-01: 0.2500 + # Forecast (2025-2026) from OBR EFO March 2026, receipts Table 3.7 + 2025-01-01: 0.1000 + 2026-01-01: 0.1000 + # Beyond the OBR forecast: the average_earnings long-run path above + 2027-01-01: 0.0000 + 2031-01-01: 0.0200 + metadata: + unit: /1 + label: residential receipts growth + reference: + - title: OBR EFO March 2026 (detailed forecast tables, receipts, Table 3.7) + href: https://obr.uk/efo/economic-and-fiscal-outlook-march-2026/ + - title: OBR EFO March 2025 (detailed forecast tables, receipts, Table 3.7; 2023-24 outturn) + href: https://obr.uk/efo/economic-and-fiscal-outlook-march-2025/ +""" + ) + assert ( + "# Outturn (2024-25 over 2023-24) from OBR EFO March 2026 over March 2025, " + 'receipts Table 3.8 (March 2025: Table 3.9) "Business rates"' + ) in rendered + assert "receipts, Table 3.9; 2023-24 outturn" in rendered + + +def test_render_changes_only_the_receipts_values_and_references(): + rendered = render() + before, after = yaml.safe_load(YOY_GROWTH_FIXTURE), yaml.safe_load(rendered) + for tree in (before, after): + for path in RECEIPTS_PATHS.values(): + del _series(tree, path)["values"] + del _series(tree, path)["metadata"]["reference"] + assert after == before + # Text outside the series' values and reference blocks is untouched. + assert rendered.startswith( + YOY_GROWTH_FIXTURE[: YOY_GROWTH_FIXTURE.index(" residential:")] + ) + tail = YOY_GROWTH_FIXTURE[YOY_GROWTH_FIXTURE.index("\n consumer_price_index:") :] + assert rendered.endswith(tail) + assert ( + "# Receipts growth; this comment and the blank lines around it stay." + in rendered + ) + + +def test_render_is_idempotent(): + once = render() + assert render(once) == once + + +def test_a_later_efo_keeps_the_years_before_its_outturn_year(): + """The next EFO's first year is 2025: 2024 keeps its value and comment, + so each index keeps its base year.""" + march_2026 = render() + march_2027 = render( + march_2026, + current=make_receipts_xlsx( + release="March 2027", + first_year=2025, + levels={key: (11.0, 12.1, 13.31) for key in RECEIPTS_LEVELS}, + ), + previous=make_receipts_xlsx(), + release=("March", 2027), + previous_release=("March", 2026), + ) + path = RECEIPTS_PATHS["residential"] + assert _values(march_2027, path) == { + 2023: 0.05, + 2024: 0.25, + # 2025-26 outturn (11.0) over 2024-25 outturn in March 2026 (10.0). + 2025: 0.1, + 2026: 0.1, + 2027: 0.1, + 2028: 0.0, + 2031: 0.02, + } + assert ( + "# Outturn (2024-25 over 2023-24) from OBR EFO March 2026 over March 2025, " + 'receipts Table 3.7 "SDLT from residential property"\n' + " 2024-01-01: 0.2500\n" + " # Outturn (2025-26 over 2024-25) from OBR EFO March 2027 over March 2026" + ) in march_2027 + assert "March 2025 (detailed" not in march_2027 + + +def test_later_outturn_years_get_their_own_comment(): + rendered = render(current=make_receipts_xlsx(outturn_years=2)) + assert ( + "# Outturn (2025) from OBR EFO March 2026, receipts Table 3.7\n" + " 2025-01-01: 0.1000\n" + " # Forecast (2026) from OBR EFO March 2026, receipts Table 3.7\n" + ) in rendered + + +def test_average_earnings_path_lists_each_change_after_the_forecast(): + content = """obr: + average_earnings: + values: + 2029-01-01: 0.022 + 2030-01-01: 0.024 + 2031-01-01: 0.0332 + 2032-01-01: 0.0371 + 2036-01-01: 0.0383 + 2037-01-01: 0.0383 + 2073-01-01: 0.0383 +""" + assert average_earnings_path(content, 2030) == { + 2031: 0.0332, + 2032: 0.0371, + 2036: 0.0383, + } + # Starting between entries takes the value then in effect. + assert average_earnings_path(content, 2033) == {2034: 0.0371, 2036: 0.0383} + with pytest.raises(ValueError, match="no value in effect in 2028"): + average_earnings_path(content, 2027) + + +@pytest.mark.parametrize( + "value, text", + [ + (0.0383, "0.0383"), + (0.038, "0.0380"), + (0.1302, "0.1302"), + (0.06083, "0.06083"), + (0.211202, "0.211202"), + (-0.1, "-0.1000"), + (1e-05, "0.00001"), + (-0.0, "0.0000"), + ], +) +def test_growth_is_written_in_fixed_point(value, text): + assert format_growth(value) == text + + +@settings(max_examples=300, deadline=None, derandomize=True) +@given(st.floats(min_value=-1, max_value=100, allow_nan=False)) +def test_written_growth_reads_back_exactly(value): + text = format_growth(value) + assert "e" not in text.lower() + assert len(text.partition(".")[2]) >= 4 + assert yaml.safe_load(f"x: {text}")["x"] == value + + +def test_read_calendar_values_includes_nested_series(tmp_path): + path = tmp_path / "yoy_growth.yaml" + path.write_text(YOY_GROWTH_FIXTURE) + values = read_calendar_values(path) + assert values["stamp_duty_land_tax.residential"] == { + 2023: 0.05, + 2024: 0.01, + 2025: 0.01, + 2031: 0.02, + } + assert set(values) == set(SERIES_KEYS) | { + ".".join(path) for path in RECEIPTS_PATHS.values() + } + + +LEVELS = st.floats(min_value=0.01, max_value=1_000, allow_nan=False) + + +@settings(max_examples=40, deadline=None, derandomize=True) +@given( + years=st.integers(2, 8), + data=st.data(), +) +def test_rendered_growth_compounds_the_published_levels(years, data): + """For any published levels: every year from the outturn year is the + six-decimal ratio of consecutive levels, every later year equals + average_earnings, years before the outturn year are kept, and a second + run changes nothing.""" + levels = { + key: tuple(data.draw(st.lists(LEVELS, min_size=years, max_size=years))) + for key in RECEIPTS_LEVELS + } + base = {key: (data.draw(LEVELS),) for key in RECEIPTS_LEVELS} + current = make_receipts_xlsx(levels=levels) + previous = make_previous_receipts_xlsx(levels=base) + rendered = render(current=current, previous=previous) + + average_earnings = _values(rendered, ("average_earnings",)) + last = 2024 + years - 1 + for key, path in RECEIPTS_PATHS.items(): + values = _values(rendered, path) + published = { + 2023: base[key][0], + **dict(zip(range(2024, last + 1), levels[key])), + } + for year in range(2024, last + 1): + expected = round(published[year] / published[year - 1] - 1, 6) + assert values[year] == expected + for year in range(last + 1, 2050): + assert _in_effect(values, year) == _in_effect(average_earnings, year) + assert values[2023] == 0.05 + assert render(rendered, current=current, previous=previous) == rendered + + +# Published OBR receipts levels as the workbooks hold them (£bn): the March +# 2026 EFO from 2024-25 and the March 2025 EFO from 2023-24. +MARCH_2026_RECEIPTS = { + "residential": ("10.379999999999999", "11.731471524432562", "13.428906667032589", "14.984943814530874", "16.673948133476184", "17.964513468126285", "19.057301113969185"), + "non_residential": ("3.5049999999999999", "3.6750451743530199", "4.72127759316189", "5.3288288104799904", "5.9009139798817092", "6.5330695355310402", "7.1046581772762902"), + "great_britain": ("32.376814327646052", "34.051963932489258", "37.413586572144581", "38.240930947824609", "39.111850954748434", "42.419067764546902", "42.463193622519263"), + "northern_ireland": ("0.38233350730688931", "0.39471664406441903", "0.40263364127921164", "0.41045270712298504", "0.41810894612429189", "0.42590403232705243", "0.43449009714292341"), +} # fmt: skip +MARCH_2025_RECEIPTS = { + "residential": ("8.57", "10.067621905140079", "11.260655778417849", "13.448356441943533", "15.629462727222798", "17.837320625934193", "19.309303893346414"), + "non_residential": ("3.0449999999999999", "3.51897590165244", "2.8597545561212101", "3.7329253327014698", "4.2294622915006306", "4.6980571035732099", "5.2115072362568595"), + "great_britain": ("29.599814327646055", "32.080747417794186", "34.009749097370083", "37.622908483210999", "37.878733699384433", "38.522006059307493", "39.56809170547703"), + "northern_ireland": ("0.37029404557706441", "0.38436854999535813", "0.39453673994468591", "0.40107161940999719", "0.40924102844267779", "0.41720986951771938", "0.42505803014305893"), +} # fmt: skip + + +def test_march_2026_tables_reproduce_the_committed_series(): + """yoy_growth.yaml holds exactly what the importer writes from the March + 2026 receipts tables (Tables 3.7, 3.8 and 3.2) and the March 2025 ones + (3.7, 3.9 and 3.2): rerunning it on the committed file changes nothing.""" + committed = get_yoy_growth_path().read_text() + rendered = render( + committed, + current=make_receipts_xlsx(levels=MARCH_2026_RECEIPTS), + previous=make_previous_receipts_xlsx(levels=MARCH_2025_RECEIPTS), + ) + assert rendered == committed + + +def test_workbook_release_comes_from_the_contents_sheet(): + assert infer_workbook_release(make_receipts_xlsx()) == ("March", 2026) + assert infer_workbook_release(make_receipts_xlsx(release=None)) is None + assert infer_workbook_release(make_test_xlsx()) is None + + +def test_main_updates_receipts_growth_from_the_new_average_earnings(tmp_path): + yoy, _ = write_tree(tmp_path) + assert run_main(tmp_path, make_test_xlsx(), "--yaml-path", str(yoy)) == 0 + + content = yoy.read_text() + assert _values(content, RECEIPTS_PATHS["great_britain"]) == { + 2023: 0.05, + 2024: 0.034483, + 2025: 0.05, + 2026: 0.05, + # 2026 average earnings as this run writes it, then the long run. + 2027: 0.0333, + 2031: 0.02, + } + assert "2026-01-01: 0.0333" in content + + +def test_main_requires_the_previous_receipts_tables(tmp_path): + yoy, gap_dir = write_tree(tmp_path) + before = {path: path.read_text() for path in [yoy, *gap_dir.iterdir()]} + with pytest.raises(ValueError, match="--skip-receipts-growth"): + run_main(tmp_path, make_test_xlsx(), "--yaml-path", str(yoy), previous=False) + assert {path: path.read_text() for path in before} == before + + assert ( + run_main( + tmp_path, + make_test_xlsx(), + "--yaml-path", + str(yoy), + "--skip-receipts-growth", + previous=False, + ) + == 0 + ) + content = yoy.read_text() + assert "2026-01-01: 0.0248" in content + for path in RECEIPTS_PATHS.values(): + assert _values(content, path) == _values(YOY_GROWTH_FIXTURE, path) + + +def test_main_dry_run_without_previous_receipts_only_warns(tmp_path, capsys): + yoy, _ = write_tree(tmp_path) + assert ( + run_main( + tmp_path, + make_test_xlsx(), + "--yaml-path", + str(yoy), + "--dry-run", + previous=False, + ) + == 0 + ) + assert "Warning: Pass the receipts tables" in capsys.readouterr().out + assert yoy.read_text() == YOY_GROWTH_FIXTURE + + +@pytest.mark.parametrize( + "receipts, previous_receipts, message", + [ + ( + make_receipts_xlsx(release="November 2025"), + None, + "receipts tables are from November 2025 but the economy tables are " + "from March 2026", + ), + ( + None, + make_previous_receipts_xlsx(release="March 2026"), + r"previous receipts tables \(March 2026\) must come from an EFO " + "before March 2026", + ), + ( + None, + make_previous_receipts_xlsx(release=None), + "Could not tell which EFO the previous receipts tables are from", + ), + ], +) +def test_main_refuses_tables_from_the_wrong_efo( + tmp_path, receipts, previous_receipts, message +): + yoy, gap_dir = write_tree(tmp_path) + before = {path: path.read_text() for path in [yoy, *gap_dir.iterdir()]} + with pytest.raises(ValueError, match=message): + run_main( + tmp_path, + make_test_xlsx(), + "--yaml-path", + str(yoy), + receipts=receipts, + previous_receipts=previous_receipts, + ) + assert {path: path.read_text() for path in before} == before + + +def test_gaps_only_refuses_the_previous_receipts_tables(tmp_path): + yoy, _ = write_tree(tmp_path) + with pytest.raises(ValueError, match="--gaps-only leaves yoy_growth.yaml alone"): + run_main(tmp_path, make_test_xlsx(), "--yaml-path", str(yoy), "--gaps-only") + + +def test_main_writes_receipts_growth_to_the_output_path(tmp_path): + yoy, _ = write_tree(tmp_path) + output = tmp_path / "out.yaml" + assert ( + run_main( + tmp_path, + make_test_xlsx(), + "--yaml-path", + str(yoy), + "--output", + str(output), + "--skip-statutory-gaps", + ) + == 0 + ) + assert yoy.read_text() == YOY_GROWTH_FIXTURE + assert _values(output.read_text(), RECEIPTS_PATHS["residential"])[2024] == 0.25 diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index cba8bce113..716da9a560 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -1,10 +1,13 @@ from __future__ import annotations import argparse +import copy import os import re import shutil from dataclasses import dataclass +from datetime import date +from decimal import Decimal from io import BytesIO from pathlib import Path from urllib.request import Request, urlopen @@ -18,10 +21,21 @@ "pkgrel": "http://schemas.openxmlformats.org/package/2006/relationships", } YEAR_RE = re.compile(r"^\d{4}$") -MONTH_RE = re.compile( - r"(january|february|march|april|may|june|july|august|september|october|november|december)[-_ ]+(\d{4})", - re.IGNORECASE, +MONTHS = ( + "january", + "february", + "march", + "april", + "may", + "june", + "july", + "august", + "september", + "october", + "november", + "december", ) +MONTH_RE = re.compile(rf"({'|'.join(MONTHS)})[-_ ]+(\d{{4}})", re.IGNORECASE) @dataclass(frozen=True) @@ -451,16 +465,27 @@ def compute_statutory_forecast_gaps( def read_calendar_values(yaml_path: Path) -> dict[str, dict[int, float]]: - """Calendar-year growth by year for each series in a yoy_growth.yaml.""" - obr = yaml.safe_load(yaml_path.read_text())["obr"] - return { - key: { - int(str(instant)[:4]): float(value) - for instant, value in series["values"].items() - } - for key, series in obr.items() - if isinstance(series, dict) and "values" in series - } + """Calendar-year growth by year for each series in a yoy_growth.yaml. + + Nested series are keyed by their path under ``obr`` joined with dots, + such as ``stamp_duty_land_tax.residential``. + """ + return _calendar_values(yaml.safe_load(yaml_path.read_text())["obr"]) + + +def _calendar_values(node: dict, prefix: str = "") -> dict[str, dict[int, float]]: + result = {} + for key, child in node.items(): + if not isinstance(child, dict) or key == "metadata": + continue + if "values" in child: + result[prefix + key] = { + int(str(instant)[:4]): float(value) + for instant, value in child["values"].items() + } + else: + result.update(_calendar_values(child, f"{prefix}{key}.")) + return result def format_gap(value: float) -> str: @@ -559,6 +584,486 @@ def update_forecast_gap_yaml( ) +TABLE_TITLE_RE = re.compile(r"^(\d+\.\d+)\s+(.+)$") +FOOTNOTE_RE = re.compile(r"(?<=[a-z])\d+$") +VALUE_LINE_RE = re.compile(r"^\s*(\d{4})-\d{2}-\d{2}:") + + +@dataclass(frozen=True) +class ReceiptsSeriesSpec: + """A line of the OBR receipts tables, grown by a yoy_growth.yaml series. + + ``path`` is the series' place under ``obr``. The table is found by its + title, not its number, because numbers move between EFOs: current + receipts on a cash basis are Table 3.8 in March 2026 and 3.9 in March + 2025. Titles and row labels are compared lowercased, without footnote + markers. + """ + + path: tuple[str, ...] + title: str + row: str + + +RECEIPTS_SERIES_SPECS = ( + ReceiptsSeriesSpec( + path=("stamp_duty_land_tax", "residential"), + title="property transactions taxes: receipts by sector", + row="sdlt from residential property", + ), + ReceiptsSeriesSpec( + path=("stamp_duty_land_tax", "non_residential"), + title="property transactions taxes: receipts by sector", + row="sdlt from non-residential property", + ), + ReceiptsSeriesSpec( + path=("business_rates", "great_britain"), + title="current receipts (on a cash basis)", + row="business rates", + ), + ReceiptsSeriesSpec( + path=("business_rates", "northern_ireland"), + title="other taxes", + row="northern ireland business rates", + ), +) + + +@dataclass(frozen=True) +class ReceiptsLine: + """A receipts line as one EFO publishes it: £bn by the calendar year in + which each fiscal year starts, and which of those years are outturn.""" + + table: str + label: str + levels: dict[int, float] + outturn_years: frozenset[int] + + +@dataclass(frozen=True) +class ReceiptsGrowth: + """Growth in a receipts line, by year, rounded to six decimals. + + The first year is outturn growth: this EFO's first (outturn) year over + the year before in the previous EFO, where it is outturn. Each later + year is the ratio of this EFO's consecutive levels. + """ + + values: dict[int, float] + line: ReceiptsLine + previous_line: ReceiptsLine + + +def fiscal_year_label(year: int) -> str: + return f"{year}-{str(year + 1)[2:]}" + + +def column_index(column: str) -> int: + index = 0 + for letter in column: + index = index * 26 + ord(letter.upper()) - ord("A") + 1 + return index + + +def receipts_label(value: str | None) -> str: + return FOOTNOTE_RE.sub("", normalise_label(value)) + + +def read_workbook_sheets(xlsx_bytes: bytes) -> dict[str, list[dict[str, str | None]]]: + with ZipFile(BytesIO(xlsx_bytes)) as archive: + sheet_names = list(_sheet_paths(archive)) + return {sheet: read_sheet_rows(xlsx_bytes, sheet) for sheet in sheet_names} + + +def table_title(rows: list[dict[str, str | None]]) -> tuple[str, str] | None: + """The number and normalised title of a table, e.g. ("3.8", "current + receipts (on a cash basis)"), from the first title cell in its top rows.""" + for row in rows[:3]: + for column in sorted(row, key=column_index): + match = TABLE_TITLE_RE.match(" ".join(str(row[column] or "").split())) + if match: + return match.group(1), receipts_label(match.group(2)) + return None + + +def _fiscal_year_columns(row: dict[str, str | None]) -> dict[str, int]: + columns = {} + for column, value in row.items(): + match = FISCAL_YEAR_RE.match(str(value or "").strip()) + if match and int(match.group(2)) == (int(match.group(1)) + 1) % 100: + columns[column] = int(match.group(1)) + return columns + + +def _outturn_years( + years: dict[str, int], markers: dict[str, int], where: str +) -> frozenset[int]: + """Years from the "Outturn" marker's column up to the "Forecast" one. + + A table without an "Outturn" marker has no outturn years, and one + without a "Forecast" marker no forecast years. + """ + if not markers: + raise ValueError(f"{where} does not mark its outturn and forecast years") + if "outturn" not in markers: + return frozenset() + end = markers.get("forecast", float("inf")) + return frozenset( + year + for column, year in years.items() + if markers["outturn"] <= column_index(column) < end + ) + + +def find_receipts_line( + sheets: dict[str, list[dict[str, str | None]]], spec: ReceiptsSeriesSpec +) -> ReceiptsLine: + """The one row labelled ``spec.row`` in the table titled ``spec.title``.""" + matches = [] + for sheet, rows in sheets.items(): + title = table_title(rows) + if title is None or title[1] != spec.title: + continue + years: dict[str, int] = {} + markers: dict[str, int] = {} + for row in rows: + row_years = _fiscal_year_columns(row) + if row_years: + years = row_years + continue + row_markers = { + normalise_label(value): column_index(column) + for column, value in row.items() + if normalise_label(value) in ("outturn", "forecast") + } + if row_markers: + markers = row_markers + continue + if not years: + continue + first_year_column = min(column_index(column) for column in years) + labels = [ + value + for column, value in sorted( + row.items(), key=lambda item: column_index(item[0]) + ) + if column_index(column) < first_year_column and normalise_label(value) + ] + if labels and receipts_label(labels[0]) == spec.row: + matches.append((title[0], sheet, labels[0], row, years, markers)) + + name = ".".join(spec.path) + if len(matches) != 1: + found = ", ".join(f"sheet {match[1]}" for match in matches) or "none" + raise ValueError( + f'Expected one "{spec.row}" row in the "{spec.title}" table for ' + f"{name}; found {found}" + ) + table, sheet, label, row, years, markers = matches[0] + where = f"Table {table} (sheet {sheet})" + levels = {} + for column, year in years.items(): + if row.get(column) in (None, ""): + raise ValueError(f"{where} has no {fiscal_year_label(year)} {name} value") + levels[year] = float(row[column]) + ordered = sorted(levels) + if ordered != list(range(ordered[0], ordered[-1] + 1)): + raise ValueError(f"{where} skips a year: {ordered}") + return ReceiptsLine( + table=table, + label=" ".join(str(label).split()).rstrip("0123456789"), + levels=levels, + outturn_years=_outturn_years(years, markers, where), + ) + + +def compute_receipts_growth( + xlsx_bytes: bytes, previous_xlsx_bytes: bytes +) -> dict[tuple[str, ...], ReceiptsGrowth]: + """Growth in each receipts line, from an EFO's receipts tables and an + earlier EFO's, whose outturn is the base for the first year's growth. + + The first year must be outturn in this EFO, and the year before it + outturn in the earlier one (March 2025 for March 2026). + """ + sheets = read_workbook_sheets(xlsx_bytes) + previous_sheets = read_workbook_sheets(previous_xlsx_bytes) + result = {} + for spec in RECEIPTS_SERIES_SPECS: + name = ".".join(spec.path) + line = find_receipts_line(sheets, spec) + previous = find_receipts_line(previous_sheets, spec) + first_year = min(line.levels) + base_year = first_year - 1 + if first_year not in line.outturn_years: + raise ValueError( + f"{name}: {fiscal_year_label(first_year)}, the first year in " + f"Table {line.table}, is not outturn" + ) + if base_year not in previous.outturn_years: + raise ValueError( + f"{name}: the previous receipts tables have no " + f"{fiscal_year_label(base_year)} outturn; pass the EFO whose " + f"outturn year is {fiscal_year_label(base_year)}" + ) + levels = {base_year: previous.levels[base_year], **line.levels} + values = {} + for year in sorted(line.levels): + if levels[year - 1] <= 0: + raise ValueError( + f"{name}: cannot grow from {fiscal_year_label(year - 1)} " + f"receipts of {levels[year - 1]}" + ) + values[year] = round(levels[year] / levels[year - 1] - 1, 6) + 0.0 + result[spec.path] = ReceiptsGrowth(values, line, previous) + return result + + +def format_growth(value: float) -> str: + """Fixed-point text with at least four decimals that reads back as the + same float: 0.038 is written 0.0380, 1e-05 as 0.00001.""" + text = format(Decimal(repr(value + 0.0)), "f") + whole, _, decimals = text.partition(".") + return f"{whole}.{decimals.ljust(4, '0')}" + + +def average_earnings_path(content: str, after_year: int) -> dict[int, float]: + """``average_earnings`` from the year after ``after_year``: its value in + effect then and each later change, so a series given these values equals + average_earnings in every later year.""" + values = yaml.safe_load(content)["obr"]["average_earnings"]["values"] + entries = sorted((int(str(instant)[:4]), float(v)) for instant, v in values.items()) + start = after_year + 1 + in_effect = [value for year, value in entries if year <= start] + if not in_effect: + raise ValueError(f"average_earnings has no value in effect in {start}") + path = {start: in_effect[-1]} + for year, value in entries: + if year > start and value != path[max(path)]: + path[year] = value + return path + + +def _indent(line: str) -> int: + return len(line) - len(line.lstrip(" ")) + + +def _yaml_block( + lines: list[str], key: str, indent: int, start: int, end: int +) -> tuple[int, int]: + """Lines [first, last) of the mapping entry ``key`` at ``indent`` within + lines[start:end], from its key through its last non-blank child line.""" + pattern = re.compile(rf"^ {{{indent}}}{re.escape(key)}:(\s|$)") + for first in range(start, end): + if not pattern.match(lines[first]): + continue + last = first + 1 + for index in range(first + 1, end): + if not lines[index].strip(): + continue + if _indent(lines[index]) <= indent: + break + last = index + 1 + return first, last + raise ValueError(f"Could not find {key} at indent {indent}") + + +def _series_block(lines: list[str], path: tuple[str, ...]) -> tuple[int, int]: + start, end = 0, len(lines) + for depth, key in enumerate(path): + start, end = _yaml_block(lines, key, 2 * depth, start, end) + return start, end + + +def render_receipts_values( + old_lines: list[str], + indent: int, + growth: ReceiptsGrowth, + tail: dict[int, float], + release: tuple[str, int], + previous_release: tuple[str, int], +) -> list[str]: + """A series' values entries: earlier years as they were, then this + EFO's growth, then the average_earnings path.""" + first_year = min(growth.values) + kept: list[str] = [] + pending: list[str] = [] + for line in old_lines: + match = VALUE_LINE_RE.match(line) + if match is None: + if line.strip() and not line.strip().startswith("#"): + raise ValueError(f"Unexpected line in a values block: {line!r}") + pending.append(line) + continue + if int(match.group(1)) < first_year: + kept += pending + [line] + pending = [] + + month, year = release + previous_month, previous_year = previous_release + published, previous = growth.line, growth.previous_line + table = f"Table {published.table}" + if previous.table != published.table: + table += f" ({previous_month} {previous_year}: Table {previous.table})" + pad = " " * indent + lines = kept + [ + f"{pad}# Outturn ({fiscal_year_label(first_year)} over " + f"{fiscal_year_label(first_year - 1)}) from OBR EFO {month} {year} over " + f'{previous_month} {previous_year}, receipts {table} "{published.label}"', + f"{pad}{first_year}-01-01: {format_growth(growth.values[first_year])}", + ] + later = sorted(y for y in growth.values if y > first_year) + for kind in ("Outturn", "Forecast"): + years = [ + y for y in later if (y in published.outturn_years) == (kind == "Outturn") + ] + if not years: + continue + span = f"{years[0]}" if len(years) == 1 else f"{years[0]}-{years[-1]}" + lines.append( + f"{pad}# {kind} ({span}) from OBR EFO {month} {year}, receipts " + f"Table {published.table}" + ) + lines += [f"{pad}{y}-01-01: {format_growth(growth.values[y])}" for y in years] + lines.append( + f"{pad}# Beyond the OBR forecast: the average_earnings long-run path above" + ) + lines += [f"{pad}{y}-01-01: {format_growth(v)}" for y, v in sorted(tail.items())] + return lines + + +def receipts_references( + growth: ReceiptsGrowth, + release: tuple[str, int], + previous_release: tuple[str, int], +) -> list[dict[str, str]]: + month, year = release + previous_month, previous_year = previous_release + base_year = min(growth.values) - 1 + return [ + { + "title": f"OBR EFO {month} {year} (detailed forecast tables, receipts, " + f"Table {growth.line.table})", + "href": build_efo_href(month, year), + }, + { + "title": f"OBR EFO {previous_month} {previous_year} (detailed forecast " + f"tables, receipts, Table {growth.previous_line.table}; " + f"{fiscal_year_label(base_year)} outturn)", + "href": build_efo_href(previous_month, previous_year), + }, + ] + + +def render_receipts_growth_yaml( + content: str, + growth: dict[tuple[str, ...], ReceiptsGrowth], + release: tuple[str, int], + previous_release: tuple[str, int], +) -> str: + """Rewrite each receipts series' values and references in a + yoy_growth.yaml. + + Years before this EFO's first year are kept, so a series' index keeps + its base year. Years after the OBR's receipts forecast follow + average_earnings as ``content`` holds it. The result is checked to + differ from ``content`` only in those values and references. + """ + before = yaml.safe_load(content) + expected = copy.deepcopy(before) + lines = content.split("\n") + for path, series in growth.items(): + tail = average_earnings_path(content, max(series.values)) + references = receipts_references(series, release, previous_release) + + start, end = _series_block(lines, ("obr", *path)) + indent = 2 * len(path) + 2 + values_start, values_end = _yaml_block(lines, "values", indent, start, end) + old_values = lines[values_start + 1 : values_end] + entry_indent = next( + (_indent(line) for line in old_values if VALUE_LINE_RE.match(line)), + indent + 2, + ) + new_values = render_receipts_values( + old_values, entry_indent, series, tail, release, previous_release + ) + lines[values_start + 1 : values_end] = new_values + + start, end = _series_block(lines, ("obr", *path)) + metadata = _yaml_block(lines, "metadata", indent, start, end) + ref_start, ref_end = _yaml_block( + lines, "reference", indent + 2, metadata[0], metadata[1] + ) + pad = " " * (indent + 2) + lines[ref_start:ref_end] = [f"{pad}reference:"] + [ + text + for reference in references + for text in ( + f"{pad} - title: {reference['title']}", + f"{pad} href: {reference['href']}", + ) + ] + + node = expected["obr"] + for key in path: + node = node[key] + kept = { + instant: value + for instant, value in node["values"].items() + if int(str(instant)[:4]) < min(series.values) + } + node["values"] = { + **kept, + **{ + date(year, 1, 1): value + for year, value in {**series.values, **tail}.items() + }, + } + node["metadata"]["reference"] = references + + rendered = "\n".join(lines) + if yaml.safe_load(rendered) != expected: + raise ValueError( + "Rewriting the receipts growth series changed more than their " + "values and references" + ) + return rendered + + +def infer_workbook_release(xlsx_bytes: bytes) -> tuple[str, int] | None: + """The release a detailed forecast tables workbook names on its contents + sheet, e.g. "March 2026 Economic and fiscal outlook - detailed forecast + tables", or None.""" + with ZipFile(BytesIO(xlsx_bytes)) as archive: + sheet_names = list(_sheet_paths(archive)) + contents = next((s for s in sheet_names if s.lower() == "contents"), None) + if contents is None: + return None + for row in read_sheet_rows(xlsx_bytes, contents)[:5]: + for value in row.values(): + text = str(value or "") + match = MONTH_RE.search(text) + if match and "outlook" in text.lower(): + return match.group(1).capitalize(), int(match.group(2)) + return None + + +def resolve_release(xlsx_bytes: bytes, source_name: str) -> tuple[str, int] | None: + """A workbook's release from its contents sheet, else from its name.""" + release = infer_workbook_release(xlsx_bytes) + if release is not None: + return release + try: + return infer_release(source_name) + except ValueError: + return None + + +def release_order(release: tuple[str, int]) -> tuple[int, int]: + month, year = release + return year, MONTHS.index(month.lower()) + + def infer_release(source_name: str) -> tuple[str, int]: match = MONTH_RE.search(source_name) if not match: @@ -710,6 +1215,16 @@ def print_summary( print(f"- {spec.key}: {', '.join(window)}") +def print_receipts_summary(growth: dict[tuple[str, ...], ReceiptsGrowth]) -> None: + print(f"Updating {len(growth)} OBR receipts growth series") + for path, series in growth.items(): + window = ", ".join( + f"{year}: {format_growth(value)}" + for year, value in sorted(series.values.items()) + ) + print(f"- {'.'.join(path)} (Table {series.line.table}): {window}") + + def build_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Import OBR economy forecast tables into yoy_growth.yaml.", @@ -756,6 +1271,25 @@ def build_arg_parser() -> argparse.ArgumentParser: ), ) parser.add_argument("--receipts-file", help="Local receipts XLSX file path") + parser.add_argument( + "--previous-receipts-url", + help=( + "Receipts detailed forecast tables (XLSX) from the EFO whose outturn " + "year is the year before this one's (March 2025 for March 2026), " + "the base for the receipts growth series' first year" + ), + ) + parser.add_argument( + "--previous-receipts-file", help="Local previous receipts XLSX file path" + ) + parser.add_argument( + "--skip-receipts-growth", + action="store_true", + help=( + "Do not update the receipts growth series (stamp_duty_land_tax, " + "business_rates) that project corporate SDLT and business rates" + ), + ) parser.add_argument( "--gaps-only", action="store_true", @@ -867,11 +1401,62 @@ def main(argv: list[str] | None = None) -> int: if not args.dry_run: raise ValueError(message) print(f"Warning: {message}") + has_previous_receipts = bool( + args.previous_receipts_url or args.previous_receipts_file + ) + if args.gaps_only and has_previous_receipts: + raise ValueError( + "--gaps-only leaves yoy_growth.yaml alone, so it cannot update the " + "receipts growth series from --previous-receipts-file or " + "--previous-receipts-url" + ) + write_receipts = not args.gaps_only and not args.skip_receipts_growth + if write_receipts and not ( + has_previous_receipts and (args.receipts_url or args.receipts_file) + ): + message = ( + "Pass the receipts tables (--receipts-file or --receipts-url) and the " + "previous EFO's (--previous-receipts-file or --previous-receipts-url) " + "so the receipts growth series move to this EFO, or " + "--skip-receipts-growth" + ) + if not args.dry_run: + raise ValueError(message) + print(f"Warning: {message}") + write_receipts = False print_summary(series_values, forecast_start_year, args.forecast_years) receipts_bytes = None if args.receipts_url or args.receipts_file: - _, receipts_bytes = load_source_bytes(args.receipts_url, args.receipts_file) + receipts_name, receipts_bytes = load_source_bytes( + args.receipts_url, args.receipts_file + ) + receipts_release = resolve_release(receipts_bytes, receipts_name) + if receipts_release not in (None, (month, year)): + raise ValueError( + "The receipts tables are from " + f"{receipts_release[0]} {receipts_release[1]} but the economy " + f"tables are from {month} {year}" + ) + receipts_growth = None + if write_receipts: + previous_name, previous_bytes = load_source_bytes( + args.previous_receipts_url, args.previous_receipts_file + ) + previous_release = resolve_release(previous_bytes, previous_name) + if previous_release is None: + raise ValueError( + "Could not tell which EFO the previous receipts tables are from; " + "name the file with its release, e.g. receipts_march_2025.xlsx" + ) + if release_order(previous_release) >= release_order((month, year)): + raise ValueError( + "The previous receipts tables " + f"({previous_release[0]} {previous_release[1]}) must come from " + f"an EFO before {month} {year}" + ) + receipts_growth = compute_receipts_growth(receipts_bytes, previous_bytes) + print_receipts_summary(receipts_growth) calendar_values = ( read_calendar_values(source_yaml_path) if args.gaps_only else series_values ) @@ -900,6 +1485,11 @@ def main(argv: list[str] | None = None) -> int: forecast_start_year, args.forecast_years, ) + if receipts_growth is not None: + yoy_path = args.output or source_yaml_path + outputs[yoy_path] = render_receipts_growth_yaml( + outputs[yoy_path], receipts_growth, (month, year), previous_release + ) if write_gaps: gap_dir = get_forecast_gap_dir(source_yaml_path) for spec in STATUTORY_GAP_SPECS: From 0da72d8aa8227a4bb31e5ee09213af194fdad073 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 15:15:42 -0400 Subject: [PATCH 2/2] Refuse non-numeric receipts cells and invalid release months; strip only footnote markers from labels Co-Authored-By: Claude Opus 5.5 --- .../tests/test_import_obr_forecasts.py | 28 ++++++++++++++++++- policyengine_uk/utils/import_obr_forecasts.py | 19 +++++++++---- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index 34abcd633b..be2fd47ad9 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -1067,7 +1067,7 @@ def test_receipts_growth_rejects_a_missing_or_non_positive_level(): ("SDLT from non-residential property", (4.0, 5.0, 4.5)), ] ) - with pytest.raises(ValueError, match="no 2026-27"): + with pytest.raises(ValueError, match="no numeric 2026-27 .* value: None"): compute_receipts_growth(missing, make_previous_receipts_xlsx()) zero = make_receipts_xlsx( sdlt_rows=[ @@ -1079,6 +1079,32 @@ def test_receipts_growth_rejects_a_missing_or_non_positive_level(): compute_receipts_growth(zero, make_previous_receipts_xlsx()) +def test_receipts_labels_lose_footnote_markers_only(): + line = find_receipts_line( + read_workbook_sheets( + make_receipts_xlsx( + sdlt_rows=[ + ("SDLT from residential property2", (10.0, 11.0, 12.1)), + ("SDLT from non-residential property", (4.0, 5.0, 4.5)), + ] + ) + ), + RECEIPTS_SERIES_SPECS[0], + ) + assert line.label == "SDLT from residential property" + + +def test_a_non_numeric_level_is_refused(): + text = make_receipts_xlsx( + sdlt_rows=[ + ("SDLT from residential property", (10.0, "[x]", 12.1)), + ("SDLT from non-residential property", (4.0, 5.0, 4.5)), + ] + ) + with pytest.raises(ValueError, match="no numeric 2025-26 .* value: '\\[x\\]'"): + compute_receipts_growth(text, make_previous_receipts_xlsx()) + + def test_render_rewrites_values_and_references(): rendered = render() start = rendered.index(" residential:") diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index 716da9a560..bfda5ab755 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -585,7 +585,7 @@ def update_forecast_gap_yaml( TABLE_TITLE_RE = re.compile(r"^(\d+\.\d+)\s+(.+)$") -FOOTNOTE_RE = re.compile(r"(?<=[a-z])\d+$") +FOOTNOTE_RE = re.compile(r"(?<=[a-z])\d+$", re.IGNORECASE) VALUE_LINE_RE = re.compile(r"^\s*(\d{4})-\d{2}-\d{2}:") @@ -763,15 +763,20 @@ def find_receipts_line( where = f"Table {table} (sheet {sheet})" levels = {} for column, year in years.items(): - if row.get(column) in (None, ""): - raise ValueError(f"{where} has no {fiscal_year_label(year)} {name} value") - levels[year] = float(row[column]) + text = row.get(column) + try: + levels[year] = float(text) + except (TypeError, ValueError): + raise ValueError( + f"{where} has no numeric {fiscal_year_label(year)} {name} value: " + f"{text!r}" + ) from None ordered = sorted(levels) if ordered != list(range(ordered[0], ordered[-1] + 1)): raise ValueError(f"{where} skips a year: {ordered}") return ReceiptsLine( table=table, - label=" ".join(str(label).split()).rstrip("0123456789"), + label=FOOTNOTE_RE.sub("", " ".join(str(label).split())), levels=levels, outturn_years=_outturn_years(years, markers, where), ) @@ -1379,6 +1384,10 @@ def main(argv: list[str] | None = None) -> int: series_values = extract_annual_series_from_xlsx(workbook_bytes) if args.release_month and args.release_year: + if args.release_month.lower() not in MONTHS: + raise ValueError( + f"--release-month must be a month name, not {args.release_month!r}" + ) month = args.release_month.capitalize() year = args.release_year elif args.release_month or args.release_year: