diff --git a/changelog.d/obr-receipts-importer.added.md b/changelog.d/obr-receipts-importer.added.md new file mode 100644 index 0000000000..523018e9e5 --- /dev/null +++ b/changelog.d/obr-receipts-importer.added.md @@ -0,0 +1 @@ +- `import_obr_forecasts.py` now refreshes the OBR receipts growth series that project corporate SDLT and business rates (`yoy_growth.obr.stamp_duty_land_tax.*` and `.business_rates.*`) from a new EFO's receipts tables, with the previous EFO's tables (`--previous-receipts-file`) as the base for outturn growth, and refuses receipts tables from a different EFO. diff --git a/policyengine_uk/parameters/gov/economic_assumptions/README.md b/policyengine_uk/parameters/gov/economic_assumptions/README.md index d59e6c8878..7bc8f2a1bf 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/README.md +++ b/policyengine_uk/parameters/gov/economic_assumptions/README.md @@ -20,6 +20,19 @@ generated from it by | `petrol_spending_litre_proxy` | Spending growth that preserves road-fuel litres | HMRC + OBR | | `diesel_spending_litre_proxy` | Same, for diesel | HMRC + OBR | | `finance_ni.domestic_rates` | Average Northern Ireland domestic rates bill (regional plus district poundage) | DoF rate poundages, LPS rates paid, NI draft Budget 2026-29 | +| `stamp_duty_land_tax.residential` | SDLT receipts from residential property | EFO receipts Table 3.7 | +| `stamp_duty_land_tax.non_residential` | SDLT receipts from non-residential property | EFO receipts Table 3.7 | +| `business_rates.great_britain` | Business rates receipts, England, Scotland and Wales (cash) | EFO receipts, current receipts (cash basis): Table 3.8 in March 2026 | +| `business_rates.northern_ireland` | Northern Ireland business rates receipts | EFO receipts Table 3.2 | + +The four receipts series project the corporate SDLT and business rates +statistics (`gov.hmrc.stamp_duty.statistics`, +`gov.hmrc.business_rates.statistics`) from their latest outturn. Each is the +ratio of consecutive published receipts levels, to six decimals: outturn +growth for the EFO's outturn year (its outturn over the year before's outturn +in an earlier EFO), the EFO's forecast years, and after them the +`average_earnings` path. Years before the outturn year are kept, so each +index keeps its base year. ## Time horizons @@ -118,6 +131,26 @@ threshold by the same rises, up to the last year of `yoy_growth.yaml`; pass the receipts tables with `--receipts-file` or `--receipts-url` so September CPI uses the OBR's September forecast. If you edit `yoy_growth.yaml` by hand instead, rerun it with `--gaps-only`. +6. Refresh the receipts series (`stamp_duty_land_tax.*`, + `business_rates.*`). The importer does this in the same run when it also + gets the receipts tables of the EFO whose outturn year is one earlier + (March 2025 for March 2026), with `--previous-receipts-file` or + `--previous-receipts-url`; their outturn is the base for the outturn + year's growth. It finds each line by table title and row label, so table + renumbering does not matter, and it refuses receipts tables from another + EFO. Then update the published levels in + `policyengine_uk/tests/test_corporate_receipts_statistics.py` + (`OBR_RECEIPTS_BN`), which pin the series to the OBR's tables. Pass + `--skip-receipts-growth` only to leave them on the previous EFO. + +For example, for March 2026: + +```bash +uv run python -m policyengine_uk.utils.import_obr_forecasts \ + --file economy_tables_march_2026.xlsx \ + --receipts-file receipts_tables_march_2026.xlsx \ + --previous-receipts-file receipts_tables_march_2025.xlsx +``` The EFO refresh does not touch `finance_ni.domestic_rates`; OBR does not publish a Northern Ireland domestic rates bill series (see below). diff --git a/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml b/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml index 4492945ab4..665aab2e75 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml +++ b/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml @@ -647,19 +647,22 @@ obr: href: https://obr.uk/efo/economic-and-fiscal-outlook-march-2026/ # Receipts used to project the statistics behind corporate incidence. - # Each is the OBR's growth in a receipts line: 2024 is outturn growth - # (2024-25 outturn in the March 2026 EFO over 2023-24 outturn in the March - # 2025 EFO), 2025-2030 the March 2026 EFO forecast, and 2031 onwards the - # average_earnings long-run path (the OBR forecasts receipts only to - # 2030-31). Values are ratios of the published levels, to six decimals. - # The 2024 value never enters a projection: indices start at 1.0 in a - # series' first year, so it only anchors the index at 2024, where the + # Each is the OBR's growth in a receipts line, the ratio of consecutive + # published levels to six decimals. The latest EFO supplies outturn growth + # for its outturn year (its outturn over the previous year's outturn in an + # earlier EFO) and growth for each forecast year. After the OBR's receipts + # forecast, each series follows the average_earnings long-run path. The + # comments in each series name the EFOs and tables behind its values. + # policyengine_uk/utils/import_obr_forecasts.py rewrites these series from + # a new EFO's receipts tables and keeps the years before its outturn year. + # The first value, 2024, never enters a projection: indices start at 1.0 + # in a series' first year, so it only anchors the index at 2024, where the # statistics with 2024-25 as their latest outturn are uprated from. stamp_duty_land_tax: residential: description: Growth in stamp duty land tax receipts from residential property (England and Northern Ireland). values: - # Outturn (2024-25 over 2023-24), Table 3.7 "SDLT from residential property" + # Outturn (2024-25 over 2023-24) from OBR EFO March 2026 over March 2025, receipts Table 3.7 "SDLT from residential property" 2024-01-01: 0.211202 # Forecast (2025-2030) from OBR EFO March 2026, receipts Table 3.7 2025-01-01: 0.1302 @@ -686,7 +689,7 @@ obr: non_residential: description: Growth in stamp duty land tax receipts from non-residential property (England and Northern Ireland). values: - # Outturn (2024-25 over 2023-24), Table 3.7 "SDLT from non-residential property" + # Outturn (2024-25 over 2023-24) from OBR EFO March 2026 over March 2025, receipts Table 3.7 "SDLT from non-residential property" 2024-01-01: 0.151067 # Forecast (2025-2030) from OBR EFO March 2026, receipts Table 3.7 2025-01-01: 0.048515 @@ -714,7 +717,7 @@ obr: great_britain: description: Growth in business rates receipts in England, Scotland and Wales (the OBR "Business rates" line, cash basis). values: - # Outturn (2024-25 over 2023-24), Table 3.8 (March 2025: Table 3.9) "Business rates" + # Outturn (2024-25 over 2023-24) from OBR EFO March 2026 over March 2025, receipts Table 3.8 (March 2025: Table 3.9) "Business rates" 2024-01-01: 0.093818 # Forecast (2025-2030) from OBR EFO March 2026, receipts Table 3.8 2025-01-01: 0.051739 @@ -741,7 +744,7 @@ obr: northern_ireland: description: Growth in Northern Ireland business rates receipts (the OBR "Northern Ireland business rates" line in other taxes). values: - # Outturn (2024-25 over 2023-24), Table 3.2 "Northern Ireland business rates" + # Outturn (2024-25 over 2023-24) from OBR EFO March 2026 over March 2025, receipts Table 3.2 "Northern Ireland business rates" 2024-01-01: 0.032513 # Forecast (2025-2030) from OBR EFO March 2026, receipts Table 3.2 2025-01-01: 0.032388 diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index 9e14ed94b9..be2fd47ad9 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -6,16 +6,27 @@ import pytest import yaml +from hypothesis import given, settings +from hypothesis import strategies as st from policyengine_uk.utils import import_obr_forecasts from policyengine_uk.utils.import_obr_forecasts import ( + RECEIPTS_SERIES_SPECS, STATUTORY_GAP_SPECS, ForecastGap, + average_earnings_path, + compute_receipts_growth, + find_receipts_line, format_gap, + format_growth, + get_yoy_growth_path, + infer_workbook_release, main, read_calendar_values, + read_workbook_sheets, render_forecast_gap_yaml, + render_receipts_growth_yaml, build_efo_href, compute_statutory_forecast_gaps, extract_annual_series_from_xlsx, @@ -365,22 +376,110 @@ def test_update_yoy_growth_yaml_keeps_existing_values_when_obr_has_blank_years( assert "2027-01-01: 0.0553" in content -def make_receipts_xlsx() -> bytes: - workbook_xml = """ - - - - - - -""" - rels_xml = """ - - - - -""" +def make_workbook(sheets: dict[str, bytes]) -> bytes: + """An XLSX with the given sheets, in order.""" + names = list(sheets) + workbook_xml = ( + '' + '' + "" + + "".join( + f'' + for i, name in enumerate(names, start=1) + ) + + "" + ) + rels_xml = ( + '' + '' + + "".join( + f'' + for i in range(1, len(names) + 1) + ) + + "" + ) + buffer = BytesIO() + with ZipFile(buffer, "w") as archive: + archive.writestr("xl/workbook.xml", workbook_xml) + archive.writestr("xl/_rels/workbook.xml.rels", rels_xml) + for i, name in enumerate(names, start=1): + archive.writestr(f"xl/worksheets/sheet{i}.xml", sheets[name]) + return buffer.getvalue() + + +def column_letter(index: int) -> str: + return chr(ord("A") + index - 1) + + +def receipts_table( + title: str, + first_year: int, + rows: list[tuple[str, tuple]], + first_column: int = 3, + outturn_years: int = 1, +) -> bytes: + """A receipts table laid out as the OBR's: title in B2, the "Outturn" + and "Forecast" markers in row 4 (each only where it has years), fiscal + years in row 5, then one row per line (label in column B). Levels may be + numbers or published cell text.""" + years = max(len(levels) for _, levels in rows) + columns = [column_letter(first_column + i) for i in range(years)] + sheet = { + 1: [make_inline_cell("A1", "Back to contents")], + 2: [make_inline_cell("B2", title)], + 3: [make_inline_cell(f"{columns[0]}3", "£ billion")], + 4: [ + *([make_inline_cell(f"{columns[0]}4", "Outturn")] if outturn_years else []), + *( + [make_inline_cell(f"{columns[outturn_years]}4", "Forecast")] + if outturn_years < years + else [] + ), + ], + 5: [ + make_inline_cell( + f"{column}5", f"{first_year + i}-{str(first_year + i + 1)[2:]}" + ) + for i, column in enumerate(columns) + ], + } + for row, (label, levels) in enumerate(rows, start=6): + sheet[row] = [make_inline_cell(f"B{row}", label)] + [ + make_number_cell(f"{column}{row}", level) + for column, level in zip(columns, levels) + ] + return make_sheet(sheet) + + +# Synthetic receipts, £bn, by property type and nation, from each EFO's +# first (outturn) year. Growth is checked against these by hand. +RECEIPTS_LEVELS = { + "residential": (10.0, 11.0, 12.1), + "non_residential": (4.0, 5.0, 4.5), + "great_britain": (30.0, 31.5, 33.075), + "northern_ireland": (0.4, 0.41, 0.4182), +} +PREVIOUS_RECEIPTS_LEVELS = { + "residential": (8.0, 9.9, 10.5), + "non_residential": (3.2, 3.0, 3.1), + "great_britain": (29.0, 30.1, 31.0), + "northern_ireland": (0.39, 0.395, 0.4), +} + + +def make_receipts_xlsx( + release: str | None = "March 2026", + first_year: int = 2024, + levels: dict[str, tuple] | None = None, + cash_table: str = "3.8", + northern_ireland_label: str = "Northern Ireland business rates", + outturn_years: int = 1, + sdlt_rows: list[tuple[str, tuple]] | None = None, +) -> bytes: + levels = levels or RECEIPTS_LEVELS sheet_318 = make_sheet({2: [make_inline_cell("B2", "Other table")]}) sheet_319 = make_sheet( { @@ -400,13 +499,89 @@ def make_receipts_xlsx() -> bytes: ], } ) - buffer = BytesIO() - with ZipFile(buffer, "w") as archive: - archive.writestr("xl/workbook.xml", workbook_xml) - archive.writestr("xl/_rels/workbook.xml.rels", rels_xml) - archive.writestr("xl/worksheets/sheet1.xml", sheet_318) - archive.writestr("xl/worksheets/sheet2.xml", sheet_319) - return buffer.getvalue() + sheets = {} + if release is not None: + sheets["Contents"] = make_sheet( + { + 2: [ + make_inline_cell( + "B2", + f"{release} Economic and fiscal outlook – detailed " + "forecast tables: receipts", + ) + ], + # The contents list names tables but holds none of their rows. + 6: [make_inline_cell("B6", "3.2 Other taxes")], + } + ) + sheets["3.2"] = receipts_table( + "3.2 Other taxes", + first_year, + [ + ( + "Northern Ireland domestic rates", + (0.5,) * len(levels["northern_ireland"]), + ), + (northern_ireland_label, levels["northern_ireland"]), + ("Other taxes1", (12.0,) * len(levels["northern_ireland"])), + ], + outturn_years=outturn_years, + ) + sheets["3.7"] = receipts_table( + "3.7 Property transactions taxes: Receipts by sector", + first_year, + sdlt_rows + if sdlt_rows is not None + else [ + ( + "Total SDLT", + tuple( + r + n + for r, n in zip(levels["residential"], levels["non_residential"]) + ), + ), + ("SDLT from residential property", levels["residential"]), + ("SDLT from non-residential property", levels["non_residential"]), + ], + outturn_years=outturn_years, + ) + # An accrued table with a "Business rates" row of its own, which the + # cash-basis series must not read. + sheets["3.5"] = receipts_table( + "3.5 Current receipts (on an accrued basis)", + first_year, + [("Business rates", (99.0,) * len(levels["great_britain"]))], + outturn_years=outturn_years, + ) + sheets[cash_table] = receipts_table( + f"{cash_table} Current receipts (on a cash basis)", + first_year, + [ + ("Stamp duty land tax", (14.0,) * len(levels["great_britain"])), + ("Business rates", levels["great_britain"]), + ("Council tax", (46.0,) * len(levels["great_britain"])), + ], + first_column=4, + outturn_years=outturn_years, + ) + sheets["3.18"] = sheet_318 + sheets["3.19"] = sheet_319 + return make_workbook(sheets) + + +def make_previous_receipts_xlsx(**overrides) -> bytes: + """The EFO before: outturn a year earlier, current receipts in Table 3.9 + and the Northern Ireland label in title case, as in March 2025.""" + return make_receipts_xlsx( + **{ + "release": "March 2025", + "first_year": 2023, + "levels": PREVIOUS_RECEIPTS_LEVELS, + "cash_table": "3.9", + "northern_ireland_label": "Northern Ireland Business Rates", + **overrides, + } + ) def test_september_cpi_is_keyed_to_the_september_before_the_uprating_year(): @@ -534,8 +709,7 @@ def test_small_gaps_are_written_in_fixed_point(value, text): "mortgage_interest", "rent", ] -YOY_GROWTH_FIXTURE = "obr:\n" + "".join( - f""" {key}: +FLAT_SERIES_FIXTURE = """ {key}: values: 2024-01-01: 0.0200 2025-01-01: 0.0000 @@ -546,7 +720,42 @@ def test_small_gaps_are_written_in_fixed_point(value, text): - title: Old href: https://example.com/old """ - for key in SERIES_KEYS +RECEIPTS_SERIES_FIXTURE = """ {key}: + description: {key} receipts growth. + values: + # Outturn (2023-24 over 2022-23), from an older EFO + 2023-01-01: 0.0500 + # Older forecast + 2024-01-01: 0.0100 + 2025-01-01: 0.0100 + 2031-01-01: 0.0200 + metadata: + unit: /1 + label: {key} receipts growth + reference: + - title: Old + href: https://example.com/old +""" +RECEIPTS_FIXTURE = ( + "\n # Receipts growth; this comment and the blank lines around it stay.\n" + " stamp_duty_land_tax:\n" + + RECEIPTS_SERIES_FIXTURE.format(key="residential") + + RECEIPTS_SERIES_FIXTURE.format(key="non_residential") + + " business_rates:\n" + + RECEIPTS_SERIES_FIXTURE.format(key="great_britain") + + RECEIPTS_SERIES_FIXTURE.format(key="northern_ireland") + + "\n" +) +YOY_GROWTH_FIXTURE = ( + "obr:\n" + + "".join(FLAT_SERIES_FIXTURE.format(key=key) for key in SERIES_KEYS[:2]) + + RECEIPTS_FIXTURE + + "".join(FLAT_SERIES_FIXTURE.format(key=key) for key in SERIES_KEYS[2:]) + + """ons: + population: + values: + 2024-01-01: 0.0100 +""" ) GAP_FIXTURE = """description: Gap. values: @@ -572,11 +781,15 @@ def write_tree(tmp_path): return yoy, gap_dir -def run_main(tmp_path, workbook, *extra): +def run_main( + tmp_path, workbook, *extra, receipts=None, previous_receipts=None, previous=True +): economy = tmp_path / "economy_march_2026.xlsx" economy.write_bytes(workbook) - receipts = tmp_path / "receipts.xlsx" - receipts.write_bytes(make_receipts_xlsx()) + receipts_path = tmp_path / "receipts.xlsx" + receipts_path.write_bytes(receipts or make_receipts_xlsx()) + previous_path = tmp_path / "previous_receipts.xlsx" + previous_path.write_bytes(previous_receipts or make_previous_receipts_xlsx()) return main( [ "--file", @@ -590,7 +803,8 @@ def run_main(tmp_path, workbook, *extra): "--release-year", "2026", "--receipts-file", - str(receipts), + str(receipts_path), + *(["--previous-receipts-file", str(previous_path)] if previous else []), *extra, ] ) @@ -634,7 +848,14 @@ def test_gaps_only_leaves_growth_alone(tmp_path): yoy, gap_dir = write_tree(tmp_path) growth = yoy.read_text() assert ( - run_main(tmp_path, make_test_xlsx(), "--yaml-path", str(yoy), "--gaps-only") + run_main( + tmp_path, + make_test_xlsx(), + "--yaml-path", + str(yoy), + "--gaps-only", + previous=False, + ) == 0 ) @@ -721,3 +942,528 @@ def failing_replace(source, target): if path not in (unrestored, Path(calls[2])): assert path.read_text() == original, path assert list(tmp_path.rglob("*.staged")) == [] + + +# Receipts growth series: stamp_duty_land_tax and business_rates under obr. + +RECEIPTS_PATHS = {spec.path[-1]: spec.path for spec in RECEIPTS_SERIES_SPECS} + + +def _series(tree, path): + node = tree["obr"] + for key in path: + node = node[key] + return node + + +def _values(content, path): + return { + instant.year: value + for instant, value in _series(yaml.safe_load(content), path)["values"].items() + } + + +def _in_effect(values, year): + return values[max(y for y in values if y <= year)] + + +def render(content=YOY_GROWTH_FIXTURE, current=None, previous=None, **releases): + growth = compute_receipts_growth( + current or make_receipts_xlsx(), previous or make_previous_receipts_xlsx() + ) + return render_receipts_growth_yaml( + content, + growth, + releases.get("release", ("March", 2026)), + releases.get("previous_release", ("March", 2025)), + ) + + +def test_receipts_lines_are_found_by_title_and_label_not_table_number(): + current = read_workbook_sheets(make_receipts_xlsx()) + previous = read_workbook_sheets(make_previous_receipts_xlsx()) + tables = { + spec.path: ( + find_receipts_line(current, spec), + find_receipts_line(previous, spec), + ) + for spec in RECEIPTS_SERIES_SPECS + } + + great_britain, great_britain_before = tables[RECEIPTS_PATHS["great_britain"]] + # Current receipts on a cash basis move from 3.9 to 3.8; the accrued + # table's own "Business rates" row is ignored. + assert (great_britain.table, great_britain_before.table) == ("3.8", "3.9") + assert great_britain.levels == {2024: 30.0, 2025: 31.5, 2026: 33.075} + northern_ireland, northern_ireland_before = tables[ + RECEIPTS_PATHS["northern_ireland"] + ] + assert northern_ireland.label == "Northern Ireland business rates" + assert northern_ireland_before.label == "Northern Ireland Business Rates" + assert northern_ireland_before.levels[2023] == 0.39 + for line, line_before in tables.values(): + assert line.outturn_years == {2024} + assert line_before.outturn_years == {2023} + + +def test_receipts_growth_is_the_ratio_of_published_levels(): + growth = compute_receipts_growth( + make_receipts_xlsx(), make_previous_receipts_xlsx() + ) + # The first year is this EFO's outturn over the previous EFO's outturn, + # e.g. residential 10.0 / 8.0; later years are this EFO's own ratios. + assert {path[-1]: series.values for path, series in growth.items()} == { + "residential": {2024: 0.25, 2025: 0.1, 2026: 0.1}, + "non_residential": {2024: 0.25, 2025: 0.25, 2026: -0.1}, + "great_britain": {2024: 0.034483, 2025: 0.05, 2026: 0.05}, + "northern_ireland": {2024: 0.025641, 2025: 0.025, 2026: 0.02}, + } + + +def test_the_previous_efo_must_publish_outturn_for_the_base_year(): + # An EFO whose first outturn year is 2022-23 has 2023-24 as a forecast. + two_years_back = make_previous_receipts_xlsx(first_year=2022) + with pytest.raises(ValueError, match="no 2023-24 outturn"): + compute_receipts_growth(make_receipts_xlsx(), two_years_back) + # An EFO from the same outturn year has no 2023-24 at all. + with pytest.raises(ValueError, match="no 2023-24 outturn"): + compute_receipts_growth(make_receipts_xlsx(), make_receipts_xlsx()) + + +def test_the_first_year_must_be_outturn(): + no_outturn = make_receipts_xlsx(outturn_years=0) + with pytest.raises( + ValueError, + match="residential: 2024-25, the first year in Table 3.7, is not outturn", + ): + compute_receipts_growth(no_outturn, make_previous_receipts_xlsx()) + + +@pytest.mark.parametrize( + "sdlt_rows, message", + [ + ([("SDLT from non-residential property", (4.0, 5.0, 4.5))], "found none"), + ( + [ + ("SDLT from residential property", (10.0, 11.0, 12.1)), + ("SDLT from residential property", (10.0, 11.0, 12.1)), + ("SDLT from non-residential property", (4.0, 5.0, 4.5)), + ], + "found sheet 3.7, sheet 3.7", + ), + ], +) +def test_each_receipts_line_must_appear_exactly_once(sdlt_rows, message): + with pytest.raises(ValueError, match=message): + compute_receipts_growth( + make_receipts_xlsx(sdlt_rows=sdlt_rows), make_previous_receipts_xlsx() + ) + + +def test_receipts_growth_rejects_a_missing_or_non_positive_level(): + missing = make_receipts_xlsx( + sdlt_rows=[ + ("SDLT from residential property", (10.0, 11.0)), + ("SDLT from non-residential property", (4.0, 5.0, 4.5)), + ] + ) + with pytest.raises(ValueError, match="no numeric 2026-27 .* value: None"): + compute_receipts_growth(missing, make_previous_receipts_xlsx()) + zero = make_receipts_xlsx( + sdlt_rows=[ + ("SDLT from residential property", (10.0, 0.0, 12.1)), + ("SDLT from non-residential property", (4.0, 5.0, 4.5)), + ] + ) + with pytest.raises(ValueError, match="cannot grow from 2025-26"): + compute_receipts_growth(zero, make_previous_receipts_xlsx()) + + +def test_receipts_labels_lose_footnote_markers_only(): + line = find_receipts_line( + read_workbook_sheets( + make_receipts_xlsx( + sdlt_rows=[ + ("SDLT from residential property2", (10.0, 11.0, 12.1)), + ("SDLT from non-residential property", (4.0, 5.0, 4.5)), + ] + ) + ), + RECEIPTS_SERIES_SPECS[0], + ) + assert line.label == "SDLT from residential property" + + +def test_a_non_numeric_level_is_refused(): + text = make_receipts_xlsx( + sdlt_rows=[ + ("SDLT from residential property", (10.0, "[x]", 12.1)), + ("SDLT from non-residential property", (4.0, 5.0, 4.5)), + ] + ) + with pytest.raises(ValueError, match="no numeric 2025-26 .* value: '\\[x\\]'"): + compute_receipts_growth(text, make_previous_receipts_xlsx()) + + +def test_render_rewrites_values_and_references(): + rendered = render() + start = rendered.index(" residential:") + end = rendered.index(" non_residential:") + assert rendered[start:end] == ( + """ residential: + description: residential receipts growth. + values: + # Outturn (2023-24 over 2022-23), from an older EFO + 2023-01-01: 0.0500 + # Outturn (2024-25 over 2023-24) from OBR EFO March 2026 over March 2025, receipts Table 3.7 "SDLT from residential property" + 2024-01-01: 0.2500 + # Forecast (2025-2026) from OBR EFO March 2026, receipts Table 3.7 + 2025-01-01: 0.1000 + 2026-01-01: 0.1000 + # Beyond the OBR forecast: the average_earnings long-run path above + 2027-01-01: 0.0000 + 2031-01-01: 0.0200 + metadata: + unit: /1 + label: residential receipts growth + reference: + - title: OBR EFO March 2026 (detailed forecast tables, receipts, Table 3.7) + href: https://obr.uk/efo/economic-and-fiscal-outlook-march-2026/ + - title: OBR EFO March 2025 (detailed forecast tables, receipts, Table 3.7; 2023-24 outturn) + href: https://obr.uk/efo/economic-and-fiscal-outlook-march-2025/ +""" + ) + assert ( + "# Outturn (2024-25 over 2023-24) from OBR EFO March 2026 over March 2025, " + 'receipts Table 3.8 (March 2025: Table 3.9) "Business rates"' + ) in rendered + assert "receipts, Table 3.9; 2023-24 outturn" in rendered + + +def test_render_changes_only_the_receipts_values_and_references(): + rendered = render() + before, after = yaml.safe_load(YOY_GROWTH_FIXTURE), yaml.safe_load(rendered) + for tree in (before, after): + for path in RECEIPTS_PATHS.values(): + del _series(tree, path)["values"] + del _series(tree, path)["metadata"]["reference"] + assert after == before + # Text outside the series' values and reference blocks is untouched. + assert rendered.startswith( + YOY_GROWTH_FIXTURE[: YOY_GROWTH_FIXTURE.index(" residential:")] + ) + tail = YOY_GROWTH_FIXTURE[YOY_GROWTH_FIXTURE.index("\n consumer_price_index:") :] + assert rendered.endswith(tail) + assert ( + "# Receipts growth; this comment and the blank lines around it stay." + in rendered + ) + + +def test_render_is_idempotent(): + once = render() + assert render(once) == once + + +def test_a_later_efo_keeps_the_years_before_its_outturn_year(): + """The next EFO's first year is 2025: 2024 keeps its value and comment, + so each index keeps its base year.""" + march_2026 = render() + march_2027 = render( + march_2026, + current=make_receipts_xlsx( + release="March 2027", + first_year=2025, + levels={key: (11.0, 12.1, 13.31) for key in RECEIPTS_LEVELS}, + ), + previous=make_receipts_xlsx(), + release=("March", 2027), + previous_release=("March", 2026), + ) + path = RECEIPTS_PATHS["residential"] + assert _values(march_2027, path) == { + 2023: 0.05, + 2024: 0.25, + # 2025-26 outturn (11.0) over 2024-25 outturn in March 2026 (10.0). + 2025: 0.1, + 2026: 0.1, + 2027: 0.1, + 2028: 0.0, + 2031: 0.02, + } + assert ( + "# Outturn (2024-25 over 2023-24) from OBR EFO March 2026 over March 2025, " + 'receipts Table 3.7 "SDLT from residential property"\n' + " 2024-01-01: 0.2500\n" + " # Outturn (2025-26 over 2024-25) from OBR EFO March 2027 over March 2026" + ) in march_2027 + assert "March 2025 (detailed" not in march_2027 + + +def test_later_outturn_years_get_their_own_comment(): + rendered = render(current=make_receipts_xlsx(outturn_years=2)) + assert ( + "# Outturn (2025) from OBR EFO March 2026, receipts Table 3.7\n" + " 2025-01-01: 0.1000\n" + " # Forecast (2026) from OBR EFO March 2026, receipts Table 3.7\n" + ) in rendered + + +def test_average_earnings_path_lists_each_change_after_the_forecast(): + content = """obr: + average_earnings: + values: + 2029-01-01: 0.022 + 2030-01-01: 0.024 + 2031-01-01: 0.0332 + 2032-01-01: 0.0371 + 2036-01-01: 0.0383 + 2037-01-01: 0.0383 + 2073-01-01: 0.0383 +""" + assert average_earnings_path(content, 2030) == { + 2031: 0.0332, + 2032: 0.0371, + 2036: 0.0383, + } + # Starting between entries takes the value then in effect. + assert average_earnings_path(content, 2033) == {2034: 0.0371, 2036: 0.0383} + with pytest.raises(ValueError, match="no value in effect in 2028"): + average_earnings_path(content, 2027) + + +@pytest.mark.parametrize( + "value, text", + [ + (0.0383, "0.0383"), + (0.038, "0.0380"), + (0.1302, "0.1302"), + (0.06083, "0.06083"), + (0.211202, "0.211202"), + (-0.1, "-0.1000"), + (1e-05, "0.00001"), + (-0.0, "0.0000"), + ], +) +def test_growth_is_written_in_fixed_point(value, text): + assert format_growth(value) == text + + +@settings(max_examples=300, deadline=None, derandomize=True) +@given(st.floats(min_value=-1, max_value=100, allow_nan=False)) +def test_written_growth_reads_back_exactly(value): + text = format_growth(value) + assert "e" not in text.lower() + assert len(text.partition(".")[2]) >= 4 + assert yaml.safe_load(f"x: {text}")["x"] == value + + +def test_read_calendar_values_includes_nested_series(tmp_path): + path = tmp_path / "yoy_growth.yaml" + path.write_text(YOY_GROWTH_FIXTURE) + values = read_calendar_values(path) + assert values["stamp_duty_land_tax.residential"] == { + 2023: 0.05, + 2024: 0.01, + 2025: 0.01, + 2031: 0.02, + } + assert set(values) == set(SERIES_KEYS) | { + ".".join(path) for path in RECEIPTS_PATHS.values() + } + + +LEVELS = st.floats(min_value=0.01, max_value=1_000, allow_nan=False) + + +@settings(max_examples=40, deadline=None, derandomize=True) +@given( + years=st.integers(2, 8), + data=st.data(), +) +def test_rendered_growth_compounds_the_published_levels(years, data): + """For any published levels: every year from the outturn year is the + six-decimal ratio of consecutive levels, every later year equals + average_earnings, years before the outturn year are kept, and a second + run changes nothing.""" + levels = { + key: tuple(data.draw(st.lists(LEVELS, min_size=years, max_size=years))) + for key in RECEIPTS_LEVELS + } + base = {key: (data.draw(LEVELS),) for key in RECEIPTS_LEVELS} + current = make_receipts_xlsx(levels=levels) + previous = make_previous_receipts_xlsx(levels=base) + rendered = render(current=current, previous=previous) + + average_earnings = _values(rendered, ("average_earnings",)) + last = 2024 + years - 1 + for key, path in RECEIPTS_PATHS.items(): + values = _values(rendered, path) + published = { + 2023: base[key][0], + **dict(zip(range(2024, last + 1), levels[key])), + } + for year in range(2024, last + 1): + expected = round(published[year] / published[year - 1] - 1, 6) + assert values[year] == expected + for year in range(last + 1, 2050): + assert _in_effect(values, year) == _in_effect(average_earnings, year) + assert values[2023] == 0.05 + assert render(rendered, current=current, previous=previous) == rendered + + +# Published OBR receipts levels as the workbooks hold them (£bn): the March +# 2026 EFO from 2024-25 and the March 2025 EFO from 2023-24. +MARCH_2026_RECEIPTS = { + "residential": ("10.379999999999999", "11.731471524432562", "13.428906667032589", "14.984943814530874", "16.673948133476184", "17.964513468126285", "19.057301113969185"), + "non_residential": ("3.5049999999999999", "3.6750451743530199", "4.72127759316189", "5.3288288104799904", "5.9009139798817092", "6.5330695355310402", "7.1046581772762902"), + "great_britain": ("32.376814327646052", "34.051963932489258", "37.413586572144581", "38.240930947824609", "39.111850954748434", "42.419067764546902", "42.463193622519263"), + "northern_ireland": ("0.38233350730688931", "0.39471664406441903", "0.40263364127921164", "0.41045270712298504", "0.41810894612429189", "0.42590403232705243", "0.43449009714292341"), +} # fmt: skip +MARCH_2025_RECEIPTS = { + "residential": ("8.57", "10.067621905140079", "11.260655778417849", "13.448356441943533", "15.629462727222798", "17.837320625934193", "19.309303893346414"), + "non_residential": ("3.0449999999999999", "3.51897590165244", "2.8597545561212101", "3.7329253327014698", "4.2294622915006306", "4.6980571035732099", "5.2115072362568595"), + "great_britain": ("29.599814327646055", "32.080747417794186", "34.009749097370083", "37.622908483210999", "37.878733699384433", "38.522006059307493", "39.56809170547703"), + "northern_ireland": ("0.37029404557706441", "0.38436854999535813", "0.39453673994468591", "0.40107161940999719", "0.40924102844267779", "0.41720986951771938", "0.42505803014305893"), +} # fmt: skip + + +def test_march_2026_tables_reproduce_the_committed_series(): + """yoy_growth.yaml holds exactly what the importer writes from the March + 2026 receipts tables (Tables 3.7, 3.8 and 3.2) and the March 2025 ones + (3.7, 3.9 and 3.2): rerunning it on the committed file changes nothing.""" + committed = get_yoy_growth_path().read_text() + rendered = render( + committed, + current=make_receipts_xlsx(levels=MARCH_2026_RECEIPTS), + previous=make_previous_receipts_xlsx(levels=MARCH_2025_RECEIPTS), + ) + assert rendered == committed + + +def test_workbook_release_comes_from_the_contents_sheet(): + assert infer_workbook_release(make_receipts_xlsx()) == ("March", 2026) + assert infer_workbook_release(make_receipts_xlsx(release=None)) is None + assert infer_workbook_release(make_test_xlsx()) is None + + +def test_main_updates_receipts_growth_from_the_new_average_earnings(tmp_path): + yoy, _ = write_tree(tmp_path) + assert run_main(tmp_path, make_test_xlsx(), "--yaml-path", str(yoy)) == 0 + + content = yoy.read_text() + assert _values(content, RECEIPTS_PATHS["great_britain"]) == { + 2023: 0.05, + 2024: 0.034483, + 2025: 0.05, + 2026: 0.05, + # 2026 average earnings as this run writes it, then the long run. + 2027: 0.0333, + 2031: 0.02, + } + assert "2026-01-01: 0.0333" in content + + +def test_main_requires_the_previous_receipts_tables(tmp_path): + yoy, gap_dir = write_tree(tmp_path) + before = {path: path.read_text() for path in [yoy, *gap_dir.iterdir()]} + with pytest.raises(ValueError, match="--skip-receipts-growth"): + run_main(tmp_path, make_test_xlsx(), "--yaml-path", str(yoy), previous=False) + assert {path: path.read_text() for path in before} == before + + assert ( + run_main( + tmp_path, + make_test_xlsx(), + "--yaml-path", + str(yoy), + "--skip-receipts-growth", + previous=False, + ) + == 0 + ) + content = yoy.read_text() + assert "2026-01-01: 0.0248" in content + for path in RECEIPTS_PATHS.values(): + assert _values(content, path) == _values(YOY_GROWTH_FIXTURE, path) + + +def test_main_dry_run_without_previous_receipts_only_warns(tmp_path, capsys): + yoy, _ = write_tree(tmp_path) + assert ( + run_main( + tmp_path, + make_test_xlsx(), + "--yaml-path", + str(yoy), + "--dry-run", + previous=False, + ) + == 0 + ) + assert "Warning: Pass the receipts tables" in capsys.readouterr().out + assert yoy.read_text() == YOY_GROWTH_FIXTURE + + +@pytest.mark.parametrize( + "receipts, previous_receipts, message", + [ + ( + make_receipts_xlsx(release="November 2025"), + None, + "receipts tables are from November 2025 but the economy tables are " + "from March 2026", + ), + ( + None, + make_previous_receipts_xlsx(release="March 2026"), + r"previous receipts tables \(March 2026\) must come from an EFO " + "before March 2026", + ), + ( + None, + make_previous_receipts_xlsx(release=None), + "Could not tell which EFO the previous receipts tables are from", + ), + ], +) +def test_main_refuses_tables_from_the_wrong_efo( + tmp_path, receipts, previous_receipts, message +): + yoy, gap_dir = write_tree(tmp_path) + before = {path: path.read_text() for path in [yoy, *gap_dir.iterdir()]} + with pytest.raises(ValueError, match=message): + run_main( + tmp_path, + make_test_xlsx(), + "--yaml-path", + str(yoy), + receipts=receipts, + previous_receipts=previous_receipts, + ) + assert {path: path.read_text() for path in before} == before + + +def test_gaps_only_refuses_the_previous_receipts_tables(tmp_path): + yoy, _ = write_tree(tmp_path) + with pytest.raises(ValueError, match="--gaps-only leaves yoy_growth.yaml alone"): + run_main(tmp_path, make_test_xlsx(), "--yaml-path", str(yoy), "--gaps-only") + + +def test_main_writes_receipts_growth_to_the_output_path(tmp_path): + yoy, _ = write_tree(tmp_path) + output = tmp_path / "out.yaml" + assert ( + run_main( + tmp_path, + make_test_xlsx(), + "--yaml-path", + str(yoy), + "--output", + str(output), + "--skip-statutory-gaps", + ) + == 0 + ) + assert yoy.read_text() == YOY_GROWTH_FIXTURE + assert _values(output.read_text(), RECEIPTS_PATHS["residential"])[2024] == 0.25 diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index cba8bce113..bfda5ab755 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -1,10 +1,13 @@ from __future__ import annotations import argparse +import copy import os import re import shutil from dataclasses import dataclass +from datetime import date +from decimal import Decimal from io import BytesIO from pathlib import Path from urllib.request import Request, urlopen @@ -18,10 +21,21 @@ "pkgrel": "http://schemas.openxmlformats.org/package/2006/relationships", } YEAR_RE = re.compile(r"^\d{4}$") -MONTH_RE = re.compile( - r"(january|february|march|april|may|june|july|august|september|october|november|december)[-_ ]+(\d{4})", - re.IGNORECASE, +MONTHS = ( + "january", + "february", + "march", + "april", + "may", + "june", + "july", + "august", + "september", + "october", + "november", + "december", ) +MONTH_RE = re.compile(rf"({'|'.join(MONTHS)})[-_ ]+(\d{{4}})", re.IGNORECASE) @dataclass(frozen=True) @@ -451,16 +465,27 @@ def compute_statutory_forecast_gaps( def read_calendar_values(yaml_path: Path) -> dict[str, dict[int, float]]: - """Calendar-year growth by year for each series in a yoy_growth.yaml.""" - obr = yaml.safe_load(yaml_path.read_text())["obr"] - return { - key: { - int(str(instant)[:4]): float(value) - for instant, value in series["values"].items() - } - for key, series in obr.items() - if isinstance(series, dict) and "values" in series - } + """Calendar-year growth by year for each series in a yoy_growth.yaml. + + Nested series are keyed by their path under ``obr`` joined with dots, + such as ``stamp_duty_land_tax.residential``. + """ + return _calendar_values(yaml.safe_load(yaml_path.read_text())["obr"]) + + +def _calendar_values(node: dict, prefix: str = "") -> dict[str, dict[int, float]]: + result = {} + for key, child in node.items(): + if not isinstance(child, dict) or key == "metadata": + continue + if "values" in child: + result[prefix + key] = { + int(str(instant)[:4]): float(value) + for instant, value in child["values"].items() + } + else: + result.update(_calendar_values(child, f"{prefix}{key}.")) + return result def format_gap(value: float) -> str: @@ -559,6 +584,491 @@ def update_forecast_gap_yaml( ) +TABLE_TITLE_RE = re.compile(r"^(\d+\.\d+)\s+(.+)$") +FOOTNOTE_RE = re.compile(r"(?<=[a-z])\d+$", re.IGNORECASE) +VALUE_LINE_RE = re.compile(r"^\s*(\d{4})-\d{2}-\d{2}:") + + +@dataclass(frozen=True) +class ReceiptsSeriesSpec: + """A line of the OBR receipts tables, grown by a yoy_growth.yaml series. + + ``path`` is the series' place under ``obr``. The table is found by its + title, not its number, because numbers move between EFOs: current + receipts on a cash basis are Table 3.8 in March 2026 and 3.9 in March + 2025. Titles and row labels are compared lowercased, without footnote + markers. + """ + + path: tuple[str, ...] + title: str + row: str + + +RECEIPTS_SERIES_SPECS = ( + ReceiptsSeriesSpec( + path=("stamp_duty_land_tax", "residential"), + title="property transactions taxes: receipts by sector", + row="sdlt from residential property", + ), + ReceiptsSeriesSpec( + path=("stamp_duty_land_tax", "non_residential"), + title="property transactions taxes: receipts by sector", + row="sdlt from non-residential property", + ), + ReceiptsSeriesSpec( + path=("business_rates", "great_britain"), + title="current receipts (on a cash basis)", + row="business rates", + ), + ReceiptsSeriesSpec( + path=("business_rates", "northern_ireland"), + title="other taxes", + row="northern ireland business rates", + ), +) + + +@dataclass(frozen=True) +class ReceiptsLine: + """A receipts line as one EFO publishes it: £bn by the calendar year in + which each fiscal year starts, and which of those years are outturn.""" + + table: str + label: str + levels: dict[int, float] + outturn_years: frozenset[int] + + +@dataclass(frozen=True) +class ReceiptsGrowth: + """Growth in a receipts line, by year, rounded to six decimals. + + The first year is outturn growth: this EFO's first (outturn) year over + the year before in the previous EFO, where it is outturn. Each later + year is the ratio of this EFO's consecutive levels. + """ + + values: dict[int, float] + line: ReceiptsLine + previous_line: ReceiptsLine + + +def fiscal_year_label(year: int) -> str: + return f"{year}-{str(year + 1)[2:]}" + + +def column_index(column: str) -> int: + index = 0 + for letter in column: + index = index * 26 + ord(letter.upper()) - ord("A") + 1 + return index + + +def receipts_label(value: str | None) -> str: + return FOOTNOTE_RE.sub("", normalise_label(value)) + + +def read_workbook_sheets(xlsx_bytes: bytes) -> dict[str, list[dict[str, str | None]]]: + with ZipFile(BytesIO(xlsx_bytes)) as archive: + sheet_names = list(_sheet_paths(archive)) + return {sheet: read_sheet_rows(xlsx_bytes, sheet) for sheet in sheet_names} + + +def table_title(rows: list[dict[str, str | None]]) -> tuple[str, str] | None: + """The number and normalised title of a table, e.g. ("3.8", "current + receipts (on a cash basis)"), from the first title cell in its top rows.""" + for row in rows[:3]: + for column in sorted(row, key=column_index): + match = TABLE_TITLE_RE.match(" ".join(str(row[column] or "").split())) + if match: + return match.group(1), receipts_label(match.group(2)) + return None + + +def _fiscal_year_columns(row: dict[str, str | None]) -> dict[str, int]: + columns = {} + for column, value in row.items(): + match = FISCAL_YEAR_RE.match(str(value or "").strip()) + if match and int(match.group(2)) == (int(match.group(1)) + 1) % 100: + columns[column] = int(match.group(1)) + return columns + + +def _outturn_years( + years: dict[str, int], markers: dict[str, int], where: str +) -> frozenset[int]: + """Years from the "Outturn" marker's column up to the "Forecast" one. + + A table without an "Outturn" marker has no outturn years, and one + without a "Forecast" marker no forecast years. + """ + if not markers: + raise ValueError(f"{where} does not mark its outturn and forecast years") + if "outturn" not in markers: + return frozenset() + end = markers.get("forecast", float("inf")) + return frozenset( + year + for column, year in years.items() + if markers["outturn"] <= column_index(column) < end + ) + + +def find_receipts_line( + sheets: dict[str, list[dict[str, str | None]]], spec: ReceiptsSeriesSpec +) -> ReceiptsLine: + """The one row labelled ``spec.row`` in the table titled ``spec.title``.""" + matches = [] + for sheet, rows in sheets.items(): + title = table_title(rows) + if title is None or title[1] != spec.title: + continue + years: dict[str, int] = {} + markers: dict[str, int] = {} + for row in rows: + row_years = _fiscal_year_columns(row) + if row_years: + years = row_years + continue + row_markers = { + normalise_label(value): column_index(column) + for column, value in row.items() + if normalise_label(value) in ("outturn", "forecast") + } + if row_markers: + markers = row_markers + continue + if not years: + continue + first_year_column = min(column_index(column) for column in years) + labels = [ + value + for column, value in sorted( + row.items(), key=lambda item: column_index(item[0]) + ) + if column_index(column) < first_year_column and normalise_label(value) + ] + if labels and receipts_label(labels[0]) == spec.row: + matches.append((title[0], sheet, labels[0], row, years, markers)) + + name = ".".join(spec.path) + if len(matches) != 1: + found = ", ".join(f"sheet {match[1]}" for match in matches) or "none" + raise ValueError( + f'Expected one "{spec.row}" row in the "{spec.title}" table for ' + f"{name}; found {found}" + ) + table, sheet, label, row, years, markers = matches[0] + where = f"Table {table} (sheet {sheet})" + levels = {} + for column, year in years.items(): + text = row.get(column) + try: + levels[year] = float(text) + except (TypeError, ValueError): + raise ValueError( + f"{where} has no numeric {fiscal_year_label(year)} {name} value: " + f"{text!r}" + ) from None + ordered = sorted(levels) + if ordered != list(range(ordered[0], ordered[-1] + 1)): + raise ValueError(f"{where} skips a year: {ordered}") + return ReceiptsLine( + table=table, + label=FOOTNOTE_RE.sub("", " ".join(str(label).split())), + levels=levels, + outturn_years=_outturn_years(years, markers, where), + ) + + +def compute_receipts_growth( + xlsx_bytes: bytes, previous_xlsx_bytes: bytes +) -> dict[tuple[str, ...], ReceiptsGrowth]: + """Growth in each receipts line, from an EFO's receipts tables and an + earlier EFO's, whose outturn is the base for the first year's growth. + + The first year must be outturn in this EFO, and the year before it + outturn in the earlier one (March 2025 for March 2026). + """ + sheets = read_workbook_sheets(xlsx_bytes) + previous_sheets = read_workbook_sheets(previous_xlsx_bytes) + result = {} + for spec in RECEIPTS_SERIES_SPECS: + name = ".".join(spec.path) + line = find_receipts_line(sheets, spec) + previous = find_receipts_line(previous_sheets, spec) + first_year = min(line.levels) + base_year = first_year - 1 + if first_year not in line.outturn_years: + raise ValueError( + f"{name}: {fiscal_year_label(first_year)}, the first year in " + f"Table {line.table}, is not outturn" + ) + if base_year not in previous.outturn_years: + raise ValueError( + f"{name}: the previous receipts tables have no " + f"{fiscal_year_label(base_year)} outturn; pass the EFO whose " + f"outturn year is {fiscal_year_label(base_year)}" + ) + levels = {base_year: previous.levels[base_year], **line.levels} + values = {} + for year in sorted(line.levels): + if levels[year - 1] <= 0: + raise ValueError( + f"{name}: cannot grow from {fiscal_year_label(year - 1)} " + f"receipts of {levels[year - 1]}" + ) + values[year] = round(levels[year] / levels[year - 1] - 1, 6) + 0.0 + result[spec.path] = ReceiptsGrowth(values, line, previous) + return result + + +def format_growth(value: float) -> str: + """Fixed-point text with at least four decimals that reads back as the + same float: 0.038 is written 0.0380, 1e-05 as 0.00001.""" + text = format(Decimal(repr(value + 0.0)), "f") + whole, _, decimals = text.partition(".") + return f"{whole}.{decimals.ljust(4, '0')}" + + +def average_earnings_path(content: str, after_year: int) -> dict[int, float]: + """``average_earnings`` from the year after ``after_year``: its value in + effect then and each later change, so a series given these values equals + average_earnings in every later year.""" + values = yaml.safe_load(content)["obr"]["average_earnings"]["values"] + entries = sorted((int(str(instant)[:4]), float(v)) for instant, v in values.items()) + start = after_year + 1 + in_effect = [value for year, value in entries if year <= start] + if not in_effect: + raise ValueError(f"average_earnings has no value in effect in {start}") + path = {start: in_effect[-1]} + for year, value in entries: + if year > start and value != path[max(path)]: + path[year] = value + return path + + +def _indent(line: str) -> int: + return len(line) - len(line.lstrip(" ")) + + +def _yaml_block( + lines: list[str], key: str, indent: int, start: int, end: int +) -> tuple[int, int]: + """Lines [first, last) of the mapping entry ``key`` at ``indent`` within + lines[start:end], from its key through its last non-blank child line.""" + pattern = re.compile(rf"^ {{{indent}}}{re.escape(key)}:(\s|$)") + for first in range(start, end): + if not pattern.match(lines[first]): + continue + last = first + 1 + for index in range(first + 1, end): + if not lines[index].strip(): + continue + if _indent(lines[index]) <= indent: + break + last = index + 1 + return first, last + raise ValueError(f"Could not find {key} at indent {indent}") + + +def _series_block(lines: list[str], path: tuple[str, ...]) -> tuple[int, int]: + start, end = 0, len(lines) + for depth, key in enumerate(path): + start, end = _yaml_block(lines, key, 2 * depth, start, end) + return start, end + + +def render_receipts_values( + old_lines: list[str], + indent: int, + growth: ReceiptsGrowth, + tail: dict[int, float], + release: tuple[str, int], + previous_release: tuple[str, int], +) -> list[str]: + """A series' values entries: earlier years as they were, then this + EFO's growth, then the average_earnings path.""" + first_year = min(growth.values) + kept: list[str] = [] + pending: list[str] = [] + for line in old_lines: + match = VALUE_LINE_RE.match(line) + if match is None: + if line.strip() and not line.strip().startswith("#"): + raise ValueError(f"Unexpected line in a values block: {line!r}") + pending.append(line) + continue + if int(match.group(1)) < first_year: + kept += pending + [line] + pending = [] + + month, year = release + previous_month, previous_year = previous_release + published, previous = growth.line, growth.previous_line + table = f"Table {published.table}" + if previous.table != published.table: + table += f" ({previous_month} {previous_year}: Table {previous.table})" + pad = " " * indent + lines = kept + [ + f"{pad}# Outturn ({fiscal_year_label(first_year)} over " + f"{fiscal_year_label(first_year - 1)}) from OBR EFO {month} {year} over " + f'{previous_month} {previous_year}, receipts {table} "{published.label}"', + f"{pad}{first_year}-01-01: {format_growth(growth.values[first_year])}", + ] + later = sorted(y for y in growth.values if y > first_year) + for kind in ("Outturn", "Forecast"): + years = [ + y for y in later if (y in published.outturn_years) == (kind == "Outturn") + ] + if not years: + continue + span = f"{years[0]}" if len(years) == 1 else f"{years[0]}-{years[-1]}" + lines.append( + f"{pad}# {kind} ({span}) from OBR EFO {month} {year}, receipts " + f"Table {published.table}" + ) + lines += [f"{pad}{y}-01-01: {format_growth(growth.values[y])}" for y in years] + lines.append( + f"{pad}# Beyond the OBR forecast: the average_earnings long-run path above" + ) + lines += [f"{pad}{y}-01-01: {format_growth(v)}" for y, v in sorted(tail.items())] + return lines + + +def receipts_references( + growth: ReceiptsGrowth, + release: tuple[str, int], + previous_release: tuple[str, int], +) -> list[dict[str, str]]: + month, year = release + previous_month, previous_year = previous_release + base_year = min(growth.values) - 1 + return [ + { + "title": f"OBR EFO {month} {year} (detailed forecast tables, receipts, " + f"Table {growth.line.table})", + "href": build_efo_href(month, year), + }, + { + "title": f"OBR EFO {previous_month} {previous_year} (detailed forecast " + f"tables, receipts, Table {growth.previous_line.table}; " + f"{fiscal_year_label(base_year)} outturn)", + "href": build_efo_href(previous_month, previous_year), + }, + ] + + +def render_receipts_growth_yaml( + content: str, + growth: dict[tuple[str, ...], ReceiptsGrowth], + release: tuple[str, int], + previous_release: tuple[str, int], +) -> str: + """Rewrite each receipts series' values and references in a + yoy_growth.yaml. + + Years before this EFO's first year are kept, so a series' index keeps + its base year. Years after the OBR's receipts forecast follow + average_earnings as ``content`` holds it. The result is checked to + differ from ``content`` only in those values and references. + """ + before = yaml.safe_load(content) + expected = copy.deepcopy(before) + lines = content.split("\n") + for path, series in growth.items(): + tail = average_earnings_path(content, max(series.values)) + references = receipts_references(series, release, previous_release) + + start, end = _series_block(lines, ("obr", *path)) + indent = 2 * len(path) + 2 + values_start, values_end = _yaml_block(lines, "values", indent, start, end) + old_values = lines[values_start + 1 : values_end] + entry_indent = next( + (_indent(line) for line in old_values if VALUE_LINE_RE.match(line)), + indent + 2, + ) + new_values = render_receipts_values( + old_values, entry_indent, series, tail, release, previous_release + ) + lines[values_start + 1 : values_end] = new_values + + start, end = _series_block(lines, ("obr", *path)) + metadata = _yaml_block(lines, "metadata", indent, start, end) + ref_start, ref_end = _yaml_block( + lines, "reference", indent + 2, metadata[0], metadata[1] + ) + pad = " " * (indent + 2) + lines[ref_start:ref_end] = [f"{pad}reference:"] + [ + text + for reference in references + for text in ( + f"{pad} - title: {reference['title']}", + f"{pad} href: {reference['href']}", + ) + ] + + node = expected["obr"] + for key in path: + node = node[key] + kept = { + instant: value + for instant, value in node["values"].items() + if int(str(instant)[:4]) < min(series.values) + } + node["values"] = { + **kept, + **{ + date(year, 1, 1): value + for year, value in {**series.values, **tail}.items() + }, + } + node["metadata"]["reference"] = references + + rendered = "\n".join(lines) + if yaml.safe_load(rendered) != expected: + raise ValueError( + "Rewriting the receipts growth series changed more than their " + "values and references" + ) + return rendered + + +def infer_workbook_release(xlsx_bytes: bytes) -> tuple[str, int] | None: + """The release a detailed forecast tables workbook names on its contents + sheet, e.g. "March 2026 Economic and fiscal outlook - detailed forecast + tables", or None.""" + with ZipFile(BytesIO(xlsx_bytes)) as archive: + sheet_names = list(_sheet_paths(archive)) + contents = next((s for s in sheet_names if s.lower() == "contents"), None) + if contents is None: + return None + for row in read_sheet_rows(xlsx_bytes, contents)[:5]: + for value in row.values(): + text = str(value or "") + match = MONTH_RE.search(text) + if match and "outlook" in text.lower(): + return match.group(1).capitalize(), int(match.group(2)) + return None + + +def resolve_release(xlsx_bytes: bytes, source_name: str) -> tuple[str, int] | None: + """A workbook's release from its contents sheet, else from its name.""" + release = infer_workbook_release(xlsx_bytes) + if release is not None: + return release + try: + return infer_release(source_name) + except ValueError: + return None + + +def release_order(release: tuple[str, int]) -> tuple[int, int]: + month, year = release + return year, MONTHS.index(month.lower()) + + def infer_release(source_name: str) -> tuple[str, int]: match = MONTH_RE.search(source_name) if not match: @@ -710,6 +1220,16 @@ def print_summary( print(f"- {spec.key}: {', '.join(window)}") +def print_receipts_summary(growth: dict[tuple[str, ...], ReceiptsGrowth]) -> None: + print(f"Updating {len(growth)} OBR receipts growth series") + for path, series in growth.items(): + window = ", ".join( + f"{year}: {format_growth(value)}" + for year, value in sorted(series.values.items()) + ) + print(f"- {'.'.join(path)} (Table {series.line.table}): {window}") + + def build_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Import OBR economy forecast tables into yoy_growth.yaml.", @@ -756,6 +1276,25 @@ def build_arg_parser() -> argparse.ArgumentParser: ), ) parser.add_argument("--receipts-file", help="Local receipts XLSX file path") + parser.add_argument( + "--previous-receipts-url", + help=( + "Receipts detailed forecast tables (XLSX) from the EFO whose outturn " + "year is the year before this one's (March 2025 for March 2026), " + "the base for the receipts growth series' first year" + ), + ) + parser.add_argument( + "--previous-receipts-file", help="Local previous receipts XLSX file path" + ) + parser.add_argument( + "--skip-receipts-growth", + action="store_true", + help=( + "Do not update the receipts growth series (stamp_duty_land_tax, " + "business_rates) that project corporate SDLT and business rates" + ), + ) parser.add_argument( "--gaps-only", action="store_true", @@ -845,6 +1384,10 @@ def main(argv: list[str] | None = None) -> int: series_values = extract_annual_series_from_xlsx(workbook_bytes) if args.release_month and args.release_year: + if args.release_month.lower() not in MONTHS: + raise ValueError( + f"--release-month must be a month name, not {args.release_month!r}" + ) month = args.release_month.capitalize() year = args.release_year elif args.release_month or args.release_year: @@ -867,11 +1410,62 @@ def main(argv: list[str] | None = None) -> int: if not args.dry_run: raise ValueError(message) print(f"Warning: {message}") + has_previous_receipts = bool( + args.previous_receipts_url or args.previous_receipts_file + ) + if args.gaps_only and has_previous_receipts: + raise ValueError( + "--gaps-only leaves yoy_growth.yaml alone, so it cannot update the " + "receipts growth series from --previous-receipts-file or " + "--previous-receipts-url" + ) + write_receipts = not args.gaps_only and not args.skip_receipts_growth + if write_receipts and not ( + has_previous_receipts and (args.receipts_url or args.receipts_file) + ): + message = ( + "Pass the receipts tables (--receipts-file or --receipts-url) and the " + "previous EFO's (--previous-receipts-file or --previous-receipts-url) " + "so the receipts growth series move to this EFO, or " + "--skip-receipts-growth" + ) + if not args.dry_run: + raise ValueError(message) + print(f"Warning: {message}") + write_receipts = False print_summary(series_values, forecast_start_year, args.forecast_years) receipts_bytes = None if args.receipts_url or args.receipts_file: - _, receipts_bytes = load_source_bytes(args.receipts_url, args.receipts_file) + receipts_name, receipts_bytes = load_source_bytes( + args.receipts_url, args.receipts_file + ) + receipts_release = resolve_release(receipts_bytes, receipts_name) + if receipts_release not in (None, (month, year)): + raise ValueError( + "The receipts tables are from " + f"{receipts_release[0]} {receipts_release[1]} but the economy " + f"tables are from {month} {year}" + ) + receipts_growth = None + if write_receipts: + previous_name, previous_bytes = load_source_bytes( + args.previous_receipts_url, args.previous_receipts_file + ) + previous_release = resolve_release(previous_bytes, previous_name) + if previous_release is None: + raise ValueError( + "Could not tell which EFO the previous receipts tables are from; " + "name the file with its release, e.g. receipts_march_2025.xlsx" + ) + if release_order(previous_release) >= release_order((month, year)): + raise ValueError( + "The previous receipts tables " + f"({previous_release[0]} {previous_release[1]}) must come from " + f"an EFO before {month} {year}" + ) + receipts_growth = compute_receipts_growth(receipts_bytes, previous_bytes) + print_receipts_summary(receipts_growth) calendar_values = ( read_calendar_values(source_yaml_path) if args.gaps_only else series_values ) @@ -900,6 +1494,11 @@ def main(argv: list[str] | None = None) -> int: forecast_start_year, args.forecast_years, ) + if receipts_growth is not None: + yoy_path = args.output or source_yaml_path + outputs[yoy_path] = render_receipts_growth_yaml( + outputs[yoy_path], receipts_growth, (month, year), previous_release + ) if write_gaps: gap_dir = get_forecast_gap_dir(source_yaml_path) for spec in STATUTORY_GAP_SPECS: