From 3edfb4b994f32681648ee6dce442da9c819f8a62 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:29:07 -0400 Subject: [PATCH 1/6] Apply dated reforms on the statutory timeline on every route Reform dicts and Reform.from_dict classes now write their changes onto the parameters as loaded from YAML, which are then processed as current law is (uprating, threshold indexation, fiscal-year annualisation). Before, they edited the already-annualised tree, so a change from 6 April missed that fiscal year and derived parameters stayed on the baseline. A bounded change with no stated value after it resumes prior law's projected path rather than restoring the last stated value, which froze the next year of an uprated parameter. Scenario parameter changes now come right after the data load, before the UC rebalancing inputs and parameter-gated structural reforms read the parameters, and before the scenario's modifier. The reprocessed tree keeps current law's baseline child (#2188). Fixes #2304, #2188, #2252, #2167. Co-Authored-By: Claude Opus 5.5 --- changelog.d/2304.fixed.md | 1 + policyengine_uk/simulation.py | 80 +++-- policyengine_uk/tax_benefit_system.py | 209 ++++++++++-- policyengine_uk/utils/dated_reforms.py | 451 +++++++++++++++++++++++++ policyengine_uk/utils/scenario.py | 159 ++++++--- 5 files changed, 789 insertions(+), 111 deletions(-) create mode 100644 changelog.d/2304.fixed.md create mode 100644 policyengine_uk/utils/dated_reforms.py diff --git a/changelog.d/2304.fixed.md b/changelog.d/2304.fixed.md new file mode 100644 index 0000000000..9963594982 --- /dev/null +++ b/changelog.d/2304.fixed.md @@ -0,0 +1 @@ +- Every reform route now writes dated changes onto the statutory parameter timeline and reprocesses it as current law is processed. A dict reform dated 6 April now sets that fiscal year, and uprating, threshold indexation and the aligned NICs thresholds follow reformed values. A bounded change on an uprated or indexed parameter now resumes prior law's projected path, not a stale stated value. Reforms applied after the data load now reach the Universal Credit rebalancing inputs and the parameter-gated structural reforms. `parameters.baseline` stays current law under `Scenario(parameter_changes=...)`. diff --git a/policyengine_uk/simulation.py b/policyengine_uk/simulation.py index 2acef5aefe..17ca9bf1b8 100644 --- a/policyengine_uk/simulation.py +++ b/policyengine_uk/simulation.py @@ -16,6 +16,7 @@ from policyengine_core.tools.hugging_face import download_huggingface_dataset from policyengine_core.tracers import FullTracer, SimpleTracer +from policyengine_uk.utils.dated_reforms import read_parameter_changes from policyengine_uk.utils.parameters import ( check_parameter_not_removed, uk_fiscal_year_period, @@ -45,6 +46,14 @@ from .tax_benefit_system import CountryTaxBenefitSystem, DEFAULT_DATASET_ENV_VAR + +def _canonical_parameter_paths(changes: dict) -> dict: + """Changes keyed by canonical parameter paths (labour supply aliases).""" + return { + canonicalize_lsr_parameter_path(path): value for path, value in changes.items() + } + + from microdf import MicroDataFrame # Cache for fully-loaded, uprated, enum-pre-encoded multi-year datasets keyed @@ -163,12 +172,13 @@ def __init__( self.branches: Dict[str, Simulation] = {} + # A scenario's parameter changes rebuild the parameters from the + # statutory tree, so they come before its modifier, whose edits a + # rebuild would discard. if scenario is not None and scenario.applied_before_data_load: + self.apply_scenario_parameters(scenario) if scenario.simulation_modifier is not None: scenario.simulation_modifier(self) - - if scenario.parameter_changes is not None: - self.apply_parameter_changes(scenario.parameter_changes) # Build simulation from appropriate source if situation is not None: self.build_from_situation(situation) @@ -188,6 +198,14 @@ def __init__( raise ValueError(f"Unsupported dataset type: {dataset.__class__}") self.built_from_dataset = situation is None + # Parameter changes applied after data load leave the data as built + # (a changed economic assumption does not re-uprate it), but come + # before the steps below that read the parameters: the Universal + # Credit rebalancing inputs and the structural reforms switched on by + # parameters. + if scenario is not None and not scenario.applied_before_data_load: + self.apply_scenario_parameters(scenario) + # Universal Credit reform (July 2025). Needs closer integration in the baseline, # but adding here for ease of toggling on/off via the 'active' parameter. from policyengine_uk.scenarios import universal_credit_july_2025_reform @@ -233,8 +251,6 @@ def __init__( if scenario is not None and not scenario.applied_before_data_load: if scenario.simulation_modifier is not None: scenario.simulation_modifier(self) - if scenario.parameter_changes is not None: - self.apply_parameter_changes(scenario.parameter_changes) def delete_arrays(self, variable: str, period: Period = None) -> None: super().delete_arrays(variable, period) @@ -260,28 +276,42 @@ def get_known_variables(self): return variables def apply_parameter_changes(self, changes: dict): - for parameter in changes: - check_parameter_not_removed(canonicalize_lsr_parameter_path(parameter)) - self.tax_benefit_system.reset_parameters() - - for parameter in changes: - canonical_parameter = canonicalize_lsr_parameter_path(parameter) - p: Parameter = self.tax_benefit_system.parameters.get_child( - canonical_parameter + """Rebuild the parameters with ``Scenario.parameter_changes``. + + Each change is written onto the statutory timeline (see + ``policyengine_uk.utils.dated_reforms`` for how its period keys are + read) and the parameters are processed as current law is. + """ + self.apply_parameter_assignments( + read_parameter_changes(_canonical_parameter_paths(changes)) + ) + + def apply_scenario_parameters(self, scenario: Scenario) -> None: + """Rebuild the parameters with a scenario's parameter changes. + + Takes ``parameter_changes`` and then each of the scenario's + ``dated_reforms`` in order, in one rebuild. Changes a dated reform + leaves to the annualised parameters (single-day keys and paths the + statutory tree does not define) are applied by the scenario's + modifier instead. + """ + assignments = [] + if scenario.parameter_changes: + assignments += read_parameter_changes( + _canonical_parameter_paths(scenario.parameter_changes) ) - if isinstance(changes[parameter], dict): - # Time-period specific changes. A bare year names the UK - # fiscal year, so the override covers 6 April to 5 April and - # survives fiscal-year conversion at full strength. - for time_period in changes[parameter]: - p.update( - period=uk_fiscal_year_period(time_period), - value=changes[parameter][time_period], - ) - else: - p.update(period="year:2000:100", value=changes[parameter]) + assignments += scenario.dated_assignments(order_start=len(assignments)) + self.apply_parameter_assignments(assignments) - self.tax_benefit_system.process_parameters() + def apply_parameter_assignments(self, assignments: list) -> None: + """Rebuild the parameters with changes on the statutory timeline.""" + if not assignments: + return + self.tax_benefit_system.apply_parameter_assignments(assignments) + if getattr(self, "populations", None) is not None: + # Building from a situation or dataset can calculate values; + # recompute them under the rebuilt parameters. + self._invalidate_all_caches() def build_from_situation(self, situation: Dict) -> None: """Build simulation from a situation dictionary. diff --git a/policyengine_uk/tax_benefit_system.py b/policyengine_uk/tax_benefit_system.py index 06acdeb86c..e9c3a039e0 100644 --- a/policyengine_uk/tax_benefit_system.py +++ b/policyengine_uk/tax_benefit_system.py @@ -1,9 +1,10 @@ # Standard library imports import copy from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Callable, Dict, List, Optional, Tuple # PolicyEngine core imports +from policyengine_core.parameters import ParameterNode from policyengine_core.parameters.operations.propagate_parameter_metadata import ( propagate_parameter_metadata, ) @@ -42,6 +43,14 @@ from policyengine_uk.parameters.gov.simulation.labour_supply_responses.aliases import ( add_lsr_deprecation_aliases, ) +from policyengine_uk.utils.dated_reforms import ( + ParameterAssignment, + Timeline, + requires_projection, + snapshot_timelines, + tree_defines, + write_assignment, +) from policyengine_uk.utils.parameters import ( add_removed_parameter_aliases, backdate_parameters, @@ -55,6 +64,70 @@ # Cache for fully-processed parameter tree, so convert_to_fiscal_year_parameters # (22,538 param.update() calls) only runs once per process. _processed_parameters_cache = None +# The parameters as loaded from the YAML files, before processing: each +# parameter's statutory values at their legal dates. Reforms are written +# onto a copy of this tree. +_statutory_parameters_cache = None +# Each gov parameter's timeline in the cached processed tree after +# projection (uprating, indexation and the other generated series) and +# before fiscal-year annualisation, by name. +_projected_timelines_cache = None + + +def process_parameter_tree( + parameters: ParameterNode, + baseline_parameters: Optional[ParameterNode] = None, + annualise: bool = True, +) -> Tuple[ParameterNode, Dict[str, Timeline]]: + """Project and annualise a statutory parameter tree. + + Applies, in order: + - private pension uprating factors + - lagged earnings and CPI indices + - statutory uprating inputs (September CPI, May-July earnings) + - triple lock calculations for state pensions + - September CPI uprating + - economic assumption indices + - income tax and NICs threshold indexation after the freeze + - parameter uprating and backdating + - conversion to fiscal year parameters + + Returns the tree and each gov parameter's timeline just before the + conversion to fiscal years. + + ``parameters.baseline`` is the current-law copy that ``baseline_vat`` + and the budget adjustments compare against. Without + ``baseline_parameters`` it is a copy of this tree taken before uprating, + which is current law only when the tree carries no reform. A reformed + tree passes the processed current-law parameters as + ``baseline_parameters``, whose ``baseline`` child it takes (#2188). + ``annualise=False`` stops after the projection and adds no baseline. + """ + parameters = add_private_pension_uprating_factor(parameters) + parameters = add_lagged_earnings(parameters) + parameters = add_lagged_cpi(parameters) + parameters = add_statutory_uprating_inputs(parameters) + parameters = add_triple_lock(parameters) + parameters = add_september_cpi_uprating(parameters) + parameters = create_economic_assumption_indices(parameters) + parameters = add_threshold_indexation(parameters) + parameters = add_lsr_deprecation_aliases(parameters) + + # Create baseline parameters for reform comparisons + if annualise and baseline_parameters is None: + parameters.add_child("baseline", parameters.clone()) + + # Apply general parameter operations + parameters = propagate_parameter_metadata(parameters) + parameters = uprate_parameters(parameters) + parameters = backdate_parameters(parameters, "2015-01-01") + projected = snapshot_timelines(parameters.gov) + if not annualise: + return parameters, projected + parameters.gov = convert_to_fiscal_year_parameters(parameters.gov) + if baseline_parameters is not None: + parameters.add_child("baseline", baseline_parameters.baseline.clone()) + return parameters, projected class CountryTaxBenefitSystem(TaxBenefitSystem): @@ -89,42 +162,47 @@ def reset_parameters(self) -> None: self.load_parameters(self.parameters_dir) self.parameters = add_removed_parameter_aliases(self.parameters) - def process_parameters(self) -> None: + def process_parameters( + self, baseline_parameters: Optional[ParameterNode] = None + ) -> None: """Process and transform parameters with UK-specific adjustments. - Applies various parameter transformations including: - - Private pension uprating factors - - Lagged earnings and CPI indices - - Statutory uprating inputs (September CPI, May-July earnings) - - Triple lock calculations for state pensions - - September CPI uprating - - Economic assumption indices - - Income tax and NICs threshold indexation after the freeze - - Parameter uprating and backdating - - Conversion to fiscal year parameters + See ``process_parameter_tree``, which this runs on the system's + parameters. ``projected_timelines`` keeps each gov parameter's + timeline from before the conversion to fiscal years. """ self._parameters_at_instant_cache = {} - # Add various UK-specific parameter adjustments - self.parameters = add_private_pension_uprating_factor(self.parameters) - self.parameters = add_lagged_earnings(self.parameters) - self.parameters = add_lagged_cpi(self.parameters) - self.parameters = add_statutory_uprating_inputs(self.parameters) - self.parameters = add_triple_lock(self.parameters) - self.parameters = add_september_cpi_uprating(self.parameters) - self.parameters = create_economic_assumption_indices(self.parameters) - self.parameters = add_threshold_indexation(self.parameters) - self.parameters = add_lsr_deprecation_aliases(self.parameters) - - # Create baseline parameters for reform comparisons - self.parameters.add_child("baseline", self.parameters.clone()) - - # Apply general parameter operations - self.parameters = propagate_parameter_metadata(self.parameters) - self.parameters = uprate_parameters(self.parameters) - self.parameters = backdate_parameters(self.parameters, "2015-01-01") - self.parameters.gov = convert_to_fiscal_year_parameters(self.parameters.gov) + self.parameters, self.projected_timelines = process_parameter_tree( + self.parameters, baseline_parameters + ) self.reset_parameter_caches() + def apply_parameter_assignments( + self, assignments: List[ParameterAssignment] + ) -> None: + """Rebuild the parameters with changes on the statutory timeline. + + Writes each change, in ``order``, onto a fresh copy of the + parameters as loaded from the YAML files, then processes them as + current law is processed, so each change is uprated, indexed and + annualised by its parameter's rule. ``parameters.baseline`` stays + current law. Where a bounded change has no stated value after it, + prior law resumes along the path the parameter takes under the + reform's other changes (see ``policyengine_uk.utils.dated_reforms``). + """ + if not assignments: + return + baseline = processed_parameters() + ordered = sorted(assignments, key=lambda assignment: assignment.order) + projected = prior_law_projection(ordered) + parameters = statutory_parameters() + for assignment in ordered: + write_assignment( + parameters.get_child(assignment.path), assignment, projected + ) + self.parameters = parameters + self.process_parameters(baseline_parameters=baseline) + def __init__(self): """Initialize the UK tax-benefit system with entities and parameters.""" self._parameters_at_instant_cache: Dict[str, Any] = {} @@ -154,18 +232,81 @@ def __init__(self): # Set up and process parameters self.parameters_dir = COUNTRY_DIR / "parameters" - global _processed_parameters_cache + global _processed_parameters_cache, _statutory_parameters_cache + global _projected_timelines_cache if _processed_parameters_cache is not None: # Fast path: clone pre-processed parameters rather than re-running # the full pipeline (saves ~0.5s from convert_to_fiscal_year_parameters). - # apply_parameter_changes() calls reset_parameters() + process_parameters() - # directly, so reforms still get the full pipeline. + # Reforms rebuild the parameters from the statutory tree through + # apply_parameter_assignments, so they still get the full pipeline. self._parameters_at_instant_cache = {} self.parameters = _processed_parameters_cache.clone() + self.projected_timelines = _projected_timelines_cache else: self.reset_parameters() + _statutory_parameters_cache = self.parameters.clone() self.process_parameters() _processed_parameters_cache = self.parameters.clone() + _projected_timelines_cache = self.projected_timelines + + +def _ensure_caches() -> None: + if _processed_parameters_cache is None: + CountryTaxBenefitSystem() + + +def processed_parameters() -> ParameterNode: + """The processed current-law parameters (shared: do not change them).""" + _ensure_caches() + return _processed_parameters_cache + + +def statutory_parameters() -> ParameterNode: + """A fresh copy of the parameters as loaded from the YAML files.""" + _ensure_caches() + return _statutory_parameters_cache.clone() + + +def statutory_parameter_defines() -> Callable[[str], bool]: + """Whether the statutory parameter tree has a parameter at a path.""" + _ensure_caches() + return tree_defines(_statutory_parameters_cache) + + +def projected_timelines() -> Dict[str, Timeline]: + """Current law's projected timeline for each gov parameter, by name.""" + _ensure_caches() + return _projected_timelines_cache + + +def prior_law_projection( + assignments: List[ParameterAssignment], +) -> Callable[[str], Optional[Timeline]]: + """Prior law's projected timeline, by parameter name, for a reform. + + A bounded change with no stated value after it resumes along the path + its parameter takes under the reform's other changes. That is current + law's projection unless the reform also changes something else (a CPI + forecast, say); then the other changes are projected first. Changes + that themselves need the projection are left out of that first pass, + so two such changes do not see each other. + """ + current_law = projected_timelines() + statutory = statutory_parameters() + needing = [ + assignment + for assignment in assignments + if requires_projection(statutory.get_child(assignment.path), assignment) + ] + others = [assignment for assignment in assignments if assignment not in needing] + if not needing or not others: + return current_law.get + for assignment in others: + write_assignment( + statutory.get_child(assignment.path), assignment, current_law.get + ) + _, environment = process_parameter_tree(statutory, annualise=False) + return environment.get # Create system instance for module-level access diff --git a/policyengine_uk/utils/dated_reforms.py b/policyengine_uk/utils/dated_reforms.py new file mode 100644 index 0000000000..fe176c989b --- /dev/null +++ b/policyengine_uk/utils/dated_reforms.py @@ -0,0 +1,451 @@ +"""Apply dated parameter changes to the statutory timeline. + +A parameter's YAML holds its statutory values at their legal dates. +Processing (``CountryTaxBenefitSystem.process_parameters``) projects them +forward: uprating, income tax and NICs threshold indexation, the triple lock +and the other generated series. It then annualises each parameter to UK +fiscal years by its documented rule: the value at 30 April, the day-weighted +average for ``fiscal_year_blend`` parameters, or the statutory dates +themselves for ``preserve_calendar_dates`` parameters. + +Every reform route writes its changes onto the statutory timeline and then +runs that same processing. So a reform is uprated, indexed and annualised as +current law is: a rate change from 6 April 2027 sets fiscal year 2027, a +change to the personal allowance after 2030-31 moves the aligned NICs +thresholds, and a change to a September CPI forecast moves every threshold +indexed by it. Editing the already-annualised parameters instead, as the +reform-dict routes used to, kept a change dated 6 April out of that fiscal +year (its 30 April value had already been taken) and left derived +parameters on the baseline. + +Prior law after a bounded change +-------------------------------- + +A change over ``[start, stop]`` restores prior law from the day after +``stop``: + +- where the timeline states later values, the value in force at ``stop`` + resumes and the later values apply on their dates; +- where it states none (the parameter's future is projected by uprating or + indexation), prior law's projected path resumes: the values the parameter + takes, with every other change in the reform, without its own changes. + +``Parameter.update`` instead restores the last stated value as a new stated +value, so a one-year change to an uprated parameter froze the next year at +that value (a UC standard allowance change for 2027-28 gave £424.90 in +2028-29 instead of the uprated amount) and every later year was uprated from +it. Whether a statutory amount changed for some years should instead let +indexation resume from the changed amount, as Income Tax Act 2007 s57 does +from "the previous tax year", is a separate contract choice; see +``policyengine_uk.reform_contract``. + +How each route reads its period keys +------------------------------------ + +The routes keep the readings they documented before: + +===================== ======================== ========================= ========================== +Key reform dict ``parameter_changes`` ``Reform.from_dict`` +===================== ======================== ========================= ========================== +``"start.stop"`` ``[start, stop]`` ``[start, stop]`` ``[start, stop]`` +``"2027"`` or 2027 fiscal year 2027 only fiscal year 2027 only from 1 January 2027 on +``"2027-04-06"`` one day of the that one day from that day on + annualised parameter +``"year:2027:2"`` calendar years 2027-28 calendar years 2027-28 calendar years 2027-28 +``"ETERNITY"`` not accepted not accepted all time +scalar from 1 January 2023 on 2000 to 2099 2000 to 2099 +===================== ======================== ========================= ========================== + +"From ... on" holds the value flat to 31 December 2100, as editing the +annualised parameters did; uprating and indexation do not apply to it. + +A reform-dict key naming a single day or month keeps its legacy meaning: it +overrides the annualised parameter on that day or month only, after +processing. Formulas read an annual parameter at 1 January, so a +``"2027-01-01"`` key sets fiscal year 2027 and a ``"2027-04-06"`` key changes +nothing; ``ReformPeriodWarning`` says so. Use ``"2027-04-06.2100-12-31"`` +for a change from 6 April 2027. A reform-dict path the statutory tree does +not define (one added by a structural reform, or a generated series such as +``gov.economic_assumptions.indices.*``) is also applied to the annualised +parameters, after the structural reforms that come before it. +""" + +from __future__ import annotations + +import datetime +import warnings +from dataclasses import dataclass +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Tuple + +from policyengine_core.parameters import Parameter, ParameterNode +from policyengine_core.parameters.parameter_at_instant import ParameterAtInstant +from policyengine_core.periods import Instant, Period, instant, period + +from policyengine_uk.utils.parameters import check_parameter_not_removed + +# A change "from a date on" holds to the end of this day, as editing the +# annualised parameters held it indefinitely within the model's horizon. +FLAT_STOP = instant("2100-12-31") +# Where the reform dict's scalar shorthand starts (unchanged). +REFORM_DICT_SCALAR_START = instant("2023-01-01") +# policyengine-core's scalar window, ``year:2000:100``. +CORE_SCALAR_PERIOD = "year:2000:100" +EARLIEST = Instant((1, 1, 1)) + +REFORM_DICT = "reform_dict" +PARAMETER_CHANGES = "parameter_changes" +REFORM_FROM_DICT = "reform_from_dict" +READINGS = (REFORM_DICT, PARAMETER_CHANGES, REFORM_FROM_DICT) + + +class ReformPeriodWarning(UserWarning): + """A reform period key that is unlikely to mean what was intended.""" + + +@dataclass(frozen=True) +class ParameterAssignment: + """A value in force over ``[start, stop]`` on the statutory timeline. + + Both ends are inclusive. ``order`` is the position at which the change + is written: a later change wins where two overlap. + """ + + path: str + start: Instant + stop: Instant + value: Any + order: int = 0 + + +@dataclass(frozen=True) +class LegacyOverride: + """A change applied to the annualised parameters, as the route used to. + + ``key`` is the route's own period key (None for a scalar) and + ``reading`` the route that gave it. + """ + + path: str + key: Any + value: Any + reading: str = REFORM_DICT + + +def uk_fiscal_year_bounds(year: int) -> Tuple[Instant, Instant]: + """6 April of ``year`` to 5 April of the next year.""" + return instant(f"{year}-04-06"), instant(f"{year + 1}-04-05") + + +def bare_year(key: Any) -> Optional[int]: + """The year a bare-year key names, or None.""" + if isinstance(key, bool): + return None + if isinstance(key, int): + year = key + elif isinstance(key, str) and key.strip().isdigit(): + year = int(key.strip()) + else: + return None + return year if 1900 < year < 2200 else None + + +def interval_bounds(key: str) -> Tuple[Instant, Instant]: + start, separator, stop = key.partition(".") + if not separator or "." in stop: + raise ValueError(f"Expected a 'start.stop' period key; got '{key}'.") + start_instant, stop_instant = instant(start), instant(stop) + if stop_instant < start_instant: + raise ValueError(f"Period key '{key}' ends before it starts.") + return start_instant, stop_instant + + +def period_bounds(key: str) -> Tuple[Instant, Instant]: + parsed = period(key) + return parsed.start, parsed.stop + + +def is_sub_year(key: str) -> bool: + """Whether a key names one day or month (a legacy reform-dict key).""" + parsed: Period = period(key) + return parsed.unit in ("day", "month") and parsed.size == 1 + + +def _check_path(path: str) -> None: + check_parameter_not_removed(path) + + +def read_reform_dict( + reform: Mapping[str, Any], + defines: Callable[[str], bool], + order_start: int = 0, + warn: bool = False, +) -> Tuple[List[ParameterAssignment], List[LegacyOverride]]: + """Read a ``Simulation(reform=dict)`` / ``Scenario.from_reform`` dict. + + ``defines(path)`` says whether the statutory parameter tree defines a + path. Changes to paths it does not define, and single-day or single-month + keys, are returned as legacy overrides of the annualised parameters; + ``warn`` warns about single-day keys that change no annual value. + """ + assignments: List[ParameterAssignment] = [] + legacy: List[LegacyOverride] = [] + order = order_start + for path, spec in reform.items(): + _check_path(path) + statutory = defines(path) + if not isinstance(spec, Mapping): + if statutory: + assignments.append( + ParameterAssignment( + path, REFORM_DICT_SCALAR_START, FLAT_STOP, spec, order + ) + ) + order += 1 + else: + legacy.append(LegacyOverride(path, None, spec)) + continue + for key, value in spec.items(): + text = str(key) + year = bare_year(key) + if "." in text: + start, stop = interval_bounds(text) + elif year is not None: + start, stop = uk_fiscal_year_bounds(year) + elif is_sub_year(text): + if warn: + _warn_sub_year_key(path, text) + legacy.append(LegacyOverride(path, text, value)) + continue + else: + start, stop = period_bounds(text) + if statutory: + assignments.append(ParameterAssignment(path, start, stop, value, order)) + order += 1 + else: + legacy.append(LegacyOverride(path, text, value)) + return assignments, legacy + + +def _warn_sub_year_key(path: str, key: str) -> None: + parsed = period(key) + if parsed.unit == "day" and str(parsed.start)[5:] == "01-01": + return + warnings.warn( + f"Reform key '{key}' for {path} overrides the annualised parameter on " + f"that {parsed.unit} only, and formulas read annual parameters at " + f"1 January, so it may change nothing. For a change from that date " + f"use '{parsed.start}.{FLAT_STOP}'; for one fiscal year use the bare " + "year.", + ReformPeriodWarning, + stacklevel=4, + ) + + +def read_parameter_changes( + changes: Mapping[str, Any], order_start: int = 0 +) -> List[ParameterAssignment]: + """Read ``Scenario(parameter_changes=...)``.""" + assignments: List[ParameterAssignment] = [] + order = order_start + for path, spec in changes.items(): + _check_path(path) + if not isinstance(spec, Mapping): + start, stop = period_bounds(CORE_SCALAR_PERIOD) + assignments.append(ParameterAssignment(path, start, stop, spec, order)) + order += 1 + continue + for key, value in spec.items(): + text = str(key) + year = bare_year(key) + if "." in text: + start, stop = interval_bounds(text) + elif year is not None: + start, stop = uk_fiscal_year_bounds(year) + else: + start, stop = period_bounds(text) + assignments.append(ParameterAssignment(path, start, stop, value, order)) + order += 1 + return assignments + + +def read_reform_from_dict( + parameter_values: Mapping[str, Any], + defines: Callable[[str], bool], + order_start: int = 0, +) -> Tuple[List[ParameterAssignment], List[LegacyOverride]]: + """Read the dict behind a policyengine-core ``Reform.from_dict`` class. + + Follows policyengine-core's documented key reading. Within a parameter, + changes apply in ascending start order, as policyengine-core applies + them. + """ + assignments: List[ParameterAssignment] = [] + legacy: List[LegacyOverride] = [] + order = order_start + for path, spec in parameter_values.items(): + _check_path(path) + if not defines(path): + if isinstance(spec, Mapping): + legacy.extend( + LegacyOverride(path, str(key), value, REFORM_FROM_DICT) + for key, value in spec.items() + ) + else: + legacy.append(LegacyOverride(path, None, spec, REFORM_FROM_DICT)) + continue + entries: List[Tuple[Instant, Instant, Any]] = [] + if not isinstance(spec, Mapping): + start, stop = period_bounds(CORE_SCALAR_PERIOD) + entries.append((start, stop, spec)) + else: + for key, value in spec.items(): + text = str(key) + if text == "ETERNITY": + start, stop = EARLIEST, FLAT_STOP + elif "." in text: + start, stop = interval_bounds(text) + elif ":" in text: + start, stop = period_bounds(text) + else: + start, stop = instant(text), FLAT_STOP + entries.append((start, stop, value)) + entries.sort(key=lambda entry: entry[0]) + for start, stop, value in entries: + assignments.append(ParameterAssignment(path, start, stop, value, order)) + order += 1 + return assignments, legacy + + +def apply_legacy_overrides( + parameters: ParameterNode, overrides: Iterable[LegacyOverride] +) -> None: + """Apply changes to the annualised parameters, as each route used to.""" + from policyengine_core.reforms.reform import _period_key_update_kwargs + + from policyengine_uk.utils.parameters import uk_fiscal_year_period + + for override in overrides: + target = parameters.get_child(override.path) + if override.reading == REFORM_FROM_DICT: + if override.key is None: + target.update(period=CORE_SCALAR_PERIOD, value=override.value) + else: + target.update(**_period_key_update_kwargs(override.key, override.value)) + elif override.key is None: + target.update( + start=REFORM_DICT_SCALAR_START, stop=None, value=override.value + ) + elif "." in override.key: + start, stop = interval_bounds(override.key) + target.update(start=start, stop=stop, value=override.value) + else: + target.update( + period=( + uk_fiscal_year_period(override.key) + if target.metadata.get("preserve_calendar_dates") + else period(override.key) + ), + value=override.value, + ) + + +Timeline = List[ParameterAtInstant] + + +def _entry(name: str, instant_str: str, value: Any) -> ParameterAtInstant: + return ParameterAtInstant(name, instant_str, data={"value": value}) + + +def _value_in_force(values: Timeline, instant_str: str) -> Any: + """The value of a descending timeline at an instant (None before it).""" + for entry in values: + if entry.instant_str <= instant_str: + return entry.value + return None + + +def requires_projection(parameter: Parameter, assignment: ParameterAssignment) -> bool: + """Whether prior law after the change comes from the projected path.""" + if assignment.stop >= FLAT_STOP: + return False + stop_str = str(assignment.stop) + return not any(entry.instant_str > stop_str for entry in parameter.values_list) + + +def write_assignment( + parameter: Parameter, + assignment: ParameterAssignment, + projected: Optional[Callable[[str], Optional[Timeline]]] = None, +) -> None: + """Write a change onto a parameter's statutory timeline. + + Entries before ``start`` are kept and entries within the change are + replaced by its value. After ``stop``: stated later entries are kept, + with the value in force at ``stop`` restored on the next day unless an + entry already starts then; with no later entry, ``projected(name)`` + supplies prior law's projected timeline, whose entries after ``stop`` + are copied (with its value on the next day). Without a projection the + value in force at ``stop`` is restored, as ``Parameter.update`` does. + """ + name = parameter.name + start_str = str(assignment.start) + stop_str = str(assignment.stop) + next_str = str(assignment.stop.offset(1, "day")) + old: Timeline = sorted( + parameter.values_list, key=lambda entry: entry.instant_str, reverse=True + ) + later = [entry for entry in old if entry.instant_str > stop_str] + earlier = [entry for entry in old if entry.instant_str < start_str] + resumed: Timeline = [] + if assignment.stop < FLAT_STOP or later: + if later: + if later[-1].instant_str != next_str: + resumed.append(_entry(name, next_str, _value_in_force(old, stop_str))) + else: + path = projected(name) if projected is not None else None + if path is None: + resumed.append(_entry(name, next_str, _value_in_force(old, stop_str))) + else: + copied = [ + _entry(name, entry.instant_str, entry.value) + for entry in path + if entry.instant_str > next_str + ] + resumed.extend(copied) + resumed.append(_entry(name, next_str, _value_in_force(path, next_str))) + else: + # A change held flat to the end of the horizon: what follows it is + # the value in force at its stop, as Parameter.update leaves it. + resumed.append(_entry(name, next_str, _value_in_force(old, stop_str))) + values = later + resumed + [_entry(name, start_str, assignment.value)] + earlier + values.sort(key=lambda entry: entry.instant_str, reverse=True) + parameter.values_list = values + if parameter.parent is not None: + parameter.parent.clear_parent_cache() + parameter.mark_as_modified() + + +def snapshot_timelines(node: ParameterNode) -> Dict[str, Timeline]: + """Each parameter's current timeline, by name, without copying it. + + ``Parameter.update`` replaces a parameter's ``values_list`` rather than + changing it, so the lists captured here keep these values after later + updates (fiscal-year annualisation) replace them. + """ + return { + parameter.name: parameter.values_list + for parameter in node.get_descendants() + if isinstance(parameter, Parameter) + } + + +def tree_defines(parameters: ParameterNode) -> Callable[[str], bool]: + """``defines(path)``: whether a parameter tree has a leaf at ``path``.""" + + def defines(path: str) -> bool: + try: + return isinstance(parameters.get_child(path), Parameter) + except Exception: + return False + + return defines diff --git a/policyengine_uk/utils/scenario.py b/policyengine_uk/utils/scenario.py index 8484578fb8..672f6471f5 100644 --- a/policyengine_uk/utils/scenario.py +++ b/policyengine_uk/utils/scenario.py @@ -1,12 +1,63 @@ from pydantic import BaseModel, ConfigDict -from typing import Optional, Callable, Dict, Type, Union +from typing import Any, Literal, List, Optional, Callable, Dict, Type, Union from policyengine_core.simulations import Simulation from policyengine_core.reforms import Reform -from policyengine_core.periods import period, instant -from policyengine_uk.utils.parameters import ( - check_parameter_not_removed, - uk_fiscal_year_period, +from policyengine_uk.utils.dated_reforms import ( + REFORM_DICT, + REFORM_FROM_DICT, + ParameterAssignment, + apply_legacy_overrides, + read_reform_dict, + read_reform_from_dict, ) +from policyengine_uk.utils.parameters import check_parameter_not_removed + + +class DatedReform(BaseModel): + """A parameter reform dict and the route whose key reading it takes. + + ``reading`` is ``"reform_dict"`` for a dict passed as a reform and + ``"reform_from_dict"`` for the dict behind a policyengine-core + ``Reform.from_dict`` class. See ``policyengine_uk.utils.dated_reforms``. + """ + + reading: Literal["reform_dict", "reform_from_dict"] + changes: Dict[str, Any] + + def read(self, defines, order_start: int = 0, warn: bool = False): + if self.reading == REFORM_DICT: + return read_reform_dict(self.changes, defines, order_start, warn=warn) + return read_reform_from_dict(self.changes, defines, order_start) + + +def _is_reform_from_dict(reform: Type[Reform]) -> bool: + """Whether a Reform class was built by ``Reform.from_dict``.""" + return reform.__qualname__ == "Reform.from_dict..reform" and isinstance( + getattr(reform, "parameter_values", None), dict + ) + + +def _legacy_override_modifier(dated: DatedReform) -> Callable[[Simulation], None]: + """Apply what a dated reform leaves to the annualised parameters. + + That is single-day or single-month reform-dict keys and paths the + statutory tree does not define, such as a parameter a structural reform + in the same tuple adds. The simulation applies the rest from the + statutory timeline before any modifier runs. + """ + + def modifier(simulation: Simulation) -> None: + from policyengine_uk.tax_benefit_system import statutory_parameter_defines + + _, legacy = dated.read(statutory_parameter_defines(), warn=True) + if not legacy: + return + apply_legacy_overrides(simulation.tax_benefit_system.parameters, legacy) + simulation.tax_benefit_system.reset_parameter_caches() + if getattr(simulation, "populations", None) is not None: + simulation._invalidate_all_caches() + + return modifier def _apply_reform_class(reform: Type[Reform], simulation: Simulation) -> None: @@ -54,6 +105,9 @@ class Scenario(BaseModel): ] = None """A dictionary of parameter changes to apply to the simulation. These are applied *before* parameter operations.""" + dated_reforms: Optional[List[DatedReform]] = None + """Reform dicts written onto the statutory timeline, in order, after ``parameter_changes``. ``Scenario.from_reform`` builds these from a dict or a ``Reform.from_dict`` class.""" + simulation_modifier: Optional[Callable[["Simulation"], None]] = None """A function that modifies the simulation before running it.""" @@ -101,12 +155,26 @@ def combined_modifier(simulation: Simulation) -> None: elif other.simulation_modifier: combined_modifier = other.simulation_modifier + dated_reforms = (self.dated_reforms or []) + (other.dated_reforms or []) + # Return new scenario with merged configuration return Scenario( parameter_changes=merged_params if merged_params else None, + dated_reforms=dated_reforms or None, simulation_modifier=combined_modifier, ) + def dated_assignments(self, order_start: int = 0) -> List[ParameterAssignment]: + """The statutory-timeline changes of ``dated_reforms``, in order.""" + from policyengine_uk.tax_benefit_system import statutory_parameter_defines + + defines = statutory_parameter_defines() + assignments: List[ParameterAssignment] = [] + for dated in self.dated_reforms or []: + read, _ = dated.read(defines, order_start + len(assignments)) + assignments += read + return assignments + @classmethod def from_reform(cls, reform: Union[tuple, dict, Type[Reform]]) -> "Scenario": """Create a Scenario from various reform representations. @@ -120,12 +188,17 @@ def from_reform(cls, reform: Union[tuple, dict, Type[Reform]]) -> "Scenario": - A tuple of any of these (nested tuples allowed), applied in order - A dict keeps this module's reading of its keys, including inside a - tuple: a bare-year key such as ``"2026"`` changes that fiscal year - only, and a scalar value applies from 2023. ``Reform.from_dict`` - (and policyengine-core's ``Simulation.apply_reform``, which turns a - dict into one) reads a bare year as that year onwards. The two agree - for ``"YYYY-MM-DD.YYYY-MM-DD"`` keys. + A dict, or the dict behind a ``Reform.from_dict`` class, is written + onto the statutory timeline and the parameters are processed as + current law is, so a change dated 6 April sets that fiscal year and + derived parameters (uprating, the indexed thresholds) follow it. Each + keeps its reading of period keys, including inside a tuple: in a + dict, a bare-year key such as ``"2026"`` changes that fiscal year + only, a single-day key overrides the annualised parameter on that + day only, and a scalar value applies from 2023. ``Reform.from_dict`` + reads a bare year or a date as that instant onwards. The two agree + for ``"YYYY-MM-DD.YYYY-MM-DD"`` keys. See + ``policyengine_uk.utils.dated_reforms``. Returns: A new Scenario configured with the reform @@ -144,6 +217,15 @@ def from_reform(cls, reform: Union[tuple, dict, Type[Reform]]) -> "Scenario": ) if isinstance(reform, type) and issubclass(reform, Reform): + if _is_reform_from_dict(reform): + dated = DatedReform( + reading=REFORM_FROM_DICT, changes=reform.parameter_values + ) + return cls( + dated_reforms=[dated], + simulation_modifier=_legacy_override_modifier(dated), + ) + reform_class = reform def modifier(simulation: Simulation) -> None: @@ -154,47 +236,12 @@ def modifier(simulation: Simulation) -> None: ) elif isinstance(reform, dict): - # Dictionary of parameter changes - # Make sure to capture YYYY-MM-DD.YYYY-MM-DD. - - def modifier(sim: Simulation): - for parameter in reform: - check_parameter_not_removed(parameter) - target = sim.tax_benefit_system.parameters.get_child(parameter) - if isinstance(reform[parameter], dict): - for period_str, value in reform[parameter].items(): - if "." in period_str: - start = instant(period_str.split(".")[0]) - stop = instant(period_str.split(".")[1]) - period_ = None - else: - start = None - stop = None - period_ = ( - uk_fiscal_year_period(period_str) - if target.metadata.get("preserve_calendar_dates") - else period(period_str) - ) - target.update( - start=start, - stop=stop, - period=period_, - value=value, - ) - else: - start = instant("2023-01-01") - stop = None - period_ = None - - target.update( - start=start, - stop=stop, - period=period_, - value=reform[parameter], - ) - - return Scenario( - simulation_modifier=modifier, + for parameter in reform: + check_parameter_not_removed(parameter) + dated = DatedReform(reading=REFORM_DICT, changes=dict(reform)) + return cls( + dated_reforms=[dated], + simulation_modifier=_legacy_override_modifier(dated), ) elif isinstance(reform, tuple): @@ -231,9 +278,15 @@ def apply(self, simulation: Simulation) -> None: First applies parameter changes, then runs the simulation modifier if present. + A scenario's ``dated_reforms`` rebuild the simulation's parameters + from the statutory tree first, which discards earlier edits to them. + Args: simulation: The simulation to modify """ + if self.dated_reforms: + simulation.apply_parameter_assignments(self.dated_assignments()) + # Apply parameter changes first if self.parameter_changes: for path, value in self.parameter_changes.items(): @@ -275,6 +328,8 @@ def __repr__(self) -> str: parts = [] if self.parameter_changes: parts.append(f"parameter_changes={len(self.parameter_changes)} items") + if self.dated_reforms: + parts.append(f"dated_reforms={len(self.dated_reforms)} items") if self.simulation_modifier: parts.append("simulation_modifier=") return f"Scenario({', '.join(parts)})" From 205dffc4ea886ee30c0d8cccb9a5e99d8e0c7e99 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:51:53 -0400 Subject: [PATCH 2/6] Materialise prior law's projection before writing a change, and test the route contract A change starting after a parameter's last stated value lost the uprating or indexation between that value and the change, because the generators extend a timeline only from its last stated value. Prior law's projected path is now written in first wherever a change reaches the projected part of the timeline. Single-day reform keys are logged as well as warned, since some variable modules switch warnings off when they load. Adds test_dated_reform_routes.py: the Budget cases on four routes, documented readings, an independent annualisation oracle checked by Hypothesis sampling, route differentials, prior law under the reform's other changes, and baseline isolation (sim.baseline, parameters.baseline, query order, module caches, a fresh process). Co-Authored-By: Claude Opus 5.5 --- .../tests/test_dated_reform_routes.py | 674 ++++++++++++++++++ policyengine_uk/utils/dated_reforms.py | 93 +-- 2 files changed, 724 insertions(+), 43 deletions(-) create mode 100644 policyengine_uk/tests/test_dated_reform_routes.py diff --git a/policyengine_uk/tests/test_dated_reform_routes.py b/policyengine_uk/tests/test_dated_reform_routes.py new file mode 100644 index 0000000000..fedbb7b228 --- /dev/null +++ b/policyengine_uk/tests/test_dated_reform_routes.py @@ -0,0 +1,674 @@ +"""Every reform route writes dated changes onto the statutory timeline. + +``Simulation(reform=dict)``, ``Scenario.from_reform`` (a dict or a +``Reform.from_dict`` class) and ``Scenario(parameter_changes=...)`` write +their changes onto the parameters as loaded from YAML, which are then +processed as current law is (``policyengine_uk.utils.dated_reforms``). + +Invariants: + +1. Statutory timeline: a change over ``[start, stop]`` holds on those dates; + outside them the parameter follows prior law, which after ``stop`` is + current law's projected path when nothing is stated later. +2. Annualisation: the annual value formulas read (1 January of fiscal year + Y) equals an independent oracle applied to that timeline: the 30 April + value, or the day-weighted average over 6 April to 5 April for + ``fiscal_year_blend`` parameters. +3. Route equivalence (differential): ``"start.stop"`` keys give identical + parameters and results through the reform dict, ``parameter_changes``, + ``Reform.from_dict`` and either Scenario timing. +4. Documented differences between readings are kept: a bare-year dict key + is that fiscal year only, ``Reform.from_dict`` reads it as onwards, and a + single-day dict key overrides the annualised parameter on that day. +5. Baseline isolation: a reform never changes the simulation's baseline, + ``parameters.baseline`` (#2188), a simulation built later, or the module + caches, whatever the query order, and a fresh process agrees. +6. Derived parameters follow reformed values: the aligned NICs thresholds + follow the personal allowance and a September CPI change moves the + indexed thresholds by ITA 2007 ss21 and 57. +7. Timing: reforms applied after the data load reach the Universal Credit + rebalancing inputs (#2252) and parameter-gated structural reforms. +""" + +import datetime +import json +import subprocess +import sys +import warnings +from decimal import ROUND_CEILING, Decimal + +import numpy as np +import pytest +from hypothesis import HealthCheck, assume, given, settings +from hypothesis import strategies as st +from policyengine_core.parameters import Parameter +from policyengine_core.reforms import Reform + +from policyengine_uk import Simulation +from policyengine_uk import tax_benefit_system as tbs_module +from policyengine_uk.utils.dated_reforms import ReformPeriodWarning +from policyengine_uk.utils.scenario import Scenario + +BASIC_RATE = "gov.hmrc.income_tax.rates.uk[0].rate" +PERSONAL_ALLOWANCE = "gov.hmrc.income_tax.allowances.personal_allowance.amount" +BASIC_RATE_LIMIT = "gov.hmrc.income_tax.rates.uk[1].threshold" +UC_SINGLE_OLD = "gov.dwp.universal_credit.standard_allowance.amount.SINGLE_OLD" +CGT_BASIC_RATE = "gov.hmrc.cgt.basic_rate" +VAT_STANDARD_RATE = "gov.hmrc.vat.standard_rate" +PROTECTED_LCWRA = "gov.dwp.universal_credit.rebalancing.protected_health_element" +CPI_SEPTEMBER = "gov.economic_assumptions.statutory_uprating_inputs.cpi_september" +OBR_CPI = "gov.economic_assumptions.yoy_growth.obr.consumer_price_index" +NICS = "gov.hmrc.national_insurance" +PRIMARY_THRESHOLD = f"{NICS}.class_1.thresholds.primary_threshold" +UPPER_EARNINGS_LIMIT = f"{NICS}.class_1.thresholds.upper_earnings_limit" +LOWER_PROFITS_LIMIT = f"{NICS}.class_4.thresholds.lower_profits_limit" +UPPER_PROFITS_LIMIT = f"{NICS}.class_4.thresholds.upper_profits_limit" +YEARS = range(2024, 2039) +FAR = "2100-12-31" + +PROPERTY_SETTINGS = settings( + max_examples=12, + deadline=None, + derandomize=True, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.filter_too_much], +) + + +def adult(income=30_000, years=range(2025, 2036), **extra): + person = {"age": {2025: 40}, "employment_income": {y: income for y in years}} + person.update(extra) + return { + "people": {"adult": person}, + "benunits": {"benunit": {"members": ["adult"]}}, + "households": {"household": {"members": ["adult"]}}, + } + + +SITUATION = adult() + + +def annual(simulation, path, year): + parameter = simulation.tax_benefit_system.parameters.get_child(path) + return float(parameter(f"{year}-01-01")) + + +def calculate(simulation, variable, year): + return float(np.sum(simulation.calculate(variable, year))) + + +def as_dict(path, start, stop, value): + return {path: {f"{start}.{stop}": value}} + + +# The routes that read a "start.stop" key the same way. +def via_reform_dict(change, situation=SITUATION): + return Simulation(situation=situation, reform=change) + + +def via_parameter_changes(change, situation=SITUATION): + return Simulation(situation=situation, scenario=Scenario(parameter_changes=change)) + + +def via_reform_from_dict(change, situation=SITUATION): + return Simulation(situation=situation, reform=Reform.from_dict(change)) + + +def via_scenario_before_data_load(change, situation=SITUATION): + scenario = Scenario.from_reform(change) + scenario.applied_before_data_load = True + return Simulation(situation=situation, scenario=scenario) + + +ROUTES = { + "reform_dict": via_reform_dict, + "parameter_changes": via_parameter_changes, + "reform_from_dict": via_reform_from_dict, + "scenario_before_data_load": via_scenario_before_data_load, +} + + +# An annualisation oracle written independently of +# policyengine_uk.utils.parameters. + + +def _day(text): + return datetime.date.fromisoformat(text) + + +def timeline_value(entries, day): + """Value of a list of (date, value) entries at a day (None before).""" + found = None + found_at = None + for start, value in entries: + if start <= day and (found_at is None or start > found_at): + found, found_at = value, start + return found + + +def oracle_annual(entries, year, blend): + if not blend: + return timeline_value(entries, datetime.date(year, 4, 30)) + day = datetime.date(year, 4, 6) + end = datetime.date(year + 1, 4, 6) + total = 0.0 + while day < end: + total += timeline_value(entries, day) + day += datetime.timedelta(days=1) + return total / (end - datetime.date(year, 4, 6)).days + + +def projected_entries(path): + """Current law's projected timeline of a parameter as (date, value).""" + return [ + (_day(entry.instant_str), entry.value) + for entry in tbs_module.projected_timelines()[path] + ] + + +def overlay(entries, start, stop, value): + """A change over [start, stop] on top of prior law's timeline.""" + after = stop + datetime.timedelta(days=1) + prior_after = timeline_value(entries, after) + kept = [(s, v) for s, v in entries if s < start or s > after] + return kept + [(start, value), (after, prior_after)] + + +# Invariants 1-3 on the Budget cases. + + +@pytest.fixture(scope="module") +def baseline(): + return Simulation(situation=SITUATION) + + +@pytest.mark.parametrize("route", ROUTES) +def test_a_rate_change_from_6_april_sets_that_fiscal_year(route, baseline): + simulation = ROUTES[route](as_dict(BASIC_RATE, "2027-04-06", FAR, 0.21)) + assert annual(simulation, BASIC_RATE, 2026) == 0.20 + assert annual(simulation, BASIC_RATE, 2027) == 0.21 + assert annual(simulation, BASIC_RATE, 2028) == 0.21 + # (30,000 - 12,570) x 1%: the allowance is frozen at £12,570 to 2030-31. + for year, change in ((2026, 0), (2027, 174.30), (2028, 174.30)): + assert calculate(simulation, "income_tax", year) - calculate( + baseline, "income_tax", year + ) == pytest.approx(change, abs=0.005) + + +@pytest.mark.parametrize("route", ROUTES) +def test_a_finite_change_ends_with_its_fiscal_year(route, baseline): + simulation = ROUTES[route](as_dict(BASIC_RATE, "2027-04-06", "2028-04-05", 0.21)) + assert [annual(simulation, BASIC_RATE, y) for y in (2026, 2027, 2028)] == [ + 0.20, + 0.21, + 0.20, + ] + for year, change in ((2027, 174.30), (2028, 0)): + assert calculate(simulation, "income_tax", year) - calculate( + baseline, "income_tax", year + ) == pytest.approx(change, abs=0.005) + + +@pytest.mark.parametrize("route", ROUTES) +def test_a_one_year_change_to_an_uprated_benefit_resumes_its_uprated_path( + route, baseline +): + # Parameter.update would restore the 2026-27 amount (£424.90) as a stated + # value for 2028-29 and uprate every later year from it. + simulation = ROUTES[route](as_dict(UC_SINGLE_OLD, "2027-04-06", "2028-04-05", 500)) + assert annual(simulation, UC_SINGLE_OLD, 2027) == 500 + assert calculate(simulation, "uc_standard_allowance", 2027) == 6_000 + for year in (2026, 2028, 2029, 2033): + assert annual(simulation, UC_SINGLE_OLD, year) == annual( + baseline, UC_SINGLE_OLD, year + ), year + + +@pytest.mark.parametrize("route", ROUTES) +def test_an_april_uprating_schedule(route, baseline): + change = { + UC_SINGLE_OLD: { + "2027-04-06.2028-04-05": 500, + f"2028-04-06.{FAR}": 520, + } + } + simulation = ROUTES[route](change) + expected = {2027: 6_000, 2028: 6_240, 2029: 6_240} + for year, amount in expected.items(): + assert calculate(simulation, "uc_standard_allowance", year) == amount + assert calculate(simulation, "uc_standard_allowance", 2026) == calculate( + baseline, "uc_standard_allowance", 2026 + ) + + +@pytest.mark.parametrize("route", ROUTES) +def test_a_reverse_ordered_schedule_matches_the_ordered_one(route): + ordered = {UC_SINGLE_OLD: {"2027-04-06.2028-04-05": 500, f"2028-04-06.{FAR}": 520}} + reverse = {UC_SINGLE_OLD: dict(reversed(list(ordered[UC_SINGLE_OLD].items())))} + a, b = ROUTES[route](ordered), ROUTES[route](reverse) + for year in YEARS: + assert annual(a, UC_SINGLE_OLD, year) == annual(b, UC_SINGLE_OLD, year) + + +@pytest.mark.parametrize("route", ROUTES) +def test_a_mid_year_change_to_a_blended_rate_is_day_weighted(route): + simulation = ROUTES[route](as_dict(CGT_BASIC_RATE, "2027-10-30", FAR, 0.20)) + # 207 days at 18% (6 April to 29 October 2027) and 159 at 20% (to + # 5 April 2028); 2027-28 has 366 days. + assert annual(simulation, CGT_BASIC_RATE, 2027) == pytest.approx( + (0.18 * 207 + 0.20 * 159) / 366, abs=1e-12 + ) + assert annual(simulation, CGT_BASIC_RATE, 2026) == pytest.approx(0.18) + assert annual(simulation, CGT_BASIC_RATE, 2028) == pytest.approx(0.20) + + +@pytest.mark.parametrize("route", ROUTES) +def test_an_extended_freeze_moves_the_aligned_nics_thresholds(route, baseline): + change = { + PERSONAL_ALLOWANCE: {"2031-04-06.2034-04-05": 12_570}, + BASIC_RATE_LIMIT: {"2031-04-06.2034-04-05": 37_700}, + } + simulation = ROUTES[route](change) + for year in (2031, 2032, 2033): + assert annual(simulation, PERSONAL_ALLOWANCE, year) == 12_570 + assert annual(simulation, BASIC_RATE_LIMIT, year) == 37_700 + assert annual(simulation, LOWER_PROFITS_LIMIT, year) == 12_570 + assert annual(simulation, UPPER_PROFITS_LIMIT, year) == 50_270 + assert annual(simulation, PRIMARY_THRESHOLD, year) == pytest.approx(241.73) + assert annual(simulation, UPPER_EARNINGS_LIMIT, year) == pytest.approx(966.73) + # Income tax rises by 20% of the allowance current law would index. + frozen_out = annual(baseline, PERSONAL_ALLOWANCE, year) - 12_570 + assert calculate(simulation, "income_tax", year) - calculate( + baseline, "income_tax", year + ) == pytest.approx(0.2 * frozen_out, abs=0.005) + # Contract v1: after the freeze, prior law's projected path resumes. + # Resuming indexation from the frozen amount instead (ITA 2007 s57 reads + # "the previous tax year") is the open choice in cos d1344. + for path in (PERSONAL_ALLOWANCE, BASIC_RATE_LIMIT, PRIMARY_THRESHOLD): + for year in (2034, 2035): + assert annual(simulation, path, year) == annual(baseline, path, year) + + +def test_bare_year_dict_keys_match_the_fiscal_year_interval(): + by_year = via_reform_dict( + {PERSONAL_ALLOWANCE: {"2031": 12_570, "2032": 12_570, "2033": 12_570}} + ) + by_interval = via_reform_dict( + {PERSONAL_ALLOWANCE: {"2031-04-06.2034-04-05": 12_570}} + ) + for path in (PERSONAL_ALLOWANCE, PRIMARY_THRESHOLD, LOWER_PROFITS_LIMIT): + for year in YEARS: + assert annual(by_year, path, year) == annual(by_interval, path, year) + + +def test_a_september_cpi_change_moves_the_indexed_thresholds_by_statute(): + simulation = via_reform_dict( + as_dict(CPI_SEPTEMBER, "2030-09-01", "2031-08-31", 0.05) + ) + # s57(3): 12,570 x 5% = 628.50, rounded up to 630. s21(3): 37,700 x 1.05 + # = 39,585, rounded up to 39,600. + assert annual(simulation, PERSONAL_ALLOWANCE, 2031) == 13_200 + assert annual(simulation, BASIC_RATE_LIMIT, 2031) == 39_600 + assert annual(simulation, PRIMARY_THRESHOLD, 2031) == pytest.approx(253.85) + + +# Invariant 7: timing. + + +def test_a_dict_reform_reaches_the_uc_rebalancing_inputs(): + situation = adult(income=0, uc_limited_capability_for_WRA={2027: True}) + for route in ("reform_dict", "parameter_changes", "reform_from_dict"): + simulation = ROUTES[route]( + as_dict(PROTECTED_LCWRA, "2027-04-06", "2028-04-05", 500.0), situation + ) + assert calculate(simulation, "uc_LCWRA_element", 2027) == 6_000, route + + +def test_a_dict_reform_switches_on_a_parameter_gated_structural_reform(): + switch = "gov.contrib.cps.marriage_tax_reforms.expanded_ma.remove_income_condition" + situation = { + "people": { + "earner": {"age": {2026: 40}, "employment_income": {2026: 80_000}}, + "partner": {"age": {2026: 40}}, + }, + "benunits": { + "benunit": {"members": ["earner", "partner"], "is_married": {2026: True}} + }, + "households": {"household": {"members": ["earner", "partner"]}}, + } + simulation = via_reform_dict( + as_dict(switch, "2025-01-01", "2030-12-31", True), situation + ) + # 10% of the £12,570 allowance, rounded up to the next £10. + assert calculate(simulation, "marriage_allowance", 2026) == 1_260 + + +# Invariant 4: documented readings. + + +def test_a_bare_year_is_that_year_in_a_dict_and_onwards_in_reform_from_dict(): + change = {BASIC_RATE: {"2027": 0.21}} + as_dict_ = via_reform_dict(change) + as_class = via_reform_from_dict(change) + assert [annual(as_dict_, BASIC_RATE, y) for y in (2026, 2027, 2028)] == [ + 0.20, + 0.21, + 0.20, + ] + assert [annual(as_class, BASIC_RATE, y) for y in (2026, 2027, 2028)] == [ + 0.20, + 0.21, + 0.21, + ] + + +def test_a_single_day_dict_key_keeps_its_legacy_meaning_and_warns(caplog): + # Logged as well as warned: loading some variable modules switches + # warnings off for the process. + with caplog.at_level("WARNING", logger="policyengine_uk.utils.dated_reforms"): + april = via_reform_dict({BASIC_RATE: {"2027-04-06": 0.21}}) + assert "2027-04-06.2100-12-31" in caplog.text + assert [annual(april, BASIC_RATE, y) for y in YEARS] == [0.20] * len(YEARS) + caplog.clear() + with caplog.at_level("WARNING", logger="policyengine_uk.utils.dated_reforms"): + january = via_reform_dict({BASIC_RATE: {"2027-01-01": 0.21}}) + assert caplog.text == "" + assert [annual(january, BASIC_RATE, y) for y in (2026, 2027, 2028)] == [ + 0.20, + 0.21, + 0.20, + ] + + +def test_scalar_readings(): + as_dict_ = via_reform_dict({BASIC_RATE: 0.21}) + as_changes = via_parameter_changes({BASIC_RATE: 0.21}) + for simulation in (as_dict_, as_changes): + assert [annual(simulation, BASIC_RATE, y) for y in (2023, 2027, 2038)] == [ + 0.21 + ] * 3 + assert annual(as_dict_, BASIC_RATE, 2022) == 0.20 + assert annual(as_changes, BASIC_RATE, 2022) == 0.21 + + +def test_a_path_the_model_does_not_define_is_rejected(): + for route in ROUTES: + with pytest.raises(Exception, match="new_credit"): + ROUTES[route]( + {"gov.budget_2026.new_credit.amount": {f"2027-04-06.{FAR}": 500}} + ) + + +def test_a_parameter_added_by_a_structural_reform_in_the_tuple_can_be_set(): + from policyengine_core.parameters import ParameterNode + + class add_bonus(Reform): + def apply(self): + self.parameters.add_child( + "test_budget", + ParameterNode( + "test_budget", data={"bonus": {"values": {"2000-01-01": 1}}} + ), + ) + + simulation = Simulation( + situation=SITUATION, reform=(add_bonus, {"test_budget.bonus": 7}) + ) + assert simulation.tax_benefit_system.parameters.test_budget.bonus("2027-01-01") == 7 + + +# Invariant 1 and 2 by sampling: random changes against the oracle. + +SAMPLED = { + BASIC_RATE: (False, st.sampled_from([0.15, 0.19, 0.21, 0.25])), + UC_SINGLE_OLD: (False, st.sampled_from([350.0, 480.0, 512.5, 600.0])), + PERSONAL_ALLOWANCE: (False, st.sampled_from([11_000, 12_570, 15_000])), + CGT_BASIC_RATE: (True, st.sampled_from([0.1, 0.2, 0.24])), +} +dates = st.dates(datetime.date(2025, 1, 1), datetime.date(2033, 12, 31)) + + +@st.composite +def changes(draw): + path = draw(st.sampled_from(sorted(SAMPLED))) + blend, values = SAMPLED[path] + start = draw(dates) + open_ended = draw(st.booleans()) + if open_ended: + stop = datetime.date(2100, 12, 31) + else: + stop = start + datetime.timedelta(days=draw(st.integers(0, 1_500))) + return path, blend, start, stop, draw(values) + + +@PROPERTY_SETTINGS +@given(change=changes()) +def test_reformed_annual_values_match_the_oracle(change): + path, blend, start, stop, value = change + simulation = via_reform_dict(as_dict(path, start, stop, value)) + expected_timeline = overlay(projected_entries(path), start, stop, value) + projected = [ + (_day(entry.instant_str), entry.value) + for entry in simulation.tax_benefit_system.projected_timelines[path] + ] + for year in YEARS: + # Invariant 1: the statutory timeline. + sample = datetime.date(year, 4, 30) + assert timeline_value(projected, sample) == pytest.approx( + timeline_value(expected_timeline, sample) + ), year + # Invariant 2: the annual value. + assert annual(simulation, path, year) == pytest.approx( + oracle_annual(expected_timeline, year, blend), abs=1e-9 + ), year + + +@PROPERTY_SETTINGS +@given(change=changes(), income=st.integers(0, 150_000)) +def test_routes_agree_on_start_stop_keys(change, income): + path, _, start, stop, value = change + situation = adult(income=income) + reform = as_dict(path, start, stop, value) + simulations = {name: route(reform, situation) for name, route in ROUTES.items()} + first = simulations["reform_dict"] + for name, simulation in simulations.items(): + for year in (2026, 2028, 2031, 2034): + assert annual(simulation, path, year) == annual(first, path, year), name + assert calculate(simulation, "income_tax", year) == calculate( + first, "income_tax", year + ), name + + +# Invariant 5: baseline isolation. + + +def parameter_digest(parameters, paths, years=YEARS): + return { + path: [float(parameters.get_child(path)(f"{year}-01-01")) for year in years] + for path in paths + } + + +DIGEST_PATHS = [ + BASIC_RATE, + PERSONAL_ALLOWANCE, + UC_SINGLE_OLD, + PRIMARY_THRESHOLD, + CGT_BASIC_RATE, + VAT_STANDARD_RATE, +] +OUTPUTS = ["income_tax", "national_insurance", "uc_standard_allowance"] + + +def outputs(simulation, order=1): + pairs = [(variable, year) for variable in OUTPUTS for year in (2026, 2027, 2031)] + return { + f"{variable}:{year}": calculate(simulation, variable, year) + for variable, year in pairs[::order] + } + + +VAT_SITUATION = { + "people": {"adult": {"age": {2026: 40}}}, + "benunits": {"benunit": {"members": ["adult"]}}, + "households": { + "household": { + "members": ["adult"], + "full_rate_vat_consumption": {2026: 10_000}, + "reduced_rate_vat_consumption": {2026: 0}, + } + }, +} + + +@pytest.mark.parametrize( + "route, change", + [ + ("reform_dict", {VAT_STANDARD_RATE: 0.22}), + ("parameter_changes", {VAT_STANDARD_RATE: 0.22}), + ("parameter_changes", {VAT_STANDARD_RATE: {"2026": 0.22}}), + ("reform_from_dict", as_dict(VAT_STANDARD_RATE, "2026-01-01", FAR, 0.22)), + ("scenario_before_data_load", {VAT_STANDARD_RATE: 0.22}), + ], +) +def test_baseline_vat_is_current_law_on_every_route(route, change): + unreformed = Simulation(situation=VAT_SITUATION) + simulation = ROUTES[route](change, VAT_SITUATION) + vat = calculate(simulation, "vat", 2026) + assert vat > calculate(unreformed, "vat", 2026) + assert calculate(simulation, "baseline_vat", 2026) == calculate( + unreformed, "vat", 2026 + ) + assert simulation.tax_benefit_system.parameters.baseline.gov.hmrc.vat.standard_rate( + "2026-06-01" + ) == pytest.approx(0.20) + + +def test_reforms_leave_baselines_caches_and_later_simulations_alone(): + reference = outputs(Simulation(situation=SITUATION)) + processed = parameter_digest(tbs_module.processed_parameters(), DIGEST_PATHS) + statutory = parameter_digest(tbs_module.statutory_parameters(), DIGEST_PATHS) + reforms = [ + as_dict(BASIC_RATE, "2027-04-06", FAR, 0.21), + as_dict(UC_SINGLE_OLD, "2027-04-06", "2028-04-05", 500), + {PERSONAL_ALLOWANCE: {"2031": 12_570}}, + as_dict(CPI_SEPTEMBER, "2030-09-01", "2031-08-31", 0.05), + ] + for reform in reforms: + for route in ROUTES.values(): + simulation = route(reform) + assert outputs(simulation.baseline) == reference + assert outputs(simulation.baseline, order=-1) == reference + assert outputs(Simulation(situation=SITUATION)) == reference + assert outputs(Simulation(situation=SITUATION), order=-1) == reference + assert ( + parameter_digest(tbs_module.processed_parameters(), DIGEST_PATHS) == processed + ) + assert ( + parameter_digest(tbs_module.statutory_parameters(), DIGEST_PATHS) == statutory + ) + + +FRESH_PROCESS = """ +import json +from policyengine_uk import Simulation +from policyengine_uk.tests.test_dated_reform_routes import SITUATION, outputs, BASIC_RATE +reform = Simulation(situation=SITUATION, reform={BASIC_RATE: {"2027-04-06.2100-12-31": 0.21}}) +print(json.dumps({"baseline": outputs(Simulation(situation=SITUATION)), "reform": outputs(reform)})) +""" + + +def test_a_fresh_process_agrees_with_a_warm_one(): + for _ in range(2): + warm_reform = via_reform_dict(as_dict(BASIC_RATE, "2027-04-06", FAR, 0.21)) + warm = { + "baseline": outputs(Simulation(situation=SITUATION)), + "reform": outputs(warm_reform), + } + result = subprocess.run( + [sys.executable, "-c", FRESH_PROCESS], + capture_output=True, + text=True, + check=True, + ) + cold = json.loads(result.stdout.strip().splitlines()[-1]) + assert cold == warm + + +def test_rebuilding_from_the_statutory_cache_matches_loading_the_yaml(): + from policyengine_uk.tax_benefit_system import CountryTaxBenefitSystem + + fresh = CountryTaxBenefitSystem() + fresh.reset_parameters() + fresh.process_parameters() + rebuilt = CountryTaxBenefitSystem() + rebuilt.parameters = tbs_module.statutory_parameters() + rebuilt.process_parameters(baseline_parameters=tbs_module.processed_parameters()) + for parameter in fresh.parameters.gov.get_descendants(): + if not isinstance(parameter, Parameter): + continue + other = rebuilt.parameters.get_child(parameter.name) + for year in (2015, 2026, 2031, 2040): + assert other(f"{year}-01-01") == parameter(f"{year}-01-01"), parameter.name + + +# Reference arithmetic for the statute the indexation follows, kept here so +# the CPI case above has an oracle independent of create_threshold_indexation. +def test_statutory_arithmetic_for_the_cpi_case(): + increase = Decimal("0.05") + step = (Decimal(12_570) * increase / 10).to_integral_value(ROUND_CEILING) * 10 + assert 12_570 + step == 13_200 + brl = (Decimal(37_700) * (1 + increase) / 100).to_integral_value(ROUND_CEILING) + assert brl * 100 == 39_600 + + +def test_prior_law_after_a_bounded_change_follows_the_reforms_other_changes(): + # Benefits are uprated by the OBR CPI forecast (gov.benefit_uprating_cpi + # follows its lagged index). With a higher 2027 CPI in the same reform, + # the uprated path the benefit resumes after its one-year change is the + # one under that CPI, not current law's. + cpi = as_dict(OBR_CPI, "2027-01-01", "2027-12-31", 0.08) + benefit = as_dict(UC_SINGLE_OLD, "2027-04-06", "2028-04-05", 500) + cpi_only = via_reform_dict(cpi) + both = via_reform_dict({**cpi, **benefit}) + assert annual(both, UC_SINGLE_OLD, 2027) == 500 + for year in (2028, 2029, 2031): + assert annual(both, UC_SINGLE_OLD, year) == pytest.approx( + annual(cpi_only, UC_SINGLE_OLD, year) + ), year + assert annual(cpi_only, UC_SINGLE_OLD, 2028) != annual( + Simulation(situation=SITUATION), UC_SINGLE_OLD, 2028 + ) + + +@st.composite +def two_changes(draw): + path = draw(st.sampled_from(sorted(SAMPLED))) + blend, values = SAMPLED[path] + first = draw(dates) + gap = draw(st.integers(1, 900)) + length = draw(st.integers(0, 900)) + second = first + datetime.timedelta(days=gap) + first_stop = second - datetime.timedelta(days=1) + second_stop = second + datetime.timedelta(days=length) + return ( + path, + blend, + [(first, first_stop, draw(values)), (second, second_stop, draw(values))], + ) + + +@PROPERTY_SETTINGS +@given(change=two_changes(), reverse=st.booleans()) +def test_a_schedule_of_two_changes_matches_the_oracle(change, reverse): + path, blend, schedule = change + keys = [(f"{start}.{stop}", value) for start, stop, value in schedule] + if reverse: + keys.reverse() + simulation = via_reform_dict({path: dict(keys)}) + expected = projected_entries(path) + for start, stop, value in schedule: + expected = overlay(expected, start, stop, value) + for year in YEARS: + assert annual(simulation, path, year) == pytest.approx( + oracle_annual(expected, year, blend), abs=1e-9 + ), year diff --git a/policyengine_uk/utils/dated_reforms.py b/policyengine_uk/utils/dated_reforms.py index fe176c989b..3c4f1cded3 100644 --- a/policyengine_uk/utils/dated_reforms.py +++ b/policyengine_uk/utils/dated_reforms.py @@ -21,14 +21,15 @@ Prior law after a bounded change -------------------------------- -A change over ``[start, stop]`` restores prior law from the day after -``stop``: +Outside ``[start, stop]`` a change leaves prior law in force: -- where the timeline states later values, the value in force at ``stop`` - resumes and the later values apply on their dates; +- where the timeline states values, they apply on their dates, and the + value in force at ``stop`` resumes the day after it; - where it states none (the parameter's future is projected by uprating or - indexation), prior law's projected path resumes: the values the parameter - takes, with every other change in the reform, without its own changes. + indexation), prior law's projected path applies: the values the parameter + takes with every other change in the reform but without its own. That + covers both the years between the last stated value and a change that + starts after it, and the years after a bounded change. ``Parameter.update`` instead restores the last stated value as a new stated value, so a one-year change to an uprated parameter froze the next year at @@ -73,6 +74,7 @@ from __future__ import annotations import datetime +import logging import warnings from dataclasses import dataclass from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Tuple @@ -92,6 +94,8 @@ CORE_SCALAR_PERIOD = "year:2000:100" EARLIEST = Instant((1, 1, 1)) +logger = logging.getLogger(__name__) + REFORM_DICT = "reform_dict" PARAMETER_CHANGES = "parameter_changes" REFORM_FROM_DICT = "reform_from_dict" @@ -230,15 +234,16 @@ def _warn_sub_year_key(path: str, key: str) -> None: parsed = period(key) if parsed.unit == "day" and str(parsed.start)[5:] == "01-01": return - warnings.warn( + message = ( f"Reform key '{key}' for {path} overrides the annualised parameter on " f"that {parsed.unit} only, and formulas read annual parameters at " f"1 January, so it may change nothing. For a change from that date " f"use '{parsed.start}.{FLAT_STOP}'; for one fiscal year use the bare " - "year.", - ReformPeriodWarning, - stacklevel=4, + "year." ) + # Logged as well: some variable modules switch warnings off when loaded. + logger.warning(message) + warnings.warn(message, ReformPeriodWarning, stacklevel=4) def read_parameter_changes( @@ -364,12 +369,23 @@ def _value_in_force(values: Timeline, instant_str: str) -> Any: return None +def _last_stated(values: Timeline) -> Optional[str]: + return max((entry.instant_str for entry in values), default=None) + + def requires_projection(parameter: Parameter, assignment: ParameterAssignment) -> bool: - """Whether prior law after the change comes from the projected path.""" - if assignment.stop >= FLAT_STOP: + """Whether a change reaches the part of the timeline projection fills. + + That is a change starting after the parameter's last stated value, whose + prior law before it is projected, or a bounded change with no stated + value after it, whose prior law after it is projected. + """ + last = _last_stated(parameter.values_list) + if last is None: return False - stop_str = str(assignment.stop) - return not any(entry.instant_str > stop_str for entry in parameter.values_list) + if str(assignment.start) > last: + return True + return assignment.stop < FLAT_STOP and str(assignment.stop) >= last def write_assignment( @@ -379,43 +395,34 @@ def write_assignment( ) -> None: """Write a change onto a parameter's statutory timeline. - Entries before ``start`` are kept and entries within the change are - replaced by its value. After ``stop``: stated later entries are kept, - with the value in force at ``stop`` restored on the next day unless an - entry already starts then; with no later entry, ``projected(name)`` - supplies prior law's projected timeline, whose entries after ``stop`` - are copied (with its value on the next day). Without a projection the - value in force at ``stop`` is restored, as ``Parameter.update`` does. + Where the change reaches past the parameter's last stated value (see + ``requires_projection``), ``projected(name)`` supplies prior law's + projected timeline, whose values after the last stated one are first + written in as stated values: uprating and indexation extend a timeline + only from its last stated value, so without them the years between that + value and the change, or after a bounded change, would lose their + projection. Then, as ``Parameter.update`` does, entries within the + change are replaced by its value and the value in force at ``stop`` is + restored on the next day unless an entry already starts then. """ name = parameter.name start_str = str(assignment.start) stop_str = str(assignment.stop) next_str = str(assignment.stop.offset(1, "day")) - old: Timeline = sorted( - parameter.values_list, key=lambda entry: entry.instant_str, reverse=True - ) + old: Timeline = list(parameter.values_list) + path = projected(name) if projected is not None else None + if path is not None and requires_projection(parameter, assignment): + last = _last_stated(old) + old += [ + _entry(name, entry.instant_str, entry.value) + for entry in path + if entry.instant_str > last + ] + old.sort(key=lambda entry: entry.instant_str, reverse=True) later = [entry for entry in old if entry.instant_str > stop_str] earlier = [entry for entry in old if entry.instant_str < start_str] resumed: Timeline = [] - if assignment.stop < FLAT_STOP or later: - if later: - if later[-1].instant_str != next_str: - resumed.append(_entry(name, next_str, _value_in_force(old, stop_str))) - else: - path = projected(name) if projected is not None else None - if path is None: - resumed.append(_entry(name, next_str, _value_in_force(old, stop_str))) - else: - copied = [ - _entry(name, entry.instant_str, entry.value) - for entry in path - if entry.instant_str > next_str - ] - resumed.extend(copied) - resumed.append(_entry(name, next_str, _value_in_force(path, next_str))) - else: - # A change held flat to the end of the horizon: what follows it is - # the value in force at its stop, as Parameter.update leaves it. + if not later or later[-1].instant_str != next_str: resumed.append(_entry(name, next_str, _value_in_force(old, stop_str))) values = later + resumed + [_entry(name, start_str, assignment.value)] + earlier values.sort(key=lambda entry: entry.instant_str, reverse=True) From 48c1417380fa3e3a67e95e3505e704918f6b0be8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 16:07:07 -0400 Subject: [PATCH 3/6] Document how dated reforms are applied and read Adds a Dated changes section to the scenarios guide covering the statutory-timeline processing, each route's period-key reading, prior law around a change, and when changes are applied relative to data load. Fixes the UC rebalancing example whose single-date key changed nothing (#2252), and a stale personal allowance path in the phasing example. Co-Authored-By: Claude Opus 5.5 --- docs/book/policy/uc-rebalancing.md | 6 +++-- docs/book/usage/scenarios.md | 42 +++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/docs/book/policy/uc-rebalancing.md b/docs/book/policy/uc-rebalancing.md index b58e3170e0..2c29ca7079 100644 --- a/docs/book/policy/uc-rebalancing.md +++ b/docs/book/policy/uc-rebalancing.md @@ -96,11 +96,13 @@ sim = Simulation(scenario=scenario) ```python from policyengine_uk import Simulation, Scenario -# Set the new claimant health element to £250 per month +# Set the new claimant health element to £250 per month from April 2026. +# A "start.stop" key sets the value over those dates. A single date such as +# "2026-01-01" would change that one day only and leave every year unchanged. scenario = Scenario( parameter_changes={ "gov.dwp.universal_credit.rebalancing.new_claimant_health_element": { - "2026-01-01": 250.00 + "2026-04-06.2100-12-31": 250.00 } } ) diff --git a/docs/book/usage/scenarios.md b/docs/book/usage/scenarios.md index 026b624173..eb19e5df17 100644 --- a/docs/book/usage/scenarios.md +++ b/docs/book/usage/scenarios.md @@ -329,7 +329,7 @@ A scenario that gradually increases the personal allowance over four years - thi # This kind of phasing helps manage fiscal costs and economic adjustment phased_scenario = Scenario( parameter_changes={ - "gov.hmrc.income_tax.personal_allowance": { + "gov.hmrc.income_tax.allowances.personal_allowance.amount": { "2025": 13_000, "2026": 14_000, "2027": 15_000, @@ -354,6 +354,46 @@ for year in [2025, 2026, 2027, 2028]: print(f"{year}: Personal allowance £{allowance:.0f}, annual tax £{tax:.2f}") ``` +### Dated changes + +Budgets date their measures: a rate from 6 April, a freeze extended to a later tax year, a change that ends. Every way of passing a reform writes its changes onto each parameter's statutory timeline, the dated values in the YAML files. That covers `Scenario(parameter_changes=...)`, `Simulation(reform=dict)`, `Scenario.from_reform` and policyengine-core's `Reform.from_dict`. The parameters are then processed as current law is: + +- uprating and the income tax and NICs threshold indexation project each parameter forward; +- each parameter is annualised to UK fiscal years by its rule: its value on 30 April, or a day-weighted average over 6 April to 5 April for `fiscal_year_blend` parameters such as the CGT rates. Parameters marked `preserve_calendar_dates` keep their dates. + +So a change from 6 April 2027 sets fiscal year 2027. A change to the personal allowance after 2030-31 moves the NICs thresholds aligned with it. A change to a CPI forecast moves everything uprated from it. + +```python +from policyengine_uk import Simulation + +rate = "gov.hmrc.income_tax.rates.uk[0].rate" +situation = { + "people": {"person": {"age": {2027: 40}, "employment_income": {2027: 30_000}}}, + "benunits": {"benunit": {"members": ["person"]}}, + "households": {"household": {"members": ["person"]}}, +} + +# A basic rate of 21% from 6 April 2027 onwards. +from_april = Simulation(situation=situation, reform={rate: {"2027-04-06.2100-12-31": 0.21}}) +# The same rate for 2027-28 only: 2028-29 is back to current law. +one_year = Simulation(situation=situation, reform={rate: {"2027-04-06.2028-04-05": 0.21}}) +``` + +Outside a change's dates, prior law stays in force. Where the timeline states later values, they apply on their dates. Where the parameter's future is projected by uprating or indexation, prior law's projected path resumes after a change that ends: a one-year change to the UC standard allowance returns to the uprated amounts the next year. A change "from a date onwards" is held flat to the end of 2100 and is not uprated. + +Each way of passing a reform reads period keys as it always has: + +| Key | `reform=` dict | `parameter_changes` | `Reform.from_dict` | +|---|---|---|---| +| `"2027-04-06.2028-04-05"` | those dates | those dates | those dates | +| `"2027"` or `2027` | fiscal year 2027 only | fiscal year 2027 only | from 1 January 2027 onwards | +| `"2027-04-06"` | that one day of the annualised value | that one day | from that day onwards | +| a value with no period | from 1 January 2023 onwards | 2000 to 2099 | 2000 to 2099 | + +A single-date key in a `reform=` dict overrides the annualised value on that day only. Formulas read annual values on 1 January, so `"2027-01-01"` sets fiscal year 2027 and `"2027-04-06"` changes nothing. The model logs a warning when a key can change nothing. Use `"start.stop"` keys for dated measures. + +Changes applied after the data is loaded (the default) leave the data as built, so a changed economic assumption does not re-uprate incomes. They still come before the steps that read parameters while the simulation is built: the Universal Credit rebalancing awards and the structural reforms that parameters switch on. Set `applied_before_data_load=True` on a scenario to project the data with changed economic assumptions too. + ### Creating complex scenarios with simulation modifiers ```{important} From db9dcb0757e1f098aeb24b3ce0313e1236988cd6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 18:58:22 -0400 Subject: [PATCH 4/6] Keep every dated change across rebuilds, and project prior law per parameter Review fixes (GPT-6.1 Sol round 1), each reproduced before fixing: - A modifier that reloads and reprocesses the parameters (no_economic_assumptions, reindex_benefit_cap) discarded the reform, and Scenario.apply dropped changes a simulation already had. The system now keeps its dated changes and legacy overrides, and every rebuild writes them again. process_parameters builds the baseline copy from the parameters as they were before the changes. - Two changes that both reach the projected part of their timelines did not see each other: a one-year primary threshold change inside an allowance freeze resumed the unfrozen path. Prior law is now projected per parameter, with the others' changes, to a fixed point. - A change starting before a parameter's first stated value lost that value's backdating (found by a new boolean property). The backdated value is written as prior law first. - The projection cache is a copy, no longer sharing lists with the module-level system's tree. - Rebuilt parameters are memoised by their changes, so repeated simulations of one reform skip the rebuild. - Mixed single-day and dated keys on one parameter warn that single-day keys apply last. Tests: each of the above, a differential against editing the annualised parameters for calendar-year changes, and oracle properties for a boolean and a null-valued preserve_calendar_dates parameter. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/tax_benefit_system.py | 266 +++++++++++++----- .../tests/test_dated_reform_routes.py | 167 ++++++++++- policyengine_uk/utils/dated_reforms.py | 57 +++- policyengine_uk/utils/scenario.py | 9 +- 4 files changed, 422 insertions(+), 77 deletions(-) diff --git a/policyengine_uk/tax_benefit_system.py b/policyengine_uk/tax_benefit_system.py index e9c3a039e0..d61be98419 100644 --- a/policyengine_uk/tax_benefit_system.py +++ b/policyengine_uk/tax_benefit_system.py @@ -1,5 +1,8 @@ # Standard library imports import copy +import logging +from collections import OrderedDict +from dataclasses import replace from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Tuple @@ -44,8 +47,12 @@ add_lsr_deprecation_aliases, ) from policyengine_uk.utils.dated_reforms import ( + BACKDATED_FROM, + LegacyOverride, ParameterAssignment, Timeline, + apply_legacy_overrides, + copy_timelines, requires_projection, snapshot_timelines, tree_defines, @@ -57,6 +64,8 @@ convert_to_fiscal_year_parameters, ) +logger = logging.getLogger(__name__) + # Module constants COUNTRY_DIR = Path(__file__).parent DEFAULT_DATASET_ENV_VAR = "POLICYENGINE_UK_DEFAULT_DATASET" @@ -74,9 +83,22 @@ _projected_timelines_cache = None +def _add_generated_parameters(parameters: ParameterNode) -> ParameterNode: + parameters = add_private_pension_uprating_factor(parameters) + parameters = add_lagged_earnings(parameters) + parameters = add_lagged_cpi(parameters) + parameters = add_statutory_uprating_inputs(parameters) + parameters = add_triple_lock(parameters) + parameters = add_september_cpi_uprating(parameters) + parameters = create_economic_assumption_indices(parameters) + parameters = add_threshold_indexation(parameters) + parameters = add_lsr_deprecation_aliases(parameters) + return parameters + + def process_parameter_tree( parameters: ParameterNode, - baseline_parameters: Optional[ParameterNode] = None, + unreformed: Optional[ParameterNode] = None, annualise: bool = True, ) -> Tuple[ParameterNode, Dict[str, Timeline]]: """Project and annualise a statutory parameter tree. @@ -95,38 +117,32 @@ def process_parameter_tree( Returns the tree and each gov parameter's timeline just before the conversion to fiscal years. - ``parameters.baseline`` is the current-law copy that ``baseline_vat`` - and the budget adjustments compare against. Without - ``baseline_parameters`` it is a copy of this tree taken before uprating, - which is current law only when the tree carries no reform. A reformed - tree passes the processed current-law parameters as - ``baseline_parameters``, whose ``baseline`` child it takes (#2188). - ``annualise=False`` stops after the projection and adds no baseline. + ``parameters.baseline`` is the copy of the parameters without reforms + that ``baseline_vat`` and the budget adjustments compare against, taken + after the generated series are added and then uprated with the rest. A + tree carrying dated reform changes passes ``unreformed``, the same tree + without them, which the copy is taken from instead (#2188). + ``annualise=False`` stops after the projection and adds no copy. """ - parameters = add_private_pension_uprating_factor(parameters) - parameters = add_lagged_earnings(parameters) - parameters = add_lagged_cpi(parameters) - parameters = add_statutory_uprating_inputs(parameters) - parameters = add_triple_lock(parameters) - parameters = add_september_cpi_uprating(parameters) - parameters = create_economic_assumption_indices(parameters) - parameters = add_threshold_indexation(parameters) - parameters = add_lsr_deprecation_aliases(parameters) + parameters = _add_generated_parameters(parameters) # Create baseline parameters for reform comparisons - if annualise and baseline_parameters is None: - parameters.add_child("baseline", parameters.clone()) + if annualise: + baseline = ( + parameters.clone() + if unreformed is None + else _add_generated_parameters(unreformed) + ) + parameters.add_child("baseline", baseline) # Apply general parameter operations parameters = propagate_parameter_metadata(parameters) parameters = uprate_parameters(parameters) - parameters = backdate_parameters(parameters, "2015-01-01") + parameters = backdate_parameters(parameters, BACKDATED_FROM) projected = snapshot_timelines(parameters.gov) if not annualise: return parameters, projected parameters.gov = convert_to_fiscal_year_parameters(parameters.gov) - if baseline_parameters is not None: - parameters.add_child("baseline", baseline_parameters.baseline.clone()) return parameters, projected @@ -162,46 +178,88 @@ def reset_parameters(self) -> None: self.load_parameters(self.parameters_dir) self.parameters = add_removed_parameter_aliases(self.parameters) - def process_parameters( - self, baseline_parameters: Optional[ParameterNode] = None - ) -> None: + def process_parameters(self) -> None: """Process and transform parameters with UK-specific adjustments. See ``process_parameter_tree``, which this runs on the system's parameters. ``projected_timelines`` keeps each gov parameter's timeline from before the conversion to fiscal years. + + The system's dated reform changes (``dated_assignments``) are written + onto the parameters first, so a modifier that reloads and edits the + parameters before processing them again (as ``no_economic_assumptions`` + does) keeps the reform. Its legacy overrides of the annualised + parameters are then applied again. """ self._parameters_at_instant_cache = {} - self.parameters, self.projected_timelines = process_parameter_tree( - self.parameters, baseline_parameters - ) + self._rebuild(self.parameters, statutory=False) + self._apply_legacy_overrides() self.reset_parameter_caches() def apply_parameter_assignments( self, assignments: List[ParameterAssignment] ) -> None: - """Rebuild the parameters with changes on the statutory timeline. - - Writes each change, in ``order``, onto a fresh copy of the - parameters as loaded from the YAML files, then processes them as - current law is processed, so each change is uprated, indexed and - annualised by its parameter's rule. ``parameters.baseline`` stays - current law. Where a bounded change has no stated value after it, - prior law resumes along the path the parameter takes under the - reform's other changes (see ``policyengine_uk.utils.dated_reforms``). + """Rebuild the parameters with more changes on the statutory timeline. + + Adds the changes to the system's ``dated_assignments`` and writes + them all, in order, onto a fresh copy of the parameters as loaded + from the YAML files. The parameters are then processed as current law + is, so each change is uprated, indexed and annualised by its + parameter's rule, and ``parameters.baseline`` stays current law. + Where a change reaches the part of a parameter's timeline that + projection fills, prior law there follows the path the parameter + takes under the reform's other changes (see + ``policyengine_uk.utils.dated_reforms``). """ if not assignments: return - baseline = processed_parameters() - ordered = sorted(assignments, key=lambda assignment: assignment.order) - projected = prior_law_projection(ordered) - parameters = statutory_parameters() - for assignment in ordered: - write_assignment( - parameters.get_child(assignment.path), assignment, projected + offset = len(self.dated_assignments) + self.dated_assignments = self.dated_assignments + [ + replace(assignment, order=offset + index) + for index, assignment in enumerate( + sorted(assignments, key=lambda assignment: assignment.order) ) - self.parameters = parameters - self.process_parameters(baseline_parameters=baseline) + ] + self._parameters_at_instant_cache = {} + key = _assignments_key(self.dated_assignments) + cached = _rebuilt_parameters_cache.get(key) + if cached is not None: + self.parameters = cached[0].clone() + self.projected_timelines = cached[1] + else: + self._rebuild(statutory_parameters(), statutory=True) + _remember_rebuilt(key, self.parameters.clone(), self.projected_timelines) + self._apply_legacy_overrides() + self.reset_parameter_caches() + + def add_legacy_overrides(self, overrides: List[LegacyOverride]) -> None: + """Apply changes to the annualised parameters and keep them. + + Kept so a later rebuild of the parameters applies them again, after + the dated changes. + """ + if not overrides: + return + self.legacy_overrides = self.legacy_overrides + list(overrides) + apply_legacy_overrides(self.parameters, overrides) + self.reset_parameter_caches() + + def _rebuild(self, raw: ParameterNode, statutory: bool) -> None: + assignments = self.dated_assignments + if not assignments: + self.parameters, self.projected_timelines = process_parameter_tree(raw) + return + unreformed = raw.clone() + projected = prior_law_projection(assignments, unreformed, statutory) + for assignment in assignments: + write_assignment(raw.get_child(assignment.path), assignment, projected) + self.parameters, self.projected_timelines = process_parameter_tree( + raw, unreformed + ) + + def _apply_legacy_overrides(self) -> None: + if self.legacy_overrides: + apply_legacy_overrides(self.parameters, self.legacy_overrides) def __init__(self): """Initialize the UK tax-benefit system with entities and parameters.""" @@ -230,6 +288,13 @@ def __init__(self): # Load all variables from the variables directory self.add_variables_from_directory(COUNTRY_DIR / "variables") + # Dated reform changes on the statutory timeline, and legacy + # overrides of the annualised parameters, that every rebuild of the + # parameters applies again. Replaced, never appended to in place, so a + # cloned system does not share later additions. + self.dated_assignments: List[ParameterAssignment] = [] + self.legacy_overrides: List[LegacyOverride] = [] + # Set up and process parameters self.parameters_dir = COUNTRY_DIR / "parameters" global _processed_parameters_cache, _statutory_parameters_cache @@ -247,7 +312,9 @@ def __init__(self): _statutory_parameters_cache = self.parameters.clone() self.process_parameters() _processed_parameters_cache = self.parameters.clone() - _projected_timelines_cache = self.projected_timelines + # A copy: these timelines are prior law for every later reform, + # and the lists just captured belong to this system's own tree. + _projected_timelines_cache = copy_timelines(self.projected_timelines) def _ensure_caches() -> None: @@ -279,34 +346,97 @@ def projected_timelines() -> Dict[str, Timeline]: return _projected_timelines_cache +# Fixed-point passes for prior law's projection; dependencies between the +# changes that need it (a frozen allowance and an aligned NICs threshold, say) +# settle in two. +PRIOR_LAW_PASSES = 4 + + +def _same_timelines(a: Dict[str, Timeline], b: Dict[str, Timeline]) -> bool: + def key(timeline): + return [(entry.instant_str, entry.value) for entry in timeline or []] + + return a.keys() == b.keys() and all(key(a[name]) == key(b[name]) for name in a) + + def prior_law_projection( assignments: List[ParameterAssignment], + unreformed: ParameterNode, + statutory: bool = True, ) -> Callable[[str], Optional[Timeline]]: """Prior law's projected timeline, by parameter name, for a reform. - A bounded change with no stated value after it resumes along the path - its parameter takes under the reform's other changes. That is current - law's projection unless the reform also changes something else (a CPI - forecast, say); then the other changes are projected first. Changes - that themselves need the projection are left out of that first pass, - so two such changes do not see each other. + Where a change reaches the part of a parameter's timeline that + projection fills (see ``requires_projection``), prior law there is the + path that parameter takes with every other change in the reform but + without its own. Each such parameter is projected with the others' + changes, which are written with the previous pass's projections, until + the projections stop changing. ``unreformed`` is the tree the changes are + written onto; ``statutory`` says it is the YAML as loaded, whose + projection without changes is cached. """ - current_law = projected_timelines() - statutory = statutory_parameters() - needing = [ - assignment - for assignment in assignments - if requires_projection(statutory.get_child(assignment.path), assignment) - ] - others = [assignment for assignment in assignments if assignment not in needing] - if not needing or not others: - return current_law.get - for assignment in others: - write_assignment( - statutory.get_child(assignment.path), assignment, current_law.get + needing = sorted( + { + assignment.path + for assignment in assignments + if requires_projection(unreformed.get_child(assignment.path), assignment) + } + ) + if not needing: + return lambda name: None + start = ( + projected_timelines() + if statutory + else process_parameter_tree(unreformed.clone(), annualise=False)[1] + ) + environment = {path: start.get(path) for path in needing} + + def lookup(name: str) -> Optional[Timeline]: + found = environment.get(name) + return found if found is not None else start.get(name) + + for _ in range(PRIOR_LAW_PASSES): + updated = {} + for path in needing: + tree = unreformed.clone() + for assignment in assignments: + if assignment.path != path: + write_assignment( + tree.get_child(assignment.path), assignment, lookup + ) + updated[path] = process_parameter_tree(tree, annualise=False)[1].get(path) + if _same_timelines(updated, environment): + break + environment = updated + else: + logger.warning( + "Prior law's projection for %s did not settle in %d passes.", + ", ".join(needing), + PRIOR_LAW_PASSES, ) - _, environment = process_parameter_tree(statutory, annualise=False) - return environment.get + return lookup + + +# Recently rebuilt parameters, by their dated changes, so simulations of the +# same reform (one per year or region, say) skip the rebuild. Entries are +# cloned on the way in and out and never changed. +_REBUILT_CACHE_SIZE = 4 +_rebuilt_parameters_cache: "OrderedDict[tuple, Tuple[ParameterNode, Dict[str, Timeline]]]" = OrderedDict() + + +def _assignments_key(assignments: List[ParameterAssignment]) -> tuple: + return tuple( + (a.path, str(a.start), str(a.stop), repr(a.value), a.order) for a in assignments + ) + + +def _remember_rebuilt( + key: tuple, parameters: ParameterNode, projected: Dict[str, Timeline] +) -> None: + _rebuilt_parameters_cache[key] = (parameters, projected) + _rebuilt_parameters_cache.move_to_end(key) + while len(_rebuilt_parameters_cache) > _REBUILT_CACHE_SIZE: + _rebuilt_parameters_cache.popitem(last=False) # Create system instance for module-level access diff --git a/policyengine_uk/tests/test_dated_reform_routes.py b/policyengine_uk/tests/test_dated_reform_routes.py index fedbb7b228..6922c32e84 100644 --- a/policyengine_uk/tests/test_dated_reform_routes.py +++ b/policyengine_uk/tests/test_dated_reform_routes.py @@ -602,7 +602,7 @@ def test_rebuilding_from_the_statutory_cache_matches_loading_the_yaml(): fresh.process_parameters() rebuilt = CountryTaxBenefitSystem() rebuilt.parameters = tbs_module.statutory_parameters() - rebuilt.process_parameters(baseline_parameters=tbs_module.processed_parameters()) + rebuilt.process_parameters() for parameter in fresh.parameters.gov.get_descendants(): if not isinstance(parameter, Parameter): continue @@ -672,3 +672,168 @@ def test_a_schedule_of_two_changes_matches_the_oracle(change, reverse): assert annual(simulation, path, year) == pytest.approx( oracle_annual(expected, year, blend), abs=1e-9 ), year + + +# Review round 1 (GPT-6.1 Sol): rebuilds keep every change. + + +@pytest.mark.parametrize("name", ["no_economic_assumptions", "reindex_benefit_cap"]) +def test_a_modifier_that_reprocesses_the_parameters_keeps_the_reform(name): + import importlib + + module = importlib.import_module(f"policyengine_uk.scenarios.{name}") + reprocessing = getattr(module, name) + reform = Scenario.from_reform(as_dict(BASIC_RATE, "2027-04-06", FAR, 0.21)) + for scenario in (reprocessing + reform, reform + reprocessing): + simulation = Simulation(situation=SITUATION, scenario=scenario) + assert annual(simulation, BASIC_RATE, 2028) == 0.21, name + assert annual(simulation, BASIC_RATE, 2026) == 0.20, name + assert ( + simulation.tax_benefit_system.parameters.baseline.gov.hmrc.income_tax.rates.uk.brackets[ + 0 + ].rate("2028-06-01") + == pytest.approx(0.20) + ) + if name == "no_economic_assumptions": + # The scenario's own effect also holds: thresholds stay frozen. + assert annual(simulation, PERSONAL_ALLOWANCE, 2035) == 12_570 + + +def test_scenario_apply_adds_to_the_changes_a_simulation_already_has(): + simulation = via_reform_dict(as_dict(BASIC_RATE, "2027-04-06", FAR, 0.21)) + Scenario.from_reform({UC_SINGLE_OLD: {"2027": 500}}).apply(simulation) + assert annual(simulation, BASIC_RATE, 2028) == 0.21 + assert annual(simulation, UC_SINGLE_OLD, 2027) == 500 + assert len(simulation.tax_benefit_system.dated_assignments) == 2 + + +def test_two_changes_that_need_projection_see_each_other(): + # A one-year primary threshold change inside an allowance freeze: after + # it, prior law is the threshold aligned with the frozen allowance. + simulation = via_reform_dict( + { + PERSONAL_ALLOWANCE: {"2031-04-06.2035-04-05": 12_570}, + PRIMARY_THRESHOLD: {"2031": 250}, + } + ) + assert annual(simulation, PRIMARY_THRESHOLD, 2031) == 250 + for year in (2032, 2033, 2034): + assert annual(simulation, PRIMARY_THRESHOLD, year) == pytest.approx(241.73), ( + year + ) + + +def test_mixed_single_day_and_dated_keys_warn_and_apply_single_days_last(caplog): + with caplog.at_level("WARNING", logger="policyengine_uk.utils.dated_reforms"): + simulation = via_reform_dict( + {BASIC_RATE: {"2027-01-01": 0.21, "2026-01-01.2100-12-31": 0.22}} + ) + assert "mixes single-day keys" in caplog.text + assert annual(simulation, BASIC_RATE, 2027) == 0.21 + assert annual(simulation, BASIC_RATE, 2028) == 0.22 + + +def test_the_projected_timelines_cache_is_a_copy_and_reforms_leave_it_alone(): + cache = tbs_module.projected_timelines() + live = tbs_module.system.parameters + for name in (CPI_SEPTEMBER, UC_SINGLE_OLD): + assert cache[name] is not live.get_child(name).values_list + before = { + name: [(e.instant_str, e.value) for e in cache[name]] + for name in (CPI_SEPTEMBER, UC_SINGLE_OLD, PERSONAL_ALLOWANCE) + } + via_reform_dict(as_dict(UC_SINGLE_OLD, "2027-04-06", "2028-04-05", 500)) + via_reform_dict({PERSONAL_ALLOWANCE: {"2031": 12_570}}) + after = {name: [(e.instant_str, e.value) for e in cache[name]] for name in before} + assert after == before + + +def test_a_repeated_reform_reuses_the_rebuilt_parameters(): + reform = as_dict(BASIC_RATE, "2027-04-06", "2029-04-05", 0.23) + first = via_reform_dict(reform) + second = via_reform_dict(reform) + for year in YEARS: + assert annual(first, BASIC_RATE, year) == annual(second, BASIC_RATE, year) + # Each simulation has its own parameter objects. + assert first.tax_benefit_system.parameters.get_child( + BASIC_RATE + ) is not second.tax_benefit_system.parameters.get_child(BASIC_RATE) + + +# Differential: calendar-year changes give each changed parameter the annual +# values that editing the annualised parameters (the old reform-dict route) +# gave it. Blended and preserve_calendar_dates parameters are left out: the +# first are now day-weighted and the second never were annualised. + + +def legacy_annual(path, start, stop, value, year): + parameters = tbs_module.processed_parameters().clone() + parameters.get_child(path).update( + start=instant_(start), stop=instant_(stop), value=value + ) + return float(parameters.get_child(path)(f"{year}-01-01")) + + +from policyengine_core.periods import instant as instant_ # noqa: E402 + + +@PROPERTY_SETTINGS +@given( + path=st.sampled_from([BASIC_RATE, UC_SINGLE_OLD, PERSONAL_ALLOWANCE]), + first=st.integers(2024, 2034), + years=st.integers(1, 6), + open_ended=st.booleans(), + value=st.sampled_from([0.17, 450.0, 14_000]), +) +def test_calendar_year_changes_match_the_annualised_route( + path, first, years, open_ended, value +): + start = f"{first}-01-01" + stop = FAR if open_ended else f"{first + years - 1}-12-31" + simulation = via_reform_dict(as_dict(path, start, stop, value)) + for year in YEARS: + assert annual(simulation, path, year) == pytest.approx( + legacy_annual(path, start, stop, value, year) + ), year + + +# More parameter kinds for the oracle: a boolean, a null-valued +# preserve_calendar_dates amount, and changes that start before a +# parameter's first stated value. + +REBALANCING_ACTIVE = "gov.dwp.universal_credit.rebalancing.active" + + +@PROPERTY_SETTINGS +@given( + start=st.dates(datetime.date(2010, 1, 1), datetime.date(2032, 12, 31)), + length=st.integers(0, 2_000), + value=st.booleans(), +) +def test_boolean_changes_match_the_oracle(start, length, value): + stop = start + datetime.timedelta(days=length) + simulation = via_reform_dict(as_dict(REBALANCING_ACTIVE, start, stop, value)) + expected = overlay(projected_entries(REBALANCING_ACTIVE), start, stop, value) + for year in YEARS: + assert bool(annual(simulation, REBALANCING_ACTIVE, year)) == bool( + oracle_annual(expected, year, blend=False) + ), year + + +@PROPERTY_SETTINGS +@given( + start=st.dates(datetime.date(2025, 1, 1), datetime.date(2031, 12, 31)), + length=st.integers(0, 900), + value=st.sampled_from([400.0, 450.0]), +) +def test_preserve_calendar_dates_changes_keep_their_dates(start, length, value): + # The protected LCWRA amount keeps statutory dates and holds nulls for + # years not yet legislated. + stop = start + datetime.timedelta(days=length) + simulation = via_reform_dict(as_dict(PROTECTED_LCWRA, start, stop, value)) + expected = overlay(projected_entries(PROTECTED_LCWRA), start, stop, value) + parameter = simulation.tax_benefit_system.parameters.get_child(PROTECTED_LCWRA) + day = datetime.date(2024, 1, 1) + while day < datetime.date(2035, 1, 1): + assert parameter(day.isoformat()) == timeline_value(expected, day), day + day += datetime.timedelta(days=37) diff --git a/policyengine_uk/utils/dated_reforms.py b/policyengine_uk/utils/dated_reforms.py index 3c4f1cded3..a2eadf3b5f 100644 --- a/policyengine_uk/utils/dated_reforms.py +++ b/policyengine_uk/utils/dated_reforms.py @@ -68,7 +68,17 @@ for a change from 6 April 2027. A reform-dict path the statutory tree does not define (one added by a structural reform, or a generated series such as ``gov.economic_assumptions.indices.*``) is also applied to the annualised -parameters, after the structural reforms that come before it. +parameters, after the structural reforms that come before it. Changes to the +annualised parameters always follow the statutory-timeline changes, so a +reform-dict parameter that mixes single-day keys with other keys gets the +single-day keys last, whatever their order (``ReformPeriodWarning`` says +so), and a structural reform class that edits a parameter overrides a dict +change to it even when the dict comes later in a tuple. + +The tax-benefit system keeps every dated change and legacy override it has +been given. Any later rebuild of its parameters (another reform, +``Scenario.apply``, or a modifier that reloads and reprocesses the +parameters, as ``no_economic_assumptions`` does) writes them all again. """ from __future__ import annotations @@ -93,6 +103,8 @@ # policyengine-core's scalar window, ``year:2000:100``. CORE_SCALAR_PERIOD = "year:2000:100" EARLIEST = Instant((1, 1, 1)) +# Where process_parameter_tree backdates each parameter's first value to. +BACKDATED_FROM = "2015-01-01" logger = logging.getLogger(__name__) @@ -227,23 +239,37 @@ def read_reform_dict( order += 1 else: legacy.append(LegacyOverride(path, text, value)) + if warn: + mixed = sorted( + {override.path for override in legacy} + & {assignment.path for assignment in assignments} + ) + for path in mixed: + _warn( + f"Reform for {path} mixes single-day keys with other keys. The " + "single-day keys apply to the annualised parameter after the " + "others, whatever their order in the dict." + ) return assignments, legacy +def _warn(message: str) -> None: + # Logged as well: some variable modules switch warnings off when loaded. + logger.warning(message) + warnings.warn(message, ReformPeriodWarning, stacklevel=4) + + def _warn_sub_year_key(path: str, key: str) -> None: parsed = period(key) if parsed.unit == "day" and str(parsed.start)[5:] == "01-01": return - message = ( + _warn( f"Reform key '{key}' for {path} overrides the annualised parameter on " f"that {parsed.unit} only, and formulas read annual parameters at " f"1 January, so it may change nothing. For a change from that date " f"use '{parsed.start}.{FLAT_STOP}'; for one fiscal year use the bare " "year." ) - # Logged as well: some variable modules switch warnings off when loaded. - logger.warning(message) - warnings.warn(message, ReformPeriodWarning, stacklevel=4) def read_parameter_changes( @@ -395,7 +421,9 @@ def write_assignment( ) -> None: """Write a change onto a parameter's statutory timeline. - Where the change reaches past the parameter's last stated value (see + A change starting before the parameter's first stated value first gets + that value written at ``BACKDATED_FROM``, as processing would backdate + it. Where the change reaches past the parameter's last stated value (see ``requires_projection``), ``projected(name)`` supplies prior law's projected timeline, whose values after the last stated one are first written in as stated values: uprating and indexation extend a timeline @@ -410,6 +438,15 @@ def write_assignment( stop_str = str(assignment.stop) next_str = str(assignment.stop.offset(1, "day")) old: Timeline = list(parameter.values_list) + first = min((entry.instant_str for entry in old), default=None) + if first is not None and start_str < first and BACKDATED_FROM < first: + # Processing backdates a parameter's first stated value to + # BACKDATED_FROM. Written here, it stays prior law for the years + # before a change that starts before that first value; otherwise the + # change's own value would be backdated, or the years after an early + # change left empty. + earliest = min(old, key=lambda entry: entry.instant_str) + old.append(_entry(name, BACKDATED_FROM, earliest.value)) path = projected(name) if projected is not None else None if path is not None and requires_projection(parameter, assignment): last = _last_stated(old) @@ -446,6 +483,14 @@ def snapshot_timelines(node: ParameterNode) -> Dict[str, Timeline]: } +def copy_timelines(timelines: Dict[str, Timeline]) -> Dict[str, Timeline]: + """Copies of the timelines, sharing no list or entry with any tree.""" + return { + name: [_entry(name, entry.instant_str, entry.value) for entry in timeline] + for name, timeline in timelines.items() + } + + def tree_defines(parameters: ParameterNode) -> Callable[[str], bool]: """``defines(path)``: whether a parameter tree has a leaf at ``path``.""" diff --git a/policyengine_uk/utils/scenario.py b/policyengine_uk/utils/scenario.py index 672f6471f5..46493b72a9 100644 --- a/policyengine_uk/utils/scenario.py +++ b/policyengine_uk/utils/scenario.py @@ -52,8 +52,13 @@ def modifier(simulation: Simulation) -> None: _, legacy = dated.read(statutory_parameter_defines(), warn=True) if not legacy: return - apply_legacy_overrides(simulation.tax_benefit_system.parameters, legacy) - simulation.tax_benefit_system.reset_parameter_caches() + system = simulation.tax_benefit_system + if hasattr(system, "add_legacy_overrides"): + # Kept by the system, so a later rebuild applies them again. + system.add_legacy_overrides(legacy) + else: + apply_legacy_overrides(system.parameters, legacy) + system.reset_parameter_caches() if getattr(simulation, "populations", None) is not None: simulation._invalidate_all_caches() From ed153b43139abfe755ddd3ea8dc05c56679df2f6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 19:31:02 -0400 Subject: [PATCH 5/6] Format the dated-changes example as CI's ruff does Co-Authored-By: Claude Opus 5.5 --- docs/book/usage/scenarios.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/book/usage/scenarios.md b/docs/book/usage/scenarios.md index eb19e5df17..ffc01ede7b 100644 --- a/docs/book/usage/scenarios.md +++ b/docs/book/usage/scenarios.md @@ -374,9 +374,13 @@ situation = { } # A basic rate of 21% from 6 April 2027 onwards. -from_april = Simulation(situation=situation, reform={rate: {"2027-04-06.2100-12-31": 0.21}}) +from_april = Simulation( + situation=situation, reform={rate: {"2027-04-06.2100-12-31": 0.21}} +) # The same rate for 2027-28 only: 2028-29 is back to current law. -one_year = Simulation(situation=situation, reform={rate: {"2027-04-06.2028-04-05": 0.21}}) +one_year = Simulation( + situation=situation, reform={rate: {"2027-04-06.2028-04-05": 0.21}} +) ``` Outside a change's dates, prior law stays in force. Where the timeline states later values, they apply on their dates. Where the parameter's future is projected by uprating or indexation, prior law's projected path resumes after a change that ends: a one-year change to the UC standard allowance returns to the uprated amounts the next year. A change "from a date onwards" is held flat to the end of 2100 and is not uprated. From 603fecbb264c1d352f17be28273d1d3ac2f713ce Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 21:03:21 -0400 Subject: [PATCH 6/6] Keep a simulation's unrecorded edits across rebuilds, isolate the memo, and roll back failed changes Review round 2 (GPT-6.1 Sol, review-uk-r3-out.md): - Scenario.apply keeps main's semantics: it edits the annualised parameters and never rebuilds, so a structural reform's parameters and a modifier's edits survive it. Later rebuilds start from the raw tree as the simulation left it. - The rebuild memo keys only plain scalar values, and returns copies. - Construction and Scenario.apply give parameter_changes and dated reforms the same precedence. - A change that fails leaves the system as it was. - Prior law that does not settle within the pass limit is an error, not a cached partial projection. Also tests W5's shape: one scenario with a modifier that sets a parameter and parameter changes, in either order and either timing, matches the reform dict with both (on 2.125.1 the modifier's 8.75% dividend rate reverted to 10.75%). Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/tax_benefit_system.py | 173 +++++++++++----- .../tests/test_dated_reform_routes.py | 189 +++++++++++++++++- policyengine_uk/utils/dated_reforms.py | 19 +- policyengine_uk/utils/scenario.py | 85 +++++--- 4 files changed, 380 insertions(+), 86 deletions(-) diff --git a/policyengine_uk/tax_benefit_system.py b/policyengine_uk/tax_benefit_system.py index d61be98419..9e4175a690 100644 --- a/policyengine_uk/tax_benefit_system.py +++ b/policyengine_uk/tax_benefit_system.py @@ -189,7 +189,8 @@ def process_parameters(self) -> None: onto the parameters first, so a modifier that reloads and edits the parameters before processing them again (as ``no_economic_assumptions`` does) keeps the reform. Its legacy overrides of the annualised - parameters are then applied again. + parameters are then applied again. The parameters as they stood + before the changes are kept as the base for later rebuilds. """ self._parameters_at_instant_cache = {} self._rebuild(self.parameters, statutory=False) @@ -202,58 +203,110 @@ def apply_parameter_assignments( """Rebuild the parameters with more changes on the statutory timeline. Adds the changes to the system's ``dated_assignments`` and writes - them all, in order, onto a fresh copy of the parameters as loaded - from the YAML files. The parameters are then processed as current law - is, so each change is uprated, indexed and annualised by its - parameter's rule, and ``parameters.baseline`` stays current law. - Where a change reaches the part of a parameter's timeline that - projection fills, prior law there follows the path the parameter - takes under the reform's other changes (see + them all, in order, onto a fresh copy of the parameters the system + was last processed from: the YAML as loaded, or that tree as a + modifier left it before reprocessing. The parameters are then + processed as current law is, so each change is uprated, indexed and + annualised by its parameter's rule, and ``parameters.baseline`` stays + without the changes. Where a change reaches the part of a parameter's + timeline that projection fills, prior law there follows the path the + parameter takes under the reform's other changes (see ``policyengine_uk.utils.dated_reforms``). + + A rebuild keeps the system's dated changes and legacy overrides. It + does not keep other edits made to the processed parameters, such as + a structural reform's, as ``reset_parameters`` never has. If the + rebuild fails, the system is left as it was. """ if not assignments: return + previous = ( + self.dated_assignments, + self.parameters, + self._projected_timelines, + self._raw_parameters, + ) offset = len(self.dated_assignments) - self.dated_assignments = self.dated_assignments + [ - replace(assignment, order=offset + index) - for index, assignment in enumerate( - sorted(assignments, key=lambda assignment: assignment.order) - ) - ] - self._parameters_at_instant_cache = {} - key = _assignments_key(self.dated_assignments) - cached = _rebuilt_parameters_cache.get(key) - if cached is not None: - self.parameters = cached[0].clone() - self.projected_timelines = cached[1] - else: - self._rebuild(statutory_parameters(), statutory=True) - _remember_rebuilt(key, self.parameters.clone(), self.projected_timelines) - self._apply_legacy_overrides() - self.reset_parameter_caches() + try: + self.dated_assignments = self.dated_assignments + [ + replace(assignment, order=offset + index) + for index, assignment in enumerate( + sorted(assignments, key=lambda assignment: assignment.order) + ) + ] + self._parameters_at_instant_cache = {} + statutory = self._raw_parameters is None + key = _assignments_key(self.dated_assignments) if statutory else None + cached = _rebuilt_parameters_cache.get(key) if key is not None else None + if cached is not None: + self.parameters = cached[0].clone() + self._projected_timelines = cached[1] + else: + raw = ( + statutory_parameters() + if statutory + else self._raw_parameters.clone() + ) + self._rebuild(raw, statutory=statutory) + if key is not None: + _remember_rebuilt( + key, + self.parameters.clone(), + copy_timelines(self._projected_timelines), + ) + self._apply_legacy_overrides() + except Exception: + ( + self.dated_assignments, + self.parameters, + self._projected_timelines, + self._raw_parameters, + ) = previous + raise + finally: + self.reset_parameter_caches() def add_legacy_overrides(self, overrides: List[LegacyOverride]) -> None: """Apply changes to the annualised parameters and keep them. Kept so a later rebuild of the parameters applies them again, after - the dated changes. + the dated changes. Overrides that fail to apply are not kept. """ if not overrides: return + try: + apply_legacy_overrides(self.parameters, overrides) + finally: + self.reset_parameter_caches() self.legacy_overrides = self.legacy_overrides + list(overrides) - apply_legacy_overrides(self.parameters, overrides) - self.reset_parameter_caches() + + @property + def projected_timelines(self) -> Dict[str, Timeline]: + """Each gov parameter's timeline before fiscal-year annualisation. + + A copy: the timelines of an unreformed system, and of a rebuild + taken from the memo, are shared with later systems. + """ + return copy_timelines( + self._projected_timelines + if self._projected_timelines is not None + else projected_timelines() + ) def _rebuild(self, raw: ParameterNode, statutory: bool) -> None: assignments = self.dated_assignments + if not statutory: + # The base later rebuilds start from: a modifier's edits to the + # reloaded parameters are part of it. + self._raw_parameters = raw.clone() if not assignments: - self.parameters, self.projected_timelines = process_parameter_tree(raw) + self.parameters, self._projected_timelines = process_parameter_tree(raw) return unreformed = raw.clone() projected = prior_law_projection(assignments, unreformed, statutory) for assignment in assignments: write_assignment(raw.get_child(assignment.path), assignment, projected) - self.parameters, self.projected_timelines = process_parameter_tree( + self.parameters, self._projected_timelines = process_parameter_tree( raw, unreformed ) @@ -294,6 +347,11 @@ def __init__(self): # cloned system does not share later additions. self.dated_assignments: List[ParameterAssignment] = [] self.legacy_overrides: List[LegacyOverride] = [] + # The parameters this system was last processed from, when that is + # not the YAML as loaded (None), and its own projected timelines, + # when it has been processed itself (None: current law's). + self._raw_parameters: Optional[ParameterNode] = None + self._projected_timelines: Optional[Dict[str, Timeline]] = None # Set up and process parameters self.parameters_dir = COUNTRY_DIR / "parameters" @@ -306,15 +364,16 @@ def __init__(self): # apply_parameter_assignments, so they still get the full pipeline. self._parameters_at_instant_cache = {} self.parameters = _processed_parameters_cache.clone() - self.projected_timelines = _projected_timelines_cache else: self.reset_parameters() _statutory_parameters_cache = self.parameters.clone() - self.process_parameters() + self._rebuild(self.parameters, statutory=True) + self.reset_parameter_caches() _processed_parameters_cache = self.parameters.clone() # A copy: these timelines are prior law for every later reform, # and the lists just captured belong to this system's own tree. - _projected_timelines_cache = copy_timelines(self.projected_timelines) + _projected_timelines_cache = copy_timelines(self._projected_timelines) + self._projected_timelines = None def _ensure_caches() -> None: @@ -346,12 +405,6 @@ def projected_timelines() -> Dict[str, Timeline]: return _projected_timelines_cache -# Fixed-point passes for prior law's projection; dependencies between the -# changes that need it (a frozen allowance and an aligned NICs threshold, say) -# settle in two. -PRIOR_LAW_PASSES = 4 - - def _same_timelines(a: Dict[str, Timeline], b: Dict[str, Timeline]) -> bool: def key(timeline): return [(entry.instant_str, entry.value) for entry in timeline or []] @@ -371,9 +424,12 @@ def prior_law_projection( path that parameter takes with every other change in the reform but without its own. Each such parameter is projected with the others' changes, which are written with the previous pass's projections, until - the projections stop changing. ``unreformed`` is the tree the changes are - written onto; ``statutory`` says it is the YAML as loaded, whose - projection without changes is cached. + the projections stop changing. A chain of n such parameters, each + projected from the one before, settles within n passes; projections + that still change after that depend on each other in a cycle, which is + an error. ``unreformed`` is the tree the changes are written onto; + ``statutory`` says it is the YAML as loaded, whose projection without + changes is cached. """ needing = sorted( { @@ -395,7 +451,7 @@ def lookup(name: str) -> Optional[Timeline]: found = environment.get(name) return found if found is not None else start.get(name) - for _ in range(PRIOR_LAW_PASSES): + for _ in range(len(needing) + 1): updated = {} for path in needing: tree = unreformed.clone() @@ -406,15 +462,13 @@ def lookup(name: str) -> Optional[Timeline]: ) updated[path] = process_parameter_tree(tree, annualise=False)[1].get(path) if _same_timelines(updated, environment): - break + return lookup environment = updated - else: - logger.warning( - "Prior law's projection for %s did not settle in %d passes.", - ", ".join(needing), - PRIOR_LAW_PASSES, - ) - return lookup + raise ValueError( + "Prior law's projected paths for " + + ", ".join(needing) + + " depend on each other in a cycle, so the reform cannot be applied." + ) # Recently rebuilt parameters, by their dated changes, so simulations of the @@ -424,10 +478,19 @@ def lookup(name: str) -> Optional[Timeline]: _rebuilt_parameters_cache: "OrderedDict[tuple, Tuple[ParameterNode, Dict[str, Timeline]]]" = OrderedDict() -def _assignments_key(assignments: List[ParameterAssignment]) -> tuple: - return tuple( - (a.path, str(a.start), str(a.stop), repr(a.value), a.order) for a in assignments - ) +def _assignments_key(assignments: List[ParameterAssignment]) -> Optional[tuple]: + """A memo key for the changes, or None when they cannot be memoised. + + Only plain numbers, booleans and strings are keyed, by type and value. + Any other value (a list, or a number subclass whose repr could hide its + value) is rebuilt every time. + """ + key = [] + for a in assignments: + if type(a.value) not in (int, float, bool, str): + return None + key.append((a.path, str(a.start), str(a.stop), type(a.value).__name__, a.value)) + return tuple(key) def _remember_rebuilt( diff --git a/policyengine_uk/tests/test_dated_reform_routes.py b/policyengine_uk/tests/test_dated_reform_routes.py index 6922c32e84..383231dfe6 100644 --- a/policyengine_uk/tests/test_dated_reform_routes.py +++ b/policyengine_uk/tests/test_dated_reform_routes.py @@ -699,12 +699,199 @@ def test_a_modifier_that_reprocesses_the_parameters_keeps_the_reform(name): assert annual(simulation, PERSONAL_ALLOWANCE, 2035) == 12_570 +DIVIDEND_BASIC_RATE = "gov.hmrc.income_tax.rates.dividends[0].rate" + + +def _dividend_baseline(simulation): + # Sets a parameter on the annualised tree, as a dashboard baseline does. + simulation.tax_benefit_system.parameters.get_child(DIVIDEND_BASIC_RATE).update( + period="year:2026:20", value=0.0875 + ) + + +@pytest.mark.parametrize("before_data_load", [False, True]) +def test_a_scenario_keeps_its_modifiers_parameters_and_its_parameter_changes( + before_data_load, +): + # In 2.125.1 the parameter changes rebuilt the parameters after the + # modifier ran, so its 8.75% dividend rate reverted to 10.75%. + situation = adult(dividend_income={year: 10_000 for year in range(2025, 2036)}) + both = { + DIVIDEND_BASIC_RATE: {f"2026-01-01.{FAR}": 0.0875}, + BASIC_RATE: {f"2027-04-06.{FAR}": 0.21}, + } + reference = via_reform_dict(both, situation) + modifier = Scenario(simulation_modifier=_dividend_baseline) + changes = Scenario(parameter_changes=as_dict(BASIC_RATE, "2027-04-06", FAR, 0.21)) + for scenario in (modifier + changes, changes + modifier): + scenario.applied_before_data_load = before_data_load + simulation = Simulation(situation=situation, scenario=scenario) + for year in (2026, 2027, 2028): + assert annual(simulation, DIVIDEND_BASIC_RATE, year) == 0.0875 + assert annual(simulation, BASIC_RATE, year) == annual( + reference, BASIC_RATE, year + ) + assert calculate(simulation, "income_tax", year) == pytest.approx( + calculate(reference, "income_tax", year), abs=1e-6 + ) + assert annual(simulation.baseline, DIVIDEND_BASIC_RATE, 2027) == 0.1075 + + def test_scenario_apply_adds_to_the_changes_a_simulation_already_has(): simulation = via_reform_dict(as_dict(BASIC_RATE, "2027-04-06", FAR, 0.21)) Scenario.from_reform({UC_SINGLE_OLD: {"2027": 500}}).apply(simulation) assert annual(simulation, BASIC_RATE, 2028) == 0.21 assert annual(simulation, UC_SINGLE_OLD, 2027) == 500 - assert len(simulation.tax_benefit_system.dated_assignments) == 2 + # Scenario.apply edits the annualised parameters, as it always has; the + # system keeps the edit so a later rebuild applies it again. + system = simulation.tax_benefit_system + assert len(system.dated_assignments) == 1 + assert len(system.legacy_overrides) == 1 + simulation.apply_parameter_changes({PERSONAL_ALLOWANCE: {"2027": 13_000}}) + assert annual(simulation, BASIC_RATE, 2028) == 0.21 + assert annual(simulation, UC_SINGLE_OLD, 2027) == 500 + assert annual(simulation, PERSONAL_ALLOWANCE, 2027) == 13_000 + + +# Review round 2 (GPT-6.1 Sol): Scenario.apply never rebuilds, rebuilds keep +# a modifier's base, failures roll back, and the memo is isolated. + + +def test_scenario_apply_keeps_a_structural_reforms_parameter(): + from policyengine_core.parameters import ParameterNode + + class add_bonus(Reform): + def apply(self): + self.parameters.add_child( + "test_budget", + ParameterNode( + "test_budget", data={"bonus": {"values": {"2000-01-01": 1}}} + ), + ) + + simulation = Simulation( + situation=SITUATION, reform=(add_bonus, {"test_budget.bonus": 7}) + ) + Scenario.from_reform({BASIC_RATE: {"2027": 0.21}}).apply(simulation) + parameters = simulation.tax_benefit_system.parameters + assert parameters.test_budget.bonus("2027-01-01") == 7 + assert annual(simulation, BASIC_RATE, 2027) == 0.21 + + +def test_later_changes_keep_a_reprocessing_modifiers_edits(): + from policyengine_uk.scenarios.no_economic_assumptions import ( + no_economic_assumptions, + ) + + simulation = Simulation(situation=SITUATION, scenario=no_economic_assumptions) + assert annual(simulation, PERSONAL_ALLOWANCE, 2035) == 12_570 + Scenario.from_reform({BASIC_RATE: {"2027": 0.21}}).apply(simulation) + assert annual(simulation, PERSONAL_ALLOWANCE, 2035) == 12_570 + # A rebuild starts from the parameters as the modifier left them. + simulation.apply_parameter_changes({UC_SINGLE_OLD: {"2027": 500}}) + assert annual(simulation, PERSONAL_ALLOWANCE, 2035) == 12_570 + assert annual(simulation, BASIC_RATE, 2027) == 0.21 + assert annual(simulation, UC_SINGLE_OLD, 2027) == 500 + + +def test_scenario_apply_and_construction_give_a_scenario_the_same_precedence(): + scenario = Scenario(parameter_changes={BASIC_RATE: 0.21}) + Scenario.from_reform( + {BASIC_RATE: {"2027": 0.22}} + ) + constructed = Simulation(situation=SITUATION, scenario=scenario) + applied = Simulation(situation=SITUATION) + scenario.apply(applied) + for year in (2026, 2027, 2028): + assert annual(applied, BASIC_RATE, year) == annual( + constructed, BASIC_RATE, year + ), year + assert annual(constructed, BASIC_RATE, 2027) == 0.22 + assert annual(constructed, BASIC_RATE, 2028) == 0.21 + + +def test_a_failed_change_leaves_the_system_as_it_was(): + simulation = via_reform_dict(as_dict(BASIC_RATE, "2027-04-06", FAR, 0.21)) + system = simulation.tax_benefit_system + with pytest.raises(ValueError): + Scenario.from_reform({"gov.not_a_real_parameter": {"2027": 1}}).apply( + simulation + ) + with pytest.raises(ValueError): + simulation.apply_parameter_changes({"gov.not_a_real_parameter": {"2027": 1}}) + assert len(system.dated_assignments) == 1 + assert system.legacy_overrides == [] + assert annual(simulation, BASIC_RATE, 2028) == 0.21 + # A valid change still applies afterwards. + simulation.apply_parameter_changes({UC_SINGLE_OLD: {"2027": 500}}) + assert annual(simulation, UC_SINGLE_OLD, 2027) == 500 + assert annual(simulation, BASIC_RATE, 2028) == 0.21 + + +def _system_with(path, value, dates=f"2027-04-06.{FAR}"): + from policyengine_uk.utils.dated_reforms import read_reform_dict + + system = tbs_module.system.clone() + changes, _ = read_reform_dict({path: {dates: value}}, lambda _: True) + system.apply_parameter_assignments(changes) + return system + + +def test_the_rebuild_memo_cannot_be_changed_through_a_system(): + sources = "gov.dwp.universal_credit.means_test.capital.person_sources" + dates = f"2016-04-06.{FAR}" + value = ["lifetime_isa_countable_capital"] + first = _system_with(sources, list(value), dates) + first.parameters.get_child(sources)("2027-01-01").append("employment_income") + later = _system_with(sources, list(value), dates) + assert later.parameters.get_child(sources)("2027-01-01") == value + # Values that are not plain numbers, booleans or strings are not keyed. + assert tbs_module._assignments_key(later.dated_assignments) is None + + class Rate(float): + def __repr__(self): + return "0.21111" + + _system_with(BASIC_RATE, 0.21111) + disguised = _system_with(BASIC_RATE, Rate(0.22222)) + assert float(disguised.parameters.get_child(BASIC_RATE)("2028-01-01")) == 0.22222 + + # Two systems with one memoised reform share no timeline with each other + # or with the cache. + a, b = _system_with(BASIC_RATE, 0.23), _system_with(BASIC_RATE, 0.23) + assert a.projected_timelines[BASIC_RATE] is not b.projected_timelines[BASIC_RATE] + a.parameters.get_child(BASIC_RATE).update(period="2028", value=0.5) + assert float(b.parameters.get_child(BASIC_RATE)("2028-01-01")) == 0.23 + assert ( + float( + _system_with(BASIC_RATE, 0.23).parameters.get_child(BASIC_RATE)( + "2028-01-01" + ) + ) + == 0.23 + ) + + +def test_prior_law_that_never_settles_is_an_error(monkeypatch): + # Projections that keep changing from pass to pass mean the changes' + # prior law depends on itself. + from policyengine_core.parameters.parameter_at_instant import ParameterAtInstant + from policyengine_uk.utils.dated_reforms import read_reform_dict + + calls = {"n": 0} + + def unsettled(tree, unreformed=None, annualise=True): + calls["n"] += 1 + entry = ParameterAtInstant( + UC_SINGLE_OLD, "2026-04-01", data={"value": float(calls["n"])} + ) + return tree, {UC_SINGLE_OLD: [entry]} + + changes, _ = read_reform_dict( + as_dict(UC_SINGLE_OLD, "2027-04-06", "2028-04-05", 500), lambda _: True + ) + monkeypatch.setattr(tbs_module, "process_parameter_tree", unsettled) + with pytest.raises(ValueError, match="depend on each other in a cycle"): + tbs_module.prior_law_projection(changes, tbs_module.statutory_parameters()) def test_two_changes_that_need_projection_see_each_other(): diff --git a/policyengine_uk/utils/dated_reforms.py b/policyengine_uk/utils/dated_reforms.py index a2eadf3b5f..ae243793f6 100644 --- a/policyengine_uk/utils/dated_reforms.py +++ b/policyengine_uk/utils/dated_reforms.py @@ -76,9 +76,16 @@ change to it even when the dict comes later in a tuple. The tax-benefit system keeps every dated change and legacy override it has -been given. Any later rebuild of its parameters (another reform, -``Scenario.apply``, or a modifier that reloads and reprocesses the -parameters, as ``no_economic_assumptions`` does) writes them all again. +been given. Any later rebuild of its parameters (another call to +``Simulation.apply_parameter_changes``, or a modifier that reloads and +reprocesses the parameters, as ``no_economic_assumptions`` does) writes them +all again, onto the parameters as that modifier left them. A rebuild does +not keep other edits to the processed parameters, such as a structural +reform's. + +``Scenario.apply`` on an already-built simulation never rebuilds: it edits +the annualised parameters, as it always has, so every change it makes is a +legacy override and dated keys are not annualised again. """ from __future__ import annotations @@ -111,6 +118,8 @@ REFORM_DICT = "reform_dict" PARAMETER_CHANGES = "parameter_changes" REFORM_FROM_DICT = "reform_from_dict" +# Scenario.apply's parameter_changes on a built simulation: every period. +ALL_PERIODS = "all_periods" READINGS = (REFORM_DICT, PARAMETER_CHANGES, REFORM_FROM_DICT) @@ -357,7 +366,9 @@ def apply_legacy_overrides( for override in overrides: target = parameters.get_child(override.path) - if override.reading == REFORM_FROM_DICT: + if override.reading == ALL_PERIODS: + target.update(period=None, value=override.value) + elif override.reading == REFORM_FROM_DICT: if override.key is None: target.update(period=CORE_SCALAR_PERIOD, value=override.value) else: diff --git a/policyengine_uk/utils/scenario.py b/policyengine_uk/utils/scenario.py index 46493b72a9..b70e692ac1 100644 --- a/policyengine_uk/utils/scenario.py +++ b/policyengine_uk/utils/scenario.py @@ -1,10 +1,14 @@ +import logging + from pydantic import BaseModel, ConfigDict from typing import Any, Literal, List, Optional, Callable, Dict, Type, Union from policyengine_core.simulations import Simulation from policyengine_core.reforms import Reform from policyengine_uk.utils.dated_reforms import ( + ALL_PERIODS, REFORM_DICT, REFORM_FROM_DICT, + LegacyOverride, ParameterAssignment, apply_legacy_overrides, read_reform_dict, @@ -12,6 +16,11 @@ ) from policyengine_uk.utils.parameters import check_parameter_not_removed +logger = logging.getLogger(__name__) + +# Set on a simulation while Scenario.apply runs its modifier. +APPLYING_TO_BUILT_SIMULATION = "_scenario_apply_to_built_simulation" + class DatedReform(BaseModel): """A parameter reform dict and the route whose key reading it takes. @@ -49,22 +58,32 @@ def _legacy_override_modifier(dated: DatedReform) -> Callable[[Simulation], None def modifier(simulation: Simulation) -> None: from policyengine_uk.tax_benefit_system import statutory_parameter_defines - _, legacy = dated.read(statutory_parameter_defines(), warn=True) + # Scenario.apply edits an already-built simulation's annualised + # parameters, so there every change of the reform is an override. + every_change = getattr(simulation, APPLYING_TO_BUILT_SIMULATION, False) + defines = ( + (lambda path: False) if every_change else statutory_parameter_defines() + ) + _, legacy = dated.read(defines, warn=not every_change) if not legacy: return - system = simulation.tax_benefit_system - if hasattr(system, "add_legacy_overrides"): - # Kept by the system, so a later rebuild applies them again. - system.add_legacy_overrides(legacy) - else: - apply_legacy_overrides(system.parameters, legacy) - system.reset_parameter_caches() - if getattr(simulation, "populations", None) is not None: - simulation._invalidate_all_caches() + _add_overrides(simulation, legacy) return modifier +def _add_overrides(simulation: Simulation, overrides: List[LegacyOverride]) -> None: + system = simulation.tax_benefit_system + if hasattr(system, "add_legacy_overrides"): + # Kept by the system, so a later rebuild applies them again. + system.add_legacy_overrides(overrides) + else: + apply_legacy_overrides(system.parameters, overrides) + system.reset_parameter_caches() + if getattr(simulation, "populations", None) is not None: + simulation._invalidate_all_caches() + + def _apply_reform_class(reform: Type[Reform], simulation: Simulation) -> None: """Apply a structural ``Reform`` class to a simulation's own system. @@ -279,21 +298,34 @@ def modifier(simulation: Simulation) -> None: ) def apply(self, simulation: Simulation) -> None: - """Apply this scenario to a simulation. + """Apply this scenario to an already-built simulation. First applies parameter changes, then runs the simulation modifier if present. - A scenario's ``dated_reforms`` rebuild the simulation's parameters - from the statutory tree first, which discards earlier edits to them. + This edits the simulation's annualised parameters, as it always has, + adding to whatever the simulation already carries. A dated reform's + keys override the annualised values over their dates and are not + annualised again, so a change dated 6 April does not set that fiscal + year here. Pass the scenario (or ``reform=``) when constructing the + ``Simulation`` to have its changes written onto the statutory + timeline. The tax-benefit system keeps the edits, so a later rebuild + of its parameters applies them again. Args: simulation: The simulation to modify """ if self.dated_reforms: - simulation.apply_parameter_assignments(self.dated_assignments()) + logger.warning( + "Scenario.apply edits the built simulation's annualised " + "parameters, so its dated reform keys are not annualised " + "again. Pass the scenario when constructing the Simulation to " + "apply them on the statutory timeline." + ) # Apply parameter changes first + overrides: List[LegacyOverride] = [] if self.parameter_changes: + parameters = simulation.tax_benefit_system.parameters for path, value in self.parameter_changes.items(): if isinstance(value, dict): # Handle nested parameter changes @@ -303,30 +335,31 @@ def apply(self, simulation: Simulation) -> None: full_path = f"{path}.{sub_path}" check_parameter_not_removed(full_path) try: - target = simulation.tax_benefit_system.parameters.get_child( - full_path - ) + parameters.get_child(full_path) except ValueError: # A saved period-valued policy on a removed scalar # path still needs the migration message; valid # children such as male.age remain reformable. check_parameter_not_removed(path) raise - target.update( - period=None, # Apply to all periods - value=sub_value, + # Apply to all periods + overrides.append( + LegacyOverride(full_path, None, sub_value, ALL_PERIODS) ) else: - # Simple parameter change + # Simple parameter change, applied to all periods check_parameter_not_removed(path) - simulation.tax_benefit_system.parameters.get_child(path).update( - period=None, - value=value, # Apply to all periods - ) + overrides.append(LegacyOverride(path, None, value, ALL_PERIODS)) + if overrides: + _add_overrides(simulation, overrides) # Then apply simulation modifier if self.simulation_modifier: - self.simulation_modifier(simulation) + setattr(simulation, APPLYING_TO_BUILT_SIMULATION, True) + try: + self.simulation_modifier(simulation) + finally: + delattr(simulation, APPLYING_TO_BUILT_SIMULATION) def __repr__(self) -> str: """String representation of the Scenario."""