diff --git a/changelog.d/2353.fixed.md b/changelog.d/2353.fixed.md new file mode 100644 index 0000000000..9a11406c91 --- /dev/null +++ b/changelog.d/2353.fixed.md @@ -0,0 +1 @@ +- Fixed a simulation given an input for a year before its data holding no person, benefit unit or household identifiers in that year, so that formulas reading them, such as `months_since_last_birthday` and the Universal Credit deduction draws, gave different values there from the simulation its exported frame loads back to. diff --git a/changelog.d/fix-roundtrip-recorded-inputs.fixed.md b/changelog.d/fix-roundtrip-recorded-inputs.fixed.md new file mode 100644 index 0000000000..69bb07a587 --- /dev/null +++ b/changelog.d/fix-roundtrip-recorded-inputs.fixed.md @@ -0,0 +1 @@ +- Fixed `Simulation.to_input_dataframe()` leaving out the person, benefit unit and household identifiers for years in which a simulation was given inputs after it was built, such as the Universal Credit health element status for 2026 to 2029, so that an exported frame loads back to a simulation that exports the same frame. diff --git a/docs/book/usage/simulations.md b/docs/book/usage/simulations.md index 4128daf55b..2288feb70c 100644 --- a/docs/book/usage/simulations.md +++ b/docs/book/usage/simulations.md @@ -200,6 +200,19 @@ print(sim.calculate("council_tax", 2025)) # One value per household period that is not one year are set on the simulation directly. A variable given for both months and the year is set column by column in the frame's order, since a year's value is spread over the months not already set. +- `to_input_dataframe()` gives the identifier columns for every year that one + of its columns gives a whole year's value for, since a simulation built from + the frame holds the identifiers in each of those years. So an exported frame + loads back to a simulation that exports the same frame, even when the + simulation was given inputs for years outside its data, such as the + Universal Credit health element status the rebalancing records for 2026 to + 2029. +- Inputs carry forward to later years but not back to earlier ones. So when + `set_input` gives a simulation an input for a year before every year it + holds the identifiers for, it also gives the identifiers for that year. + Formulas that read identifiers, such as the draws that place each person in + their year of age, then read the same people and groups there as in the + simulation its exported frame loads back to. ```{important} The model stores `person_household_id` and `person_benunit_id` as 32-bit diff --git a/policyengine_uk/data/input_dataframe.py b/policyengine_uk/data/input_dataframe.py index fcc6b133d1..fae8c23a74 100644 --- a/policyengine_uk/data/input_dataframe.py +++ b/policyengine_uk/data/input_dataframe.py @@ -15,6 +15,16 @@ Identifiers are read as whole numbers, value by value, and a float that may stand for more than one whole number is refused rather than allowed to merge two people or groups. + +The identifiers hold for every period, and a simulation built from a frame +holds them all as inputs in each whole year a column names. +``Simulation.to_input_dataframe`` gives them for each such year too +(``with_identifiers_in_each_year``), so that an exported frame loads back to +a simulation that exports it again. Inputs carry forward but not back, so +``Simulation.set_input`` gives a simulation its identifiers for a year +before every year it holds them for when it is given an input there +(``hold_identifiers_in``). Formulas that read them then read the same people +and groups in that year as the simulation the frame loads back to. """ import warnings @@ -22,7 +32,7 @@ import numpy as np import pandas as pd -from policyengine_core.periods import period as period_ +from policyengine_core.periods import ETERNITY, period as period_ from policyengine_uk.data.dataset_schema import ( UKMultiYearDataset, @@ -267,6 +277,17 @@ def _holds_floats(identifiers: pd.Series) -> bool: return values.dtype.kind == "f" +def _is_whole_year(period) -> bool: + """Whether ``period`` is one calendar year, the period of a dataset + year.""" + start = period.start + return ( + period.unit == "year" + and period.size == 1 + and (start.month, start.day) == (1, 1) + ) + + def _same(left: np.ndarray, right: np.ndarray) -> np.ndarray: """Whether each pair of values is equal, counting two missing values as equal.""" @@ -362,11 +383,7 @@ def no_tables() -> Dict[str, Dict[str, np.ndarray]]: continue period = period_(period_text) start = period.start - whole_year = ( - period.unit == "year" - and period.size == 1 - and (start.month, start.day) == (1, 1) - ) + whole_year = _is_whole_year(period) if variable in IDENTIFIER_VARIABLES: if whole_year: identifier_years.add(start.year) @@ -425,3 +442,137 @@ def no_tables() -> Dict[str, Dict[str, np.ndarray]]: for fiscal_year, tables in sorted(years.items()) ] return UKMultiYearDataset(datasets=datasets), replayed, identifier_inputs + + +def _ignored(variable, period) -> bool: + """Whether policyengine-core's ``Simulation.set_input`` stores nothing + for ``variable`` in ``period``, as it does for a period that starts after + the variable ends.""" + return variable.end is not None and period.start.date > variable.end + + +def hold_identifiers_in(simulation, variable_name: str, period) -> None: + """Give ``simulation`` its identifiers for the year an input of + ``variable_name`` for ``period`` starts in, if they start later. + + policyengine-core carries an input forward to later periods but never + back (``Simulation._calculate``). So a simulation given an input for a + year before its data, such as 2023 for a frame of 2025, would hold no + identifiers there. Formulas would read every person, benefit unit and + household identifier there as zero, the default. Yet once the frame it + exports gives a whole year's value for that year, the frame loads back to + a simulation that holds the people and groups there: each whole year a + column of another variable names is a year of the dataset, which holds + them, and each other year an identifier column names gets the identifiers + as inputs (``dataset_from_input_dataframe``). + + So each identifier that ``simulation`` holds as an input for some period, + but for none that starts by the first day of the year ``period`` starts + in, is set for that year to its value in the period it is held for that + starts first. A simulation holds one set of people and groups, so that + value names the same ones. From that year on the identifier carries + forward, so the simulation holds its identifiers in each year it holds an + input for. An identifier the simulation does not hold as an input, as in + one built from a situation, is left alone. + + An input core does not store, for a period after its variable ends, gives + none. Each identifier is set through ``simulation.set_input``, which + calls this again for the identifier. That call finds the identifier held + for the year, or returns here if core did not store it, so the calls end. + """ + period = period_(period) + variables = simulation.tax_benefit_system.variables + if period.unit == ETERNITY or _ignored(variables[variable_name], period): + return + year = period_(str(period.start.year)) + branch = simulation.branch_name + for identifier in sorted(IDENTIFIER_VARIABLES): + holder = simulation.get_holder(identifier) + held = holder.get_input_periods(branch) + if not held: + continue + first = min(held, key=lambda held_period: held_period.start) + if first.start > year.start: + simulation.set_input( + identifier, year, np.array(holder.get_array(first, branch)) + ) + + +def with_identifiers_in_each_year( + df: pd.DataFrame, tax_benefit_system: "TaxBenefitSystem" +) -> pd.DataFrame: + """``df``, an exported frame, with each identifier it gives for some year + given for every whole calendar year a column of the frame names. + + ``dataset_from_input_dataframe`` holds every identifier in each such year: + a year a column of another variable names is a year of the dataset, which + holds the people and groups, and each other year an identifier column + names gets them all as inputs. So a simulation built from the frame holds + the identifiers as inputs in those years, and exports them there. + + The simulation that exported the frame may not have. One given an input + for a year after its data once it was built, as the Universal Credit + rebalancing records each benefit unit's health element status for 2026 + to 2029, holds its identifiers in that year only by carrying them forward + from an earlier one. (In a year before its data, where inputs do not + carry back, ``hold_identifiers_in`` gives it them as inputs.) + policyengine-core exports an input only for the periods it was set for. + Giving every identifier for each year makes the frame load back to a + simulation that exports the same frame. + + A simulation holds one set of people and groups, so every column of an + identifier names the same ones (the loader refuses a frame whose columns + differ). An identifier added for a year copies its column for the latest + year before it the frame gives, or else the earliest year it gives. Each + goes beside the identifier's other columns, before the first whose period + sorts after its own. The default export sorts each variable's periods, so + that is where it puts an input set for that year. (With + ``include_computed_variables=True``, policyengine-core exports each + variable's periods in the order they are stored, and the frame is not + expected to load back to the same frame.) + + Only columns that name a variable of ``tax_benefit_system`` count, and an + identifier the frame does not give for any year is not added. ``df`` is + not changed: if every identifier it gives is given for each such year, + ``df`` itself is returned, and otherwise a new frame. + """ + named: Dict[str, Dict[int, object]] = {} + years = set() + for column in df.columns: + variable, period_text = split_column(column) + if variable not in tax_benefit_system.variables: + continue + period = period_(period_text) + if not _is_whole_year(period): + continue + if variable in IDENTIFIER_VARIABLES: + named.setdefault(variable, {})[period.start.year] = column + years.add(period.start.year) + added = { + variable: sorted(years - set(columns)) + for variable, columns in named.items() + if years - set(columns) + } + if not added: + return df + result = df.copy(deep=False) + for variable, missing in added.items(): + columns = named[variable] + for year in missing: + earlier = [named_year for named_year in columns if named_year < year] + source = columns[max(earlier) if earlier else min(columns)] + name = f"{variable}{COLUMN_SEPARATOR}{year}" + # The identifier's columns, by period text, as the exporter sorts + # them; the new one goes before the first that sorts after it. + own = [ + position + for position, column in enumerate(result.columns) + if split_column(column)[0] == variable + ] + later = [ + position + for position in own + if split_column(result.columns[position])[1] > str(year) + ] + result.insert(later[0] if later else own[-1] + 1, name, df[source].copy()) + return result diff --git a/policyengine_uk/simulation.py b/policyengine_uk/simulation.py index 2acef5aefe..5f957dca78 100644 --- a/policyengine_uk/simulation.py +++ b/policyengine_uk/simulation.py @@ -32,7 +32,11 @@ reset_growthfactor_uprating, ) from policyengine_uk.data.dataset_sources import materialize_gcs_dataset_url -from policyengine_uk.data.input_dataframe import dataset_from_input_dataframe +from policyengine_uk.data.input_dataframe import ( + dataset_from_input_dataframe, + hold_identifiers_in, + with_identifiers_in_each_year, +) from policyengine_uk.utils.dependencies import get_variable_dependencies from policyengine_uk.utils.supplied_inputs import ( SUPPLIED_INPUT_VARIABLES, @@ -247,6 +251,14 @@ def delete_arrays(self, variable: str, period: Period = None) -> None: # and the current branch, ancestor branches and default branch. drop_missing_supplied_inputs(self, variable) + def set_input(self, variable_name: str, period, value) -> None: + """Set an input as policyengine-core does, then give the simulation + its identifiers for the year the input starts in if it holds them + only for later years, since inputs do not carry back. See + ``policyengine_uk.data.input_dataframe.hold_identifiers_in``.""" + super().set_input(variable_name, period, value) + hold_identifiers_in(self, variable_name, period) + def reset_calculations(self): for variable in self.tax_benefit_system.variables: if variable not in self.input_variables: @@ -426,6 +438,30 @@ def build_from_dataframe(self, df: pd.DataFrame) -> None: for variable, time_period, values in replayed: self.set_input(variable, time_period, values) + def to_input_dataframe( + self, include_computed_variables: bool = False + ) -> pd.DataFrame: + """Export the simulation's inputs as a DataFrame that + ``Simulation(dataset=...)`` loads back. + + policyengine-core exports each input for the periods it was set for. + The identifiers are also given for every whole year a column names, + where a simulation built from the frame holds them, so that the + default export loads back to a simulation that exports the same + frame. See + ``policyengine_uk.data.input_dataframe.with_identifiers_in_each_year``. + + Args: + include_computed_variables: If True, export every stored value, + calculated ones included, as policyengine-core does. + """ + return with_identifiers_in_each_year( + super().to_input_dataframe( + include_computed_variables=include_computed_variables + ), + self.tax_benefit_system, + ) + def build_from_dataset(self, dataset: Dataset) -> None: """Build simulation from a Dataset object. diff --git a/policyengine_uk/tests/code_health/test_simulation_methods_defined_once.py b/policyengine_uk/tests/code_health/test_simulation_methods_defined_once.py new file mode 100644 index 0000000000..a5ad69bc84 --- /dev/null +++ b/policyengine_uk/tests/code_health/test_simulation_methods_defined_once.py @@ -0,0 +1,77 @@ +"""Each method of the simulation classes is defined once. + +Pull requests often override methods of ``policyengine_uk.simulation.Simulation`` +(``set_input``, ``delete_arrays``, ``clone`` and others). Two overrides of one +method, added at different places in the class, merge without a git conflict, +and the later definition replaces the earlier one: whatever the earlier one +did is lost without an error. Combine them into one method instead. + +``ruff check`` reports this as F811 when it runs. This test guards the merged +class with the tests themselves, wherever lint is not run. Property setters +and deleters, and ``typing.overload`` stubs, name a method again on purpose +and are left out. +""" + +import ast +from collections import Counter +from pathlib import Path + +import pytest + +PACKAGE = Path(__file__).resolve().parents[2] +MODULES = ["simulation.py", "microsimulation.py"] + + +def _names_again(decorator: ast.expr) -> bool: + """Whether ``decorator`` defines a method under a name already used.""" + if isinstance(decorator, ast.Attribute): + return decorator.attr in ("setter", "deleter", "getter") + return isinstance(decorator, ast.Name) and decorator.id == "overload" + + +def methods_defined_more_than_once(source: str) -> dict: + """The methods each class in ``source`` defines more than once, by + class.""" + repeated = {} + for node in ast.walk(ast.parse(source)): + if not isinstance(node, ast.ClassDef): + continue + names = Counter( + method.name + for method in node.body + if isinstance(method, (ast.FunctionDef, ast.AsyncFunctionDef)) + and not any(map(_names_again, method.decorator_list)) + ) + twice = sorted(name for name, count in names.items() if count > 1) + if twice: + repeated[node.name] = twice + return repeated + + +@pytest.mark.parametrize("module", MODULES) +def test_each_method_of_the_simulation_classes_is_defined_once(module): + repeated = methods_defined_more_than_once((PACKAGE / module).read_text()) + assert not repeated, ( + f"{module} defines these methods more than once, so only the last " + f"definition of each takes effect: {repeated}. Combine them." + ) + + +def test_a_method_defined_twice_is_found(): + source = """ +class Simulation: + def set_input(self): + pass + + @property + def size(self): + pass + + @size.setter + def size(self, value): + pass + + def set_input(self): + pass +""" + assert methods_defined_more_than_once(source) == {"Simulation": ["set_input"]} diff --git a/policyengine_uk/tests/test_dataframe_dataset.py b/policyengine_uk/tests/test_dataframe_dataset.py index c3793a4f51..bcdaada276 100644 --- a/policyengine_uk/tests/test_dataframe_dataset.py +++ b/policyengine_uk/tests/test_dataframe_dataset.py @@ -30,6 +30,13 @@ 5. Round trip: the frame a simulation exports loads back to the same people, groups and inputs, and exports the same frame again; so does that frame saved as CSV, which holds the exported 32-bit identifiers as 64-bit floats. + This holds for frames that name the identifiers and give values for + different years, and for simulations given inputs for other years once + built, as the Universal Credit rebalancing gives each benefit unit's + health element status for 2026 to 2029. The export gives every + identifier for each whole year a column names, which is where a + simulation built from it holds them, and person_id and the benefit unit + and household identifiers exactly. 6. Differential: a simulation built from a dataset and one built from the frame it exports give the same value for every variable of the model. 7. Exact names: a column is read only if the part of its name before ``__`` @@ -59,7 +66,10 @@ anything, since it also exports every calculated value. """ +import datetime +import inspect import io +import re import warnings from dataclasses import dataclass from typing import Dict @@ -67,7 +77,7 @@ import numpy as np import pandas as pd import pytest -from hypothesis import HealthCheck, given, settings +from hypothesis import HealthCheck, example, given, settings from hypothesis import strategies as st from policyengine_uk import CountryTaxBenefitSystem, Microsimulation, Simulation @@ -77,6 +87,7 @@ IDENTIFIER_VARIABLES, dataset_from_input_dataframe, split_column, + with_identifiers_in_each_year, ) from policyengine_uk.scenarios import no_economic_assumptions @@ -601,6 +612,64 @@ def test_a_frame_that_names_no_whole_year_is_held_in_the_default_input_year(): ) == {1: True} +def test_an_export_gives_its_identifiers_for_each_year_it_gives_values_for( + tax_benefit_system, +): + # Values for 2024, 2025, 2027 and 2029 make those years of the dataset, + # which holds the identifiers in each, and the loader sets them all for + # 2028, which an identifier names. A month and a column that names no + # variable add no year. Each added column copies the identifier's latest + # earlier column, or else its earliest: here person_id__2028 is given as + # floats, so person_id__2029 is too. + frame_ = pd.DataFrame( + { + "benunit_id__2025": [7, 7], + "person_id__2025": [1, 2], + "person_id__2028": [1.0, 2.0], + "age__2024": [39, 37], + "age__2025": [40, 38], + "age__2027": [42, 40], + "ebr_energy_bills_credit_eligible__2026-01": [True, True], + "council_tax__2029": [1_500.0, 1_500.0], + "previous_household_id__2030": [5, 5], + } + ) + before = frame_.copy() + + exported = with_identifiers_in_each_year(frame_, tax_benefit_system) + + assert exported.columns.tolist() == [ + "benunit_id__2024", + "benunit_id__2025", + "benunit_id__2027", + "benunit_id__2028", + "benunit_id__2029", + "person_id__2024", + "person_id__2025", + "person_id__2027", + "person_id__2028", + "person_id__2029", + *frame_.columns[3:], + ] + for identifier, year, source in [ + ("person_id", 2024, 2025), + ("person_id", 2027, 2025), + ("person_id", 2029, 2028), + ("benunit_id", 2028, 2025), + ]: + pd.testing.assert_series_equal( + exported[f"{identifier}__{year}"], + frame_[f"{identifier}__{source}"], + check_names=False, + ) + pd.testing.assert_frame_equal(exported[frame_.columns], frame_) + pd.testing.assert_frame_equal(frame_, before) + # A frame that gives its identifiers for each such year gains nothing. + pd.testing.assert_frame_equal( + with_identifiers_in_each_year(exported, tax_benefit_system), exported + ) + + # The dataset a frame-built simulation holds. @@ -1140,6 +1209,29 @@ def test_members_may_all_leave_a_group_value_missing(tax_benefit_system): suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large], ) ROUND_TRIP_SETTINGS = settings(SIMULATION_SETTINGS, max_examples=15) +# The smallest example of the round trip failing once the Universal Credit +# rebalancing recorded each benefit unit's health element status for 2026 to +# 2029 (#2251). The export gave that status for those years but the person +# and their groups for 2025 alone. Loaded back, the frame was a dataset with a +# year for each of 2025 to 2029, which holds the identifiers in each, so the +# second export gave them for 2026 to 2029 as well: 38 columns, not 18. +ONE_PERSON = Population( + person_ids=np.array([1]), + groups={"benunit": np.array([1]), "household": np.array([1])}, + values={ + "age": np.array([0]), + "hours_worked": np.array([0]), + "gender": np.array(["MALE"]), + "is_disabled_for_benefits": np.array([False]), + "benunit_reported_capital": np.array([0]), + "would_claim_uc": np.array([False]), + "council_tax": np.array([0]), + "household_weight": np.array([0]), + "region": np.array(["LONDON"]), + }, +) +# Years around the data year, for frames and inputs that span several. +YEARS = list(range(YEAR - 1, YEAR + 6)) TABLE_SETTINGS = settings( max_examples=200, deadline=None, @@ -1165,6 +1257,7 @@ def test_a_frame_builds_its_people_groups_and_values(population, id_columns): @ROUND_TRIP_SETTINGS @given(population=populations(), id_columns=id_columns) +@example(population=ONE_PERSON, id_columns="both") def test_an_exported_frame_loads_back_and_exports_itself(population, id_columns): simulation = Simulation(dataset=frame(population, id_columns)) exported = simulation.to_input_dataframe() @@ -1187,6 +1280,463 @@ def test_an_exported_frame_loads_back_and_exports_itself(population, id_columns) assert_holds(Simulation(dataset=from_csv), population) +def columns_of(frame_: pd.DataFrame, variable: str) -> list: + return [column for column in frame_ if split_column(column)[0] == variable] + + +def test_an_export_names_the_people_in_each_year_the_rebalancing_records(): + # The rebalancing records the status of a simulation built from data for + # 2025 for 2026 to 2029, where it holds its people and groups by carrying + # them forward from 2025. + exported = Simulation(dataset=frame(ONE_PERSON)).to_input_dataframe() + + status = "uc_receives_new_claimant_health_element" + years = range(YEAR, 2030) + assert columns_of(exported, status) == [f"{status}__{year}" for year in years[1:]] + for identifier in IDENTIFIER_VARIABLES: + columns = columns_of(exported, identifier) + assert columns == [f"{identifier}__{year}" for year in years] + # Together, where the exporter puts an input set for each year. + start = exported.columns.get_loc(columns[0]) + assert exported.columns[start : start + len(columns)].tolist() == columns + for column in columns: + assert exported[column].tolist() == [1] + reloaded = Simulation(dataset=exported) + pd.testing.assert_frame_equal(reloaded.to_input_dataframe(), exported) + + +@pytest.mark.parametrize( + "variable, year, value", + [ + ("hours_worked", YEAR - 1, 1_000), + ("hours_worked", YEAR + 2, 1_000), + ("hours_worked", YEAR + 7, 1_000), + # One identifier for a year: the loader sets them all for it. + ("person_id", YEAR + 7, 1), + ], +) +def test_an_input_set_once_built_for_another_year_round_trips(variable, year, value): + # Not only the rebalancing: any input set for a year the data does not + # give, before it, among the rebalancing's years or after them. + simulation = Simulation(dataset=frame(ONE_PERSON)) + simulation.set_input(variable, year, np.array([value])) + exported = simulation.to_input_dataframe() + + for identifier in IDENTIFIER_VARIABLES: + assert exported[f"{identifier}__{year}"].tolist() == [1] + reloaded = Simulation(dataset=exported) + assert reloaded.calculate(variable, year).tolist() == [value] + pd.testing.assert_frame_equal(reloaded.to_input_dataframe(), exported) + + +def multi_year_frame( + population: Population, id_columns: str, identifier_years, data_years +) -> pd.DataFrame: + """The frame of ``population`` with its identifiers given for + ``identifier_years`` and its values for ``data_years``.""" + columns = {} + for year in sorted(set(identifier_years) | set(data_years)): + for column, values in frame(population, id_columns, year).items(): + identifier = split_column(column)[0] in IDENTIFIER_VARIABLES + if year in (identifier_years if identifier else data_years): + columns[column] = values + return pd.DataFrame(columns) + + +@ROUND_TRIP_SETTINGS +@given( + population=populations(), + id_columns=id_columns, + identifier_years=st.sets(st.sampled_from(YEARS), min_size=1, max_size=2), + data_years=st.sets(st.sampled_from(YEARS), min_size=1, max_size=3), + later_years=st.sets(st.sampled_from(YEARS), max_size=2), + later_identifier_years=st.sets(st.sampled_from(YEARS), max_size=1), +) +@example( + population=ONE_PERSON, + id_columns="both", + identifier_years={YEAR}, + data_years={YEAR}, + later_years=set(), + later_identifier_years=set(), +) +@example( + population=ONE_PERSON, + id_columns="own", + identifier_years={YEAR + 2}, + data_years={YEAR - 1, YEAR + 3}, + later_years={YEAR + 5}, + later_identifier_years={YEAR + 4}, +) +def test_a_frame_over_several_years_exports_the_same_frame_again( + population, + id_columns, + identifier_years, + data_years, + later_years, + later_identifier_years, +): + # 5. Round trip, for frames that name the identifiers and give values + # for different years, and simulations given an input, or a person's + # identifier, for other years once built. + simulation = Simulation( + dataset=multi_year_frame(population, id_columns, identifier_years, data_years) + ) + for year in sorted(later_years): + simulation.set_input( + "hours_worked", year, population.values["hours_worked"] + year + ) + for year in sorted(later_identifier_years): + simulation.set_input("person_id", year, population.person_ids) + exported = simulation.to_input_dataframe() + + reloaded = Simulation(dataset=exported) + + pd.testing.assert_frame_equal(reloaded.to_input_dataframe(), exported) + # The export gives the exact identifiers in every whole year it names. + years = { + int(period) + for _, period in map(split_column, exported.columns) + if period.isdigit() + } + assert years >= set(data_years) | set(later_years) | set(identifier_years) + for year in years: + assert exported[f"person_id__{year}"].tolist() == population.person_ids.tolist() + for group in GROUPS: + exact = population.groups[group].tolist() + assert exported[f"{group}_id__{year}"].tolist() == exact + + +# A formula that reads an identifier, such as ``person("person_id", period)``. +READS_AN_IDENTIFIER = re.compile( + r"""\(\s*["'](?:%s)["']""" % "|".join(sorted(IDENTIFIER_VARIABLES)) +) + + +@pytest.fixture(scope="module") +def identifier_readers(tax_benefit_system) -> list: + """The variables whose formulas read an identifier.""" + readers = sorted( + name + for name, variable in tax_benefit_system.variables.items() + if any( + READS_AN_IDENTIFIER.search(inspect.getsource(formula)) + for formula in variable.formulas.values() + ) + ) + # The scan finds at least the readers known when it was written. + assert set(readers) >= { + "is_benunit_eldest_child", + "months_since_last_birthday", + "uc_deduction_random_draw", + "uc_deduction_type_random_draw", + } + return readers + + +def give_values_for( + simulation, population: Population, year: int, weight: int = 1 +) -> None: + """Set each of ``population``'s values on ``simulation`` for ``year``, with + household weights ``weight`` times theirs. + + A weight of a million makes the benefit units of any household of weight + one or more add up to the weight of representative data, for which the + Universal Credit deduction draws read the benefit unit identifiers. + """ + for variable, entity in VARIABLES.items(): + by_id = population.by_id(variable) + values = np.array([by_id[id_] for id_ in ids(simulation, entity)]) + if variable == "household_weight": + values = values * weight + simulation.set_input(variable, year, values) + + +# Years before the data year, the data year and years after it. +YEARS_AROUND = list(range(YEAR - 3, YEAR + 3)) +# Two children of one age in a benefit unit, in a household of weight one. +# The tie on age falls to their identifiers (is_benunit_eldest_child), the +# draw that places each in their year of age reads them +# (months_since_last_birthday), and at a million times the weight so do the +# Universal Credit deduction draws. Before simulations held their identifiers +# in a year before their data, each of the four differed there between a +# simulation and the one its export loads back to. +TWINS = Population( + person_ids=np.array([1, 2]), + groups={"benunit": np.array([1, 1]), "household": np.array([1, 1])}, + values={ + "age": np.array([10, 10]), + "hours_worked": np.array([0, 0]), + "gender": np.array(["MALE", "MALE"]), + "is_disabled_for_benefits": np.array([False, False]), + "benunit_reported_capital": np.array([0, 0]), + "would_claim_uc": np.array([False, False]), + "council_tax": np.array([0, 0]), + "household_weight": np.array([1, 1]), + "region": np.array(["LONDON", "LONDON"]), + }, +) + + +@ROUND_TRIP_SETTINGS +@given( + population=populations(), + id_columns=id_columns, + input_years=st.sets(st.sampled_from(YEARS_AROUND), min_size=1, max_size=2), + weight=st.sampled_from([1, 1_000_000]), +) +# The smallest example of the identifiers differing: one person's draw for +# their place in their year of age, in a year before the data. +@example(population=ONE_PERSON, id_columns="both", input_years={YEAR - 3}, weight=1) +@example(population=TWINS, id_columns="own", input_years={YEAR - 1}, weight=1_000_000) +def test_a_reloaded_simulation_reads_the_same_identifiers_in_each_year_given_inputs( + identifier_readers, population, id_columns, input_years, weight +): + # 10. Identifiers in each year: a simulation built from a frame and given + # values for other years, before the data year or after it, and the + # simulation its export loads back to compute the same values from the + # identifiers in each of those years. Both hold the people and groups' + # own identifiers there. + simulation = Simulation(dataset=frame(population, id_columns)) + for year in sorted(input_years): + give_values_for(simulation, population, year, weight) + exported = simulation.to_input_dataframe() + + reloaded = Simulation(dataset=exported) + + for year in sorted(input_years): + for variable in identifier_readers: + np.testing.assert_array_equal( + reloaded.calculate(variable, year), + simulation.calculate(variable, year), + err_msg=f"{variable} for {year}", + ) + for built in (simulation, reloaded): + people = built.calculate("person_id", year) + assert people.tolist() == population.person_ids.tolist() + for group in GROUPS: + assert built.calculate(f"{group}_id", year).tolist() == ids( + built, group + ) + membership = built.calculate(f"person_{group}_id", year) + exact = population.groups[group].astype(np.float32) + assert membership.tolist() == exact.tolist() + pd.testing.assert_frame_equal(reloaded.to_input_dataframe(), exported) + + +def identifier_inputs(simulation, branch: str = "default") -> dict: + """The periods each identifier is held for as an input, by identifier.""" + return { + identifier: sorted( + str(period) + for period in simulation.get_holder(identifier).get_input_periods(branch) + ) + for identifier in sorted(IDENTIFIER_VARIABLES) + } + + +def test_an_input_before_the_data_gives_the_identifiers_for_its_year(): + simulation = Simulation(dataset=frame(TWINS)) + data_years = [str(YEAR)] + assert set(map(tuple, identifier_inputs(simulation).values())) == { + tuple(data_years) + } + + # A month two years before the data: the identifiers are given for its + # year, and carry forward from there to the next. + simulation.set_input( + "ebr_energy_bills_credit_eligible", f"{YEAR - 2}-01", np.array([True]) + ) + + for periods in identifier_inputs(simulation).values(): + assert periods == [str(YEAR - 2), *data_years] + for year in (YEAR - 2, YEAR - 1): + assert simulation.calculate("person_id", year).tolist() == [1, 2] + assert simulation.calculate("benunit_id", year).tolist() == [1] + assert simulation.calculate("person_household_id", year).tolist() == [1, 1] + # Before that year the simulation holds no input, nor any identifier. + assert simulation.calculate("person_id", YEAR - 3).tolist() == [0, 0] + # An input for a later year gives none: they carry forward to it. + simulation.set_input("hours_worked", YEAR + 7, np.array([1, 2])) + for periods in identifier_inputs(simulation).values(): + assert periods == [str(YEAR - 2), *data_years] + assert simulation.calculate("person_id", YEAR + 7).tolist() == [1, 2] + # The month and the identifiers for its year load back as given. + exported = simulation.to_input_dataframe() + assert exported[f"person_id__{YEAR - 2}"].tolist() == [1, 2] + reloaded = Simulation(dataset=exported) + for periods in identifier_inputs(reloaded).values(): + assert periods[0] == str(YEAR - 2) + assert reloaded.calculate("person_id", YEAR - 1).tolist() == [1, 2] + pd.testing.assert_frame_equal(reloaded.to_input_dataframe(), exported) + + +def test_an_identifier_given_alone_before_the_data_brings_the_others(): + # One identifier for an earlier year, as a frame can name for a year it + # gives no values for: the person's groups are given for it too. + simulation = Simulation(dataset=frame(TWINS)) + simulation.set_input("person_id", YEAR - 1, np.array([1, 2])) + + for periods in identifier_inputs(simulation).values(): + assert periods[0] == str(YEAR - 1) + assert simulation.calculate("household_id", YEAR - 1).tolist() == [1] + assert simulation.calculate("person_benunit_id", YEAR - 1).tolist() == [1, 1] + + +def test_a_branch_given_an_input_before_the_data_holds_the_identifiers_itself(): + simulation = Simulation(dataset=frame(TWINS)) + branch = simulation.get_branch("earlier") + branch.set_input("hours_worked", YEAR - 1, np.array([10, 20])) + + assert branch.calculate("person_id", YEAR - 1).tolist() == [1, 2] + assert identifier_inputs(branch, "earlier")["person_id"][0] == str(YEAR - 1) + # The branch it was made from holds no input for that year, nor any + # identifier. + assert identifier_inputs(simulation)["person_id"][0] == str(YEAR) + assert simulation.calculate("person_id", YEAR - 1).tolist() == [0, 0] + + +@pytest.fixture +def ending(): + """Give a variable of a simulation an end date for one test, then take + it away: ``ending(simulation, variable, year)`` ends it on the last day + of ``year``.""" + ended = [] + + def end(simulation, name: str, year: int) -> None: + variable = simulation.tax_benefit_system.get_variable(name) + ended.append((variable, variable.end)) + variable.end = datetime.date(year, 12, 31) + + yield end + for variable, end in ended: + variable.end = end + + +def test_an_input_core_stores_nothing_for_gives_no_identifiers(ending): + # policyengine-core ignores an input for a period after its variable + # ends, so the simulation holds no input in that year. + simulation = Simulation(dataset=frame(TWINS)) + ending(simulation, "hours_worked", YEAR - 3) + + simulation.set_input("hours_worked", YEAR - 2, np.array([1, 2])) + + assert set(map(tuple, identifier_inputs(simulation).values())) == {(str(YEAR),)} + + +def test_an_identifier_core_would_not_store_is_left_out(ending): + # Core stores nothing for an identifier that has ended. The attempt to + # give it ends there, since an input core ignores gives no identifiers, + # and the other identifiers are still given. + simulation = Simulation(dataset=frame(TWINS)) + ending(simulation, "household_id", YEAR - 3) + + simulation.set_input("hours_worked", YEAR - 2, np.array([1, 2])) + + held = identifier_inputs(simulation) + assert held.pop("household_id") == [str(YEAR)] + for periods in held.values(): + assert periods == [str(YEAR - 2), str(YEAR)] + + +def test_a_simulation_that_holds_no_identifiers_is_given_none(): + # A household situation names its people and groups but sets no + # identifier as an input, so an input for another year gives none. + simulation = Simulation( + situation={ + "people": {"adult": {"age": {YEAR: 40}}}, + "benunits": {"benunit": {"members": ["adult"]}}, + "households": {"household": {"members": ["adult"]}}, + } + ) + simulation.set_input("hours_worked", YEAR - 1, np.array([10])) + + assert identifier_inputs(simulation) == { + identifier: [] for identifier in sorted(IDENTIFIER_VARIABLES) + } + + +@pytest.mark.parametrize("descending", [False, True]) +def test_a_dataset_gives_the_identifiers_for_its_own_years_only(descending): + # As H5 files load: the identifiers are inputs in each year of the + # dataset, and loading gives them for no other year, in whichever order + # the dataset holds its years. + built = Simulation( + dataset=multi_year_frame(TWINS, "both", {YEAR}, {YEAR, YEAR + 2}) + ).dataset + years = [YEAR + 2, YEAR] if descending else [YEAR, YEAR + 2] + dataset = UKMultiYearDataset(datasets=[built[year] for year in years]) + + simulation = Simulation(dataset=dataset) + + assert identifier_inputs(simulation) == { + identifier: [str(YEAR), str(YEAR + 2)] + for identifier in sorted(IDENTIFIER_VARIABLES) + } + for year in (YEAR, YEAR + 2): + assert simulation.calculate("person_id", year).tolist() == [1, 2] + + +@TABLE_SETTINGS +@given( + population=populations(), + id_columns=id_columns, + identifier_years=st.sets(st.sampled_from(YEARS), min_size=1, max_size=2), + data_years=st.sets(st.sampled_from(YEARS), max_size=3), + lone_years=st.sets(st.sampled_from(YEARS), max_size=1), + month_years=st.sets(st.sampled_from(YEARS), max_size=1), +) +def test_the_export_and_the_loader_find_the_same_years( + tax_benefit_system, + population, + id_columns, + identifier_years, + data_years, + lone_years, + month_years, +): + # The export gives each identifier for the years a simulation built from + # the frame holds it in: the dataset's years and the years only an + # identifier column names. The loader and the export each work these out + # from the frame's columns, so the round trip relies on them agreeing. + # A person_id column alone names a year for every identifier; months + # name no year in either. + frame_ = multi_year_frame(population, id_columns, identifier_years, data_years) + for year in sorted(lone_years): + frame_[f"person_id__{year}"] = population.person_ids + for year in sorted(month_years): + frame_[f"ebr_energy_bills_credit_eligible__{year}-01"] = True + + dataset, _, identifier_inputs = dataset_from_input_dataframe( + frame_, tax_benefit_system, YEAR + ) + exported = with_identifiers_in_each_year(frame_, tax_benefit_system) + + held = {int(year) for year in dataset.years} | { + int(period) for _, period, _ in identifier_inputs + } + assert held == set(identifier_years) | set(data_years) | set(lone_years) + named = {split_column(column)[0] for column in frame_} + for identifier in IDENTIFIER_VARIABLES: + given_for = { + int(split_column(column)[1]) for column in columns_of(exported, identifier) + } + assert given_for == (held if identifier in named else set()) + + +@pytest.mark.parametrize("year", [YEAR - 2, YEAR + 2]) +def test_an_export_whose_identifiers_differ_between_years_is_refused(year): + # The export copies an identifier into the years it does not give it + # for; that never makes columns that differ agree, so the loader still + # refuses them. + simulation = Simulation(dataset=frame(TWINS)) + simulation.set_input("person_id", year, np.array([1, 3])) + exported = simulation.to_input_dataframe() + + with pytest.raises(ValueError, match="differ"): + Simulation(dataset=exported) + + @TABLE_SETTINGS @given(population=populations(), id_columns=id_columns, data=st.data()) def test_tables_hold_the_frame_in_any_row_order(