From fcd4c4a0c16769d2bb6dd055a4bee363deefb9f1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 3 Oct 2026 08:09:08 -0400 Subject: [PATCH 1/6] Break equal-start uprating ties as auto-carry-over does Two earlier inputs in a variable's own unit can start on the same day: Holder.set_input stores a year:2012:2 input to a yearly variable as given (a set_input helper is only called for another unit), beside one for 2012. The uprating source was max(..., key=start), so the tie went to whichever was stored first, or to the one in memory over the one on disk: 2015 came out as [1.115, 1.115] or [5.576, 5.576] by set order. Move #562's carry-over sort key into _latest_input_key and use it for the uprating source too: on a tie, the input that ends last, then the larger unit, then the period's string form. Distinct periods never tie, so the source depends only on the stored inputs. The factor still runs from the source's start, and the candidates are unchanged (own unit, starting before the period, inputs only). Tests: regressions for both set orders with and without a set_input helper, monthly inputs, a 10-year input whose string sorts first, memory versus disk and a branch input; Hypothesis properties for set-order and disk-placement independence and for flat-index uprating agreeing with auto-carry-over; the fixture's reference rule breaks ties the same way. Review (GPT-6.1 Sol, f051cc28): the key's period.stop raised OverflowError for a period ending after 9999-12-31 (day:9999-12-30:3), where the start-only key did not. The key now orders ends with #562's _end_order, which puts such a period last; regressions use a daily variable. The flat-uprating differential now skips only the case where the two paths really diverge. Co-Authored-By: Claude Opus 5.5 --- changelog.d/fix-uprating-equal-start.fixed.md | 1 + policyengine_core/simulations/simulation.py | 55 ++-- tests/core/test_uprating_order.py | 171 +++++++++++++ tests/core/test_uprating_order_property.py | 238 +++++++++++++++++- tests/fixtures/uprating_order.py | 81 +++++- 5 files changed, 510 insertions(+), 36 deletions(-) create mode 100644 changelog.d/fix-uprating-equal-start.fixed.md diff --git a/changelog.d/fix-uprating-equal-start.fixed.md b/changelog.d/fix-uprating-equal-start.fixed.md new file mode 100644 index 000000000..4aec72b37 --- /dev/null +++ b/changelog.d/fix-uprating-equal-start.fixed.md @@ -0,0 +1 @@ +When two earlier inputs start on the same day (such as `2012` and `year:2012:2`), uprating now starts from the one that ends last, the input auto-carry-over would carry, so the uprated value no longer depends on which input was stored first or on whether it is in memory or on disk. diff --git a/policyengine_core/simulations/simulation.py b/policyengine_core/simulations/simulation.py index 1a21bc8fc..ba8c400d2 100644 --- a/policyengine_core/simulations/simulation.py +++ b/policyengine_core/simulations/simulation.py @@ -135,6 +135,28 @@ def _end_order(period: Period) -> float: return days_before_year + _DAYS_BEFORE_MONTH[month - 1] + (leap and month > 2) + day +def _latest_input_key(period: Period, definition_period: str) -> tuple: + """Sort key for the stored input that auto-carry-over or uprating reads. + + The input at the variable's own definition-period unit beats one in + another unit; then the one that starts last wins; on a tie, the one that + ends last (``_end_order``), then the larger unit, then the period's + string form. Distinct periods never tie, so the input chosen depends only + on which periods are stored, never on the order they were stored in (or + on whether a value is in memory or on disk). Compare ``period.start`` + (temporal order): sorting Period tuples lexicographically puts "year" + before "month" alphabetically, so a known "2023" annual value would win + over a later "2024-06" monthly value (bug H1). + """ + return ( + period.unit == definition_period, + period.start, + _end_order(period), + periods.unit_weight(period.unit), + str(period), + ) + + if TYPE_CHECKING: from policyengine_core.taxbenefitsystems import TaxBenefitSystem @@ -957,9 +979,15 @@ def _calculate(self, variable_name: str, period: Period = None) -> ArrayLike: # Take the latest period from the filtered list itself. # Indexing ``known_periods`` with a position in the # filtered list picked the wrong period whenever a later - # one was stored first. + # one was stored first. Two inputs can start on the same + # day: a yearly variable stores a ``year:2012:2`` input as + # given, beside one for ``2012``. Break that tie as + # auto-carry-over does (the one that ends last), so the + # source never depends on which was stored first. The + # factor below runs from the source's start either way. latest_known_period = max( - earlier_input_periods, key=lambda p: p.start + earlier_input_periods, + key=lambda p: _latest_input_key(p, variable.definition_period), ) try: uprating_parameter = get_parameter( @@ -996,16 +1024,11 @@ def _calculate(self, variable_name: str, period: Period = None) -> ArrayLike: ): # Carry over the latest input: of the stored periods that # start no later than ``period``, the one that starts last - # (on a tie, the one that ends last, then the larger unit, - # so the choice never depends on the order inputs were - # stored), preferring periods at - # the variable's own definition-period unit and using - # another unit only when there is none, as for an input to - # a variable with no ``set_input`` helper. Compare - # period.start (temporal order): sorting Period tuples - # lexicographically puts "year" before "month" - # alphabetically, so a known "2023" annual value would win - # over a later "2024-06" monthly value (bug H1). + # (on a tie, the one that ends last, then the larger unit; + # see ``_latest_input_key``), preferring periods at the + # variable's own definition-period unit and using another + # unit only when there is none, as for an input to a + # variable with no ``set_input`` helper. # # Only inputs carry. Every value this simulation # calculated is marked derived when cached (formula @@ -1025,13 +1048,7 @@ def _calculate(self, variable_name: str, period: Period = None) -> ArrayLike: ) if input_period.start <= period.start ), - key=lambda p: ( - p.unit == variable.definition_period, - p.start, - _end_order(p), - periods.unit_weight(p.unit), - str(p), - ), + key=lambda p: _latest_input_key(p, variable.definition_period), default=None, ) if last_known_period is not None: diff --git a/tests/core/test_uprating_order.py b/tests/core/test_uprating_order.py index 05285239e..274e8d1c1 100644 --- a/tests/core/test_uprating_order.py +++ b/tests/core/test_uprating_order.py @@ -27,6 +27,12 @@ Each regression here compares with a fresh simulation that calculates only the period in question, byte for byte. +Two earlier inputs can also start on the same day: a yearly variable stores a +``year:2012:2`` input as given, beside one for ``2012`` (a ``set_input`` +helper is only called for an input in another unit). The uprating source was +then whichever was stored first, or the one in memory over the one on disk. +It is now the one that ends last, the input auto-carry-over would carry. + A yearly input for a monthly variable is stored month by month by its ``set_input`` helper. The helpers took a month the simulation had calculated for an input and kept it, so the uprating rule above skipped it. They now @@ -51,6 +57,7 @@ assert_bitwise_equal, build_system, index, + reference, simulation, ) @@ -293,6 +300,170 @@ def test_uprated_value_is_cached_as_derived(system): assert not holder.is_derived(periods.period("2012")) +# Inputs that start on the same day. + +TIED = {"year:2012:2": [1.0, 2.0], "2012": [5.0, 6.0]} + + +def _in_order(values, longer_first): + order = list(values) if longer_first else list(values)[::-1] + return {key: values[key] for key in order} + + +@pytest.mark.parametrize("variable", ["uprated_any_unit", "uprated"]) +@pytest.mark.parametrize("longer_first", [True, False], ids=["longer", "shorter"]) +def test_inputs_starting_together_uprate_the_one_ending_last( + system, variable, longer_first +): + """``uprated`` has a ``set_input`` helper and ``uprated_any_unit`` none: + both store ``year:2012:2`` as given.""" + inputs = {variable: _in_order(TIED, longer_first)} + result = alone(system, inputs, variable, "2015") + # Before the fix: [5, 6] uprated whenever 2012 was stored first. + np.testing.assert_allclose( + result, np.array([1, 2]) * index(2015) / index(2012), rtol=1e-6 + ) + assert_bitwise_equal(result, reference(system, inputs, variable, "2015")) + + +@pytest.mark.parametrize("reverse", [False, True], ids=["forward", "reverse"]) +def test_inputs_starting_together_uprate_the_longest_not_the_first_by_name( + system, reverse +): + """``year:2012:10`` ends last; as a string it sorts between ``2012`` and + ``year:2012:2``.""" + values = {"2012": [5.0, 6.0], "year:2012:10": [1.0, 2.0], "year:2012:2": [3, 4]} + order = list(values)[::-1] if reverse else list(values) + inputs = {"uprated_any_unit": {key: values[key] for key in order}} + result = alone(system, inputs, "uprated_any_unit", "2015") + np.testing.assert_allclose( + result, np.array([1, 2]) * index(2015) / index(2012), rtol=1e-6 + ) + assert_bitwise_equal(result, reference(system, inputs, "uprated_any_unit", "2015")) + + +@pytest.mark.parametrize("longer_first", [True, False], ids=["longer", "shorter"]) +def test_monthly_inputs_starting_together_uprate_the_one_ending_last( + system, longer_first +): + values = {"month:2012-01:3": [1.0, 2.0], "2012-01": [5.0, 6.0]} + inputs = {"uprated_monthly": _in_order(values, longer_first)} + result = alone(system, inputs, "uprated_monthly", "2014-03") + np.testing.assert_allclose( + result, np.array([1, 2]) * index(2014) / index(2012), rtol=1e-6 + ) + assert_bitwise_equal( + result, reference(system, inputs, "uprated_monthly", "2014-03") + ) + + +@pytest.mark.filterwarnings( + "ignore::policyengine_core.warnings.memory_config_warning.MemoryConfigWarning" +) +@pytest.mark.parametrize("longer_on_disk", [True, False], ids=["longer", "shorter"]) +def test_inputs_starting_together_uprate_the_same_one_from_memory_or_disk( + system, longer_on_disk +): + """``get_known_periods`` lists the periods in memory before those on + disk, whatever order they were stored in.""" + built = simulation(system, {}) + built.memory_config = MemoryConfig(max_memory_occupation=0) + holder = built.get_holder("uprated_any_unit") + holder._disk_storage = holder.create_disk_storage() + on_disk, in_memory = ("year:2012:2", "2012")[:: 1 if longer_on_disk else -1] + holder._on_disk_storable = False + built.set_input("uprated_any_unit", in_memory, np.array(TIED[in_memory])) + holder._on_disk_storable = True + built.set_input("uprated_any_unit", on_disk, np.array(TIED[on_disk])) + assert holder._disk_storage.has(periods.period(on_disk)) + assert holder._memory_storage.has(periods.period(in_memory)) + result = built.calculate("uprated_any_unit", "2015") + expected = alone(system, {"uprated_any_unit": TIED}, "uprated_any_unit", "2015") + assert_bitwise_equal(result, expected) + np.testing.assert_allclose( + result, np.array([1, 2]) * index(2015) / index(2012), rtol=1e-6 + ) + + +@pytest.mark.parametrize("longer_first", [True, False], ids=["longer", "shorter"]) +def test_branch_input_starting_with_a_parent_input_is_uprated_by_the_same_rule( + system, longer_first +): + first, second = ("year:2012:2", "2012")[:: 1 if longer_first else -1] + built = simulation(system, {"uprated": {first: TIED[first]}}) + branch = built.get_branch("reform") + branch.set_input("uprated", second, np.array(TIED[second])) + np.testing.assert_allclose( + branch.calculate("uprated", "2015"), + np.array([1, 2]) * index(2015) / index(2012), + rtol=1e-6, + ) + + +@pytest.mark.parametrize("longer_first", [True, False], ids=["longer", "shorter"]) +def test_flat_uprating_reads_the_input_auto_carry_over_reads(longer_first): + """Uprating by an index that never moves returns what auto-carry-over + returns for the same inputs, ties included, as long as no input in the + variable's own unit starts on the period's first day (see the next + test).""" + system = SYSTEMS[True] + values = {**TIED, "2012-06": [7.0, 8.0], "2014-02": [9.0, 10.0]} + ordered = _in_order(values, longer_first) + built = simulation(system, {"uprated_flat": ordered, "carried_any_unit": ordered}) + for year in ("2013", "2014", "2015"): + assert_bitwise_equal( + built.calculate("uprated_flat", year), + built.calculate("carried_any_unit", year), + year, + ) + np.testing.assert_array_equal(built.calculate("uprated_flat", "2015"), [1, 2]) + + +def test_longer_input_starting_on_the_period_is_not_an_uprating_source(system): + """The rule this change keeps: only inputs that start before the period + are uprated from, so ``year:2012:2`` is not a source for 2012 itself. + 2012 gets 2011's input uprated; auto-carry-over would carry + ``year:2012:2``. Without an earlier input, 2012 gets what auto-carry-over + gives it. Whether that should change is PolicyEngine/policyengine-core#583.""" + inputs = {"uprated_any_unit": {"2011": [1.0, 2.0], "year:2012:2": [5.0, 6.0]}} + np.testing.assert_allclose( + alone(system, inputs, "uprated_any_unit", "2012"), + np.array([1, 2]) * index(2012) / index(2011), + rtol=1e-6, + ) + inputs = {"uprated_any_unit": {"year:2012:2": [5.0, 6.0]}} + expected = [5, 6] if system.auto_carry_over_input_variables else [0, 0] + np.testing.assert_array_equal( + alone(system, inputs, "uprated_any_unit", "2012"), expected + ) + + +@pytest.mark.parametrize( + "stored,asked", + [("day:9999-12-30:3", "9999-12-31"), ("day:2012-01-01:4000000", "2013-01-01")], +) +def test_an_input_ending_after_the_last_date_is_uprated(system, stored, asked): + """A period that ends after 9999-12-31 has no ``stop`` (it raises + ``OverflowError``); it is still an uprating source.""" + inputs = {"uprated_daily_flat": {stored: [10.0, 20.0]}} + np.testing.assert_array_equal( + alone(system, inputs, "uprated_daily_flat", asked), [10, 20] + ) + assert_bitwise_equal( + alone(system, inputs, "uprated_daily_flat", asked), + reference(system, inputs, "uprated_daily_flat", asked), + ) + + +@pytest.mark.parametrize("longer_first", [True, False], ids=["longer", "shorter"]) +def test_an_input_ending_after_the_last_date_ends_last(system, longer_first): + values = {"day:2012-01-01:4000000": [10.0, 20.0], "day:2012-01-01:2": [1.0, 2.0]} + inputs = {"uprated_daily_flat": _in_order(values, longer_first)} + np.testing.assert_array_equal( + alone(system, inputs, "uprated_daily_flat", "2013-01-01"), [10, 20] + ) + + # A yearly input for a monthly variable goes through its ``set_input`` helper, # which stores it in each month of the year: copied to each month for a stock # (``set_input_dispatch_by_period``), divided between them for a flow diff --git a/tests/core/test_uprating_order_property.py b/tests/core/test_uprating_order_property.py index 1b39b23c2..feb85410a 100644 --- a/tests/core/test_uprating_order_property.py +++ b/tests/core/test_uprating_order_property.py @@ -7,9 +7,16 @@ in ``tests/fixtures/uprating_order.py``. Checked with and without auto-carry-over. Regressions are in ``test_uprating_order.py``. +Two more properties: the result does not depend on the order the inputs +were set in, or on which of them are on disk, even when several start on the +same day; and uprating by an index that never moves returns what +auto-carry-over returns for the same inputs. + The domain: -* Inputs are stored in each variable's own unit. ``uprated_any_unit`` has no +* Inputs are stored in each variable's own unit, some for a period several + units long that starts with another input (``year:2012:2`` beside + ``2012``): a variable stores those as given. ``uprated_any_unit`` has no ``set_input`` helper, so its inputs are years or months. * Earlier requests are plain calculations in either unit, ``ADD`` and ``DIVIDE``. @@ -21,7 +28,7 @@ module tests. * Targets are in the variable's definition unit. -A second property, ``test_helper_input_replaces_values_calculated_in_its_year``, +One more property, ``test_helper_input_replaces_values_calculated_in_its_year``, covers yearly inputs given through a ``set_input`` helper after calculations in that year; its domain is stated above it. """ @@ -33,6 +40,11 @@ # The smoke job installs Core without the dev extra but collects every module. pytest.importorskip("hypothesis") +# A simulation given inputs on disk warns that MemoryConfig is experimental. +pytestmark = pytest.mark.filterwarnings( + "ignore::policyengine_core.warnings.memory_config_warning.MemoryConfigWarning" +) + from hypothesis import HealthCheck, example, given, settings # noqa: E402 from hypothesis import strategies as st # noqa: E402 import numpy as np # noqa: E402 @@ -40,6 +52,7 @@ from policyengine_core import periods # noqa: E402 from policyengine_core.experimental import MemoryConfig # noqa: E402 from policyengine_core.reforms import Reform # noqa: E402 +from policyengine_core.simulations import SimulationBuilder # noqa: E402 from tests.fixtures.uprating_order import ( # noqa: E402 COUNT, assert_bitwise_equal, @@ -56,6 +69,21 @@ MONTHS = [f"{year}-{month:02d}" for year in range(2011, 2016) for month in range(1, 13)] INPUT_MONTHS = [month for month in MONTHS if "2011-06" <= month <= "2013-12"] REQUEST_MONTHS = [month for month in MONTHS if "2011-11" <= month <= "2015-06"] +# Several units long, each starting with a single-unit input period. As +# strings, "year:2012:10" sorts before "year:2012:2" though it ends later, +# and "month:2012-01:10" before "month:2012-01:3". (Twelve months from +# January would not do: a period is stored under its string form, and +# "month:2012-01:12" prints as "2012", so it reads back as the year.) +MULTI_YEARS = ["year:2012:2", "year:2012:10", "year:2014:3"] +MULTI_MONTHS = ["month:2012-01:3", "month:2012-01:10", "month:2013-06:2"] +# Input periods that start on the same day, by the unit they are in. +TIES = { + "year": [["2012", "year:2012:2", "year:2012:10"], ["2014", "year:2014:3"]], + "month": [ + ["2012-01", "month:2012-01:3", "month:2012-01:10"], + ["2013-06", "month:2013-06:2"], + ], +} YEAR_VARIABLES = [ "uprated", @@ -70,9 +98,11 @@ VALUES = {variable: FLOATS for variable in ALL_VARIABLES} VALUES["uprated_count"] = st.integers(min_value=-2000, max_value=2000) VALUES["eligible"] = st.booleans() -INPUT_PERIODS = {variable: INPUT_YEARS for variable in YEAR_VARIABLES} -INPUT_PERIODS["uprated_monthly"] = INPUT_MONTHS -INPUT_PERIODS["uprated_any_unit"] = INPUT_YEARS + INPUT_MONTHS +INPUT_PERIODS = {variable: INPUT_YEARS + MULTI_YEARS for variable in YEAR_VARIABLES} +INPUT_PERIODS["uprated_monthly"] = INPUT_MONTHS + MULTI_MONTHS +INPUT_PERIODS["uprated_any_unit"] = ( + INPUT_YEARS + MULTI_YEARS + INPUT_MONTHS + MULTI_MONTHS +) # Other variables read ``eligible`` (``uprated_if_eligible`` is defined for # it), so a parent request for them caches it at their period: no branch # input for it. @@ -273,6 +303,42 @@ def _read_from(built, branch, variable, branch_input): ), auto_carry_over=True, ) +# Inputs that start on the same day: the one that ends last is uprated, +# whichever was stored first. +@example( + scenario=( + {"uprated": {"2012": [5.0, 6.0], "year:2012:2": [1.0, 2.0]}}, + [], + "uprated", + "2015", + None, + None, + ), + auto_carry_over=False, +) +@example( + scenario=( + {"uprated_monthly": {"2012-01": [5.0, 6.0], "month:2012-01:3": [1.0, 2.0]}}, + [("calculate", "uprated_monthly", "2012-02")], + "uprated_monthly", + "2013-03", + None, + None, + ), + auto_carry_over=True, +) +# A branch input that starts with the parent's input. +@example( + scenario=( + {"uprated_count": {"year:2012:2": [1001, 77]}}, + [], + "uprated_count", + "2015", + "nested", + ("2012", [5, 6]), + ), + auto_carry_over=True, +) # Signed zero survives uprating from an input. @example( scenario=( @@ -306,6 +372,168 @@ def test_uprating_depends_only_on_the_inputs(scenario, auto_carry_over): assert_bitwise_equal(result, reference(system, seen, variable, period), message) +def _set_inputs(system, sets, on_disk=frozenset()): + """A simulation given ``sets``, ``(variable, period, values)`` in the + order listed. The inputs at the ``(variable, period)`` pairs in + ``on_disk`` are stored on disk, the rest in memory.""" + built = SimulationBuilder().build_default_simulation(system, count=COUNT) + if on_disk: + built.memory_config = MemoryConfig(max_memory_occupation=0) + for variable, period, values in sets: + holder = built.get_holder(variable) + if on_disk and holder._disk_storage is None: + holder._disk_storage = holder.create_disk_storage() + holder._on_disk_storable = (variable, period) in on_disk + built.set_input(variable, period, np.array(values)) + holder._on_disk_storable = False + return built + + +def _flatten(inputs): + return [ + (variable, period, values) + for variable, by_period in inputs.items() + for period, values in by_period.items() + ] + + +@st.composite +def reordered_scenarios(draw): + """A scenario whose target variable has two or three inputs that start on + the same day, the order to set all the inputs in, and the inputs to put + on disk.""" + inputs, requests, variable, period, branch, branch_input = draw(scenarios()) + unit = "month" if variable == "uprated_monthly" else "year" + group = draw(st.sampled_from(TIES[unit])) + tied = draw(st.lists(st.sampled_from(group), min_size=2, max_size=3, unique=True)) + values = st.lists(VALUES[variable], min_size=COUNT, max_size=COUNT) + inputs = {name: dict(by_period) for name, by_period in inputs.items()} + for tied_period in tied: + inputs[variable][tied_period] = draw(values) + sets = draw(st.permutations(_flatten(inputs))) + on_disk = draw( + st.sets(st.sampled_from([(variable, stored) for stored in inputs[variable]])) + ) + return (inputs, requests, variable, period, branch, branch_input), sets, on_disk + + +@settings( + max_examples=300, + deadline=None, + derandomize=True, + suppress_health_check=[HealthCheck.too_slow], +) +@given(scenario=reordered_scenarios(), auto_carry_over=st.booleans()) +def test_uprating_does_not_depend_on_the_order_inputs_were_set( + scenario, auto_carry_over +): + (inputs, requests, variable, period, branch, branch_input), sets, on_disk = scenario + system = SYSTEMS[auto_carry_over] + results = [] + for built in ( + simulation(system, inputs), + _set_inputs(system, sets, frozenset(on_disk)), + ): + for kind, requested_variable, requested_period in requests: + request(built, kind, requested_variable, requested_period) + read = _read_from(built, branch, variable, branch_input) + results.append(read.calculate(variable, period)) + message = f"{variable} {period}, set in order {sets}, on disk {on_disk}" + assert_bitwise_equal(results[1], results[0], message) + + seen = {name: dict(values) for name, values in inputs.items()} + if branch_input is not None: + seen.setdefault(variable, {})[branch_input[0]] = branch_input[1] + assert_bitwise_equal(results[1], reference(system, seen, variable, period), message) + + +FLAT_INPUT_PERIODS = INPUT_YEARS + MULTI_YEARS + INPUT_MONTHS + MULTI_MONTHS + + +@st.composite +def flat_scenarios(draw): + """Inputs for ``uprated_flat`` and ``carried_any_unit`` alike, set in + any order, earlier requests on either, and a target year.""" + values = st.lists(FLOATS, min_size=COUNT, max_size=COUNT) + by_period = draw( + st.dictionaries( + st.sampled_from(FLAT_INPUT_PERIODS), values, min_size=1, max_size=4 + ) + ) + group = draw(st.sampled_from(TIES["year"])) + tied = draw( + st.just([]) + | st.lists(st.sampled_from(group), min_size=2, max_size=3, unique=True) + ) + for tied_period in tied: + by_period[tied_period] = draw(values) + inputs = {"uprated_flat": by_period, "carried_any_unit": by_period} + sets = draw(st.permutations(_flatten(inputs))) + requests = draw( + st.lists( + st.one_of(_requests_on("uprated_flat"), _requests_on("carried_any_unit")), + max_size=6, + ) + ) + return inputs, sets, requests, draw(st.sampled_from(YEARS)) + + +@settings( + max_examples=300, + deadline=None, + derandomize=True, + suppress_health_check=[HealthCheck.too_slow], +) +@given(scenario=flat_scenarios()) +@example( + scenario=( + { + "uprated_flat": {"year:2012:2": [1.0, 2.0], "2012": [5.0, 6.0]}, + "carried_any_unit": {"year:2012:2": [1.0, 2.0], "2012": [5.0, 6.0]}, + }, + [ + ("uprated_flat", "2012", [5.0, 6.0]), + ("carried_any_unit", "2012", [5.0, 6.0]), + ("uprated_flat", "year:2012:2", [1.0, 2.0]), + ("carried_any_unit", "year:2012:2", [1.0, 2.0]), + ], + [], + "2015", + ) +) +def test_flat_uprating_reads_the_input_auto_carry_over_reads(scenario): + """Two implementations of "the latest input": the uprating source and + the input auto-carry-over carries. They agree except in one case: the + period itself is not stored, an input in the variable's own unit starts + before it (so uprating starts from one), and another starts on its first + day (which auto-carry-over carries, and uprating, taking only inputs that + start before the period, does not).""" + inputs, sets, requests, period = scenario + system = SYSTEMS[True] + built = _set_inputs(system, sets) + for kind, requested_variable, requested_period in requests: + request(built, kind, requested_variable, requested_period) + uprated = built.calculate("uprated_flat", period) + carried = built.calculate("carried_any_unit", period) + target = periods.period(period) + stored = list(map(periods.period, inputs["uprated_flat"])) + own_unit = [each for each in stored if each.unit == target.unit] + diverge = ( + target not in stored + and any(each.start < target.start for each in own_unit) + and any(each.start == target.start for each in own_unit) + ) + message = f"{period}, set in order {sets}, after {requests}" + if not diverge: + assert_bitwise_equal(uprated, carried, message) + assert_bitwise_equal( + uprated, reference(system, inputs, "uprated_flat", period), message + ) + assert_bitwise_equal( + carried, reference(system, inputs, "carried_any_unit", period), message + ) + + # Inputs given through a ``set_input`` helper after calculations. # # A yearly input for a monthly variable is stored in each month of the year by diff --git a/tests/fixtures/uprating_order.py b/tests/fixtures/uprating_order.py index 0f2a66fc9..05c97cad4 100644 --- a/tests/fixtures/uprating_order.py +++ b/tests/fixtures/uprating_order.py @@ -6,13 +6,22 @@ The reference rule, for a variable with ``uprating`` and no formula result for a period: an input stored for the period itself is read back as stored; -otherwise the latest earlier input stored in the variable's own unit, times -the ratio of the uprating index at the two period starts, cast to the -variable's type. With no such input, the period gets what auto-carry-over -gives it (the input stored for the latest-starting period that starts no -later than it, on a tie the one that ends last, preferring the variable's own -unit) or, without auto-carry-over or such an input, the default. Either way the value is masked -by the variable's ``defined_for`` at the period. +otherwise the latest input stored in the variable's own unit for a period +that starts before it, times the ratio of the uprating index at the two +period starts, cast to the variable's type. With no such input, the period +gets what auto-carry-over gives it (the input stored for the latest-starting +period that starts no later than it, preferring the variable's own unit) or, +without auto-carry-over or such an input, the default. Either way the value +is masked by the variable's ``defined_for`` at the period. + +"Latest" breaks ties the same way on both paths: of inputs that start on the +same day (a yearly variable stores ``year:2012:2`` as given, beside +``2012``), the one that ends last, then the larger unit, then the period's +string form. An input in the variable's own unit that starts on the period's +own first day but is longer than it (``year:2012:2`` for ``2012``) is not an +uprating source: the period gets the uprated earlier input if there is one, +and only otherwise what auto-carry-over gives it (see +PolicyEngine/policyengine-core#583). """ from __future__ import annotations @@ -28,6 +37,7 @@ __all__ = [ "COUNT", + "FLAT", "INDEX", "MONTHLY_INDEX", "UPRATING", @@ -51,6 +61,8 @@ for month in range(1, 13) } MONTHLY_UPRATING = "uprating_order.monthly_index" +# An index that never moves: uprating by it multiplies by exactly 1. +FLAT = "uprating_order.flat" class uprated(Variable): @@ -120,6 +132,32 @@ class uprated_monthly_stock(Variable): label = "Uprated monthly input (a stock: set_input_dispatch_by_period)" +class uprated_flat(Variable): + value_type = float + entity = entities.Person + definition_period = periods.YEAR + uprating = FLAT + set_input = None + label = "Yearly input uprated by an index that never moves, stored at any period" + + +class uprated_daily_flat(Variable): + value_type = float + entity = entities.Person + definition_period = periods.DAY + uprating = FLAT + set_input = None + label = "Daily input uprated by an index that never moves, stored at any period" + + +class carried_any_unit(Variable): + value_type = float + entity = entities.Person + definition_period = periods.YEAR + set_input = None + label = "uprated_flat without uprating: a yearly input stored at any period" + + VARIABLES = ( uprated, uprated_count, @@ -129,6 +167,9 @@ class uprated_monthly_stock(Variable): uprated_any_unit, uprated_monthly, uprated_monthly_stock, + uprated_flat, + uprated_daily_flat, + carried_any_unit, ) @@ -142,6 +183,7 @@ def build_system(auto_carry_over: bool = True) -> CountryTaxBenefitSystem: data={ "index": {"values": INDEX}, "monthly_index": {"values": MONTHLY_INDEX}, + "flat": {"values": {"2010-01-01": 100}}, }, ), ) @@ -186,6 +228,24 @@ def _stored_inputs(system, inputs, variable): return stored +def _latest(stored_period): + """Of inputs in one unit class, the latest: the one that starts last; on + a tie, the one that ends last, then the larger unit, then the period's + string form. Distinct periods never tie, so the choice does not depend on + the order the inputs are listed in.""" + try: + # A period ending after 9999-12-31 has no ``stop``: it ends last. + ends = (0, stored_period.stop) + except OverflowError: + ends = (1,) + return ( + stored_period.start, + ends, + periods.unit_weight(stored_period.unit), + str(stored_period), + ) + + def _uprating_factor(system, definition, earlier, period): parameter = get_parameter(system.parameters, definition.uprating) value_then = _uprating_index_value(parameter, earlier.start) @@ -218,7 +278,7 @@ def reference(system, inputs, variable, period): stored_period for stored_period in stored if stored_period.start <= period.start ] if definition.uprating is not None and earlier: - latest = max(earlier, key=lambda stored_period: stored_period.start) + latest = max(earlier, key=_latest) value = stored[latest] * _uprating_factor(system, definition, latest, period) elif system.auto_carry_over_input_variables and carried: own_unit = [ @@ -226,10 +286,7 @@ def reference(system, inputs, variable, period): for stored_period in carried if stored_period.unit == definition.definition_period ] - latest = max( - own_unit or carried, - key=lambda stored_period: (stored_period.start, stored_period.stop), - ) + latest = max(own_unit or carried, key=_latest) value = stored[latest] else: value = default From b43c3562655f7d86b81434cbd92cdb2b26cc9299 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 5 Oct 2026 15:07:59 -0400 Subject: [PATCH 2/6] Keep multi-unit periods off disk on Windows in the tie tests OnDiskStorage names a value's file "{branch}_{period}.npy". On Windows, numpy.save raises OSError for default_year:2012:2.npy and default_year:2014:3.npy, so storing year:2012:2 or year:2014:3 on disk fails there (PolicyEngine/policyengine-core#526). The tie tests put such periods on disk, which failed the four Windows test jobs: the "longer" memory-or-disk regression (both systems) and the set-order property. The fixture's can_store_on_disk says whether the tests may put a period on disk on this platform. On Windows the "longer" regression is skipped (the "shorter" one still puts 2012 on disk beside year:2012:2 in memory), and the property drops periods with a colon in their string form from its drawn on-disk set, so memory-against-disk ties are still drawn there, with the winner in memory. Elsewhere nothing changes: the derandomized draws are the same. Checked with a plugin that makes numpy.save and tempfile.mkstemp reject any file name containing ":": without the guard the same 3 tests fail with the same paths as CI; with it 3 pass and 2 skip. Co-Authored-By: Claude Opus 5.5 --- tests/core/test_uprating_order.py | 16 +++++++++++++++- tests/core/test_uprating_order_property.py | 6 ++++++ tests/fixtures/uprating_order.py | 15 +++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/tests/core/test_uprating_order.py b/tests/core/test_uprating_order.py index 274e8d1c1..27fc47e96 100644 --- a/tests/core/test_uprating_order.py +++ b/tests/core/test_uprating_order.py @@ -56,6 +56,7 @@ alone, assert_bitwise_equal, build_system, + can_store_on_disk, index, reference, simulation, @@ -360,7 +361,20 @@ def test_monthly_inputs_starting_together_uprate_the_one_ending_last( @pytest.mark.filterwarnings( "ignore::policyengine_core.warnings.memory_config_warning.MemoryConfigWarning" ) -@pytest.mark.parametrize("longer_on_disk", [True, False], ids=["longer", "shorter"]) +@pytest.mark.parametrize( + "longer_on_disk", + [ + pytest.param( + True, + id="longer", + marks=pytest.mark.skipif( + not can_store_on_disk("year:2012:2"), + reason="OnDiskStorage cannot store year:2012:2 on Windows (#526)", + ), + ), + pytest.param(False, id="shorter"), + ], +) def test_inputs_starting_together_uprate_the_same_one_from_memory_or_disk( system, longer_on_disk ): diff --git a/tests/core/test_uprating_order_property.py b/tests/core/test_uprating_order_property.py index feb85410a..7d9fb52ad 100644 --- a/tests/core/test_uprating_order_property.py +++ b/tests/core/test_uprating_order_property.py @@ -27,6 +27,9 @@ cached for the branch whatever its inputs are, which is not what this module tests. * Targets are in the variable's definition unit. +* Inputs on disk are any of the target variable's, except on Windows, where + ``OnDiskStorage`` cannot store a period several units long + (PolicyEngine/policyengine-core#526): those stay in memory there. One more property, ``test_helper_input_replaces_values_calculated_in_its_year``, covers yearly inputs given through a ``set_input`` helper after calculations @@ -57,6 +60,7 @@ COUNT, assert_bitwise_equal, build_system, + can_store_on_disk, reference, request, simulation, @@ -414,6 +418,8 @@ def reordered_scenarios(draw): on_disk = draw( st.sets(st.sampled_from([(variable, stored) for stored in inputs[variable]])) ) + # On Windows, a period several units long stays in memory. + on_disk = {pair for pair in on_disk if can_store_on_disk(pair[1])} return (inputs, requests, variable, period, branch, branch_input), sets, on_disk diff --git a/tests/fixtures/uprating_order.py b/tests/fixtures/uprating_order.py index 05c97cad4..3a8db2dab 100644 --- a/tests/fixtures/uprating_order.py +++ b/tests/fixtures/uprating_order.py @@ -26,6 +26,8 @@ from __future__ import annotations +import sys + import numpy as np from policyengine_core import periods @@ -44,6 +46,7 @@ "alone", "assert_bitwise_equal", "build_system", + "can_store_on_disk", "index", "reference", "request", @@ -64,6 +67,18 @@ # An index that never moves: uprating by it multiplies by exactly 1. FLAT = "uprating_order.flat" +# ``OnDiskStorage`` names a value's file after its period, and Windows does +# not allow ":" in a file name. So on Windows a period several units long +# (``year:2012:2``) cannot be stored on disk: ``numpy.save`` raises +# ``OSError``. The tests keep those periods in memory there (see +# PolicyEngine/policyengine-core#526). +DISK_STORES_ANY_PERIOD = sys.platform != "win32" + + +def can_store_on_disk(period) -> bool: + """Whether ``OnDiskStorage`` can store a value for ``period`` here.""" + return DISK_STORES_ANY_PERIOD or ":" not in str(periods.period(period)) + class uprated(Variable): value_type = float From 7e2aa23887c63f09d7d0f145d1357d018d0ef816 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 5 Oct 2026 16:28:34 -0400 Subject: [PATCH 3/6] Probe disk storage for the Windows guard; test an input on disk winning a tie Review of 6148851d (APPROVE WITH NITS): - The guard keyed on sys.platform, so it would not lift when #526 lets OnDiskStorage store a period with a colon in its name. The fixture now tries storing year:2012:2 in a temporary directory once, at import, and DISK_STORES_ANY_PERIOD says whether that worked. - The comments said "a period several units long", but the code checks for a colon in the period's string form (year:2012-03 has one; month:2012-01:12 prints as 2012 and has none). They now say so. - No property example had the winning input on disk and a losing one in memory: in the drawn ties split between memory and disk, the winner was always in memory, which get_known_periods lists first. A new example puts year:2012:2 on disk beside 2012 in memory. Where storage cannot store year:2012:2 it is an expected failure with OSError. There, no tie this property draws can have its winner on disk. Two inputs tie when the key ranks them equal up to their start: the same start, and both in the variable's own unit or both not. This property's inputs are years and months, so the two are in the same unit, and the winner, which ends last, is the longer; every such period it draws has a colon in its string form. Co-Authored-By: Claude Opus 5.5 --- tests/core/test_uprating_order.py | 2 +- tests/core/test_uprating_order_property.py | 36 +++++++++++++++++++--- tests/fixtures/uprating_order.py | 29 ++++++++++++----- 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/tests/core/test_uprating_order.py b/tests/core/test_uprating_order.py index 27fc47e96..61b5928f5 100644 --- a/tests/core/test_uprating_order.py +++ b/tests/core/test_uprating_order.py @@ -369,7 +369,7 @@ def test_monthly_inputs_starting_together_uprate_the_one_ending_last( id="longer", marks=pytest.mark.skipif( not can_store_on_disk("year:2012:2"), - reason="OnDiskStorage cannot store year:2012:2 on Windows (#526)", + reason="OnDiskStorage cannot store year:2012:2 here (Windows; #526)", ), ), pytest.param(False, id="shorter"), diff --git a/tests/core/test_uprating_order_property.py b/tests/core/test_uprating_order_property.py index 7d9fb52ad..d353d1bc4 100644 --- a/tests/core/test_uprating_order_property.py +++ b/tests/core/test_uprating_order_property.py @@ -27,9 +27,10 @@ cached for the branch whatever its inputs are, which is not what this module tests. * Targets are in the variable's definition unit. -* Inputs on disk are any of the target variable's, except on Windows, where - ``OnDiskStorage`` cannot store a period several units long - (PolicyEngine/policyengine-core#526): those stay in memory there. +* Inputs on disk are any of the target variable's, except where + ``OnDiskStorage`` cannot store a period whose string form has a colon, + such as ``year:2012:2`` (on Windows; PolicyEngine/policyengine-core#526): + those stay in memory there. One more property, ``test_helper_input_replaces_values_calculated_in_its_year``, covers yearly inputs given through a ``set_input`` helper after calculations @@ -58,6 +59,7 @@ from policyengine_core.simulations import SimulationBuilder # noqa: E402 from tests.fixtures.uprating_order import ( # noqa: E402 COUNT, + DISK_STORES_ANY_PERIOD, assert_bitwise_equal, build_system, can_store_on_disk, @@ -418,7 +420,8 @@ def reordered_scenarios(draw): on_disk = draw( st.sets(st.sampled_from([(variable, stored) for stored in inputs[variable]])) ) - # On Windows, a period several units long stays in memory. + # Where storage cannot store a period with a colon in its string form + # (on Windows), such periods stay in memory. on_disk = {pair for pair in on_disk if can_store_on_disk(pair[1])} return (inputs, requests, variable, period, branch, branch_input), sets, on_disk @@ -430,6 +433,31 @@ def reordered_scenarios(draw): suppress_health_check=[HealthCheck.too_slow], ) @given(scenario=reordered_scenarios(), auto_carry_over=st.booleans()) +# The input that ends last is on disk and the other in memory, which +# ``get_known_periods`` lists first. Where storage cannot store +# ``year:2012:2``, setting it raises ``OSError``. +@example( + scenario=( + ( + {"uprated_any_unit": {"2012": [5.0, 6.0], "year:2012:2": [1.0, 2.0]}}, + [], + "uprated_any_unit", + "2015", + None, + None, + ), + [ + ("uprated_any_unit", "2012", [5.0, 6.0]), + ("uprated_any_unit", "year:2012:2", [1.0, 2.0]), + ], + {("uprated_any_unit", "year:2012:2")}, + ), + auto_carry_over=True, +).xfail( + not DISK_STORES_ANY_PERIOD, + reason="OnDiskStorage cannot store year:2012:2 here (Windows; #526)", + raises=OSError, +) def test_uprating_does_not_depend_on_the_order_inputs_were_set( scenario, auto_carry_over ): diff --git a/tests/fixtures/uprating_order.py b/tests/fixtures/uprating_order.py index 3a8db2dab..d97a5113e 100644 --- a/tests/fixtures/uprating_order.py +++ b/tests/fixtures/uprating_order.py @@ -26,12 +26,13 @@ from __future__ import annotations -import sys +import tempfile import numpy as np from policyengine_core import periods from policyengine_core.country_template import CountryTaxBenefitSystem, entities +from policyengine_core.data_storage import OnDiskStorage from policyengine_core.parameters import ParameterNode, get_parameter from policyengine_core.simulations.simulation import _uprating_index_value from policyengine_core.variables import QuantityType, Variable @@ -39,6 +40,7 @@ __all__ = [ "COUNT", + "DISK_STORES_ANY_PERIOD", "FLAT", "INDEX", "MONTHLY_INDEX", @@ -67,12 +69,25 @@ # An index that never moves: uprating by it multiplies by exactly 1. FLAT = "uprating_order.flat" -# ``OnDiskStorage`` names a value's file after its period, and Windows does -# not allow ":" in a file name. So on Windows a period several units long -# (``year:2012:2``) cannot be stored on disk: ``numpy.save`` raises -# ``OSError``. The tests keep those periods in memory there (see -# PolicyEngine/policyengine-core#526). -DISK_STORES_ANY_PERIOD = sys.platform != "win32" + +def _disk_stores_any_period() -> bool: + """Whether ``OnDiskStorage`` can store a value here for a period whose + string form has a colon, such as ``year:2012:2``. + + It names a value's file after its period, and Windows does not allow ":" + in a file name, so there ``numpy.save`` raises ``OSError`` + (PolicyEngine/policyengine-core#526). Where it cannot, the tests keep + those periods in memory; once storage can, they are on disk again.""" + with tempfile.TemporaryDirectory() as directory: + storage = OnDiskStorage(directory, preserve_storage_dir=True) + try: + storage.put(np.zeros(1), periods.period("year:2012:2")) + except OSError: + return False + return True + + +DISK_STORES_ANY_PERIOD = _disk_stores_any_period() def can_store_on_disk(period) -> bool: From 49ed6a4a5b1f382fc7901f5fc32457b70868d7d8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 5 Oct 2026 18:58:14 -0400 Subject: [PATCH 4/6] Word the Windows guard by what CI showed; require Hypothesis 6.62 Review of 3f222e1f (APPROVE WITH NITS): - The fixture said Windows does not allow ":" in a file name, so numpy.save raises OSError there. CI showed that for default_year:2012:2.npy, a name with two colons. The docstrings and comments now name what the probe stores (year:2012:2) and say what the tests do where it fails. - example(...).xfail, which the disk-winner example uses, needs Hypothesis 6.62.0 or later: under 6.61.3 it raises AttributeError, so the module could not be collected with an older Hypothesis the dev extra allowed. The dev extra now requires hypothesis>=6.62,<7, and uv.lock's specifier matches. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 2 +- tests/core/test_uprating_order_property.py | 10 +++++----- tests/fixtures/uprating_order.py | 16 +++++++++------- uv.lock | 2 +- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index aee17316d..7cf59b4fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ dev = [ "ruff>=0.9.0", "coverage", "furo", - "hypothesis>=6,<7", + "hypothesis>=6.62,<7", "jupyter-book<1", "markupsafe==2.0.1", "mypy==0.991", diff --git a/tests/core/test_uprating_order_property.py b/tests/core/test_uprating_order_property.py index d353d1bc4..0fdc69468 100644 --- a/tests/core/test_uprating_order_property.py +++ b/tests/core/test_uprating_order_property.py @@ -28,9 +28,9 @@ module tests. * Targets are in the variable's definition unit. * Inputs on disk are any of the target variable's, except where - ``OnDiskStorage`` cannot store a period whose string form has a colon, - such as ``year:2012:2`` (on Windows; PolicyEngine/policyengine-core#526): - those stay in memory there. + ``OnDiskStorage`` cannot store ``year:2012:2`` (on Windows; + PolicyEngine/policyengine-core#526): there, periods whose string form has + a colon stay in memory. One more property, ``test_helper_input_replaces_values_calculated_in_its_year``, covers yearly inputs given through a ``set_input`` helper after calculations @@ -420,8 +420,8 @@ def reordered_scenarios(draw): on_disk = draw( st.sets(st.sampled_from([(variable, stored) for stored in inputs[variable]])) ) - # Where storage cannot store a period with a colon in its string form - # (on Windows), such periods stay in memory. + # Where storage cannot store year:2012:2 (on Windows), periods with a + # colon in their string form stay in memory. on_disk = {pair for pair in on_disk if can_store_on_disk(pair[1])} return (inputs, requests, variable, period, branch, branch_input), sets, on_disk diff --git a/tests/fixtures/uprating_order.py b/tests/fixtures/uprating_order.py index d97a5113e..76dac02a7 100644 --- a/tests/fixtures/uprating_order.py +++ b/tests/fixtures/uprating_order.py @@ -71,13 +71,13 @@ def _disk_stores_any_period() -> bool: - """Whether ``OnDiskStorage`` can store a value here for a period whose - string form has a colon, such as ``year:2012:2``. + """Whether ``OnDiskStorage`` can store a value here for ``year:2012:2``. - It names a value's file after its period, and Windows does not allow ":" - in a file name, so there ``numpy.save`` raises ``OSError`` - (PolicyEngine/policyengine-core#526). Where it cannot, the tests keep - those periods in memory; once storage can, they are on disk again.""" + It names a value's file after its period, and on Windows ``numpy.save`` + raises ``OSError`` for ``default_year:2012:2.npy``, whose name has two + colons (PolicyEngine/policyengine-core#526). Where it cannot, the tests + keep every period whose string form has a colon in memory; once storage + can, they are on disk again.""" with tempfile.TemporaryDirectory() as directory: storage = OnDiskStorage(directory, preserve_storage_dir=True) try: @@ -91,7 +91,9 @@ def _disk_stores_any_period() -> bool: def can_store_on_disk(period) -> bool: - """Whether ``OnDiskStorage`` can store a value for ``period`` here.""" + """Whether the tests may store a value for ``period`` on disk here: any + period where ``year:2012:2`` can be stored, otherwise only one whose string + form has no colon.""" return DISK_STORES_ANY_PERIOD or ":" not in str(periods.period(period)) diff --git a/uv.lock b/uv.lock index 8a837867c..e03f3e404 100644 --- a/uv.lock +++ b/uv.lock @@ -1503,7 +1503,7 @@ requires-dist = [ { name = "furo", marker = "extra == 'dev'" }, { name = "h5py", specifier = ">=3,<4" }, { name = "huggingface-hub", specifier = ">=0.25.1" }, - { name = "hypothesis", marker = "extra == 'dev'", specifier = ">=6,<7" }, + { name = "hypothesis", marker = "extra == 'dev'", specifier = ">=6.62,<7" }, { name = "ipython", specifier = ">=8,<9" }, { name = "jupyter-book", marker = "extra == 'dev'", specifier = "<1" }, { name = "markupsafe", marker = "extra == 'dev'", specifier = "==2.0.1" }, From f081b7c4d2e9bd4562998e7bb4601e7d3889e10e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 5 Oct 2026 20:18:53 -0400 Subject: [PATCH 5/6] Order inputs ending after 9999-12-31 by their end in the reference rule Reviews of 578911f2 (Opus 5.5 and GPT-6.1 Sol, both APPROVE WITH NITS): - The reference rule ordered ends by Period.stop, which raises OverflowError for days that end after 9999-12-31, and gave every such input the same end, after any input whose stop has a value. So two of them fell through to string order, and one beat a month input that ends later (month:9999-01:13's stop is 10000-01-31). The simulation orders them with _end_order. For uprated_daily_flat at 9999-12-31, {day:9999-12-30:10, day:9999-12-30:3} gave [10, 20] from the simulation and [1, 2] from the reference; for carried_any_unit at 9999, {month:9999-01:13, day:9999-01-01:370} gave [1, 2] and [3, 4]. The reference's last_day now numbers each period's last day itself: for days, the start's number plus the days after it; for months and years, stop, moved back by whole 400-year Gregorian cycles (146097 days) to a year datetime has. - Tests: both cases as regressions, in either set order, compared with the reference; and a derandomized property that last_day, written separately, plus one equals _end_order, for days, months and years up to 5,000,000 units long. Against the old reference the regressions fail 6 tests; without the 400-year shift, 3; with last_day off by one, the property fails. - The fixture said that where the probe fails the tests keep every period with a colon in memory. There, the "longer" regression is skipped and the disk-winner example expects the OSError; it now says so. Co-Authored-By: Claude Opus 5.5 --- tests/core/test_uprating_order.py | 41 +++++++++++++++++++--- tests/core/test_uprating_order_property.py | 36 +++++++++++++++++-- tests/fixtures/uprating_order.py | 36 ++++++++++++++----- 3 files changed, 96 insertions(+), 17 deletions(-) diff --git a/tests/core/test_uprating_order.py b/tests/core/test_uprating_order.py index 61b5928f5..3803c2fc4 100644 --- a/tests/core/test_uprating_order.py +++ b/tests/core/test_uprating_order.py @@ -469,13 +469,44 @@ def test_an_input_ending_after_the_last_date_is_uprated(system, stored, asked): ) +@pytest.mark.parametrize( + "values,asked", + [ + ( + {"day:2012-01-01:4000000": [10.0, 20.0], "day:2012-01-01:2": [1.0, 2.0]}, + "2013-01-01", + ), + # Both end after 9999-12-31. As strings, ":10" sorts before ":3". + ( + {"day:9999-12-30:10": [10.0, 20.0], "day:9999-12-30:3": [1.0, 2.0]}, + "9999-12-31", + ), + ], + ids=["one_after", "both_after"], +) @pytest.mark.parametrize("longer_first", [True, False], ids=["longer", "shorter"]) -def test_an_input_ending_after_the_last_date_ends_last(system, longer_first): - values = {"day:2012-01-01:4000000": [10.0, 20.0], "day:2012-01-01:2": [1.0, 2.0]} +def test_an_input_ending_after_the_last_date_ends_last( + system, values, asked, longer_first +): inputs = {"uprated_daily_flat": _in_order(values, longer_first)} - np.testing.assert_array_equal( - alone(system, inputs, "uprated_daily_flat", "2013-01-01"), [10, 20] - ) + result = alone(system, inputs, "uprated_daily_flat", asked) + np.testing.assert_array_equal(result, [10, 20]) + assert_bitwise_equal(result, reference(system, inputs, "uprated_daily_flat", asked)) + + +@pytest.mark.parametrize("longer_first", [True, False], ids=["longer", "shorter"]) +def test_inputs_ending_after_the_last_date_carry_the_one_ending_last( + system, longer_first +): + """``stop`` raises ``OverflowError`` for days that end after 9999-12-31, + but not for months: ``month:9999-01:13`` ends on 10000-01-31, after + ``day:9999-01-01:370`` (10000-01-05), so it is the one carried.""" + values = {"month:9999-01:13": [1.0, 2.0], "day:9999-01-01:370": [3.0, 4.0]} + inputs = {"carried_any_unit": _in_order(values, longer_first)} + result = alone(system, inputs, "carried_any_unit", "9999") + expected = [1, 2] if system.auto_carry_over_input_variables else [0, 0] + np.testing.assert_array_equal(result, expected) + assert_bitwise_equal(result, reference(system, inputs, "carried_any_unit", "9999")) # A yearly input for a monthly variable goes through its ``set_input`` helper, diff --git a/tests/core/test_uprating_order_property.py b/tests/core/test_uprating_order_property.py index 0fdc69468..96c8eb211 100644 --- a/tests/core/test_uprating_order_property.py +++ b/tests/core/test_uprating_order_property.py @@ -32,9 +32,12 @@ PolicyEngine/policyengine-core#526): there, periods whose string form has a colon stay in memory. -One more property, ``test_helper_input_replaces_values_calculated_in_its_year``, -covers yearly inputs given through a ``set_input`` helper after calculations -in that year; its domain is stated above it. +Two more properties have their domains stated above them: +``test_helper_input_replaces_values_calculated_in_its_year`` covers yearly +inputs given through a ``set_input`` helper after calculations in that year, +and ``test_the_reference_rule_ends_a_period_where_end_order_does`` checks +that the reference rule orders period ends as the simulation does, after +9999-12-31 too. """ from __future__ import annotations @@ -49,6 +52,8 @@ "ignore::policyengine_core.warnings.memory_config_warning.MemoryConfigWarning" ) +from datetime import date # noqa: E402 + from hypothesis import HealthCheck, example, given, settings # noqa: E402 from hypothesis import strategies as st # noqa: E402 import numpy as np # noqa: E402 @@ -57,12 +62,14 @@ from policyengine_core.experimental import MemoryConfig # noqa: E402 from policyengine_core.reforms import Reform # noqa: E402 from policyengine_core.simulations import SimulationBuilder # noqa: E402 +from policyengine_core.simulations.simulation import _end_order # noqa: E402 from tests.fixtures.uprating_order import ( # noqa: E402 COUNT, DISK_STORES_ANY_PERIOD, assert_bitwise_equal, build_system, can_store_on_disk, + last_day, reference, request, simulation, @@ -768,3 +775,26 @@ def test_helper_input_replaces_values_calculated_in_its_year( fresh.calculate(variable, target), f"{message}: {target}", ) + + +# The domain: a period of days, months or years, starting on any date +# ``datetime`` has, up to 5,000,000 units long, so many end after 9999-12-31. +@settings(max_examples=500, deadline=None, derandomize=True) +@given( + unit=st.sampled_from([periods.DAY, periods.MONTH, periods.YEAR]), + start=st.dates(min_value=date(1, 1, 1), max_value=date(9999, 12, 31)), + size=st.integers(min_value=1, max_value=5_000_000), +) +@example(unit=periods.DAY, start=date(9999, 12, 30), size=10) +@example(unit=periods.MONTH, start=date(9999, 1, 1), size=13) +@example(unit=periods.YEAR, start=date(9999, 1, 1), size=1) +@example(unit=periods.MONTH, start=date(2012, 1, 31), size=1) +def test_the_reference_rule_ends_a_period_where_end_order_does(unit, start, size): + """The reference rule's ``last_day`` is written apart from the + simulation's ``_end_order``, which ``test_carry_over_order_property.py`` + checks against ``Period.stop`` and numpy's calendar. The day after a + period's last day is the one ``_end_order`` gives.""" + period = periods.Period( + (unit, periods.Instant((start.year, start.month, start.day)), size) + ) + assert last_day(period) + 1 == _end_order(period) diff --git a/tests/fixtures/uprating_order.py b/tests/fixtures/uprating_order.py index 76dac02a7..eb4bd8a3b 100644 --- a/tests/fixtures/uprating_order.py +++ b/tests/fixtures/uprating_order.py @@ -26,6 +26,7 @@ from __future__ import annotations +import datetime import tempfile import numpy as np @@ -75,9 +76,11 @@ def _disk_stores_any_period() -> bool: It names a value's file after its period, and on Windows ``numpy.save`` raises ``OSError`` for ``default_year:2012:2.npy``, whose name has two - colons (PolicyEngine/policyengine-core#526). Where it cannot, the tests - keep every period whose string form has a colon in memory; once storage - can, they are on disk again.""" + colons (PolicyEngine/policyengine-core#526). Where it cannot, the + property keeps every drawn period whose string form has a colon in + memory, the regression that needs one on disk is skipped, and the + example that puts one there expects the ``OSError``. Once storage can, + they all put such periods on disk again.""" with tempfile.TemporaryDirectory() as directory: storage = OnDiskStorage(directory, preserve_storage_dir=True) try: @@ -260,19 +263,34 @@ def _stored_inputs(system, inputs, variable): return stored +def last_day(stored_period): + """The number of the period's last day, counting days as + ``date.toordinal`` does, also after 9999-12-31, the last date ``datetime`` + can represent. + + For days, the start's number plus the days after it: their ``stop`` + raises ``OverflowError`` past that date. For months and years, ``stop`` + (an ``Instant``, which can be later), moved back by whole 400-year cycles + of the Gregorian calendar to a year ``datetime`` has: the calendar + repeats every 400 years, which are 146097 days.""" + unit, start, size = stored_period + if unit == periods.ETERNITY: + return float("inf") + if unit == periods.DAY: + return start.date.toordinal() + size - 1 + year, month, day = stored_period.stop + cycles = max(0, (year - 9999 + 399) // 400) + return datetime.date(year - 400 * cycles, month, day).toordinal() + 146097 * cycles + + def _latest(stored_period): """Of inputs in one unit class, the latest: the one that starts last; on a tie, the one that ends last, then the larger unit, then the period's string form. Distinct periods never tie, so the choice does not depend on the order the inputs are listed in.""" - try: - # A period ending after 9999-12-31 has no ``stop``: it ends last. - ends = (0, stored_period.stop) - except OverflowError: - ends = (1,) return ( stored_period.start, - ends, + last_day(stored_period), periods.unit_weight(stored_period.unit), str(stored_period), ) From db5c3de8f20132a85856c8662aeca0397b72c4c7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 5 Oct 2026 20:38:41 -0400 Subject: [PATCH 6/6] Say in the changelog that the tied inputs are in the variable's own unit Review of f081b7c4 (GPT-6.1 Sol, APPROVE WITH NITS): the fragment said "two earlier inputs start on the same day", but uprating reads only inputs in the variable's own unit. A yearly variable with no set_input helper, given 2012 and month:2012-01:25, which start together, is still uprated from 2012, though the month input ends later. Co-Authored-By: Claude Opus 5.5 --- changelog.d/fix-uprating-equal-start.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/fix-uprating-equal-start.fixed.md b/changelog.d/fix-uprating-equal-start.fixed.md index 4aec72b37..0d98ddf08 100644 --- a/changelog.d/fix-uprating-equal-start.fixed.md +++ b/changelog.d/fix-uprating-equal-start.fixed.md @@ -1 +1 @@ -When two earlier inputs start on the same day (such as `2012` and `year:2012:2`), uprating now starts from the one that ends last, the input auto-carry-over would carry, so the uprated value no longer depends on which input was stored first or on whether it is in memory or on disk. +When two earlier inputs in a variable's own unit start on the same day (such as `2012` and `year:2012:2` for a yearly variable), uprating now starts from the one that ends last, the input auto-carry-over would carry, so the uprated value no longer depends on which input was stored first or on whether it is in memory or on disk.