From c8def95f7c106865c69db442e76afeeab9f452c9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 6 Oct 2026 12:59:38 -0400 Subject: [PATCH 1/3] Reduce and rank over a population with no persons GroupPopulation.reduce (behind all, max and min), members_position and Population.get_rank each took numpy.max of the persons' positions or group indices to size a loop. With no persons that array is empty, so they raised "zero-size array to reduction operation maximum which has no identity" instead of returning the neutral element reduce already fills each group with. They now start the maximum at -1, so the loop runs zero times; get_rank keeps its value matrix two-dimensional with no columns. Adds the audit witness (one household, no persons) and a Hypothesis property: for 1 to 6 groups and 0 to 30 persons, sum, nb_persons, any, all, max, min and get_rank equal a plain-Python oracle, empty groups included. Co-Authored-By: Claude Opus 5.5 --- .../fix-empty-group-reductions.fixed.md | 1 + .../populations/group_population.py | 5 +- policyengine_core/populations/population.py | 19 +++-- tests/core/test_group_population_empty.py | 51 +++++++++++++ ...st_group_population_reductions_property.py | 75 +++++++++++++++++++ 5 files changed, 142 insertions(+), 9 deletions(-) create mode 100644 changelog.d/fix-empty-group-reductions.fixed.md create mode 100644 tests/core/test_group_population_empty.py create mode 100644 tests/core/test_group_population_reductions_property.py diff --git a/changelog.d/fix-empty-group-reductions.fixed.md b/changelog.d/fix-empty-group-reductions.fixed.md new file mode 100644 index 000000000..6f336c892 --- /dev/null +++ b/changelog.d/fix-empty-group-reductions.fixed.md @@ -0,0 +1 @@ +Group `all`, `max` and `min`, `members_position` and `Population.get_rank` now work when a simulation has no persons, giving the neutral element (True, -inf, inf) for each group instead of raising "zero-size array to reduction". diff --git a/policyengine_core/populations/group_population.py b/policyengine_core/populations/group_population.py index 70ff98125..7c0aaec78 100644 --- a/policyengine_core/populations/group_population.py +++ b/policyengine_core/populations/group_population.py @@ -67,7 +67,7 @@ def clone( def members_position(self) -> ArrayLike: if self._members_position is None and self.members_entity_id is not None: # We could use self.count and self.members.count , but with the current initilization, we are not sure count will be set before members_position is called - nb_entities = numpy.max(self.members_entity_id) + 1 + nb_entities = numpy.max(self.members_entity_id, initial=-1) + 1 nb_persons = len(self.members_entity_id) self._members_position = numpy.empty_like(self.members_entity_id) counter_by_entity = numpy.zeros(nb_entities) @@ -191,7 +191,8 @@ def reduce( # We loop over the positions in the entity # Looping over the entities is tempting, but potentielly slow if there are a lot of entities - biggest_entity_size = numpy.max(position_in_entity) + 1 + # With no persons there is no position, and every entity keeps the neutral element. + biggest_entity_size = numpy.max(position_in_entity, initial=-1) + 1 for p in range(biggest_entity_size): values = self.value_nth_person(p, filtered_array, default=neutral_element) diff --git a/policyengine_core/populations/population.py b/policyengine_core/populations/population.py index d2e8df1da..1106e4d91 100644 --- a/policyengine_core/populations/population.py +++ b/policyengine_core/populations/population.py @@ -233,17 +233,22 @@ def get_rank( ) positions = entity.members_position - biggest_entity_size = numpy.max(positions) + 1 + biggest_entity_size = numpy.max(positions, initial=-1) + 1 filtered_criteria = numpy.where(condition, criteria, numpy.inf) ids = entity.members_entity_id # Matrix: the value in line i and column j is the value of criteria for the jth person of the ith entity - matrix = numpy.asarray( - [ - entity.value_nth_person(k, filtered_criteria, default=numpy.inf) - for k in range(biggest_entity_size) - ] - ).transpose() + # (``reshape`` keeps it two-dimensional when there are no persons) + matrix = ( + numpy.asarray( + [ + entity.value_nth_person(k, filtered_criteria, default=numpy.inf) + for k in range(biggest_entity_size) + ] + ) + .reshape(biggest_entity_size, entity.count) + .transpose() + ) # We double-argsort all lines of the matrix. # Double-argsorting gets the rank of each value once sorted diff --git a/tests/core/test_group_population_empty.py b/tests/core/test_group_population_empty.py new file mode 100644 index 000000000..e69df429b --- /dev/null +++ b/tests/core/test_group_population_empty.py @@ -0,0 +1,51 @@ +"""Group reductions and ranks when a simulation has no persons. + +``GroupPopulation.reduce`` (behind ``all``, ``max`` and ``min``) returns its +neutral element for a group with no members, and ``sum`` and ``nb_persons`` +return 0. With no persons at all, ``reduce``, ``members_position`` and +``Population.get_rank`` took ``numpy.max`` of an empty array and raised +"zero-size array to reduction operation maximum which has no identity". +""" + +from __future__ import annotations + +import numpy as np + +from policyengine_core.entities import build_entity +from policyengine_core.populations import GroupPopulation, Population + +PERSON = build_entity("person", "persons", "Person", is_person=True) +HOUSEHOLD = build_entity( + "household", + "households", + "Household", + roles=[{"key": "member", "plural": "members"}], +) + + +def populations(members_entity_id, count): + persons = Population(PERSON) + persons.count = len(members_entity_id) + households = GroupPopulation(HOUSEHOLD, persons) + households.count = count + households.members_entity_id = np.asarray(members_entity_id, dtype=int) + return persons, households + + +def test_reductions_over_no_persons_give_neutral_elements(): + _, households = populations([], 1) + no_values = np.array([], dtype=float) + + assert households.all(no_values > 0).tolist() == [True] + assert households.any(no_values > 0).tolist() == [False] + assert households.max(no_values).tolist() == [-np.inf] + assert households.min(no_values).tolist() == [np.inf] + assert households.sum(no_values).tolist() == [0] + assert households.nb_persons().tolist() == [0] + assert households.members_position.tolist() == [] + + +def test_rank_over_no_persons_is_empty(): + persons, households = populations([], 2) + + assert persons.get_rank(households, np.array([], dtype=float)).tolist() == [] diff --git a/tests/core/test_group_population_reductions_property.py b/tests/core/test_group_population_reductions_property.py new file mode 100644 index 000000000..92aaffc4b --- /dev/null +++ b/tests/core/test_group_population_reductions_property.py @@ -0,0 +1,75 @@ +"""Property: group reductions and ranks match a plain-Python oracle. + +For any number of groups (1 to 6), any assignment of 0 to 30 persons to them +(so groups may be empty, and there may be no persons at all) and any values: + +* ``sum``, ``nb_persons``, ``any``, ``all``, ``max`` and ``min`` equal the + oracle, with 0, False, True, -inf and inf for groups with no members; +* ``Population.get_rank`` gives each person the position of their value among + their group's values, ties going to the person listed first. + +Examples, including the empty-population witness, are in +``test_group_population_empty.py``. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +# The smoke job installs Core without the dev extra but collects every module. +pytest.importorskip("hypothesis") + +from hypothesis import given, settings # noqa: E402 +from hypothesis import strategies as st # noqa: E402 + +from tests.core.test_group_population_empty import populations # noqa: E402 + + +@st.composite +def _groups(draw): + count = draw(st.integers(1, 6)) + members_entity_id = draw(st.lists(st.integers(0, count - 1), max_size=30)) + values = draw( + st.lists( + st.integers(-5, 5), + min_size=len(members_entity_id), + max_size=len(members_entity_id), + ) + ) + return count, members_entity_id, values + + +@settings(max_examples=300, deadline=None) +@given(_groups()) +def test_reductions_and_ranks_match_oracle(case): + count, members_entity_id, values = case + persons, households = populations(members_entity_id, count) + array = np.array(values, dtype=float) + + members = {group: [] for group in range(count)} + for person, group in enumerate(members_entity_id): + members[group].append(person) + group_values = [[values[person] for person in members[g]] for g in range(count)] + + assert households.sum(array).tolist() == [sum(v) for v in group_values] + assert households.nb_persons().tolist() == [len(v) for v in group_values] + assert households.any(array > 0).tolist() == [ + any(x > 0 for x in v) for v in group_values + ] + assert households.all(array > 0).tolist() == [ + all(x > 0 for x in v) for v in group_values + ] + assert households.max(array).tolist() == [ + max(v, default=-np.inf) for v in group_values + ] + assert households.min(array).tolist() == [ + min(v, default=np.inf) for v in group_values + ] + + expected_rank = [0] * len(values) + for group in range(count): + ordered = sorted(members[group], key=lambda person: values[person]) + for rank, person in enumerate(ordered): + expected_rank[person] = rank + assert persons.get_rank(households, array).tolist() == expected_rank From eca6a6215d43aca85b9ebf2f007cb55a7763be40 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 6 Oct 2026 13:42:39 -0400 Subject: [PATCH 2/3] Size the loop by a length check instead of initial=-1 numpy.max(x, initial=-1) raises OverflowError for an unsigned index array, which master handled for any non-empty population. A length check gives 0 for no persons and leaves every other case exactly as on master. Co-Authored-By: Claude Opus 5.5 --- policyengine_core/populations/group_population.py | 10 ++++++++-- policyengine_core/populations/population.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/policyengine_core/populations/group_population.py b/policyengine_core/populations/group_population.py index 7c0aaec78..739156bf9 100644 --- a/policyengine_core/populations/group_population.py +++ b/policyengine_core/populations/group_population.py @@ -67,7 +67,11 @@ def clone( def members_position(self) -> ArrayLike: if self._members_position is None and self.members_entity_id is not None: # We could use self.count and self.members.count , but with the current initilization, we are not sure count will be set before members_position is called - nb_entities = numpy.max(self.members_entity_id, initial=-1) + 1 + nb_entities = ( + numpy.max(self.members_entity_id) + 1 + if len(self.members_entity_id) + else 0 + ) nb_persons = len(self.members_entity_id) self._members_position = numpy.empty_like(self.members_entity_id) counter_by_entity = numpy.zeros(nb_entities) @@ -192,7 +196,9 @@ def reduce( # We loop over the positions in the entity # Looping over the entities is tempting, but potentielly slow if there are a lot of entities # With no persons there is no position, and every entity keeps the neutral element. - biggest_entity_size = numpy.max(position_in_entity, initial=-1) + 1 + biggest_entity_size = ( + numpy.max(position_in_entity) + 1 if len(position_in_entity) else 0 + ) for p in range(biggest_entity_size): values = self.value_nth_person(p, filtered_array, default=neutral_element) diff --git a/policyengine_core/populations/population.py b/policyengine_core/populations/population.py index 1106e4d91..fe0b37a03 100644 --- a/policyengine_core/populations/population.py +++ b/policyengine_core/populations/population.py @@ -233,7 +233,7 @@ def get_rank( ) positions = entity.members_position - biggest_entity_size = numpy.max(positions, initial=-1) + 1 + biggest_entity_size = numpy.max(positions) + 1 if len(positions) else 0 filtered_criteria = numpy.where(condition, criteria, numpy.inf) ids = entity.members_entity_id From 8d662ec7e8729ae640ec270ca06ee2f5c81352fd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 6 Oct 2026 16:52:09 -0400 Subject: [PATCH 3/3] Keep empty memberships integer so ranks and projections work Addresses the independent review of eca6a621 (one should-fix finding). An empty list or numpy.array([]) is float64, and so were the IDs and memberships build_default_simulation(count=0) made (np.array(range(0))). NumPy indexes only with integers or booleans, so get_rank, project, value_nth_person and projector reductions over no persons still raised IndexError or TypeError. The members_entity_id setter now stores an empty membership as an integer array, and build_default_simulation uses np.arange(count), which is integer even for 0. Adds regressions for a plain [] membership and a default simulation of no one, and lets the Hypothesis property pass memberships as plain lists. All fail on eca6a621 (the property shrinks to one group, no persons, given as a list). Co-Authored-By: Claude Opus 5.5 --- .../fix-empty-group-reductions.fixed.md | 2 +- .../populations/group_population.py | 4 +++ .../simulations/simulation_builder.py | 2 +- tests/core/test_group_population_empty.py | 32 +++++++++++++++++++ ...st_group_population_reductions_property.py | 11 +++++-- 5 files changed, 46 insertions(+), 5 deletions(-) diff --git a/changelog.d/fix-empty-group-reductions.fixed.md b/changelog.d/fix-empty-group-reductions.fixed.md index 6f336c892..be0927fb2 100644 --- a/changelog.d/fix-empty-group-reductions.fixed.md +++ b/changelog.d/fix-empty-group-reductions.fixed.md @@ -1 +1 @@ -Group `all`, `max` and `min`, `members_position` and `Population.get_rank` now work when a simulation has no persons, giving the neutral element (True, -inf, inf) for each group instead of raising "zero-size array to reduction". +Group `all`, `max` and `min`, `members_position` and `Population.get_rank` now work when a simulation has no persons, giving the neutral element (True, -inf, inf) for each group instead of raising "zero-size array to reduction"; memberships given as an empty list, and default simulations of no one, are kept as integers so ranks and projections over them work. diff --git a/policyengine_core/populations/group_population.py b/policyengine_core/populations/group_population.py index 739156bf9..8d42a811f 100644 --- a/policyengine_core/populations/group_population.py +++ b/policyengine_core/populations/group_population.py @@ -92,6 +92,10 @@ def members_entity_id(self) -> ArrayLike: @members_entity_id.setter def members_entity_id(self, members_entity_id: ArrayLike) -> None: + if members_entity_id is not None and len(members_entity_id) == 0: + # An empty list or ``numpy.array([])`` is float, and NumPy indexes + # only with integers (or booleans): keep no members as integers. + members_entity_id = numpy.zeros(0, dtype=numpy.intp) self._members_entity_id = members_entity_id @property diff --git a/policyengine_core/simulations/simulation_builder.py b/policyengine_core/simulations/simulation_builder.py index e3dee0cc5..5f3840501 100644 --- a/policyengine_core/simulations/simulation_builder.py +++ b/policyengine_core/simulations/simulation_builder.py @@ -245,7 +245,7 @@ def build_default_simulation( ) for population in simulation.populations.values(): population.count = count - population.ids = np.array(range(count)) + population.ids = np.arange(count) # integers even when count is 0 if not population.entity.is_person: population.members_entity_id = ( population.ids diff --git a/tests/core/test_group_population_empty.py b/tests/core/test_group_population_empty.py index e69df429b..60a6f005f 100644 --- a/tests/core/test_group_population_empty.py +++ b/tests/core/test_group_population_empty.py @@ -5,6 +5,8 @@ return 0. With no persons at all, ``reduce``, ``members_position`` and ``Population.get_rank`` took ``numpy.max`` of an empty array and raised "zero-size array to reduction operation maximum which has no identity". +Memberships given as an empty list, or built for no one, were also float, +which NumPy cannot index with. """ from __future__ import annotations @@ -13,6 +15,8 @@ from policyengine_core.entities import build_entity from policyengine_core.populations import GroupPopulation, Population +from policyengine_core.simulations import SimulationBuilder +from policyengine_core.taxbenefitsystems import TaxBenefitSystem PERSON = build_entity("person", "persons", "Person", is_person=True) HOUSEHOLD = build_entity( @@ -49,3 +53,31 @@ def test_rank_over_no_persons_is_empty(): persons, households = populations([], 2) assert persons.get_rank(households, np.array([], dtype=float)).tolist() == [] + + +def test_no_members_given_as_a_plain_empty_list(): + # ``[]`` and ``numpy.array([])`` are float; NumPy indexes only with + # integers, so ranks and projections raised IndexError. + persons, households = populations([], 1) + households.members_entity_id = [] + + assert households.members_position.dtype.kind == "i" + assert persons.get_rank(households, np.array([], dtype=float)).tolist() == [] + assert households.project(np.array([5.0])).tolist() == [] + assert households.max(np.array([], dtype=float)).tolist() == [-np.inf] + + +def test_default_simulation_of_no_one(): + simulation = SimulationBuilder().build_default_simulation( + TaxBenefitSystem([PERSON, HOUSEHOLD]), count=0 + ) + persons = simulation.persons + households = simulation.populations["household"] + no_values = np.array([], dtype=float) + + assert households.ids.dtype.kind == "i" + assert persons.get_rank(households, no_values).tolist() == [] + assert households.value_nth_person(0, no_values).tolist() == [] + assert households.project(no_values).tolist() == [] + assert persons.household.max(no_values).tolist() == [] + assert households.all(no_values > 0).tolist() == [] diff --git a/tests/core/test_group_population_reductions_property.py b/tests/core/test_group_population_reductions_property.py index 92aaffc4b..2732cfc03 100644 --- a/tests/core/test_group_population_reductions_property.py +++ b/tests/core/test_group_population_reductions_property.py @@ -1,7 +1,8 @@ """Property: group reductions and ranks match a plain-Python oracle. For any number of groups (1 to 6), any assignment of 0 to 30 persons to them -(so groups may be empty, and there may be no persons at all) and any values: +(so groups may be empty, and there may be no persons at all), given as an +integer array or a plain list, and any values: * ``sum``, ``nb_persons``, ``any``, ``all``, ``max`` and ``min`` equal the oracle, with 0, False, True, -inf and inf for groups with no members; @@ -37,14 +38,18 @@ def _groups(draw): max_size=len(members_entity_id), ) ) - return count, members_entity_id, values + as_list = draw(st.booleans()) + return count, members_entity_id, values, as_list @settings(max_examples=300, deadline=None) @given(_groups()) def test_reductions_and_ranks_match_oracle(case): - count, members_entity_id, values = case + count, members_entity_id, values, as_list = case persons, households = populations(members_entity_id, count) + if as_list: + # Memberships may come as a plain list; an empty one is float. + households.members_entity_id = list(members_entity_id) array = np.array(values, dtype=float) members = {group: [] for group in range(count)}