From 5cc375e766b287de53315a441e7508aa6b0c182c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 8 Oct 2026 21:02:37 -0400 Subject: [PATCH 1/6] Refuse unsigned ids int64 cannot hold; match concept ids as int64 validate_concept_tables (_check_pointers) and ConceptMapping.encode (_Context.person_rows) matched pointers by casting them to int64. The concept-frame contract accepts unsigned id dtypes, so a pointer of 2**64 - 1 was read as -1: with a person whose id is -1, a dangling pointer validated clean and encoded as a link. Both helpers now refuse, with a ValueError naming every such column, any unsigned id or pointer above 2**63 - 1 before converting anything. The same property exposed a second wrap: pandas matches against a narrower unsigned index by casting the targets down to it, so with uint8 person ids a pointer of 261 named person 5 (and nullable UInt8 ids raised a TypeError). Both helpers now widen the integer id columns to int64 before matching, which is exact once the wide ids are refused. Regressions fail on main: 2**63 and 2**64 - 1 as uint64 and UInt64 in each id and pointer column, the #1122 review probe (2**64 - 1 onto -1), and narrow uint8/16/32 and UInt8/16/32 ids; 2**63 - 1 is kept exactly. A Hypothesis property over 16 integer dtypes checks that every frame is refused exactly when some unsigned value exceeds 2**63 - 1, and otherwise validates and encodes exactly as the same ids typed int64. Co-Authored-By: Claude Opus 5.5 --- .../src/microcosm/frame/concept_mapping.py | 26 +- .../src/microcosm/frame/concepts.py | 62 +++- .../shared/test_concept_mapping.py | 189 +++++++++++ .../tests/engine_free/shared/test_concepts.py | 118 +++++++ .../microcosm_frame/concept_id_dtypes.py | 306 ++++++++++++++++++ 5 files changed, 693 insertions(+), 8 deletions(-) create mode 100644 test_support/microcosm_frame/concept_id_dtypes.py diff --git a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py index fd5f5f8a6..d0b530875 100644 --- a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py +++ b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py @@ -50,6 +50,7 @@ Unit, _parsing, _record_fields, + _require_int64_ids, _text_fields, concept, concept_schema_sha256, @@ -1365,16 +1366,19 @@ class _Context: """A concept frame, with pointer lookups done by exact integer ids. Pointers are matched through row positions, never through float - arrays, so person ids beyond 2**53 resolve exactly. + arrays, so person ids beyond 2**53 resolve exactly. Ids are matched as + int64, so an unsigned id above 2**63 - 1, which would wrap (2**64 - 1 + onto -1), is refused. """ def __init__(self, person: pd.DataFrame, household: pd.DataFrame) -> None: + _require_int64_ids(person, household, "Encoding matches") self.person = person self.household = household - self._person_ids = pd.Index(person[_PERSON_ID].to_numpy()) + self._person_ids = pd.Index(_matched_ids(person[_PERSON_ID])) self._household_rows = pd.Index( - household[_HOUSEHOLD_ID].to_numpy() - ).get_indexer(person[_PERSON_HOUSEHOLD_ID].to_numpy()) + _matched_ids(household[_HOUSEHOLD_ID]) + ).get_indexer(_matched_ids(person[_PERSON_HOUSEHOLD_ID])) def has(self, concept_id: str) -> bool: item = CONCEPT_BY_ID[concept_id] @@ -1409,6 +1413,20 @@ def reference_rows(self) -> np.ndarray: return by_household[self._household_rows] +def _matched_ids(values: pd.Series) -> np.ndarray: + """Ids ready for exact matching: integers as int64, anything else as is. + + pandas matches against a narrower unsigned index by casting the targets + down to it (261 onto 5 for uint8), so null-free integer ids are widened + first; :func:`_require_int64_ids` has already refused any int64 cannot + hold. + """ + + if values.dtype.kind in "iu" and not values.hasnans: + return values.to_numpy(dtype=np.int64) + return values.to_numpy() + + def _apply( binding: InputBinding, context: _Context, diff --git a/packages/microcosm-frame/src/microcosm/frame/concepts.py b/packages/microcosm-frame/src/microcosm/frame/concepts.py index 943f3e57e..575c5bbb6 100644 --- a/packages/microcosm-frame/src/microcosm/frame/concepts.py +++ b/packages/microcosm-frame/src/microcosm/frame/concepts.py @@ -1398,7 +1398,8 @@ def validate_concept_tables( Raises: ValueError: If a table or an id column is missing, ids are not unique, - or a person points at an unknown household. + a person points at an unknown household, or an unsigned id or + pointer exceeds ``2**63 - 1``: pointers are matched as int64. """ person, household = _require_structure(tables) @@ -1599,9 +1600,15 @@ def add(code: str, message: str, rows: int = 0) -> None: def _check_pointers( person: pd.DataFrame, household: pd.DataFrame ) -> list[ConceptViolation]: + # Ids and pointers are matched as int64, where an unsigned id above + # 2**63 - 1 would wrap (2**64 - 1 onto -1), so such ids are refused + # first. Every other id converts exactly. The person ids are widened + # too: pandas matches against a narrower unsigned index by casting the + # pointers down to it (261 onto 5 for uint8). + _require_int64_ids(person, household, "Pointer checks match") out: list[ConceptViolation] = [] - ids = pd.Index(person[_PERSON_ID].to_numpy()) - own_household = person[_PERSON_HOUSEHOLD_ID].to_numpy() + ids = pd.Index(person[_PERSON_ID].to_numpy(dtype=np.int64)) + own_household = person[_PERSON_HOUSEHOLD_ID].to_numpy(dtype=np.int64) rows = np.arange(len(person)) def add( @@ -1711,7 +1718,7 @@ def positions(values: pd.Series) -> tuple[np.ndarray, np.ndarray]: household["reference_person_id"] ): found, present = positions(household["reference_person_id"]) - own = household[_HOUSEHOLD_ID].to_numpy() + own = household[_HOUSEHOLD_ID].to_numpy(dtype=np.int64) member = (found >= 0) & (own_household[np.maximum(found, 0)] == own) add( "household", @@ -1728,6 +1735,53 @@ def _is_integer_column(values: pd.Series) -> bool: return values.dtype.kind in "iu" +#: The largest id a pointer can be matched by: ids are matched as int64. +_INT64_MAX = int(np.iinfo(np.int64).max) + +#: Every column holding a person or household id, as (entity, column). +_ID_COLUMNS = ( + ("person", _PERSON_ID), + ("person", _PERSON_HOUSEHOLD_ID), + ("person", "partner_person_id"), + ("person", "parent_1_person_id"), + ("person", "parent_2_person_id"), + ("household", _HOUSEHOLD_ID), + ("household", "reference_person_id"), +) + + +def _exceeds_int64(values: pd.Series) -> bool: + """Whether an unsigned integer column holds an id int64 cannot hold.""" + + if values.dtype.kind != "u": + return False + present = values.dropna() + return len(present) > 0 and int(present.max()) > _INT64_MAX + + +def _require_int64_ids( + person: pd.DataFrame, household: pd.DataFrame, what: str +) -> None: + """Refuse a frame whose ids int64 cannot hold, naming every such column. + + Only an unsigned id above ``2**63 - 1`` is refused; every other integer + id converts to int64 exactly. ``what`` starts the message, as in + "Encoding matches". + """ + + tables = {"person": person, "household": household} + wide = [ + f"{entity}.{column}" + for entity, column in _ID_COLUMNS + if column in tables[entity].columns and _exceeds_int64(tables[entity][column]) + ] + if wide: + raise ValueError( + f"{what} ids as int64, and {wide} hold unsigned ids above " + f"{_INT64_MAX}, which int64 cannot represent." + ) + + def _parent_cycle_members(n_persons: int, parent_rows: list[np.ndarray]) -> int: """Count persons on a cycle of parent pointers, or on a path between two. diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py b/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py index 13ea6b033..27b6394cb 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py @@ -63,6 +63,12 @@ concept_frames, shares, ) +from test_support.microcosm_frame.concept_id_dtypes import ( + WIDE_CASES, + id_typed_frames, + typed, + wide_id_frame, +) from test_support.microcosm_frame.concept_mappings import ( COVERAGE_DOCS, concept_mappings, @@ -531,6 +537,189 @@ def test_group_bindings_are_deferred_not_executed(self) -> None: } +#: The two dtypes that hold an id int64 cannot: NumPy's and pandas' nullable. +UNSIGNED_64 = [np.dtype(np.uint64), pd.UInt64Dtype()] + + +def _pointer_mapping() -> ConceptMapping: + """One engine input per transform that resolves a pointer.""" + + bindings = [ + _binding( + engine_input=role.value, + concepts=RelationshipRole(role=role).concepts, + transform=RelationshipRole(role=role), + ) + for role in Role + ] + bindings.append( + _binding( + engine_input="children", concepts=_PARENTS, transform=CoresidentChildCount() + ) + ) + bindings.append( + _binding( + engine_input="rent_paid", + concepts=("fact:household.rent",), + transform=AllocateToReferencePerson(), + ) + ) + return _mapping(bindings) + + +def _assert_same_encoding(actual, expected) -> None: + assert actual.keys() == expected.keys() + for entity, table in expected.items(): + pd.testing.assert_frame_equal(actual[entity], table, check_dtype=False) + + +def _as_int64(tables: dict[str, pd.DataFrame]) -> dict[str, pd.DataFrame]: + """The same frame with every integer id and pointer typed int64.""" + + out = {entity: table.copy() for entity, table in tables.items()} + for entity, column in ( + ("person", "person_id"), + ("person", "person_household_id"), + ("household", "household_id"), + ): + out[entity][column] = out[entity][column].astype(np.int64) + for entity, column in ( + ("person", "partner_person_id"), + ("person", "parent_1_person_id"), + ("person", "parent_2_person_id"), + ("household", "reference_person_id"), + ): + if column in out[entity].columns: + out[entity][column] = out[entity][column].astype("Int64") + return out + + +class TestIdDtypes: + """Encoding matches ids of any accepted integer dtype exactly. + + Pointers are matched as int64. An unsigned id above ``2**63 - 1`` would + wrap (``2**64 - 1`` onto -1) and so is refused, by name; every other id + must encode exactly as the same values typed int64. + """ + + @pytest.mark.parametrize("dtype", UNSIGNED_64, ids=str) + @pytest.mark.parametrize("value", [2**63, 2**64 - 1]) + @pytest.mark.parametrize("case", sorted(WIDE_CASES)) + def test_an_id_int64_cannot_hold_is_refused_by_column( + self, case, value, dtype + ) -> None: + tables, wide = wide_id_frame(case, value, dtype) + with pytest.raises( + ValueError, match=re.escape(f"{wide} hold unsigned ids above {2**63 - 1}") + ): + _pointer_mapping().encode(tables) + + def test_a_dangling_pointer_cannot_wrap_onto_person_minus_one(self) -> None: + # Read as int64, 2**64 - 1 is -1: reference person 11's dangling + # partner pointer made person -1 the reference person's partner. + person = pd.DataFrame( + { + "person_id": np.array([-1, 11, 12, 13], dtype=np.int64), + "person_household_id": [1, 1, 1, 1], + "partner_person_id": pd.array( + [None, 2**64 - 1, None, None], dtype="UInt64" + ), + } + ) + household = pd.DataFrame( + {"household_id": [1], "reference_person_id": pd.array([11], dtype="Int64")} + ) + with pytest.raises(ValueError, match=re.escape("['person.partner_person_id']")): + _pointer_mapping().encode({"person": person, "household": household}) + + @pytest.mark.parametrize( + "dtype", ["uint8", "uint16", "uint32", "UInt8", "UInt16", "UInt32"] + ) + def test_a_narrow_unsigned_id_cannot_catch_a_pointer_past_its_range( + self, dtype + ) -> None: + # pandas matched against a narrower unsigned index by casting the + # other side down to it, so 5 + 2**8 named person 5 under uint8, and + # household 1 + 2**8 was household 1; the nullable dtypes raised a + # TypeError instead. + dtype = pd.api.types.pandas_dtype(dtype) + span = 2 ** (8 * dtype.itemsize) + person = pd.DataFrame( + { + "person_id": typed([5, 6, 7, 8], dtype), + "person_household_id": np.array([1, 1, 1, 1 + span], dtype=np.int64), + "partner_person_id": pd.array([6, 5, None, 5], dtype="Int64"), + "parent_1_person_id": pd.array( + [None, None, 5 + span, None], dtype="Int64" + ), + "parent_2_person_id": pd.array([None] * 4, dtype="Int64"), + } + ) + household = pd.DataFrame( + { + "household_id": typed([1, 2], dtype), + "reference_person_id": pd.array([5 + span, 8], dtype="Int64"), + "rent": [100.0, 200.0], + } + ) + tables = {"person": person, "household": household} + mapping = _pointer_mapping() + encoded = mapping.encode(tables).tables + _assert_same_encoding(encoded, mapping.encode(_as_int64(tables)).tables) + # Person 7's parent and the reference person of household 1 are nobody. + assert encoded["person"]["children"].tolist() == [0, 0, 0, 0] + assert encoded["person"]["reference_person_partner"].tolist() == [False] * 4 + + @pytest.mark.parametrize("dtype", UNSIGNED_64, ids=str) + def test_ids_up_to_int64_max_are_kept_exactly(self, dtype) -> None: + top = 2**63 - 1 + person = pd.DataFrame( + { + "person_id": typed([top, top - 1, top - 2], dtype), + "person_household_id": typed([top, top, top], dtype), + "partner_person_id": pd.array([top - 1, top, None], dtype="UInt64"), + "parent_1_person_id": pd.array([None, None, top], dtype="UInt64"), + "parent_2_person_id": pd.array([None, None, top - 1], dtype="UInt64"), + } + ) + household = pd.DataFrame( + { + "household_id": typed([top], dtype), + "reference_person_id": pd.array([top], dtype="UInt64"), + "rent": [100.0], + } + ) + out = _pointer_mapping().encode({"person": person, "household": household}) + flags = out.tables["person"] + assert flags["reference_person"].tolist() == [True, False, False] + assert flags["reference_person_partner"].tolist() == [False, True, False] + assert flags["children"].tolist() == [1, 1, 0] + assert flags["rent_paid"].tolist() == [100.0, 0.0, 0.0] + assert out.tables["person"]["person_id"].tolist() == [top, top - 1, top - 2] + + @settings(max_examples=300, deadline=None) + @given( + name=st.sampled_from([*sorted(MAPPINGS), "pointer transforms"]), + case=id_typed_frames(dangling_households=True), + data=st.data(), + ) + def test_every_frame_is_refused_or_encodes_as_its_int64_twin( + self, name, case, data + ) -> None: + mapping = _pointer_mapping() if name == "pointer transforms" else MAPPINGS[name] + parameters = _parameters(mapping, data) + if case.wide: + with pytest.raises( + ValueError, match=re.escape(f"{case.wide} hold unsigned ids above") + ): + mapping.encode(case.tables, **parameters) + return + _assert_same_encoding( + mapping.encode(case.tables, **parameters).tables, + mapping.encode(case.twin, **parameters).tables, + ) + + class TestDecodeGuards: def test_a_whole_number_concept_refuses_fractions(self) -> None: mapping = _mapping([_binding()]) diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_concepts.py b/packages/microcosm-frame/tests/engine_free/shared/test_concepts.py index 1f97a493b..e21de076a 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_concepts.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_concepts.py @@ -6,6 +6,7 @@ """ import json +import re from dataclasses import replace import numpy as np @@ -40,6 +41,12 @@ validate_concept_tables, ) from test_support.microcosm_frame.concept_frames import concept_frames +from test_support.microcosm_frame.concept_id_dtypes import ( + WIDE_CASES, + id_typed_frames, + typed, + wide_id_frame, +) from test_support.paths import paths_for PROPERTY = settings(max_examples=150, deadline=None) @@ -754,6 +761,117 @@ def test_cycle_count_matches_a_reachability_reference(self, n, data) -> None: assert rows == _cycle_reference(parents, n) +#: The two dtypes that hold an id int64 cannot: NumPy's and pandas' nullable. +UNSIGNED_64 = [np.dtype(np.uint64), pd.UInt64Dtype()] + + +class TestIdDtypes: + """Ids and pointers of any accepted integer dtype are matched exactly. + + Pointers are matched as int64. An unsigned id above ``2**63 - 1`` would + wrap (``2**64 - 1`` onto -1) and so is refused, by name; every other id + must validate exactly as the same values typed int64. + """ + + @pytest.mark.parametrize("dtype", UNSIGNED_64, ids=str) + @pytest.mark.parametrize("value", [2**63, 2**64 - 1]) + @pytest.mark.parametrize("case", sorted(WIDE_CASES)) + def test_an_id_int64_cannot_hold_is_refused_by_column( + self, case, value, dtype + ) -> None: + tables, wide = wide_id_frame(case, value, dtype) + with pytest.raises( + ValueError, match=re.escape(f"{wide} hold unsigned ids above {2**63 - 1}") + ): + validate_concept_tables(tables) + + def test_a_dangling_pointer_cannot_wrap_onto_person_minus_one(self) -> None: + # The microcosm#1122 review probe: read as int64, 2**64 - 1 is -1, so + # person 11's dangling partner pointer named person -1, who names 11 + # back, and the frame validated clean. + person = pd.DataFrame( + { + "person_id": np.array([-1, 11, 12, 13], dtype=np.int64), + "person_household_id": [1, 1, 1, 1], + "partner_person_id": pd.array( + [11, 2**64 - 1, None, None], dtype="UInt64" + ), + } + ) + household = pd.DataFrame( + {"household_id": [1], "reference_person_id": pd.array([-1], dtype="Int64")} + ) + with pytest.raises(ValueError, match=re.escape("['person.partner_person_id']")): + validate_concept_tables({"person": person, "household": household}) + + @pytest.mark.parametrize( + "dtype", ["uint8", "uint16", "uint32", "UInt8", "UInt16", "UInt32"] + ) + def test_a_narrow_unsigned_id_cannot_catch_a_pointer_past_its_range( + self, dtype + ) -> None: + # pandas matched pointers against a narrower unsigned index by casting + # them down to it, so 5 + 2**8 named person 5 under uint8; the + # nullable dtypes raised a TypeError instead. + dtype = pd.api.types.pandas_dtype(dtype) + past = 5 + 2 ** (8 * dtype.itemsize) + person = pd.DataFrame( + { + "person_id": typed([5, 6], dtype), + "person_household_id": [1, 1], + "partner_person_id": pd.array([6, past], dtype="Int64"), + } + ) + household = pd.DataFrame( + { + "household_id": [1], + "reference_person_id": pd.array([past], dtype="Int64"), + } + ) + assert _violations({"person": person, "household": household}) == { + ("partner_person_id", "dangling"), + ("partner_person_id", "asymmetric"), + ("reference_person_id", "not_member"), + } + + @pytest.mark.parametrize("dtype", UNSIGNED_64, ids=str) + def test_ids_up_to_int64_max_are_kept_exactly(self, dtype) -> None: + top = 2**63 - 1 + person = pd.DataFrame( + { + "person_id": typed([top, top - 1, top - 2], dtype), + "person_household_id": typed([top, top, top], dtype), + "partner_person_id": pd.array([top - 1, top, None], dtype="UInt64"), + "parent_1_person_id": pd.array([None, None, top], dtype="UInt64"), + "parent_2_person_id": pd.array([None, None, top - 1], dtype="UInt64"), + } + ) + household = pd.DataFrame( + { + "household_id": typed([top], dtype), + "reference_person_id": pd.array([top], dtype="UInt64"), + } + ) + tables = {"person": person, "household": household} + assert validate_concept_tables(tables) == () + # Within float rounding of the others, top - 3 is still nobody. + person["partner_person_id"] = pd.array([top - 1, top, top - 3], dtype="UInt64") + assert _violations(tables) == {("partner_person_id", "dangling")} + + @settings(max_examples=300, deadline=None) + @given(case=id_typed_frames()) + def test_every_frame_is_refused_or_validates_as_its_int64_twin(self, case) -> None: + if case.wide: + with pytest.raises( + ValueError, match=re.escape(f"{case.wide} hold unsigned ids above") + ): + validate_concept_tables(case.tables) + else: + assert validate_concept_tables(case.tables) == validate_concept_tables( + case.twin + ) + + class TestTransportSplit: @PROPERTY @given(tables=concept_frames()) diff --git a/test_support/microcosm_frame/concept_id_dtypes.py b/test_support/microcosm_frame/concept_id_dtypes.py new file mode 100644 index 000000000..96a88122a --- /dev/null +++ b/test_support/microcosm_frame/concept_id_dtypes.py @@ -0,0 +1,306 @@ +"""Concept frames whose id and pointer columns take every accepted integer dtype. + +The concept-frame contract accepts ids and pointers of any integer dtype: +NumPy or pandas nullable, signed or unsigned, any width. Matching ids across +those dtypes is where values wrap. A bare int64 cast turns ``2**64 - 1`` into +-1, and pandas matches against a narrower unsigned index by casting the other +side down to it (261 onto 5 for uint8). :func:`id_typed_frames` relabels a +valid frame's ids onto values at those edges, may plant pointers on the +values a wrap would collide with, and gives each id column a dtype drawn from +those that hold its values. A consumer must then either refuse the frame or +treat it exactly as the same ids typed int64. +""" + +# ruff: noqa: F401 + +from __future__ import annotations + +from typing import NamedTuple + +import numpy as np +import pandas as pd +from hypothesis import strategies as st + +from test_support.microcosm_frame.concept_frames import concept_frames + +INT64_MIN = -(2**63) +INT64_MAX = 2**63 - 1 +UINT64_MAX = 2**64 - 1 + +#: Every column holding a person or household id, as (entity, column), in the +#: order a refusal names them. +ID_COLUMNS: tuple[tuple[str, str], ...] = ( + ("person", "person_id"), + ("person", "person_household_id"), + ("person", "partner_person_id"), + ("person", "parent_1_person_id"), + ("person", "parent_2_person_id"), + ("household", "household_id"), + ("household", "reference_person_id"), +) +_PERSON_POINTERS = ("partner_person_id", "parent_1_person_id", "parent_2_person_id") +#: The id columns a frame requires, never null; the rest are nullable pointers. +_REQUIRED = { + ("person", "person_id"), + ("person", "person_household_id"), + ("household", "household_id"), +} + +#: Every integer dtype the contract accepts for an id, NumPy then nullable. +ID_DTYPES = ( + *(np.dtype(f"{sign}int{bits}") for sign in ("", "u") for bits in (8, 16, 32, 64)), + *( + pd.api.types.pandas_dtype(f"{sign}Int{bits}") + for sign in ("", "U") + for bits in (8, 16, 32, 64) + ), +) + +#: Values within one of a power of two where some integer dtype wraps. +_EDGES = sorted( + { + sign * 2**power + offset + for power in (7, 8, 15, 16, 31, 32, 63, 64) + for sign in (1, -1) + for offset in (-1, 0, 1) + if INT64_MIN <= sign * 2**power + offset <= UINT64_MAX + } +) + +#: The shifts a wrap applies to a value: a multiple of a dtype's span. +_WRAPS = tuple(sign * 2**power for power in (8, 16, 32, 64) for sign in (1, -1)) + + +def _ids(unsigned: bool) -> st.SearchStrategy[int]: + """Ids near zero, at the dtype edges, or anywhere int64 or uint64 holds.""" + + low, high = (0, UINT64_MAX) if unsigned else (INT64_MIN, INT64_MAX) + return st.one_of( + st.integers(max(low, -300), 300), + st.sampled_from([value for value in _EDGES if low <= value <= high]), + st.integers(low, high), + ) + + +def holds(dtype, values: list[int | None]) -> bool: + """Whether ``dtype`` can hold every one of ``values`` (``None`` is null).""" + + if isinstance(dtype, np.dtype): + if any(value is None for value in values): + return False + info = np.iinfo(dtype) + else: + info = np.iinfo(dtype.numpy_dtype) + return all(info.min <= value <= info.max for value in values if value is not None) + + +def typed( + values: list[int | None], dtype +) -> np.ndarray | pd.api.extensions.ExtensionArray: + """``values`` as a column of ``dtype``.""" + + if isinstance(dtype, np.dtype): + return np.array(values, dtype=dtype) + return pd.array(values, dtype=dtype) + + +class IdTypedFrame(NamedTuple): + """A frame with retyped ids, its int64 twin, and the columns int64 cannot hold. + + ``twin`` holds the same values with every required id column typed + ``int64`` and every pointer typed ``Int64``, or is ``None`` when ``wide`` + is not empty. ``wide`` names, as ``"entity.column"`` in + :data:`ID_COLUMNS` order, every column holding a value above + ``2**63 - 1``. + """ + + tables: dict[str, pd.DataFrame] + twin: dict[str, pd.DataFrame] | None + wide: list[str] + + +def _frame( + base: dict[str, pd.DataFrame], columns: dict[tuple[str, str], object] +) -> dict[str, pd.DataFrame]: + out = {entity: table.copy() for entity, table in base.items()} + for (entity, column), values in columns.items(): + out[entity][column] = values + return out + + +@st.composite +def id_typed_frames(draw, *, dangling_households: bool = False) -> IdTypedFrame: + """A concept frame whose ids sit at the dtype edges, in any accepted dtype. + + Starting from a valid frame, person and household ids are relabelled + (each set drawn wholly from the int64 range or wholly from the uint64 + range, so some dtype holds them). Each pointer column may then gain one + planted value: an id shifted by a dtype's span, the value a wrap would + land on an existing id, or any id. With ``dangling_households`` the + household pointer may be planted too, which leaves a person naming no + household. Each id column then takes a dtype drawn from the accepted ones + that hold its values. + """ + + base = draw(concept_frames(max_households=3, max_members=4)) + person, household = base["person"], base["household"] + old_persons = person["person_id"].tolist() + old_households = household["household_id"].tolist() + new_persons = draw( + st.lists( + _ids(draw(st.booleans(), label="unsigned person ids")), + min_size=len(old_persons), + max_size=len(old_persons), + unique=True, + ), + label="person ids", + ) + new_households = draw( + st.lists( + _ids(draw(st.booleans(), label="unsigned household ids")), + min_size=len(old_households), + max_size=len(old_households), + unique=True, + ), + label="household ids", + ) + to_person = dict(zip(old_persons, new_persons, strict=True)) + to_household = dict(zip(old_households, new_households, strict=True)) + + def relabel(values: pd.Series, to: dict[int, int]) -> list[int | None]: + return [None if pd.isna(value) else to[int(value)] for value in values] + + values: dict[tuple[str, str], list[int | None]] = { + ("person", "person_id"): list(new_persons), + ("person", "person_household_id"): relabel( + person["person_household_id"], to_household + ), + **{ + ("person", column): relabel(person[column], to_person) + for column in _PERSON_POINTERS + }, + ("household", "household_id"): list(new_households), + ("household", "reference_person_id"): relabel( + household["reference_person_id"], to_person + ), + } + + planted = [("person", column) for column in _PERSON_POINTERS] + planted.append(("household", "reference_person_id")) + if dangling_households: + planted.append(("person", "person_household_id")) + for key in planted: + if not draw(st.booleans(), label=f"plant {key[1]}"): + continue + column = values[key] + targets = new_households if key[1] == "person_household_id" else new_persons + row = draw(st.integers(0, len(column) - 1), label="row") + if draw(st.booleans(), label="plant a wrap image"): + value = draw(st.sampled_from(targets)) + draw(st.sampled_from(_WRAPS)) + else: + value = draw(_ids(draw(st.booleans()))) + if not INT64_MIN <= value <= UINT64_MAX: + continue + before, column[row] = column[row], value + if not any(holds(dtype, column) for dtype in ID_DTYPES): + column[row] = before + + columns = { + key: typed( + values[key], + draw( + st.sampled_from( + [dtype for dtype in ID_DTYPES if holds(dtype, values[key])] + ), + label=f"{key[1]} dtype", + ), + ) + for key in ID_COLUMNS + } + wide = [ + f"{entity}.{column}" + for entity, column in ID_COLUMNS + if any( + value is not None and value > INT64_MAX + for value in values[(entity, column)] + ) + ] + twin = ( + None + if wide + else _frame( + base, + { + key: typed( + values[key], + np.dtype(np.int64) if key in _REQUIRED else pd.Int64Dtype(), + ) + for key in ID_COLUMNS + }, + ) + ) + return IdTypedFrame(_frame(base, columns), twin, wide) + + +def three_person_frame() -> dict[str, pd.DataFrame]: + """Persons 10 and 20, partners, are the parents of 30; 10 heads household 1.""" + + person = pd.DataFrame( + { + "person_id": np.array([10, 20, 30], dtype=np.int64), + "person_household_id": np.array([1, 1, 1], dtype=np.int64), + "partner_person_id": pd.array([20, 10, None], dtype="Int64"), + "parent_1_person_id": pd.array([None, None, 10], dtype="Int64"), + "parent_2_person_id": pd.array([None, None, 20], dtype="Int64"), + } + ) + household = pd.DataFrame( + { + "household_id": np.array([1], dtype=np.int64), + "reference_person_id": pd.array([10], dtype="Int64"), + } + ) + return {"person": person, "household": household} + + +#: Stands for the id int64 cannot hold in :data:`WIDE_CASES`. +WIDE = object() + +#: For each way an id int64 cannot hold reaches :func:`three_person_frame`, +#: the columns that then hold it, null-free so a NumPy dtype can hold them. +#: The household id appears on both tables, so both are named. +WIDE_CASES: dict[str, dict[tuple[str, str], list[object]]] = { + "person_id": {("person", "person_id"): [10, 20, WIDE]}, + "household_id": { + ("person", "person_household_id"): [WIDE, WIDE, WIDE], + ("household", "household_id"): [WIDE], + }, + "partner_person_id": {("person", "partner_person_id"): [20, 10, WIDE]}, + "parent_1_person_id": {("person", "parent_1_person_id"): [WIDE, WIDE, 10]}, + "parent_2_person_id": {("person", "parent_2_person_id"): [WIDE, WIDE, 20]}, + "reference_person_id": {("household", "reference_person_id"): [WIDE]}, +} + + +def wide_id_frame( + case: str, value: int, dtype +) -> tuple[dict[str, pd.DataFrame], list[str]]: + """:func:`three_person_frame` with ``value`` placed as ``WIDE_CASES[case]`` + says, in columns of ``dtype``; and the columns a refusal must name.""" + + columns = WIDE_CASES[case] + tables = _frame( + three_person_frame(), + { + key: typed([value if item is WIDE else item for item in items], dtype) + for key, items in columns.items() + }, + ) + return tables, [ + f"{entity}.{column}" + for entity, column in ID_COLUMNS + if (entity, column) in columns + ] + + +__all__ = [name for name in globals() if not name.startswith("__")] From 52cdec479181ed818bb8ceb6be8695d7de4eb89a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 8 Oct 2026 22:32:04 -0400 Subject: [PATCH 2/6] Refuse wide ids before the structure checks; make the id property exact Review of #1158 (in-session counterexample, mutation and claims lenses): - The committed validate property was flaky. test_support's typed() used pd.array(values, dtype="UInt64"), which in pandas 3.0.3 routes some lists above 2**63 through float64 ([10, 2**63, 2**63 + 1] becomes [10, 2**63, 2**63]), so a drawn frame could hold duplicate ids and fail under about 1 seed in 13. typed() now builds nullable columns from an exact NumPy array and a mask, and asserts the round trip. - validate_concept_tables' household membership check compared int64 with uint64 household ids through pandas' float64 isin, so a person naming household 2**62 + 256 passed as a member of household 2**62. The refusal of wide ids moves into _require_structure, ahead of the uniqueness and membership checks, which now run on int64; a wide id is refused by name even in an otherwise malformed frame. - The id strategy now draws ids beyond 2**53, plants shifts of one, and plants household pointers in both properties; both properties compare raised errors too. New regressions cover the float64 merge and the refusal ordering; all fail on main. - Docstrings: encode's Raises section, _matched_ids' summary, the strategy's plant options. Adds a changelog fragment. Co-Authored-By: Claude Opus 5.5 --- .../concept-id-int64-matching.fixed.md | 1 + .../src/microcosm/frame/concept_mapping.py | 7 +- .../src/microcosm/frame/concepts.py | 34 ++++++--- .../shared/test_concept_mapping.py | 39 ++++++---- .../tests/engine_free/shared/test_concepts.py | 76 +++++++++++++++---- .../microcosm_frame/concept_id_dtypes.py | 70 +++++++++++------ 6 files changed, 160 insertions(+), 67 deletions(-) create mode 100644 changelog.d/concept-id-int64-matching.fixed.md diff --git a/changelog.d/concept-id-int64-matching.fixed.md b/changelog.d/concept-id-int64-matching.fixed.md new file mode 100644 index 000000000..03d873eda --- /dev/null +++ b/changelog.d/concept-id-int64-matching.fixed.md @@ -0,0 +1 @@ +Match concept-frame ids exactly in `validate_concept_tables` and `ConceptMapping.encode`. Both now refuse, with a ValueError naming the columns, any unsigned id or pointer above `2**63 - 1`, which they used to wrap (`2**64 - 1` read as -1, so a dangling pointer could name person -1). They match every other integer id as int64, where pandas had cast pointers down to a narrower unsigned index (261 onto 5 for uint8) and compared int64 with uint64 household ids through float64. Valid frames holding uint64 ids above `2**63 - 1`, which validation used to accept, are now refused. diff --git a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py index d0b530875..6673d37c1 100644 --- a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py +++ b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py @@ -1296,8 +1296,9 @@ def encode( Raises: ValueError: If a needed share or rate is missing or outside - [0, 1], or a relationship role or allocation lacks the - pointers it reads. + [0, 1], a relationship role or allocation lacks the + pointers it reads, or an unsigned id or pointer exceeds + ``2**63 - 1`` (ids are matched as int64). """ shares = dict(shares or {}) @@ -1414,7 +1415,7 @@ def reference_rows(self) -> np.ndarray: def _matched_ids(values: pd.Series) -> np.ndarray: - """Ids ready for exact matching: integers as int64, anything else as is. + """Ids ready for exact matching: null-free integers as int64, else as is. pandas matches against a narrower unsigned index by casting the targets down to it (261 onto 5 for uint8), so null-free integer ids are widened diff --git a/packages/microcosm-frame/src/microcosm/frame/concepts.py b/packages/microcosm-frame/src/microcosm/frame/concepts.py index 575c5bbb6..469e6ccdd 100644 --- a/packages/microcosm-frame/src/microcosm/frame/concepts.py +++ b/packages/microcosm-frame/src/microcosm/frame/concepts.py @@ -1397,9 +1397,10 @@ def validate_concept_tables( Every violation found, empty for a valid frame. Raises: - ValueError: If a table or an id column is missing, ids are not unique, - a person points at an unknown household, or an unsigned id or - pointer exceeds ``2**63 - 1``: pointers are matched as int64. + ValueError: If a table or an id column is missing, an id column + holds anything but non-null integers, an unsigned id or pointer + exceeds ``2**63 - 1`` (ids are matched as int64), ids are not + unique, or a person points at an unknown household. """ person, household = _require_structure(tables) @@ -1523,9 +1524,20 @@ def _require_structure( f"Concept frame id column {column!r} must hold non-null " f"integers, found dtype {ids.dtype}." ) - if not person[_PERSON_ID].is_unique or not household[_HOUSEHOLD_ID].is_unique: + # Ids are matched as int64 from here on, here and in _check_pointers. An + # unsigned id above 2**63 - 1 would wrap (2**64 - 1 onto -1), so it is + # refused before anything converts; every other id converts exactly. + # Matching the native dtypes is not exact either: pandas compares int64 + # with uint64 through float64, which merges ids above 2**53. + _require_int64_ids(person, household, "Validation matches") + person_ids = pd.Index(person[_PERSON_ID].to_numpy(dtype=np.int64)) + household_ids = pd.Index(household[_HOUSEHOLD_ID].to_numpy(dtype=np.int64)) + if not person_ids.is_unique or not household_ids.is_unique: raise ValueError("Concept frame ids must be unique.") - unknown = ~person[_PERSON_HOUSEHOLD_ID].isin(household[_HOUSEHOLD_ID]) + unknown = ( + household_ids.get_indexer(person[_PERSON_HOUSEHOLD_ID].to_numpy(dtype=np.int64)) + < 0 + ) if unknown.any(): raise ValueError(f"{int(unknown.sum())} person(s) name an unknown household.") return person, household @@ -1600,12 +1612,10 @@ def add(code: str, message: str, rows: int = 0) -> None: def _check_pointers( person: pd.DataFrame, household: pd.DataFrame ) -> list[ConceptViolation]: - # Ids and pointers are matched as int64, where an unsigned id above - # 2**63 - 1 would wrap (2**64 - 1 onto -1), so such ids are refused - # first. Every other id converts exactly. The person ids are widened - # too: pandas matches against a narrower unsigned index by casting the - # pointers down to it (261 onto 5 for uint8). - _require_int64_ids(person, household, "Pointer checks match") + # Ids and pointers are matched as int64; _require_structure has refused + # any id int64 cannot hold, so every conversion here is exact. The + # person ids are widened too: pandas matches against a narrower unsigned + # index by casting the pointers down to it (261 onto 5 for uint8). out: list[ConceptViolation] = [] ids = pd.Index(person[_PERSON_ID].to_numpy(dtype=np.int64)) own_household = person[_PERSON_HOUSEHOLD_ID].to_numpy(dtype=np.int64) @@ -1766,7 +1776,7 @@ def _require_int64_ids( Only an unsigned id above ``2**63 - 1`` is refused; every other integer id converts to int64 exactly. ``what`` starts the message, as in - "Encoding matches". + "Encoding matches". Columns of other dtypes are not inspected. """ tables = {"person": person, "household": household} diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py b/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py index 27b6394cb..28d841af8 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py @@ -64,6 +64,7 @@ shares, ) from test_support.microcosm_frame.concept_id_dtypes import ( + UNSIGNED_64, WIDE_CASES, id_typed_frames, typed, @@ -537,10 +538,6 @@ def test_group_bindings_are_deferred_not_executed(self) -> None: } -#: The two dtypes that hold an id int64 cannot: NumPy's and pandas' nullable. -UNSIGNED_64 = [np.dtype(np.uint64), pd.UInt64Dtype()] - - def _pointer_mapping() -> ConceptMapping: """One engine input per transform that resolves a pointer.""" @@ -573,6 +570,15 @@ def _assert_same_encoding(actual, expected) -> None: pd.testing.assert_frame_equal(actual[entity], table, check_dtype=False) +def _encoding(mapping: ConceptMapping, tables, parameters) -> object: + """The encoded tables, or the ValueError encoding raises, as plain data.""" + + try: + return mapping.encode(tables, **parameters).tables + except ValueError as error: + return ("ValueError", str(error)) + + def _as_int64(tables: dict[str, pd.DataFrame]) -> dict[str, pd.DataFrame]: """The same frame with every integer id and pointer typed int64.""" @@ -621,8 +627,8 @@ def test_a_dangling_pointer_cannot_wrap_onto_person_minus_one(self) -> None: { "person_id": np.array([-1, 11, 12, 13], dtype=np.int64), "person_household_id": [1, 1, 1, 1], - "partner_person_id": pd.array( - [None, 2**64 - 1, None, None], dtype="UInt64" + "partner_person_id": typed( + [None, 2**64 - 1, None, None], pd.UInt64Dtype() ), } ) @@ -673,19 +679,20 @@ def test_a_narrow_unsigned_id_cannot_catch_a_pointer_past_its_range( @pytest.mark.parametrize("dtype", UNSIGNED_64, ids=str) def test_ids_up_to_int64_max_are_kept_exactly(self, dtype) -> None: top = 2**63 - 1 + pointers = pd.UInt64Dtype() person = pd.DataFrame( { "person_id": typed([top, top - 1, top - 2], dtype), "person_household_id": typed([top, top, top], dtype), - "partner_person_id": pd.array([top - 1, top, None], dtype="UInt64"), - "parent_1_person_id": pd.array([None, None, top], dtype="UInt64"), - "parent_2_person_id": pd.array([None, None, top - 1], dtype="UInt64"), + "partner_person_id": typed([top - 1, top, None], pointers), + "parent_1_person_id": typed([None, None, top], pointers), + "parent_2_person_id": typed([None, None, top - 1], pointers), } ) household = pd.DataFrame( { "household_id": typed([top], dtype), - "reference_person_id": pd.array([top], dtype="UInt64"), + "reference_person_id": typed([top], pointers), "rent": [100.0], } ) @@ -700,7 +707,7 @@ def test_ids_up_to_int64_max_are_kept_exactly(self, dtype) -> None: @settings(max_examples=300, deadline=None) @given( name=st.sampled_from([*sorted(MAPPINGS), "pointer transforms"]), - case=id_typed_frames(dangling_households=True), + case=id_typed_frames(), data=st.data(), ) def test_every_frame_is_refused_or_encodes_as_its_int64_twin( @@ -714,10 +721,12 @@ def test_every_frame_is_refused_or_encodes_as_its_int64_twin( ): mapping.encode(case.tables, **parameters) return - _assert_same_encoding( - mapping.encode(case.tables, **parameters).tables, - mapping.encode(case.twin, **parameters).tables, - ) + encoded = _encoding(mapping, case.tables, parameters) + expected = _encoding(mapping, case.twin, parameters) + if isinstance(expected, tuple) or isinstance(encoded, tuple): + assert encoded == expected + else: + _assert_same_encoding(encoded, expected) class TestDecodeGuards: diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_concepts.py b/packages/microcosm-frame/tests/engine_free/shared/test_concepts.py index e21de076a..723a1ed50 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_concepts.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_concepts.py @@ -42,6 +42,7 @@ ) from test_support.microcosm_frame.concept_frames import concept_frames from test_support.microcosm_frame.concept_id_dtypes import ( + UNSIGNED_64, WIDE_CASES, id_typed_frames, typed, @@ -761,16 +762,22 @@ def test_cycle_count_matches_a_reachability_reference(self, n, data) -> None: assert rows == _cycle_reference(parents, n) -#: The two dtypes that hold an id int64 cannot: NumPy's and pandas' nullable. -UNSIGNED_64 = [np.dtype(np.uint64), pd.UInt64Dtype()] +def _validation(tables) -> object: + """What validation returns, or the ValueError it raises, as plain data.""" + + try: + return validate_concept_tables(tables) + except ValueError as error: + return ("ValueError", str(error)) class TestIdDtypes: """Ids and pointers of any accepted integer dtype are matched exactly. - Pointers are matched as int64. An unsigned id above ``2**63 - 1`` would - wrap (``2**64 - 1`` onto -1) and so is refused, by name; every other id - must validate exactly as the same values typed int64. + Ids are matched as int64. An unsigned id above ``2**63 - 1`` would wrap + (``2**64 - 1`` onto -1) and so is refused, by name, before anything + converts; every other id must validate exactly as the same values typed + int64. """ @pytest.mark.parametrize("dtype", UNSIGNED_64, ids=str) @@ -785,6 +792,16 @@ def test_an_id_int64_cannot_hold_is_refused_by_column( ): validate_concept_tables(tables) + def test_a_wide_id_is_refused_by_name_before_the_structure_checks(self) -> None: + # Uniqueness and household membership compare ids too, so the + # refusal comes first: here two persons share the id 2**63 and one + # names no household, and the refusal still names the column. + tables, wide = wide_id_frame("person_id", 2**63, pd.UInt64Dtype()) + tables["person"]["person_id"] = typed([10, 2**63, 2**63], pd.UInt64Dtype()) + tables["person"]["person_household_id"] = np.array([1, 1, 2], dtype=np.int64) + with pytest.raises(ValueError, match=re.escape(f"{wide} hold unsigned ids")): + validate_concept_tables(tables) + def test_a_dangling_pointer_cannot_wrap_onto_person_minus_one(self) -> None: # The microcosm#1122 review probe: read as int64, 2**64 - 1 is -1, so # person 11's dangling partner pointer named person -1, who names 11 @@ -793,8 +810,8 @@ def test_a_dangling_pointer_cannot_wrap_onto_person_minus_one(self) -> None: { "person_id": np.array([-1, 11, 12, 13], dtype=np.int64), "person_household_id": [1, 1, 1, 1], - "partner_person_id": pd.array( - [11, 2**64 - 1, None, None], dtype="UInt64" + "partner_person_id": typed( + [11, 2**64 - 1, None, None], pd.UInt64Dtype() ), } ) @@ -834,28 +851,59 @@ def test_a_narrow_unsigned_id_cannot_catch_a_pointer_past_its_range( ("reference_person_id", "not_member"), } + @pytest.mark.parametrize( + ("persons", "households"), + [ + ("int64", "uint64"), + ("uint64", "int64"), + ("Int64", "UInt64"), + ("UInt64", "Int64"), + ], + ) + def test_a_household_float64_would_merge_with_another_is_still_unknown( + self, persons, households + ) -> None: + # pandas compared int64 with uint64 household ids through float64, + # where 2**62 + 256 rounds to 2**62, so a person naming no household + # passed as a member of household 2**62. + base = 2**62 + person = pd.DataFrame( + { + "person_id": np.array([10, 20], dtype=np.int64), + "person_household_id": typed( + [base, base + 256], pd.api.types.pandas_dtype(persons) + ), + } + ) + household = pd.DataFrame( + {"household_id": typed([base], pd.api.types.pandas_dtype(households))} + ) + with pytest.raises(ValueError, match=re.escape("1 person(s) name an unknown")): + validate_concept_tables({"person": person, "household": household}) + @pytest.mark.parametrize("dtype", UNSIGNED_64, ids=str) def test_ids_up_to_int64_max_are_kept_exactly(self, dtype) -> None: top = 2**63 - 1 + pointers = pd.UInt64Dtype() person = pd.DataFrame( { "person_id": typed([top, top - 1, top - 2], dtype), "person_household_id": typed([top, top, top], dtype), - "partner_person_id": pd.array([top - 1, top, None], dtype="UInt64"), - "parent_1_person_id": pd.array([None, None, top], dtype="UInt64"), - "parent_2_person_id": pd.array([None, None, top - 1], dtype="UInt64"), + "partner_person_id": typed([top - 1, top, None], pointers), + "parent_1_person_id": typed([None, None, top], pointers), + "parent_2_person_id": typed([None, None, top - 1], pointers), } ) household = pd.DataFrame( { "household_id": typed([top], dtype), - "reference_person_id": pd.array([top], dtype="UInt64"), + "reference_person_id": typed([top], pointers), } ) tables = {"person": person, "household": household} assert validate_concept_tables(tables) == () # Within float rounding of the others, top - 3 is still nobody. - person["partner_person_id"] = pd.array([top - 1, top, top - 3], dtype="UInt64") + person["partner_person_id"] = typed([top - 1, top, top - 3], pointers) assert _violations(tables) == {("partner_person_id", "dangling")} @settings(max_examples=300, deadline=None) @@ -867,9 +915,7 @@ def test_every_frame_is_refused_or_validates_as_its_int64_twin(self, case) -> No ): validate_concept_tables(case.tables) else: - assert validate_concept_tables(case.tables) == validate_concept_tables( - case.twin - ) + assert _validation(case.tables) == _validation(case.twin) class TestTransportSplit: diff --git a/test_support/microcosm_frame/concept_id_dtypes.py b/test_support/microcosm_frame/concept_id_dtypes.py index 96a88122a..f384eee63 100644 --- a/test_support/microcosm_frame/concept_id_dtypes.py +++ b/test_support/microcosm_frame/concept_id_dtypes.py @@ -2,12 +2,14 @@ The concept-frame contract accepts ids and pointers of any integer dtype: NumPy or pandas nullable, signed or unsigned, any width. Matching ids across -those dtypes is where values wrap. A bare int64 cast turns ``2**64 - 1`` into --1, and pandas matches against a narrower unsigned index by casting the other -side down to it (261 onto 5 for uint8). :func:`id_typed_frames` relabels a -valid frame's ids onto values at those edges, may plant pointers on the -values a wrap would collide with, and gives each id column a dtype drawn from -those that hold its values. A consumer must then either refuse the frame or +those dtypes is where values wrap or merge. A bare int64 cast turns +``2**64 - 1`` into -1; pandas matches against a narrower unsigned index by +casting the other side down to it (261 onto 5 for uint8); and it compares +int64 with uint64 through float64, which merges ids above ``2**53``. +:func:`id_typed_frames` relabels a valid frame's ids onto values at those +edges, may plant pointers on the values a wrap or a float64 rounding would +collide with, and gives each id column a dtype drawn from those that hold its +values. A consumer must then either refuse the frame or treat it exactly as the same ids typed int64. """ @@ -46,6 +48,9 @@ ("household", "household_id"), } +#: The two dtypes that hold an id int64 cannot: NumPy's and pandas' nullable. +UNSIGNED_64 = (np.dtype(np.uint64), pd.UInt64Dtype()) + #: Every integer dtype the contract accepts for an id, NumPy then nullable. ID_DTYPES = ( *(np.dtype(f"{sign}int{bits}") for sign in ("", "u") for bits in (8, 16, 32, 64)), @@ -67,17 +72,27 @@ } ) -#: The shifts a wrap applies to a value: a multiple of a dtype's span. -_WRAPS = tuple(sign * 2**power for power in (8, 16, 32, 64) for sign in (1, -1)) +#: Shifts that make a different id collide with an existing one: a dtype's +#: span (a wrap), or one (float64 merges neighbours above 2**53). +_SHIFTS = tuple(sign * 2**power for power in (0, 8, 16, 32, 64) for sign in (1, -1)) def _ids(unsigned: bool) -> st.SearchStrategy[int]: - """Ids near zero, at the dtype edges, or anywhere int64 or uint64 holds.""" + """Ids near zero, at the dtype edges, past 2**53, or anywhere in range. + + Hypothesis draws small integers far more often than large ones, so ids + beyond 2**53, where float64 no longer tells neighbours apart, get their + own branch. + """ low, high = (0, UINT64_MAX) if unsigned else (INT64_MIN, INT64_MAX) + beyond = st.integers(2**53, high) + if not unsigned: + beyond |= st.integers(low, -(2**53)) return st.one_of( st.integers(max(low, -300), 300), st.sampled_from([value for value in _EDGES if low <= value <= high]), + beyond, st.integers(low, high), ) @@ -100,8 +115,21 @@ def typed( """``values`` as a column of ``dtype``.""" if isinstance(dtype, np.dtype): - return np.array(values, dtype=dtype) - return pd.array(values, dtype=dtype) + column = np.array(values, dtype=dtype) + else: + # pd.array(values, dtype="UInt64") routes some lists of ints above + # 2**63 through float64 (pandas 3.0.3 turns [10, 2**63, 2**63 + 1] + # into [10, 2**63, 2**63]), so the column is built from exact NumPy + # values and a mask instead. + missing = np.array([value is None for value in values], dtype=bool) + filled = [0 if value is None else value for value in values] + column = pd.arrays.IntegerArray( + np.array(filled, dtype=dtype.numpy_dtype), missing + ) + held = [None if pd.isna(value) else int(value) for value in column] + if held != values: + raise AssertionError(f"{dtype} did not hold {values} exactly: {held}.") + return column class IdTypedFrame(NamedTuple): @@ -129,17 +157,16 @@ def _frame( @st.composite -def id_typed_frames(draw, *, dangling_households: bool = False) -> IdTypedFrame: +def id_typed_frames(draw) -> IdTypedFrame: """A concept frame whose ids sit at the dtype edges, in any accepted dtype. Starting from a valid frame, person and household ids are relabelled (each set drawn wholly from the int64 range or wholly from the uint64 range, so some dtype holds them). Each pointer column may then gain one - planted value: an id shifted by a dtype's span, the value a wrap would - land on an existing id, or any id. With ``dangling_households`` the - household pointer may be planted too, which leaves a person naming no - household. Each id column then takes a dtype drawn from the accepted ones - that hold its values. + planted value: an existing id shifted by one of :data:`_SHIFTS`, or any + id. The pointers planted include each person's household, which can leave + a person naming no household. Each id column then takes a dtype drawn + from the accepted ones that hold its values. """ base = draw(concept_frames(max_households=3, max_members=4)) @@ -186,17 +213,16 @@ def relabel(values: pd.Series, to: dict[int, int]) -> list[int | None]: } planted = [("person", column) for column in _PERSON_POINTERS] - planted.append(("household", "reference_person_id")) - if dangling_households: - planted.append(("person", "person_household_id")) + planted += [("household", "reference_person_id")] + planted += [("person", "person_household_id")] for key in planted: if not draw(st.booleans(), label=f"plant {key[1]}"): continue column = values[key] targets = new_households if key[1] == "person_household_id" else new_persons row = draw(st.integers(0, len(column) - 1), label="row") - if draw(st.booleans(), label="plant a wrap image"): - value = draw(st.sampled_from(targets)) + draw(st.sampled_from(_WRAPS)) + if draw(st.booleans(), label="plant a shifted id"): + value = draw(st.sampled_from(targets)) + draw(st.sampled_from(_SHIFTS)) else: value = draw(_ids(draw(st.booleans()))) if not INT64_MIN <= value <= UINT64_MAX: From a9d759f7445239b322ed74683a4fdd5d212f1c63 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 00:14:27 -0400 Subject: [PATCH 3/6] Reconcile with #1122: one int64 helper, exact unit households After merging main (with #1122): - test_unit_construction: TestIdRange's unsigned-household case asserted validate_concept_tables accepts a uint64 household id of 2**63, which this PR refuses; it now expects validation to refuse the same columns (the 4-line change agreed with the NZ hub). - _INT64_MAX / _exceeds_int64 were defined twice with the same bodies, once in concepts (this PR) and once in concept_mapping (#1122); concept_mapping and unit_construction now import them from concepts. - _Units' wide-id list dropped its person.* entries: _Units is only built from a _Context, which has already refused them, so they could not fire. - _Units matched unit households against the household ids in their native dtype, so under uint8 household ids a family nesting in household 261 was placed in household 5 (UInt8 raised a TypeError). It now matches through _matched_ids, as _Context does; a regression covers uint8, UInt8 and the int64 control. Co-Authored-By: Claude Opus 5.5 --- .../src/microcosm/frame/concept_mapping.py | 27 ++++--------------- .../src/microcosm/frame/unit_construction.py | 9 +++---- .../engine_free/shared/test_group_encode.py | 23 ++++++++++++++++ .../shared/test_unit_construction.py | 9 ++++--- 4 files changed, 36 insertions(+), 32 deletions(-) diff --git a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py index a47ea6bfb..0c7294cb6 100644 --- a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py +++ b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py @@ -39,6 +39,7 @@ import pandas as pd from microcosm.frame.concepts import ( + _INT64_MAX, CONCEPT_BY_ID, CONCEPT_ENTITIES, CONCEPTS, @@ -51,6 +52,7 @@ ConceptAlignment, TemporalBasis, Unit, + _exceeds_int64, _parsing, _record_fields, _require_int64_ids, @@ -2200,19 +2202,6 @@ def _state_column(column: str) -> tuple[str, str]: return entity, name -#: The largest id a unit table carries: units hold and compare ids as int64. -_INT64_MAX = int(np.iinfo(np.int64).max) - - -def _exceeds_int64(values: pd.Series) -> bool: - """Whether an unsigned integer column holds an id int64 cannot hold.""" - - if values.dtype.kind != "u": - return False - present = values.dropna() - return len(present) > 0 and int(present.max()) > _INT64_MAX - - class _Units: """A membership checked against a concept frame, as row positions.""" @@ -2250,19 +2239,13 @@ def __init__(self, membership: GroupMembership, context: _Context) -> None: ) # The ids are compared as int64 below. An unsigned id int64 cannot # hold would wrap onto another id (2**64 - 1 onto -1), so a - # membership naming no unit could pass; it is refused instead. + # membership naming no unit could pass; it is refused instead. The + # concept frame's own ids were refused already, by _Context. wide = [ name for name, values in ( ("person unit ids", membership.person_unit), *((column, units[column]) for column in columns), - (f"person.{_PERSON_ID}", person[_PERSON_ID]), - (f"person.{_PERSON_HOUSEHOLD_ID}", person[_PERSON_HOUSEHOLD_ID]), - *( - (f"person.{column}", person[column]) - for column in ("partner_person_id",) - if column in person.columns - ), ) if _exceeds_int64(values) ] @@ -2327,7 +2310,7 @@ def __init__(self, membership: GroupMembership, context: _Context) -> None: f"Every {membership.entity} must nest in its declared household." ) household_rows = pd.Index( - context.household[_HOUSEHOLD_ID].to_numpy() + _matched_ids(context.household[_HOUSEHOLD_ID]) ).get_indexer(unit_households) if (household_rows < 0).any(): raise ValueError(f"A {membership.entity} names an unknown household.") diff --git a/packages/microcosm-frame/src/microcosm/frame/unit_construction.py b/packages/microcosm-frame/src/microcosm/frame/unit_construction.py index 9cd4ecad4..917a3038e 100644 --- a/packages/microcosm-frame/src/microcosm/frame/unit_construction.py +++ b/packages/microcosm-frame/src/microcosm/frame/unit_construction.py @@ -75,16 +75,13 @@ import numpy as np import pandas as pd -from microcosm.frame.concept_mapping import ( - _INT64_MAX, - GroupMembership, - UnitRole, - _exceeds_int64, -) +from microcosm.frame.concept_mapping import GroupMembership, UnitRole from microcosm.frame.concepts import ( + _INT64_MAX, HOUSEHOLD_ID_COLUMN, PERSON_HOUSEHOLD_ID_COLUMN, PERSON_ID_COLUMN, + _exceeds_int64, _parsing, _record_fields, validate_concept_tables, diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py b/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py index e35669c31..26258e7be 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py @@ -900,6 +900,29 @@ def test_a_partner_pointer_int64_cannot_hold_is_refused(self) -> None: {"Family": membership}, ) + @pytest.mark.parametrize("dtype", ["uint8", "UInt8", "int64"]) + def test_a_narrow_unsigned_household_cannot_catch_a_unit_past_its_range( + self, dtype + ) -> None: + # The family nests in household 261, and only household 5 exists. + # pandas matched unit households against a narrower unsigned index by + # casting them down to it, so under uint8 household 261 was household + # 5 (and UInt8 raised a TypeError); as int64 it is unknown. + tables, _, family, _, group = _example() + person = tables["person"].assign(person_household_id=np.int64(261)) + household = tables["household"].assign(household_id=pd.array([5], dtype=dtype)) + membership = GroupMembership( + entity="family", + units=family.assign(family_household_id=np.int64(261)), + person_unit=group.person_unit, + person_role=group.person_role, + ) + with pytest.raises(ValueError, match="names an unknown household"): + MAPPING.encode_groups( + {**tables, "person": person, "household": household}, + {"Family": membership}, + ) + @PROPERTY @given(case=unit_frames(), dtype=st.sampled_from(["uint64", "UInt64", "Int64"])) def test_membership_ids_are_read_exactly_or_refused(self, case, dtype) -> None: diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py b/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py index 943a53b3c..5bf24fbb4 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py @@ -633,12 +633,13 @@ def test_the_largest_unsigned_id_int64_holds_is_kept_exactly(self) -> None: def test_an_unsigned_household_id_int64_cannot_hold_is_refused( self, household_id ) -> None: - # The concept-frame contract accepts this frame. Cast to int64, 2**63 - # became household -2**63: the unit nested in no household and its - # household counted no unit. + # Cast to int64, 2**63 became household -2**63: the unit nested in no + # household and its household counted no unit. Validation refuses the + # same columns (microcosm#1158). person, household = self._one_adult(household_id) - assert not validate_concept_tables({"person": person, "household": household}) columns = "['person.person_household_id', 'household.household_id']" + with pytest.raises(ValueError, match=re.escape(columns)): + validate_concept_tables({"person": person, "household": household}) with pytest.raises(ValueError, match=re.escape(columns)): build_benefit_units(person, household, self.RULE) From 3aa7386af84eb46cd534871cb113d443279822bc Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 02:22:12 -0400 Subject: [PATCH 4/6] Count unit households as int64; scope the strategy's dtype claim From the hard-tier review of a9d759f7 (APPROVE_WITH_NITS): - units_per_household reindexed household ids against the unit table's household column in its native dtype, so with a uint8 unit column household 261 counted household 5's unit (UInt8 raised a TypeError). Both sides now go through _int64_ids, as the rest of the module does; a regression covers uint8, UInt8 and the int64 control. - concept_id_dtypes: the strategy draws the NumPy and pandas nullable integer dtypes; the docstrings no longer claim every accepted dtype (Arrow-backed integers are accepted but not drawn). Co-Authored-By: Claude Opus 5.5 --- .../src/microcosm/frame/unit_construction.py | 11 ++++++---- .../shared/test_unit_construction.py | 15 ++++++++++++++ .../microcosm_frame/concept_id_dtypes.py | 20 ++++++++++--------- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/packages/microcosm-frame/src/microcosm/frame/unit_construction.py b/packages/microcosm-frame/src/microcosm/frame/unit_construction.py index 917a3038e..35646521d 100644 --- a/packages/microcosm-frame/src/microcosm/frame/unit_construction.py +++ b/packages/microcosm-frame/src/microcosm/frame/unit_construction.py @@ -521,11 +521,14 @@ def units_per_household( ) -> pd.Series: """How many units each household holds, aligned to ``household``'s rows.""" - counts = family[rule.household_column].value_counts() + # Both sides as int64: reindexing by ids pandas would cast down to a + # narrower unsigned unit column (household 261 onto 5 for uint8). + counts = pd.Series( + _int64_ids(family[rule.household_column], rule.household_column) + ).value_counts() + households = _int64_ids(household[_HOUSEHOLD_ID], f"household.{_HOUSEHOLD_ID}") return pd.Series( - counts.reindex(household[_HOUSEHOLD_ID].to_numpy(), fill_value=0) - .to_numpy() - .astype(np.int64), + counts.reindex(households, fill_value=0).to_numpy().astype(np.int64), index=household.index, name=f"n_{rule.entity}_units", ) diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py b/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py index 5bf24fbb4..770aad9e6 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py @@ -643,6 +643,21 @@ def test_an_unsigned_household_id_int64_cannot_hold_is_refused( with pytest.raises(ValueError, match=re.escape(columns)): build_benefit_units(person, household, self.RULE) + @pytest.mark.parametrize("dtype", ["uint8", "UInt8", "int64"]) + def test_a_narrow_unsigned_unit_household_counts_only_its_own(self, dtype) -> None: + # pandas reindexed household 261 against a uint8 unit column by + # casting it down to 5, so household 261 counted household 5's unit. + family = pd.DataFrame( + { + self.RULE.id_column: np.array([1], dtype=np.int64), + self.RULE.household_column: pd.array([5], dtype=dtype), + self.RULE.head_column: np.array([1], dtype=np.int64), + } + ) + household = pd.DataFrame({"household_id": np.array([5, 261], dtype=np.int64)}) + counts = units_per_household(household, family, self.RULE) + assert counts.tolist() == [1, 0] + def test_an_unsigned_pointer_int64_cannot_hold_is_refused(self) -> None: # 2**64 - 1 names no person here, but cast to int64 it is -1, who is # a person here: the pair would have passed as partners. diff --git a/test_support/microcosm_frame/concept_id_dtypes.py b/test_support/microcosm_frame/concept_id_dtypes.py index f384eee63..bd99cbd19 100644 --- a/test_support/microcosm_frame/concept_id_dtypes.py +++ b/test_support/microcosm_frame/concept_id_dtypes.py @@ -1,16 +1,17 @@ -"""Concept frames whose id and pointer columns take every accepted integer dtype. +"""Concept frames whose id and pointer columns take NumPy and nullable int dtypes. The concept-frame contract accepts ids and pointers of any integer dtype: -NumPy or pandas nullable, signed or unsigned, any width. Matching ids across -those dtypes is where values wrap or merge. A bare int64 cast turns -``2**64 - 1`` into -1; pandas matches against a narrower unsigned index by -casting the other side down to it (261 onto 5 for uint8); and it compares -int64 with uint64 through float64, which merges ids above ``2**53``. +NumPy, pandas nullable or Arrow-backed, signed or unsigned, any width; the +strategy here draws the NumPy and nullable ones. Matching ids across those +dtypes is where values wrap or merge. A bare int64 cast turns ``2**64 - 1`` +into -1; pandas matches against a narrower unsigned index by casting the +other side down to it (261 onto 5 for uint8); and it compares int64 with +uint64 through float64, which merges ids above ``2**53``. :func:`id_typed_frames` relabels a valid frame's ids onto values at those edges, may plant pointers on the values a wrap or a float64 rounding would collide with, and gives each id column a dtype drawn from those that hold its -values. A consumer must then either refuse the frame or -treat it exactly as the same ids typed int64. +values. A consumer must then either refuse the frame or treat it exactly as +the same ids typed int64. """ # ruff: noqa: F401 @@ -51,7 +52,8 @@ #: The two dtypes that hold an id int64 cannot: NumPy's and pandas' nullable. UNSIGNED_64 = (np.dtype(np.uint64), pd.UInt64Dtype()) -#: Every integer dtype the contract accepts for an id, NumPy then nullable. +#: The NumPy and pandas nullable integer dtypes, which the contract accepts +#: for an id. Arrow-backed integers are accepted too but are not drawn here. ID_DTYPES = ( *(np.dtype(f"{sign}int{bits}") for sign in ("", "u") for bits in (8, 16, 32, 64)), *( From b1b8d5439b0b989ca98fe7a60ba0838c1d73b202 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 02:34:50 -0400 Subject: [PATCH 5/6] Cover units_per_household's wide-id refusal; document its Raises From the delta review of 3aa7386a: a mutant casting both sides with a bare to_numpy(dtype=np.int64) survived every test, so a uint64 household id of 2**64 - 1 would have counted the unit in household -1. A test now requires the refusal. units_per_household's docstring gains a Raises section, and the strategy's UNSIGNED_64 comment notes Arrow's uint64. Co-Authored-By: Claude Opus 5.5 --- .../src/microcosm/frame/unit_construction.py | 8 +++++++- .../engine_free/shared/test_unit_construction.py | 15 +++++++++++++++ test_support/microcosm_frame/concept_id_dtypes.py | 3 ++- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-frame/src/microcosm/frame/unit_construction.py b/packages/microcosm-frame/src/microcosm/frame/unit_construction.py index 35646521d..33057abd1 100644 --- a/packages/microcosm-frame/src/microcosm/frame/unit_construction.py +++ b/packages/microcosm-frame/src/microcosm/frame/unit_construction.py @@ -519,7 +519,13 @@ def benefit_unit_attributes( def units_per_household( household: pd.DataFrame, family: pd.DataFrame, rule: BenefitUnitRule ) -> pd.Series: - """How many units each household holds, aligned to ``household``'s rows.""" + """How many units each household holds, aligned to ``household``'s rows. + + Raises: + ValueError: If the household ids or the units' household column hold + anything but non-null integer ids, or an unsigned id above + ``2**63 - 1``: both are matched as int64. + """ # Both sides as int64: reindexing by ids pandas would cast down to a # narrower unsigned unit column (household 261 onto 5 for uint8). diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py b/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py index 770aad9e6..8b4c811db 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py @@ -658,6 +658,21 @@ def test_a_narrow_unsigned_unit_household_counts_only_its_own(self, dtype) -> No counts = units_per_household(household, family, self.RULE) assert counts.tolist() == [1, 0] + def test_unit_households_int64_cannot_hold_are_refused(self) -> None: + # Read as int64, household 2**64 - 1 is -1, the unit's household. + family = pd.DataFrame( + { + self.RULE.id_column: np.array([1], dtype=np.int64), + self.RULE.household_column: np.array([-1], dtype=np.int64), + self.RULE.head_column: np.array([1], dtype=np.int64), + } + ) + household = pd.DataFrame( + {"household_id": np.array([2**64 - 1], dtype=np.uint64)} + ) + with pytest.raises(ValueError, match="household_id holds ids above"): + units_per_household(household, family, self.RULE) + def test_an_unsigned_pointer_int64_cannot_hold_is_refused(self) -> None: # 2**64 - 1 names no person here, but cast to int64 it is -1, who is # a person here: the pair would have passed as partners. diff --git a/test_support/microcosm_frame/concept_id_dtypes.py b/test_support/microcosm_frame/concept_id_dtypes.py index bd99cbd19..edc2d91af 100644 --- a/test_support/microcosm_frame/concept_id_dtypes.py +++ b/test_support/microcosm_frame/concept_id_dtypes.py @@ -49,7 +49,8 @@ ("household", "household_id"), } -#: The two dtypes that hold an id int64 cannot: NumPy's and pandas' nullable. +#: The drawn dtypes that hold an id int64 cannot: NumPy's and pandas' nullable +#: uint64 (Arrow's uint64 is a third, not drawn). UNSIGNED_64 = (np.dtype(np.uint64), pd.UInt64Dtype()) #: The NumPy and pandas nullable integer dtypes, which the contract accepts From d8484852734939cd822cfb74fc2050829a384299 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 02:39:41 -0400 Subject: [PATCH 6/6] Test units_per_household's wide-id refusal on both sides From the delta re-check of b1b8d543: a mutant bare-casting only the unit table's household column survived. The refusal test is now parametrized over which side holds 2**64 - 1; each side's bare-cast mutant fails its own case. Co-Authored-By: Claude Opus 5.5 --- .../engine_free/shared/test_unit_construction.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py b/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py index 8b4c811db..72ba6f76a 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py @@ -658,19 +658,23 @@ def test_a_narrow_unsigned_unit_household_counts_only_its_own(self, dtype) -> No counts = units_per_household(household, family, self.RULE) assert counts.tolist() == [1, 0] - def test_unit_households_int64_cannot_hold_are_refused(self) -> None: - # Read as int64, household 2**64 - 1 is -1, the unit's household. + @pytest.mark.parametrize("wide_side", ["household", "unit"]) + def test_unit_households_int64_cannot_hold_are_refused(self, wide_side) -> None: + # Read as int64, household 2**64 - 1 is -1, the household on the other + # side, so a bare cast would count the unit there. + wide = np.array([2**64 - 1], dtype=np.uint64) + minus_one = np.array([-1], dtype=np.int64) family = pd.DataFrame( { self.RULE.id_column: np.array([1], dtype=np.int64), - self.RULE.household_column: np.array([-1], dtype=np.int64), + self.RULE.household_column: wide if wide_side == "unit" else minus_one, self.RULE.head_column: np.array([1], dtype=np.int64), } ) household = pd.DataFrame( - {"household_id": np.array([2**64 - 1], dtype=np.uint64)} + {"household_id": wide if wide_side == "household" else minus_one} ) - with pytest.raises(ValueError, match="household_id holds ids above"): + with pytest.raises(ValueError, match="holds ids above"): units_per_household(household, family, self.RULE) def test_an_unsigned_pointer_int64_cannot_hold_is_refused(self) -> None: