diff --git a/changelog.d/concept-id-int64-matching.fixed.md b/changelog.d/concept-id-int64-matching.fixed.md new file mode 100644 index 000000000..03d873eda --- /dev/null +++ b/changelog.d/concept-id-int64-matching.fixed.md @@ -0,0 +1 @@ +Match concept-frame ids exactly in `validate_concept_tables` and `ConceptMapping.encode`. Both now refuse, with a ValueError naming the columns, any unsigned id or pointer above `2**63 - 1`, which they used to wrap (`2**64 - 1` read as -1, so a dangling pointer could name person -1). They match every other integer id as int64, where pandas had cast pointers down to a narrower unsigned index (261 onto 5 for uint8) and compared int64 with uint64 household ids through float64. Valid frames holding uint64 ids above `2**63 - 1`, which validation used to accept, are now refused. diff --git a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py index 86908ead2..0c7294cb6 100644 --- a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py +++ b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py @@ -39,6 +39,7 @@ import pandas as pd from microcosm.frame.concepts import ( + _INT64_MAX, CONCEPT_BY_ID, CONCEPT_ENTITIES, CONCEPTS, @@ -51,8 +52,10 @@ ConceptAlignment, TemporalBasis, Unit, + _exceeds_int64, _parsing, _record_fields, + _require_int64_ids, _text_fields, concept, concept_schema_sha256, @@ -1722,8 +1725,9 @@ def encode( Raises: ValueError: If a needed share or rate is missing or outside - [0, 1], or a relationship role or allocation lacks the - pointers it reads. + [0, 1], a relationship role or allocation lacks the + pointers it reads, or an unsigned id or pointer exceeds + ``2**63 - 1`` (ids are matched as int64). """ shares = dict(shares or {}) @@ -1992,16 +1996,19 @@ class _Context: """A concept frame, with pointer lookups done by exact integer ids. Pointers are matched through row positions, never through float - arrays, so person ids beyond 2**53 resolve exactly. + arrays, so person ids beyond 2**53 resolve exactly. Ids are matched as + int64, so an unsigned id above 2**63 - 1, which would wrap (2**64 - 1 + onto -1), is refused. """ def __init__(self, person: pd.DataFrame, household: pd.DataFrame) -> None: + _require_int64_ids(person, household, "Encoding matches") self.person = person self.household = household - self._person_ids = pd.Index(person[_PERSON_ID].to_numpy()) + self._person_ids = pd.Index(_matched_ids(person[_PERSON_ID])) self._household_rows = pd.Index( - household[_HOUSEHOLD_ID].to_numpy() - ).get_indexer(person[_PERSON_HOUSEHOLD_ID].to_numpy()) + _matched_ids(household[_HOUSEHOLD_ID]) + ).get_indexer(_matched_ids(person[_PERSON_HOUSEHOLD_ID])) def has(self, concept_id: str) -> bool: item = CONCEPT_BY_ID[concept_id] @@ -2036,6 +2043,20 @@ def reference_rows(self) -> np.ndarray: return by_household[self._household_rows] +def _matched_ids(values: pd.Series) -> np.ndarray: + """Ids ready for exact matching: null-free integers as int64, else as is. + + pandas matches against a narrower unsigned index by casting the targets + down to it (261 onto 5 for uint8), so null-free integer ids are widened + first; :func:`_require_int64_ids` has already refused any int64 cannot + hold. + """ + + if values.dtype.kind in "iu" and not values.hasnans: + return values.to_numpy(dtype=np.int64) + return values.to_numpy() + + def _apply( binding: InputBinding, context: _Context, @@ -2181,19 +2202,6 @@ def _state_column(column: str) -> tuple[str, str]: return entity, name -#: The largest id a unit table carries: units hold and compare ids as int64. -_INT64_MAX = int(np.iinfo(np.int64).max) - - -def _exceeds_int64(values: pd.Series) -> bool: - """Whether an unsigned integer column holds an id int64 cannot hold.""" - - if values.dtype.kind != "u": - return False - present = values.dropna() - return len(present) > 0 and int(present.max()) > _INT64_MAX - - class _Units: """A membership checked against a concept frame, as row positions.""" @@ -2231,19 +2239,13 @@ def __init__(self, membership: GroupMembership, context: _Context) -> None: ) # The ids are compared as int64 below. An unsigned id int64 cannot # hold would wrap onto another id (2**64 - 1 onto -1), so a - # membership naming no unit could pass; it is refused instead. + # membership naming no unit could pass; it is refused instead. The + # concept frame's own ids were refused already, by _Context. wide = [ name for name, values in ( ("person unit ids", membership.person_unit), *((column, units[column]) for column in columns), - (f"person.{_PERSON_ID}", person[_PERSON_ID]), - (f"person.{_PERSON_HOUSEHOLD_ID}", person[_PERSON_HOUSEHOLD_ID]), - *( - (f"person.{column}", person[column]) - for column in ("partner_person_id",) - if column in person.columns - ), ) if _exceeds_int64(values) ] @@ -2308,7 +2310,7 @@ def __init__(self, membership: GroupMembership, context: _Context) -> None: f"Every {membership.entity} must nest in its declared household." ) household_rows = pd.Index( - context.household[_HOUSEHOLD_ID].to_numpy() + _matched_ids(context.household[_HOUSEHOLD_ID]) ).get_indexer(unit_households) if (household_rows < 0).any(): raise ValueError(f"A {membership.entity} names an unknown household.") diff --git a/packages/microcosm-frame/src/microcosm/frame/concepts.py b/packages/microcosm-frame/src/microcosm/frame/concepts.py index 2d37b95dd..34ae07f0e 100644 --- a/packages/microcosm-frame/src/microcosm/frame/concepts.py +++ b/packages/microcosm-frame/src/microcosm/frame/concepts.py @@ -1398,8 +1398,10 @@ def validate_concept_tables( Every violation found, empty for a valid frame. Raises: - ValueError: If a table or an id column is missing, ids are not unique, - or a person points at an unknown household. + ValueError: If a table or an id column is missing, an id column + holds anything but non-null integers, an unsigned id or pointer + exceeds ``2**63 - 1`` (ids are matched as int64), ids are not + unique, or a person points at an unknown household. """ person, household = _require_structure(tables) @@ -1523,9 +1525,20 @@ def _require_structure( f"Concept frame id column {column!r} must hold non-null " f"integers, found dtype {ids.dtype}." ) - if not person[_PERSON_ID].is_unique or not household[_HOUSEHOLD_ID].is_unique: + # Ids are matched as int64 from here on, here and in _check_pointers. An + # unsigned id above 2**63 - 1 would wrap (2**64 - 1 onto -1), so it is + # refused before anything converts; every other id converts exactly. + # Matching the native dtypes is not exact either: pandas compares int64 + # with uint64 through float64, which merges ids above 2**53. + _require_int64_ids(person, household, "Validation matches") + person_ids = pd.Index(person[_PERSON_ID].to_numpy(dtype=np.int64)) + household_ids = pd.Index(household[_HOUSEHOLD_ID].to_numpy(dtype=np.int64)) + if not person_ids.is_unique or not household_ids.is_unique: raise ValueError("Concept frame ids must be unique.") - unknown = ~person[_PERSON_HOUSEHOLD_ID].isin(household[_HOUSEHOLD_ID]) + unknown = ( + household_ids.get_indexer(person[_PERSON_HOUSEHOLD_ID].to_numpy(dtype=np.int64)) + < 0 + ) if unknown.any(): raise ValueError(f"{int(unknown.sum())} person(s) name an unknown household.") return person, household @@ -1600,9 +1613,13 @@ def add(code: str, message: str, rows: int = 0) -> None: def _check_pointers( person: pd.DataFrame, household: pd.DataFrame ) -> list[ConceptViolation]: + # Ids and pointers are matched as int64; _require_structure has refused + # any id int64 cannot hold, so every conversion here is exact. The + # person ids are widened too: pandas matches against a narrower unsigned + # index by casting the pointers down to it (261 onto 5 for uint8). out: list[ConceptViolation] = [] - ids = pd.Index(person[_PERSON_ID].to_numpy()) - own_household = person[_PERSON_HOUSEHOLD_ID].to_numpy() + ids = pd.Index(person[_PERSON_ID].to_numpy(dtype=np.int64)) + own_household = person[_PERSON_HOUSEHOLD_ID].to_numpy(dtype=np.int64) rows = np.arange(len(person)) def add( @@ -1712,7 +1729,7 @@ def positions(values: pd.Series) -> tuple[np.ndarray, np.ndarray]: household["reference_person_id"] ): found, present = positions(household["reference_person_id"]) - own = household[_HOUSEHOLD_ID].to_numpy() + own = household[_HOUSEHOLD_ID].to_numpy(dtype=np.int64) member = (found >= 0) & (own_household[np.maximum(found, 0)] == own) add( "household", @@ -1729,6 +1746,53 @@ def _is_integer_column(values: pd.Series) -> bool: return values.dtype.kind in "iu" +#: The largest id a pointer can be matched by: ids are matched as int64. +_INT64_MAX = int(np.iinfo(np.int64).max) + +#: Every column holding a person or household id, as (entity, column). +_ID_COLUMNS = ( + ("person", _PERSON_ID), + ("person", _PERSON_HOUSEHOLD_ID), + ("person", "partner_person_id"), + ("person", "parent_1_person_id"), + ("person", "parent_2_person_id"), + ("household", _HOUSEHOLD_ID), + ("household", "reference_person_id"), +) + + +def _exceeds_int64(values: pd.Series) -> bool: + """Whether an unsigned integer column holds an id int64 cannot hold.""" + + if values.dtype.kind != "u": + return False + present = values.dropna() + return len(present) > 0 and int(present.max()) > _INT64_MAX + + +def _require_int64_ids( + person: pd.DataFrame, household: pd.DataFrame, what: str +) -> None: + """Refuse a frame whose ids int64 cannot hold, naming every such column. + + Only an unsigned id above ``2**63 - 1`` is refused; every other integer + id converts to int64 exactly. ``what`` starts the message, as in + "Encoding matches". Columns of other dtypes are not inspected. + """ + + tables = {"person": person, "household": household} + wide = [ + f"{entity}.{column}" + for entity, column in _ID_COLUMNS + if column in tables[entity].columns and _exceeds_int64(tables[entity][column]) + ] + if wide: + raise ValueError( + f"{what} ids as int64, and {wide} hold unsigned ids above " + f"{_INT64_MAX}, which int64 cannot represent." + ) + + def _parent_cycle_members(n_persons: int, parent_rows: list[np.ndarray]) -> int: """Count persons on a cycle of parent pointers, or on a path between two. diff --git a/packages/microcosm-frame/src/microcosm/frame/unit_construction.py b/packages/microcosm-frame/src/microcosm/frame/unit_construction.py index 9cd4ecad4..33057abd1 100644 --- a/packages/microcosm-frame/src/microcosm/frame/unit_construction.py +++ b/packages/microcosm-frame/src/microcosm/frame/unit_construction.py @@ -75,16 +75,13 @@ import numpy as np import pandas as pd -from microcosm.frame.concept_mapping import ( - _INT64_MAX, - GroupMembership, - UnitRole, - _exceeds_int64, -) +from microcosm.frame.concept_mapping import GroupMembership, UnitRole from microcosm.frame.concepts import ( + _INT64_MAX, HOUSEHOLD_ID_COLUMN, PERSON_HOUSEHOLD_ID_COLUMN, PERSON_ID_COLUMN, + _exceeds_int64, _parsing, _record_fields, validate_concept_tables, @@ -522,13 +519,22 @@ def benefit_unit_attributes( def units_per_household( household: pd.DataFrame, family: pd.DataFrame, rule: BenefitUnitRule ) -> pd.Series: - """How many units each household holds, aligned to ``household``'s rows.""" + """How many units each household holds, aligned to ``household``'s rows. + + Raises: + ValueError: If the household ids or the units' household column hold + anything but non-null integer ids, or an unsigned id above + ``2**63 - 1``: both are matched as int64. + """ - counts = family[rule.household_column].value_counts() + # Both sides as int64: reindexing by ids pandas would cast down to a + # narrower unsigned unit column (household 261 onto 5 for uint8). + counts = pd.Series( + _int64_ids(family[rule.household_column], rule.household_column) + ).value_counts() + households = _int64_ids(household[_HOUSEHOLD_ID], f"household.{_HOUSEHOLD_ID}") return pd.Series( - counts.reindex(household[_HOUSEHOLD_ID].to_numpy(), fill_value=0) - .to_numpy() - .astype(np.int64), + counts.reindex(households, fill_value=0).to_numpy().astype(np.int64), index=household.index, name=f"n_{rule.entity}_units", ) diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py b/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py index ccce264d6..2e91d0837 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py @@ -64,6 +64,13 @@ concept_frames, shares, ) +from test_support.microcosm_frame.concept_id_dtypes import ( + UNSIGNED_64, + WIDE_CASES, + id_typed_frames, + typed, + wide_id_frame, +) from test_support.microcosm_frame.concept_mappings import ( COVERAGE_DOCS, concept_mappings, @@ -532,6 +539,197 @@ def test_group_bindings_are_deferred_not_executed(self) -> None: } +def _pointer_mapping() -> ConceptMapping: + """One engine input per transform that resolves a pointer.""" + + bindings = [ + _binding( + engine_input=role.value, + concepts=RelationshipRole(role=role).concepts, + transform=RelationshipRole(role=role), + ) + for role in Role + ] + bindings.append( + _binding( + engine_input="children", concepts=_PARENTS, transform=CoresidentChildCount() + ) + ) + bindings.append( + _binding( + engine_input="rent_paid", + concepts=("fact:household.rent",), + transform=AllocateToReferencePerson(), + ) + ) + return _mapping(bindings) + + +def _assert_same_encoding(actual, expected) -> None: + assert actual.keys() == expected.keys() + for entity, table in expected.items(): + pd.testing.assert_frame_equal(actual[entity], table, check_dtype=False) + + +def _encoding(mapping: ConceptMapping, tables, parameters) -> object: + """The encoded tables, or the ValueError encoding raises, as plain data.""" + + try: + return mapping.encode(tables, **parameters).tables + except ValueError as error: + return ("ValueError", str(error)) + + +def _as_int64(tables: dict[str, pd.DataFrame]) -> dict[str, pd.DataFrame]: + """The same frame with every integer id and pointer typed int64.""" + + out = {entity: table.copy() for entity, table in tables.items()} + for entity, column in ( + ("person", "person_id"), + ("person", "person_household_id"), + ("household", "household_id"), + ): + out[entity][column] = out[entity][column].astype(np.int64) + for entity, column in ( + ("person", "partner_person_id"), + ("person", "parent_1_person_id"), + ("person", "parent_2_person_id"), + ("household", "reference_person_id"), + ): + if column in out[entity].columns: + out[entity][column] = out[entity][column].astype("Int64") + return out + + +class TestIdDtypes: + """Encoding matches ids of any accepted integer dtype exactly. + + Pointers are matched as int64. An unsigned id above ``2**63 - 1`` would + wrap (``2**64 - 1`` onto -1) and so is refused, by name; every other id + must encode exactly as the same values typed int64. + """ + + @pytest.mark.parametrize("dtype", UNSIGNED_64, ids=str) + @pytest.mark.parametrize("value", [2**63, 2**64 - 1]) + @pytest.mark.parametrize("case", sorted(WIDE_CASES)) + def test_an_id_int64_cannot_hold_is_refused_by_column( + self, case, value, dtype + ) -> None: + tables, wide = wide_id_frame(case, value, dtype) + with pytest.raises( + ValueError, match=re.escape(f"{wide} hold unsigned ids above {2**63 - 1}") + ): + _pointer_mapping().encode(tables) + + def test_a_dangling_pointer_cannot_wrap_onto_person_minus_one(self) -> None: + # Read as int64, 2**64 - 1 is -1: reference person 11's dangling + # partner pointer made person -1 the reference person's partner. + person = pd.DataFrame( + { + "person_id": np.array([-1, 11, 12, 13], dtype=np.int64), + "person_household_id": [1, 1, 1, 1], + "partner_person_id": typed( + [None, 2**64 - 1, None, None], pd.UInt64Dtype() + ), + } + ) + household = pd.DataFrame( + {"household_id": [1], "reference_person_id": pd.array([11], dtype="Int64")} + ) + with pytest.raises(ValueError, match=re.escape("['person.partner_person_id']")): + _pointer_mapping().encode({"person": person, "household": household}) + + @pytest.mark.parametrize( + "dtype", ["uint8", "uint16", "uint32", "UInt8", "UInt16", "UInt32"] + ) + def test_a_narrow_unsigned_id_cannot_catch_a_pointer_past_its_range( + self, dtype + ) -> None: + # pandas matched against a narrower unsigned index by casting the + # other side down to it, so 5 + 2**8 named person 5 under uint8, and + # household 1 + 2**8 was household 1; the nullable dtypes raised a + # TypeError instead. + dtype = pd.api.types.pandas_dtype(dtype) + span = 2 ** (8 * dtype.itemsize) + person = pd.DataFrame( + { + "person_id": typed([5, 6, 7, 8], dtype), + "person_household_id": np.array([1, 1, 1, 1 + span], dtype=np.int64), + "partner_person_id": pd.array([6, 5, None, 5], dtype="Int64"), + "parent_1_person_id": pd.array( + [None, None, 5 + span, None], dtype="Int64" + ), + "parent_2_person_id": pd.array([None] * 4, dtype="Int64"), + } + ) + household = pd.DataFrame( + { + "household_id": typed([1, 2], dtype), + "reference_person_id": pd.array([5 + span, 8], dtype="Int64"), + "rent": [100.0, 200.0], + } + ) + tables = {"person": person, "household": household} + mapping = _pointer_mapping() + encoded = mapping.encode(tables).tables + _assert_same_encoding(encoded, mapping.encode(_as_int64(tables)).tables) + # Person 7's parent and the reference person of household 1 are nobody. + assert encoded["person"]["children"].tolist() == [0, 0, 0, 0] + assert encoded["person"]["reference_person_partner"].tolist() == [False] * 4 + + @pytest.mark.parametrize("dtype", UNSIGNED_64, ids=str) + def test_ids_up_to_int64_max_are_kept_exactly(self, dtype) -> None: + top = 2**63 - 1 + pointers = pd.UInt64Dtype() + person = pd.DataFrame( + { + "person_id": typed([top, top - 1, top - 2], dtype), + "person_household_id": typed([top, top, top], dtype), + "partner_person_id": typed([top - 1, top, None], pointers), + "parent_1_person_id": typed([None, None, top], pointers), + "parent_2_person_id": typed([None, None, top - 1], pointers), + } + ) + household = pd.DataFrame( + { + "household_id": typed([top], dtype), + "reference_person_id": typed([top], pointers), + "rent": [100.0], + } + ) + out = _pointer_mapping().encode({"person": person, "household": household}) + flags = out.tables["person"] + assert flags["reference_person"].tolist() == [True, False, False] + assert flags["reference_person_partner"].tolist() == [False, True, False] + assert flags["children"].tolist() == [1, 1, 0] + assert flags["rent_paid"].tolist() == [100.0, 0.0, 0.0] + assert out.tables["person"]["person_id"].tolist() == [top, top - 1, top - 2] + + @settings(max_examples=300, deadline=None) + @given( + name=st.sampled_from([*sorted(MAPPINGS), "pointer transforms"]), + case=id_typed_frames(), + data=st.data(), + ) + def test_every_frame_is_refused_or_encodes_as_its_int64_twin( + self, name, case, data + ) -> None: + mapping = _pointer_mapping() if name == "pointer transforms" else MAPPINGS[name] + parameters = _parameters(mapping, data) + if case.wide: + with pytest.raises( + ValueError, match=re.escape(f"{case.wide} hold unsigned ids above") + ): + mapping.encode(case.tables, **parameters) + return + encoded = _encoding(mapping, case.tables, parameters) + expected = _encoding(mapping, case.twin, parameters) + if isinstance(expected, tuple) or isinstance(encoded, tuple): + assert encoded == expected + else: + _assert_same_encoding(encoded, expected) + + class TestDecodeGuards: def test_a_whole_number_concept_refuses_fractions(self) -> None: mapping = _mapping([_binding()]) diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_concepts.py b/packages/microcosm-frame/tests/engine_free/shared/test_concepts.py index 1f97a493b..723a1ed50 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_concepts.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_concepts.py @@ -6,6 +6,7 @@ """ import json +import re from dataclasses import replace import numpy as np @@ -40,6 +41,13 @@ validate_concept_tables, ) from test_support.microcosm_frame.concept_frames import concept_frames +from test_support.microcosm_frame.concept_id_dtypes import ( + UNSIGNED_64, + WIDE_CASES, + id_typed_frames, + typed, + wide_id_frame, +) from test_support.paths import paths_for PROPERTY = settings(max_examples=150, deadline=None) @@ -754,6 +762,162 @@ def test_cycle_count_matches_a_reachability_reference(self, n, data) -> None: assert rows == _cycle_reference(parents, n) +def _validation(tables) -> object: + """What validation returns, or the ValueError it raises, as plain data.""" + + try: + return validate_concept_tables(tables) + except ValueError as error: + return ("ValueError", str(error)) + + +class TestIdDtypes: + """Ids and pointers of any accepted integer dtype are matched exactly. + + Ids are matched as int64. An unsigned id above ``2**63 - 1`` would wrap + (``2**64 - 1`` onto -1) and so is refused, by name, before anything + converts; every other id must validate exactly as the same values typed + int64. + """ + + @pytest.mark.parametrize("dtype", UNSIGNED_64, ids=str) + @pytest.mark.parametrize("value", [2**63, 2**64 - 1]) + @pytest.mark.parametrize("case", sorted(WIDE_CASES)) + def test_an_id_int64_cannot_hold_is_refused_by_column( + self, case, value, dtype + ) -> None: + tables, wide = wide_id_frame(case, value, dtype) + with pytest.raises( + ValueError, match=re.escape(f"{wide} hold unsigned ids above {2**63 - 1}") + ): + validate_concept_tables(tables) + + def test_a_wide_id_is_refused_by_name_before_the_structure_checks(self) -> None: + # Uniqueness and household membership compare ids too, so the + # refusal comes first: here two persons share the id 2**63 and one + # names no household, and the refusal still names the column. + tables, wide = wide_id_frame("person_id", 2**63, pd.UInt64Dtype()) + tables["person"]["person_id"] = typed([10, 2**63, 2**63], pd.UInt64Dtype()) + tables["person"]["person_household_id"] = np.array([1, 1, 2], dtype=np.int64) + with pytest.raises(ValueError, match=re.escape(f"{wide} hold unsigned ids")): + validate_concept_tables(tables) + + def test_a_dangling_pointer_cannot_wrap_onto_person_minus_one(self) -> None: + # The microcosm#1122 review probe: read as int64, 2**64 - 1 is -1, so + # person 11's dangling partner pointer named person -1, who names 11 + # back, and the frame validated clean. + person = pd.DataFrame( + { + "person_id": np.array([-1, 11, 12, 13], dtype=np.int64), + "person_household_id": [1, 1, 1, 1], + "partner_person_id": typed( + [11, 2**64 - 1, None, None], pd.UInt64Dtype() + ), + } + ) + household = pd.DataFrame( + {"household_id": [1], "reference_person_id": pd.array([-1], dtype="Int64")} + ) + with pytest.raises(ValueError, match=re.escape("['person.partner_person_id']")): + validate_concept_tables({"person": person, "household": household}) + + @pytest.mark.parametrize( + "dtype", ["uint8", "uint16", "uint32", "UInt8", "UInt16", "UInt32"] + ) + def test_a_narrow_unsigned_id_cannot_catch_a_pointer_past_its_range( + self, dtype + ) -> None: + # pandas matched pointers against a narrower unsigned index by casting + # them down to it, so 5 + 2**8 named person 5 under uint8; the + # nullable dtypes raised a TypeError instead. + dtype = pd.api.types.pandas_dtype(dtype) + past = 5 + 2 ** (8 * dtype.itemsize) + person = pd.DataFrame( + { + "person_id": typed([5, 6], dtype), + "person_household_id": [1, 1], + "partner_person_id": pd.array([6, past], dtype="Int64"), + } + ) + household = pd.DataFrame( + { + "household_id": [1], + "reference_person_id": pd.array([past], dtype="Int64"), + } + ) + assert _violations({"person": person, "household": household}) == { + ("partner_person_id", "dangling"), + ("partner_person_id", "asymmetric"), + ("reference_person_id", "not_member"), + } + + @pytest.mark.parametrize( + ("persons", "households"), + [ + ("int64", "uint64"), + ("uint64", "int64"), + ("Int64", "UInt64"), + ("UInt64", "Int64"), + ], + ) + def test_a_household_float64_would_merge_with_another_is_still_unknown( + self, persons, households + ) -> None: + # pandas compared int64 with uint64 household ids through float64, + # where 2**62 + 256 rounds to 2**62, so a person naming no household + # passed as a member of household 2**62. + base = 2**62 + person = pd.DataFrame( + { + "person_id": np.array([10, 20], dtype=np.int64), + "person_household_id": typed( + [base, base + 256], pd.api.types.pandas_dtype(persons) + ), + } + ) + household = pd.DataFrame( + {"household_id": typed([base], pd.api.types.pandas_dtype(households))} + ) + with pytest.raises(ValueError, match=re.escape("1 person(s) name an unknown")): + validate_concept_tables({"person": person, "household": household}) + + @pytest.mark.parametrize("dtype", UNSIGNED_64, ids=str) + def test_ids_up_to_int64_max_are_kept_exactly(self, dtype) -> None: + top = 2**63 - 1 + pointers = pd.UInt64Dtype() + person = pd.DataFrame( + { + "person_id": typed([top, top - 1, top - 2], dtype), + "person_household_id": typed([top, top, top], dtype), + "partner_person_id": typed([top - 1, top, None], pointers), + "parent_1_person_id": typed([None, None, top], pointers), + "parent_2_person_id": typed([None, None, top - 1], pointers), + } + ) + household = pd.DataFrame( + { + "household_id": typed([top], dtype), + "reference_person_id": typed([top], pointers), + } + ) + tables = {"person": person, "household": household} + assert validate_concept_tables(tables) == () + # Within float rounding of the others, top - 3 is still nobody. + person["partner_person_id"] = typed([top - 1, top, top - 3], pointers) + assert _violations(tables) == {("partner_person_id", "dangling")} + + @settings(max_examples=300, deadline=None) + @given(case=id_typed_frames()) + def test_every_frame_is_refused_or_validates_as_its_int64_twin(self, case) -> None: + if case.wide: + with pytest.raises( + ValueError, match=re.escape(f"{case.wide} hold unsigned ids above") + ): + validate_concept_tables(case.tables) + else: + assert _validation(case.tables) == _validation(case.twin) + + class TestTransportSplit: @PROPERTY @given(tables=concept_frames()) diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py b/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py index e35669c31..26258e7be 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py @@ -900,6 +900,29 @@ def test_a_partner_pointer_int64_cannot_hold_is_refused(self) -> None: {"Family": membership}, ) + @pytest.mark.parametrize("dtype", ["uint8", "UInt8", "int64"]) + def test_a_narrow_unsigned_household_cannot_catch_a_unit_past_its_range( + self, dtype + ) -> None: + # The family nests in household 261, and only household 5 exists. + # pandas matched unit households against a narrower unsigned index by + # casting them down to it, so under uint8 household 261 was household + # 5 (and UInt8 raised a TypeError); as int64 it is unknown. + tables, _, family, _, group = _example() + person = tables["person"].assign(person_household_id=np.int64(261)) + household = tables["household"].assign(household_id=pd.array([5], dtype=dtype)) + membership = GroupMembership( + entity="family", + units=family.assign(family_household_id=np.int64(261)), + person_unit=group.person_unit, + person_role=group.person_role, + ) + with pytest.raises(ValueError, match="names an unknown household"): + MAPPING.encode_groups( + {**tables, "person": person, "household": household}, + {"Family": membership}, + ) + @PROPERTY @given(case=unit_frames(), dtype=st.sampled_from(["uint64", "UInt64", "Int64"])) def test_membership_ids_are_read_exactly_or_refused(self, case, dtype) -> None: diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py b/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py index 943a53b3c..72ba6f76a 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py @@ -633,15 +633,50 @@ def test_the_largest_unsigned_id_int64_holds_is_kept_exactly(self) -> None: def test_an_unsigned_household_id_int64_cannot_hold_is_refused( self, household_id ) -> None: - # The concept-frame contract accepts this frame. Cast to int64, 2**63 - # became household -2**63: the unit nested in no household and its - # household counted no unit. + # Cast to int64, 2**63 became household -2**63: the unit nested in no + # household and its household counted no unit. Validation refuses the + # same columns (microcosm#1158). person, household = self._one_adult(household_id) - assert not validate_concept_tables({"person": person, "household": household}) columns = "['person.person_household_id', 'household.household_id']" + with pytest.raises(ValueError, match=re.escape(columns)): + validate_concept_tables({"person": person, "household": household}) with pytest.raises(ValueError, match=re.escape(columns)): build_benefit_units(person, household, self.RULE) + @pytest.mark.parametrize("dtype", ["uint8", "UInt8", "int64"]) + def test_a_narrow_unsigned_unit_household_counts_only_its_own(self, dtype) -> None: + # pandas reindexed household 261 against a uint8 unit column by + # casting it down to 5, so household 261 counted household 5's unit. + family = pd.DataFrame( + { + self.RULE.id_column: np.array([1], dtype=np.int64), + self.RULE.household_column: pd.array([5], dtype=dtype), + self.RULE.head_column: np.array([1], dtype=np.int64), + } + ) + household = pd.DataFrame({"household_id": np.array([5, 261], dtype=np.int64)}) + counts = units_per_household(household, family, self.RULE) + assert counts.tolist() == [1, 0] + + @pytest.mark.parametrize("wide_side", ["household", "unit"]) + def test_unit_households_int64_cannot_hold_are_refused(self, wide_side) -> None: + # Read as int64, household 2**64 - 1 is -1, the household on the other + # side, so a bare cast would count the unit there. + wide = np.array([2**64 - 1], dtype=np.uint64) + minus_one = np.array([-1], dtype=np.int64) + family = pd.DataFrame( + { + self.RULE.id_column: np.array([1], dtype=np.int64), + self.RULE.household_column: wide if wide_side == "unit" else minus_one, + self.RULE.head_column: np.array([1], dtype=np.int64), + } + ) + household = pd.DataFrame( + {"household_id": wide if wide_side == "household" else minus_one} + ) + with pytest.raises(ValueError, match="holds ids above"): + units_per_household(household, family, self.RULE) + def test_an_unsigned_pointer_int64_cannot_hold_is_refused(self) -> None: # 2**64 - 1 names no person here, but cast to int64 it is -1, who is # a person here: the pair would have passed as partners. diff --git a/test_support/microcosm_frame/concept_id_dtypes.py b/test_support/microcosm_frame/concept_id_dtypes.py new file mode 100644 index 000000000..edc2d91af --- /dev/null +++ b/test_support/microcosm_frame/concept_id_dtypes.py @@ -0,0 +1,335 @@ +"""Concept frames whose id and pointer columns take NumPy and nullable int dtypes. + +The concept-frame contract accepts ids and pointers of any integer dtype: +NumPy, pandas nullable or Arrow-backed, signed or unsigned, any width; the +strategy here draws the NumPy and nullable ones. Matching ids across those +dtypes is where values wrap or merge. A bare int64 cast turns ``2**64 - 1`` +into -1; pandas matches against a narrower unsigned index by casting the +other side down to it (261 onto 5 for uint8); and it compares int64 with +uint64 through float64, which merges ids above ``2**53``. +:func:`id_typed_frames` relabels a valid frame's ids onto values at those +edges, may plant pointers on the values a wrap or a float64 rounding would +collide with, and gives each id column a dtype drawn from those that hold its +values. A consumer must then either refuse the frame or treat it exactly as +the same ids typed int64. +""" + +# ruff: noqa: F401 + +from __future__ import annotations + +from typing import NamedTuple + +import numpy as np +import pandas as pd +from hypothesis import strategies as st + +from test_support.microcosm_frame.concept_frames import concept_frames + +INT64_MIN = -(2**63) +INT64_MAX = 2**63 - 1 +UINT64_MAX = 2**64 - 1 + +#: Every column holding a person or household id, as (entity, column), in the +#: order a refusal names them. +ID_COLUMNS: tuple[tuple[str, str], ...] = ( + ("person", "person_id"), + ("person", "person_household_id"), + ("person", "partner_person_id"), + ("person", "parent_1_person_id"), + ("person", "parent_2_person_id"), + ("household", "household_id"), + ("household", "reference_person_id"), +) +_PERSON_POINTERS = ("partner_person_id", "parent_1_person_id", "parent_2_person_id") +#: The id columns a frame requires, never null; the rest are nullable pointers. +_REQUIRED = { + ("person", "person_id"), + ("person", "person_household_id"), + ("household", "household_id"), +} + +#: The drawn dtypes that hold an id int64 cannot: NumPy's and pandas' nullable +#: uint64 (Arrow's uint64 is a third, not drawn). +UNSIGNED_64 = (np.dtype(np.uint64), pd.UInt64Dtype()) + +#: The NumPy and pandas nullable integer dtypes, which the contract accepts +#: for an id. Arrow-backed integers are accepted too but are not drawn here. +ID_DTYPES = ( + *(np.dtype(f"{sign}int{bits}") for sign in ("", "u") for bits in (8, 16, 32, 64)), + *( + pd.api.types.pandas_dtype(f"{sign}Int{bits}") + for sign in ("", "U") + for bits in (8, 16, 32, 64) + ), +) + +#: Values within one of a power of two where some integer dtype wraps. +_EDGES = sorted( + { + sign * 2**power + offset + for power in (7, 8, 15, 16, 31, 32, 63, 64) + for sign in (1, -1) + for offset in (-1, 0, 1) + if INT64_MIN <= sign * 2**power + offset <= UINT64_MAX + } +) + +#: Shifts that make a different id collide with an existing one: a dtype's +#: span (a wrap), or one (float64 merges neighbours above 2**53). +_SHIFTS = tuple(sign * 2**power for power in (0, 8, 16, 32, 64) for sign in (1, -1)) + + +def _ids(unsigned: bool) -> st.SearchStrategy[int]: + """Ids near zero, at the dtype edges, past 2**53, or anywhere in range. + + Hypothesis draws small integers far more often than large ones, so ids + beyond 2**53, where float64 no longer tells neighbours apart, get their + own branch. + """ + + low, high = (0, UINT64_MAX) if unsigned else (INT64_MIN, INT64_MAX) + beyond = st.integers(2**53, high) + if not unsigned: + beyond |= st.integers(low, -(2**53)) + return st.one_of( + st.integers(max(low, -300), 300), + st.sampled_from([value for value in _EDGES if low <= value <= high]), + beyond, + st.integers(low, high), + ) + + +def holds(dtype, values: list[int | None]) -> bool: + """Whether ``dtype`` can hold every one of ``values`` (``None`` is null).""" + + if isinstance(dtype, np.dtype): + if any(value is None for value in values): + return False + info = np.iinfo(dtype) + else: + info = np.iinfo(dtype.numpy_dtype) + return all(info.min <= value <= info.max for value in values if value is not None) + + +def typed( + values: list[int | None], dtype +) -> np.ndarray | pd.api.extensions.ExtensionArray: + """``values`` as a column of ``dtype``.""" + + if isinstance(dtype, np.dtype): + column = np.array(values, dtype=dtype) + else: + # pd.array(values, dtype="UInt64") routes some lists of ints above + # 2**63 through float64 (pandas 3.0.3 turns [10, 2**63, 2**63 + 1] + # into [10, 2**63, 2**63]), so the column is built from exact NumPy + # values and a mask instead. + missing = np.array([value is None for value in values], dtype=bool) + filled = [0 if value is None else value for value in values] + column = pd.arrays.IntegerArray( + np.array(filled, dtype=dtype.numpy_dtype), missing + ) + held = [None if pd.isna(value) else int(value) for value in column] + if held != values: + raise AssertionError(f"{dtype} did not hold {values} exactly: {held}.") + return column + + +class IdTypedFrame(NamedTuple): + """A frame with retyped ids, its int64 twin, and the columns int64 cannot hold. + + ``twin`` holds the same values with every required id column typed + ``int64`` and every pointer typed ``Int64``, or is ``None`` when ``wide`` + is not empty. ``wide`` names, as ``"entity.column"`` in + :data:`ID_COLUMNS` order, every column holding a value above + ``2**63 - 1``. + """ + + tables: dict[str, pd.DataFrame] + twin: dict[str, pd.DataFrame] | None + wide: list[str] + + +def _frame( + base: dict[str, pd.DataFrame], columns: dict[tuple[str, str], object] +) -> dict[str, pd.DataFrame]: + out = {entity: table.copy() for entity, table in base.items()} + for (entity, column), values in columns.items(): + out[entity][column] = values + return out + + +@st.composite +def id_typed_frames(draw) -> IdTypedFrame: + """A concept frame whose ids sit at the dtype edges, in any accepted dtype. + + Starting from a valid frame, person and household ids are relabelled + (each set drawn wholly from the int64 range or wholly from the uint64 + range, so some dtype holds them). Each pointer column may then gain one + planted value: an existing id shifted by one of :data:`_SHIFTS`, or any + id. The pointers planted include each person's household, which can leave + a person naming no household. Each id column then takes a dtype drawn + from the accepted ones that hold its values. + """ + + base = draw(concept_frames(max_households=3, max_members=4)) + person, household = base["person"], base["household"] + old_persons = person["person_id"].tolist() + old_households = household["household_id"].tolist() + new_persons = draw( + st.lists( + _ids(draw(st.booleans(), label="unsigned person ids")), + min_size=len(old_persons), + max_size=len(old_persons), + unique=True, + ), + label="person ids", + ) + new_households = draw( + st.lists( + _ids(draw(st.booleans(), label="unsigned household ids")), + min_size=len(old_households), + max_size=len(old_households), + unique=True, + ), + label="household ids", + ) + to_person = dict(zip(old_persons, new_persons, strict=True)) + to_household = dict(zip(old_households, new_households, strict=True)) + + def relabel(values: pd.Series, to: dict[int, int]) -> list[int | None]: + return [None if pd.isna(value) else to[int(value)] for value in values] + + values: dict[tuple[str, str], list[int | None]] = { + ("person", "person_id"): list(new_persons), + ("person", "person_household_id"): relabel( + person["person_household_id"], to_household + ), + **{ + ("person", column): relabel(person[column], to_person) + for column in _PERSON_POINTERS + }, + ("household", "household_id"): list(new_households), + ("household", "reference_person_id"): relabel( + household["reference_person_id"], to_person + ), + } + + planted = [("person", column) for column in _PERSON_POINTERS] + planted += [("household", "reference_person_id")] + planted += [("person", "person_household_id")] + for key in planted: + if not draw(st.booleans(), label=f"plant {key[1]}"): + continue + column = values[key] + targets = new_households if key[1] == "person_household_id" else new_persons + row = draw(st.integers(0, len(column) - 1), label="row") + if draw(st.booleans(), label="plant a shifted id"): + value = draw(st.sampled_from(targets)) + draw(st.sampled_from(_SHIFTS)) + else: + value = draw(_ids(draw(st.booleans()))) + if not INT64_MIN <= value <= UINT64_MAX: + continue + before, column[row] = column[row], value + if not any(holds(dtype, column) for dtype in ID_DTYPES): + column[row] = before + + columns = { + key: typed( + values[key], + draw( + st.sampled_from( + [dtype for dtype in ID_DTYPES if holds(dtype, values[key])] + ), + label=f"{key[1]} dtype", + ), + ) + for key in ID_COLUMNS + } + wide = [ + f"{entity}.{column}" + for entity, column in ID_COLUMNS + if any( + value is not None and value > INT64_MAX + for value in values[(entity, column)] + ) + ] + twin = ( + None + if wide + else _frame( + base, + { + key: typed( + values[key], + np.dtype(np.int64) if key in _REQUIRED else pd.Int64Dtype(), + ) + for key in ID_COLUMNS + }, + ) + ) + return IdTypedFrame(_frame(base, columns), twin, wide) + + +def three_person_frame() -> dict[str, pd.DataFrame]: + """Persons 10 and 20, partners, are the parents of 30; 10 heads household 1.""" + + person = pd.DataFrame( + { + "person_id": np.array([10, 20, 30], dtype=np.int64), + "person_household_id": np.array([1, 1, 1], dtype=np.int64), + "partner_person_id": pd.array([20, 10, None], dtype="Int64"), + "parent_1_person_id": pd.array([None, None, 10], dtype="Int64"), + "parent_2_person_id": pd.array([None, None, 20], dtype="Int64"), + } + ) + household = pd.DataFrame( + { + "household_id": np.array([1], dtype=np.int64), + "reference_person_id": pd.array([10], dtype="Int64"), + } + ) + return {"person": person, "household": household} + + +#: Stands for the id int64 cannot hold in :data:`WIDE_CASES`. +WIDE = object() + +#: For each way an id int64 cannot hold reaches :func:`three_person_frame`, +#: the columns that then hold it, null-free so a NumPy dtype can hold them. +#: The household id appears on both tables, so both are named. +WIDE_CASES: dict[str, dict[tuple[str, str], list[object]]] = { + "person_id": {("person", "person_id"): [10, 20, WIDE]}, + "household_id": { + ("person", "person_household_id"): [WIDE, WIDE, WIDE], + ("household", "household_id"): [WIDE], + }, + "partner_person_id": {("person", "partner_person_id"): [20, 10, WIDE]}, + "parent_1_person_id": {("person", "parent_1_person_id"): [WIDE, WIDE, 10]}, + "parent_2_person_id": {("person", "parent_2_person_id"): [WIDE, WIDE, 20]}, + "reference_person_id": {("household", "reference_person_id"): [WIDE]}, +} + + +def wide_id_frame( + case: str, value: int, dtype +) -> tuple[dict[str, pd.DataFrame], list[str]]: + """:func:`three_person_frame` with ``value`` placed as ``WIDE_CASES[case]`` + says, in columns of ``dtype``; and the columns a refusal must name.""" + + columns = WIDE_CASES[case] + tables = _frame( + three_person_frame(), + { + key: typed([value if item is WIDE else item for item in items], dtype) + for key, items in columns.items() + }, + ) + return tables, [ + f"{entity}.{column}" + for entity, column in ID_COLUMNS + if (entity, column) in columns + ] + + +__all__ = [name for name in globals() if not name.startswith("__")]