Refuse unsigned ids int64 cannot hold; match concept ids as int64

changelog.d/concept-id-int64-matching.fixed.md

@@ -0,0 +1 @@
+Match concept-frame ids exactly in `validate_concept_tables` and `ConceptMapping.encode`. Both now refuse, with a ValueError naming the columns, any unsigned id or pointer above `2**63 - 1`, which they used to wrap (`2**64 - 1` read as -1, so a dangling pointer could name person -1). They match every other integer id as int64, where pandas had cast pointers down to a narrower unsigned index (261 onto 5 for uint8) and compared int64 with uint64 household ids through float64. Valid frames holding uint64 ids above `2**63 - 1`, which validation used to accept, are now refused.

packages/microcosm-frame/src/microcosm/frame/concept_mapping.py

@@ -39,6 +39,7 @@
 import pandas as pd
 
 from microcosm.frame.concepts import (
+    _INT64_MAX,
     CONCEPT_BY_ID,
     CONCEPT_ENTITIES,
     CONCEPTS,
@@ -51,8 +52,10 @@
     ConceptAlignment,
     TemporalBasis,
     Unit,
+    _exceeds_int64,
     _parsing,
     _record_fields,
+    _require_int64_ids,
     _text_fields,
     concept,
     concept_schema_sha256,
@@ -1722,8 +1725,9 @@ def encode(
 
         Raises:
             ValueError: If a needed share or rate is missing or outside
-                [0, 1], or a relationship role or allocation lacks the
-                pointers it reads.
+                [0, 1], a relationship role or allocation lacks the
+                pointers it reads, or an unsigned id or pointer exceeds
+                ``2**63 - 1`` (ids are matched as int64).
         """
 
         shares = dict(shares or {})
@@ -1992,16 +1996,19 @@ class _Context:
     """A concept frame, with pointer lookups done by exact integer ids.
 
     Pointers are matched through row positions, never through float
-    arrays, so person ids beyond 2**53 resolve exactly.
+    arrays, so person ids beyond 2**53 resolve exactly. Ids are matched as
+    int64, so an unsigned id above 2**63 - 1, which would wrap (2**64 - 1
+    onto -1), is refused.
     """
 
     def __init__(self, person: pd.DataFrame, household: pd.DataFrame) -> None:
+        _require_int64_ids(person, household, "Encoding matches")
         self.person = person
         self.household = household
-        self._person_ids = pd.Index(person[_PERSON_ID].to_numpy())
+        self._person_ids = pd.Index(_matched_ids(person[_PERSON_ID]))
         self._household_rows = pd.Index(
-            household[_HOUSEHOLD_ID].to_numpy()
-        ).get_indexer(person[_PERSON_HOUSEHOLD_ID].to_numpy())
+            _matched_ids(household[_HOUSEHOLD_ID])
+        ).get_indexer(_matched_ids(person[_PERSON_HOUSEHOLD_ID]))
 
     def has(self, concept_id: str) -> bool:
         item = CONCEPT_BY_ID[concept_id]
@@ -2036,6 +2043,20 @@ def reference_rows(self) -> np.ndarray:
         return by_household[self._household_rows]
 
 
+def _matched_ids(values: pd.Series) -> np.ndarray:
+    """Ids ready for exact matching: null-free integers as int64, else as is.
+
+    pandas matches against a narrower unsigned index by casting the targets
+    down to it (261 onto 5 for uint8), so null-free integer ids are widened
+    first; :func:`_require_int64_ids` has already refused any int64 cannot
+    hold.
+    """
+
+    if values.dtype.kind in "iu" and not values.hasnans:
+        return values.to_numpy(dtype=np.int64)
+    return values.to_numpy()
+
+
 def _apply(
     binding: InputBinding,
     context: _Context,
@@ -2181,19 +2202,6 @@ def _state_column(column: str) -> tuple[str, str]:
     return entity, name
 
 
-#: The largest id a unit table carries: units hold and compare ids as int64.
-_INT64_MAX = int(np.iinfo(np.int64).max)
-
-
-def _exceeds_int64(values: pd.Series) -> bool:
-    """Whether an unsigned integer column holds an id int64 cannot hold."""
-
-    if values.dtype.kind != "u":
-        return False
-    present = values.dropna()
-    return len(present) > 0 and int(present.max()) > _INT64_MAX
-
-
 class _Units:
     """A membership checked against a concept frame, as row positions."""
 
@@ -2231,19 +2239,13 @@ def __init__(self, membership: GroupMembership, context: _Context) -> None:
                 )
         # The ids are compared as int64 below. An unsigned id int64 cannot
         # hold would wrap onto another id (2**64 - 1 onto -1), so a
-        # membership naming no unit could pass; it is refused instead.
+        # membership naming no unit could pass; it is refused instead. The
+        # concept frame's own ids were refused already, by _Context.
         wide = [
             name
             for name, values in (
                 ("person unit ids", membership.person_unit),
                 *((column, units[column]) for column in columns),
-                (f"person.{_PERSON_ID}", person[_PERSON_ID]),
-                (f"person.{_PERSON_HOUSEHOLD_ID}", person[_PERSON_HOUSEHOLD_ID]),
-                *(
-                    (f"person.{column}", person[column])
-                    for column in ("partner_person_id",)
-                    if column in person.columns
-                ),
             )
             if _exceeds_int64(values)
         ]
@@ -2308,7 +2310,7 @@ def __init__(self, membership: GroupMembership, context: _Context) -> None:
                 f"Every {membership.entity} must nest in its declared household."
             )
         household_rows = pd.Index(
-            context.household[_HOUSEHOLD_ID].to_numpy()
+            _matched_ids(context.household[_HOUSEHOLD_ID])
         ).get_indexer(unit_households)
         if (household_rows < 0).any():
             raise ValueError(f"A {membership.entity} names an unknown household.")

packages/microcosm-frame/src/microcosm/frame/concepts.py

@@ -1398,8 +1398,10 @@ def validate_concept_tables(
         Every violation found, empty for a valid frame.
 
     Raises:
-        ValueError: If a table or an id column is missing, ids are not unique,
-            or a person points at an unknown household.
+        ValueError: If a table or an id column is missing, an id column
+            holds anything but non-null integers, an unsigned id or pointer
+            exceeds ``2**63 - 1`` (ids are matched as int64), ids are not
+            unique, or a person points at an unknown household.
     """
 
     person, household = _require_structure(tables)
@@ -1523,9 +1525,20 @@ def _require_structure(
                     f"Concept frame id column {column!r} must hold non-null "
                     f"integers, found dtype {ids.dtype}."
                 )
-    if not person[_PERSON_ID].is_unique or not household[_HOUSEHOLD_ID].is_unique:
+    # Ids are matched as int64 from here on, here and in _check_pointers. An
+    # unsigned id above 2**63 - 1 would wrap (2**64 - 1 onto -1), so it is
+    # refused before anything converts; every other id converts exactly.
+    # Matching the native dtypes is not exact either: pandas compares int64
+    # with uint64 through float64, which merges ids above 2**53.
+    _require_int64_ids(person, household, "Validation matches")
+    person_ids = pd.Index(person[_PERSON_ID].to_numpy(dtype=np.int64))
+    household_ids = pd.Index(household[_HOUSEHOLD_ID].to_numpy(dtype=np.int64))
+    if not person_ids.is_unique or not household_ids.is_unique:
         raise ValueError("Concept frame ids must be unique.")
-    unknown = ~person[_PERSON_HOUSEHOLD_ID].isin(household[_HOUSEHOLD_ID])
+    unknown = (
+        household_ids.get_indexer(person[_PERSON_HOUSEHOLD_ID].to_numpy(dtype=np.int64))
+        < 0
+    )
     if unknown.any():
         raise ValueError(f"{int(unknown.sum())} person(s) name an unknown household.")
     return person, household
@@ -1600,9 +1613,13 @@ def add(code: str, message: str, rows: int = 0) -> None:
 def _check_pointers(
     person: pd.DataFrame, household: pd.DataFrame
 ) -> list[ConceptViolation]:
+    # Ids and pointers are matched as int64; _require_structure has refused
+    # any id int64 cannot hold, so every conversion here is exact. The
+    # person ids are widened too: pandas matches against a narrower unsigned
+    # index by casting the pointers down to it (261 onto 5 for uint8).
     out: list[ConceptViolation] = []
-    ids = pd.Index(person[_PERSON_ID].to_numpy())
-    own_household = person[_PERSON_HOUSEHOLD_ID].to_numpy()
+    ids = pd.Index(person[_PERSON_ID].to_numpy(dtype=np.int64))
+    own_household = person[_PERSON_HOUSEHOLD_ID].to_numpy(dtype=np.int64)
     rows = np.arange(len(person))
 
     def add(
@@ -1712,7 +1729,7 @@ def positions(values: pd.Series) -> tuple[np.ndarray, np.ndarray]:
         household["reference_person_id"]
     ):
         found, present = positions(household["reference_person_id"])
-        own = household[_HOUSEHOLD_ID].to_numpy()
+        own = household[_HOUSEHOLD_ID].to_numpy(dtype=np.int64)
         member = (found >= 0) & (own_household[np.maximum(found, 0)] == own)
         add(
             "household",
@@ -1729,6 +1746,53 @@ def _is_integer_column(values: pd.Series) -> bool:
     return values.dtype.kind in "iu"
 
 
+#: The largest id a pointer can be matched by: ids are matched as int64.
+_INT64_MAX = int(np.iinfo(np.int64).max)
+
+#: Every column holding a person or household id, as (entity, column).
+_ID_COLUMNS = (
+    ("person", _PERSON_ID),
+    ("person", _PERSON_HOUSEHOLD_ID),
+    ("person", "partner_person_id"),
+    ("person", "parent_1_person_id"),
+    ("person", "parent_2_person_id"),
+    ("household", _HOUSEHOLD_ID),
+    ("household", "reference_person_id"),
+)
+
+
+def _exceeds_int64(values: pd.Series) -> bool:
+    """Whether an unsigned integer column holds an id int64 cannot hold."""
+
+    if values.dtype.kind != "u":
+        return False
+    present = values.dropna()
+    return len(present) > 0 and int(present.max()) > _INT64_MAX
+
+
+def _require_int64_ids(
+    person: pd.DataFrame, household: pd.DataFrame, what: str
+) -> None:
+    """Refuse a frame whose ids int64 cannot hold, naming every such column.
+
+    Only an unsigned id above ``2**63 - 1`` is refused; every other integer
+    id converts to int64 exactly. ``what`` starts the message, as in
+    "Encoding matches". Columns of other dtypes are not inspected.
+    """
+
+    tables = {"person": person, "household": household}
+    wide = [
+        f"{entity}.{column}"
+        for entity, column in _ID_COLUMNS
+        if column in tables[entity].columns and _exceeds_int64(tables[entity][column])
+    ]
+    if wide:
+        raise ValueError(
+            f"{what} ids as int64, and {wide} hold unsigned ids above "
+            f"{_INT64_MAX}, which int64 cannot represent."
+        )
+
+
 def _parent_cycle_members(n_persons: int, parent_rows: list[np.ndarray]) -> int:
     """Count persons on a cycle of parent pointers, or on a path between two.
 

packages/microcosm-frame/src/microcosm/frame/unit_construction.py

@@ -75,16 +75,13 @@
 import numpy as np
 import pandas as pd
 
-from microcosm.frame.concept_mapping import (
-    _INT64_MAX,
-    GroupMembership,
-    UnitRole,
-    _exceeds_int64,
-)
+from microcosm.frame.concept_mapping import GroupMembership, UnitRole
 from microcosm.frame.concepts import (
+    _INT64_MAX,
     HOUSEHOLD_ID_COLUMN,
     PERSON_HOUSEHOLD_ID_COLUMN,
     PERSON_ID_COLUMN,
+    _exceeds_int64,
     _parsing,
     _record_fields,
     validate_concept_tables,
@@ -522,13 +519,22 @@ def benefit_unit_attributes(
 def units_per_household(
     household: pd.DataFrame, family: pd.DataFrame, rule: BenefitUnitRule
 ) -> pd.Series:
-    """How many units each household holds, aligned to ``household``'s rows."""
+    """How many units each household holds, aligned to ``household``'s rows.
+
+    Raises:
+        ValueError: If the household ids or the units' household column hold
+            anything but non-null integer ids, or an unsigned id above
+            ``2**63 - 1``: both are matched as int64.
+    """
 
-    counts = family[rule.household_column].value_counts()
+    # Both sides as int64: reindexing by ids pandas would cast down to a
+    # narrower unsigned unit column (household 261 onto 5 for uint8).
+    counts = pd.Series(
+        _int64_ids(family[rule.household_column], rule.household_column)
+    ).value_counts()
+    households = _int64_ids(household[_HOUSEHOLD_ID], f"household.{_HOUSEHOLD_ID}")
     return pd.Series(
-        counts.reindex(household[_HOUSEHOLD_ID].to_numpy(), fill_value=0)
-        .to_numpy()
-        .astype(np.int64),
+        counts.reindex(households, fill_value=0).to_numpy().astype(np.int64),
         index=household.index,
         name=f"n_{rule.entity}_units",
     )