diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3d57fcb5..c980a1a8 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -5,8 +5,6 @@ on: branches: - master pull_request: - branches: - - master permissions: contents: read diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5baf54f3..68cbb54b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,8 +5,6 @@ on: branches: - master pull_request: - branches: - - master jobs: pytest-shard: diff --git a/docs/accounting-history.md b/docs/accounting-history.md new file mode 100644 index 00000000..31246231 --- /dev/null +++ b/docs/accounting-history.md @@ -0,0 +1,104 @@ +# Supplied annual accounting history + +`populace_dynamics.engine.accounting_history` is an optional wrapper around +the unchanged [annual accountant](stock-flow-accounting.md). It accepts +already supplied frames and declarations. It does not run a projection, +generate events, or establish scientific acceptance. + +```python +from populace_dynamics.engine.accounting_history import ( + AnnualTransition, + reconcile_history, +) + +history = reconcile_history( + [ + AnnualTransition( + opening_year=2020, + closing_year=2021, + opening=people_2020, + closing=people_2021, + additions=declared_2021_additions, + exits=declared_2021_exits, + ), + AnnualTransition( + opening_year=2021, + closing_year=2022, + opening=people_2021, + closing=people_2022, + additions=declared_2022_additions, + exits=declared_2022_exits, + ), + ], + identity_contract="single_presence_episode", +) +``` + +The keyword `identity_contract` is required. Its only supported value is +`single_presence_episode`: within the supplied history, every person ID +may have one presence episode. Every declared departure retires that ID +under this explicitly selected constraint, including a transient who +arrives and departs between endpoints. Declaring that ID again is refused, +even after empty intervening years or as another transient. New, distinct +IDs remain supported after extinction; the wrapper never allocates IDs. + +This contract does not assert that emigration or another departure is +permanent in reality. Same-person return needs a separately reviewed typed +linkage to its earlier departure and explicit return semantics. The current +event kinds do not supply that contract. Changing a returnee's ID to evade +the check would not model their identity correctly. Within a single year, +the annual accountant still supports only an arrival followed by a departure, +not exit followed by return. + +Periods must form a nonempty sequence in chronological order, each one year +long, with no gaps or repeated periods. At a shared boundary the previous +closing frame and next opening frame must have the same person-ID set and +the same weight for each ID, regardless of row order. The wrapper reuses the +accountant's signed-int64 ID/year and finite nonnegative binary64 weight +validation before constructing snapshots. Weight equality is exact in that +binary64 domain, with no tolerance. A discrepancy between two representations +of the same boundary is refused; weight changes within an annual transition +retain the accountant's separate revaluation components and residuals. +Zero-weight people remain in the roster. + +Each returned `HistoryAccount.periods` item is the unchanged annual account. +The result, its nested mappings, and its ID tuples are immutable; `to_dict()` +creates fresh summary containers and omits identifier rosters. The supplied +`AnnualTransition` descriptors freeze field assignment but do not make +caller-owned DataFrames or declaration lists immutable. Reconciliation does +not mutate those inputs or retain their containers in the result. Callers +must not mutate inputs concurrently while they are being read. + +`HistoryAccountingError` identifies the failure kind and zero-based period +index, with validated year coordinates when available. Annual failures keep +their original typed exception in `__cause__`, including reconciliation +discrepancies. Its `to_dict()` returns fresh JSON-safe diagnostic containers. +A failure raises without returning a completed or partial history account. + +The first opening roster establishes no prior lifecycle history. This wrapper +checks only the supplied periods and declarations: it cannot establish event +truth, cause, completeness, or behavior outside that interval. A transient +omitted from both declaration lists is still unobservable. No horizon totals, +family behavior, benefit calculations, benchmark comparison, acceptance gate, +or population admission claim is introduced. + +Only person IDs, years, and person weights enter this accounting interface. +Other frame columns are ignored and their continuity is not checked; this is +not household or geographic context transport. The separately proposed +[mortality graph](https://github.com/PolicyEngine/microcosm-dynamics/pull/436) +keeps its age/sex source and snapshots at their exact schemas and rejects +extra columns; this wrapper does not extend those schemas. +A future separate context contract must preserve household/member links and +household atomic-location anchors assigned by Microcosm once before support +cloning. Dynamics must inherit those anchors, derive larger geographies with +the same versioned mapping, and allow location changes only through separately +declared mobility or migration events. No context adapter, household-weight +mapping, partial-household lifecycle, or group-quarters lifecycle is supplied +here. Canonical export fields and provenance still require parent contract +verification. + +The module is not re-exported by the engine initializer and is not called by +the historical engine. Integration into the separately proposed graph is not +included. Direct imports also execute the existing engine initializer and +its broader source dependencies; this wrapper adds only stdlib, pandas, and +the existing accountant to that path. diff --git a/scripts/first_estimates_birth_evidence.py b/scripts/first_estimates_birth_evidence.py index 144a0af5..dfa5f697 100644 --- a/scripts/first_estimates_birth_evidence.py +++ b/scripts/first_estimates_birth_evidence.py @@ -159,6 +159,7 @@ # This opt-in accountant is unreachable from the historical projection. # The existing engine loop, steps, and package initializer remain sealed. Path("src/populace_dynamics/engine/accounting.py"), + Path("src/populace_dynamics/engine/accounting_history.py"), ) POST_REVIEW_SHARED_SOURCE_BLOBS = { Path( diff --git a/src/populace_dynamics/engine/accounting_history.py b/src/populace_dynamics/engine/accounting_history.py new file mode 100644 index 00000000..41c1a29b --- /dev/null +++ b/src/populace_dynamics/engine/accounting_history.py @@ -0,0 +1,315 @@ +"""Optional accounting of supplied annual frames under an explicit ID contract. + +This module adds boundary and identity-history checks to ``reconcile_period``. +It neither generates events nor establishes their truth or completeness. The +only supported identity contract allows one presence episode per person ID +within the supplied history; same-ID return requires a different contract. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from enum import Enum +from types import MappingProxyType + +import pandas as pd + +from . import accounting + +__all__ = [ + "HISTORY_INTERFACE_VERSION", + "SINGLE_PRESENCE_EPISODE", + "AnnualTransition", + "HistoryAccount", + "HistoryAccountingError", + "HistoryErrorKind", + "reconcile_history", +] + +HISTORY_INTERFACE_VERSION = "accounting-history/0.1.0-experimental" +SINGLE_PRESENCE_EPISODE = "single_presence_episode" + + +class HistoryErrorKind(str, Enum): + """The stage at which a supplied history is refused.""" + + INPUT = "input" + PERIOD = "period" + NONCONTIGUOUS = "noncontiguous" + BOUNDARY_PERSON_IDS = "boundary_person_ids" + BOUNDARY_WEIGHTS = "boundary_weights" + RETIRED_PERSON_ID = "retired_person_id" + + +class HistoryAccountingError(accounting.PopulationAccountingError): + """A history refusal, with a zero-based period index when applicable. + + Annual input/reconciliation failures retain the original exception as + ``__cause__``, including its typed discrepancies. No partial history + account is returned on failure. + """ + + def __init__( + self, + message: str, + *, + kind: HistoryErrorKind, + period_index: int | None = None, + opening_year: int | None = None, + closing_year: int | None = None, + person_ids: tuple[int, ...] = (), + ) -> None: + self.kind = kind + self.period_index = period_index + self.opening_year = opening_year + self.closing_year = closing_year + self.person_ids = person_ids + prefix = ( + "history" if period_index is None else f"period[{period_index}]" + ) + super().__init__(f"{prefix}: {message}") + + def to_dict(self) -> dict[str, object]: + """Return a JSON-safe refusal without serializing input containers.""" + cause = self.__cause__ + return { + "interface_version": HISTORY_INTERFACE_VERSION, + "error": "history_accounting_error", + "kind": self.kind.value, + "message": str(self), + "period_index": self.period_index, + "opening_year": self.opening_year, + "closing_year": self.closing_year, + "person_ids": list(self.person_ids), + "cause": ( + { + "type": type(cause).__name__, + "message": str(cause), + "details": ( + cause.to_dict() + if isinstance( + cause, accounting.PopulationReconciliationError + ) + else None + ), + } + if cause is not None + else None + ), + } + + +@dataclass(frozen=True) +class AnnualTransition: + """One supplied period, without ownership of the caller's containers. + + Freezing this descriptor prevents field reassignment. The DataFrames and + any declaration lists remain caller-owned and mutable; reconciliation + reads snapshots and never stores those containers in its result. + """ + + opening_year: int + closing_year: int + opening: pd.DataFrame + closing: pd.DataFrame + additions: Sequence[accounting.PopulationEvent] = () + exits: Sequence[accounting.PopulationEvent] = () + + +@dataclass(frozen=True) +class HistoryAccount: + """Immutable annual accounts and the IDs observed within this history.""" + + identity_contract: str + periods: tuple[accounting.PeriodAccount, ...] + seen_person_ids: tuple[int, ...] + retired_person_ids: tuple[int, ...] + provenance: Mapping[str, object] + + def to_dict(self) -> dict[str, object]: + """Return fresh summary containers, omitting the identifier rosters.""" + return { + "interface_version": HISTORY_INTERFACE_VERSION, + "status": accounting.ENGINEERING_STATUS, + "identity_contract": self.identity_contract, + "periods": [period.to_dict() for period in self.periods], + "provenance": dict(self.provenance), + } + + +def _snapshot_frame( + frame: pd.DataFrame, year: int, label: str +) -> tuple[pd.DataFrame, dict[int, float]]: + # Reuse the accountant's strict domain before conversion. These helpers + # are internal to the same package; annual law remains reconcile_period. + ids, weights = accounting._read_frame(frame, year, label) + return ( + pd.DataFrame( + { + accounting.PERSON_ID_COLUMN: ids, + accounting.YEAR_COLUMN: year, + accounting.WEIGHT_COLUMN: weights, + } + ), + dict(zip(ids.tolist(), weights.tolist(), strict=True)), + ) + + +def _snapshot_events(events: object) -> object: + # Preserve malformed inputs for the accountant's own typed validation. + if isinstance(events, Sequence) and not isinstance(events, (str, bytes)): + return tuple(events) + return events + + +def reconcile_history( + transitions: Sequence[AnnualTransition], *, identity_contract: str +) -> HistoryAccount: + """Reconcile nonempty, ordered, contiguous annual periods. + + ``identity_contract`` must explicitly be ``single_presence_episode``. + Every declared departure retires its ID for the supplied history under + that chosen constraint, regardless of cause. This does not assert that + emigration or another exit is demographically permanent. A same-ID return + is unsupported, including a return declared as a transient. + + Adjacent snapshots must agree on the person-ID set and exact binary64 + weight per ID, independent of row order. Other columns are not compared. + The first opening roster supplies no prehistory. Callers must not mutate + input containers concurrently while this function reads them. + """ + if ( + not isinstance(identity_contract, str) + or identity_contract != SINGLE_PRESENCE_EPISODE + ): + raise HistoryAccountingError( + "identity_contract must explicitly be 'single_presence_episode'; " + "general same-person re-entry requires a separate typed contract", + kind=HistoryErrorKind.INPUT, + ) + if ( + isinstance(transitions, (str, bytes)) + or not isinstance(transitions, Sequence) + or not transitions + ): + raise HistoryAccountingError( + "transitions must be a nonempty ordered sequence of " + "AnnualTransition", + kind=HistoryErrorKind.INPUT, + ) + + periods: list[accounting.PeriodAccount] = [] + seen: set[int] = set() + retired: set[int] = set() + previous_closing: dict[int, float] = {} + for index, transition in enumerate(tuple(transitions)): + opening_year = closing_year = None + try: + if not isinstance(transition, AnnualTransition): + raise accounting.PopulationAccountingInputError( + "transition must be an AnnualTransition" + ) + opening_year = accounting._as_year( + transition.opening_year, "opening_year" + ) + closing_year = accounting._as_year( + transition.closing_year, "closing_year" + ) + opening, opening_by_id = _snapshot_frame( + transition.opening, opening_year, "opening frame" + ) + closing, closing_by_id = _snapshot_frame( + transition.closing, closing_year, "closing frame" + ) + account = accounting.reconcile_period( + opening, + closing, + opening_year=opening_year, + closing_year=closing_year, + additions=_snapshot_events(transition.additions), + exits=_snapshot_events(transition.exits), + ) + except accounting.PopulationAccountingError as error: + raise HistoryAccountingError( + str(error), + kind=HistoryErrorKind.PERIOD, + period_index=index, + opening_year=opening_year, + closing_year=closing_year, + ) from error + + context = { + "period_index": index, + "opening_year": opening_year, + "closing_year": closing_year, + } + if periods: + if opening_year != periods[-1].closing_year: + raise HistoryAccountingError( + "periods must be ordered and contiguous: " + f"expected opening year {periods[-1].closing_year}", + kind=HistoryErrorKind.NONCONTIGUOUS, + **context, + ) + differing_ids = previous_closing.keys() ^ opening_by_id.keys() + if differing_ids: + raise HistoryAccountingError( + "shared boundary person-ID sets differ", + kind=HistoryErrorKind.BOUNDARY_PERSON_IDS, + person_ids=tuple(sorted(differing_ids)), + **context, + ) + differing_weights = tuple( + sorted( + person_id + for person_id, weight in opening_by_id.items() + if weight != previous_closing[person_id] + ) + ) + if differing_weights: + raise HistoryAccountingError( + "shared boundary weights differ in the accountant's " + "binary64 domain", + kind=HistoryErrorKind.BOUNDARY_WEIGHTS, + person_ids=differing_weights, + **context, + ) + else: + seen.update(opening_by_id) + + reused = tuple(sorted(retired.intersection(account.added_person_ids))) + if reused: + raise HistoryAccountingError( + "a departed person ID was declared again; reuse and " + "same-person return are unsupported under the explicitly " + "selected single_presence_episode contract", + kind=HistoryErrorKind.RETIRED_PERSON_ID, + person_ids=reused, + **context, + ) + seen.update(account.added_person_ids) + retired.update(account.departed_person_ids) + periods.append(account) + previous_closing = closing_by_id + + return HistoryAccount( + identity_contract=SINGLE_PRESENCE_EPISODE, + periods=tuple(periods), + seen_person_ids=tuple(sorted(seen)), + retired_person_ids=tuple(sorted(retired)), + provenance=MappingProxyType( + { + "annual_accounting_interface": ( + accounting.ACCOUNTING_INTERFACE_VERSION + ), + "scope": "supplied history only; first opening has no prehistory", + "boundary_comparison": "person-ID set and binary64 weight per ID", + "other_column_continuity_verified": False, + "event_log_completeness_verified": False, + "event_truth_verified": False, + "retirement_policy": "all declared departures under the selected contract", + "general_reentry_supported": False, + } + ), + ) diff --git a/tests/README-tiers.md b/tests/README-tiers.md index 39a10c2c..beafa451 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -38,9 +38,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 1,656 | +| `unit` | 1,706 | | `artifact` | 2,672 | | `integration_psid` | 848 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 159 | -| **Total** | **5,855** | +| **Total** | **5,905** | diff --git a/tests/estimates/test_birth_evidence_artifact.py b/tests/estimates/test_birth_evidence_artifact.py index 64558840..fb56fdc2 100644 --- a/tests/estimates/test_birth_evidence_artifact.py +++ b/tests/estimates/test_birth_evidence_artifact.py @@ -89,6 +89,7 @@ def test_post_review_sources_are_outside_historical_reducer_identity(): Path("src/populace_dynamics/estimates/anchor_context_rehearsal.py"), Path("src/populace_dynamics/estimates/anchor_context_report.py"), Path("src/populace_dynamics/engine/accounting.py"), + Path("src/populace_dynamics/engine/accounting_history.py"), ) assert reducer.POST_REVIEW_SHARED_SOURCE_BLOBS == { Path( @@ -252,6 +253,8 @@ def test_psid_identity_exclusions_are_unreachable_from_birth_evidence(): ) assert "populace_dynamics.engine.accounting" in module_paths assert "populace_dynamics.engine.accounting" not in reachable + assert "populace_dynamics.engine.accounting_history" in module_paths + assert "populace_dynamics.engine.accounting_history" not in reachable assert "populace_dynamics.engine" in reachable assert "populace_dynamics.engine.steps" in reachable diff --git a/tests/test_m6_accounting_history.py b/tests/test_m6_accounting_history.py new file mode 100644 index 00000000..d0c8fa9c --- /dev/null +++ b/tests/test_m6_accounting_history.py @@ -0,0 +1,431 @@ +"""Synthetic supplied-history accounting; no data or fitted models.""" + +import json +from dataclasses import FrozenInstanceError + +import numpy as np +import pandas as pd +import pytest + +from populace_dynamics.engine import accounting +from populace_dynamics.engine.accounting_history import ( + SINGLE_PRESENCE_EPISODE, + AnnualTransition, + HistoryAccountingError, + HistoryErrorKind, + reconcile_history, +) + + +def _frame(year, rows=()): + return pd.DataFrame( + { + "person_id": [person_id for person_id, _ in rows], + "year": [year] * len(rows), + "weight": [weight for _, weight in rows], + } + ) + + +def _steady(year, rows=((1, 1.0),)): + return AnnualTransition( + year, year + 1, _frame(year, rows), _frame(year + 1, rows) + ) + + +def _event(person_id, kind, year, weight=None): + return accounting.PopulationEvent( + person_id, + kind, + year, + weight=weight, + reason="supplied synthetic event", + source="synthetic fixture", + ) + + +def _reconcile(transitions): + return reconcile_history( + transitions, identity_contract=SINGLE_PRESENCE_EPISODE + ) + + +def _history(): + first = AnnualTransition( + 2020, + 2021, + _frame(2020, [(1, 0), (2, 2), (3, 3)]), + _frame(2021, [(1, 1), (2, 2), (4, 4)]), + additions=[ + _event(4, "birth", 2021, 5), + _event(8, "scheduled_entry", 2021, 2), + ], + exits=[ + _event(3, "death", 2021), + _event(8, "other_exit", 2021, 1), + ], + ) + second = AnnualTransition( + 2021, + 2022, + _frame(2021, [(4, 4), (2, 2), (1, 1)]), + _frame(2022, [(2, 2), (4, 4)]), + additions=[_event(9, "other_entry", 2022, 3)], + exits=[ + _event(1, "emigration", 2022), + _event(9, "death", 2022, 3), + ], + ) + return [first, second, _steady(2022, [(4, 4), (2, 2)])] + + +def test_accounts_are_unchanged_annual_results_in_supplied_order(monkeypatch): + transitions = _history() + expected = tuple( + accounting.reconcile_period( + item.opening, + item.closing, + opening_year=item.opening_year, + closing_year=item.closing_year, + additions=item.additions, + exits=item.exits, + ) + for item in transitions + ) + calls = [] + original = accounting.reconcile_period + + def record(*args, **kwargs): + calls.append(kwargs["closing_year"]) + return original(*args, **kwargs) + + monkeypatch.setattr(accounting, "reconcile_period", record) + result = _reconcile(transitions) + assert calls == [2021, 2022, 2023] + assert result.periods == expected + assert result.seen_person_ids == (1, 2, 3, 4, 8, 9) + assert result.retired_person_ids == (1, 3, 8, 9) + assert result.periods[0].counts.transient == 1 + assert result.periods[0].weights.revaluation.transient == -1.0 + assert result.provenance["event_log_completeness_verified"] is False + assert result.provenance["event_truth_verified"] is False + + +def test_contract_keyword_is_required(): + with pytest.raises(TypeError, match="identity_contract"): + reconcile_history([_steady(2020)]) + + +@pytest.mark.parametrize("contract", [None, "", "general_reentry", True, []]) +def test_unsupported_identity_contract_is_a_typed_refusal(contract): + with pytest.raises(HistoryAccountingError) as caught: + reconcile_history([_steady(2020)], identity_contract=contract) + assert caught.value.kind is HistoryErrorKind.INPUT + assert caught.value.period_index is None + json.dumps(caught.value.to_dict(), allow_nan=False) + + +@pytest.mark.parametrize("transitions", [[], (), None, "history", iter(())]) +def test_nonempty_sequence_is_required(transitions): + with pytest.raises(HistoryAccountingError) as caught: + _reconcile(transitions) + assert caught.value.kind is HistoryErrorKind.INPUT + + +@pytest.mark.parametrize("second_year", [2019, 2020, 2022]) +def test_reversal_duplicate_and_gap_are_rejected(second_year): + with pytest.raises(HistoryAccountingError) as caught: + _reconcile([_steady(2020), _steady(second_year)]) + assert caught.value.kind is HistoryErrorKind.NONCONTIGUOUS + assert caught.value.period_index == 1 + assert caught.value.opening_year == second_year + assert caught.value.closing_year == second_year + 1 + + +def test_nonannual_period_preserves_accountant_input_error(): + transition = AnnualTransition(2020, 2022, _frame(2020), _frame(2022)) + with pytest.raises(HistoryAccountingError) as caught: + _reconcile([transition]) + error = caught.value + assert error.kind is HistoryErrorKind.PERIOD + assert error.period_index == 0 + assert (error.opening_year, error.closing_year) == (2020, 2022) + assert isinstance( + error.__cause__, accounting.PopulationAccountingInputError + ) + assert "annual" in str(error.__cause__) + + +def test_invalid_descriptor_preserves_period_index_and_cause(): + with pytest.raises(HistoryAccountingError) as caught: + _reconcile([_steady(2020), object()]) + error = caught.value + assert error.period_index == 1 + assert error.opening_year is None + assert error.closing_year is None + assert isinstance( + error.__cause__, accounting.PopulationAccountingInputError + ) + + +def test_annual_discrepancies_remain_available_on_original_cause(): + broken = AnnualTransition(2021, 2022, _frame(2021, [(1, 1)]), _frame(2022)) + with pytest.raises(HistoryAccountingError) as caught: + _reconcile([_steady(2020), broken]) + error = caught.value + assert error.kind is HistoryErrorKind.PERIOD + assert error.period_index == 1 + assert (error.opening_year, error.closing_year) == (2021, 2022) + assert isinstance( + error.__cause__, accounting.PopulationReconciliationError + ) + assert ( + error.__cause__.discrepancies[0].kind + is accounting.DiscrepancyKind.UNDECLARED_EXIT + ) + payload = error.to_dict() + assert payload["cause"]["type"] == "PopulationReconciliationError" + assert ( + payload["cause"]["details"]["discrepancies"][0]["kind"] + == "undeclared_exit" + ) + assert json.loads(json.dumps(payload, allow_nan=False)) == payload + payload["person_ids"].append(999) + payload["cause"]["details"]["discrepancies"].clear() + assert error.to_dict()["person_ids"] == [] + assert len(error.to_dict()["cause"]["details"]["discrepancies"]) == 1 + + +def test_boundary_person_sets_must_match_even_at_equal_counts_and_mass(): + with pytest.raises(HistoryAccountingError) as caught: + _reconcile([_steady(2020), _steady(2021, [(2, 1)])]) + assert caught.value.kind is HistoryErrorKind.BOUNDARY_PERSON_IDS + assert caught.value.person_ids == (1, 2) + assert caught.value.period_index == 1 + + +def test_boundary_weights_are_compared_per_person_not_in_aggregate(): + with pytest.raises(HistoryAccountingError) as caught: + _reconcile( + [_steady(2020, [(1, 2), (2, 3)]), _steady(2021, [(1, 3), (2, 2)])] + ) + assert caught.value.kind is HistoryErrorKind.BOUNDARY_WEIGHTS + assert caught.value.person_ids == (1, 2) + + +def test_boundary_weight_comparison_has_no_tolerance(): + with pytest.raises(HistoryAccountingError) as caught: + _reconcile( + [_steady(2020), _steady(2021, [(1, np.nextafter(1.0, 2.0))])] + ) + assert caught.value.kind is HistoryErrorKind.BOUNDARY_WEIGHTS + + +def test_boundary_comparison_uses_accountant_binary64_domain(): + first = _steady(2020, [(1, 2**53 + 1)]) + second = _steady(2021, [(1, float(2**53))]) + result = _reconcile([first, second]) + assert ( + result.periods[0].weights.closing == result.periods[1].weights.opening + ) + + +def test_boundary_order_and_opaque_columns_do_not_imply_other_continuity(): + first = _steady(2020, [(1, 0), (2, 3)]) + second = _steady(2021, [(2, 3), (1, 0)]) + first.closing["opaque"] = ["earlier", "earlier"] + second.opening["opaque"] = ["later", "later"] + before = [ + frame.copy(deep=True) for frame in (first.closing, second.opening) + ] + result = _reconcile([first, second]) + assert result.periods[0].counts.closing == 2 + assert result.provenance["other_column_continuity_verified"] is False + for frame, original in zip( + (first.closing, second.opening), before, strict=True + ): + pd.testing.assert_frame_equal(frame, original) + + +@pytest.mark.parametrize("exit_kind", ["death", "emigration", "other_exit"]) +@pytest.mark.parametrize("return_is_transient", [False, True]) +def test_departed_id_return_is_unsupported_under_selected_contract( + exit_kind, return_is_transient +): + departure = AnnualTransition( + 2020, + 2021, + _frame(2020, [(1, 1)]), + _frame(2021), + exits=[_event(1, exit_kind, 2021)], + ) + returning = AnnualTransition( + 2022, + 2023, + _frame(2022), + _frame(2023, [] if return_is_transient else [(1, 1)]), + additions=[_event(1, "scheduled_entry", 2023, 1)], + exits=[_event(1, "death", 2023, 1)] if return_is_transient else (), + ) + with pytest.raises( + HistoryAccountingError, match="same-person return" + ) as caught: + _reconcile([departure, _steady(2021, []), returning]) + assert caught.value.kind is HistoryErrorKind.RETIRED_PERSON_ID + assert caught.value.period_index == 2 + assert caught.value.person_ids == (1,) + + +def test_transient_id_is_seen_and_retired_before_any_endpoint_presence(): + first = AnnualTransition( + 2020, + 2021, + _frame(2020), + _frame(2021), + additions=[_event(10, "other_entry", 2021, 0)], + exits=[_event(10, "other_exit", 2021, 0)], + ) + result = _reconcile([first]) + assert result.seen_person_ids == result.retired_person_ids == (10,) + second = AnnualTransition( + 2021, + 2022, + _frame(2021), + _frame(2022, [(10, 0)]), + additions=[_event(10, "birth", 2022)], + ) + with pytest.raises(HistoryAccountingError) as caught: + _reconcile([first, second]) + assert caught.value.kind is HistoryErrorKind.RETIRED_PERSON_ID + + +def test_extinction_empty_years_and_fresh_ids_remain_supported(): + first = AnnualTransition( + 2020, + 2021, + _frame(2020, [(1, 0)]), + _frame(2021), + exits=[_event(1, "death", 2021)], + ) + third = AnnualTransition( + 2022, + 2023, + _frame(2022), + _frame(2023, [(2, 0)]), + additions=[_event(2, "scheduled_entry", 2023)], + ) + result = _reconcile([first, _steady(2021, []), third]) + assert ( + result.periods[1].counts.opening + == result.periods[1].counts.closing + == 0 + ) + assert result.seen_person_ids == (1, 2) + assert result.retired_person_ids == (1,) + + +def test_omitting_both_transient_declarations_remains_undetectable(): + result = _reconcile([_steady(2020, []), _steady(2021, [])]) + assert result.seen_person_ids == result.retired_person_ids == () + assert result.provenance["event_log_completeness_verified"] is False + assert all(period.counts.transient == 0 for period in result.periods) + + +def test_first_opening_has_no_inferred_prehistory(): + result = _reconcile([_steady(2020, [(100, 1)])]) + assert result.seen_person_ids == (100,) + assert result.retired_person_ids == () + assert "no prehistory" in result.provenance["scope"] + + +@pytest.mark.parametrize( + ("column", "value"), + [ + ("person_id", True), + ("person_id", "1"), + ("person_id", 2**64 - 1), + ("year", 2021.0), + ("weight", True), + ("weight", "1.0"), + ("weight", float("inf")), + ], +) +def test_malformed_boundary_values_are_not_coerced_before_validation( + column, value +): + second = _steady(2021) + second.opening[column] = pd.Series([value], dtype=object) + with pytest.raises(HistoryAccountingError) as caught: + _reconcile([_steady(2020), second]) + assert caught.value.kind is HistoryErrorKind.PERIOD + assert caught.value.period_index == 1 + assert isinstance( + caught.value.__cause__, accounting.PopulationAccountingInputError + ) + json.dumps(caught.value.to_dict(), allow_nan=False) + + +@pytest.mark.parametrize("year", [True, "2020", 2020.0, 2**64 - 1]) +def test_malformed_period_years_are_wrapped_without_unsafe_coordinates(year): + transition = AnnualTransition(year, 2021, _frame(2020), _frame(2021)) + with pytest.raises(HistoryAccountingError) as caught: + _reconcile([transition]) + assert caught.value.opening_year is None + assert caught.value.closing_year is None + json.dumps(caught.value.to_dict(), allow_nan=False) + + +def test_ids_at_signed_int64_limits_remain_exact(): + rows = [(np.int64(-(2**63)), 0), (np.int64(2**63 - 1), 1)] + result = _reconcile([_steady(2020, rows), _steady(2021, rows[::-1])]) + assert result.seen_person_ids == (-(2**63), 2**63 - 1) + assert "seen_person_ids" not in result.to_dict() + + +def test_result_and_serialization_are_isolated_from_caller_mutation(): + transitions = _history() + result = _reconcile(transitions) + expected = result.to_dict() + for transition in transitions: + transition.opening["weight"] = 999 + transition.closing["person_id"] = 999 + transitions[0].additions.clear() + transitions.clear() + assert result.to_dict() == expected + with pytest.raises(FrozenInstanceError): + result.identity_contract = "changed" + with pytest.raises(FrozenInstanceError): + result.periods[0].opening_year = 1999 + with pytest.raises(TypeError): + result.provenance["scope"] = "changed" + with pytest.raises(TypeError): + result.periods[0].counts.additions_by_kind["birth"] = 100 + payload = result.to_dict() + payload["provenance"]["scope"] = "changed" + payload["periods"][0]["provenance"]["declaration_sources"].append( + "changed" + ) + payload["periods"][0]["counts"]["opening"] = 999 + assert result.to_dict() == expected + json.dumps(result.to_dict(), allow_nan=False) + + +def test_descriptor_freezes_fields_but_does_not_own_input_frames(): + transition = _steady(2020) + with pytest.raises(FrozenInstanceError): + transition.opening_year = 2000 + transition.opening.loc[0, "weight"] = 2 + assert transition.opening.loc[0, "weight"] == 2 + + +@pytest.mark.parametrize("events", ["events", iter(()), [object()]]) +def test_malformed_declaration_containers_stay_accountant_errors(events): + transition = AnnualTransition( + 2020, 2021, _frame(2020), _frame(2021), additions=events + ) + with pytest.raises(HistoryAccountingError) as caught: + _reconcile([transition]) + assert caught.value.kind is HistoryErrorKind.PERIOD + assert isinstance( + caught.value.__cause__, accounting.PopulationAccountingInputError + ) diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 082d3bfd..dafada3c 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 1656, + "unit": 1706, "artifact": 2672, "integration_psid": 848, "reproduction_legacy": 520,