diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3d57fcb5..c980a1a8 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -5,8 +5,6 @@ on: branches: - master pull_request: - branches: - - master permissions: contents: read diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5baf54f3..ccd2d0e8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,8 +5,6 @@ on: branches: - master pull_request: - branches: - - master jobs: pytest-shard: @@ -37,11 +35,30 @@ jobs: - name: Run tests (shard ${{ matrix.shard }}/4) run: pytest -q --splits 4 --group ${{ matrix.shard }} --splitting-algorithm duration_based_chunks --durations-path .test_durations + graph-integration: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.13', '3.14'] + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install the pinned optional graph integration + run: pip install -e ".[graph]" pytest + - name: Require typed graph capabilities + run: python -c "from populace_dynamics.graph._compat import require_graph; require_graph()" + - name: Run graph and existing mortality regressions + run: pytest -q tests/test_graph_mortality.py tests/test_graph_mortality_trajectory.py tests/test_graph_trajectory_accounting.py tests/test_m6_engine_refit.py tests/test_m6_engine_steps.py + # Fan-in jobs keeping the branch-protection context names # ("pytest (3.11)" / "pytest (3.13)") stable across the shard split. pytest: name: pytest (${{ matrix.python-version }}) - needs: pytest-shard + needs: [pytest-shard, graph-integration] if: always() runs-on: ubuntu-latest strategy: @@ -49,4 +66,6 @@ jobs: python-version: ['3.14'] steps: - name: Verify all shards passed - run: test "${{ needs.pytest-shard.result }}" = "success" + run: | + test "${{ needs.pytest-shard.result }}" = "success" + test "${{ needs.graph-integration.result }}" = "success" diff --git a/docs/accounting-history.md b/docs/accounting-history.md new file mode 100644 index 00000000..31246231 --- /dev/null +++ b/docs/accounting-history.md @@ -0,0 +1,104 @@ +# Supplied annual accounting history + +`populace_dynamics.engine.accounting_history` is an optional wrapper around +the unchanged [annual accountant](stock-flow-accounting.md). It accepts +already supplied frames and declarations. It does not run a projection, +generate events, or establish scientific acceptance. + +```python +from populace_dynamics.engine.accounting_history import ( + AnnualTransition, + reconcile_history, +) + +history = reconcile_history( + [ + AnnualTransition( + opening_year=2020, + closing_year=2021, + opening=people_2020, + closing=people_2021, + additions=declared_2021_additions, + exits=declared_2021_exits, + ), + AnnualTransition( + opening_year=2021, + closing_year=2022, + opening=people_2021, + closing=people_2022, + additions=declared_2022_additions, + exits=declared_2022_exits, + ), + ], + identity_contract="single_presence_episode", +) +``` + +The keyword `identity_contract` is required. Its only supported value is +`single_presence_episode`: within the supplied history, every person ID +may have one presence episode. Every declared departure retires that ID +under this explicitly selected constraint, including a transient who +arrives and departs between endpoints. Declaring that ID again is refused, +even after empty intervening years or as another transient. New, distinct +IDs remain supported after extinction; the wrapper never allocates IDs. + +This contract does not assert that emigration or another departure is +permanent in reality. Same-person return needs a separately reviewed typed +linkage to its earlier departure and explicit return semantics. The current +event kinds do not supply that contract. Changing a returnee's ID to evade +the check would not model their identity correctly. Within a single year, +the annual accountant still supports only an arrival followed by a departure, +not exit followed by return. + +Periods must form a nonempty sequence in chronological order, each one year +long, with no gaps or repeated periods. At a shared boundary the previous +closing frame and next opening frame must have the same person-ID set and +the same weight for each ID, regardless of row order. The wrapper reuses the +accountant's signed-int64 ID/year and finite nonnegative binary64 weight +validation before constructing snapshots. Weight equality is exact in that +binary64 domain, with no tolerance. A discrepancy between two representations +of the same boundary is refused; weight changes within an annual transition +retain the accountant's separate revaluation components and residuals. +Zero-weight people remain in the roster. + +Each returned `HistoryAccount.periods` item is the unchanged annual account. +The result, its nested mappings, and its ID tuples are immutable; `to_dict()` +creates fresh summary containers and omits identifier rosters. The supplied +`AnnualTransition` descriptors freeze field assignment but do not make +caller-owned DataFrames or declaration lists immutable. Reconciliation does +not mutate those inputs or retain their containers in the result. Callers +must not mutate inputs concurrently while they are being read. + +`HistoryAccountingError` identifies the failure kind and zero-based period +index, with validated year coordinates when available. Annual failures keep +their original typed exception in `__cause__`, including reconciliation +discrepancies. Its `to_dict()` returns fresh JSON-safe diagnostic containers. +A failure raises without returning a completed or partial history account. + +The first opening roster establishes no prior lifecycle history. This wrapper +checks only the supplied periods and declarations: it cannot establish event +truth, cause, completeness, or behavior outside that interval. A transient +omitted from both declaration lists is still unobservable. No horizon totals, +family behavior, benefit calculations, benchmark comparison, acceptance gate, +or population admission claim is introduced. + +Only person IDs, years, and person weights enter this accounting interface. +Other frame columns are ignored and their continuity is not checked; this is +not household or geographic context transport. The separately proposed +[mortality graph](https://github.com/PolicyEngine/microcosm-dynamics/pull/436) +keeps its age/sex source and snapshots at their exact schemas and rejects +extra columns; this wrapper does not extend those schemas. +A future separate context contract must preserve household/member links and +household atomic-location anchors assigned by Microcosm once before support +cloning. Dynamics must inherit those anchors, derive larger geographies with +the same versioned mapping, and allow location changes only through separately +declared mobility or migration events. No context adapter, household-weight +mapping, partial-household lifecycle, or group-quarters lifecycle is supplied +here. Canonical export fields and provenance still require parent contract +verification. + +The module is not re-exported by the engine initializer and is not called by +the historical engine. Integration into the separately proposed graph is not +included. Direct imports also execute the existing engine initializer and +its broader source dependencies; this wrapper adds only stdlib, pandas, and +the existing accountant to that path. diff --git a/docs/claiming-successor.md b/docs/claiming-successor.md new file mode 100644 index 00000000..b1c61d25 --- /dev/null +++ b/docs/claiming-successor.md @@ -0,0 +1,76 @@ +# Opt-in claiming state successor + +`populace_dynamics.engine.claiming.apply_claiming` corrects the existing +behavioral no-draw contract for observed disability conversions and previously +claimed people. It is available through an explicit module import. The +registered historical function in `engine.steps` and its assembly remain +unchanged. + +## State contract + +The adapter draws a behavioral plan only when all four conditions hold: + +- `claim_age` is missing. +- Age is at least 50. +- `di_converted` is false. +- The person has not already claimed. + +As in the historical adapter, absent or null conversion and claiming flags +are treated as false. This preserves the existing missing-value disposition; +it does not establish that an unknown upstream event was observed not to occur. +The `age` and `sex` columns are still required for all input frames. + +`claim_age` represents a planned behavioral age. A conversion with no existing +plan retains a null age, becomes claimed, and receives the current year as its +claim year if it was not already claimed. Existing plans are preserved. A +previously claimed person's claim year is preserved, including a missing year. +Persisted `claimed=True` prevents a new plan from being drawn after the annual +conversion event flag clears. No placeholder age or new lifetime conversion +field is introduced. + +## Explicit injection + +The successor reuses `ClaimingSchedule` and the `PeriodModules.claiming` +interface: + +```python +from dataclasses import replace +from functools import partial + +from populace_dynamics.engine.claiming import apply_claiming +from populace_dynamics.engine.loop import ProjectionEngine + +# Supply an existing set of period modules and a caller-owned schedule. +successor_modules = replace( + modules, + claiming=partial(apply_claiming, schedule=schedule), +) +engine = ProjectionEngine(successor_modules) +``` + +The opt-in module must remain outside the historical registered call graph. +It does not promote or replay a scientific candidate, change a locked gate, +or establish benefit-level acceptance. The integration test uses the actual +annual loop and its registry with explicitly synthetic injections for the +other steps; it does not run all eight fitted models. + +## Random streams and scope + +The PMFs, sex grouping, nearest-reference-year rule (including earlier-year +tie breaking), age-50 threshold, and `ProjectionModule.CLAIMING` stream are +unchanged. Under `ProjectionRNGRegistry`, each eligible person's draw agrees +with the historical adapter when the same stable ordinal is retained. +Removing an ineligible row or changing row order does not change another +person's keyed draw. Converted and previously claimed rows do not need a +claiming PMF for their sex or a claiming ordinal. + +Without a registry, the injected batch generator draws fewer values. This +can shift same-seed realizations for other people and subsequent sex groups. +The successor remains reproducible for the same input and seed; cross-version +bitwise parity is not claimed for this fallback. It does not burn discarded +draws to mimic the historical bug. + +The adapter consumes the conversion event supplied by the disability step. +It does not infer statutory conversion timing, insured status, quarters of +coverage, PIA, or benefits. A changed upstream conversion definition or +promotion into registered historical execution requires separate work. diff --git a/docs/design/assembled-history-observer.md b/docs/design/assembled-history-observer.md new file mode 100644 index 00000000..05c82a6d --- /dev/null +++ b/docs/design/assembled-history-observer.md @@ -0,0 +1,180 @@ +# Histories from assembled projections + +`observe_assembled_history` retrospectively composes a +`ClosedCohortEarningsHistory` from an initialized 2014 slice, consecutive +assembled projection slices through at most 2022, and the existing +`mortality_slices` collector. It calls no model, generator or RNG method. +Every original person retains their recorded earnings before removal; +removal before the target year's earnings step creates no earnings row or +post-death zero. The underlying history validators remain unchanged. + +This is an opt-in engineering interface, not an execution or acceptance of a +registered population projection. Caller-supplied source and realization +labels remain declarations. No coverage inference, benefit calculation, +source admission or accepted Axiom bridge is provided. + +## Observe the original population explicitly + +The caller must choose one of two modes: + +- `strict_full_roster` permits mortality attrition but refuses scheduled + entrants and materialized births anywhere in the supplied projection. +- `original_2014_view` follows exactly the initialized 2014 population while + retaining the full projection. Before selecting that cohort, it reconciles + all deaths, scheduled entrants and materialized children. Its separate + exclusion ledger identifies every later person, their entry kind/year, + native parent when applicable, and any subsequent observed removal year. + This mode does not call the full population closed or construct complete + earnings histories for later entrants. + +Supply the entire scheduled-entry mapping, reserved real-ID namespace and +synthetic allocator start used by the projection. The initialized slice must +match the complete declared original cohort. Each next mortality input must +equal the previous full roster plus that year's declared scheduled entries. +Each final slice must equal mortality survivors plus evidenced newborns. +Unknown disappearance, resurrection, repeated identity, unexplained +additions, incomplete mortality evidence and changed demographics refuse. +An entrant who dies immediately remains in the exclusion ledger. Complete +cohort extinction can coexist with a continuing noncohort population. + +Native `person_id` columns must retain NumPy signed `int64` dtype. Exact +source identities map to private history keys only on copied observation +views; native IDs and RNG ordinals are never replaced in the projection. +Registry ordinals are derived from the numerically sorted union of original +and all scheduled IDs, including future entries, matching assembly setup. +They are distinct from dense private keys. Canonical audit IDs must also fit +signed `int64`, including excluded people. + +The full mortality pre-roster must match the preceding slice's age, sex and +weight, including the first step's initialized baseline. Scheduled entries +must supply the immediately preceding year. Survivors must advance age by +one and retain sex/weight. Exact integer scalar ages and years in native +object columns are accepted and converted to `int64` only on copied cohort +views; floating ages/years are refused. Every pre-mortality age must lie +within the supplied snapshot's supported domain. + +## Capture the actual fertility boundary when births occur + +The current `PeriodTrace.authoritative_marital_state.births` is empty in +assembled runs. Maternal births occur in the subsequent fertility callback; +its private draw records are not returned by the collector. Therefore the +trace and a final `synthetic_entry` marker cannot alone substantiate an added +child. Observation without additional evidence refuses materialized births. + +`capture_fertility(modules)` supplies the bounded additional seam: + +```python +instrumented_modules, capture = capture_fertility(assembled_modules) +# Execute the normal ProjectionEngine once with instrumented_modules. +# Retain its ProjectionResult and the existing assembly draw collector. +``` + +The adapter replaces only the `PeriodModules.fertility` callable. It copies +the callback's input values, invokes the original exactly once with the +identical frame/context/marital/RNG objects, copies its output values, and +returns that original output object by identity. All other callbacks and +the initializer retain identity. In particular, the assembled mortality +callback still owns `ensure_draw`, RNG consumption and mortality collection. +The adapter neither calculates fertility nor exposes private maternal draws. + +Create a fresh adapter/capture for each projection invocation. Out-of-order +calls, attempted reuse, mixed draw/horizon, callback exceptions or incomplete +capture make it unusable. `capture.to_json()` archives complete copied +boundary records for an external execution receipt. There is no standalone +capture loader in this prototype. + +The observer requires the captured input to equal the aged mortality +survivors, and captured output to match the final full roster and its +demographics. Newborn metadata must agree between capture and final slice, +with age zero, target birth year, synthetic marker, consecutive allocator +ID, no reserved collision and a surviving parent. Native concatenation can +promote parent IDs to float. Only finite integral floats with absolute value +strictly below `2**53` and a unique exact native-parent match are supported; +larger/ambiguous floating parents refuse. This does not reconstruct a lost +integer ID. Exact integer parent columns retain signed-int64 support. + +Using this adapter creates a separately instrumented execution. Its adapter, +source versions and boundary evidence require a new execution identity even +though the existing engine, assembly and generator source files remain +byte-identical. It must not be presented as the old registered artifact. + +## API and persistence + +```python +observation = observe_assembled_history( + projection, draw_outputs, + mode="original_2014_view", + identity_map=original_identity_map, + realization_id=realization_id, + generator_digest=generator_digest, + earnings_source_contract_digest=earnings_contract_digest, + mortality_snapshot=before_model_snapshot, + mortality_snapshot_after=after_model_snapshot, + mortality_source_contract_digest=mortality_contract_digest, + unit=unit, price_basis=price_basis, + lineage_by_year=annual_lineage_digests, + initial_native_ids=initial_native_ids, + scheduled_entries_by_year=scheduled_entries, + reserved_real_ids=reserved_real_ids, + synthetic_id_start=synthetic_id_start, + fertility_capture=capture, +) +person_history = observation.history.for_person(original_identity) +restored = AssembledHistoryObservation.from_json( + observation.to_json(), + baseline=trusted_initial_history, + expected_digest=observation.digest, +) +``` + +The immutable result binds the cohort history to a canonical selection audit. +Loading requires a trusted whole-envelope digest and external baseline, +rejects unknown/duplicate fields and noncanonical JSON, and rechecks full +roster transitions, exclusions, ordinal bindings and cohort consistency. +Even a recomputed digest cannot permit an internally inconsistent audit or +an ID outside the native type. A trusted digest remains necessary to reject +other valid-looking substitutions; hashes alone do not authenticate a run. + +`input_digests` binds precisely the following supplied evidence: + +| Key | Hashed content | +|---|---| +| `projection` | All slice column names, dtype strings, row order and scalar values | +| `mortality` | The same representation for every supplied mortality slice | +| `traces` | Each trace's year, ordered step names and authoritative marital `births` frame | +| `entry_metadata` | Initial IDs, reserved real IDs, synthetic start and full scheduled-entry frames | +| `mortality_model` | The immutable effective probability snapshot's digest | +| `fertility_capture` | Capture coordinates and copied boundary frame representations, or explicit absence | + +Frame encodings do not include pandas indices or attributes. The `traces` +digest does not bind the rest of authoritative marital state; no digest here +binds other collector outputs, fitted-model files or the execution command. +These representations are input-value bindings, not original-file hashes. +An actual execution receipt must separately bind source files, commands, +all relevant model/input artifacts and pre/post effective parameters. Equal +supplied mortality snapshots are required here, but the observer does not +itself establish when they were obtained. It adds no exposure values and +infers no death date from a step's target year. + +## Verification and remaining scope + +Tests execute the actual eight assembly callbacks with invented fixtures. +Mortality, aging, earnings and claiming execute normally. The fitted marital, +fertility, disability and household simulation cores are stubbed at bounded +test seams; native fertility materialization still runs. This verifies +wiring, not a fully fitted eight-model or registered-input projection. + +Ordinary and correlated earnings fixtures compare all annual frames, +collector outputs and registry-stream end states with and without capture. +They exercise repeated same-draw reset, exact callback arguments/results, +one-call behavior, no observation-time model/RNG calls, no source mutation, +large IDs, unsafe parent refusal, immediate entrant death, extinction and +tampered persistence. An all-real-core rehearsal would be a separate +reviewed task; none is performed by these tests. + +The implementation favors auditable composition over population-scale +storage: boundary JSON copies full frames, and nested per-person histories +repeat metadata. Birth linkage is callback-boundary evidence, not private +maternal draw provenance. The existing historical reducer excludes this new +module explicitly, with a transitive reachability test. Historical gates, +inputs and evidence artifacts remain unchanged. diff --git a/docs/design/closed-cohort-history.md b/docs/design/closed-cohort-history.md new file mode 100644 index 00000000..9805148f --- /dev/null +++ b/docs/design/closed-cohort-history.md @@ -0,0 +1,114 @@ +# Closed-cohort earnings histories + +`ClosedCohortEarningsHistory` links observed mortality steps to annual +earnings for an original 2014 cohort through at most 2022. It composes the +existing `ForwardEarningsHistory` and `MortalityStepObservation` classes. It +does not call a generator or change the registered projection engine. + +```python +cohort = ClosedCohortEarningsHistory.start(initial_history, draw_index=3) +cohort = cohort.append( + mortality=mortality_record, + earnings_frame=actual_post_earnings_frame, + lineage_digest=annual_frame_digest, +) +person_history = cohort.for_person(original_typed_identity) +state = cohort.amount_state(original_typed_identity, year=2016) +death = cohort.death_step(original_typed_identity) +restored = ClosedCohortEarningsHistory.from_json( + cohort.to_json(), baseline=initial_history, + expected_digest=cohort.digest, previous=previous_cohort, +) +``` + +## Composition and observation order + +The initial history must end at 2014. The wrapper retains that baseline and +one existing fixed-roster history per original person. This reuses the +existing amount, nominal unit, provenance, domain and odd-year carry +validation. Each survivor's history appends the actual output for exactly +the next year; deceased persons' earlier rows remain unchanged. The +original identity map and cohort remain fixed even as the active roster +shrinks. `active_keys` reports the current survivors; `last_year` reports the +last explicitly observed transition, including empty post-extinction years. + +The caller first executes the existing mortality step, then aging and the +actual earnings step on survivors, preserving all private generator state. +It supplies those observations to `append`; recording neither repeats the +generator nor replaces lag state with stored annual amounts. Tests compare +this minimal sequence with and without the observer using the actual +ordinary and correlated-refresh generators on invented fixtures, with +registry streams and fallback RNGs. + +The survivor frame requires unique `person_id`, `year`, `age`, `sex`, +`earnings` and `earnings_domain` columns. IDs, years and ages must have NumPy +`int64` dtype, earnings `float64`, and domain `bool`. Sex must have a +string-compatible dtype and contain the recorded mortality labels. These +type requirements also apply to empty frames. No float-ID conversion is +performed; exact original signed/unsigned integers and textual IDs remain +reversible through the baseline identity map. + +Mortality input IDs must equal the previous active roster. Earnings output +IDs must equal mortality survivors exactly. Target years advance one at a +time, with period index `year - 2014`, unchanged draw/realization/identity, +and the same mortality model, mortality source contract and RNG mode/horizon +throughout. Continuing registry ordinals cannot change. Mortality and +earnings source contracts remain distinct. + +Each survivor's output age must equal its pre-mortality age plus one, with +unchanged sex. The actual survivor demographic tuples are retained in each +transition alongside mortality and annual lineage. Consecutive mortality +records must also advance pre-ages by one and preserve sex and RNG ordinals. +The first mortality record establishes demographics: the initial earnings +history has no age or sex evidence against which to verify it. + +## Death, zero and unavailable amounts + +Removal by the mortality step targeting year Y means no Y earnings row was +generated for that person, including when Y would otherwise carry the prior +amount. The person's history ends at Y−1. `amount_state` returns the existing +`known_amount`, `known_zero` or `unavailable` state for recorded years, and +`not_generated_after_mortality_step` for observed years after removal. +`death_step` returns the first removal record, or `None`; its target year and +digest identify the event without inventing a death date. + +This distinguishes a supported zero from an outside-domain control zero, +death before generation, and an unobserved future year. Future queries and +years before 2014 refuse. Absence of generated earnings is not a factual +claim about income earned during a real person's death year. No post-death +zero or outside-domain row is created. + +If all persons die, histories remain intact. Explicit empty mortality and +correctly typed empty earnings frames can continue the observed envelope. +Their year is established by the mortality transition. Ending observation +early does not establish any later year. + +## Persistence and limits + +Canonical JSON composes existing history and mortality documents and binds +the exact external baseline by digest. Loading invokes those validators, +then rechecks mortality continuity, survivor demographics, original cohort, +each person's final year and annual lineage. Unknown/duplicate fields and +inconsistent envelopes refuse. A trusted whole-record digest detects other +valid-looking edits; `previous` enforces preservation of earlier transitions +and amounts. Caller-supplied realization and source labels remain declarations, +not source admission, model acceptance or proof of a full execution. + +`CoveredWageHistory` sidecars remain unchanged and bound to their original +history prefixes. The wrapper does not widen them, infer coverage, or erase +independently sourced wages after a modeled death. Further source attachments +need their own explicit contract. No statutory arithmetic or accepted Axiom +bridge is provided. + +This prototype refuses births, entrants, resurrection, unrecorded attrition, +domain changes, mortality-model changes and extension beyond 2022. It retains +per-person nested history metadata for simplicity; this is not a population +storage format or a demonstration of the full registered projection. + +The existing assembly mortality callback also performs the first-period +`ensure_draw` reset and writes its optional collector. Replacing it with the +standalone observer loses that behavior; calling both mortality paths draws +twice. Assembly-aware integration remains separate work. Frozen engine +code, gates, inputs and historical evidence are unchanged. The new module's +explicit historical-source exclusion is guarded by the transitive import +reachability test. diff --git a/docs/design/compact-cohort-history.md b/docs/design/compact-cohort-history.md new file mode 100644 index 00000000..d7c65aea --- /dev/null +++ b/docs/design/compact-cohort-history.md @@ -0,0 +1,46 @@ +# Compact closed-cohort history storage + +The optional compact envelope stores an existing validated +`ClosedCohortEarningsHistory` without repeating its full identity map inside +every per-person `ForwardEarningsHistory`. It does not replace or reinterpret +either existing serializer. Legacy JSON bytes and legacy digests remain the +authoritative value representation for existing callers. + +Use `compact_history_to_json(history)` to produce the distinct +`populace_dynamics.compact_closed_cohort_history.v1` envelope and +`compact_history_digest(history)` to identify those exact compact bytes. +Compact and legacy digests are separate namespaces. + +Loading is explicit: + +```python +restored = compact_history_from_json( + compact_text, + baseline=trusted_initial_history, + expected_digest=trusted_compact_digest, + previous=optional_previous_history, +) +``` + +The loader requires the externally trusted baseline because that object owns +the one identity map used by all reconstructed children. The envelope and +each child retain the identity-map digest. Loading checks the exact envelope, +child, observation and transition fields; canonical integer and binary64 +representations; schema and status constants; provenance; baseline and map +digests; the expected compact digest; and a canonical byte-for-byte round +trip. It constructs `ForwardEarningsObservation`, `ForwardEarningsHistory`, +`MortalityStepObservation`, `CohortTransition` and +`ClosedCohortEarningsHistory` through their existing validators. When a +previous value is supplied, the existing extension validator runs unchanged. + +The serializer reads already validated typed fields directly. It never +creates a legacy child JSON document merely to remove its identity map. The +mortality representation was already externally identity-map bound and is +retained unchanged. + +This is storage normalization only. It performs no generator or model call, +source admission, coverage classification, observation arithmetic, or value +normalization. It does not make population-scale memory or runtime claims. +Bounded invented-record tests at 80 and 160 people verify that compact bytes +grow approximately linearly while the unchanged legacy envelope retains its +measured quadratic repeated-map term. diff --git a/docs/design/covered-wage-history.md b/docs/design/covered-wage-history.md new file mode 100644 index 00000000..3a75a5e2 --- /dev/null +++ b/docs/design/covered-wage-history.md @@ -0,0 +1,65 @@ +# Recording covered-wage observations + +`CoveredWageHistory` attaches explicit source observations to one exact +`ForwardEarningsHistory`. It records a separate concept: source-reported, +uncapped employee wages declared OASDI-covered. It does not classify generated +labor income, calculate creditable earnings or verify statutory coverage. + +Every person/year in the declared forward history must have one observation. +An amount requires a typed source value and receipt digest. An unavailable +amount requires a receipt digest and one reason: `source_field_unavailable`, +`coverage_unresolved` or `crosswalk_registration_required`. Omitting a row +refuses; it never creates a zero. A source-reported zero remains distinct from +unavailability. Odd-year labor carries do not create covered-wage observations. + +The observations are independent of the generator's labor concept. Covered +wages may differ from labor income, including when the person is outside the +forward generator's domain. This sidecar performs no reconciliation, component +allocation, copying of control zeros or change to the original history. + +## Source values and provenance + +`SourceAmount(logical_dtype, serialization)` supports `int64`, `uint64`, +`decimal` and `binary64`. Integer strings are canonical and range checked. +Decimal strings retain their fractional scale; binary64 retains canonical +`float.hex()` values, including signed zero. Negative, nonfinite and ambiguous +values refuse. No floating-point conversion of source integers or decimals, +monetary rounding, indexing or cap occurs. + +The sidecar inherits the history's explicit nominal measurement unit. The +caller must supply a source-contract digest declaring that unit and the +uncapped covered-employee-wage concept. Each observation's source digest must +identify a retained receipt binding the artifact, stable record/field locator, +original amount and dtype, reference year, unit and information date. The +module checks digest syntax; it does not read or admit those receipts. A +source declaration does not establish legal correctness or authorize use of +future information in a fit. + +## Exact history binding + +Construct `CoveredWageHistory(history, source_contract_digest, observations)` +with immutable `CoveredWageObservation` rows. `for_person` uses the history's +typed `PersonIdentityMap`, preserving source strings and large unsigned IDs. +`missing_coordinates` reports unavailable `(private key, year)` pairs. + +`to_json` and `digest` retain the exact external history digest, identity-map +digest, realization, unit, price basis, source contract and source values. +The loader requires `from_json(text, history=..., expected_digest=...)`; +the expected digest is optional, while the actual history is mandatory. +Rebinding the serialized observations to another draw, identity map, changed +history or appended history refuses. Duplicate members, unknown fields, +missing rows and inconsistent amount states also refuse. + +There is no implicit append or correction operation. Construct and retain a +new bundle for each explicit revision. Row order does not change its digest; +changing a source value's serialization or provenance does. + +## Limits + +Even a sidecar with no missing observations does not establish complete +OASDI coverage. The original history retains `not_materialized` coverage and +`registration_required` crosswalk status. Pre-2014 records, empirical source +admission, self-employment facts, full component reconciliation, event dates +and accepted Axiom input rules remain separate work. This module calculates +no benefits, payroll or population estimates and does not feed the frozen +historical estimates runner. diff --git a/docs/design/forward-earnings-history.md b/docs/design/forward-earnings-history.md new file mode 100644 index 00000000..17458368 --- /dev/null +++ b/docs/design/forward-earnings-history.md @@ -0,0 +1,75 @@ +# Recording annual forward earnings + +`ForwardEarningsHistory` records the existing generator's annual labor-income +outputs for one realization. Previously the projection kept current earnings +and a few private lag values; this opt-in observer retains a dated history +without changing those inputs, drawing randomness or computing policy. + +The first implementation covers the existing 2014–22 forward contract on a +fixed roster. It starts from an already materialized 2014 frame and appends +exactly one year at a time. The caller supplies the realization ID, generator, +source-contract and per-snapshot lineage digests, nominal measurement unit, +and exact identity manifest. Those digests identify the claimed inputs; they +do not verify a fitted artifact, grant source admission or register a crosswalk. + +The recorder consumes the existing `person_id`, `year`, `earnings` and +`earnings_domain` columns. IDs and years must already be int64, earnings +binary64, and domain membership a complete Boolean column. The private IDs +reverse through `PersonIdentityMap`, preserving original uint64 and string +identities. Row order is irrelevant; duplicate IDs, missing markers and +implicit dtype conversion refuse. + +## Values and missingness + +A supported zero is recorded as `known_zero`. The outside-domain wrapper's +control zero is recorded as `unavailable`, with no amount and reason +`outside_forward_earnings_domain`. A nonzero outside-domain value refuses; +the recorder cannot discard an unexplained amount. Missing or nonfinite +supported values also refuse, rather than creating a zero or a guessed reason. + +Amounts retain their exact `float.hex()` serialization, including the sign of +zero. There is no monetary rounding. This narrow version accepts the existing +generator's binary64 output only; integer, decimal, coverage-allocation and +signed self-employment concepts need separate source contracts. + +Supported 2014 observations are labeled `boundary_method`, subsequent even +years `biennial_draw`, and odd years `odd_year_carry`. Carries must preserve the +previous year's exact amount bits. These labels describe the producer's +declared schedule; they do not establish how an arbitrary caller produced a +frame. The private 2012 lag is never read or converted into a historical record. + +## Persistence and append + +The immutable history declares its roster and last reference year separately +from its observations. Every declared person must have exactly one row in +every year from 2014 through that end year. Removing a complete last year or +an entire person therefore cannot silently shrink the declared envelope. + +`append` returns a new history, preserving every prior row and metadata field. +It refuses overlap, gaps, roster/domain changes and extension past 2022. +Canonical JSON encodes private keys and years as decimal strings and amounts +as hexadecimal strings. The loader validates the embedded identity digest, +closed field set, dense envelope, amount values and carry semantics. An optional +expected digest verifies the whole history; an optional `previous` history +enforces the same append-only prefix and metadata binding after loading. + +## Integration and limits + +Invented tests run the actual `ForwardEarningsGenerator`, domain adapter and +`apply_earnings` functions with and without the observer, then compare frames. +Recording leaves the generated values and private state unchanged. Additional +cases exercise invalid states, source identities above signed int64, exact +floating-point transport, input-frame preservation and serialized refusals. + +The history always reports coverage as `not_materialized` and the official +crosswalk as `registration_required`. Known labor income is not evidence of +Social Security covered earnings. No AIME, benefit, payroll, national estimate +or empirical score is produced. This module does not replace the proposed +full OASDI history bundle: pre-2014 observations, changing rosters, death and +entry events, coverage classification and the accepted Axiom input contract +remain separate work. + +The historical estimates runner does not import this module. Its addition is +explicitly excluded from the historical source inventory, with the existing +transitive reachability test guarding that separation. Registered research +inputs, outputs, thresholds and generator implementation remain unchanged. diff --git a/docs/design/mortality-observer.md b/docs/design/mortality-observer.md new file mode 100644 index 00000000..c55334f1 --- /dev/null +++ b/docs/design/mortality-observer.md @@ -0,0 +1,76 @@ +# Mortality step observer + +`populace_dynamics.mortality_observer.observe_mortality` is an opt-in, +single-step helper around the existing `engine.steps.apply_mortality`. It +returns that function's survivor frame and an immutable +`MortalityStepObservation`. Registered projection and assembly code remain +unchanged; this is not automatically installed in `ProjectionEngine`. + +```python +survivors, record = observe_mortality( + frame, context, rng, + model=mortality_model, + identity_map=identity_map, + realization_id="caller-supplied-realization", + source_contract_digest=source_contract_digest, +) +restored = MortalityStepObservation.from_json( + record.to_json(), identity_map=identity_map, + expected_digest=record.digest, +) +``` + +The caller supplies an existing `AgeSexMortalityModel`, `PeriodContext` and +NumPy generator. Input `person_id`, `age` and `year` columns must have NumPy +`int64` dtype; floating ages such as `30.0` are deliberately rejected rather +than cast. Each private ID must be unique and already present in the explicit +`PersonIdentityMap`. Reverse mapping preserves typed signed/unsigned source +integers beyond `2**53` and textual identities without numeric coercion. + +Every frame year must immediately precede the target year. Ages must lie +between zero and the actual final inclusive model bound, and sex must be +`female` or `male`. The final band's `+` label does not imply an infinite +upper bound. The helper rejects unsupported ages before any draw; the +underlying model otherwise casts ages and leaves unmatched rows at zero +probability. It preserves deliberately supplied zero-probability cells, +including the registered model's separately documented under-25 pad. + +Registry-backed calls validate the draw index, period bounds, required +person ordinals and uniqueness of all supplied ordinals before requesting +any generator. Empty, correctly typed frames remain valid. Period indices +start at one; draw indices start at zero. The helper snapshots the model's +effective float probabilities and invokes the actual mortality function +exactly once with detached parameters. It does not fit a model or implement +an alternative mortality calculation. For valid inputs, survivor values, +ordering, index, dtypes and random draws match a direct call. + +The record contains pre-aging age and sex, the actual survived/died outcome +for every input person, target year, period and draw indices, registry size +and active-person ordinals when applicable, realization label, source +contract digest, effective model cells and identity-map binding. Its +`pre_keys` and `post_keys` properties derive the exact before/after ID sets. +Deceased identities remain reversible after the survivor frame drops them. +The model and observations are detached from mutable inputs. Model floats +use canonical hexadecimal strings; integer JSON fields use decimal strings. +The loader checks the external identity map before constructing rows and +rejects duplicate fields, extra fields, invalid types and inconsistent +model digests. Detecting otherwise valid edits requires a trusted expected +record digest. These digests establish identity, not source admission, +empirical validity or complete RNG replay provenance. Realization and source +contract labels are supplied by the caller, not verified against a whole run. + +A death outcome means removal at the mortality step targeting the recorded +projection year. It does not establish a death date, birthday, within-year +timing or exposure. The existing assembly already has an optional +`M6_DRAW_OUTPUTS_KEY` collector with mortality slices and a conventional +0.5 death exposure. That callback also manages `ensure_draw` state; replacing +it directly with this helper would discard that behavior. Assembly-aware +integration is separate work. + +This record neither changes a fixed-roster earnings history nor fills +post-death earnings with zero. A future longitudinal observer can retain +earlier histories for deceased persons and explicitly handle new entrants +and births. This slice provides no coverage inference, statutory arithmetic, +Axiom bridge, benefit or population-parity claim. Its explicit exclusion +from the historical source inventory is guarded by the transitive import +reachability test. diff --git a/docs/design/person-identity-map.md b/docs/design/person-identity-map.md new file mode 100644 index 00000000..2c7c1813 --- /dev/null +++ b/docs/design/person-identity-map.md @@ -0,0 +1,67 @@ +# Lossless person identity map + +`populace_dynamics.person_identity` is an opt-in bridge between explicit source +identities and private nonnegative signed-int64 keys. It does not connect a +population to an earnings generator or calculate policy outcomes. + +An identity contains its logical type (`int64`, `uint64` or `string`) and exact +value. Integer values never pass through floating point. Strings remain opaque: +`"1"`, integer `1`, and unsigned integer `1` are distinct identities; Unicode +normalization is not applied. Empty strings are preserved rather than treated +as missing. The producer must establish which source fields identify persons. +Household IDs and missing values are not substituted automatically. + +```python +from populace_dynamics.person_identity import PersonIdentity, PersonIdentityMap + +people = [ + PersonIdentity("uint64", 2**64 - 1), + PersonIdentity("string", "person-A"), +] +original = PersonIdentityMap.from_identities(people) +keys = original.map_rows(people) +assert original.reverse_rows(keys) == tuple(people) + +extended = original.append([PersonIdentity("string", "new-entrant")]) +assert extended.map_rows(people) == keys +loaded = PersonIdentityMap.from_json( + extended.to_json(), + expected_digest=extended.digest, + previous=original, +) +assert loaded == extended +``` + +Each admission batch is sorted lexically by logical type and canonical source +value. The first batch receives keys starting at zero. Later batches append +keys and never renumber existing entries, even when an entrant sorts before an +existing person. Input order within a batch does not affect the map. Admission +batch history does affect it and must remain pinned for reproducible careers. +Duplicate identities within an admission or already present in the map refuse; +repeated observations in `map_rows` are allowed and retain their row order. + +The canonical JSON manifest encodes both original integer values and private +keys as decimal strings, avoiding precision loss in JSON consumers. It rejects +duplicate or unknown fields, noncanonical integers, unsupported logical types, +duplicate identities and non-dense keys. The SHA-256 digest covers canonical +UTF-8 JSON; insignificant input whitespace is not part of that identity. + +On intake, use a separately pinned `expected_digest`. For an extension, also +provide the previous accepted map: `previous` checks that every old entry is +preserved. A new JSON document cannot prove its own provenance merely by +carrying a matching digest. Neither this digest nor successful loading admits +a baseline, certifies source person identity, or proves historical record +linkage. + +All values are immutable snapshots. Accepted string subclasses are copied to +built-in strings, preserving their underlying text rather than overridden +rendering methods. Mapping refuses unknown identities, and reverse mapping +refuses unknown or coerced keys. +Identity, entry and prior-manifest boundaries require the declared value +classes; subclasses cannot supply mutable overridden identity semantics. +The map class cannot be subclassed, so its factories and digest verification +cannot dispatch to subclass overrides. +The original generator's integer handling is unchanged. A later adapter must +explicitly use these +private keys and restore the exact original identity on export; until then, +this module is a tested standalone prerequisite only. diff --git a/docs/entrant-seam.md b/docs/entrant-seam.md new file mode 100644 index 00000000..6ec8af94 --- /dev/null +++ b/docs/entrant-seam.md @@ -0,0 +1,85 @@ +# Experimental entrant schedule and support restrictions + +This source slice recovers the two engine modules and synthetic tests from +local commit `61bbf1c7e25a7e55033c134bbc2e846022b8850b`. It accepts an explicitly +supplied donor frame and annual controls in thousands. It does not include the +original native donor/control readers, snapshots, build script, or run artifact. +The original frame-reader round-trip test is deferred with that source path. + +`build_entrant_schedule` returns frames for the existing +`metadata[SCHEDULED_ENTRIES_KEY]` interface, plus arithmetic alignment and +provenance records. Positive-weight donors are reweighted for each positive +control. Zero controls remain in alignment but produce no frame and consume +no person IDs; zero-weight donors do not become demographic actors. The input +boundary rejects noninteger ages/activation keys and ambiguous boolean flags. +Controls are validated before cohort allocation begins. + +The schedule retains the historical gross-positive-inflow convention. That +control convention is not a count of observed border arrivals, a net migration +law, or a complete population stock model. The recorded donor composition and +control provenance describe caller-supplied inputs; this interface does not +admit their sources or fit/calibrate a population. + +## Existing loop timing + +A cohort scheduled for year Y carries `year=Y-1` and `age=entry_age`. +The existing loop activates it before mortality, then increments age. Its +activation-year slice therefore carries `entry_age+1`. This is the existing +engine convention, not a newly established arrival-exposure assumption. +The caller must reserve real IDs in the shared allocator. The loop rejects +duplicate scheduled IDs and builds stable person ordinals across cohorts. + +## Explicit support boundaries + +`EntrantClaimingAdapter(step)` calls a supplied claiming adapter only on +incumbents. It rejects existing entrant claim ages, claim years, claimed state, +or disability-conversion events before executing the incumbent step. Such +observations need a separately admitted claim-history path. The wrapper does +not erase possible observed entitlement or declare those people ineligible. +Supported exclusion inputs retain missing plans/years and a structural false +claiming flag. With only entrants, the incumbent step is not called. + +Explicit synthetic rows require a known `entry_kind`; losing or misspelling +that marker is an error. Legacy closed panels without synthetic markers retain +their incumbent interpretation. Birth and realized-opener markers remain +distinct from immigrant cohorts. Provenance counts reject missing/unknown kinds. + +The historical birth materializer leaves the new provenance column missing. +For this experimental path, `materialize_births_with_provenance` delegates to +that actual materializer and labels only the children it just allocated. It +validates existing rows first; it cannot retrospectively relabel unidentified +synthetic rows. It accepts supplied birth records and does not establish a +fitted fertility adapter or enforce the fertility risk-set restriction. + +`suppress_entrant_benefit_outputs` marks specified unsupported outputs missing. +It does not calculate benefits. Consumers must report the unsupported population +separately; missing benefits must not become zero benefits in an average or be +used to label a partial-population result national. + +The fertility/disability ID helpers and earnings-domain assertion express +support restrictions. `exclusion_report` is explicitly `inventory_only` with +`execution_verified=False`; it does not claim those restrictions were applied +in a projection. The actual claiming wrapper enforces its own narrower contract. +The historical fertility function treats an empty `holdout_ids` set as all +roster IDs and ignores that argument in its precomputed-birth branch. An ID-set +complement alone therefore cannot enforce the entrant fertility restriction. + +## Validation and remaining work + +Synthetic tests exercise the real scheduled-entry loop, cohort extinction and +later activation, mortality-before-ageing order, positive/zero cohort weights, +claiming exclusion, missing benefit outputs, and incumbent random consumption. +The mortality and other fitted transitions in those integration fixtures are +explicitly synthetic. Placeholder benefit values test output suppression only; +no policy calculation, Axiom runtime, or national score is claimed. + +The historical registered assembly, engine steps, and scientific gates remain +unchanged. Exact historical source exclusions are justified by assertions that +these two experimental modules remain unreachable from the birth-evidence +reducer and its registered input roots. + +An entrant-supported Social Security score still requires admitted donor and +control sources, migration-universe/exposure decisions, covered-work histories, +insured-status and benefit support, and coherent family, disability, earnings, +and other post-entry transitions. This recovery addresses an engineering +boundary; it does not establish DynaSim parity. diff --git a/docs/population-graph.md b/docs/population-graph.md new file mode 100644 index 00000000..4d873d8a --- /dev/null +++ b/docs/population-graph.md @@ -0,0 +1,227 @@ +# First population graph: mortality and ageing + +The optional `populace_dynamics.graph` package fits the existing M6 mortality +model on a historical synthetic panel, applies the fitted artifact to a +separate starting population, and adds surviving observations for the next +year, or repeats those transitions through an explicit end year. It produces +an engineering report and a content-addressed execution +manifest. It does not change the existing projection loop, candidate +registries, scientific gates, or committed evidence. + +## Dependencies and execution + +The graph example requires Python **3.13 or 3.14**, NumPy 2+, pandas 2.3+, +and the Microcosm graph/frame revisions containing typed artifact edges and +`microcosm.graph.randomness.keyed_uniform`. The default Dynamics installation +continues to support Python 3.10–3.14 without importing Microcosm. The entry +point checks capabilities and gives installation guidance when they are absent. + +The `graph` extra pins both graph and Frame to core commit +`3ff92b0aea14407d09479bff5623dc7d1a92d008`. An older package merely sharing +the version number `0.1.0` is insufficient. In an isolated Python 3.13 or 3.14 +environment, install with `uv pip install '.[graph]'`. The core change must be +reviewed before this dependent integration is released; replace the Git pins +with a compatible published release when one exists. CI installs this exact +extra, refuses missing capabilities, and runs the integration on both supported +Python versions. Do not modify the existing scientific gate environment. No +rules engine, restricted microdata, or optional forest fitter is needed here. + +Run from that environment, choosing an output directory: + +```sh +python -m populace_dynamics.graph --synthetic --output-dir ./mortality-example +``` + +The command creates small synthetic inputs under `mortality-example/inputs`. +It preserves existing input files so that edits can test cache invalidation. +Repeated execution reuses the verified store under `mortality-example/store`. +The report, manifest, fitted JSON model, entity tables, and next-period slice +are written inside the chosen output directory. A failed engineering or +fixture verdict exits nonzero while retaining the diagnostics. + +If malformed held-out input prevents evaluation, the executor records a failed +`evaluate` gate. The command exports that manifest and a diagnostic report with +the gate's name, outcome and original exception evidence on both cold and warm +runs, then exits nonzero. The report marks `engineering_verdict` and +`fixture_verdict` as `not_evaluated`; it does not invent evaluated deaths or +mass metrics. Successful evaluation retains the existing report fields. + +Four explicit source paths can replace the generated inputs: + +```sh +python -m populace_dynamics.graph \ + --training ./inputs/training.json --rates ./inputs/rates.json \ + --initial ./inputs/initial.json --holdout ./inputs/holdout.json \ + --boundary-year 2014 --external-vintage-year 2014 \ + --experiment-id comparison-a --replicate 0 --base-seed 0 \ + --output-dir ./mortality-example +``` + +These inputs still exercise the synthetic engineering contract. The example +does not confer validity on a real-population projection. Source JSON rejects +duplicate members and nonfinite values. Each source is declared separately; +holdout bytes are available only to evaluation. Domain kernels read the +content-verified JSON directly: the registered source marker deliberately +does not pretend an external rate table or a holdout report is a population. + +## Executable ownership + +The graph has two CREATE roots, each carrying a `person_period` observation +entity and `person` and `period` groups. `person` retains stable identities; +`period.period` is the immutable mass-partition label. The training root +contains exposure records; the initial root contains recipients. Their only +connection is the explicitly typed mortality-model artifact. + +The fit node calls `prepare_mortality_refit_inputs` and +`fit_mortality_model`. Event year, required interview year, and declared +external vintage retain the existing cutoff checks. The JSON model contains +validated contiguous age bands, sex-specific probabilities, fit boundary, +external vintage, and retained row count. The manifest binds its producer +to source identities and implementation digests. The fitter's external-rate +factor cancels in its fitted-window level, so this is not evidence of +independent external calibration. + +Application calls `apply_mortality` with a graph-specific context. Every +uniform is keyed by the original person identity, process, year, and draw +index under the chosen experiment/replicate/seed. It does not use the legacy +ID-sorted ordinal registry. Reordering, splitting, or adding unrelated +people preserves the existing people's draws. Fit and application declare +platform-specific bitwise numeric behavior conservatively; cross-platform +equality is not claimed. + +EXPAND calls `advance_age` on survivors, adds their next-period observations +with lineage to the original observations, and attaches them to one newly +admitted period group. A same-version rewrite node claims the materialized +age values. Historical ages and memberships remain unchanged. The temporary +`year` returned by `advance_age` is never written over a carried observation +column. No new person, birth, or immigrant is implied by admission of the +period group. + +Typed person-period weights are the single authority. Every survivor carries +the same trajectory weight into the new period. The declared mass receipt +shows historical mass unchanged and new-period mass equal to surviving +weight. Total stored observation mass therefore grows by the additional +period. If everyone dies, the graph adds no observations and no orphan period +group; the report explicitly records next-period mass zero. + +## Evaluation and limits + +The report separates `engineering_verdict` (survivor/age parity and population +structure) from `fixture_verdict` (the independently sourced synthetic +death-rate and age expectations). It records weighted expected, observed, +and generated deaths, row counts, period mass, node/model identities, and +cache reuse. The fixture death-rate tolerance is an input named +`fixture_max_abs_death_rate_gap`; it is not a scientific acceptance threshold. +Changing all held-out outcomes to deaths fails that fixture check while +leaving fitting, application, draws, and accounting unchanged. + +Household accounting is explicitly unsupported and refused by the Python +entry point. Household weight sharing, marriage, births, immigration, +alignment replay, and the full M6 loop remain later work. +No certified data release or scientific candidate is produced by this graph. + +## Annual trajectories + +The optional `run_mortality_trajectory` Python entry point builds one graph +with a single mortality fit and separate application, expansion, age-ownership, +snapshot, and evaluation nodes for each year. It uses the same exact +graph/Frame pin as the one-year example. The fit cutoff stays fixed while the +application year +advances. This extends execution of the existing age/sex law; it does not add +a calendar-year mortality improvement model or establish long-horizon validity. + +Each application reads only the preceding period's observations. A typed +transition artifact binds each probability and survival decision to its +person and observation identities. EXPAND appends survivor observations with +lineage to that preceding period. Earlier ages, memberships, and trajectory +weights stay unchanged. The mass receipt covers every historical period, +not just the newest pair. After extinction, later years contain no at-risk +people and add no orphan period groups. + +Declare one aggregate synthetic holdout for each application year. For example: + +```python +import json +from pathlib import Path + +from populace_dynamics.graph import run_mortality_trajectory +from populace_dynamics.graph.synthetic import write_synthetic_inputs + +root = Path("mortality-trajectory") +sources = write_synthetic_inputs(root / "inputs") +sources.pop("holdout") +holdouts = {} +for year in range(2015, 2018): + path = root / "inputs" / f"aggregate-{year}.json" + path.write_text(json.dumps({ + "scope": "synthetic_engineering", + "year": year, + "expected_death_rate": 0.2, + "fixture_max_abs_death_rate_gap": 0.25, + })) + holdouts[year] = path + +result = run_mortality_trajectory( + **sources, holdouts=holdouts, end_year=2017, output_dir=root, +) +print(result.report) +``` + +These small aggregate fixtures are deliberately artificial, with input +tolerances used only for engineering tests. They contain no empirical +acceptance targets. Each evaluation reads a typed snapshot of the actual +materialized population on a separate population version. This keeps the +evaluation outside the next expansion's dependencies under the pinned core. +An annual evaluation depends on its own holdout; changing +or failing that evaluation does not alter later simulation. Extending the +horizon reuses the existing fit and annual nodes in the same verified store. +Changing experiment, replicate, or seed changes application identities while +reusing the fit. All sources remain declared and content-hashed by the executor, +including evaluation sources whose kernels are subsequently guarded. + +The output directory contains `trajectory.csv`, `model.json`, `report.json`, +and `manifest.json`. The trajectory includes the initial period and every +completed survivor period, with person identity, age, year, and weight. +Annual reports keep expected and generated deaths, survivor counts, and +period mass separate from fixture and engineering verdicts. + +Application ages outside the fitted bands must fail explicitly. In particular, +a survivor aged 120 can be advanced to 121, but cannot enter another mortality +draw under a law with support ending at 120. The graph does not silently assign +such people a zero death probability. A typed failure outcome guards later +applications and expansions, preserving the latest valid population and the +original diagnostic. Blocked application status is reported separately from +the core's execution/cache receipts: this pinned executor still runs guarded +nodes and does not provide native `unreached` receipts. A failed evaluation +does not propagate this application block. + +Snapshots include the full materialized history, so their storage grows with +both population size and horizon. This synthetic integration has not been +benchmarked for national-scale projection. Root creation, fitting, store +corruption, and unexpected structural failures can still abort execution; +the retained diagnostic path covers application and evaluation failures. + +## Tests + +```sh +python -m pytest -q tests/test_graph_mortality.py \ + tests/test_graph_mortality_trajectory.py \ + tests/test_m6_engine_refit.py tests/test_m6_engine_steps.py +``` + +The integration tests cover direct execution with an independently injected +uniform vector, JSON validation, cutoff and holdout isolation, fitted-artifact +reuse, changed fitting weights, row/chunk/person invariance, cold/warm stores, +and zero/all-survivor expansion. Independent graph-to-direct model parity and +future-event/late-interview mutations protect cutoff mapping. Nondefault +experiment, replicate and seed tests independently derive draw coordinates and +check direct-step parity, fit reuse and changed application identity. They skip +the optional runtime cases when the required core capabilities are unavailable; +the JSON and dependency +boundary tests still run. Importing `populace_dynamics.graph` remains safe +under Python 3.10–3.12. + +The annual tests independently repeat the existing fit, mortality, ageing, +and keyed-draw operations; compare every retained person-period and weighted +diagnostic; and exercise horizon reuse, stream changes, holdout isolation, +extinction, and retained support-failure evidence. diff --git a/docs/stock-flow-accounting.md b/docs/stock-flow-accounting.md new file mode 100644 index 00000000..37723b9c --- /dev/null +++ b/docs/stock-flow-accounting.md @@ -0,0 +1,157 @@ +# Annual population stock-flow accounting + +`populace_dynamics.engine.accounting.reconcile_period` checks whether one +annual transition's opening population, declared arrivals and departures, +and closing population reconcile. It reports person counts, weight flows, +weight changes, and arithmetic residuals without altering either frame. + +This interface is experimental and opt-in. It does not change the historical +projection engine, generate demographic events, or establish an admitted +population. Its status is `engineering-accounting-coherence-only`, and its +interface version is `stock-flow-accounting/0.1.0-experimental`. No scientific +tolerance or acceptance gate is added. + +## Input contract + +```python +reconcile_period( + opening, + closing, + *, + opening_year, + closing_year, + additions=(), + exits=(), +) +``` + +Both pandas frames require `person_id`, `year`, and `weight` columns. +Identifiers are unique within each frame; IDs and years must be integers +within the signed int64 range. Floats and booleans are rejected as IDs or +years. These are the accountant's own validation rules, which are stricter +than the historical loop's slice check. + +Weights must be real numeric, finite and nonnegative. Strings, complex +values, and booleans are rejected before conversion to binary64. Negative +weights are checked before conversion; nonzero values that underflow to zero +in binary64 are rejected. Zero-weight +rows remain visible in counts. Empty frames still require the three columns, +but empty columns may have any dtype. Each nonempty frame must carry its +stated year, and `closing_year` must equal `opening_year + 1`. + +`additions` and `exits` are sequences of `PopulationEvent` declarations. +Each event has an integer `person_id`, `kind`, closing `year`, optional +`weight`, and optional string `reason` and `source`. + +| Addition kinds | Exit kinds | +|---|---| +| `birth` | `death` | +| `scheduled_entry` | `emigration` | +| `other_entry` | `other_exit` | + +The two `other_*` kinds require a nonempty reason. At most one addition and +one exit are supported for a person in a period, with arrival before exit. +There is no finer event-timing model. An addition cannot collide with the +opening roster, and a declared departure cannot remain in the closing roster. +Unexplained changes in endpoint membership and duplicate declarations fail. +The accountant never infers that a disappearing person died or that a new +identifier represents an immigrant. + +## Counts and weights + +Counts reconcile exactly: + +```text +closing = opening + additions_total - exits_total +``` + +People who arrive and depart in the same period are counted in both flows. +These transients appear in neither endpoint frame and must have explicit +weights on both declarations. Otherwise, an omitted arrival weight uses the +closing-frame weight; an omitted departure weight uses the opening-frame +weight. These conventions are recorded in provenance through the counts of +explicit event weights. + +Weight accounting reports: + +```text +reconstructed_closing = opening + additions_total - exits_total + revaluation +weight_residual = closing - reconstructed_closing +``` + +| Revaluation component | Difference summed over the relevant people | +|---|---| +| `carried` | Closing weight minus opening weight for survivors | +| `entrant` | Closing weight minus declared arrival weight | +| `exiting` | Declared departure weight minus opening weight | +| `transient` | Declared departure weight minus declared arrival weight | + +Weights are never rebalanced. Sums use `math.fsum` over binary64 components; +subtraction and component totals still involve rounding. Nonzero arithmetic +residuals are reported without a tolerance-based verdict. An intermediate or +summary that cannot be represented with finite binary64 arithmetic raises a +`PopulationAccountingInputError`. + +## Example + +```python +import pandas as pd +from populace_dynamics.engine.accounting import PopulationEvent, reconcile_period + +opening = pd.DataFrame({ + "person_id": [1, 2, 3], "year": [2020] * 3, + "weight": [10.0, 20.0, 30.0], +}) +closing = pd.DataFrame({ + "person_id": [1, 2, 90], "year": [2021] * 3, + "weight": [11.0, 20.0, 5.0], +}) +account = reconcile_period( + opening, closing, opening_year=2020, closing_year=2021, + additions=[PopulationEvent(90, "birth", 2021, source="synthetic.birth")], + exits=[PopulationEvent(3, "death", 2021, source="synthetic.mortality")], +) +assert account.weights.closing == 36.0 +assert account.weights.revaluation.carried == 1.0 +assert account.count_residual == 0 +assert account.weight_residual == 0.0 +``` + +The weight identity is `60 + 5 - 30 + 1 = 36`. `account.to_dict()` returns +JSON-serializable counts, weights, residuals, and provenance. Person-ID tuples +are available as attributes and are omitted from this summary. Serialized +provenance is isolated from the immutable account and other serializations. + +Malformed inputs raise `PopulationAccountingInputError`. Well-formed inputs +whose declarations conflict with the frames raise +`PopulationReconciliationError`, with typed `.discrepancies` and a `.to_dict()` +representation. Validation stops at the first failing stage: frames, +declarations, reconciliation, then weights. + +## Projection-loop integration and limits + +The caller must capture declarations from its adapters or supplied schedule. +The loop activates scheduled entries before mortality in the wave ending in +`Y`, while entry frames carry `Y - 1`. Their declarations must use `Y` and +explicit weights so an entrant who dies in the same wave can be accounted for. +Birth and mortality declarations should come from the operations that perform +those transitions. An endpoint difference alone cannot establish their cause. + +Tests in `tests/test_m6_stock_flow.py` drive the real `ProjectionEngine` with +synthetic recording adapters, then reconcile adjacent output slices. They +include a birth, a death, and a scheduled entrant who dies before the wave +closes. No fitted transition law or native population is used. + +Accounting coherence does not verify event-log completeness. If both records +for a transient are omitted, the endpoint frames cannot reveal the omission; +provenance records `event_log_completeness_verified=False`. Event reasons are +also caller assertions. The accountant has no cross-period memory, so a past +ID explicitly declared as a new addition can be reused without detection. +Only `weight` is reconciled; `start_weight` and other frame columns are ignored. + +The module directly imports only the standard library, NumPy, and pandas. +Normal package import also executes the historical engine initializer and its +broader source dependencies. The historical source-identity test verifies that +this new module remains unreachable from the sealed projection roots; the +static guard covers ordinary imports and explicitly listed dynamic roots, not +arbitrary runtime imports. This page is not added to the Quarto chapter list. diff --git a/docs/trajectory-accounting.md b/docs/trajectory-accounting.md new file mode 100644 index 00000000..16dc8ded --- /dev/null +++ b/docs/trajectory-accounting.md @@ -0,0 +1,89 @@ +# Accounting for annual mortality transitions + +The optional `populace_dynamics.graph.trajectory_accounting` module adds one +accounting node per year to the existing synthetic mortality/ageing graph. +It executes the real Microcosm DAG with the same reviewed graph/Frame pin. +It does not change fitted laws, draws, population frames, weights, scientific +gates, or the historical projection engine. + +```python +from populace_dynamics.graph.trajectory_accounting import ( + run_accounted_mortality_trajectory, +) + +result = run_accounted_mortality_trajectory( + training="synthetic-training.json", + rates="synthetic-rates.json", + initial="synthetic-initial.json", + holdouts={2015: "synthetic-annual-2015.json"}, + end_year=2015, + output_dir="synthetic-accounting-output", +) +``` + +These paths must contain the exact synthetic contracts documented in +[the population graph and mortality trajectory](population-graph.md). +The optional module +requires Python 3.13 or later and the reviewed graph dependencies. Importing +the ordinary `populace_dynamics.graph` package remains lazy and unchanged. +`build_accounted_trajectory_graph` also exposes the declaration and kernel +registry for callers using the executor directly. + +## Declared events and frozen snapshots + +Each `account_YEAR` node reads only the corresponding typed mortality +transition and frozen population snapshot. It runs against the graph's +separate training population version without reading that population's +columns. It has no holdout, model, source, or RNG input. Accounting is not a +prerequisite of any subsequent mortality transition. + +The adapter validates the transition's person and observation bindings and +the snapshot's year, row, period, and weight structure. It copies the opening +and closing rows, preserving the supported columns and weight positions. +Only transition records declaring `survives=false` produce death events. +Missing or additional endpoint people cannot supply their own event causes. +No births, migration, or other entry/exit events are assumed. + +Completed transitions call the +[annual stock-flow accountant](stock-flow-accounting.md). Count conservation +is exact; weight residuals and revaluation components are reported without +an acceptance threshold. A changed survivor weight can therefore produce a +complete account while the original mortality evaluation independently +fails its unchanged-weight check. Accounting completion establishes neither +scientific acceptance nor completeness or truth of the declared event log. +The adapter cannot detect an arbitrary same-length rearrangement of supplied +weights without independent binding evidence; it accounts for the supplied, +content-addressed snapshot. + +After complete extinction, subsequent completed empty periods receive explicit +zero-to-zero accounts even though the population contains no new period +groups. A failed or blocked mortality transition instead produces +`account=null` and `accounting_status=not_evaluated`, retaining its diagnostic +and last completed year. It never turns a stale snapshot into deaths. +Malformed inputs or reconciliation refusals produce a separate failed +accounting artifact; original mortality and evaluation receipts remain intact. + +The runner writes `accounting-report.json` and the actual `manifest.json`. +Its `AccountedTrajectoryRun` contains those accounting summaries and the +manifest. It does not rewrite the original runner's `report.json`, model, or +trajectory exports. The manifest retains original application/evaluation +receipts and artifact references; an accounting status is not their rollup. +Source hashes cover the accountant and the reused snapshot/transition helpers. + +## Household and location contract still required + +The current initial source accepts exactly `person_id`, `age`, `sex`, and +`weight`. Its snapshot accepts only the existing person-period identity, +age, sex, period, and weight structure. Additional atomic-location or +household-link columns are refused, not silently discarded. This integration +does not yet transport household location or membership. + +A separate extension must preserve household atomic-location columns **and +household/member links** in both source and snapshot contracts. Microcosm +assigns the household anchor once **before support clones are created**; +clones and Dynamics inherit it. Larger geographies must derive from that +anchor through the **same versioned mapping**. Location may change only +through a separate declared mobility or migration event. Accounting must not +allocate locations, create independent geography assignments, or infer a move +from a roster difference. That extension needs its own schema, lineage, +membership, mapping-version, and declared-event tests. diff --git a/pyproject.toml b/pyproject.toml index 370d161a..3bdf706c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,12 @@ dependencies = [ ] [project.optional-dependencies] +# Typed artifacts are not yet in the published 0.1.0 packages. Pin both +# graph and Frame to the same reviewed core revision; this extra needs 3.13+. +graph = [ + "microcosm-frame @ git+https://github.com/PolicyEngine/microcosm.git@3ff92b0aea14407d09479bff5623dc7d1a92d008#subdirectory=packages/microcosm-frame", + "microcosm-graph @ git+https://github.com/PolicyEngine/microcosm.git@3ff92b0aea14407d09479bff5623dc7d1a92d008#subdirectory=packages/microcosm-graph", +] dev = [ "pytest>=7.4.0", "black>=23.7.0", diff --git a/scripts/first_estimates_birth_evidence.py b/scripts/first_estimates_birth_evidence.py index efae0662..5967bea3 100644 --- a/scripts/first_estimates_birth_evidence.py +++ b/scripts/first_estimates_birth_evidence.py @@ -156,6 +156,38 @@ Path("src/populace_dynamics/estimates/anchor_context_registry.py"), Path("src/populace_dynamics/estimates/anchor_context_rehearsal.py"), Path("src/populace_dynamics/estimates/anchor_context_report.py"), + # The opt-in graph integration is outside the historical reducer and + # registered production call paths. Keep exact file exclusions, with + # import-reachability coverage, rather than changing any evidence pin. + Path("src/populace_dynamics/graph/__init__.py"), + Path("src/populace_dynamics/graph/__main__.py"), + Path("src/populace_dynamics/graph/_compat.py"), + Path("src/populace_dynamics/graph/model.py"), + Path("src/populace_dynamics/graph/runtime.py"), + Path("src/populace_dynamics/graph/synthetic.py"), + Path("src/populace_dynamics/graph/trajectory.py"), + Path("src/populace_dynamics/graph/trajectory_accounting.py"), + # This opt-in accountant is unreachable from the historical projection. + # The existing engine loop, steps, and package initializer remain sealed. + Path("src/populace_dynamics/engine/accounting.py"), + # This explicit opt-in successor is outside the reviewed projection + # call graph. Keep the historical steps.py implementation in the seal. + Path("src/populace_dynamics/engine/claiming.py"), + # Experimental entrants are outside the registered projection call graph. + Path("src/populace_dynamics/engine/entrant_schedule.py"), + Path("src/populace_dynamics/engine/entrant_domains.py"), + # The opt-in accounting-history validator is likewise unreachable. + Path("src/populace_dynamics/engine/accounting_history.py"), + # The opt-in identity transport is not imported by the historical reducer. + # Its exclusion remains guarded by the transitive reachability test. + Path("src/populace_dynamics/person_identity.py"), + # Opt-in history recorders and observers sit outside the historical loop. + Path("src/populace_dynamics/forward_earnings_history.py"), + Path("src/populace_dynamics/covered_wage_history.py"), + Path("src/populace_dynamics/mortality_observer.py"), + Path("src/populace_dynamics/closed_cohort_history.py"), + Path("src/populace_dynamics/assembled_history_observer.py"), + Path("src/populace_dynamics/compact_cohort_history.py"), ) POST_REVIEW_SHARED_SOURCE_BLOBS = { Path( diff --git a/src/populace_dynamics/assembled_history_observer.py b/src/populace_dynamics/assembled_history_observer.py new file mode 100644 index 00000000..2aa588d8 --- /dev/null +++ b/src/populace_dynamics/assembled_history_observer.py @@ -0,0 +1,915 @@ +"""Opt-in, retrospective histories from unchanged assembled projections. + +The observer performs no model evaluation. Fertility boundary capture is a +separately instrumented execution and never exposes private maternal draws. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from collections.abc import Mapping +from dataclasses import dataclass, replace + +import numpy as np +import pandas as pd + +from .closed_cohort_history import ClosedCohortEarningsHistory +from .engine.loop import PeriodModules, ProjectionResult +from .engine.rng import ProjectionModule +from .forward_earnings_history import ForwardEarningsHistory +from .mortality_observer import ( + MortalityModelSnapshot, + MortalityObservation, + MortalityStepObservation, +) +from .person_identity import PersonIdentity, PersonIdentityMap + +_MODES = {"strict_full_roster", "original_2014_view"} +_SCHEMA = "assembled-history-observation/v1" +_AUDIT_KEYS = { + "mode", + "baseline_digest", + "history_digest", + "identity_bindings", + "initial_ids", + "scheduled", + "periods", + "excluded", + "input_digests", +} + + +def _json(value: object) -> str: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ) + + +def _load(text: str): + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError("duplicate JSON field") + result[key] = value + return result + + def nonfinite(value): + raise ValueError("nonfinite JSON number") + + return json.loads(text, object_pairs_hook=pairs, parse_constant=nonfinite) + + +def _hash(text: str) -> str: + return hashlib.sha256(text.encode()).hexdigest() + + +def _digest(value: object) -> None: + if ( + type(value) is not str + or len(value) != 64 + or any(c not in "0123456789abcdef" for c in value) + ): + raise ValueError("canonical SHA-256 required") + + +def _int(value: object) -> int: + if isinstance(value, (bool, np.bool_)) or not isinstance( + value, (int, np.integer) + ): + raise ValueError("exact integer required") + return int(value) + + +def _ids(values) -> tuple[int, ...]: + result = tuple(_int(x) for x in values) + if any(x < -(2**63) or x >= 2**63 for x in result): + raise ValueError("native identity must fit signed int64") + if len(set(result)) != len(result): + raise ValueError("duplicate native identity") + return tuple(sorted(result)) + + +def _strings(values) -> list[str]: + return [str(x) for x in sorted(values)] + + +def _parse_ids(values) -> tuple[int, ...]: + if type(values) is not list: + raise ValueError("ID list required") + result = [] + for value in values: + if type(value) is not str or str(int(value)) != value: + raise ValueError("canonical integer string required") + result.append(int(value)) + if tuple(result) != _ids(result): + raise ValueError("sorted unique IDs required") + return tuple(result) + + +def _keys(value, expected): + if type(value) is not dict or set(value) != set(expected): + raise ValueError("unexpected document fields") + + +def _arrays(value, width: int) -> list: + # A digit string or an object also unpacks, so "100" would read as + # ["1", "0", "0"]: a second valid encoding of the same audit row. + if type(value) is not list or any( + type(row) is not list or len(row) != width for row in value + ): + raise ValueError("audit rows must be fixed-width arrays") + return value + + +def _scalar(value): + if value is None: + return ["null", None] + if value is pd.NA: + return ["NA", None] + if isinstance(value, (bool, np.bool_)): + return ["bool", bool(value)] + if isinstance(value, (int, np.integer)): + return ["int", str(int(value))] + if isinstance(value, (float, np.floating)): + return ["float", float(value).hex()] + if isinstance(value, str): + return ["str", value] + raise ValueError("unsupported frame scalar for exact input binding") + + +def _frame_json(frame: pd.DataFrame) -> str: + if ( + type(frame) is not pd.DataFrame + or not frame.columns.is_unique + or any(type(c) is not str for c in frame) + ): + raise ValueError("ordinary frame with unique string columns required") + return _json( + { + "columns": [[c, str(frame[c].dtype)] for c in frame], + "rows": [ + [_scalar(x) for x in row] + for row in frame.itertuples(index=False, name=None) + ], + } + ) + + +def _frame_from_json(text: str) -> pd.DataFrame: + doc = _load(text) + + def decode(cell): + kind, value = cell + return { + "null": lambda: None, + "NA": lambda: pd.NA, + "bool": lambda: value, + "int": lambda: int(value), + "float": lambda: float.fromhex(value), + "str": lambda: value, + }[kind]() + + return pd.DataFrame( + { + c: pd.Series([decode(row[i]) for row in doc["rows"]], dtype=dtype) + for i, (c, dtype) in enumerate(doc["columns"]) + } + ) + + +def _rows(frame: pd.DataFrame, *, year: int | None = None) -> dict[int, dict]: + required = {"person_id", "age", "sex", "weight"} + if year is not None: + required.add("year") + if ( + type(frame) is not pd.DataFrame + or not frame.columns.is_unique + or not required.issubset(frame) + ): + raise ValueError("complete unique demographic frame required") + if frame.person_id.dtype != np.dtype("int64"): + raise ValueError("native person_id must preserve int64") + keys = _ids(frame.person_id) + if len(keys) != len(frame): + raise ValueError("duplicate person rows") + result = {} + for row in frame.to_dict("records"): + key = _int(row["person_id"]) + age = _int(row["age"]) + if age < 0 or row["sex"] not in ("female", "male"): + raise ValueError("invalid mortality demographics") + if isinstance(row["weight"], (bool, np.bool_)) or not isinstance( + row["weight"], (float, int, np.number) + ): + raise ValueError("numeric weight required") + if not math.isfinite(float(row["weight"])) or row["weight"] < 0: + raise ValueError("invalid weight") + if year is not None and _int(row["year"]) != year: + raise ValueError("frame year mismatch") + result[key] = row + return result + + +def _same_demographics(left: dict, right: dict, *, aging: int = 0): + for key in left: + a, b = left[key], right[key] + if ( + a["age"] + aging != b["age"] + or a["sex"] != b["sex"] + or _scalar(a["weight"]) != _scalar(b["weight"]) + ): + raise ValueError("demographic continuity mismatch") + + +@dataclass(frozen=True) +class FertilityBoundary: + """Exact value/dtype copies of one actual callback boundary.""" + + draw_index: int + period_index: int + year: int + before_json: str + after_json: str + + +class FertilityCapture: + """Single-invocation capture; failure and attempted reuse are permanent.""" + + def __init__(self): + self._records: list[FertilityBoundary] = [] + self._failed = False + self._draw: int | None = None + self._periods: int | None = None + + @property + def records(self) -> tuple[FertilityBoundary, ...]: + return tuple(self._records) + + def _call(self, original, frame, context, marital, rng): + try: + if self._failed or context.period_index != len(self._records) + 1: + raise ValueError( + "fertility capture cannot be reused or reordered" + ) + if context.rng_registry is None: + raise ValueError("assembled capture requires registry") + # The engine passes draw_index through, and derives n_periods from + # end_year, unconverted, so NumPy integers arrive here. Keep only + # exact Python ints so the archive and digest can serialize them. + draw = _int(context.draw_index) + period = _int(context.period_index) + year = _int(context.year) + periods = _int(context.rng_registry.n_periods) + if self._draw is None: + self._draw, self._periods = draw, periods + if ( + draw != self._draw + or periods != self._periods + or period > self._periods + ): + raise ValueError("fertility capture mixes invocations") + if ( + _int(context.rng_registry.draw_index) != draw + or year != 2014 + period + ): + raise ValueError( + "fertility capture registry/calendar mismatch" + ) + before = _frame_json(frame) + output = original(frame, context, marital, rng) + self._records.append( + FertilityBoundary( + draw, + period, + year, + before, + _frame_json(output), + ) + ) + return output + except BaseException: + self._failed = True + raise + + def to_json(self) -> str: + """Persist complete copied boundaries for an external run receipt.""" + if self._periods is None: + raise ValueError("no fertility invocation captured") + self._require_complete(self._draw, self._periods) + return _json( + { + "schema": "assembled-fertility-boundaries/v1", + "draw_index": self._draw, + "n_periods": self._periods, + "records": [ + [ + x.draw_index, + x.period_index, + x.year, + x.before_json, + x.after_json, + ] + for x in self.records + ], + } + ) + + def _require_complete(self, draw: int, periods: int): + if self._failed or len(self._records) != periods: + raise ValueError("incomplete or failed fertility capture") + if periods and (self._draw != draw or self._periods != periods): + raise ValueError("fertility capture belongs to another invocation") + + +def capture_fertility( + modules: PeriodModules, +) -> tuple[PeriodModules, FertilityCapture]: + """Wrap only fertility, calling the original exactly once by identity.""" + if type(modules) is not PeriodModules: + raise ValueError("existing PeriodModules required") + capture = FertilityCapture() + original = modules.fertility + + def fertility(frame, context, marital, rng): + return capture._call(original, frame, context, marital, rng) + + return replace(modules, fertility=fertility), capture + + +def _validate_audit(history, audit): + _keys(audit, _AUDIT_KEYS) + if ( + audit["mode"] not in _MODES + or audit["baseline_digest"] != history.baseline.digest + or audit["history_digest"] != history.digest + ): + raise ValueError("audit/history binding mismatch") + initial = set(_parse_ids(audit["initial_ids"])) + bindings = _arrays(audit["identity_bindings"], 3) + if bindings != sorted(bindings, key=lambda row: int(row[0])): + raise ValueError("native bindings must be sorted") + native_to_key, ordinals = {}, {} + for native, key, ordinal in bindings: + # Parse singly: mapping order is native-numeric, not private-key order. + native, key, ordinal = ( + _parse_ids([x])[0] for x in (native, key, ordinal) + ) + if native in native_to_key or ordinal < 0: + raise ValueError("invalid identity binding") + native_to_key[native], ordinals[native] = key, ordinal + if history.baseline.identity_map.reverse_rows([key])[ + 0 + ] != PersonIdentity("int64", native): + raise ValueError("native identity binding mismatch") + if ( + set(native_to_key) != initial + or set(native_to_key.values()) != set(history.baseline.roster_keys) + or len(set(ordinals.values())) != len(ordinals) + ): + raise ValueError("cohort identity coverage mismatch") + if _arrays(audit["scheduled"], 2) != sorted( + audit["scheduled"], key=lambda row: row[0] + ): + raise ValueError("scheduled years must be sorted") + scheduled = {} + scheduled_ids = set() + for year, ids in audit["scheduled"]: + if ( + type(year) is not int + or not 2015 <= year <= history.last_year + or year in scheduled + ): + raise ValueError("invalid scheduled period") + scheduled[year] = set(_parse_ids(ids)) + if scheduled[year] & (initial | scheduled_ids): + raise ValueError("scheduled identity collision") + scheduled_ids.update(scheduled[year]) + native_order = { + key: i for i, key in enumerate(sorted(initial | scheduled_ids)) + } + if any(ordinals[x] != native_order[x] for x in initial): + raise ValueError("native ordinal binding mismatch") + if len(audit["periods"]) != len(history.transitions): + raise ValueError("audit period coverage mismatch") + active, seen, excluded = set(initial), set(initial | scheduled_ids), {} + for transition, row in zip( + history.transitions, audit["periods"], strict=True + ): + _keys(row, {"year", "pre_ids", "death_ids", "post_ids", "births"}) + year = transition.mortality.target_year + # Python equality accepts 2016.0 == 2016; a float year would be a + # second valid encoding (and digest) of the same observation. + if type(row["year"]) is not int or row["year"] != year: + raise ValueError("audit year mismatch") + entrants = scheduled.get(year, set()) + pre, dead, post = ( + set(_parse_ids(row[x])) + for x in ("pre_ids", "death_ids", "post_ids") + ) + if pre != active | entrants or not dead <= pre: + raise ValueError("full mortality roster mismatch") + for key in entrants: + excluded[key] = { + "native_id": str(key), + "kind": "scheduled_entry", + "entry_year": year, + "death_year": None, + "parent_id": None, + } + survivors = pre - dead + added = set() + for native, parent in _arrays(row["births"], 2): + native, parent = _parse_ids([native])[0], _parse_ids([parent])[0] + if native in seen or native in added or parent not in survivors: + raise ValueError("birth identity or parent mismatch") + added.add(native) + excluded[native] = { + "native_id": str(native), + "kind": "native_synthetic_birth", + "entry_year": year, + "death_year": None, + "parent_id": str(parent), + } + if post != survivors | added: + raise ValueError("full final roster mismatch") + for key in dead - initial: + excluded[key]["death_year"] = year + if ( + tuple(sorted(native_to_key[x] for x in pre & initial)) + != transition.mortality.pre_keys + ): + raise ValueError("cohort pre-roster mismatch") + if ( + tuple(sorted(native_to_key[x] for x in survivors & initial)) + != transition.mortality.post_keys + ): + raise ValueError("cohort survivor mismatch") + for observed in transition.mortality.rows: + native = history.baseline.identity_map.reverse_rows( + [observed.dynamics_person_key] + )[0].value + if observed.person_ordinal != ordinals[native]: + raise ValueError("cohort ordinal mismatch") + if audit["mode"] == "strict_full_roster" and (entrants or added): + raise ValueError("strict full roster prohibits additions") + active, seen = post, seen | added + # Compare canonical bytes, not dict equality, so float entry/death years + # (2016.0 == 2016) cannot pass as a second encoding of the same ledger. + if _json(audit["excluded"]) != _json( + [excluded[x] for x in sorted(excluded)] + ): + raise ValueError("excluded-person audit mismatch") + expected = { + "projection", + "mortality", + "traces", + "entry_metadata", + "mortality_model", + "fertility_capture", + } + _keys(audit["input_digests"], expected) + for digest in audit["input_digests"].values(): + _digest(digest) + if ( + history.transitions + and audit["input_digests"]["mortality_model"] + != history.transitions[0].mortality.model.digest + ): + raise ValueError("model digest mismatch") + + +@dataclass(frozen=True) +class AssembledHistoryObservation: + """Canonical cohort history plus inseparable full-roster selection audit.""" + + history: ClosedCohortEarningsHistory + audit_json: str + + def __post_init__(self): + if type(self.history) is not ClosedCohortEarningsHistory: + raise ValueError("explicit closed-cohort history required") + audit = _load(self.audit_json) + if _json(audit) != self.audit_json: + raise ValueError("canonical audit JSON required") + _validate_audit(self.history, audit) + + def to_json(self) -> str: + return _json( + { + "schema": _SCHEMA, + "history": _load(self.history.to_json()), + "audit": _load(self.audit_json), + } + ) + + @property + def digest(self) -> str: + return _hash(self.to_json()) + + @classmethod + def from_json( + cls, + text: str, + *, + baseline: ForwardEarningsHistory, + expected_digest: str, + ): + """Require trusted baseline and whole-envelope digest before loading.""" + _digest(expected_digest) + if _hash(text) != expected_digest: + raise ValueError("observation digest mismatch") + doc = _load(text) + _keys(doc, {"schema", "history", "audit"}) + if doc["schema"] != _SCHEMA: + raise ValueError("unsupported observation schema") + result = cls( + ClosedCohortEarningsHistory.from_json( + _json(doc["history"]), baseline=baseline + ), + _json(doc["audit"]), + ) + if result.to_json() != text: + raise ValueError("noncanonical observation envelope") + return result + + +def _birth_year(value, year): + # Native concatenation adds missing values for nonchildren, promoting this + # column to float. The supported calendar is small and exact. + if isinstance(value, (float, np.floating)): + if ( + not math.isfinite(value) + or not float(value).is_integer() + or abs(value) >= 2**53 + ): + raise ValueError("unsafe birth year") + value = int(value) + if _int(value) != year: + raise ValueError("birth year mismatch") + return value + + +def _parent(value, survivors): + if isinstance(value, (int, np.integer)) and not isinstance( + value, (bool, np.bool_) + ): + parent = int(value) + elif ( + isinstance(value, (float, np.floating)) + and math.isfinite(value) + and float(value).is_integer() + and abs(value) < 2**53 + ): + parent = int(value) + if sum(float(x) == value for x in survivors) != 1: + raise ValueError("ambiguous float parent identity") + else: + raise ValueError("unsafe or ambiguous parent identity") + if parent not in survivors: + raise ValueError("birth parent is not a survivor") + return parent + + +def observe_assembled_history( + projection: ProjectionResult, + draw_outputs: Mapping[str, object], + *, + mode: str, + identity_map: PersonIdentityMap, + realization_id: str, + generator_digest: str, + earnings_source_contract_digest: str, + mortality_snapshot: MortalityModelSnapshot, + mortality_snapshot_after: MortalityModelSnapshot, + mortality_source_contract_digest: str, + unit: str, + price_basis: str, + lineage_by_year: Mapping[int, str], + initial_native_ids, + scheduled_entries_by_year: Mapping[int, pd.DataFrame], + reserved_real_ids, + synthetic_id_start: int, + fertility_capture: FertilityCapture | None = None, +) -> AssembledHistoryObservation: + """Reconcile a complete run before selecting the original 2014 cohort. + + This reads supplied evidence only. Effective parameters and execution + provenance must independently be bound before/after the actual run. + """ + if type(projection) is not ProjectionResult or mode not in _MODES: + raise ValueError("explicit projection and population mode required") + if ( + type(identity_map) is not PersonIdentityMap + or type(mortality_snapshot) is not MortalityModelSnapshot + ): + raise ValueError( + "explicit identity map and immutable mortality snapshot required" + ) + if ( + type(mortality_snapshot_after) is not MortalityModelSnapshot + or mortality_snapshot_after != mortality_snapshot + ): + raise ValueError("pre/post mortality parameter binding mismatch") + _digest(mortality_source_contract_digest) + if any(type(year) is not int for year in lineage_by_year): + raise ValueError("integer lineage years required") + for value in lineage_by_year.values(): + _digest(value) + periods = len(projection.traces) + if not 0 <= periods <= 8 or len(projection.slices) != periods + 1: + raise ValueError("complete 2014-22 projection required") + draw = _int(projection.draw_index) + initial = set(_ids(initial_native_ids)) + if not initial: + raise ValueError("nonempty original cohort required") + if set(lineage_by_year) != set(range(2014, 2015 + periods)): + raise ValueError("exact annual lineage coverage required") + if fertility_capture is not None: + if type(fertility_capture) is not FertilityCapture: + raise ValueError("native fertility boundary capture required") + fertility_capture._require_complete(draw, periods) + scheduled, scheduled_frames, all_ids = {}, {}, set(initial) + for year, frame in scheduled_entries_by_year.items(): + # The engine accepts NumPy year keys via int(); normalize them here so + # the entry-metadata digest never sees a non-JSON integer key. + year = _int(year) + if not 2015 <= year <= 2014 + periods or year in scheduled: + raise ValueError("invalid scheduled entry year") + rows = _rows(frame, year=year - 1) + if not rows: + raise ValueError("registered scheduled entries cannot be empty") + if all_ids & set(rows): + raise ValueError("scheduled identity collision") + scheduled[year], scheduled_frames[year] = rows, frame + all_ids.update(rows) + if mode == "strict_full_roster" and any(scheduled.values()): + raise ValueError("strict full roster prohibits scheduled entrants") + reserved = set(_ids(reserved_real_ids)) + if not all_ids <= reserved: + raise ValueError( + "reserved real namespace must cover all supplied real IDs" + ) + next_id = _int(synthetic_id_start) + if next_id < 0: + raise ValueError("invalid synthetic namespace") + ordinals = {key: ordinal for ordinal, key in enumerate(sorted(all_ids))} + mapped = dict( + zip( + sorted(initial), + identity_map.map_rows( + PersonIdentity("int64", x) for x in sorted(initial) + ), + strict=True, + ) + ) + + def copied(frame): + subset = frame.loc[frame.person_id.isin(initial)].copy() + subset["person_id"] = np.asarray( + [mapped[_int(x)] for x in subset.person_id], dtype="int64" + ) + # Registered merges may carry object columns; accept only exact integer + # scalar ages/years, never float coercion, on these observation copies. + for column in ("age", "year"): + subset[column] = np.asarray( + [_int(x) for x in subset[column]], dtype="int64" + ) + return subset + + previous = _rows(projection.slices[0], year=2014) + if set(previous) != initial: + raise ValueError( + "declared initial cohort differs from initialized slice" + ) + baseline = ForwardEarningsHistory.start( + identity_map, + copied(projection.slices[0]), + realization_id=realization_id, + generator_digest=generator_digest, + source_contract_digest=earnings_source_contract_digest, + unit=unit, + price_basis=price_basis, + lineage_digest=lineage_by_year[2014], + ) + history = ClosedCohortEarningsHistory.start(baseline, draw_index=draw) + mortalities = draw_outputs.get("mortality_slices", []) + if type(mortalities) is not list or len(mortalities) != periods: + raise ValueError("complete mortality slices required") + period_audits, excluded, seen = [], {}, set(all_ids) + for offset, (trace, frame, mortality) in enumerate( + zip( + projection.traces, projection.slices[1:], mortalities, strict=True + ), + 1, + ): + year = 2014 + offset + if trace.year != year or trace.steps != tuple( + x.value for x in ProjectionModule + ): + raise ValueError("actual ordered eight-step trace required") + pre = _rows(mortality) + if ( + "cal_year" not in mortality + or "death" not in mortality + or mortality.death.dtype != np.dtype("bool") + ): + raise ValueError( + "native mortality year and boolean death flags required" + ) + if any(_int(x) != year for x in mortality.cal_year): + raise ValueError("mortality target year mismatch") + expected = {**previous, **scheduled.get(year, {})} + if set(previous) & set(scheduled.get(year, {})) or set(pre) != set( + expected + ): + raise ValueError("complete mortality pre-roster mismatch") + _same_demographics(expected, pre) + if any( + row["age"] > mortality_snapshot.cells[-1][1] + for row in pre.values() + ): + raise ValueError("age outside mortality snapshot") + dead = {key for key, row in pre.items() if row["death"]} + survivors = set(pre) - dead + final = _rows(frame, year=year) + if not survivors <= set(final): + raise ValueError("unexplained survivor disappearance") + _same_demographics({x: pre[x] for x in survivors}, final, aging=1) + additions = set(final) - survivors + births = [] + if additions and fertility_capture is None: + raise ValueError( + "birth additions require actual fertility capture" + ) + if fertility_capture is not None: + boundary = fertility_capture.records[offset - 1] + if (boundary.draw_index, boundary.period_index, boundary.year) != ( + draw, + offset, + year, + ): + raise ValueError("fertility capture coordinates mismatch") + before = _rows(_frame_from_json(boundary.before_json), year=year) + after = _rows(_frame_from_json(boundary.after_json), year=year) + if set(before) != survivors or set(after) != set(final): + raise ValueError("fertility boundary roster mismatch") + _same_demographics({x: pre[x] for x in survivors}, before, aging=1) + _same_demographics(before, after) + _same_demographics(after, final) + for key in additions: + for column in ( + "birth_year", + "parent_person_id", + "synthetic_entry", + ): + if ( + column not in after[key] + or column not in final[key] + or _scalar(after[key][column]) + != _scalar(final[key][column]) + ): + raise ValueError("birth boundary metadata mismatch") + for key in sorted(additions): + row = final[key] + if ( + key != next_id + or key in seen + or key in reserved + or row["age"] != 0 + or _birth_year(row["birth_year"], year) != year + or type(row["synthetic_entry"]) is not bool + or not row["synthetic_entry"] + ): + raise ValueError( + "birth identity, allocator or metadata mismatch" + ) + parent = _parent(row["parent_person_id"], survivors) + births.append([str(key), str(parent)]) + next_id += 1 + if mode == "strict_full_roster" and additions: + raise ValueError("strict full roster prohibits births") + for key in scheduled.get(year, {}): + excluded[key] = { + "native_id": str(key), + "kind": "scheduled_entry", + "entry_year": year, + "death_year": None, + "parent_id": None, + } + for key, parent in births: + excluded[int(key)] = { + "native_id": key, + "kind": "native_synthetic_birth", + "entry_year": year, + "death_year": None, + "parent_id": parent, + } + for key in dead - initial: + excluded[key]["death_year"] = year + mortality_record = MortalityStepObservation( + identity_map, + realization_id, + mortality_source_contract_digest, + mortality_snapshot, + year, + offset, + draw, + periods, + tuple( + sorted( + ( + MortalityObservation( + mapped[key], + row["age"], + row["sex"], + key not in dead, + ordinals[key], + ) + for key, row in pre.items() + if key in initial + ), + key=lambda x: x.dynamics_person_key, + ) + ), + ) + history = history.append( + mortality=mortality_record, + earnings_frame=copied(frame), + lineage_digest=lineage_by_year[year], + ) + period_audits.append( + { + "year": year, + "pre_ids": _strings(pre), + "death_ids": _strings(dead), + "post_ids": _strings(final), + "births": births, + } + ) + previous, seen = final, seen | additions + audit = { + "mode": mode, + "baseline_digest": baseline.digest, + "history_digest": history.digest, + "identity_bindings": [ + [str(x), str(mapped[x]), str(ordinals[x])] for x in sorted(initial) + ], + "initial_ids": _strings(initial), + "scheduled": [ + [y, _strings(rows)] for y, rows in sorted(scheduled.items()) + ], + "periods": period_audits, + "excluded": [excluded[x] for x in sorted(excluded)], + "input_digests": { + "projection": _hash( + _json([_frame_json(x) for x in projection.slices]) + ), + "mortality": _hash(_json([_frame_json(x) for x in mortalities])), + "traces": _hash( + _json( + [ + [ + _int(x.year), + list(x.steps), + _frame_json(x.authoritative_marital_state.births), + ] + for x in projection.traces + ] + ) + ), + "entry_metadata": _hash( + _json( + { + "initial": _strings(initial), + "reserved": _strings(reserved), + "synthetic_start": str(synthetic_id_start), + "scheduled": [ + [y, _frame_json(f)] + for y, f in sorted(scheduled_frames.items()) + ], + } + ) + ), + "mortality_model": mortality_snapshot.digest, + "fertility_capture": _hash( + _json( + None + if fertility_capture is None + else [ + [ + x.draw_index, + x.period_index, + x.year, + x.before_json, + x.after_json, + ] + for x in fertility_capture.records + ] + ) + ), + }, + } + return AssembledHistoryObservation(history, _json(audit)) diff --git a/src/populace_dynamics/closed_cohort_history.py b/src/populace_dynamics/closed_cohort_history.py new file mode 100644 index 00000000..8edbff36 --- /dev/null +++ b/src/populace_dynamics/closed_cohort_history.py @@ -0,0 +1,459 @@ +"""Compose existing earnings histories with closed-cohort mortality records. + +This observer makes no generator calls and supplies no post-death amounts. +It does not integrate the registered assembly or admit a model or source. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, replace +from hashlib import sha256 +from numbers import Integral + +import numpy as np +import pandas as pd + +from .forward_earnings_history import ForwardEarningsHistory +from .mortality_observer import MortalityStepObservation +from .person_identity import PersonIdentity + +_SCHEMA = "populace_dynamics.closed_cohort_history.v1" + + +def _integer(value: object) -> int: + if isinstance(value, bool) or not isinstance(value, Integral): + raise ValueError("integer required without coercion") + if not 0 <= value < 2**63: + raise ValueError("integer outside nonnegative int64") + return int(value) + + +def _parse_integer(value: object) -> int: + if type(value) is not str: + raise ValueError("canonical integer string required") + result = _integer(int(value)) + if str(result) != value: + raise ValueError("noncanonical integer") + return result + + +def _digest(value: object) -> None: + if type(value) is not str or re.fullmatch(r"[0-9a-f]{64}", value) is None: + raise ValueError("lowercase SHA-256 digest required") + + +def _json(value: object) -> str: + return json.dumps( + value, + sort_keys=True, + ensure_ascii=False, + separators=(",", ":"), + allow_nan=False, + ) + + +def _person_baselines( + baseline: ForwardEarningsHistory, +) -> dict[int, ForwardEarningsHistory]: + # A validated 2014 baseline contains exactly one observation per person. + return { + row.dynamics_person_key: replace( + baseline, + roster_keys=(row.dynamics_person_key,), + observations=(row,), + ) + for row in baseline.observations + } + + +@dataclass(frozen=True) +class CohortTransition: + """Actual mortality plus post-aging demographics and frame lineage.""" + + mortality: MortalityStepObservation + lineage_digest: str + survivor_demographics: tuple[tuple[int, int, str], ...] + + def __post_init__(self) -> None: + if type(self.mortality) is not MortalityStepObservation: + raise ValueError("explicit mortality step required") + _digest(self.lineage_digest) + demographics = [] + for key, age, sex in self.survivor_demographics: + if type(sex) is not str: + raise ValueError("explicit mortality sex label required") + demographics.append((_integer(key), _integer(age), sex)) + expected = tuple( + (row.dynamics_person_key, row.age + 1, row.sex) + for row in self.mortality.rows + if row.survived + ) + if tuple(demographics) != expected: + raise ValueError("survivor demographics must match actual aging") + object.__setattr__(self, "survivor_demographics", tuple(demographics)) + + +@dataclass(frozen=True) +class ClosedCohortEarningsHistory: + """Original persons remain recorded while only mortality survivors append.""" + + baseline: ForwardEarningsHistory + draw_index: int + histories: tuple[ForwardEarningsHistory, ...] + transitions: tuple[CohortTransition, ...] = () + + def __init_subclass__(cls, **kwargs) -> None: + raise TypeError("ClosedCohortEarningsHistory cannot be subclassed") + + def __post_init__(self) -> None: + if ( + type(self.baseline) is not ForwardEarningsHistory + or self.baseline.last_year != 2014 + ): + raise ValueError("baseline must be an explicit 2014 history") + object.__setattr__(self, "draw_index", _integer(self.draw_index)) + histories, transitions = tuple(self.histories), tuple(self.transitions) + if len(transitions) > 8: + raise ValueError("closed-cohort history cannot extend past 2022") + if any( + type(h) is not ForwardEarningsHistory or len(h.roster_keys) != 1 + for h in histories + ): + raise ValueError( + "one existing history per original person required" + ) + keys = [h.roster_keys[0] for h in histories] + if tuple(sorted(keys)) != self.baseline.roster_keys: + raise ValueError( + "person histories must cover the original roster once" + ) + if any(type(t) is not CohortTransition for t in transitions): + raise ValueError("typed annual transitions required") + active, previous, deaths = self.baseline.roster_keys, {}, {} + signature = None + for offset, transition in enumerate(transitions, 1): + mortality = transition.mortality + if ( + mortality.target_year != 2014 + offset + or mortality.period_index != offset + or mortality.draw_index != self.draw_index + or mortality.identity_map != self.baseline.identity_map + or mortality.realization_id != self.baseline.realization_id + ): + raise ValueError( + "mortality calendar, identity or realization mismatch" + ) + current_signature = ( + mortality.model, + mortality.source_contract_digest, + mortality.registry_n_periods, + ) + if signature is not None and current_signature != signature: + raise ValueError("mortality model, source or RNG mode changed") + signature = current_signature + if mortality.pre_keys != active: + raise ValueError( + "mortality input roster must equal prior survivors" + ) + for row in mortality.rows: + key = row.dynamics_person_key + if previous: + prior = previous[key] + if ( + row.age != prior.age + 1 + or row.sex != prior.sex + or row.person_ordinal != prior.person_ordinal + ): + raise ValueError( + "mortality demographic or ordinal continuity changed" + ) + if not row.survived: + deaths[key] = mortality.target_year + previous = { + r.dynamics_person_key: r for r in mortality.rows if r.survived + } + active = mortality.post_keys + person_baselines = _person_baselines(self.baseline) + for history in histories: + key = history.roster_keys[0] + history.require_extension_of(person_baselines[key]) + end = deaths[key] - 1 if key in deaths else 2014 + len(transitions) + if history.last_year != end: + raise ValueError( + "person history extent disagrees with mortality" + ) + for row in history.observations[1:]: + if ( + row.lineage_digest + != transitions[row.year - 2015].lineage_digest + ): + raise ValueError( + "annual earnings lineage disagrees with transition" + ) + object.__setattr__( + self, + "histories", + tuple(sorted(histories, key=lambda h: h.roster_keys[0])), + ) + object.__setattr__(self, "transitions", transitions) + + @classmethod + def start( + cls, initial_history: ForwardEarningsHistory, *, draw_index: int + ) -> ClosedCohortEarningsHistory: + """Bind an already materialized 2014 history without changing it.""" + if ( + type(initial_history) is not ForwardEarningsHistory + or initial_history.last_year != 2014 + ): + raise ValueError("baseline must be an explicit 2014 history") + return cls( + initial_history, + draw_index, + tuple(_person_baselines(initial_history).values()), + ) + + @property + def last_year(self) -> int: + return 2014 + len(self.transitions) + + @property + def active_keys(self) -> tuple[int, ...]: + return ( + self.transitions[-1].mortality.post_keys + if self.transitions + else self.baseline.roster_keys + ) + + def append( + self, + *, + mortality: MortalityStepObservation, + earnings_frame: pd.DataFrame, + lineage_digest: str, + ) -> ClosedCohortEarningsHistory: + """Observe one actual survivor frame; never generate or fill earnings.""" + if self.last_year == 2022: + raise ValueError("closed-cohort history cannot extend past 2022") + if type(mortality) is not MortalityStepObservation: + raise ValueError("explicit mortality step required") + if mortality.pre_keys != self.active_keys: + raise ValueError( + "mortality input roster must equal prior survivors" + ) + frame = earnings_frame + required = { + "person_id", + "year", + "age", + "sex", + "earnings", + "earnings_domain", + } + if ( + not isinstance(frame, pd.DataFrame) + or not frame.columns.is_unique + or not required.issubset(frame.columns) + ): + raise ValueError( + "earnings frame requires unique observation columns" + ) + for column, dtype in ( + ("person_id", "int64"), + ("year", "int64"), + ("age", "int64"), + ("earnings", "float64"), + ("earnings_domain", "bool"), + ): + if frame[column].dtype != np.dtype(dtype): + raise ValueError(f"{column} must preserve {dtype} dtype") + if not pd.api.types.is_string_dtype(frame["sex"].dtype): + raise ValueError("sex must preserve a string-compatible dtype") + if tuple(sorted(frame["person_id"])) != mortality.post_keys: + raise ValueError( + "earnings frame must contain exactly the survivors" + ) + if not (frame["year"] == self.last_year + 1).all(): + raise ValueError("earnings frame must contain the next year") + ordered = frame.sort_values("person_id", kind="stable") + transition = CohortTransition( + mortality, + lineage_digest, + tuple( + zip( + ordered["person_id"], + ordered["age"], + ordered["sex"], + strict=True, + ) + ), + ) + survivors = set(mortality.post_keys) + by_person = frame.set_index("person_id", drop=False) + histories = tuple( + ( + history.append( + by_person.loc[[history.roster_keys[0]]], + lineage_digest=lineage_digest, + ) + if history.roster_keys[0] in survivors + else history + ) + for history in self.histories + ) + return type(self)( + self.baseline, + self.draw_index, + histories, + self.transitions + (transition,), + ) + + def for_person(self, identity: PersonIdentity) -> ForwardEarningsHistory: + """Return the retained existing history using exact source identity.""" + key = self.baseline.identity_map.map_rows([identity])[0] + for history in self.histories: + if history.roster_keys == (key,): + return history + raise ValueError("person is outside the original cohort") + + def death_step( + self, identity: PersonIdentity + ) -> MortalityStepObservation | None: + """Return the recorded removal step, without inferring a death date.""" + key = self.for_person(identity).roster_keys[0] + for transition in self.transitions: + if ( + key in transition.mortality.pre_keys + and key not in transition.mortality.post_keys + ): + return transition.mortality + return None + + def amount_state(self, identity: PersonIdentity, *, year: int) -> str: + """Distinguish generated amounts, unsupported inputs and mortality.""" + year = _integer(year) + if not 2014 <= year <= self.last_year: + raise ValueError("year is outside the observed cohort envelope") + history = self.for_person(identity) + if year <= history.last_year: + return history.observations[year - 2014].amount_state + return "not_generated_after_mortality_step" + + def require_extension_of( + self, previous: ClosedCohortEarningsHistory + ) -> None: + """Reject changes to the baseline, draw or any prior observation.""" + if ( + type(previous) is not ClosedCohortEarningsHistory + or self.baseline != previous.baseline + or self.draw_index != previous.draw_index + ): + raise ValueError("cohort extension changed baseline or draw") + if ( + self.transitions[: len(previous.transitions)] + != previous.transitions + ): + raise ValueError( + "cohort extension removed or changed prior transitions" + ) + for current, prior in zip( + self.histories, previous.histories, strict=True + ): + current.require_extension_of(prior) + + def to_json(self) -> str: + """Compose existing canonical records with an external baseline bind.""" + return _json( + { + "schema": _SCHEMA, + "baseline_digest": self.baseline.digest, + "draw_index": str(self.draw_index), + "last_year": str(self.last_year), + "histories": [json.loads(h.to_json()) for h in self.histories], + "transitions": [ + { + "mortality": json.loads(t.mortality.to_json()), + "lineage_digest": t.lineage_digest, + "survivor_demographics": [ + [str(key), str(age), sex] + for key, age, sex in t.survivor_demographics + ], + } + for t in self.transitions + ], + } + ) + + @property + def digest(self) -> str: + return sha256(self.to_json().encode("utf-8")).hexdigest() + + @classmethod + def from_json( + cls, + text: str, + *, + baseline: ForwardEarningsHistory, + expected_digest: str | None = None, + previous: ClosedCohortEarningsHistory | None = None, + ) -> ClosedCohortEarningsHistory: + """Load through existing validators, then check cross-step continuity.""" + + def unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON field") + result[key] = value + return result + + def reject(value): + raise ValueError(f"invalid JSON constant: {value}") + + document = json.loads( + text, object_pairs_hook=unique, parse_constant=reject + ) + if ( + type(baseline) is not ForwardEarningsHistory + or not isinstance(document, dict) + or document.get("baseline_digest") != baseline.digest + ): + raise ValueError("trusted baseline digest mismatch") + try: + result = cls( + baseline, + _parse_integer(document["draw_index"]), + tuple( + ForwardEarningsHistory.from_json(_json(h)) + for h in document["histories"] + ), + tuple( + CohortTransition( + MortalityStepObservation.from_json( + _json(t["mortality"]), + identity_map=baseline.identity_map, + ), + t["lineage_digest"], + tuple( + (_parse_integer(key), _parse_integer(age), sex) + for key, age, sex in t["survivor_demographics"] + ), + ) + for t in document["transitions"] + ), + ) + except (KeyError, TypeError) as exc: + raise ValueError("invalid closed-cohort record structure") from exc + if result.to_json() != _json(document): + raise ValueError( + "noncanonical or inconsistent closed-cohort record" + ) + if expected_digest is not None: + _digest(expected_digest) + if result.digest != expected_digest: + raise ValueError("closed-cohort digest mismatch") + if previous is not None: + result.require_extension_of(previous) + return result diff --git a/src/populace_dynamics/compact_cohort_history.py b/src/populace_dynamics/compact_cohort_history.py new file mode 100644 index 00000000..cc577349 --- /dev/null +++ b/src/populace_dynamics/compact_cohort_history.py @@ -0,0 +1,297 @@ +"""Linear-size persistence for validated closed-cohort earnings histories. + +This optional envelope leaves the legacy serializers and their digests intact. +Loading requires the trusted legacy baseline that owns the identity map. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping + +from .closed_cohort_history import ( + ClosedCohortEarningsHistory, + CohortTransition, +) +from .forward_earnings_history import ( + ForwardEarningsHistory, + ForwardEarningsObservation, +) +from .mortality_observer import MortalityStepObservation + +_SCHEMA = "populace_dynamics.compact_closed_cohort_history.v1" +_HISTORY_FIELDS = { + "identity_map_digest", + "realization_id", + "generator_digest", + "source_contract_digest", + "unit", + "price_basis", + "calendar", + "source_registry_status", + "coverage_status", + "roster_keys", + "last_year", + "observations", +} +_OBSERVATION_FIELDS = { + "dynamics_person_key", + "year", + "amount_hex", + "earnings_domain", + "lineage_digest", +} + + +def _json(value: object) -> str: + return json.dumps( + value, + sort_keys=True, + ensure_ascii=False, + separators=(",", ":"), + allow_nan=False, + ) + + +def _load(text: str) -> object: + if type(text) is not str: + raise ValueError("compact history must be JSON text") + + def unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON field") + result[key] = value + return result + + def reject(value): + raise ValueError(f"invalid JSON constant: {value}") + + return json.loads(text, object_pairs_hook=unique, parse_constant=reject) + + +def _digest(value: object, label: str) -> str: + if type(value) is not str or re.fullmatch(r"[0-9a-f]{64}", value) is None: + raise ValueError(f"{label} must be a lowercase SHA-256 digest") + return value + + +def _integer(value: object, label: str) -> int: + if type(value) is not str: + raise ValueError(f"{label} must be a canonical integer string") + try: + result = int(value) + except ValueError as exc: + raise ValueError( + f"{label} must be a canonical integer string" + ) from exc + if str(result) != value: + raise ValueError(f"{label} must be a canonical integer string") + return result + + +def _keys(value: object, fields: set[str], label: str) -> Mapping: + if not isinstance(value, Mapping) or set(value) != fields: + raise ValueError(f"invalid {label} fields") + return value + + +def _history_document( + history: ForwardEarningsHistory, identity_map_digest: str +) -> dict: + return { + "identity_map_digest": identity_map_digest, + "realization_id": history.realization_id, + "generator_digest": history.generator_digest, + "source_contract_digest": history.source_contract_digest, + "unit": history.unit, + "price_basis": history.price_basis, + "calendar": "calendar_year", + "source_registry_status": history.source_registry_status, + "coverage_status": history.coverage_status, + "roster_keys": [str(key) for key in history.roster_keys], + "last_year": str(history.last_year), + "observations": [ + { + "dynamics_person_key": str(row.dynamics_person_key), + "year": str(row.year), + "amount_hex": row.amount_hex, + "earnings_domain": row.earnings_domain, + "lineage_digest": row.lineage_digest, + } + for row in history.observations + ], + } + + +def compact_history_to_json(history: ClosedCohortEarningsHistory) -> str: + """Serialize a validated history without repeated identity-map documents.""" + if type(history) is not ClosedCohortEarningsHistory: + raise ValueError("explicit closed-cohort history required") + identity_map_digest = history.baseline.identity_map.digest + return _json( + { + "schema": _SCHEMA, + "baseline_digest": history.baseline.digest, + "identity_map_digest": identity_map_digest, + "draw_index": str(history.draw_index), + "last_year": str(history.last_year), + "histories": [ + _history_document(item, identity_map_digest) + for item in history.histories + ], + "transitions": [ + { + "mortality": json.loads(item.mortality.to_json()), + "lineage_digest": item.lineage_digest, + "survivor_demographics": [ + [str(key), str(age), sex] + for key, age, sex in item.survivor_demographics + ], + } + for item in history.transitions + ], + } + ) + + +def compact_history_digest(history: ClosedCohortEarningsHistory) -> str: + """Return the distinct SHA-256 digest of the compact envelope.""" + return hashlib.sha256( + compact_history_to_json(history).encode() + ).hexdigest() + + +def _history_from_document( + document: object, + baseline: ForwardEarningsHistory, + identity_map_digest: str, +): + item = _keys(document, _HISTORY_FIELDS, "compact child history") + if ( + item["identity_map_digest"] != identity_map_digest + or item["calendar"] != "calendar_year" + or item["source_registry_status"] != "registration_required" + or item["coverage_status"] != "not_materialized" + ): + raise ValueError("compact child scope or identity mismatch") + if ( + type(item["roster_keys"]) is not list + or type(item["observations"]) is not list + ): + raise ValueError("compact child arrays required") + observations = [] + for value in item["observations"]: + row = _keys(value, _OBSERVATION_FIELDS, "compact observation") + observations.append( + ForwardEarningsObservation( + _integer(row["dynamics_person_key"], "person key"), + _integer(row["year"], "year"), + row["amount_hex"], + row["earnings_domain"], + row["lineage_digest"], + ) + ) + return ForwardEarningsHistory( + baseline.identity_map, + item["realization_id"], + item["generator_digest"], + item["source_contract_digest"], + item["unit"], + item["price_basis"], + tuple(_integer(value, "roster key") for value in item["roster_keys"]), + _integer(item["last_year"], "last year"), + tuple(observations), + ) + + +def compact_history_from_json( + text: str, + *, + baseline: ForwardEarningsHistory, + expected_digest: str, + previous: ClosedCohortEarningsHistory | None = None, +) -> ClosedCohortEarningsHistory: + """Load a compact envelope against a trusted baseline and exact digest.""" + if type(baseline) is not ForwardEarningsHistory: + raise ValueError("trusted ForwardEarningsHistory baseline required") + if type(text) is not str: + raise ValueError("compact history must be JSON text") + expected_digest = _digest(expected_digest, "compact envelope digest") + if hashlib.sha256(text.encode()).hexdigest() != expected_digest: + raise ValueError("compact envelope digest mismatch") + document = _keys( + _load(text), + { + "schema", + "baseline_digest", + "identity_map_digest", + "draw_index", + "last_year", + "histories", + "transitions", + }, + "compact envelope", + ) + identity_map_digest = baseline.identity_map.digest + if ( + document["schema"] != _SCHEMA + or document["baseline_digest"] != baseline.digest + or document["identity_map_digest"] != identity_map_digest + ): + raise ValueError("compact envelope baseline binding mismatch") + if ( + type(document["histories"]) is not list + or type(document["transitions"]) is not list + ): + raise ValueError("compact envelope arrays required") + histories = tuple( + _history_from_document(item, baseline, identity_map_digest) + for item in document["histories"] + ) + transitions = [] + for value in document["transitions"]: + item = _keys( + value, + {"mortality", "lineage_digest", "survivor_demographics"}, + "compact transition", + ) + if type(item["survivor_demographics"]) is not list: + raise ValueError("survivor demographics array required") + demographics = [] + for row in item["survivor_demographics"]: + if type(row) is not list or len(row) != 3: + raise ValueError("invalid survivor demographic row") + demographics.append( + ( + _integer(row[0], "survivor key"), + _integer(row[1], "survivor age"), + row[2], + ) + ) + transitions.append( + CohortTransition( + MortalityStepObservation.from_json( + _json(item["mortality"]), + identity_map=baseline.identity_map, + ), + item["lineage_digest"], + tuple(demographics), + ) + ) + result = ClosedCohortEarningsHistory( + baseline, + _integer(document["draw_index"], "draw index"), + histories, + tuple(transitions), + ) + if result.last_year != _integer(document["last_year"], "last year"): + raise ValueError("compact envelope last year mismatch") + if compact_history_to_json(result) != text: + raise ValueError("noncanonical or inconsistent compact envelope") + if previous is not None: + result.require_extension_of(previous) + return result diff --git a/src/populace_dynamics/covered_wage_history.py b/src/populace_dynamics/covered_wage_history.py new file mode 100644 index 00000000..221f27b4 --- /dev/null +++ b/src/populace_dynamics/covered_wage_history.py @@ -0,0 +1,326 @@ +"""Attach explicit covered-wage source observations to a labor history. + +This opt-in sidecar records declarations, not statutory coverage decisions. +It never infers wages from labor income, imputes missing values, or computes +creditable earnings. Source digests identify caller-retained receipts; their +contents and admission are not verified here. +""" + +from __future__ import annotations + +import json +import math +import re +from dataclasses import dataclass +from decimal import Decimal +from hashlib import sha256 +from numbers import Integral + +from .forward_earnings_history import ForwardEarningsHistory +from .person_identity import PersonIdentity + +_SCHEMA = "populace_dynamics.covered_wage_history.v1" +_CONCEPT = "source_reported_uncapped_employee_wages_covered" +_MISSING = frozenset( + { + "source_field_unavailable", + "coverage_unresolved", + "crosswalk_registration_required", + } +) + + +def _digest(value: object) -> None: + if type(value) is not str or re.fullmatch(r"[0-9a-f]{64}", value) is None: + raise ValueError("source digest must be lowercase SHA-256") + + +def _integer(value: object) -> int: + if isinstance(value, bool) or not isinstance(value, Integral): + raise ValueError("coordinate must be an integer without coercion") + return int(value) + + +def _load_integer(value: object) -> int: + if type(value) is not str or re.fullmatch(r"0|[1-9][0-9]*", value) is None: + raise ValueError("coordinate must be a canonical integer string") + return int(value) + + +def _fields(document: object, expected: set[str]) -> None: + if type(document) is not dict or set(document) != expected: + raise ValueError("invalid document fields") + + +@dataclass(frozen=True) +class SourceAmount: + """Nonnegative amount retaining its source dtype and exact serialization.""" + + logical_dtype: str + serialization: str + + def __post_init__(self) -> None: + kind, value = self.logical_dtype, self.serialization + if type(kind) is not str or type(value) is not str: + raise ValueError( + "explicit dtype and source serialization required" + ) + if kind in ("int64", "uint64"): + amount = _load_integer(value) + if amount >= 2 ** (63 if kind == "int64" else 64): + raise ValueError("amount exceeds source integer dtype") + elif kind == "decimal": + if ( + re.fullmatch(r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?", value) + is None + ): + raise ValueError("decimal must retain plain source text") + if Decimal(value) < 0: + raise ValueError("covered wages cannot be negative") + elif kind == "binary64": + try: + amount = float.fromhex(value) + except (ValueError, OverflowError) as exc: + raise ValueError("invalid binary64 source amount") from exc + if ( + not math.isfinite(amount) + or amount < 0 + or amount.hex() != value + ): + raise ValueError( + "binary64 must be canonical, finite and nonnegative" + ) + else: + raise ValueError("unsupported source amount dtype") + + @property + def is_zero(self) -> bool: + if self.logical_dtype == "binary64": + return float.fromhex(self.serialization) == 0 + return Decimal(self.serialization) == 0 + + +@dataclass(frozen=True) +class CoveredWageObservation: + """Explicit source amount or unavailability for one private person/year.""" + + dynamics_person_key: int + year: int + amount: SourceAmount | None + missing_reason: str | None + source_digest: str + + def __post_init__(self) -> None: + key, year = _integer(self.dynamics_person_key), _integer(self.year) + if not 0 <= key < 2**63 or not 2014 <= year <= 2022: + raise ValueError("coordinate outside forward-history scope") + _digest(self.source_digest) + if self.amount is None: + if ( + type(self.missing_reason) is not str + or self.missing_reason not in _MISSING + ): + raise ValueError( + "unavailable amount needs an explicit missing reason" + ) + elif ( + type(self.amount) is not SourceAmount + or self.missing_reason is not None + ): + raise ValueError("typed source amount requires no missing reason") + object.__setattr__(self, "dynamics_person_key", key) + object.__setattr__(self, "year", year) + + @property + def amount_state(self) -> str: + if self.amount is None: + return "unavailable" + return "known_zero" if self.amount.is_zero else "known_amount" + + +@dataclass(frozen=True) +class CoveredWageHistory: + """Dense source observations bound to one exact forward history. + + The source contract must declare uncapped covered employee wages in the + history's nominal unit. Each source receipt must bind the artifact, + record/field locator, original value/dtype, reference year, unit and + information date. This class validates representation, not those claims. + """ + + history: ForwardEarningsHistory + source_contract_digest: str + observations: tuple[CoveredWageObservation, ...] + + def __init_subclass__(cls, **kwargs) -> None: + raise TypeError("CoveredWageHistory cannot be subclassed") + + def __post_init__(self) -> None: + if type(self.history) is not ForwardEarningsHistory: + raise ValueError("explicit forward history required") + _digest(self.source_contract_digest) + rows = tuple(self.observations) + if any(type(row) is not CoveredWageObservation for row in rows): + raise ValueError("typed covered-wage observations required") + coordinates = [(r.year, r.dynamics_person_key) for r in rows] + required = { + (r.year, r.dynamics_person_key) for r in self.history.observations + } + if len(coordinates) != len(required) or set(coordinates) != required: + raise ValueError( + "observations must cover the exact history envelope once" + ) + object.__setattr__( + self, + "observations", + tuple(sorted(rows, key=lambda r: (r.year, r.dynamics_person_key))), + ) + + def for_person( + self, identity: PersonIdentity + ) -> tuple[CoveredWageObservation, ...]: + """Select exact source identities without inferring identity from order.""" + key = self.history.identity_map.map_rows([identity])[0] + rows = tuple( + r for r in self.observations if r.dynamics_person_key == key + ) + if not rows: + raise ValueError("person is outside the history roster") + return rows + + @property + def missing_coordinates(self) -> tuple[tuple[int, int], ...]: + """Return (private key, year) for explicit unavailable observations.""" + return tuple( + (r.dynamics_person_key, r.year) + for r in self.observations + if r.amount is None + ) + + def _metadata(self) -> dict[str, str]: + return { + "schema": _SCHEMA, + "concept": _CONCEPT, + "history_digest": self.history.digest, + "identity_map_digest": self.history.identity_map.digest, + "realization_id": self.history.realization_id, + "unit": self.history.unit, + "price_basis": self.history.price_basis, + } + + def to_json(self) -> str: + """Serialize source strings without monetary or identity conversion.""" + document = { + **self._metadata(), + "source_contract_digest": self.source_contract_digest, + "observations": [ + { + "dynamics_person_key": str(r.dynamics_person_key), + "year": str(r.year), + "amount": ( + None + if r.amount is None + else { + "logical_dtype": r.amount.logical_dtype, + "serialization": r.amount.serialization, + } + ), + "missing_reason": r.missing_reason, + "source_digest": r.source_digest, + } + for r in self.observations + ], + } + return json.dumps( + document, + sort_keys=True, + ensure_ascii=False, + separators=(",", ":"), + allow_nan=False, + ) + + @property + def digest(self) -> str: + return sha256(self.to_json().encode("utf-8")).hexdigest() + + @classmethod + def from_json( + cls, + text: str, + *, + history: ForwardEarningsHistory, + expected_digest: str | None = None, + ) -> CoveredWageHistory: + """Load only against the bound history; never fill missing rows.""" + + def unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON member") + result[key] = value + return result + + def reject_constant(value): + raise ValueError(f"nonfinite JSON constant: {value}") + + if type(history) is not ForwardEarningsHistory: + raise ValueError("explicit forward history required") + document = json.loads( + text, object_pairs_hook=unique, parse_constant=reject_constant + ) + metadata = { + "schema": _SCHEMA, + "concept": _CONCEPT, + "history_digest": history.digest, + "identity_map_digest": history.identity_map.digest, + "realization_id": history.realization_id, + "unit": history.unit, + "price_basis": history.price_basis, + } + _fields( + document, + set(metadata) | {"source_contract_digest", "observations"}, + ) + if any( + type(document[k]) is not str or document[k] != v + for k, v in metadata.items() + ): + raise ValueError( + "document does not bind the supplied history and scope" + ) + if type(document["observations"]) is not list: + raise ValueError("observations must be an array") + rows = [] + for row in document["observations"]: + _fields( + row, + { + "dynamics_person_key", + "year", + "amount", + "missing_reason", + "source_digest", + }, + ) + amount = row["amount"] + if amount is not None: + _fields(amount, {"logical_dtype", "serialization"}) + amount = SourceAmount(**amount) + rows.append( + CoveredWageObservation( + _load_integer(row["dynamics_person_key"]), + _load_integer(row["year"]), + amount, + row["missing_reason"], + row["source_digest"], + ) + ) + result = cls(history, document["source_contract_digest"], tuple(rows)) + if expected_digest is not None: + _digest(expected_digest) + if result.digest != expected_digest: + raise ValueError( + "covered-wage digest differs from expected digest" + ) + return result diff --git a/src/populace_dynamics/engine/accounting.py b/src/populace_dynamics/engine/accounting.py new file mode 100644 index 00000000..6e91126d --- /dev/null +++ b/src/populace_dynamics/engine/accounting.py @@ -0,0 +1,1207 @@ +"""Experimental accounting for one annual population transition. + +``reconcile_period`` checks opening and closing person sets against the +caller's declared arrivals and departures. It does not generate transitions, +infer their causes, fit a model, or read data. Count conservation is exact; +weight stocks, flows, and separate revaluations use binary64 arithmetic with +``math.fsum`` and a reported residual. An unrepresentable summary is refused. +No scientific tolerance or acceptance gate is introduced. + +Frames require unique signed-int64 person IDs, matching integer years, and +finite nonnegative real weights. These are this interface's validation rules; +the historical projection loop has a less restrictive input check. Zero-weight +rows remain people. Every declaration uses the closing year, even when a +scheduled-entry frame carries the loop's required previous-year stamp. + +The supported lifecycle is at most one arrival followed by at most one +departure in the period. Event ordering within the year is not observed. +Declared transients appear in neither endpoint frame and need explicit +weights for both events. Omitting BOTH events is unobservable from endpoints: +accounting coherence does not establish event-log completeness or true causes. +There is no cross-period history, so declared reuse of a past ID is not caught. + +The module has only stdlib, NumPy, and pandas direct imports. A normal package +import also executes the existing engine package initializer and its broader +source dependencies. This optional module is not called by the historical +engine. See ``docs/stock-flow-accounting.md`` for the interface and limits. +""" + +from __future__ import annotations + +import math +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from enum import Enum +from types import MappingProxyType + +import numpy as np +import pandas as pd + +__all__ = [ + "ACCOUNTING_INTERFACE_VERSION", + "ADDITION_KINDS", + "ENGINEERING_STATUS", + "ENGINEERING_STATUS_NOTE", + "EXIT_KINDS", + "PERSON_ID_COLUMN", + "REQUIRED_COLUMNS", + "WEIGHT_COLUMN", + "YEAR_COLUMN", + "AccountingDiscrepancy", + "DiscrepancyKind", + "PeriodAccount", + "PersonCounts", + "PopulationAccountingError", + "PopulationAccountingInputError", + "PopulationEvent", + "PopulationEventKind", + "PopulationReconciliationError", + "WeightRevaluation", + "WeightTotals", + "reconcile_period", +] + +ACCOUNTING_INTERFACE_VERSION = "stock-flow-accounting/0.1.0-experimental" +ENGINEERING_STATUS = "engineering-accounting-coherence-only" +ENGINEERING_STATUS_NOTE = ( + "Person sets reconcile exactly and the weight identity closes to the " + "reported arithmetic residual. This is engineering coherence only: it " + "is not scientific acceptance, not a benchmark comparison, not a gate " + "outcome, and not evidence that the population is admitted." +) +SUMMATION_METHOD = "math.fsum over binary64 components; residual reported" +INFERENCE_POLICY = ( + "none: every addition and every exit must be declared by the caller" +) + +PERSON_ID_COLUMN = "person_id" +YEAR_COLUMN = "year" +WEIGHT_COLUMN = "weight" +REQUIRED_COLUMNS = (PERSON_ID_COLUMN, YEAR_COLUMN, WEIGHT_COLUMN) +_INT64_MIN = -(2**63) +_INT64_MAX = 2**63 - 1 + + +class PopulationEventKind(str, Enum): + """The declared reason a person joins or leaves within one period.""" + + BIRTH = "birth" + SCHEDULED_ENTRY = "scheduled_entry" + OTHER_ENTRY = "other_entry" + DEATH = "death" + EMIGRATION = "emigration" + OTHER_EXIT = "other_exit" + + +ADDITION_KINDS = frozenset( + { + PopulationEventKind.BIRTH, + PopulationEventKind.SCHEDULED_ENTRY, + PopulationEventKind.OTHER_ENTRY, + } +) +EXIT_KINDS = frozenset( + { + PopulationEventKind.DEATH, + PopulationEventKind.EMIGRATION, + PopulationEventKind.OTHER_EXIT, + } +) +#: Kinds whose whole purpose is "something else happened", and which are +#: therefore only meaningful if the caller says what. +REASON_REQUIRED_KINDS = frozenset( + {PopulationEventKind.OTHER_ENTRY, PopulationEventKind.OTHER_EXIT} +) + + +class DiscrepancyKind(str, Enum): + """Ways a declared story can fail to match the two frames.""" + + UNDECLARED_ADDITION = "undeclared_addition" + UNDECLARED_EXIT = "undeclared_exit" + ADDITION_COLLIDES_WITH_OPENING = "addition_collides_with_opening" + DUPLICATE_ADDITION = "duplicate_addition" + DUPLICATE_EXIT = "duplicate_exit" + EXIT_WITHOUT_PRESENCE = "exit_without_presence" + EXIT_CONTRADICTED_BY_CLOSING = "exit_contradicted_by_closing" + ADDITION_ABSENT_AT_CLOSE = "addition_absent_at_close" + COUNT_IDENTITY_VIOLATION = "count_identity_violation" + + +_DISCREPANCY_ORDER = { + kind: index for index, kind in enumerate(DiscrepancyKind) +} + + +def _restore_refusal( + cls: type[PopulationAccountingError], + args: tuple[object, ...], + attributes: dict[str, object], + cause: BaseException | None, + suppress_context: bool, +) -> PopulationAccountingError: + """Rebuild a pickled or copied refusal from its recorded state. + + ``__init__`` is not replayed: subclasses take typed or keyword-only + fields and may prefix their message, so replaying it from ``args`` + would fail or alter the message. The recorded ``args`` and instance + attributes are restored exactly instead. + """ + error = cls.__new__(cls, *args) + error.__dict__.update(attributes) + error.__cause__ = cause + error.__suppress_context__ = suppress_context + return error + + +class PopulationAccountingError(ValueError): + """Base class for every refusal raised by this module. + + Pickling and copying rebuild a refusal with its message arguments, its + typed attributes (``discrepancies``, a history ``kind`` and period + coordinates, notes) and its explicit ``raise ... from`` cause, so the + refusal can cross a process boundary. As for built-in exceptions, the + implicit ``__context__`` and the traceback are not carried. + """ + + def __reduce__(self) -> tuple[object, ...]: + return ( + _restore_refusal, + ( + type(self), + self.args, + dict(self.__dict__), + self.__cause__, + self.__suppress_context__, + ), + ) + + +class PopulationAccountingInputError(PopulationAccountingError): + """A frame or a declaration is malformed on its own terms.""" + + +class PopulationReconciliationError(PopulationAccountingError): + """Well-formed inputs whose declared story does not reconcile. + + The typed findings stay on the exception so that a caller can + distinguish an unexplained arrival from an omitted exit without + parsing a message. + """ + + def __init__( + self, + message: str, + discrepancies: Sequence[AccountingDiscrepancy], + ) -> None: + self.discrepancies: tuple[AccountingDiscrepancy, ...] = tuple( + discrepancies + ) + super().__init__(message) + + def to_dict(self) -> dict[str, object]: + """Return a JSON-safe view of the refusal.""" + return { + "interface_version": ACCOUNTING_INTERFACE_VERSION, + "error": "population_reconciliation_error", + "message": str(self), + "discrepancies": [item.to_dict() for item in self.discrepancies], + } + + +def _as_person_id(value: object, label: str) -> int: + """Coerce one identifier to ``int``, rejecting bools and floats.""" + if isinstance(value, bool) or not isinstance(value, (int, np.integer)): + raise PopulationAccountingInputError( + f"{label} must be an integer person identifier, " + f"got {value!r} of type {type(value).__name__}" + ) + return _in_integer_domain(int(value), label) + + +def _as_year(value: object, label: str) -> int: + """Coerce one calendar year to ``int``, rejecting bools and floats.""" + if isinstance(value, bool) or not isinstance(value, (int, np.integer)): + raise PopulationAccountingInputError( + f"{label} must be an integer year, got {value!r} of type " + f"{type(value).__name__}" + ) + return _in_integer_domain(int(value), label) + + +def _in_integer_domain(value: int, label: str) -> int: + """Enforce the same signed int64 domain before any array cast.""" + if not _INT64_MIN <= value <= _INT64_MAX: + raise PopulationAccountingInputError( + f"{label} must fit in signed int64, got {value!r}" + ) + return value + + +def _stable_sum(values: Iterable[float]) -> float: + """Sum binary64 components and refuse an unrepresentable summary.""" + try: + result = math.fsum(float(value) for value in values) + except (OverflowError, ValueError) as error: + raise PopulationAccountingInputError( + "weight summary is not representable with finite binary64 " + "arithmetic" + ) from error + if not math.isfinite(result): + raise PopulationAccountingInputError("weight summary is not finite") + return result + + +def _as_weight(value: object, label: str) -> float: + """Validate a real numeric weight before converting it to binary64.""" + if isinstance(value, (bool, np.bool_)) or not isinstance( + value, (int, float, np.integer, np.floating) + ): + raise PopulationAccountingInputError( + f"{label} must be a real number, got {value!r}" + ) + if value < 0: + raise PopulationAccountingInputError( + f"{label} must be finite and non-negative, got {value!r}" + ) + try: + weight = float(value) + except (OverflowError, ValueError) as error: + raise PopulationAccountingInputError( + f"{label} is not representable as a finite binary64 weight" + ) from error + if not math.isfinite(weight) or weight < 0.0: + raise PopulationAccountingInputError( + f"{label} must be finite and non-negative, got {value!r}" + ) + if weight == 0.0 and value != 0: + raise PopulationAccountingInputError( + f"{label} underflows to zero in binary64 arithmetic" + ) + return weight + + +@dataclass(frozen=True) +class PopulationEvent: + """One declared arrival or departure inside one annual period. + + Parameters + ---------- + person_id: + The identifier that joins or leaves. Must match the identifier + used in the frames. + kind: + A member of :class:`PopulationEventKind`. Plain strings are + accepted and coerced. + year: + The period's *closing* year, which labels the period. + weight: + Optional explicit weight at the moment of the event. When + omitted, an addition inherits the weight of its closing-frame + row and an exit inherits the weight of its opening-frame row. + A person who both arrives and departs within the period appears + in neither frame and must therefore declare both weights. + reason: + Free text. Required, and required to be non-empty, for + ``other_entry`` and ``other_exit`` so that "something else" + is never silent. + source: + Free-text provenance, e.g. the adapter that emitted the record. + Collected into the account's provenance block. + """ + + person_id: int + kind: PopulationEventKind + year: int + weight: float | None = None + reason: str = "" + source: str = "" + + def __post_init__(self) -> None: + object.__setattr__( + self, + "person_id", + _as_person_id(self.person_id, "PopulationEvent.person_id"), + ) + try: + kind = PopulationEventKind(self.kind) + except ValueError as error: + raise PopulationAccountingInputError( + f"unknown population event kind {self.kind!r}; expected one " + f"of {sorted(item.value for item in PopulationEventKind)}" + ) from error + object.__setattr__(self, "kind", kind) + object.__setattr__( + self, "year", _as_year(self.year, "PopulationEvent.year") + ) + if self.weight is not None: + object.__setattr__( + self, + "weight", + _as_weight(self.weight, "PopulationEvent.weight"), + ) + for field_name in ("reason", "source"): + value = getattr(self, field_name) + if not isinstance(value, str): + raise PopulationAccountingInputError( + f"PopulationEvent.{field_name} must be a string, " + f"got {value!r}" + ) + if kind in REASON_REQUIRED_KINDS and not self.reason.strip(): + raise PopulationAccountingInputError( + f"a {kind.value!r} event must state a non-empty reason; " + "this module never books an unexplained change" + ) + + @property + def is_addition(self) -> bool: + """Whether this kind adds a person to the roster.""" + return self.kind in ADDITION_KINDS + + def to_dict(self) -> dict[str, object]: + """Return a JSON-safe view of the declaration.""" + return { + "person_id": self.person_id, + "kind": self.kind.value, + "year": self.year, + "weight": self.weight, + "reason": self.reason, + "source": self.source, + } + + +@dataclass(frozen=True) +class AccountingDiscrepancy: + """One typed reason the declared story does not reconcile.""" + + kind: DiscrepancyKind + person_id: int | None + detail: str + + def to_dict(self) -> dict[str, object]: + """Return a JSON-safe view of the finding.""" + return { + "kind": self.kind.value, + "person_id": self.person_id, + "detail": self.detail, + } + + +@dataclass(frozen=True) +class PersonCounts: + """Exact integer person counts for one period. + + ``closing == opening + additions_total - exits_total`` holds + exactly, including transients, which are counted in both + ``additions_total`` and ``exits_total`` and cancel. + """ + + opening: int + closing: int + carried: int + entered: int + exited: int + transient: int + additions_total: int + exits_total: int + additions_by_kind: Mapping[str, int] + exits_by_kind: Mapping[str, int] + + def to_dict(self) -> dict[str, object]: + """Return a JSON-safe view of the counts.""" + return { + "opening": self.opening, + "closing": self.closing, + "carried": self.carried, + "entered": self.entered, + "exited": self.exited, + "transient": self.transient, + "additions_total": self.additions_total, + "exits_total": self.exits_total, + "additions_by_kind": dict(self.additions_by_kind), + "exits_by_kind": dict(self.exits_by_kind), + } + + +@dataclass(frozen=True) +class WeightRevaluation: + """Weight movement that is *not* an arrival or a departure. + + Each component is the sum of ``weight at the end of the person's + presence minus weight at the start of it`` over one presence class. + ``carried`` is the component the caller usually wants: it is the + entire change in the weight of people who were present at both ends + of the period. The other three are zero unless the caller declared + an explicit event weight that differs from the frame weight. + """ + + carried: float + entrant: float + exiting: float + transient: float + + @property + def total(self) -> float: + """Stable sum of the four binary64 components.""" + return _stable_sum( + (self.carried, self.entrant, self.exiting, self.transient) + ) + + def to_dict(self) -> dict[str, object]: + """Return a JSON-safe view of the revaluation.""" + return { + "carried": self.carried, + "entrant": self.entrant, + "exiting": self.exiting, + "transient": self.transient, + "total": self.total, + } + + +@dataclass(frozen=True) +class WeightTotals: + """Weight stocks and flows for one period. + + The identity is:: + + closing == opening + + additions_total + - exits_total + + revaluation.total + + reported against an arithmetic residual rather than a tolerance. + """ + + opening: float + closing: float + additions_total: float + exits_total: float + additions_by_kind: Mapping[str, float] + exits_by_kind: Mapping[str, float] + revaluation: WeightRevaluation + + def to_dict(self) -> dict[str, object]: + """Return a JSON-safe view of the weight totals.""" + return { + "opening": self.opening, + "closing": self.closing, + "additions_total": self.additions_total, + "exits_total": self.exits_total, + "additions_by_kind": dict(self.additions_by_kind), + "exits_by_kind": dict(self.exits_by_kind), + "revaluation": self.revaluation.to_dict(), + } + + +@dataclass(frozen=True) +class PeriodAccount: + """The reconciled stock-flow account for one annual period.""" + + opening_year: int + closing_year: int + status: str + status_note: str + counts: PersonCounts + weights: WeightTotals + count_residual: int + weight_residual: float + reconstructed_closing_weight: float + carried_person_ids: tuple[int, ...] + entered_person_ids: tuple[int, ...] + exited_person_ids: tuple[int, ...] + transient_person_ids: tuple[int, ...] + provenance: Mapping[str, object] + + @property + def added_person_ids(self) -> tuple[int, ...]: + """Every declared addition: entrants plus transients, sorted.""" + return tuple( + sorted(self.entered_person_ids + self.transient_person_ids) + ) + + @property + def departed_person_ids(self) -> tuple[int, ...]: + """Every declared exit: exiters plus transients, sorted.""" + return tuple( + sorted(self.exited_person_ids + self.transient_person_ids) + ) + + def to_dict(self) -> dict[str, object]: + """Return the serializable counts, weights and provenance. + + Person identifier tuples stay off this payload on purpose: they + are available as attributes for programmatic use, but a + serialized account is a summary, not a roster. + """ + return { + "interface_version": ACCOUNTING_INTERFACE_VERSION, + "status": self.status, + "status_note": self.status_note, + "opening_year": self.opening_year, + "closing_year": self.closing_year, + "counts": self.counts.to_dict(), + "weights": self.weights.to_dict(), + "residuals": { + "count": self.count_residual, + "weight": self.weight_residual, + "reconstructed_closing_weight": ( + self.reconstructed_closing_weight + ), + }, + "provenance": { + **self.provenance, + "declaration_sources": list( + self.provenance["declaration_sources"] + ), + }, + } + + +def _integer_column( + frame: pd.DataFrame, column: str, label: str +) -> np.ndarray: + """Return one non-null integral column as ``int64``. + + An empty column is accepted whatever its dtype: it holds no value + that could be non-integral, and the naive way to spell an empty + population, ``pd.DataFrame({"person_id": [], ...})``, yields float + columns that carry no information about the caller's intent. + """ + series = frame[column] + if len(series) == 0: + return np.empty(0, dtype=np.int64) + if series.isna().any(): + raise PopulationAccountingInputError( + f"{label} column {column!r} contains null values" + ) + if pd.api.types.is_bool_dtype(series.dtype): + raise PopulationAccountingInputError( + f"{label} column {column!r} is boolean, not integral" + ) + if pd.api.types.is_integer_dtype(series.dtype): + _in_integer_domain(int(series.min()), f"{label}.{column}") + _in_integer_domain(int(series.max()), f"{label}.{column}") + return series.to_numpy(dtype=np.int64, copy=True) + if series.dtype == object: + values = series.tolist() + for value in values: + if isinstance(value, bool) or not isinstance( + value, (int, np.integer) + ): + raise PopulationAccountingInputError( + f"{label} column {column!r} holds a non-integral value " + f"{value!r} of type {type(value).__name__}" + ) + return np.asarray( + [ + _in_integer_domain(int(value), f"{label}.{column}") + for value in values + ], + dtype=np.int64, + ) + raise PopulationAccountingInputError( + f"{label} column {column!r} must be an integer dtype, got " + f"{series.dtype!r}; float identifiers and years are rejected " + "because they cannot be compared exactly" + ) + + +def _weight_column(frame: pd.DataFrame, label: str) -> np.ndarray: + """Return the weight column as finite, non-negative ``float64``. + + As with identifiers, an empty column is accepted whatever its + dtype. + """ + series = frame[WEIGHT_COLUMN] + if len(series) == 0: + return np.empty(0, dtype=np.float64) + if series.isna().any(): + raise PopulationAccountingInputError( + f"{label} column {WEIGHT_COLUMN!r} contains null values" + ) + if pd.api.types.is_bool_dtype(series.dtype): + raise PopulationAccountingInputError( + f"{label} column {WEIGHT_COLUMN!r} is boolean, not numeric" + ) + if series.dtype == object: + return np.asarray( + [ + _as_weight(value, f"{label}.{WEIGHT_COLUMN}") + for value in series.tolist() + ], + dtype=np.float64, + ) + if not pd.api.types.is_numeric_dtype( + series.dtype + ) or pd.api.types.is_complex_dtype(series.dtype): + raise PopulationAccountingInputError( + f"{label} column {WEIGHT_COLUMN!r} must contain real numbers" + ) + if (series < 0).any(): + raise PopulationAccountingInputError( + f"{label} column {WEIGHT_COLUMN!r} contains negative weights" + ) + try: + values = series.to_numpy(dtype=np.float64, copy=True) + except (TypeError, ValueError, OverflowError) as error: + raise PopulationAccountingInputError( + f"{label} column {WEIGHT_COLUMN!r} is not numeric " + f"({series.dtype!r})" + ) from error + if not np.isfinite(values).all(): + raise PopulationAccountingInputError( + f"{label} column {WEIGHT_COLUMN!r} contains non-finite weights" + ) + if (values < 0.0).any(): + raise PopulationAccountingInputError( + f"{label} column {WEIGHT_COLUMN!r} contains negative weights" + ) + if ((values == 0.0) & series.ne(0).to_numpy(dtype=bool)).any(): + raise PopulationAccountingInputError( + f"{label} column {WEIGHT_COLUMN!r} underflows to zero in binary64" + ) + return values + + +def _read_frame( + frame: pd.DataFrame, year: int, label: str +) -> tuple[np.ndarray, np.ndarray]: + """Validate one population frame and snapshot its two columns. + + The frame is only read. Nothing is assigned, sorted in place, or + otherwise mutated, and the returned arrays are fresh. + """ + if not isinstance(frame, pd.DataFrame): + raise PopulationAccountingInputError( + f"{label} must be a pandas DataFrame, got " + f"{type(frame).__name__}" + ) + missing = [ + column for column in REQUIRED_COLUMNS if column not in frame.columns + ] + if missing: + raise PopulationAccountingInputError( + f"{label} is missing columns {missing}" + ) + if frame.columns.duplicated().any(): + raise PopulationAccountingInputError( + f"{label} has duplicate column labels" + ) + person_ids = _integer_column(frame, PERSON_ID_COLUMN, label) + unique_ids, counts = np.unique(person_ids, return_counts=True) + if unique_ids.size != person_ids.size: + repeated = unique_ids[counts > 1][:10].tolist() + raise PopulationAccountingInputError( + f"{label} contains duplicate {PERSON_ID_COLUMN} rows: {repeated}" + ) + years = _integer_column(frame, YEAR_COLUMN, label) + off_year = np.unique(years[years != year])[:10].tolist() + if off_year: + raise PopulationAccountingInputError( + f"{label} must carry year {year}; found {off_year}" + ) + weights = _weight_column(frame, label) + return person_ids, weights + + +def _index_events( + events: Sequence[PopulationEvent], + *, + expected_kinds: frozenset[PopulationEventKind], + closing_year: int, + label: str, + duplicate_kind: DiscrepancyKind, + discrepancies: list[AccountingDiscrepancy], +) -> dict[int, PopulationEvent]: + """Validate one declaration sequence and index it by person.""" + if isinstance(events, (str, bytes)) or not isinstance(events, Sequence): + raise PopulationAccountingInputError( + f"{label} must be a sequence of PopulationEvent, got " + f"{type(events).__name__}" + ) + indexed: dict[int, PopulationEvent] = {} + for position, event in enumerate(events): + if not isinstance(event, PopulationEvent): + raise PopulationAccountingInputError( + f"{label}[{position}] must be a PopulationEvent, got " + f"{type(event).__name__}" + ) + if event.kind not in expected_kinds: + raise PopulationAccountingInputError( + f"{label}[{position}] declares kind {event.kind.value!r}, " + f"which is not one of " + f"{sorted(item.value for item in expected_kinds)}; " + "arrivals and departures are declared separately" + ) + if event.year != closing_year: + raise PopulationAccountingInputError( + f"{label}[{position}] is booked to year {event.year}, but " + f"this period closes in {closing_year}; events are booked " + "to the period's closing year" + ) + if event.person_id in indexed: + discrepancies.append( + AccountingDiscrepancy( + kind=duplicate_kind, + person_id=event.person_id, + detail=( + f"person {event.person_id} is declared more than " + f"once in {label} " + f"({indexed[event.person_id].kind.value!r} then " + f"{event.kind.value!r})" + ), + ) + ) + continue + indexed[event.person_id] = event + return indexed + + +def _resolve_weight( + person_id: int, + event: PopulationEvent, + fallback: Mapping[int, float], + *, + fallback_label: str, +) -> float: + """Return the declared event weight, or its frame stand-in.""" + if event.weight is not None: + return event.weight + try: + return fallback[person_id] + except KeyError: + raise PopulationAccountingInputError( + f"person {person_id} is declared as a " + f"{event.kind.value!r} but appears in neither the opening nor " + "the closing frame, so this module cannot recover a weight for " + f"the event from the {fallback_label} frame; a person who both " + "arrives and departs within the period must declare an " + "explicit weight on both declarations" + ) from None + + +def reconcile_period( + opening: pd.DataFrame, + closing: pd.DataFrame, + *, + opening_year: int, + closing_year: int, + additions: Sequence[PopulationEvent] = (), + exits: Sequence[PopulationEvent] = (), +) -> PeriodAccount: + """Reconcile one annual period against its declared transitions. + + This is the module's only entry point. It is a pure function: it + reads the two frames and the two declaration sequences, mutates + nothing, touches no filesystem, and returns a + :class:`PeriodAccount` or raises. + + Parameters + ---------- + opening, closing: + Person-level frames carrying ``person_id``, ``year`` and + ``weight``. Identifiers must be integral and unique within + each frame; weights must be finite and non-negative. Rows with + zero weight are ordinary persons and are never dropped. + opening_year, closing_year: + The period's endpoints, stated explicitly rather than inferred, + so an empty frame is unambiguous. ``closing_year`` must be + ``opening_year + 1``. + additions, exits: + The declared arrivals and departures, each a sequence of + :class:`PopulationEvent`. Passing a departure kind in + ``additions`` (or the reverse) is an input error. + + Returns + ------- + PeriodAccount + Counts, weight stocks and flows, arithmetic residuals and + provenance. Its ``status`` is always + :data:`ENGINEERING_STATUS`: a returned account is an + engineering statement about arithmetic and identity, never a + scientific verdict. + + Raises + ------ + PopulationAccountingInputError + A frame or a declaration is malformed: a missing column, a null + or non-integral identifier, a duplicate identifier within one + frame, a non-finite or negative weight, a row carrying the + wrong year, a declaration booked to the wrong year, a + declaration of the wrong direction, or a transient whose weight + cannot be recovered. + PopulationReconciliationError + The inputs are well formed but the declared story does not + match the frames: an unexplained arrival, an omitted exit, a + duplicate or colliding declaration, or an impossible sequence. + The typed findings are on the exception's ``discrepancies``. + + Notes + ----- + Validation is staged, and the first stage to find a problem raises: + frames, then declarations, then reconciliation, then weights. A + caller with several problems at once therefore sees the earliest, + not all of them. + + The accountant sees exactly one period. It has no memory of + earlier ones and no view of what the loop *should* have scheduled; + it is not a schedule builder. A person who exits in one period and + reappears later is an unexplained arrival only if no addition is + declared. The accountant cannot detect declared reuse of a past ID, + or a transient omitted from both event sequences. + """ + opening_year = _as_year(opening_year, "opening_year") + closing_year = _as_year(closing_year, "closing_year") + if closing_year != opening_year + 1: + raise PopulationAccountingInputError( + f"closing_year must be opening_year + 1 (this is an annual " + f"accountant); got opening_year={opening_year} and " + f"closing_year={closing_year}" + ) + + opening_ids, opening_weights = _read_frame( + opening, opening_year, "opening frame" + ) + closing_ids, closing_weights = _read_frame( + closing, closing_year, "closing frame" + ) + opening_weight_by_person = dict( + zip(opening_ids.tolist(), opening_weights.tolist(), strict=True) + ) + closing_weight_by_person = dict( + zip(closing_ids.tolist(), closing_weights.tolist(), strict=True) + ) + opening_set = set(opening_weight_by_person) + closing_set = set(closing_weight_by_person) + + discrepancies: list[AccountingDiscrepancy] = [] + additions_by_person = _index_events( + additions, + expected_kinds=ADDITION_KINDS, + closing_year=closing_year, + label="additions", + duplicate_kind=DiscrepancyKind.DUPLICATE_ADDITION, + discrepancies=discrepancies, + ) + exits_by_person = _index_events( + exits, + expected_kinds=EXIT_KINDS, + closing_year=closing_year, + label="exits", + duplicate_kind=DiscrepancyKind.DUPLICATE_EXIT, + discrepancies=discrepancies, + ) + + for person_id in sorted(additions_by_person): + if person_id in opening_set: + discrepancies.append( + AccountingDiscrepancy( + kind=DiscrepancyKind.ADDITION_COLLIDES_WITH_OPENING, + person_id=person_id, + detail=( + f"person {person_id} is declared as a " + f"{additions_by_person[person_id].kind.value!r} but " + "was already present in the opening frame" + ), + ) + ) + for person_id in sorted(exits_by_person): + if person_id not in opening_set and person_id not in ( + additions_by_person + ): + discrepancies.append( + AccountingDiscrepancy( + kind=DiscrepancyKind.EXIT_WITHOUT_PRESENCE, + person_id=person_id, + detail=( + f"person {person_id} is declared as a " + f"{exits_by_person[person_id].kind.value!r} but was " + "never present: absent from the opening frame and " + "never declared as an addition" + ), + ) + ) + if person_id in closing_set: + discrepancies.append( + AccountingDiscrepancy( + kind=DiscrepancyKind.EXIT_CONTRADICTED_BY_CLOSING, + person_id=person_id, + detail=( + f"person {person_id} is declared as a " + f"{exits_by_person[person_id].kind.value!r} but is " + "still present in the closing frame" + ), + ) + ) + for person_id in sorted(closing_set - opening_set): + if person_id not in additions_by_person: + discrepancies.append( + AccountingDiscrepancy( + kind=DiscrepancyKind.UNDECLARED_ADDITION, + person_id=person_id, + detail=( + f"person {person_id} appears in the closing frame " + "with no declared addition; this module will not " + "guess whether the identifier is a birth, a " + "scheduled entry, or a defect" + ), + ) + ) + for person_id in sorted(opening_set - closing_set): + if person_id not in exits_by_person: + discrepancies.append( + AccountingDiscrepancy( + kind=DiscrepancyKind.UNDECLARED_EXIT, + person_id=person_id, + detail=( + f"person {person_id} disappears between the frames " + "with no declared exit; this module will not assume " + "the person died" + ), + ) + ) + for person_id in sorted(additions_by_person): + if ( + person_id not in closing_set + and person_id not in exits_by_person + and person_id not in opening_set + ): + discrepancies.append( + AccountingDiscrepancy( + kind=DiscrepancyKind.ADDITION_ABSENT_AT_CLOSE, + person_id=person_id, + detail=( + f"person {person_id} is declared as a " + f"{additions_by_person[person_id].kind.value!r} but " + "is absent from the closing frame and has no " + "declared exit" + ), + ) + ) + + if discrepancies: + raise PopulationReconciliationError( + _summarize(discrepancies, opening_year, closing_year), + _sorted_discrepancies(discrepancies), + ) + + carried = tuple(sorted(opening_set & closing_set)) + entered = tuple(sorted(closing_set - opening_set)) + exited = tuple(sorted(opening_set - closing_set)) + transient = tuple( + sorted(set(additions_by_person) - opening_set - closing_set) + ) + + counts = PersonCounts( + opening=len(opening_set), + closing=len(closing_set), + carried=len(carried), + entered=len(entered), + exited=len(exited), + transient=len(transient), + additions_total=len(additions_by_person), + exits_total=len(exits_by_person), + additions_by_kind=_count_by_kind(additions_by_person, ADDITION_KINDS), + exits_by_kind=_count_by_kind(exits_by_person, EXIT_KINDS), + ) + count_residual = counts.closing - ( + counts.opening + counts.additions_total - counts.exits_total + ) + if count_residual != 0: + raise PopulationReconciliationError( + "the exact person-count identity does not hold: " + f"closing {counts.closing} != opening {counts.opening} + " + f"additions {counts.additions_total} - exits " + f"{counts.exits_total}", + ( + AccountingDiscrepancy( + kind=DiscrepancyKind.COUNT_IDENTITY_VIOLATION, + person_id=None, + detail=f"count residual {count_residual}", + ), + ), + ) + + entry_weight = { + person_id: _resolve_weight( + person_id, + event, + closing_weight_by_person, + fallback_label="closing", + ) + for person_id, event in additions_by_person.items() + } + exit_weight = { + person_id: _resolve_weight( + person_id, + event, + opening_weight_by_person, + fallback_label="opening", + ) + for person_id, event in exits_by_person.items() + } + + revaluation = WeightRevaluation( + carried=_stable_sum( + closing_weight_by_person[person_id] + - opening_weight_by_person[person_id] + for person_id in carried + ), + entrant=_stable_sum( + closing_weight_by_person[person_id] - entry_weight[person_id] + for person_id in entered + ), + exiting=_stable_sum( + exit_weight[person_id] - opening_weight_by_person[person_id] + for person_id in exited + ), + transient=_stable_sum( + exit_weight[person_id] - entry_weight[person_id] + for person_id in transient + ), + ) + additions_by_kind_weight = _weight_by_kind( + additions_by_person, entry_weight, ADDITION_KINDS + ) + exits_by_kind_weight = _weight_by_kind( + exits_by_person, exit_weight, EXIT_KINDS + ) + weights = WeightTotals( + opening=_stable_sum(opening_weights.tolist()), + closing=_stable_sum(closing_weights.tolist()), + additions_total=_stable_sum(entry_weight.values()), + exits_total=_stable_sum(exit_weight.values()), + additions_by_kind=additions_by_kind_weight, + exits_by_kind=exits_by_kind_weight, + revaluation=revaluation, + ) + reconstructed = _stable_sum( + ( + weights.opening, + weights.additions_total, + -weights.exits_total, + revaluation.total, + ) + ) + weight_residual = _stable_sum((weights.closing, -reconstructed)) + + provenance = MappingProxyType( + { + "interface_version": ACCOUNTING_INTERFACE_VERSION, + "person_id_column": PERSON_ID_COLUMN, + "year_column": YEAR_COLUMN, + "weight_column": WEIGHT_COLUMN, + "summation": SUMMATION_METHOD, + "inference": INFERENCE_POLICY, + "opening_rows": int(opening_ids.size), + "closing_rows": int(closing_ids.size), + "opening_zero_weight_rows": int( + np.count_nonzero(opening_weights == 0.0) + ), + "closing_zero_weight_rows": int( + np.count_nonzero(closing_weights == 0.0) + ), + "declared_additions": len(additions_by_person), + "declared_exits": len(exits_by_person), + "declared_addition_weights": sum( + 1 + for event in additions_by_person.values() + if event.weight is not None + ), + "declared_exit_weights": sum( + 1 + for event in exits_by_person.values() + if event.weight is not None + ), + "event_log_completeness_verified": False, + "declaration_sources": tuple( + sorted( + { + event.source + for event in ( + *additions_by_person.values(), + *exits_by_person.values(), + ) + if event.source + } + ) + ), + } + ) + + return PeriodAccount( + opening_year=opening_year, + closing_year=closing_year, + status=ENGINEERING_STATUS, + status_note=ENGINEERING_STATUS_NOTE, + counts=counts, + weights=weights, + count_residual=count_residual, + weight_residual=weight_residual, + reconstructed_closing_weight=reconstructed, + carried_person_ids=carried, + entered_person_ids=entered, + exited_person_ids=exited, + transient_person_ids=transient, + provenance=provenance, + ) + + +def _count_by_kind( + indexed: Mapping[int, PopulationEvent], + kinds: frozenset[PopulationEventKind], +) -> Mapping[str, int]: + """Count declarations by kind, with every kind present as a key.""" + tally = {kind.value: 0 for kind in sorted(kinds, key=lambda k: k.value)} + for event in indexed.values(): + tally[event.kind.value] += 1 + return MappingProxyType(tally) + + +def _weight_by_kind( + indexed: Mapping[int, PopulationEvent], + resolved: Mapping[int, float], + kinds: frozenset[PopulationEventKind], +) -> Mapping[str, float]: + """Sum resolved event weights by kind, stably, with all keys present.""" + grouped: dict[str, list[float]] = { + kind.value: [] for kind in sorted(kinds, key=lambda k: k.value) + } + for person_id, event in indexed.items(): + grouped[event.kind.value].append(resolved[person_id]) + return MappingProxyType( + {kind: _stable_sum(values) for kind, values in grouped.items()} + ) + + +def _sorted_discrepancies( + discrepancies: Iterable[AccountingDiscrepancy], +) -> tuple[AccountingDiscrepancy, ...]: + """Order findings deterministically by kind then person.""" + return tuple( + sorted( + discrepancies, + key=lambda item: ( + _DISCREPANCY_ORDER[item.kind], + -1 if item.person_id is None else item.person_id, + ), + ) + ) + + +def _summarize( + discrepancies: Sequence[AccountingDiscrepancy], + opening_year: int, + closing_year: int, +) -> str: + """Build a stable one-line summary of a refusal.""" + tally: dict[str, int] = {} + for item in discrepancies: + tally[item.kind.value] = tally.get(item.kind.value, 0) + 1 + rendered = ", ".join( + f"{kind}={count}" for kind, count in sorted(tally.items()) + ) + return ( + f"population accounting for {opening_year}->{closing_year} does not " + f"reconcile: {rendered}" + ) diff --git a/src/populace_dynamics/engine/accounting_history.py b/src/populace_dynamics/engine/accounting_history.py new file mode 100644 index 00000000..c61441cd --- /dev/null +++ b/src/populace_dynamics/engine/accounting_history.py @@ -0,0 +1,321 @@ +"""Optional accounting of supplied annual frames under an explicit ID contract. + +This module adds boundary and identity-history checks to ``reconcile_period``. +It neither generates events nor establishes their truth or completeness. The +only supported identity contract allows one presence episode per person ID +within the supplied history; same-ID return requires a different contract. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from enum import Enum +from types import MappingProxyType + +import pandas as pd + +from . import accounting + +__all__ = [ + "HISTORY_INTERFACE_VERSION", + "SINGLE_PRESENCE_EPISODE", + "AnnualTransition", + "HistoryAccount", + "HistoryAccountingError", + "HistoryErrorKind", + "reconcile_history", +] + +HISTORY_INTERFACE_VERSION = "accounting-history/0.1.0-experimental" +SINGLE_PRESENCE_EPISODE = "single_presence_episode" + + +class HistoryErrorKind(str, Enum): + """The stage at which a supplied history is refused.""" + + INPUT = "input" + PERIOD = "period" + NONCONTIGUOUS = "noncontiguous" + BOUNDARY_PERSON_IDS = "boundary_person_ids" + BOUNDARY_WEIGHTS = "boundary_weights" + RETIRED_PERSON_ID = "retired_person_id" + + +class HistoryAccountingError(accounting.PopulationAccountingError): + """A history refusal, with a zero-based period index when applicable. + + Annual input/reconciliation failures retain the original exception as + ``__cause__``, including its typed discrepancies. No partial history + account is returned on failure. + + Pickling and copying use the base refusal's ``__reduce__``: they keep + the kind, period index, years, person IDs, the already-prefixed message + and the ``__cause__``. ``concurrent.futures`` and ``multiprocessing`` + pools replace ``__cause__`` with their remote traceback text after + unpickling, so a pool caller sees these fields but not the typed cause. + """ + + def __init__( + self, + message: str, + *, + kind: HistoryErrorKind, + period_index: int | None = None, + opening_year: int | None = None, + closing_year: int | None = None, + person_ids: tuple[int, ...] = (), + ) -> None: + self.kind = kind + self.period_index = period_index + self.opening_year = opening_year + self.closing_year = closing_year + self.person_ids = person_ids + prefix = ( + "history" if period_index is None else f"period[{period_index}]" + ) + super().__init__(f"{prefix}: {message}") + + def to_dict(self) -> dict[str, object]: + """Return a JSON-safe refusal without serializing input containers.""" + cause = self.__cause__ + return { + "interface_version": HISTORY_INTERFACE_VERSION, + "error": "history_accounting_error", + "kind": self.kind.value, + "message": str(self), + "period_index": self.period_index, + "opening_year": self.opening_year, + "closing_year": self.closing_year, + "person_ids": list(self.person_ids), + "cause": ( + { + "type": type(cause).__name__, + "message": str(cause), + "details": ( + cause.to_dict() + if isinstance( + cause, accounting.PopulationReconciliationError + ) + else None + ), + } + if cause is not None + else None + ), + } + + +@dataclass(frozen=True) +class AnnualTransition: + """One supplied period, without ownership of the caller's containers. + + Freezing this descriptor prevents field reassignment. The DataFrames and + any declaration lists remain caller-owned and mutable; reconciliation + reads snapshots and never stores those containers in its result. + """ + + opening_year: int + closing_year: int + opening: pd.DataFrame + closing: pd.DataFrame + additions: Sequence[accounting.PopulationEvent] = () + exits: Sequence[accounting.PopulationEvent] = () + + +@dataclass(frozen=True) +class HistoryAccount: + """Immutable annual accounts and the IDs observed within this history.""" + + identity_contract: str + periods: tuple[accounting.PeriodAccount, ...] + seen_person_ids: tuple[int, ...] + retired_person_ids: tuple[int, ...] + provenance: Mapping[str, object] + + def to_dict(self) -> dict[str, object]: + """Return fresh summary containers, omitting the identifier rosters.""" + return { + "interface_version": HISTORY_INTERFACE_VERSION, + "status": accounting.ENGINEERING_STATUS, + "identity_contract": self.identity_contract, + "periods": [period.to_dict() for period in self.periods], + "provenance": dict(self.provenance), + } + + +def _snapshot_frame( + frame: pd.DataFrame, year: int, label: str +) -> tuple[pd.DataFrame, dict[int, float]]: + # Reuse the accountant's strict domain before conversion. These helpers + # are internal to the same package; annual law remains reconcile_period. + ids, weights = accounting._read_frame(frame, year, label) + return ( + pd.DataFrame( + { + accounting.PERSON_ID_COLUMN: ids, + accounting.YEAR_COLUMN: year, + accounting.WEIGHT_COLUMN: weights, + } + ), + dict(zip(ids.tolist(), weights.tolist(), strict=True)), + ) + + +def _snapshot_events(events: object) -> object: + # Preserve malformed inputs for the accountant's own typed validation. + if isinstance(events, Sequence) and not isinstance(events, (str, bytes)): + return tuple(events) + return events + + +def reconcile_history( + transitions: Sequence[AnnualTransition], *, identity_contract: str +) -> HistoryAccount: + """Reconcile nonempty, ordered, contiguous annual periods. + + ``identity_contract`` must explicitly be ``single_presence_episode``. + Every declared departure retires its ID for the supplied history under + that chosen constraint, regardless of cause. This does not assert that + emigration or another exit is demographically permanent. A same-ID return + is unsupported, including a return declared as a transient. + + Adjacent snapshots must agree on the person-ID set and exact binary64 + weight per ID, independent of row order. Other columns are not compared. + The first opening roster supplies no prehistory. Callers must not mutate + input containers concurrently while this function reads them. + """ + if ( + not isinstance(identity_contract, str) + or identity_contract != SINGLE_PRESENCE_EPISODE + ): + raise HistoryAccountingError( + "identity_contract must explicitly be 'single_presence_episode'; " + "general same-person re-entry requires a separate typed contract", + kind=HistoryErrorKind.INPUT, + ) + if ( + isinstance(transitions, (str, bytes)) + or not isinstance(transitions, Sequence) + or not transitions + ): + raise HistoryAccountingError( + "transitions must be a nonempty ordered sequence of " + "AnnualTransition", + kind=HistoryErrorKind.INPUT, + ) + + periods: list[accounting.PeriodAccount] = [] + seen: set[int] = set() + retired: set[int] = set() + previous_closing: dict[int, float] = {} + for index, transition in enumerate(tuple(transitions)): + opening_year = closing_year = None + try: + if not isinstance(transition, AnnualTransition): + raise accounting.PopulationAccountingInputError( + "transition must be an AnnualTransition" + ) + opening_year = accounting._as_year( + transition.opening_year, "opening_year" + ) + closing_year = accounting._as_year( + transition.closing_year, "closing_year" + ) + opening, opening_by_id = _snapshot_frame( + transition.opening, opening_year, "opening frame" + ) + closing, closing_by_id = _snapshot_frame( + transition.closing, closing_year, "closing frame" + ) + account = accounting.reconcile_period( + opening, + closing, + opening_year=opening_year, + closing_year=closing_year, + additions=_snapshot_events(transition.additions), + exits=_snapshot_events(transition.exits), + ) + except accounting.PopulationAccountingError as error: + raise HistoryAccountingError( + str(error), + kind=HistoryErrorKind.PERIOD, + period_index=index, + opening_year=opening_year, + closing_year=closing_year, + ) from error + + context = { + "period_index": index, + "opening_year": opening_year, + "closing_year": closing_year, + } + if periods: + if opening_year != periods[-1].closing_year: + raise HistoryAccountingError( + "periods must be ordered and contiguous: " + f"expected opening year {periods[-1].closing_year}", + kind=HistoryErrorKind.NONCONTIGUOUS, + **context, + ) + differing_ids = previous_closing.keys() ^ opening_by_id.keys() + if differing_ids: + raise HistoryAccountingError( + "shared boundary person-ID sets differ", + kind=HistoryErrorKind.BOUNDARY_PERSON_IDS, + person_ids=tuple(sorted(differing_ids)), + **context, + ) + differing_weights = tuple( + sorted( + person_id + for person_id, weight in opening_by_id.items() + if weight != previous_closing[person_id] + ) + ) + if differing_weights: + raise HistoryAccountingError( + "shared boundary weights differ in the accountant's " + "binary64 domain", + kind=HistoryErrorKind.BOUNDARY_WEIGHTS, + person_ids=differing_weights, + **context, + ) + else: + seen.update(opening_by_id) + + reused = tuple(sorted(retired.intersection(account.added_person_ids))) + if reused: + raise HistoryAccountingError( + "a departed person ID was declared again; reuse and " + "same-person return are unsupported under the explicitly " + "selected single_presence_episode contract", + kind=HistoryErrorKind.RETIRED_PERSON_ID, + person_ids=reused, + **context, + ) + seen.update(account.added_person_ids) + retired.update(account.departed_person_ids) + periods.append(account) + previous_closing = closing_by_id + + return HistoryAccount( + identity_contract=SINGLE_PRESENCE_EPISODE, + periods=tuple(periods), + seen_person_ids=tuple(sorted(seen)), + retired_person_ids=tuple(sorted(retired)), + provenance=MappingProxyType( + { + "annual_accounting_interface": ( + accounting.ACCOUNTING_INTERFACE_VERSION + ), + "scope": "supplied history only; first opening has no prehistory", + "boundary_comparison": "person-ID set and binary64 weight per ID", + "other_column_continuity_verified": False, + "event_log_completeness_verified": False, + "event_truth_verified": False, + "retirement_policy": "all declared departures under the selected contract", + "general_reentry_supported": False, + } + ), + ) diff --git a/src/populace_dynamics/engine/claiming.py b/src/populace_dynamics/engine/claiming.py new file mode 100644 index 00000000..f368f211 --- /dev/null +++ b/src/populace_dynamics/engine/claiming.py @@ -0,0 +1,90 @@ +"""Opt-in claiming state adapter that excludes observed benefit entrants. + +The registered historical adapter remains in ``engine.steps``. Import this +module explicitly and inject its adapter into ``PeriodModules.claiming`` to +use the corrected no-new-draw partition; historical assembly is unchanged. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from populace_dynamics.engine.loop import PeriodContext +from populace_dynamics.engine.rng import ProjectionModule +from populace_dynamics.engine.steps import ClaimingSchedule + + +def apply_claiming( + frame: pd.DataFrame, + context: PeriodContext, + rng: np.random.Generator, + *, + schedule: ClaimingSchedule, +) -> pd.DataFrame: + """Draw behavioral plans only for unclaimed, nonconverted people. + + ``claim_age`` is a planned behavioral age, not the age of every benefit + entry. A conversion with no plan keeps a null age and becomes claimed; + the carried ``claimed`` state prevents a later draw after the conversion + event flag clears. Existing plans and previously claimed years remain. + + The historical age-50 threshold, PMFs, nearest-year selection, and keyed + claiming streams are retained. Without a registry, excluding rows changes + batch RNG consumption and can change other people's same-seed draws. + """ + missing = {"age", "sex"} - set(frame.columns) + if missing: + raise ValueError( + f"claiming frame is missing columns {sorted(missing)}" + ) + out = frame.copy() + if "claim_age" not in out: + out["claim_age"] = pd.array([pd.NA] * len(out), dtype="Int64") + age = out["age"].to_numpy(dtype=np.int64) + converted = ( + out.get("di_converted", pd.Series(False, index=out.index)) + .fillna(False) + .to_numpy(dtype=bool) + ) + previously_claimed = ( + out.get("claimed", pd.Series(False, index=out.index)) + .fillna(False) + .to_numpy(dtype=bool) + ) + unassigned = ( + out["claim_age"].isna().to_numpy() + & (age >= 50) + & ~converted + & ~previously_claimed + ) + sex_labels = out["sex"].astype(str).to_numpy() + for sex in sorted(np.unique(sex_labels[unassigned])): + rows = unassigned & (sex_labels == sex) + ages, probability = schedule.distribution(sex, context.year) + row_indices = np.flatnonzero(rows) + if context.rng_registry is None: + chosen = rng.choice(ages, size=len(row_indices), p=probability) + else: + chosen = np.asarray( + [ + context.person_generator( + ProjectionModule.CLAIMING, + out.iloc[index]["person_id"], + ).choice(ages, p=probability) + for index in row_indices + ] + ) + out.loc[rows, "claim_age"] = chosen + + assigned = out["claim_age"].notna().to_numpy() + plan_reached = np.zeros(len(out), dtype=bool) + plan_reached[assigned] = age[assigned] >= out.loc[ + assigned, "claim_age" + ].to_numpy(dtype=np.int64) + out["claimed"] = previously_claimed | converted | plan_reached + new_claim = out["claimed"].to_numpy(dtype=bool) & ~previously_claimed + if "claim_year" not in out: + out["claim_year"] = pd.array([pd.NA] * len(out), dtype="Int64") + out.loc[new_claim, "claim_year"] = context.year + return out diff --git a/src/populace_dynamics/engine/entrant_domains.py b/src/populace_dynamics/engine/entrant_domains.py new file mode 100644 index 00000000..0be9cd7b --- /dev/null +++ b/src/populace_dynamics/engine/entrant_domains.py @@ -0,0 +1,380 @@ +"""Experimental support restrictions for explicitly scheduled entrants. + +Recovered from local entrant work at 61bbf1c7e25a7e55033c134bbc2e846022b8850b. +The existing fitted 2014 earnings state and observed disability panel do not +supply histories for newly allocated IDs. The historical claiming adapter also +has no insured-status gate. This module therefore identifies unsupported rows +and offers an explicit claiming wrapper; it does not supply entrant behavior, +entitlement, or an admitted population. Fertility/disability ID inventories +and demographic scope declarations do not demonstrate step execution. + +A source-only recovery accepts caller-supplied synthetic frames. No native +donor, control release, fitted model, or scientific gate is invoked here. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from typing import Any + +import numpy as np +import pandas as pd + +from populace_dynamics.engine.earnings_domain import EARNINGS_DOMAIN_COLUMN +from populace_dynamics.engine.entrant_schedule import ( + ENTRY_KIND_BIRTH, + ENTRY_KIND_COLUMN, + ENTRY_KIND_IMMIGRANT, + ENTRY_KIND_INCUMBENT, + ENTRY_KIND_REALIZED_OPENER, +) +from populace_dynamics.engine.steps import materialize_maternal_births + +__all__ = [ + "EXCLUDED_DOMAINS", + "EntrantClaimingAdapter", + "EntrantExclusionReport", + "entrant_mask", + "materialize_births_with_provenance", + "assert_entrants_out_of_earnings_domain", + "excluded_claiming_ids", + "excluded_disability_ids", + "excluded_fertility_ids", + "exclusion_report", + "suppress_entrant_benefit_outputs", +] + +#: The four domains an entrant is excluded from, and why each one is unfitted. +EXCLUDED_DOMAINS: Mapping[str, str] = { + "fertility_risk": ( + "steps.py:451-506 materializes births only for on-roster mothers and " + "initializes parity at zero; no entrant parity or birth-history seed " + "exists, so leaving entrants at risk would assert every entrant " + "arrived childless" + ), + "claiming_eligibility": ( + "steps.py:390-450 draws a claim age for everyone aged 50+ with no " + "insured-status, quarters-of-coverage, AIME or PIA test; reported " + "year of entry does not identify FIRST entry, so prior US covered " + "earnings are unknown/censored rather than zero and insured status " + "cannot be established from either survey" + ), + "disability_panel": ( + "disability.py:56-60 filters a PSID-built DisabilityPanel to " + "holdout_ids; entrants are not in the panel, and ASEC/CPS disability " + "items are not realized PSID M4 status" + ), + "earnings_domain": ( + "earnings_domain.py:150-208 keys membership on the generator's fitted " + "2014 state and forward_earnings.py:1421-1436 raises without it; the " + "section 2.8.3a certificate was never fitted on entrants and does not " + "transfer to them" + ), +} + + +def entrant_mask( + frame: pd.DataFrame, + *, + entry_kinds: Iterable[str] = (ENTRY_KIND_IMMIGRANT,), +) -> np.ndarray: + """Boolean membership: is each row a scheduled entrant? + + Reads the explicit :data:`~populace_dynamics.engine.entrant_schedule.ENTRY_KIND_COLUMN` + provenance column. A frame without that column is a closed-panel frame + and every row is an incumbent -- that is the honest reading, and it keeps + this predicate safe to call on a roster that has never seen a schedule. + A missing value is also treated as incumbent unless the row is explicitly + synthetic. Synthetic rows require a known entry kind, so losing that + provenance cannot silently remove their exclusion. + """ + known = { + ENTRY_KIND_BIRTH, + ENTRY_KIND_INCUMBENT, + ENTRY_KIND_IMMIGRANT, + ENTRY_KIND_REALIZED_OPENER, + } + if isinstance(entry_kinds, (str, bytes)): + raise ValueError( + "entry_kinds must be a collection, not a scalar string" + ) + kinds = set(entry_kinds) + if not kinds: + raise ValueError("entrant_mask needs at least one entry kind") + if not kinds.issubset(known): + raise ValueError("entry_kinds contains unknown selectors") + values = frame.get( + ENTRY_KIND_COLUMN, pd.Series(pd.NA, index=frame.index) + ).to_numpy() + synthetic = frame.get( + "synthetic_entry", pd.Series(False, index=frame.index) + ).to_numpy() + for value, synthetic_value in zip(values, synthetic, strict=True): + if not pd.isna(synthetic_value) and not isinstance( + synthetic_value, (bool, np.bool_) + ): + raise ValueError( + "synthetic_entry must contain booleans or missing" + ) + is_synthetic = not pd.isna(synthetic_value) and synthetic_value + if pd.isna(value): + if is_synthetic: + raise ValueError( + "synthetic entrants require explicit entry_kind" + ) + elif not isinstance(value, str) or value not in known: + raise ValueError(f"unknown entry_kind {value!r}") + return np.asarray( + [(not pd.isna(value)) and str(value) in kinds for value in values], + dtype=bool, + ) + + +def materialize_births_with_provenance( + frame: pd.DataFrame, + births: pd.DataFrame, + context: Any, + rng: np.random.Generator, +) -> pd.DataFrame: + """Label children at the actual historical birth-materialization boundary. + + Existing synthetic rows must already have valid provenance. Only rows + allocated by this call receive the maternal-birth marker. This helper + handles supplied birth records; it does not fit/simulate fertility or + enforce the entrant fertility risk restriction. + """ + entrant_mask(frame) + out = materialize_maternal_births(frame, births, context, rng) + added = ~out["person_id"].isin(frame["person_id"]) + if ENTRY_KIND_COLUMN not in out: + out[ENTRY_KIND_COLUMN] = pd.Series( + pd.NA, index=out.index, dtype="object" + ) + out.loc[added, ENTRY_KIND_COLUMN] = ENTRY_KIND_BIRTH + return out + + +def _entrant_ids(frame: pd.DataFrame, **kwargs: Any) -> set[int]: + mask = entrant_mask(frame, **kwargs) + return {int(value) for value in frame.loc[mask, "person_id"]} + + +def excluded_fertility_ids(frame: pd.DataFrame, **kwargs: Any) -> set[int]: + """Person IDs to remove from the fertility risk set. + + This is an inventory, not a fertility adapter. Historical + ``apply_fertility`` treats an empty ``holdout_ids`` as all roster IDs and + its precomputed-birth path does not apply that argument. Passing this + set's complement therefore does not establish entrant exclusion. + """ + return _entrant_ids(frame, **kwargs) + + +def excluded_claiming_ids(frame: pd.DataFrame, **kwargs: Any) -> set[int]: + """Person IDs whose claiming draw must be suppressed.""" + return _entrant_ids(frame, **kwargs) + + +def excluded_disability_ids(frame: pd.DataFrame, **kwargs: Any) -> set[int]: + """Person IDs to keep out of the M4 disability panel's holdout set.""" + return _entrant_ids(frame, **kwargs) + + +def assert_entrants_out_of_earnings_domain(frame: pd.DataFrame) -> int: + """Fail loudly if any entrant is marked inside the fitted earnings domain. + + Returns the number of entrants checked. There is no "exclude" step to + perform here -- ``earnings_domain.membership`` already excludes them by + construction, because a synthetic ID cannot be in the fitted 2014 state + maps. This is the assertion that the construction was not circumvented, + which is the failure mode ``validate_domain`` exists to catch. + """ + mask = entrant_mask(frame) + if not mask.any(): + return 0 + if EARNINGS_DOMAIN_COLUMN not in frame.columns: + return int(mask.sum()) + marked = frame.loc[mask, EARNINGS_DOMAIN_COLUMN] + offending = marked.fillna(False).astype(bool).to_numpy() + if offending.any(): + bad = frame.loc[mask].loc[offending, "person_id"].tolist()[:10] + raise ValueError( + "scheduled entrants are marked inside the fitted earnings " + f"domain: {bad}; the section 2.8.3a certificate does not transfer " + "to a population it was never fitted on" + ) + return int(mask.sum()) + + +def suppress_entrant_benefit_outputs( + frame: pd.DataFrame, *, columns: Iterable[str] = ("aime", "pia", "benefit") +) -> pd.DataFrame: + """Blank entrant benefit outputs to missing, never to zero. + + A zero AIME is a *measurement*: it says this person had no covered + earnings. For an entrant it would be a fabrication, because reported year + of entry does not identify first entry, so prior US covered earnings are + censored rather than absent. Missing is the only honest value until a + stock-to-arrival and insured-status bridge exists. + """ + out = frame.copy() + mask = entrant_mask(out) + if not mask.any(): + return out + for column in columns: + if column in out.columns: + out.loc[mask, column] = pd.NA + return out + + +@dataclass(frozen=True) +class EntrantExclusionReport: + """What was excluded, from where, and how many -- for the run artifact.""" + + n_rows: int + n_entrants: int + excluded: dict[str, dict[str, Any]] + + def as_dict(self) -> dict[str, Any]: + return { + "n_rows": self.n_rows, + "n_entrants": self.n_entrants, + "excluded_domains": self.excluded, + "gated": False, + "status": "inventory_only", + "interpretation": ( + "an excluded entrant is OUTSIDE the estimand for that domain, " + "not a person modelled as having no children, no disability " + "and no claim; suppressed counts are reported instead of " + "zeros for exactly that reason" + ), + "execution_verified": False, + "intended_demographic_domains": [ + "mortality (age/sex draw at the entry age)", + "aging (deterministic advance)", + ], + "not_certified": ( + "entrant benefit levels, insured status, prior US covered " + "earnings, legal status, population stocks and every " + "post-entry transition remain out of scope" + ), + } + + +def exclusion_report(frame: pd.DataFrame) -> EntrantExclusionReport: + """Measure the exclusions on one roster frame.""" + mask = entrant_mask(frame) + n_entrants = int(mask.sum()) + age = ( + frame["age"].to_numpy(dtype=np.float64) + if "age" in frame.columns + else np.full(len(frame), np.nan) + ) + missing_plan = ( + frame.get("claim_age", pd.Series(pd.NA, index=frame.index)) + .isna() + .to_numpy() + ) + claim_exposed = int((mask & (age >= 50) & missing_plan).sum()) + female = ( + frame["sex"].astype(str).to_numpy() == "female" + if "sex" in frame.columns + else np.zeros(len(frame), dtype=bool) + ) + fertile_exposed = int((mask & female & (age >= 15) & (age <= 49)).sum()) + + excluded: dict[str, dict[str, Any]] = {} + for domain, reason in EXCLUDED_DOMAINS.items(): + record: dict[str, Any] = { + "n_excluded": n_entrants, + "reason": reason, + } + if domain == "claiming_eligibility": + record["n_would_have_drawn_a_claim_age"] = claim_exposed + record["counterfactual"] = ( + "without this exclusion apply_claiming would draw a claim age " + f"for {claim_exposed} entrant rows aged 50+ without a plan; " + "this inventory does not verify execution or entitlement" + ) + if domain == "fertility_risk": + record["n_would_have_been_at_risk"] = fertile_exposed + if domain == "earnings_domain": + record["n_checked_out_of_domain"] = ( + assert_entrants_out_of_earnings_domain(frame) + ) + excluded[domain] = record + return EntrantExclusionReport( + n_rows=int(len(frame)), n_entrants=n_entrants, excluded=excluded + ) + + +@dataclass(frozen=True) +class EntrantClaimingAdapter: + """Run a claiming step on incumbents only, leaving entrants unclaimed. + + The direct analogue of :class:`~populace_dynamics.engine.earnings_domain.EarningsDomainAdapter`: + it keeps a support restriction outside the historical step rather than + editing the historical step; entrant rows never reach that step. + + Existing entrant plans, claim years, claimed state, or conversion events + are rejected before the incumbent step runs. They require a separately + admitted claim-history path; this wrapper does not erase them or rule on + observed entitlement. For admitted inputs, those three fields come back as + missing / ``False`` / missing. ``claimed = False`` is not a behavioural + claim that entrants never retire -- it is the roster's structural default + for a person outside the claiming estimand, and + :func:`exclusion_report` publishes how many rows it applied to so the + suppression is never mistaken for a measured zero. + """ + + step: Any + entry_kinds: tuple[str, ...] = (ENTRY_KIND_IMMIGRANT,) + + def __call__( + self, + frame: pd.DataFrame, + context: Any, + rng: np.random.Generator, + ) -> pd.DataFrame: + mask = entrant_mask(frame, entry_kinds=self.entry_kinds) + if not mask.any(): + return self.step(frame, context, rng) + incumbents = frame.loc[~mask].copy() + entrants = frame.loc[mask].copy() + # Preserve possible observed entitlement by rejecting unsupported + # state, rather than overwriting it with an exclusion default. + for column in ("claim_age", "claim_year"): + if column in entrants and entrants[column].notna().any(): + raise ValueError( + f"excluded entrants have existing {column}; an admitted " + "claim-history path is required" + ) + for column in ("claimed", "di_converted"): + if column in entrants: + observed = entrants[column].dropna() + if any( + not isinstance(value, (bool, np.bool_)) or value + for value in observed + ): + raise ValueError( + f"excluded entrants have unsupported {column} state" + ) + advanced = ( + self.step(incumbents, context, rng) + if not incumbents.empty + else incumbents + ) + for column, default in ( + ("claim_age", pd.NA), + ("claimed", False), + ("claim_year", pd.NA), + ): + entrants[column] = pd.array( + [default] * len(entrants), + dtype="bool" if column == "claimed" else "Int64", + ) + out = pd.concat([advanced, entrants], ignore_index=True, sort=False) + return out.sort_values("person_id", kind="stable").reset_index( + drop=True + ) diff --git a/src/populace_dynamics/engine/entrant_schedule.py b/src/populace_dynamics/engine/entrant_schedule.py new file mode 100644 index 00000000..b50a18e4 --- /dev/null +++ b/src/populace_dynamics/engine/entrant_schedule.py @@ -0,0 +1,507 @@ +"""Build entrant cohorts for the loop's scheduled-entries seam. + +REPORT-ONLY. Nothing this module produces enters a fitted law, and no gate +scores it. It converts a sized control (Trustees Table V.A2 gross inflow) and +an explicitly supplied demographic donor into +frames the existing seam already accepts. + +**The seam is not new.** ``engine/loop.py`` has activated scheduled entries +since the M6 openers: :data:`~populace_dynamics.engine.loop.SCHEDULED_ENTRIES_KEY` +at ``loop.py:27``, validation at ``loop.py:219-247``, activation at +``loop.py:262-281``. This module only produces its input, so three properties +are inherited rather than designed: + +1. **The frame coordinate is the year BEFORE activation.** ``loop.py:236-239`` + rejects any other year. Following the documented opener convention at + ``m6_population.py:328-334`` -- "the frame coordinate is the reference year + immediately before the anchor interview, while age is the realized + collection-wave age" -- an entrant row carries ``year = activation_year - 1`` + and ``age = entry_age``. The loop's first step is therefore a **mortality + draw at the entrant's entry age**, before any aging; the aging step then + advances them to ``entry_age + 1`` in the activation year. That entry-year + exposure convention is fixed by the seam, not chosen here. +2. **IDs come from the projection-wide allocator.** ``loop.py:41-63`` raises + on any overlap with ``reserved_real_ids``, which is what stops an entrant + from silently inheriting a fitted person's ``u_w``. The caller must pass + the *same* allocator object it places in + ``metadata["synthetic_id_allocator"]``. A second allocator with the same + start would hand in-loop births the entrants' IDs. The loop itself is + inside the reviewed-implementation seal of the birth-evidence reducer + and does not check this, so callers run + :func:`validate_projection_allocator` on the exact metadata they project + with; it refuses an allocator whose ``next_id`` can reach an initial or + scheduled ``person_id`` that its ``reserved_real_ids`` do not cover. +3. **RNG streams are stable per person** (``loop.py:285-292, 351-353``), so + reproducibility is free. + +**Sizing.** Cohorts are sized to V.A2's *gross positive inflow* (LPR inflow + +temporary-or-unlawfully-present inflow), never to its total net change. Net +change is a residual whose age/sex/family composition has no literal +interpretation, and adjustment of status is a reclassification between two +stocks rather than a new person. Native control and donor readers from the original branch are not included +in this isolated source slice; all controls and rows are supplied by callers. + +**Method.** The donor pool is reweighted, not resampled: every positive-weight donor row +appears once per positive-inflow activation year with its weight scaled by a single factor so +the cohort's weighted total equals the control. That is deterministic, +consumes no RNG, and reproduces the donor composition exactly, so any residual +against the control is arithmetic rather than sampling noise. + +**Provenance.** Every emitted row carries ``entry_kind``, so births, realized +openers and immigrant cohorts stop being inferred from ID arithmetic. That +inference is what ``harness/m6_runner.py:1026-1032`` currently does when it +writes ``"immigrant_cohorts": 0`` under the comment "Every synthetic ID +allocated by this closed-panel engine is a step-4 materialized maternal +birth" -- true today, and false the moment a schedule is supplied. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from numbers import Integral +from typing import Any + +import numpy as np +import pandas as pd + +from populace_dynamics.engine.loop import ( + SCHEDULED_ENTRIES_KEY, + SyntheticPersonIdAllocator, +) + +__all__ = [ + "ENTRY_KIND_COLUMN", + "ENTRY_KIND_IMMIGRANT", + "ENTRY_KIND_BIRTH", + "ENTRY_KIND_REALIZED_OPENER", + "ENTRY_KIND_INCUMBENT", + "EntrantSchedule", + "build_entrant_schedule", + "entrant_provenance_counters", + "validate_projection_allocator", +] + +#: The provenance column every open-addition row should carry. +ENTRY_KIND_COLUMN = "entry_kind" +ENTRY_KIND_IMMIGRANT = "immigrant_cohort" +ENTRY_KIND_BIRTH = "maternal_birth" +ENTRY_KIND_REALIZED_OPENER = "realized_opener" +ENTRY_KIND_INCUMBENT = "incumbent" + +#: V.A2 is published in thousands of persons. +_THOUSANDS = 1000.0 + +#: Columns this module owns on an entrant row. Anything else present on the +#: roster is emitted as NA, which is the same disposition the birth path uses +#: (``steps.py:485-487`` builds children on ``frame.columns`` and fills only +#: what a newborn genuinely has). +_OWNED_COLUMNS = ( + "person_id", + "year", + "age", + "sex", + "birth_year", + "weight", + "start_weight", + "synthetic_entry", + ENTRY_KIND_COLUMN, + "entry_year", + "entry_age", + "donor_person_id", + "donor_source_year", + "donor_peinusyr", + "donor_prcitshp", + "donor_penatvty", + "foreign_born", +) + + +@dataclass(frozen=True) +class EntrantSchedule: + """Scheduled entrant frames plus the audit record of how they were sized.""" + + frames: dict[int, pd.DataFrame] + alignment: dict[int, dict[str, float]] + provenance: dict[str, Any] + + def as_metadata(self) -> dict[int, pd.DataFrame]: + """The value for ``metadata[SCHEDULED_ENTRIES_KEY]``.""" + return {year: frame.copy() for year, frame in self.frames.items()} + + def total_rows(self) -> int: + return int(sum(len(frame) for frame in self.frames.values())) + + def total_weight(self) -> float: + return float( + sum(frame["weight"].sum() for frame in self.frames.values()) + ) + + +def _validate_donor(donor: pd.DataFrame) -> None: + required = { + "person_id", + "weight", + "entry_age", + "is_female", + "source_year", + "peinusyr", + "prcitshp", + "penatvty", + # Carried, never asserted: the recent-arrival band's universe is + # everyone not born in the fifty states, so it contains natives too + # (see nativity_frame.recent_arrival_donor). + "foreign_born", + } + missing = required - set(donor.columns) + if missing: + raise ValueError(f"entrant donor is missing columns {sorted(missing)}") + if donor.empty: + raise ValueError("entrant donor pool is empty") + weight = donor["weight"].to_numpy(dtype=np.float64) + if not np.isfinite(weight).all() or (weight < 0).any(): + raise ValueError("entrant donor weights must be finite and >= 0") + if not np.isfinite(weight.sum()) or weight.sum() <= 0: + raise ValueError( + "entrant donor pool needs a finite positive total weight" + ) + age = donor["entry_age"].to_numpy(dtype=np.float64) + if ( + not np.isfinite(age).all() + or (age < 0).any() + or (age != np.floor(age)).any() + or (age >= np.iinfo(np.int64).max).any() + ): + raise ValueError( + "entrant donor entry_age must be nonnegative integers" + ) + for column in ("is_female", "foreign_born"): + if not all( + isinstance(value, (bool, np.bool_)) for value in donor[column] + ): + raise ValueError(f"entrant donor {column} must contain booleans") + + +def build_entrant_schedule( + donor: pd.DataFrame, + inflow_thousands_by_year: Mapping[int, float], + *, + allocator: SyntheticPersonIdAllocator, + roster_columns: Sequence[str] | None = None, + entry_kind: str = ENTRY_KIND_IMMIGRANT, + control_provenance: Mapping[str, Any] | None = None, + donor_provenance: Mapping[str, Any] | None = None, +) -> EntrantSchedule: + """Materialize one entrant frame per activation year. + + ``donor`` supplies the explicit demographic/provenance columns; no native + donor reader or source admission is included in this experimental slice. + ``inflow_thousands_by_year`` supplies scenario controls in thousands using + the historical gross-positive-inflow convention. ``allocator`` is the + projection-wide + :class:`~populace_dynamics.engine.loop.SyntheticPersonIdAllocator`; passing + the projection's own allocator is what guarantees entrant IDs never + collide with a fitted person-keyed support or with a later in-loop birth. + :func:`validate_projection_allocator` refuses projection metadata whose + allocator ``next_id`` can reach an unreserved scheduled ID, so a schedule + built from a separate allocator with the same start fails before the + first period when callers validate. Integer + activation-year keys (including numpy integers) are returned as ``int``. + """ + _validate_donor(donor) + if entry_kind != ENTRY_KIND_IMMIGRANT: + raise ValueError("immigrant schedules require immigrant_cohort kind") + if any( + isinstance(year, (bool, np.bool_)) or not isinstance(year, Integral) + for year in inflow_thousands_by_year + ): + raise ValueError("activation years must be integers") + # Normalize numpy/pandas integer keys to ``int`` here, at the boundary: + # they reach the frame keys, the alignment keys and the provenance year + # lists, and a numpy integer there makes the audit record unwritable as + # JSON. + inflow_by_year: dict[int, Any] = {} + for raw_year, raw_inflow in inflow_thousands_by_year.items(): + year = int(raw_year) + if year in inflow_by_year: + raise ValueError(f"activation year {year} is supplied twice") + inflow_by_year[year] = raw_inflow + years = sorted(inflow_by_year) + if not years: + raise ValueError("no activation years requested") + controls = {} + for year in years: + inflow = float(inflow_by_year[year]) + if ( + not np.isfinite(inflow) + or inflow < 0 + or not np.isfinite(inflow * _THOUSANDS) + ): + raise ValueError( + f"activation year {year} has a non-finite or negative control " + f"inflow {inflow!r}" + ) + controls[year] = inflow + + # Zero-weight donors never become demographic actors or consume IDs. + # Keep the float representation validated above; an integer sum can + # overflow before scaling even when every individual weight is valid. + donor = donor.copy() + donor["weight"] = donor["weight"].to_numpy(dtype=np.float64) + donor = donor.loc[donor["weight"] > 0].copy() + + donor_weight_total = float(donor["weight"].sum()) + donor_weight = donor["weight"].to_numpy(dtype=np.float64) + donor_proportion = donor_weight / donor_weight_total + for inflow in controls.values(): + if inflow == 0: + continue + scale = inflow * _THOUSANDS / donor_weight_total + with np.errstate(over="ignore", under="ignore"): + extrema = donor_proportion[ + [donor_weight.argmin(), donor_weight.argmax()] + ] * (inflow * _THOUSANDS) + if ( + not np.isfinite(scale) + or scale <= 0 + or not np.isfinite(extrema).all() + or (extrema <= 0).any() + ): + raise ValueError( + "control scaling must yield finite positive donor weights" + ) + entry_age = donor["entry_age"].to_numpy(dtype=np.int64) + sex = np.where(donor["is_female"].to_numpy(dtype=bool), "female", "male") + + frames: dict[int, pd.DataFrame] = {} + alignment: dict[int, dict[str, float]] = {} + for year in years: + inflow = controls[year] + target_weight = inflow * _THOUSANDS + scale = target_weight / donor_weight_total + if target_weight == 0: + alignment[year] = { + "control_inflow_thousands": inflow, + "target_weighted_persons": 0.0, + "scheduled_weighted_persons": 0.0, + "residual_persons": 0.0, + "relative_residual": 0.0, + "n_rows": 0, + "donor_weight_scale": 0.0, + } + # The existing loop rejects empty scheduled frames. Omitting the + # executable frame also preserves its ID/RNG state for this year. + continue + # Normalize before scaling: a positive subnormal common scale can + # lose material precision even though the target is representable. + weight = donor_proportion * target_weight + person_id = allocator.allocate(len(donor)) + frame_year = year - 1 + row = pd.DataFrame( + { + "person_id": person_id, + "year": np.full(len(donor), frame_year, dtype=np.int64), + "age": entry_age, + "sex": sex, + "birth_year": frame_year - entry_age, + "weight": weight, + "start_weight": weight, + "synthetic_entry": np.ones(len(donor), dtype=bool), + ENTRY_KIND_COLUMN: np.full( + len(donor), entry_kind, dtype=object + ), + "entry_year": np.full(len(donor), year, dtype=np.int64), + "entry_age": entry_age, + "donor_person_id": donor["person_id"].to_numpy(), + "donor_source_year": donor["source_year"].to_numpy(), + "donor_peinusyr": donor["peinusyr"].to_numpy(), + "donor_prcitshp": donor["prcitshp"].to_numpy(), + "donor_penatvty": donor["penatvty"].to_numpy(), + "foreign_born": donor["foreign_born"].to_numpy(dtype=bool), + } + ) + if roster_columns is not None: + for column in roster_columns: + if column not in row.columns: + row[column] = pd.NA + row = row[ + list(roster_columns) + + [c for c in row.columns if c not in set(roster_columns)] + ] + row = row.sort_values("person_id", kind="stable").reset_index( + drop=True + ) + frames[year] = row + realized = float(row["weight"].sum()) + alignment[year] = { + "control_inflow_thousands": inflow, + "target_weighted_persons": target_weight, + "scheduled_weighted_persons": realized, + "residual_persons": realized - target_weight, + "relative_residual": ( + (realized - target_weight) / target_weight + if target_weight + else 0.0 + ), + "n_rows": int(len(row)), + "donor_weight_scale": scale, + } + + provenance: dict[str, Any] = { + "method": "donor_reweighted_to_control", + "method_detail": ( + "every positive-weight donor appears once per positive-inflow " + "activation year with its weight " + "scaled by a single factor, so the cohort's weighted total equals " + "the control and its composition equals the donor exactly; no RNG " + "is consumed and no row is resampled" + ), + "sizing_basis": "trustees_va2_gross_positive_inflow", + "sizing_excludes": [ + "outflow (the engine has no emigration law)", + "adjustment of status (a reclassification, not a new person)", + "total net change (a residual with no literal composition)", + ], + "sizing_basis_disclosure": ( + "a stock-accounting inflow proxy, NOT a count of physical " + "arrivals: V.A2's temporary-or-unlawfully-present inflow counts " + "only those who remain to year-end, so the gross total understates " + "border arrivals and the cohort must not be read as one " + "(2026 OASDI Trustees Report Table V.A2; the same qualification " + "PR #218 section 0 states for this control)" + ), + "frame_coordinate": ( + "year = activation_year - 1, age = entry_age; the loop's first " + "step is a mortality draw at the entry age, then aging advances " + "to entry_age + 1 in the activation year (loop.py:236-239, " + "262-281; convention per m6_population.py:328-334)" + ), + "entry_kind": entry_kind, + "activation_years": years, + "scheduled_activation_years": sorted(frames), + "zero_inflow_years": [year for year in years if controls[year] == 0], + "id_allocation": ( + "projection-wide SyntheticPersonIdAllocator; loop.py:52-61 raises " + "on any overlap with reserved_real_ids" + ), + "gated": False, + "report_only": True, + } + if control_provenance is not None: + provenance["control"] = dict(control_provenance) + if donor_provenance is not None: + provenance["donor"] = dict(donor_provenance) + return EntrantSchedule( + frames=frames, alignment=alignment, provenance=provenance + ) + + +def entrant_provenance_counters( + frames: Mapping[int, pd.DataFrame], +) -> dict[str, Any]: + """Counts by ``entry_kind`` and by year, for the run artifact. + + This is the replacement for inferring an entrant's kind from ID + arithmetic. ``harness/m6_runner.py:1026-1032`` and ``:1210-1216`` publish + a hardcoded ``"immigrant_cohorts": 0`` justified by a comment that holds + only while no schedule exists; a counter keyed on an explicit column + survives the schedule existing. + """ + by_kind: dict[str, int] = {} + by_year: dict[int, dict[str, Any]] = {} + weighted_by_kind: dict[str, float] = {} + for year, frame in sorted(frames.items()): + if ENTRY_KIND_COLUMN not in frame.columns: + raise ValueError( + f"scheduled entries {year} carry no {ENTRY_KIND_COLUMN!r} " + "column; entrant provenance cannot be counted" + ) + if ( + not frame[ENTRY_KIND_COLUMN] + .isin( + { + ENTRY_KIND_BIRTH, + ENTRY_KIND_INCUMBENT, + ENTRY_KIND_IMMIGRANT, + ENTRY_KIND_REALIZED_OPENER, + } + ) + .all() + ): + raise ValueError( + "scheduled provenance has missing or unknown entry_kind" + ) + counts = frame[ENTRY_KIND_COLUMN].value_counts().to_dict() + by_year[int(year)] = { + "n_rows": int(len(frame)), + "weighted_persons": float(frame["weight"].sum()), + "by_entry_kind": {str(k): int(v) for k, v in counts.items()}, + } + for kind, count in counts.items(): + by_kind[str(kind)] = by_kind.get(str(kind), 0) + int(count) + mask = frame[ENTRY_KIND_COLUMN] == kind + weighted_by_kind[str(kind)] = weighted_by_kind.get( + str(kind), 0.0 + ) + float(frame.loc[mask, "weight"].sum()) + return { + "n_rows_by_entry_kind": by_kind, + "weighted_persons_by_entry_kind": weighted_by_kind, + "by_activation_year": by_year, + "immigrant_cohorts": by_kind.get(ENTRY_KIND_IMMIGRANT, 0), + "counter_basis": ( + f"explicit {ENTRY_KIND_COLUMN!r} column, not ID arithmetic" + ), + } + + +def validate_projection_allocator( + initial_person_ids: Sequence[Any] | pd.Series, + metadata: Mapping[str, Any], +) -> None: + """Refuse projection metadata whose allocator could reuse a known ID. + + ``SyntheticPersonIdAllocator`` hands out IDs upward from ``next_id`` and + raises only on ``reserved_real_ids``. If an initial or scheduled + ``person_id`` sits at or above that cursor and is not reserved, a later + in-loop birth can receive it; when that person has already died the reuse + is silent, so one ``person_id`` names two people and shares one RNG + ordinal. The usual cause is a schedule built with a *different* + allocator than ``metadata["synthetic_id_allocator"]`` that starts at the + same ``synthetic_id_start``. + + Call this on the exact metadata passed to ``ProjectionEngine.project``. + The loop does not perform this check itself: ``engine/loop.py`` is part + of the birth-evidence reducer's reviewed-implementation identity, and + changing it needs a deliberate re-pin. Metadata without an allocator is + accepted, because the loop then builds its own from ``max(ids) + 1``. + """ + allocator = metadata.get("synthetic_id_allocator") + if allocator is None: + return + if not isinstance(allocator, SyntheticPersonIdAllocator): + raise TypeError( + "metadata synthetic_id_allocator must be a " + "SyntheticPersonIdAllocator" + ) + known = {int(person_id) for person_id in initial_person_ids} + scheduled = metadata.get(SCHEDULED_ENTRIES_KEY, {}) or {} + if not isinstance(scheduled, Mapping): + raise TypeError( + f"metadata {SCHEDULED_ENTRIES_KEY!r} must be a mapping" + ) + for frame in scheduled.values(): + known.update(int(person_id) for person_id in frame["person_id"]) + cursor = int(allocator.next_id) + reachable = sorted( + person_id + for person_id in known + if person_id >= cursor and person_id not in allocator.reserved_real_ids + ) + if reachable: + raise ValueError( + f"metadata synthetic_id_allocator next_id {cursor} can reach " + f"initial or scheduled person_id {reachable[:10]}, which " + "reserved_real_ids does not cover; a later allocation could " + "reuse an existing person's ID. Build scheduled entrants with " + "the projection's own allocator." + ) diff --git a/src/populace_dynamics/forward_earnings_history.py b/src/populace_dynamics/forward_earnings_history.py new file mode 100644 index 00000000..4141d091 --- /dev/null +++ b/src/populace_dynamics/forward_earnings_history.py @@ -0,0 +1,508 @@ +"""Record annual outputs of the existing 2014–22 forward earnings law. + +This opt-in observer does not invoke or change a generator. It records one +realization on a fixed roster, preserves binary64 values, and never treats +labor income as covered earnings. Source digests establish identity, not +empirical admission, legal authority, or acceptance of the PSID crosswalk. +""" + +from __future__ import annotations + +import json +import math +import re +from collections.abc import Mapping +from dataclasses import dataclass +from hashlib import sha256 +from numbers import Integral + +import numpy as np +import pandas as pd + +from .person_identity import PersonIdentity, PersonIdentityMap + +_SCHEMA = "populace_dynamics.forward_earnings_history.v1" +_FIRST_YEAR = 2014 +_LAST_YEAR = 2022 +_COVERAGE = "not_materialized" +_REGISTRY = "registration_required" + + +def _text(value: object, label: str) -> str: + if type(value) is not str or not value.strip(): + raise ValueError(f"{label} must be a nonempty string") + try: + value.encode("utf-8") + except UnicodeEncodeError as exc: + raise ValueError(f"{label} must be valid UTF-8") from exc + return value + + +def _digest(value: object, label: str) -> str: + if type(value) is not str or re.fullmatch(r"[0-9a-f]{64}", value) is None: + raise ValueError(f"{label} must be a lowercase SHA-256 digest") + return value + + +def _integer(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, Integral): + raise ValueError(f"{label} must be an integer without coercion") + return int(value) + + +def _canonical_integer(value: object, label: str) -> int: + if type(value) is not str: + raise ValueError(f"{label} must be a canonical integer string") + try: + result = int(value) + except ValueError as exc: + raise ValueError( + f"{label} must be a canonical integer string" + ) from exc + if str(result) != value: + raise ValueError(f"{label} must be a canonical integer string") + return result + + +@dataclass(frozen=True) +class ForwardEarningsObservation: + """One realized annual labor amount, or an explicit unsupported record.""" + + dynamics_person_key: int + year: int + amount_hex: str | None + earnings_domain: bool + lineage_digest: str + + def __post_init__(self) -> None: + key = _integer(self.dynamics_person_key, "person key") + year = _integer(self.year, "reference year") + if not 0 <= key < 2**63: + raise ValueError("person key must be nonnegative int64") + if not _FIRST_YEAR <= year <= _LAST_YEAR: + raise ValueError("reference year must be within 2014–2022") + if type(self.earnings_domain) is not bool: + raise ValueError("earnings domain must be an explicit bool") + _digest(self.lineage_digest, "lineage digest") + if not self.earnings_domain: + if self.amount_hex is not None: + raise ValueError("outside-domain amount must be unavailable") + else: + if type(self.amount_hex) is not str: + raise ValueError("supported amount needs binary64 hex") + try: + amount = float.fromhex(self.amount_hex) + except (ValueError, OverflowError) as exc: + raise ValueError("invalid binary64 amount") from exc + if ( + not math.isfinite(amount) + or amount < 0 + or amount.hex() != self.amount_hex + ): + raise ValueError( + "amount must be canonical nonnegative binary64" + ) + object.__setattr__(self, "dynamics_person_key", key) + object.__setattr__(self, "year", year) + + @property + def amount_state(self) -> str: + if self.amount_hex is None: + return "unavailable" + return ( + "known_zero" + if float.fromhex(self.amount_hex) == 0 + else "known_amount" + ) + + @property + def missing_reason(self) -> str | None: + return ( + None if self.earnings_domain else "outside_forward_earnings_domain" + ) + + @property + def generation_method(self) -> str: + if not self.earnings_domain: + return "not_materialized" + if self.year == _FIRST_YEAR: + return "boundary_method" + return "odd_year_carry" if self.year % 2 else "biennial_draw" + + +def _snapshot( + mapping: PersonIdentityMap, frame: pd.DataFrame, lineage_digest: str +) -> tuple[ForwardEarningsObservation, ...]: + if type(mapping) is not PersonIdentityMap: + raise ValueError("explicit PersonIdentityMap required") + _digest(lineage_digest, "lineage digest") + if not isinstance(frame, pd.DataFrame) or not frame.columns.is_unique: + raise ValueError("frame requires unique columns") + required = ("person_id", "year", "earnings", "earnings_domain") + if not set(required).issubset(frame.columns) or frame.empty: + raise ValueError( + "nonempty frame requires identity/year/earnings/domain" + ) + for name in ("person_id", "year"): + if frame[name].dtype != np.dtype("int64"): + raise ValueError( + f"{name} must be int64, without implicit conversion" + ) + if frame["earnings"].dtype != np.dtype("float64"): + raise ValueError( + "earnings must preserve the generator's float64 dtype" + ) + if frame["earnings_domain"].dtype != np.dtype("bool"): + raise ValueError( + "earnings_domain must be bool, with no missing values" + ) + if frame["person_id"].duplicated().any(): + raise ValueError("duplicate person in annual frame") + if frame["year"].nunique() != 1: + raise ValueError("annual frame must have exactly one reference year") + keys = frame["person_id"].tolist() + mapping.reverse_rows(keys) # Check private keys without float conversion. + rows = [] + for key, year, amount, domain in frame[list(required)].itertuples( + index=False, name=None + ): + if not domain and amount != 0: + raise ValueError("outside-domain output must be its control zero") + rows.append( + ForwardEarningsObservation( + key, + year, + float(amount).hex() if domain else None, + bool(domain), + lineage_digest, + ) + ) + return tuple(sorted(rows, key=lambda row: row.dynamics_person_key)) + + +@dataclass(frozen=True) +class ForwardEarningsHistory: + """Dense append-only 2014–22 snapshot history for one fixed-roster draw. + + A history may cover a subset of an identity map, but its roster and domain + cannot change during append. Death, entrant and domain-transition records + need a separately defined extension. Historical years before 2014 and + coverage classification are deliberately absent from this version. + """ + + identity_map: PersonIdentityMap + realization_id: str + generator_digest: str + source_contract_digest: str + unit: str + price_basis: str + roster_keys: tuple[int, ...] + last_year: int + observations: tuple[ForwardEarningsObservation, ...] + + def __init_subclass__(cls, **kwargs) -> None: + raise TypeError("ForwardEarningsHistory cannot be subclassed") + + def __post_init__(self) -> None: + if type(self.identity_map) is not PersonIdentityMap: + raise ValueError("explicit PersonIdentityMap required") + _text(self.realization_id, "realization ID") + _text(self.unit, "unit") + if self.price_basis != "nominal" or type(self.price_basis) is not str: + raise ValueError("forward earnings require explicit nominal basis") + _digest(self.generator_digest, "generator digest") + _digest(self.source_contract_digest, "source contract digest") + roster = tuple(_integer(key, "roster key") for key in self.roster_keys) + if not roster or len(set(roster)) != len(roster): + raise ValueError("declared roster must be nonempty and unique") + self.identity_map.reverse_rows(roster) + roster = tuple(sorted(roster)) + last_year = _integer(self.last_year, "last reference year") + if not _FIRST_YEAR <= last_year <= _LAST_YEAR: + raise ValueError("last reference year must be within 2014–2022") + rows = tuple(self.observations) + if not rows or any( + type(r) is not ForwardEarningsObservation for r in rows + ): + raise ValueError("nonempty typed observations required") + rows = tuple( + sorted(rows, key=lambda r: (r.year, r.dynamics_person_key)) + ) + self.identity_map.reverse_rows(r.dynamics_person_key for r in rows) + by_year: dict[int, dict[int, ForwardEarningsObservation]] = {} + for row in rows: + annual = by_year.setdefault(row.year, {}) + if row.dynamics_person_key in annual: + raise ValueError("duplicate person/year observation") + annual[row.dynamics_person_key] = row + years = sorted(by_year) + if years != list(range(_FIRST_YEAR, last_year + 1)): + raise ValueError("history must contain dense years beginning 2014") + initial = by_year[_FIRST_YEAR] + if tuple(sorted(initial)) != roster: + raise ValueError("observations must cover the declared roster") + previous = initial + for year in years: + annual = by_year[year] + if annual.keys() != initial.keys(): + raise ValueError("fixed roster must be complete in every year") + if len({row.lineage_digest for row in annual.values()}) != 1: + raise ValueError( + "annual snapshot must have one lineage digest" + ) + for key, row in annual.items(): + if row.earnings_domain != initial[key].earnings_domain: + raise ValueError("fixed earnings domain cannot change") + if year % 2 and row.amount_hex != previous[key].amount_hex: + raise ValueError( + "odd-year carry must preserve exact source bits" + ) + previous = annual + object.__setattr__(self, "observations", rows) + object.__setattr__(self, "roster_keys", roster) + object.__setattr__(self, "last_year", last_year) + + @property + def source_registry_status(self) -> str: + return _REGISTRY + + @property + def coverage_status(self) -> str: + return _COVERAGE + + @classmethod + def start( + cls, + identity_map: PersonIdentityMap, + frame: pd.DataFrame, + *, + realization_id: str, + generator_digest: str, + source_contract_digest: str, + unit: str, + price_basis: str, + lineage_digest: str, + ) -> ForwardEarningsHistory: + """Snapshot an already materialized 2014 frame without changing it. + + ``frame["person_id"]`` must already hold ``identity_map``'s private + keys (for example from ``identity_map.map_rows``), not native source + IDs. Only admission of each key is checked: nonnegative native int64 + IDs below the map size are indistinguishable from private keys and + would be attributed to whichever identity owns that key. + """ + rows = _snapshot(identity_map, frame, lineage_digest) + return cls( + identity_map, + realization_id, + generator_digest, + source_contract_digest, + unit, + price_basis, + tuple(row.dynamics_person_key for row in rows), + _FIRST_YEAR, + rows, + ) + + def append( + self, frame: pd.DataFrame, *, lineage_digest: str + ) -> ForwardEarningsHistory: + """Return a new history containing exactly the following year's rows. + + As in ``start``, ``frame["person_id"]`` holds private keys. + """ + if self.last_year == _LAST_YEAR: + raise ValueError("2014–2022 contract cannot extend beyond 2022") + rows = _snapshot(self.identity_map, frame, lineage_digest) + if rows[0].year != self.last_year + 1: + raise ValueError("append requires exactly the next reference year") + return type(self)( + self.identity_map, + self.realization_id, + self.generator_digest, + self.source_contract_digest, + self.unit, + self.price_basis, + self.roster_keys, + self.last_year + 1, + self.observations + rows, + ) + + def for_person( + self, identity: PersonIdentity + ) -> tuple[ForwardEarningsObservation, ...]: + """Look up source IDs exactly, independently of private key ordering.""" + key = self.identity_map.map_rows([identity])[0] + result = tuple( + r for r in self.observations if r.dynamics_person_key == key + ) + if not result: + raise ValueError("person is not in this history's fixed roster") + return result + + def require_extension_of(self, previous: ForwardEarningsHistory) -> None: + """Reject a loaded successor that changes provenance or any old row.""" + if type(previous) is not ForwardEarningsHistory: + raise ValueError("previous history must be ForwardEarningsHistory") + metadata = ( + "identity_map", + "realization_id", + "generator_digest", + "source_contract_digest", + "unit", + "price_basis", + "roster_keys", + ) + if any(getattr(self, k) != getattr(previous, k) for k in metadata): + raise ValueError( + "history extension changed identity or provenance" + ) + if ( + self.observations[: len(previous.observations)] + != previous.observations + ): + raise ValueError("history extension changed or removed prior rows") + + def to_json(self) -> str: + """Serialize without JSON floating-point numbers or numeric ID loss.""" + document = { + "schema": _SCHEMA, + "identity_map": json.loads(self.identity_map.to_json()), + "identity_map_digest": self.identity_map.digest, + "realization_id": self.realization_id, + "generator_digest": self.generator_digest, + "source_contract_digest": self.source_contract_digest, + "unit": self.unit, + "price_basis": self.price_basis, + "calendar": "calendar_year", + "roster_keys": [str(key) for key in self.roster_keys], + "last_year": str(self.last_year), + "source_registry_status": _REGISTRY, + "coverage_status": _COVERAGE, + "observations": [ + { + "dynamics_person_key": str(r.dynamics_person_key), + "year": str(r.year), + "amount_hex": r.amount_hex, + "earnings_domain": r.earnings_domain, + "lineage_digest": r.lineage_digest, + } + for r in self.observations + ], + } + return json.dumps( + document, + sort_keys=True, + ensure_ascii=False, + separators=(",", ":"), + allow_nan=False, + ) + + @property + def digest(self) -> str: + return sha256(self.to_json().encode("utf-8")).hexdigest() + + @classmethod + def from_json( + cls, + text: str, + *, + expected_digest: str | None = None, + previous: ForwardEarningsHistory | None = None, + ) -> ForwardEarningsHistory: + """Load and revalidate every annual row and the embedded identity map.""" + + def unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON member") + result[key] = value + return result + + def reject_constant(value): + raise ValueError(f"nonfinite JSON constant: {value}") + + document = json.loads( + text, object_pairs_hook=unique, parse_constant=reject_constant + ) + fields = { + "schema", + "identity_map", + "identity_map_digest", + "realization_id", + "generator_digest", + "source_contract_digest", + "unit", + "price_basis", + "calendar", + "source_registry_status", + "coverage_status", + "observations", + "roster_keys", + "last_year", + } + if not isinstance(document, Mapping) or set(document) != fields: + raise ValueError("invalid history document fields") + constants = { + "schema": _SCHEMA, + "calendar": "calendar_year", + "coverage_status": _COVERAGE, + "source_registry_status": _REGISTRY, + } + if any(document[k] != value for k, value in constants.items()): + raise ValueError("unsupported history scope or status") + _digest(document["identity_map_digest"], "identity map digest") + mapping = PersonIdentityMap.from_json( + json.dumps(document["identity_map"]), + expected_digest=document["identity_map_digest"], + ) + if type(document["observations"]) is not list: + raise ValueError("observations must be an array") + if type(document["roster_keys"]) is not list: + raise ValueError("roster keys must be an array") + rows = [] + row_fields = { + "dynamics_person_key", + "year", + "amount_hex", + "earnings_domain", + "lineage_digest", + } + for row in document["observations"]: + if type(row) is not dict or set(row) != row_fields: + raise ValueError("invalid observation fields") + rows.append( + ForwardEarningsObservation( + _canonical_integer( + row["dynamics_person_key"], "person key" + ), + _canonical_integer(row["year"], "reference year"), + row["amount_hex"], + row["earnings_domain"], + row["lineage_digest"], + ) + ) + result = cls( + mapping, + document["realization_id"], + document["generator_digest"], + document["source_contract_digest"], + document["unit"], + document["price_basis"], + tuple( + _canonical_integer(key, "roster key") + for key in document["roster_keys"] + ), + _canonical_integer(document["last_year"], "last reference year"), + tuple(rows), + ) + if expected_digest is not None: + _digest(expected_digest, "expected digest") + if result.digest != expected_digest: + raise ValueError( + "history digest does not match expected digest" + ) + if previous is not None: + result.require_extension_of(previous) + return result diff --git a/src/populace_dynamics/graph/__init__.py b/src/populace_dynamics/graph/__init__.py new file mode 100644 index 00000000..05eca38c --- /dev/null +++ b/src/populace_dynamics/graph/__init__.py @@ -0,0 +1,28 @@ +"""Optional synthetic population-graph integration. + +Importing this package does not import Microcosm or change legacy execution. +The graph entry point checks Python and the installed graph capabilities. +""" + + +def run_mortality_graph(**kwargs): + """Run the existing mortality/ageing operations through Microcosm.""" + from ._compat import require_graph + + require_graph() + from .runtime import run_mortality_graph as run + + return run(**kwargs) + + +def run_mortality_trajectory(**kwargs): + """Run the existing mortality/ageing steps across annual graph periods.""" + from ._compat import require_graph + + require_graph() + from .trajectory import run_mortality_trajectory as run + + return run(**kwargs) + + +__all__ = ["run_mortality_graph", "run_mortality_trajectory"] diff --git a/src/populace_dynamics/graph/__main__.py b/src/populace_dynamics/graph/__main__.py new file mode 100644 index 00000000..fb502f35 --- /dev/null +++ b/src/populace_dynamics/graph/__main__.py @@ -0,0 +1,66 @@ +"""Run the mortality graph with explicit inputs and output placement.""" + +import argparse +from pathlib import Path + +from . import run_mortality_graph +from .synthetic import write_synthetic_inputs + + +def parser(): + result = argparse.ArgumentParser(description=__doc__) + result.add_argument("--output-dir", type=Path, required=True) + result.add_argument("--synthetic", action="store_true") + for name in ("training", "rates", "initial", "holdout"): + result.add_argument(f"--{name}", type=Path) + result.add_argument("--boundary-year", type=int, default=2014) + result.add_argument("--external-vintage-year", type=int, default=2014) + result.add_argument("--experiment-id", default="mortality") + result.add_argument("--replicate", type=int, default=0) + result.add_argument("--base-seed", type=int, default=0) + return result + + +def main(argv=None): + arg_parser = parser() + args = arg_parser.parse_args(argv) + sources = { + name: getattr(args, name) + for name in ("training", "rates", "initial", "holdout") + } + if args.synthetic: + if any(sources.values()): + arg_parser.error("--synthetic cannot be combined with input paths") + if args.boundary_year != 2014: + arg_parser.error( + "the supplied synthetic fixture has boundary year 2014" + ) + sources = write_synthetic_inputs(args.output_dir / "inputs") + elif not all(sources.values()): + arg_parser.error("supply all four input paths or --synthetic") + try: + run = run_mortality_graph( + **sources, + output_dir=args.output_dir, + boundary_year=args.boundary_year, + external_vintage_year=args.external_vintage_year, + experiment_id=args.experiment_id, + replicate=args.replicate, + base_seed=args.base_seed, + ) + except (ImportError, ValueError) as error: + arg_parser.exit(2, f"{error}\n") + print( + f"{args.output_dir / 'report.json'}: engineering={run.report['engineering_verdict']}, fixture={run.report['fixture_verdict']}" + ) + return ( + 0 + if run.report["engineering_verdict"] + == run.report["fixture_verdict"] + == "pass" + else 1 + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/populace_dynamics/graph/_compat.py b/src/populace_dynamics/graph/_compat.py new file mode 100644 index 00000000..86ffff07 --- /dev/null +++ b/src/populace_dynamics/graph/_compat.py @@ -0,0 +1,36 @@ +"""An explicit dependency boundary for the optional graph example.""" + +import importlib +import sys + +_GUIDANCE = ( + "The population graph needs Microcosm's typed model-artifact and keyed " + "randomness interfaces. Install the reviewed microcosm-graph and " + "microcosm-frame revisions together; see docs/population-graph.md. " + "Legacy Dynamics does not require these packages." +) + + +def _python_version(): + return sys.version_info[:2] + + +def require_graph(): + """Refuse unsupported Python or a graph lacking the required interfaces.""" + if _python_version() < (3, 13): + raise ImportError( + "The optional population graph requires Python >=3.13." + ) + try: + decl = importlib.import_module("microcosm.graph.decl") + kernel = importlib.import_module("microcosm.graph.kernel") + randomness = importlib.import_module("microcosm.graph.randomness") + except (ImportError, SyntaxError) as error: + raise ImportError(_GUIDANCE) from error + for name in ("ArtifactType", "ArtifactInput", "ArtifactOutput"): + if not hasattr(decl, name): + raise ImportError(_GUIDANCE) + if not hasattr(kernel.SeedSource, "KEYED") or not hasattr( + randomness, "keyed_uniform" + ): + raise ImportError(_GUIDANCE) diff --git a/src/populace_dynamics/graph/model.py b/src/populace_dynamics/graph/model.py new file mode 100644 index 00000000..0f2da4be --- /dev/null +++ b/src/populace_dynamics/graph/model.py @@ -0,0 +1,179 @@ +"""Validated JSON for the existing fitted M6 mortality law; no pickle.""" + +from __future__ import annotations + +import json +import math +from dataclasses import dataclass + +_FORMAT = "populace-dynamics.mortality" + + +def _unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON member {key!r}") + result[key] = value + return result + + +def read_json(path): + """Read JSON without accepting duplicate fields or nonfinite numbers.""" + return parse_json(path.read_bytes()) + + +def parse_json(payload): + def invalid(value): + raise ValueError(f"nonfinite JSON number {value}") + + try: + return json.loads( + payload, object_pairs_hook=_unique_object, parse_constant=invalid + ) + except (UnicodeError, json.JSONDecodeError) as error: + raise ValueError("invalid mortality JSON") from error + + +def json_bytes(value): + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode("utf-8") + + +def _integer(value, label): + if type(value) is not int: + raise ValueError(f"{label} must be an integer") + return value + + +@dataclass(frozen=True) +class MortalityArtifact: + """A compact model with an explicit fit boundary and source vintage.""" + + bands: tuple[tuple[int, int], ...] + probabilities: tuple[tuple[str, str, float], ...] + boundary_year: int + external_vintage_year: int + fit_rows: int + + def __post_init__(self): + _integer(self.boundary_year, "boundary_year") + _integer(self.external_vintage_year, "external_vintage_year") + if _integer(self.fit_rows, "fit_rows") <= 0: + raise ValueError("fit_rows must be positive") + if self.external_vintage_year > self.boundary_year: + raise ValueError("external vintage is later than the fit boundary") + seen = set() + for label, sex, probability in self.probabilities: + if not isinstance(label, str) or sex not in ("female", "male"): + raise ValueError("invalid mortality probability cell") + if (label, sex) in seen: + raise ValueError("duplicate mortality probability cell") + seen.add((label, sex)) + if isinstance(probability, bool) or not isinstance( + probability, (int, float) + ): + raise ValueError("mortality probability must be numeric") + if not math.isfinite(probability) or not 0 <= probability <= 1: + raise ValueError("mortality probability must lie in [0, 1]") + for band in self.bands: + if len(band) != 2 or any(type(age) is not int for age in band): + raise ValueError("mortality bands must contain integer bounds") + # The real model validates complete, contiguous bands and sex cells. + _ = self.model + + @property + def model(self): + from populace_dynamics.engine.steps import AgeSexMortalityModel + + return AgeSexMortalityModel( + self.bands, + {(band, sex): p for band, sex, p in self.probabilities}, + ) + + def to_bytes(self): + return json_bytes( + { + "format": _FORMAT, + "schema_version": 1, + "boundary_year": self.boundary_year, + "external_vintage_year": self.external_vintage_year, + "fit_rows": self.fit_rows, + "bands": self.bands, + "probabilities": [ + {"age_band": band, "sex": sex, "probability": p} + for band, sex, p in sorted(self.probabilities) + ], + } + ) + + @classmethod + def from_bytes(cls, payload): + raw = parse_json(payload) + expected = { + "format", + "schema_version", + "boundary_year", + "external_vintage_year", + "fit_rows", + "bands", + "probabilities", + } + if not isinstance(raw, dict) or set(raw) != expected: + raise ValueError("mortality model has an invalid field set") + if raw["format"] != _FORMAT or type(raw["schema_version"]) is not int: + raise ValueError("invalid mortality format or schema version") + if raw["schema_version"] != 1: + raise ValueError("unsupported mortality schema version") + if not isinstance(raw["bands"], list) or not all( + isinstance(band, list) for band in raw["bands"] + ): + raise ValueError("mortality bands must be arrays") + if not isinstance(raw["probabilities"], list): + raise ValueError("mortality probabilities must be an array") + for row in raw["probabilities"]: + if not isinstance(row, dict) or set(row) != { + "age_band", + "sex", + "probability", + }: + raise ValueError("invalid mortality probability fields") + return cls( + bands=tuple(tuple(band) for band in raw["bands"]), + probabilities=tuple( + (row["age_band"], row["sex"], row["probability"]) + for row in raw["probabilities"] + ), + boundary_year=raw["boundary_year"], + external_vintage_year=raw["external_vintage_year"], + fit_rows=raw["fit_rows"], + ) + + +def fit_mortality( + exposure, external_rates, *, boundary_year, external_vintage_year +): + """Use the existing cutoff-safe fitter and retain its compact results.""" + from populace_dynamics.engine.refit import ( + fit_mortality_model, + prepare_mortality_refit_inputs, + ) + + prepared = prepare_mortality_refit_inputs( + exposure, + external_rates, + boundary_year=boundary_year, + external_vintage_year=external_vintage_year, + ) + model = fit_mortality_model(prepared) + return MortalityArtifact( + bands=model.bands, + probabilities=tuple( + (band, sex, p) + for (band, sex), p in sorted(model.probability.items()) + ), + boundary_year=boundary_year, + external_vintage_year=external_vintage_year, + fit_rows=len(prepared.exposure), + ) diff --git a/src/populace_dynamics/graph/runtime.py b/src/populace_dynamics/graph/runtime.py new file mode 100644 index 00000000..b24261ed --- /dev/null +++ b/src/populace_dynamics/graph/runtime.py @@ -0,0 +1,931 @@ +"""Fit the existing mortality law, transport it, and retain two periods. + +This module is imported only after the optional capability check. Its graph +is a synthetic engineering integration, separate from the locked M6 loop. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import SOURCE_CODECS +from microcosm.graph.decl import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Graph, + Node, + Owned, + Slice, + SourceRef, + StructuralDelta, + compile_graph, +) +from microcosm.graph.executor import run_graph +from microcosm.graph.kernel import ( + Capabilities, + Determinism, + KernelRegistry, + KernelResult, + KernelRole, + Numeric, + SeedSource, + source_hash, +) +from microcosm.graph.randomness import keyed_uniform +from microcosm.graph.store import ContentStore + +from populace_dynamics.engine.refit import ( + fit_mortality_model, + prepare_mortality_refit_inputs, +) +from populace_dynamics.engine.steps import advance_age, apply_mortality + +from . import model as model_module +from .model import ( + MortalityArtifact, + fit_mortality, + json_bytes, + parse_json, + read_json, +) + +OBS = "person_period" +PID = "person_period_person_id" +PERIOD_ID = "person_period_period_id" +OID = "person_period_id" +MODEL_TYPE = ArtifactType("populace-dynamics.mortality", 1) +CODEC = "dynamics-mortality-json-v1" +DEPENDENCIES = ("numpy", "pandas") +TRAIN_COLUMNS = ( + ("age_band", "string"), + ("sex", "string"), + ("required_interview_year", "int64"), + ("exposure", "float64"), + ("death", "float64"), +) + + +def _json_source_marker(path, *, store=None): + """Raw declared sources are read by domain kernels, never as Frames.""" + del path, store + raise ValueError("Dynamics JSON sources need their declared domain kernel") + + +def _series(values, ids, entity=OBS, dtype="int64"): + return pd.Series( + values, + index=pd.Index(ids, name=f"{entity}_id", dtype="int64"), + dtype=dtype, + ) + + +def _records(path, expected): + raw = read_json(path) + if not isinstance(raw, list) or not raw: + raise ValueError(f"{path.name} must contain nonempty record inputs") + if any( + not isinstance(row, dict) or set(row) != set(expected) for row in raw + ): + raise ValueError( + f"{path.name} has unsupported fields; expected {expected}" + ) + return pd.DataFrame(raw) + + +_INT64 = np.iinfo(np.int64) + + +def _integer_column(frame, column): + values = frame[column].tolist() + if any(type(v) is not int for v in values): + raise ValueError(f"{column} must contain integer identifiers/years") + # JSON integers at or above 2**63 make pandas store uint64, which an + # int64 cast would silently wrap (a future event year becoming -1). + if any(not _INT64.min <= v <= _INT64.max for v in values): + raise ValueError(f"{column} must fit in a signed 64-bit integer") + frame[column] = frame[column].astype("int64") + + +def _real_numbers(values, label): + """Return float64 values, refusing JSON strings, booleans, and nulls.""" + values = values.tolist() if hasattr(values, "tolist") else list(values) + if any( + isinstance(v, (bool, np.bool_)) + or not isinstance(v, (int, float, np.integer, np.floating)) + for v in values + ): + raise ValueError( + f"{label} must be JSON numbers, not strings or booleans" + ) + try: + return np.asarray(values, dtype=np.float64) + except OverflowError as error: + raise ValueError(f"{label} must be finite") from error + + +def _weights(values): + result = _real_numbers(values, "source weights") + if not np.isfinite(result).all() or (result <= 0).any(): + raise ValueError("source weights must be positive and finite") + return Weights(result, WeightKind.DESIGN) + + +def _frame(records, periods, weights, columns): + """Give observations an explicit persistent-person and period partition.""" + observations = pd.DataFrame( + { + OID: np.arange(1, len(records) + 1, dtype=np.int64), + PID: records["person_id"].to_numpy(dtype=np.int64), + PERIOD_ID: np.asarray(periods, dtype=np.int64), + } + ) + for column, dtype in columns: + observations[column] = records[column].astype(dtype).array + person_ids = np.sort(observations[PID].unique()) + period_ids = np.sort(observations[PERIOD_ID].unique()) + return Frame( + { + OBS: observations, + "person": pd.DataFrame({"person_id": person_ids}), + "period": pd.DataFrame( + {"period_id": period_ids, "period": period_ids} + ), + }, + EntitySchema(person_entity=OBS, group_entities=("person", "period")), + {OBS: weights}, + pd.Series(["synthetic"] * len(observations), dtype=object), + ) + + +def _create_training(context): + fields = ["person_id", "event_year", "start_weight", *dict(TRAIN_COLUMNS)] + data = _records(context.sources["training"], fields) + for column in ("person_id", "event_year", "required_interview_year"): + _integer_column(data, column) + if data.duplicated(["person_id", "event_year"]).any(): + raise ValueError("training source repeats a person-period") + if not data.sex.isin(["female", "male"]).all(): + raise ValueError("training sex must be female or male") + for column in ("exposure", "death"): + numeric = _real_numbers(data[column], f"training {column}") + if not np.isfinite(numeric).all() or (numeric < 0).any(): + raise ValueError( + f"training {column} must be finite and nonnegative" + ) + data[column] = numeric + if (data.death > 1).any(): + raise ValueError("training death must lie in [0, 1]") + data = data.sort_values(["person_id", "event_year"]).reset_index(drop=True) + return KernelResult( + frame=_frame( + data, data.event_year, _weights(data.start_weight), TRAIN_COLUMNS + ) + ) + + +def _create_initial(context): + data = _records( + context.sources["initial"], ("person_id", "age", "sex", "weight") + ) + for column in ("person_id", "age"): + _integer_column(data, column) + if data.person_id.duplicated().any(): + raise ValueError("initial population repeats a person_id") + if not data.age.between(0, 120).all(): + raise ValueError("initial ages must lie in [0, 120]") + if not data.sex.isin(["female", "male"]).all(): + raise ValueError("initial sex must be female or male") + data = data.sort_values("person_id").reset_index(drop=True) + return KernelResult( + frame=_frame( + data, + [context.params["boundary_year"]] * len(data), + _weights(data.weight), + (("age", "int64"), ("sex", "string")), + ) + ) + + +def _periods(context, observations): + periods = context.tables["period"].set_index("period_id")["period"] + return observations[PERIOD_ID].map(periods).astype("int64") + + +def _fit(context): + observations = context.tables[OBS] + exposure = observations[[column for column, _ in TRAIN_COLUMNS]].copy() + exposure["person_id"] = observations[PID].to_numpy() + exposure["event_year"] = _periods(context, observations).to_numpy() + exposure["start_weight"] = context.weights[OBS].values + rates = pd.DataFrame(read_json(context.sources["rates"])) + artifact = fit_mortality( + exposure, + rates, + boundary_year=context.params["boundary_year"], + external_vintage_year=context.params["external_vintage_year"], + ) + return KernelResult( + artifacts={"model": artifact.to_bytes()}, + receipt={ + "fit_rows": artifact.fit_rows, + "boundary_year": artifact.boundary_year, + }, + ) + + +def mortality_uniforms( + person_ids, + *, + experiment_id="mortality", + replicate=0, + base_seed=0, + period=2015, + draw_index=0, +): + """Stable original-person draws, independent of observation row ordinals.""" + if any( + isinstance(pid, (bool, np.bool_)) + or not isinstance(pid, (int, np.integer)) + for pid in person_ids + ): + raise ValueError("person identities must be integers") + if ( + type(period) is not int + or type(draw_index) is not int + or draw_index < 0 + ): + raise ValueError("period and nonnegative draw index must be integers") + return keyed_uniform( + stream=("sha256-u53-v1", experiment_id, replicate, base_seed), + keys=[ + (int(pid), "mortality", int(period), int(draw_index)) + for pid in person_ids + ], + ) + + +class _PersonGenerator: + def __init__(self, person_id, module, period, stream): + self.person_id = person_id + self.module = getattr(module, "value", str(module)) + self.period = period + self.stream = stream + self.draw_index = 0 + + def random(self): + value = keyed_uniform( + stream=self.stream, + keys=[ + ( + int(self.person_id), + self.module, + self.period, + self.draw_index, + ) + ], + )[0] + self.draw_index += 1 + return float(value) + + +class _GraphPeriodContext: + """Adapter for existing steps, intentionally bypassing ordinal mapping.""" + + def __init__(self, params): + self.year = int(params["boundary_year"]) + 1 + self.metadata = {} + self.rng_registry = self # Existing apply_mortality tests for None. + self.stream = tuple(params["stream"]) + + def person_generator(self, module, person_id): + return _PersonGenerator(person_id, module, self.year, self.stream) + + +def _slice(context, observations=None): + observations = ( + context.tables[OBS] if observations is None else observations + ) + return pd.DataFrame( + { + "person_id": observations[PID].to_numpy(dtype=np.int64), + "age": observations.age.to_numpy(dtype=np.int64), + "sex": observations.sex.astype(str).to_numpy(), + "year": _periods(context, observations).to_numpy(), + } + ) + + +def _apply(context): + artifact = MortalityArtifact.from_bytes(context.artifacts["model"].payload) + if artifact.boundary_year != context.params["boundary_year"]: + raise ValueError("mortality fit and application boundary years differ") + model = artifact.model + initial = _slice(context) + survived = apply_mortality( + initial, _GraphPeriodContext(context.params), context.rng, model=model + ) + ids = context.tables[OBS][OID].tolist() + return KernelResult( + columns={ + (OBS, "death_probability"): _series( + model.probabilities(initial), ids, dtype="float64" + ), + # Nullable: a next-period row admitted by ``advance`` has had no + # mortality draw, so its outcome cells are missing, not copied. + (OBS, "survives"): _series( + initial.person_id.isin(survived.person_id).to_numpy(), + ids, + dtype="boolean", + ), + } + ) + + +def _mass(weights, survivor_weights, strata, survivor_strata, boundary_year): + def totals(values, labels): + frame = pd.DataFrame({"weight": values, "stratum": list(labels)}) + return { + str(label): float(value) + for label, value in frame.groupby("stratum", observed=True) + .weight.sum() + .items() + } + + before = totals(weights, strata) + future = totals(survivor_weights, survivor_strata) + after = totals( + np.concatenate([weights, survivor_weights]), + [*strata, *survivor_strata], + ) + partition_after = {str(boundary_year): before} + if len(survivor_weights): + partition_after[str(boundary_year + 1)] = future + return { + "policy": "declared", + "before": float(np.sum(weights)), + "after": float(np.sum(np.concatenate([weights, survivor_weights]))), + "stratum_before": before, + "stratum_after": after, + "partition": { + "entity": "period", + "column": "period", + "stratum_before": {str(boundary_year): before}, + "stratum_after": partition_after, + }, + } + + +def _advance(context): + observations = context.tables[OBS] + mask = observations.survives.to_numpy(dtype=bool) + surviving = observations.loc[mask] + boundary = int(context.params["boundary_year"]) + # The existing adapter's year is used for the entrant period group only; + # it is never written back onto an incumbent observation. + aged = advance_age( + _slice(context, surviving), + SimpleNamespace(year=boundary + 1, metadata={}), + context.rng, + ) + old_ids = observations[OID].tolist() + new_ids = list(range(max(old_ids) + 1, max(old_ids) + 1 + len(surviving))) + target_ids = old_ids + new_ids + period_ids = context.tables["period"].period_id.tolist() + period_values = context.tables["period"].period.tolist() + next_period = [boundary + 1] if len(new_ids) else [] + weights = context.weights[OBS].values + survivor_weights = weights[mask] + return KernelResult( + expand={ + OBS: _series(surviving[OID].tolist(), new_ids), + "person": _series([], [], "person"), + "period": _series( + [pd.NA] * len(next_period), next_period, "period", "Int64" + ), + }, + columns={ + (OBS, PERIOD_ID): _series( + observations[PERIOD_ID].tolist() + + [boundary + 1] * len(new_ids), + target_ids, + ), + (OBS, "age"): _series( + observations.age.tolist() + aged.age.tolist(), target_ids + ), + # Lineage would otherwise copy the parent's period-B outcome + # (survives=True, the old age band's probability) onto rows no + # mortality draw has reached. Leave those cells missing. + (OBS, "death_probability"): _series( + np.concatenate( + [ + observations.death_probability.to_numpy( + dtype=np.float64 + ), + np.full(len(new_ids), np.nan), + ] + ), + target_ids, + dtype="float64", + ), + (OBS, "survives"): _series( + observations.survives.tolist() + [pd.NA] * len(new_ids), + target_ids, + dtype="boolean", + ), + ("period", "period"): _series( + period_values + next_period, period_ids + next_period, "period" + ), + }, + weights=Weights( + np.concatenate([weights, survivor_weights]), + context.weights[OBS].kind, + ), + receipt={ + "mass": _mass( + weights, + survivor_weights, + context.strata.tolist(), + context.strata.to_numpy()[mask], + boundary, + ) + }, + ) + + +def _age_claim(context): + observations = context.tables[OBS] + return KernelResult( + columns={ + (OBS, "age"): _series( + observations.age.tolist(), observations[OID].tolist() + ) + } + ) + + +def _outcome_claim(context): + """Claim advance's missing next-period outcome cells, unchanged.""" + observations = context.tables[OBS] + ids = observations[OID].tolist() + return KernelResult( + columns={ + (OBS, "death_probability"): _series( + observations.death_probability.to_numpy(dtype=np.float64), + ids, + dtype="float64", + ), + (OBS, "survives"): _series( + observations.survives.tolist(), ids, dtype="boolean" + ), + } + ) + + +def _evaluate(context): + observations = context.tables[OBS] + boundary = int(context.params["boundary_year"]) + periods = _periods(context, observations) + initial = observations.loc[periods == boundary].copy() + future = observations.loc[periods == boundary + 1].copy() + truth_document = read_json(context.sources["holdout"]) + if ( + not isinstance(truth_document, dict) + or set(truth_document) + != {"scope", "fixture_max_abs_death_rate_gap", "outcomes"} + or truth_document["scope"] != "synthetic_engineering" + ): + raise ValueError( + "holdout requires an explicit synthetic engineering contract" + ) + truth = pd.DataFrame(truth_document["outcomes"]) + if set(truth.columns) != {"person_id", "year", "age", "death"}: + raise ValueError("invalid held-out outcome columns") + for column in truth.columns: + _integer_column(truth, column) + if truth.person_id.duplicated().any() or set(truth.person_id) != set( + initial[PID] + ): + raise ValueError( + "held-out identities must match the initial population exactly" + ) + if ( + not (truth.year == boundary + 1).all() + or not truth.death.isin([0, 1]).all() + ): + raise ValueError( + "held-out outcomes must be binary deaths in the next period" + ) + truth = truth.set_index("person_id").loc[initial[PID]] + weights = context.weights[OBS].values + start_weights = weights[(periods == boundary).to_numpy()] + next_weights = weights[(periods == boundary + 1).to_numpy()] + mass = _mass( + start_weights, + next_weights, + context.strata[(periods == boundary).to_numpy()].tolist(), + context.strata[(periods == boundary + 1).to_numpy()].tolist(), + boundary, + ) + mass["next_period"] = float(next_weights.sum()) + probability = initial.death_probability.to_numpy() + expected_deaths = float(np.dot(start_weights, probability)) + observed_deaths = float(np.dot(start_weights, truth.death.to_numpy())) + generated_deaths = float( + np.dot(start_weights, ~initial.survives.to_numpy(dtype=bool)) + ) + discrepancy = abs(expected_deaths - observed_deaths) / float( + start_weights.sum() + ) + threshold = truth_document["fixture_max_abs_death_rate_gap"] + if ( + isinstance(threshold, bool) + or not isinstance(threshold, (int, float)) + or not np.isfinite(threshold) + or not 0 <= threshold <= 1 + ): + raise ValueError("fixture death-rate gap must be finite and in [0, 1]") + artifact = MortalityArtifact.from_bytes(context.artifacts["model"].payload) + original = _slice(context, initial).sort_values("person_id") + stream = tuple(context.params["stream"]) + uniforms = mortality_uniforms( + original.person_id.tolist(), + experiment_id=stream[1], + replicate=stream[2], + base_seed=stream[3], + period=boundary + 1, + ) + expected_ids = original.loc[ + uniforms >= artifact.model.probabilities(original), "person_id" + ].tolist() + actual_ids = sorted(future[PID].tolist()) + expected_ages = original.set_index("person_id").age + 1 + age_parity = all( + int(row.age) == int(expected_ages.loc[getattr(row, PID)]) + for row in future.itertuples(index=False) + ) + # No mortality draw has reached the admitted next-period rows. + undrawn_future = bool( + future.survives.isna().all() and future.death_probability.isna().all() + ) + engineering_pass = ( + expected_ids == actual_ids + and age_parity + and undrawn_future + and not observations[OID].duplicated().any() + ) + heldout_age_error = float( + np.abs(truth.age.to_numpy() - (initial.age.to_numpy() + 1)).mean() + ) + fixture_pass = discrepancy <= threshold and heldout_age_error == 0 + report = { + "scope": "synthetic_engineering", + "boundary_year": boundary, + "next_period": boundary + 1, + "fit_rows": artifact.fit_rows, + "initial_records": len(initial), + "survivor_records": len(future), + "heldout_records": len(truth), + "expected_deaths": expected_deaths, + "observed_deaths": observed_deaths, + "generated_deaths": generated_deaths, + "absolute_death_rate_gap": discrepancy, + "fixture_max_abs_death_rate_gap": float(threshold), + "heldout_mean_absolute_age_error": heldout_age_error, + "fixture_verdict": "pass" if fixture_pass else "fail", + "engineering_verdict": "pass" if engineering_pass else "fail", + "mass": mass, + "limitations": [ + "Synthetic engineering fixture; no scientific or national-population certification.", + "The mortality fit's external-rate factor cancels in the fitted-window level.", + "Household accounting, births, immigration, and the full M6 loop are outside this slice.", + ], + } + return KernelResult( + artifacts={"report": json_bytes(report)}, + receipt={ + "outcome": "pass" if engineering_pass and fixture_pass else "fail", + "evidence": report, + }, + ) + + +class _Kernel: + def __init__( + self, + ref, + function, + *, + structural=StructuralDelta.NONE, + numeric=Numeric.PLATFORM_BITWISE, + seeded=False, + gate=False, + ): + self.ref = ref + self.function = function + self.capabilities = Capabilities( + Determinism.SEEDED if seeded else Determinism.DETERMINISTIC, + numeric=numeric, + structural=structural, + seed_source=SeedSource.KEYED if seeded else SeedSource.NONE, + role=KernelRole.GATE if gate else KernelRole.COMPUTE, + dependencies=DEPENDENCIES, + ) + + def implementation_hash(self): + return source_hash( + self.function, + model_module, + fit_mortality_model, + prepare_mortality_refit_inputs, + apply_mortality, + advance_age, + keyed_uniform, + canonical_json, + dependencies=self.capabilities.dependencies, + ) + + def run(self, context): + return self.function(context) + + +def build_graph( + *, + boundary_year=2014, + external_vintage_year=2014, + experiment_id="mortality", + replicate=0, + base_seed=0, +): + """Return the declared graph and registered existing-operation wrappers.""" + stream = ("sha256-u53-v1", experiment_id, replicate, base_seed) + keyed_uniform(stream=stream, keys=[]) # Validate even an empty population. + if ( + type(boundary_year) is not int + or type(external_vintage_year) is not int + ): + raise ValueError( + "fit boundary and external vintage must be integer years" + ) + params = {"boundary_year": boundary_year, "stream": stream} + binding = (ArtifactInput("model", "fit", "model", MODEL_TYPE),) + roots = ( + Node( + "training", + "dynamics.training@1", + sources=("training",), + structural=StructuralDelta.CREATE, + outputs=tuple( + Owned(OBS, column, dtype) for column, dtype in TRAIN_COLUMNS + ) + + (Owned("period", "period", "int64"),), + ), + Node( + "fit", + "dynamics.mortality.fit@1", + population="training", + sources=("rates",), + inputs=( + Slice(OBS, tuple(dict(TRAIN_COLUMNS))), + Slice("period", ("period",)), + ), + params={ + "boundary_year": boundary_year, + "external_vintage_year": external_vintage_year, + }, + artifact_outputs=(ArtifactOutput("model", MODEL_TYPE),), + ), + Node( + "initial", + "dynamics.initial@1", + sources=("initial",), + structural=StructuralDelta.CREATE, + params={"boundary_year": boundary_year}, + outputs=( + Owned(OBS, "age", "int64"), + Owned(OBS, "sex", "string"), + Owned("period", "period", "int64"), + ), + ), + Node( + "apply", + "dynamics.mortality.apply@1", + population="initial", + artifact_inputs=binding, + inputs=(Slice(OBS, ("age", "sex")), Slice("period", ("period",))), + params=params, + outputs=( + Owned(OBS, "death_probability", "float64"), + Owned(OBS, "survives", "boolean"), + ), + ), + Node( + "advance", + "dynamics.advance@1", + base="initial", + structural=StructuralDelta.EXPAND, + entrants=True, + mass="declared", + inputs=( + Slice(OBS, ("age", "sex", "survives", "death_probability")), + Slice("period", ("period",)), + ), + params={ + "boundary_year": boundary_year, + "expand_cells": ( + (OBS, PERIOD_ID, "int64"), + (OBS, "age", "int64"), + (OBS, "death_probability", "float64"), + (OBS, "survives", "boolean"), + ("period", "period", "int64"), + ), + "expand_weight_entity": OBS, + "expand_weight_kind": "design", + }, + ), + Node( + "age", + "dynamics.age-claim@1", + population="advance", + inputs=(Slice(OBS, ("age",)),), + outputs=(Owned(OBS, "age", "int64", rewrite=True),), + ), + Node( + "outcomes", + "dynamics.mortality.outcome-claim@1", + population="advance", + inputs=(Slice(OBS, ("survives", "death_probability")),), + outputs=( + Owned(OBS, "death_probability", "float64", rewrite=True), + Owned(OBS, "survives", "boolean", rewrite=True), + ), + ), + Node( + "evaluate", + "dynamics.mortality.evaluate@1", + population="advance", + sources=("holdout",), + artifact_inputs=binding, + params=params, + inputs=( + Slice(OBS, ("age", "sex", "survives", "death_probability")), + Slice("period", ("period",)), + ), + ), + ) + registry = KernelRegistry() + for kernel in ( + _Kernel( + "dynamics.training@1", + _create_training, + structural=StructuralDelta.CREATE, + numeric=Numeric.BITWISE, + ), + _Kernel("dynamics.mortality.fit@1", _fit), + _Kernel( + "dynamics.initial@1", + _create_initial, + structural=StructuralDelta.CREATE, + numeric=Numeric.BITWISE, + ), + _Kernel("dynamics.mortality.apply@1", _apply, seeded=True), + _Kernel( + "dynamics.advance@1", _advance, structural=StructuralDelta.EXPAND + ), + _Kernel("dynamics.age-claim@1", _age_claim), + _Kernel("dynamics.mortality.outcome-claim@1", _outcome_claim), + _Kernel( + "dynamics.mortality.evaluate@1", _evaluate, seeded=True, gate=True + ), + ): + registry.register(kernel) + SOURCE_CODECS.register(CODEC, _json_source_marker) + return ( + Graph( + "dynamics-mortality", + tuple( + SourceRef(name, CODEC) + for name in ("training", "rates", "initial", "holdout") + ), + roots, + mass_partition=("period", "period"), + ), + registry, + ) + + +@dataclass(frozen=True) +class MortalityGraphRun: + manifest: object + report: dict + model_payload: bytes + next_slice: pd.DataFrame + + +def run_mortality_graph( + *, + training, + rates, + initial, + holdout, + output_dir, + boundary_year=2014, + external_vintage_year=2014, + experiment_id="mortality", + replicate=0, + base_seed=0, + household_accounting=False, +): + """Run/reuse the graph and write artifacts only in the explicit directory.""" + if household_accounting: + raise ValueError( + "household accounting is unsupported by this person-period slice" + ) + output = Path(output_dir).resolve() + sources = { + name: Path(path).resolve() + for name, path in ( + ("training", training), + ("rates", rates), + ("initial", initial), + ("holdout", holdout), + ) + } + graph, registry = build_graph( + boundary_year=boundary_year, + external_vintage_year=external_vintage_year, + experiment_id=experiment_id, + replicate=replicate, + base_seed=base_seed, + ) + output.mkdir(parents=True, exist_ok=True) + store = ContentStore(output / "store") + manifest = run_graph( + compile_graph(graph), sources=sources, store=store, kernels=registry + ) + # Gate exceptions are evidenced failures in the executor's contract and + # may deliberately produce no report artifact. Retain that receipt before + # reading optional diagnostics, including on a cached failed-gate hit. + (output / "manifest.json").write_text(manifest.to_json(), encoding="utf-8") + model_payload = store.load_bytes( + manifest.nodes["fit"].opaque_artifacts["model"] + ) + evaluation = manifest.nodes["evaluate"] + report_key = evaluation.opaque_artifacts.get("report") + if report_key is not None: + report = parse_json(store.load_bytes(report_key)) + elif evaluation.receipt.get("outcome") == "fail": + report = { + "scope": "synthetic_engineering", + "boundary_year": boundary_year, + "next_period": boundary_year + 1, + "engineering_verdict": "not_evaluated", + "fixture_verdict": "not_evaluated", + "evaluation_gate": { + "node_id": "evaluate", + "kernel_ref": evaluation.kernel_ref, + "outcome": "fail", + "evidence": dict(evaluation.receipt.get("evidence", {})), + }, + } + else: + raise ValueError( + "evaluation report is absent without a failed gate receipt" + ) + report["node_keys"] = { + name: node.key for name, node in manifest.nodes.items() + } + report["cache_hits"] = { + name: node.hit for name, node in manifest.nodes.items() + } + report["model_artifact_key"] = manifest.nodes["fit"].opaque_artifacts[ + "model" + ] + population = manifest.population("advance") + observations = population.table(OBS) + next_rows = observations.loc[observations[PERIOD_ID] == boundary_year + 1] + next_slice = ( + pd.DataFrame( + { + "person_id": next_rows[PID].to_numpy(dtype=np.int64), + "age": next_rows.age.to_numpy(dtype=np.int64), + "year": np.full( + len(next_rows), boundary_year + 1, dtype=np.int64 + ), + } + ) + .sort_values("person_id") + .reset_index(drop=True) + ) + (output / "report.json").write_bytes(json_bytes(report)) + (output / "model.json").write_bytes(model_payload) + for entity in (OBS, "person", "period"): + population.table(entity).to_csv(output / f"{entity}.csv", index=False) + next_slice.to_csv(output / "next_period.csv", index=False) + return MortalityGraphRun(manifest, report, model_payload, next_slice) diff --git a/src/populace_dynamics/graph/synthetic.py b/src/populace_dynamics/graph/synthetic.py new file mode 100644 index 00000000..24808993 --- /dev/null +++ b/src/populace_dynamics/graph/synthetic.py @@ -0,0 +1,78 @@ +"""Small hand-specified engineering inputs, independent of generated draws.""" + +from pathlib import Path + +from .model import json_bytes + + +def write_synthetic_inputs(directory): + """Write synthetic sources to an explicit directory; preserve edits.""" + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + training = [ + { + "person_id": 1001 + i, + "event_year": 2013, + "required_interview_year": 2013, + "age_band": "0+", + "sex": "female" if i < 4 else "male", + "start_weight": 1.0, + "exposure": 1.0, + "death": float(i % 4 == 0), + } + for i in range(8) + ] + training.extend( + [ + {**training[0], "person_id": 1009, "event_year": 2015}, + { + **training[0], + "person_id": 1010, + "required_interview_year": 2015, + }, + ] + ) + rates = [ + { + "lower_age": 0, + "upper_age": 120, + "age_band": "0+", + "sex": sex, + "central_rate": rate, + } + for sex, rate in (("female", 0.005), ("male", 0.006)) + ] + initial = [ + { + "person_id": 100 + i, + "age": 30 + 2 * i, + "sex": "female" if i % 2 == 0 else "male", + "weight": float(1 + i % 3), + } + for i in range(20) + ] + holdout = { + "scope": "synthetic_engineering", + "fixture_max_abs_death_rate_gap": 0.25, + "outcomes": [ + { + "person_id": row["person_id"], + "year": 2015, + "age": row["age"] + 1, + "death": int(i % 5 == 0), + } + for i, row in enumerate(initial) + ], + } + result = {} + for name, value in ( + ("training", training), + ("rates", rates), + ("initial", initial), + ("holdout", holdout), + ): + path = directory / f"{name}.json" + if not path.exists(): + path.write_bytes(json_bytes(value)) + result[name] = path + return result diff --git a/src/populace_dynamics/graph/trajectory.py b/src/populace_dynamics/graph/trajectory.py new file mode 100644 index 00000000..b457b08b --- /dev/null +++ b/src/populace_dynamics/graph/trajectory.py @@ -0,0 +1,898 @@ +"""Synthetic annual mortality transport with immutable period history. + +The fitted law and ageing step are the existing Dynamics implementations. +This optional graph supplies typed transition edges, stable person draws, +and explicit annual engineering diagnostics. It certifies no population. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +from microcosm.frame import Weights +from microcosm.graph.decl import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Graph, + Node, + Owned, + Slice, + SourceRef, + StructuralDelta, + compile_graph, +) +from microcosm.graph.executor import run_graph +from microcosm.graph.kernel import KernelResult, source_hash +from microcosm.graph.store import ContentStore + +from . import runtime as rt +from .model import MortalityArtifact, json_bytes, parse_json, read_json + +TRANSITION_TYPE = ArtifactType("populace-dynamics.mortality-transition", 1) +SNAPSHOT_TYPE = ArtifactType("populace-dynamics.mortality-population", 1) +_TRANSITION_FIELDS = { + "person_id", + "observation_id", + "death_probability", + "survives", +} + + +def _risk_rows(context): + observations = context.tables[rt.OBS] + periods = rt._periods(context, observations) + return observations.loc[periods == context.params["year"] - 1].sort_values( + rt.PID + ) + + +def _model(context): + artifact = MortalityArtifact.from_bytes(context.artifacts["model"].payload) + if artifact.boundary_year != context.params["boundary_year"]: + raise ValueError("mortality fit and trajectory boundary years differ") + return artifact.model + + +def _check_age_support(frame, model): + age = frame.age.to_numpy(dtype=np.int64) + if ((age < model.bands[0][0]) | (age > model.bands[-1][1])).any(): + raise ValueError( + "mortality application age exceeds the fitted age support " + f"[{model.bands[0][0]}, {model.bands[-1][1]}]" + ) + + +def _outcome( + year, status, *, records=(), completed_year=None, diagnostic=None +): + return { + "format": TRANSITION_TYPE.name, + "schema_version": 1, + "from_year": year - 1, + "year": year, + "status": status, + "completed_year": year if completed_year is None else completed_year, + "records": list(records), + "diagnostic": diagnostic, + } + + +def _apply_complete(context): + model = _model(context) + risk = _risk_rows(context) + initial = rt._slice(context, risk) + _check_age_support(initial, model) + year = context.params["year"] + survived = rt.apply_mortality( + initial, + rt._GraphPeriodContext( + {"boundary_year": year - 1, "stream": context.params["stream"]} + ), + context.rng, + model=model, + ) + probability = model.probabilities(initial) + survivors = set(survived.person_id) + records = [ + { + "person_id": int(pid), + "observation_id": int(oid), + "death_probability": float(p), + "survives": int(pid) in survivors, + } + for pid, oid, p in zip( + risk[rt.PID], risk[rt.OID], probability, strict=True + ) + ] + return _outcome(year, "complete", records=records) + + +def _apply(context): + year = context.params["year"] + completed_year = context.params["boundary_year"] + try: + previous = context.artifacts.get("previous_transition") + if previous is not None: + previous = _decode_transition(previous.payload, year - 1) + if previous["completed_year"] < completed_year: + raise ValueError( + "prior transition predates the model boundary" + ) + completed_year = previous["completed_year"] + if previous is not None and previous["status"] != "complete": + outcome = _outcome( + year, + "blocked", + completed_year=previous["completed_year"], + diagnostic={ + "blocked_by": f"apply_{year - 1}", + "message": "previous mortality transition did not complete", + }, + ) + else: + outcome = _apply_complete(context) + except Exception as error: + # The exact pinned core requires every declared artifact even on a + # failed gate. Publish an explicit typed failure, never survivor data. + outcome = _outcome( + year, + "failed", + completed_year=completed_year, + diagnostic={ + "exception_type": type(error).__name__, + "message": str(error), + }, + ) + complete = outcome["status"] == "complete" + return KernelResult( + artifacts={"transition": json_bytes(outcome)}, + receipt={ + "outcome": ( + ("pass" if outcome["records"] else "not_applicable") + if complete + else "fail" + ), + "application_status": outcome["status"], + "completed_year": outcome["completed_year"], + "evidence": outcome["diagnostic"] + or {"year": year, "risk_records": len(outcome["records"])}, + }, + ) + + +def _decode_transition(payload, year): + raw = parse_json(payload) + if ( + not isinstance(raw, dict) + or set(raw) + != { + "format", + "schema_version", + "from_year", + "year", + "records", + "status", + "completed_year", + "diagnostic", + } + or raw["format"] != TRANSITION_TYPE.name + or type(raw["schema_version"]) is not int + or raw["schema_version"] != 1 + or type(raw["year"]) is not int + or type(raw["from_year"]) is not int + or raw["year"] != year + or raw["from_year"] != year - 1 + or not isinstance(raw["records"], list) + or raw["status"] not in ("complete", "failed", "blocked") + or type(raw["completed_year"]) is not int + ): + raise ValueError("invalid annual mortality transition contract") + if raw["status"] == "complete": + if raw["completed_year"] != year or raw["diagnostic"] is not None: + raise ValueError("invalid completed mortality transition") + elif ( + raw["records"] + or raw["completed_year"] >= year + or not isinstance(raw["diagnostic"], dict) + or not isinstance(raw["diagnostic"].get("message"), str) + ): + raise ValueError("invalid failed or blocked mortality transition") + return raw + + +def _transition(context): + """Validate both the byte contract and the recipient observation binding.""" + raw = _decode_transition( + context.artifacts["transition"].payload, context.params["year"] + ) + risk = _risk_rows(context) + if raw["status"] != "complete": + return risk, raw + expected = list(zip(risk[rt.PID], risk[rt.OID], strict=True)) + records = raw["records"] + if len(records) != len(expected): + raise ValueError("mortality transition differs from the risk set") + for row, (person_id, observation_id) in zip( + records, expected, strict=True + ): + if ( + not isinstance(row, dict) + or set(row) != _TRANSITION_FIELDS + or type(row["person_id"]) is not int + or type(row["observation_id"]) is not int + or row["person_id"] != person_id + or row["observation_id"] != observation_id + or type(row["survives"]) is not bool + ): + raise ValueError( + "invalid mortality transition observation binding" + ) + p = row["death_probability"] + if ( + isinstance(p, bool) + or not isinstance(p, (int, float)) + or not np.isfinite(p) + or not 0 <= p <= 1 + ): + raise ValueError("invalid mortality transition probability") + return risk, raw + + +def _mass(weights, strata, periods): + frame = pd.DataFrame( + {"weight": weights, "stratum": strata, "period": periods} + ) + totals = { + str(key): float(value) + for key, value in frame.groupby("stratum", observed=True) + .weight.sum() + .items() + } + partition = { + str(year): { + str(key): float(value) + for key, value in part.groupby("stratum", observed=True) + .weight.sum() + .items() + } + for year, part in frame.groupby("period", observed=True) + } + return totals, partition + + +def _advance(context): + observations = context.tables[rt.OBS] + risk, outcome = _transition(context) + complete = outcome["status"] == "complete" + records = outcome["records"] + mask = np.asarray([row["survives"] for row in records], dtype=bool) + surviving = risk.loc[mask] if complete else risk.iloc[:0] + year = context.params["year"] + aged = ( + rt.advance_age( + rt._slice(context, surviving), + SimpleNamespace(year=year, metadata={}), + context.rng, + ) + if complete + else rt._slice(context, surviving) + ) + old_ids = observations[rt.OID].tolist() + new_ids = list(range(max(old_ids) + 1, max(old_ids) + 1 + len(surviving))) + target_ids = old_ids + new_ids + period_ids = context.tables["period"].period_id.tolist() + period_values = context.tables["period"].period.tolist() + next_period = [year] if len(new_ids) else [] + weights = context.weights[rt.OBS].values + position = pd.Series(np.arange(len(observations)), index=old_ids) + source_positions = position.loc[surviving[rt.OID]].to_numpy(dtype=int) + survivor_weights = weights[source_positions] + expanded_weights = np.concatenate([weights, survivor_weights]) + strata = context.strata.to_numpy() + periods = rt._periods(context, observations).to_numpy() + before, partitions_before = _mass(weights, strata, periods) + after, partitions_after = _mass( + expanded_weights, + np.concatenate([strata, strata[source_positions]]), + np.concatenate([periods, np.full(len(new_ids), year, dtype=int)]), + ) + return KernelResult( + expand={ + rt.OBS: rt._series(surviving[rt.OID].tolist(), new_ids), + "person": rt._series([], [], "person"), + "period": rt._series( + [pd.NA] * len(next_period), next_period, "period", "Int64" + ), + }, + columns={ + (rt.OBS, rt.PERIOD_ID): rt._series( + observations[rt.PERIOD_ID].tolist() + [year] * len(new_ids), + target_ids, + ), + (rt.OBS, "age"): rt._series( + observations.age.tolist() + aged.age.tolist(), target_ids + ), + ("period", "period"): rt._series( + period_values + next_period, period_ids + next_period, "period" + ), + }, + weights=Weights(expanded_weights, context.weights[rt.OBS].kind), + receipt={ + "application_status": "complete" if complete else "blocked", + "completed_year": outcome["completed_year"], + "mass": { + "policy": "declared", + "before": float(weights.sum()), + "after": float(expanded_weights.sum()), + "stratum_before": before, + "stratum_after": after, + "partition": { + "entity": "period", + "column": "period", + "stratum_before": partitions_before, + "stratum_after": partitions_after, + }, + }, + }, + ) + + +def _holdout(context): + year = context.params["year"] + raw = read_json(context.sources[f"holdout_{year}"]) + if ( + not isinstance(raw, dict) + or set(raw) + != { + "scope", + "year", + "expected_death_rate", + "fixture_max_abs_death_rate_gap", + } + or raw["scope"] != "synthetic_engineering" + or type(raw["year"]) is not int + or raw["year"] != year + ): + raise ValueError( + "holdout requires the matching year and synthetic engineering " + "aggregate contract" + ) + for key in ("expected_death_rate", "fixture_max_abs_death_rate_gap"): + value = raw[key] + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not np.isfinite(value) + or not 0 <= value <= 1 + ): + raise ValueError(f"holdout {key} must be finite and in [0, 1]") + return raw + + +def _snapshot(context): + """Transport the actual executor population to an isolated evaluation.""" + return KernelResult( + artifacts={ + "snapshot": json_bytes( + { + "format": SNAPSHOT_TYPE.name, + "schema_version": 1, + "year": context.params["year"], + "observations": context.tables[rt.OBS].to_dict( + orient="records" + ), + "periods": context.tables["period"].to_dict( + orient="records" + ), + "weights": context.weights[rt.OBS].values.tolist(), + } + ) + } + ) + + +def _evaluation_context(context): + raw = parse_json(context.artifacts["snapshot"].payload) + if ( + not isinstance(raw, dict) + or set(raw) + != { + "format", + "schema_version", + "year", + "observations", + "periods", + "weights", + } + or raw["format"] != SNAPSHOT_TYPE.name + or type(raw["schema_version"]) is not int + or raw["schema_version"] != 1 + or type(raw["year"]) is not int + or raw["year"] != context.params["year"] + or not isinstance(raw["observations"], list) + or not isinstance(raw["periods"], list) + or not isinstance(raw["weights"], list) + ): + raise ValueError("invalid annual mortality population snapshot") + observations = pd.DataFrame(raw["observations"]) + periods = pd.DataFrame(raw["periods"]) + if ( + set(observations.columns) + != {rt.OID, rt.PID, rt.PERIOD_ID, "age", "sex"} + or set(periods.columns) != {"period_id", "period"} + or len(observations) != len(raw["weights"]) + or not observations[rt.OID].is_unique + or not periods.period_id.is_unique + or not observations[rt.PERIOD_ID].isin(periods.period_id).all() + ): + raise ValueError("invalid annual mortality snapshot row binding") + for column in (rt.OID, rt.PID, rt.PERIOD_ID, "age"): + rt._integer_column(observations, column) + for column in ("period_id", "period"): + rt._integer_column(periods, column) + if not observations.sex.isin(["female", "male"]).all(): + raise ValueError("invalid annual mortality snapshot sex") + return SimpleNamespace( + tables={rt.OBS: observations, "period": periods}, + weights={rt.OBS: rt._weights(raw["weights"])}, + params=context.params, + artifacts=context.artifacts, + sources=context.sources, + ) + + +def _evaluate(context): + context = _evaluation_context(context) + risk, outcome = _transition(context) + if outcome["status"] != "complete": + report = { + "scope": "synthetic_engineering", + "from_year": context.params["year"] - 1, + "year": context.params["year"], + "application_status": outcome["status"], + "completed_year": outcome["completed_year"], + "engineering_verdict": "not_evaluated", + "fixture_verdict": "not_evaluated", + "application_gate": { + "node_id": f"apply_{context.params['year']}", + "kernel_ref": "dynamics.trajectory.mortality.apply@1", + "outcome": "fail", + "evidence": outcome["diagnostic"], + }, + } + return KernelResult( + artifacts={"report": json_bytes(report)}, + receipt={"outcome": "evidence_absent", "evidence": report}, + ) + holdout = _holdout(context) + model = _model(context) + records = outcome["records"] + observations = context.tables[rt.OBS] + year = context.params["year"] + future = observations.loc[ + rt._periods(context, observations) == year + ].sort_values(rt.PID) + initial = rt._slice(context, risk) + _check_age_support(initial, model) + probability = model.probabilities(initial) + stream = tuple(context.params["stream"]) + uniforms = rt.mortality_uniforms( + initial.person_id.tolist(), + experiment_id=stream[1], + replicate=stream[2], + base_seed=stream[3], + period=year, + ) + survives = uniforms >= probability + expected_ids = initial.loc[survives, "person_id"].tolist() + weights = pd.Series( + context.weights[rt.OBS].values, + index=observations[rt.OID], + ) + start_weights = weights.loc[risk[rt.OID]].to_numpy() + next_weights = weights.loc[future[rt.OID]].to_numpy() + start_mass = float(start_weights.sum()) + expected_deaths = float(np.dot(start_weights, probability)) + generated_deaths = float( + start_weights[~risk[rt.PID].isin(future[rt.PID]).to_numpy()].sum() + ) + expected_ages = initial.loc[survives, "age"].to_numpy() + 1 + transition_parity = [ + row["survives"] for row in records + ] == survives.tolist() and np.array_equal( + [row["death_probability"] for row in records], probability + ) + engineering_pass = ( + transition_parity + and expected_ids == future[rt.PID].tolist() + and np.array_equal(expected_ages, future.age.to_numpy()) + and np.array_equal(start_weights[survives], next_weights) + and observations[rt.OID].is_unique + and not observations.duplicated([rt.PID, rt.PERIOD_ID]).any() + ) + rate_gap = ( + abs(expected_deaths / start_mass - holdout["expected_death_rate"]) + if start_mass + else None + ) + fixture_pass = ( + rate_gap <= holdout["fixture_max_abs_death_rate_gap"] + if rate_gap is not None + else True + ) + report = { + "scope": "synthetic_engineering", + "application_status": "complete", + "completed_year": year, + "from_year": year - 1, + "year": year, + "initial_records": len(risk), + "survivor_records": len(future), + "expected_deaths": expected_deaths, + "generated_deaths": generated_deaths, + "start_mass": start_mass, + "next_period_mass": float(next_weights.sum()), + "absolute_death_rate_gap": rate_gap, + "fixture_expected_death_rate": holdout["expected_death_rate"], + "fixture_max_abs_death_rate_gap": holdout[ + "fixture_max_abs_death_rate_gap" + ], + "engineering_verdict": ( + ("pass" if len(risk) else "not_applicable") + if engineering_pass + else "fail" + ), + "fixture_verdict": ( + ("pass" if fixture_pass else "fail") + if len(risk) + else "not_applicable" + ), + } + return KernelResult( + artifacts={"report": json_bytes(report)}, + receipt={ + "outcome": ( + ("pass" if len(risk) else "not_applicable") + if engineering_pass and fixture_pass + else "fail" + ), + "evidence": report, + }, + ) + + +class _TrajectoryKernel(rt._Kernel): + def implementation_hash(self): + # source_hash includes complete defining modules, including helpers. + # Keep these new wrappers separate from the unchanged fit/root hashes. + return source_hash( + self.function, + rt, + rt.model_module, + rt.fit_mortality_model, + rt.prepare_mortality_refit_inputs, + rt.apply_mortality, + rt.advance_age, + rt.keyed_uniform, + rt.canonical_json, + dependencies=self.capabilities.dependencies, + ) + + +def build_trajectory_graph( + *, + end_year, + boundary_year=2014, + external_vintage_year=2014, + experiment_id="mortality", + replicate=0, + base_seed=0, +): + """Declare one fit and independent annual transition/evaluation nodes.""" + if type(end_year) is not int or end_year <= boundary_year: + raise ValueError("end_year must be an integer after boundary_year") + original, registry = rt.build_graph( + boundary_year=boundary_year, + external_vintage_year=external_vintage_year, + experiment_id=experiment_id, + replicate=replicate, + base_seed=base_seed, + ) + nodes = list(original.nodes[:3]) + sources = list(original.sources[:3]) + for kernel in ( + _TrajectoryKernel( + "dynamics.trajectory.mortality.apply@1", + _apply, + seeded=True, + gate=True, + ), + _TrajectoryKernel( + "dynamics.trajectory.advance@1", + _advance, + structural=StructuralDelta.EXPAND, + ), + _TrajectoryKernel( + "dynamics.trajectory.snapshot@1", + _snapshot, + ), + _TrajectoryKernel( + "dynamics.trajectory.mortality.evaluate@1", + _evaluate, + seeded=True, + gate=True, + ), + ): + registry.register(kernel) + model_binding = ArtifactInput("model", "fit", "model", rt.MODEL_TYPE) + stream = ("sha256-u53-v1", experiment_id, replicate, base_seed) + base = "initial" + previous_transition = () + for year in range(boundary_year + 1, end_year + 1): + source = f"holdout_{year}" + sources.append(SourceRef(source, rt.CODEC)) + params = { + "year": year, + "boundary_year": boundary_year, + "stream": stream, + } + apply_id, advance_id = f"apply_{year}", f"advance_{year}" + transition_binding = ArtifactInput( + "transition", apply_id, "transition", TRANSITION_TYPE + ) + slices = (Slice(rt.OBS, ("age", "sex")), Slice("period", ("period",))) + nodes.extend( + ( + Node( + apply_id, + "dynamics.trajectory.mortality.apply@1", + population=base, + inputs=slices, + params=params, + artifact_inputs=(model_binding, *previous_transition), + artifact_outputs=( + ArtifactOutput("transition", TRANSITION_TYPE), + ), + ), + Node( + advance_id, + "dynamics.trajectory.advance@1", + base=base, + structural=StructuralDelta.EXPAND, + entrants=True, + mass="declared", + inputs=slices, + artifact_inputs=(transition_binding,), + params={ + "year": year, + "expand_cells": ( + (rt.OBS, rt.PERIOD_ID, "int64"), + (rt.OBS, "age", "int64"), + ("period", "period", "int64"), + ), + "expand_weight_entity": rt.OBS, + "expand_weight_kind": "design", + }, + ), + Node( + f"age_{year}", + "dynamics.age-claim@1", + population=advance_id, + inputs=(Slice(rt.OBS, ("age",)),), + outputs=(Owned(rt.OBS, "age", "int64", rewrite=True),), + ), + Node( + f"snapshot_{year}", + "dynamics.trajectory.snapshot@1", + population=advance_id, + inputs=slices, + params={"year": year}, + artifact_outputs=( + ArtifactOutput("snapshot", SNAPSHOT_TYPE), + ), + ), + Node( + f"evaluate_{year}", + "dynamics.trajectory.mortality.evaluate@1", + # The exact pinned compiler makes EXPAND depend on all + # ordinary members of its base. Evaluate an actual + # population snapshot on a separate existing version so + # its holdout never enters the next transition's key. + population="training", + sources=(source,), + artifact_inputs=( + model_binding, + transition_binding, + ArtifactInput( + "snapshot", + f"snapshot_{year}", + "snapshot", + SNAPSHOT_TYPE, + ), + ), + params=params, + ), + ) + ) + base = advance_id + previous_transition = ( + ArtifactInput( + "previous_transition", apply_id, "transition", TRANSITION_TYPE + ), + ) + return ( + Graph( + "dynamics-mortality-trajectory", + tuple(sources), + tuple(nodes), + mass_partition=original.mass_partition, + ), + registry, + ) + + +@dataclass(frozen=True) +class MortalityTrajectoryRun: + manifest: object + report: dict + model_payload: bytes + trajectory: pd.DataFrame + + +def _gate_diagnostic(node_id, node): + return { + "node_id": node_id, + "kernel_ref": node.kernel_ref, + "outcome": node.receipt.get("outcome"), + "evidence": dict(node.receipt.get("evidence", {})), + } + + +def _rollup(periods, field): + verdicts = {period[field] for period in periods.values()} + if "fail" in verdicts: + return "fail" + if "not_evaluated" in verdicts: + return "not_evaluated" + return "pass" if "pass" in verdicts else "not_applicable" + + +def run_mortality_trajectory( + *, + training, + rates, + initial, + holdouts, + end_year, + output_dir, + boundary_year=2014, + external_vintage_year=2014, + experiment_id="mortality", + replicate=0, + base_seed=0, + household_accounting=False, +): + """Run the optional annual DAG, retaining explicit engineering evidence.""" + if household_accounting: + raise ValueError("household accounting is unsupported by this graph") + graph, registry = build_trajectory_graph( + end_year=end_year, + boundary_year=boundary_year, + external_vintage_year=external_vintage_year, + experiment_id=experiment_id, + replicate=replicate, + base_seed=base_seed, + ) + years = range(boundary_year + 1, end_year + 1) + if ( + not isinstance(holdouts, dict) + or any(type(year) is not int for year in holdouts) + or set(holdouts) != set(years) + ): + raise ValueError("holdouts must supply exactly one source per year") + sources = { + "training": Path(training).resolve(), + "rates": Path(rates).resolve(), + "initial": Path(initial).resolve(), + **{ + f"holdout_{year}": Path(holdouts[year]).resolve() for year in years + }, + } + output = Path(output_dir).resolve() + output.mkdir(parents=True, exist_ok=True) + store = ContentStore(output / "store") + manifest = run_graph( + compile_graph(graph), sources=sources, store=store, kernels=registry + ) + (output / "manifest.json").write_text(manifest.to_json(), encoding="utf-8") + model_key = manifest.nodes["fit"].opaque_artifacts["model"] + model_payload = store.load_bytes(model_key) + periods = {} + last_population = "initial" + for year in years: + evaluation = manifest.nodes[f"evaluate_{year}"] + key = evaluation.opaque_artifacts.get("report") + if key is not None: + periods[str(year)] = parse_json(store.load_bytes(key)) + else: + period = { + "scope": "synthetic_engineering", + "from_year": year - 1, + "year": year, + "engineering_verdict": "not_evaluated", + "fixture_verdict": "not_evaluated", + } + application = manifest.nodes[f"apply_{year}"] + if application.receipt.get("outcome") == "fail": + period["application_gate"] = _gate_diagnostic( + f"apply_{year}", application + ) + elif evaluation.receipt.get("outcome") == "fail": + period["evaluation_gate"] = _gate_diagnostic( + f"evaluate_{year}", evaluation + ) + periods[str(year)] = period + advance = manifest.nodes[f"advance_{year}"] + if advance.frame_key is not None: + last_population = f"advance_{year}" + population = manifest.population(last_population) + observations = population.table(rt.OBS) + period_values = population.table("period").set_index("period_id").period + trajectory = ( + pd.DataFrame( + { + "person_id": observations[rt.PID].to_numpy(dtype=np.int64), + "age": observations.age.to_numpy(dtype=np.int64), + "year": observations[rt.PERIOD_ID] + .map(period_values) + .to_numpy(dtype=np.int64), + "weight": population.weights_for(rt.OBS).values, + } + ) + .sort_values(["year", "person_id"]) + .reset_index(drop=True) + ) + report = { + "scope": "synthetic_engineering", + "boundary_year": boundary_year, + "end_year": end_year, + "completed_year": max( + boundary_year, + *( + node.receipt.get("completed_year", boundary_year) + for name, node in manifest.nodes.items() + if name.startswith("apply_") + ), + ), + "periods": periods, + "engineering_verdict": _rollup(periods, "engineering_verdict"), + "fixture_verdict": _rollup(periods, "fixture_verdict"), + "execution_status": ( + "failed" + if any( + period["engineering_verdict"] == "not_evaluated" + for period in periods.values() + ) + else "complete" + ), + "model_artifact_key": model_key, + "node_keys": {name: node.key for name, node in manifest.nodes.items()}, + "cache_hits": { + name: node.hit for name, node in manifest.nodes.items() + }, + "limitations": [ + "Synthetic engineering fixture; no scientific or national-population certification.", + "The mortality fit's external-rate factor cancels in the fitted-window level.", + "Household accounting, births, immigration, and the full M6 loop are outside this graph.", + "The pinned core executes guarded descendants after a typed failure; application-level blocked does not mean native executor unreached.", + ], + } + (output / "report.json").write_bytes(json_bytes(report)) + (output / "model.json").write_bytes(model_payload) + trajectory.to_csv(output / "trajectory.csv", index=False) + return MortalityTrajectoryRun(manifest, report, model_payload, trajectory) diff --git a/src/populace_dynamics/graph/trajectory_accounting.py b/src/populace_dynamics/graph/trajectory_accounting.py new file mode 100644 index 00000000..82b3f654 --- /dev/null +++ b/src/populace_dynamics/graph/trajectory_accounting.py @@ -0,0 +1,284 @@ +"""Optional accounting of the existing annual mortality graph's artifacts. + +No population or transition is changed. Death declarations come from the +typed mortality transition, never from the difference between two rosters. +Import this optional module only with the reviewed graph dependencies. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from pathlib import Path + +import pandas as pd +from microcosm.graph.decl import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Node, + compile_graph, +) +from microcosm.graph.executor import run_graph +from microcosm.graph.kernel import KernelResult, source_hash +from microcosm.graph.store import ContentStore + +from populace_dynamics.engine import accounting + +from . import trajectory +from ._compat import require_graph +from .model import json_bytes, parse_json + +ACCOUNT_TYPE = ArtifactType("populace-dynamics.mortality-stock-flow", 1) +ACCOUNT_KERNEL = "dynamics.trajectory.stock-flow@1" + + +def _validate_raw_snapshot(payload): + """Apply the accountant's scalar domain before pandas can coerce values.""" + raw = parse_json(payload) + rt = trajectory.rt + if ( + not isinstance(raw, dict) + or not isinstance(raw.get("observations"), list) + or not isinstance(raw.get("periods"), list) + or not isinstance(raw.get("weights"), list) + or len(raw["observations"]) != len(raw["weights"]) + ): + raise ValueError("invalid accounting snapshot row/weight structure") + accounting._as_year(raw.get("year"), "snapshot.year") + for row in raw["observations"]: + if not isinstance(row, dict) or set(row) != { + rt.OID, + rt.PID, + rt.PERIOD_ID, + "age", + "sex", + }: + raise ValueError("invalid annual mortality snapshot row binding") + for column in (rt.OID, rt.PID, rt.PERIOD_ID, "age"): + accounting._as_person_id(row[column], f"snapshot.{column}") + for row in raw["periods"]: + if not isinstance(row, dict) or set(row) != {"period_id", "period"}: + raise ValueError("invalid accounting snapshot period structure") + for column in ("period_id", "period"): + accounting._as_year(row[column], f"snapshot.{column}") + for value in raw["weights"]: + accounting._as_weight(value, "snapshot.weight") + + +def _period_frames(context): + """Copy endpoint rows, retaining columns and positional weight binding.""" + rt = trajectory.rt + observations = context.tables[rt.OBS] + periods = rt._periods(context, observations) + calendar = context.tables["period"].period + if ( + not calendar.is_unique + or (calendar > context.params["year"]).any() + or (calendar < context.params["boundary_year"]).any() + or pd.DataFrame({"person_id": observations[rt.PID], "year": periods}) + .duplicated() + .any() + ): + raise ValueError("invalid accounting snapshot person-period history") + frames = [] + for year in (context.params["year"] - 1, context.params["year"]): + mask = periods == year + frame = observations.loc[mask].copy() + frame["person_id"] = frame[rt.PID].to_numpy(copy=True) + frame["year"] = periods.loc[mask].to_numpy(copy=True) + frame["weight"] = context.weights[rt.OBS].values[mask.to_numpy()] + frames.append(frame) + return frames + + +def _account(context): + year = context.params["year"] + report = { + "format": ACCOUNT_TYPE.name, + "schema_version": 1, + "scope": "synthetic_engineering", + "from_year": year - 1, + "year": year, + "application_status": None, + "completed_year": None, + "accounting_status": "failed", + "account": None, + "diagnostic": None, + "transition_node": f"apply_{year}", + "snapshot_node": f"snapshot_{year}", + } + try: + outcome = trajectory._decode_transition( + context.artifacts["transition"].payload, year + ) + if outcome["completed_year"] < context.params["boundary_year"]: + raise ValueError("transition predates the model boundary") + # JSON exponent overflow can produce infinity despite parse_constant. + # Refuse it before copying arbitrary diagnostic fields into a report. + json_bytes(outcome["diagnostic"]) + report.update( + application_status=outcome["status"], + completed_year=outcome["completed_year"], + ) + if outcome["status"] != "complete": + # Do not reconcile a stale snapshot or parse it after failure. + report.update( + accounting_status="not_evaluated", + diagnostic=outcome["diagnostic"], + ) + else: + _validate_raw_snapshot(context.artifacts["snapshot"].payload) + frozen = trajectory._evaluation_context(context) + _, outcome = trajectory._transition(frozen) + opening, closing = _period_frames(frozen) + exits = tuple( + accounting.PopulationEvent( + person_id=row["person_id"], + kind=accounting.PopulationEventKind.DEATH, + year=year, + source=f"apply_{year}:mortality-transition@1", + ) + for row in outcome["records"] + if not row["survives"] + ) + account = accounting.reconcile_period( + opening, + closing, + opening_year=year - 1, + closing_year=year, + exits=exits, + ) + report.update( + accounting_status="complete", account=account.to_dict() + ) + except Exception as error: + # Keep an accounting refusal inspectable without changing mortality + # gates or stopping another year's already declared transition. + report["diagnostic"] = { + "exception_type": type(error).__name__, + "message": str(error), + } + if isinstance(error, accounting.PopulationReconciliationError): + report["diagnostic"]["reconciliation"] = error.to_dict() + return KernelResult( + artifacts={"account": json_bytes(report)}, + receipt={ + "accounting_status": report["accounting_status"], + "application_status": report["application_status"], + "completed_year": report["completed_year"], + }, + ) + + +class _AccountingKernel(trajectory.rt._Kernel): + def implementation_hash(self): + return source_hash( + self.function, + accounting, + trajectory, + trajectory.rt, + trajectory.rt.model_module, + dependencies=self.capabilities.dependencies, + ) + + +def build_accounted_trajectory_graph(**kwargs): + """Append isolated accounting nodes to the unchanged trajectory graph.""" + require_graph() + graph, registry = trajectory.build_trajectory_graph(**kwargs) + registry.register(_AccountingKernel(ACCOUNT_KERNEL, _account)) + boundary = kwargs.get("boundary_year", 2014) + nodes = tuple( + Node( + f"account_{year}", + ACCOUNT_KERNEL, + # EXPAND consumes ordinary members of its base. This separate + # population keeps accounting outside all later transition keys. + population="training", + params={"year": year, "boundary_year": boundary}, + artifact_inputs=( + ArtifactInput( + "transition", + f"apply_{year}", + "transition", + trajectory.TRANSITION_TYPE, + ), + ArtifactInput( + "snapshot", + f"snapshot_{year}", + "snapshot", + trajectory.SNAPSHOT_TYPE, + ), + ), + artifact_outputs=(ArtifactOutput("account", ACCOUNT_TYPE),), + ) + for year in range(boundary + 1, kwargs["end_year"] + 1) + ) + return replace(graph, nodes=(*graph.nodes, *nodes)), registry + + +@dataclass(frozen=True) +class AccountedTrajectoryRun: + """Accounting summaries and the actual graph's unmodified receipts.""" + + manifest: object + report: dict + + +def run_accounted_mortality_trajectory( + *, training, rates, initial, holdouts, output_dir, **kwargs +): + """Run once and export accounting separately from mortality evaluations. + + Graph coordinates are those of ``build_trajectory_graph``. Inputs retain + its exact synthetic schemas; arbitrary household/location columns are + unsupported and refused by the original source reader. + """ + graph, registry = build_accounted_trajectory_graph(**kwargs) + boundary = kwargs.get("boundary_year", 2014) + years = range(boundary + 1, kwargs["end_year"] + 1) + if ( + not isinstance(holdouts, dict) + or any(type(year) is not int for year in holdouts) + or set(holdouts) != set(years) + ): + raise ValueError("holdouts must supply exactly one source per year") + sources = { + "training": Path(training).resolve(), + "rates": Path(rates).resolve(), + "initial": Path(initial).resolve(), + **{ + f"holdout_{year}": Path(holdouts[year]).resolve() for year in years + }, + } + output = Path(output_dir).resolve() + output.mkdir(parents=True, exist_ok=True) + store = ContentStore(output / "store") + manifest = run_graph( + compile_graph(graph), sources=sources, store=store, kernels=registry + ) + periods = { + str(year): parse_json( + store.load_bytes( + manifest.nodes[f"account_{year}"].opaque_artifacts["account"] + ) + ) + for year in years + } + statuses = {period["accounting_status"] for period in periods.values()} + report = { + "scope": "synthetic_engineering", + "boundary_year": boundary, + "end_year": kwargs["end_year"], + "accounting_status": ( + "failed" + if "failed" in statuses + else ( + "not_evaluated" if "not_evaluated" in statuses else "complete" + ) + ), + "periods": periods, + } + (output / "manifest.json").write_text(manifest.to_json(), encoding="utf-8") + (output / "accounting-report.json").write_bytes(json_bytes(report)) + return AccountedTrajectoryRun(manifest, report) diff --git a/src/populace_dynamics/mortality_observer.py b/src/populace_dynamics/mortality_observer.py new file mode 100644 index 00000000..2bbb6e89 --- /dev/null +++ b/src/populace_dynamics/mortality_observer.py @@ -0,0 +1,422 @@ +"""Opt-in, exact-identity observations of the existing mortality step. + +Records describe simulated removal at a projection step, not death dates or +source admission. Registered engine and assembly behavior remain unchanged. +""" + +from __future__ import annotations + +import json +import math +import re +from dataclasses import dataclass, replace +from hashlib import sha256 +from numbers import Integral +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from populace_dynamics.engine.loop import PeriodContext +from populace_dynamics.engine.rng import ProjectionRNGRegistry +from populace_dynamics.engine.steps import ( + AgeSexMortalityModel, + apply_mortality, +) +from populace_dynamics.person_identity import PersonIdentityMap + +_SCHEMA = "populace_dynamics.mortality_step.v1" + + +def _integer(value: object, minimum: int = 0) -> int: + if isinstance(value, bool) or not isinstance(value, Integral): + raise ValueError("integer required without coercion") + value = int(value) + if not minimum <= value <= 2**63 - 1: + raise ValueError("integer outside supported range") + return value + + +def _parse_integer(value: object) -> int: + if type(value) is not str: + raise ValueError("canonical integer string required") + result = _integer(int(value)) + if str(result) != value: + raise ValueError("noncanonical integer") + return result + + +def _optional_integer(value: object) -> int | None: + return None if value is None else _parse_integer(value) + + +def _text(value: object) -> str: + if type(value) is not str or not value: + raise ValueError("nonempty string required") + value.encode("utf-8") + return value + + +def _digest(value: object) -> str: + if not re.fullmatch("[0-9a-f]{64}", _text(value)): + raise ValueError("lowercase SHA-256 digest required") + return value + + +def _json(value: object) -> str: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + + +def _hash(value: str) -> str: + return sha256(value.encode("utf-8")).hexdigest() + + +@dataclass(frozen=True) +class MortalityModelSnapshot: + """Ordered (lower, upper, female hex, male hex) effective model cells.""" + + cells: tuple[tuple[int, int, str, str], ...] + + def __post_init__(self) -> None: + cells = [] + for lower, upper, female, male in self.cells: + for value in (female, male): + if type(value) is not str: + raise ValueError("probability must be canonical float hex") + try: + number = float.fromhex(value) + except (ValueError, OverflowError) as exc: + raise ValueError("invalid canonical probability") from exc + if ( + not math.isfinite(number) + or not 0 <= number <= 1 + or number.hex() != value + ): + raise ValueError("invalid canonical probability") + cells.append((_integer(lower), _integer(upper), female, male)) + object.__setattr__(self, "cells", tuple(cells)) + self.to_model() # Validate the existing model's structural contract. + + @classmethod + def from_model(cls, model: AgeSexMortalityModel) -> MortalityModelSnapshot: + if type(model) is not AgeSexMortalityModel: + raise ValueError("the existing AgeSexMortalityModel is required") + # Validate exact keys too, including mappings mutated after creation. + detached = AgeSexMortalityModel( + tuple(model.bands), dict(model.probability) + ) + cells = [] + for lower, upper in detached.bands: + label = detached.band_label(lower, upper) + values = tuple( + float(detached.probability[(label, sex)]).hex() + for sex in ("female", "male") + ) + cells.append((lower, upper, *values)) + return cls(tuple(cells)) + + def to_model(self) -> AgeSexMortalityModel: + """Construct the unchanged evaluator with a detached read-only map.""" + bands, probabilities = [], {} + for lower, upper, female, male in self.cells: + bands.append((lower, upper)) + label = AgeSexMortalityModel.band_label(lower, upper) + probabilities[(label, "female")] = float.fromhex(female) + probabilities[(label, "male")] = float.fromhex(male) + return AgeSexMortalityModel( + tuple(bands), MappingProxyType(probabilities) + ) + + def document(self) -> list[list[str]]: + return [ + [str(lower), str(upper), female, male] + for lower, upper, female, male in self.cells + ] + + @property + def digest(self) -> str: + return _hash(_json(self.document())) + + +@dataclass(frozen=True) +class MortalityObservation: + """Pre-aging inputs and the actual simulated survival outcome.""" + + dynamics_person_key: int + age: int + sex: str + survived: bool + person_ordinal: int | None = None + + def __post_init__(self) -> None: + for name in ("dynamics_person_key", "age"): + object.__setattr__(self, name, _integer(getattr(self, name))) + if type(self.sex) is not str or self.sex not in ("female", "male"): + raise ValueError("unsupported mortality sex") + if type(self.survived) is not bool: + raise ValueError("survival outcome must be boolean") + if self.person_ordinal is not None: + object.__setattr__( + self, "person_ordinal", _integer(self.person_ordinal) + ) + + +@dataclass(frozen=True) +class MortalityStepObservation: + """An immutable step; no death-date, exposure or longitudinal inference.""" + + identity_map: PersonIdentityMap + realization_id: str + source_contract_digest: str + model: MortalityModelSnapshot + target_year: int + period_index: int + draw_index: int + registry_n_periods: int | None + rows: tuple[MortalityObservation, ...] + + def __init_subclass__(cls, **kwargs) -> None: + raise TypeError("MortalityStepObservation cannot be subclassed") + + def __post_init__(self) -> None: + if type(self.identity_map) is not PersonIdentityMap: + raise ValueError("explicit identity map required") + _text(self.realization_id) + _digest(self.source_contract_digest) + if type(self.model) is not MortalityModelSnapshot: + raise ValueError("explicit effective model snapshot required") + for name in ("target_year", "period_index", "draw_index"): + minimum = 0 if name == "draw_index" else 1 + object.__setattr__( + self, name, _integer(getattr(self, name), minimum) + ) + if self.registry_n_periods is not None: + object.__setattr__( + self, + "registry_n_periods", + _integer(self.registry_n_periods, 1), + ) + if self.period_index > self.registry_n_periods: + raise ValueError("period outside registry bounds") + rows = tuple(self.rows) + if any(type(row) is not MortalityObservation for row in rows): + raise ValueError("explicit observation rows required") + keys = tuple(row.dynamics_person_key for row in rows) + if keys != tuple(sorted(set(keys))): + raise ValueError("rows must have sorted unique private keys") + self.identity_map.reverse_rows(keys) + ordinals = [] + for row in rows: + if row.age > self.model.cells[-1][1]: + raise ValueError("age outside configured mortality bands") + if (row.person_ordinal is None) != ( + self.registry_n_periods is None + ): + raise ValueError("ordinal and RNG mode mismatch") + if row.person_ordinal is not None: + ordinals.append(row.person_ordinal) + if len(set(ordinals)) != len(ordinals): + raise ValueError("person ordinals must be unique") + object.__setattr__(self, "rows", rows) + + @property + def pre_keys(self) -> tuple[int, ...]: + return tuple(row.dynamics_person_key for row in self.rows) + + @property + def post_keys(self) -> tuple[int, ...]: + return tuple( + row.dynamics_person_key for row in self.rows if row.survived + ) + + def to_json(self) -> str: + """Canonical JSON binds an external exact identity map by digest.""" + return _json( + { + "schema": _SCHEMA, + "identity_map_digest": self.identity_map.digest, + "realization_id": self.realization_id, + "source_contract_digest": self.source_contract_digest, + "model": self.model.document(), + "model_digest": self.model.digest, + "target_year": str(self.target_year), + "period_index": str(self.period_index), + "draw_index": str(self.draw_index), + "registry_n_periods": ( + None + if self.registry_n_periods is None + else str(self.registry_n_periods) + ), + "rows": [ + { + "dynamics_person_key": str(row.dynamics_person_key), + "age": str(row.age), + "sex": row.sex, + "survived": row.survived, + "person_ordinal": ( + None + if row.person_ordinal is None + else str(row.person_ordinal) + ), + } + for row in self.rows + ], + } + ) + + @property + def digest(self) -> str: + return _hash(self.to_json()) + + @classmethod + def from_json( + cls, + text: str, + *, + identity_map: PersonIdentityMap, + expected_digest: str | None = None, + ) -> MortalityStepObservation: + """Load against the caller's trusted map and optional record digest.""" + + def unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON field") + result[key] = value + return result + + def reject(value): + raise ValueError(f"invalid JSON constant: {value}") + + document = json.loads( + text, object_pairs_hook=unique, parse_constant=reject + ) + if type(identity_map) is not PersonIdentityMap: + raise ValueError("explicit identity map required") + if ( + not isinstance(document, dict) + or document.get("identity_map_digest") != identity_map.digest + ): + raise ValueError("identity map digest mismatch") + try: + model = MortalityModelSnapshot( + tuple( + (_parse_integer(lo), _parse_integer(hi), female, male) + for lo, hi, female, male in document["model"] + ) + ) + result = cls( + identity_map, + document["realization_id"], + document["source_contract_digest"], + model, + *( + _parse_integer(document[key]) + for key in ("target_year", "period_index", "draw_index") + ), + _optional_integer(document["registry_n_periods"]), + tuple( + MortalityObservation( + _parse_integer(row["dynamics_person_key"]), + _parse_integer(row["age"]), + row["sex"], + row["survived"], + _optional_integer(row["person_ordinal"]), + ) + for row in document["rows"] + ), + ) + except (KeyError, TypeError) as exc: + raise ValueError("invalid mortality record structure") from exc + # Also rejects unknown fields, schema/model-digest changes and shapes. + if result.to_json() != _json(document): + raise ValueError("noncanonical or inconsistent mortality record") + if expected_digest is not None and result.digest != _digest( + expected_digest + ): + raise ValueError("mortality record digest mismatch") + return result + + +def observe_mortality( + frame: pd.DataFrame, + context: PeriodContext, + rng: np.random.Generator, + *, + model: AgeSexMortalityModel, + identity_map: PersonIdentityMap, + realization_id: str, + source_contract_digest: str, +) -> tuple[pd.DataFrame, MortalityStepObservation]: + """Validate before RNG use, invoke actual mortality once and retain IDs.""" + if type(context) is not PeriodContext or not isinstance( + rng, np.random.Generator + ): + raise ValueError( + "existing period context and NumPy generator required" + ) + if not frame.columns.is_unique or not { + "person_id", + "age", + "year", + "sex", + }.issubset(frame.columns): + raise ValueError("unique mortality input columns required") + for name in ("person_id", "age", "year"): + if frame[name].dtype != np.dtype("int64"): + raise ValueError(f"{name} must preserve int64 dtype") + if not (frame["year"] == _integer(context.year, 1) - 1).all(): + raise ValueError("frame year must immediately precede target year") + snapshot = MortalityModelSnapshot.from_model(model) + ordinals, periods = {}, None + registry = context.rng_registry + if registry is not None: + if type(registry) is not ProjectionRNGRegistry: + raise ValueError("existing RNG registry required") + periods = _integer(registry.n_periods, 1) + if _integer(registry.draw_index) != _integer(context.draw_index): + raise ValueError("context and registry draw mismatch") + ordinals = { + _integer(key): _integer(value) + for key, value in context.person_ordinals.items() + } + if len(set(ordinals.values())) != len(ordinals): + raise ValueError("person ordinals must be unique") + if not set(frame["person_id"]).issubset(ordinals): + raise ValueError("missing person RNG ordinal") + ordered = frame.sort_values("person_id", kind="stable") + rows = tuple( + MortalityObservation(key, age, sex, True, ordinals.get(key)) + for key, age, sex in zip( + ordered["person_id"], ordered["age"], ordered["sex"], strict=True + ) + ) + # Constructing the provisional record completes validation before drawing. + record = MortalityStepObservation( + identity_map, + realization_id, + source_contract_digest, + snapshot, + context.year, + context.period_index, + context.draw_index, + periods, + rows, + ) + survivors = apply_mortality(frame, context, rng, model=snapshot.to_model()) + survivor_keys = set(survivors["person_id"]) + record = replace( + record, + rows=tuple( + replace(row, survived=row.dynamics_person_key in survivor_keys) + for row in rows + ), + ) + return survivors, record diff --git a/src/populace_dynamics/person_identity.py b/src/populace_dynamics/person_identity.py new file mode 100644 index 00000000..cfc43be0 --- /dev/null +++ b/src/populace_dynamics/person_identity.py @@ -0,0 +1,286 @@ +"""Lossless, opt-in identities for integer-keyed Dynamics consumers. + +This module transports identities. It does not admit a population, infer an +identity from household membership, or change an earnings generator's inputs. +""" + +import json +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from hashlib import sha256 +from numbers import Integral +from types import MappingProxyType + +_INT64_MIN = -(2**63) +_INT64_MAX = 2**63 - 1 +_UINT64_MAX = 2**64 - 1 +_SCHEMA = "populace_dynamics.person_identity_map.v1" + + +def _integer(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, Integral): + raise ValueError(f"{label} must be an integer, not a coerced value") + return int(value) + + +def _canonical_integer(value: object, label: str) -> int: + if not isinstance(value, str): + raise ValueError(f"{label} must be a canonical decimal string") + try: + result = int(value) + except ValueError as exc: + raise ValueError( + f"{label} must be a canonical decimal string" + ) from exc + if str(result) != value: + raise ValueError(f"{label} must be a canonical decimal string") + return result + + +@dataclass(frozen=True) +class PersonIdentity: + """An explicitly typed source identity, with no numeric/string coercion.""" + + logical_type: str + value: int | str + + def __post_init__(self) -> None: + if not isinstance(self.logical_type, str): + raise ValueError("unsupported person identity logical type") + object.__setattr__( + self, "logical_type", str.__str__(self.logical_type) + ) + if self.logical_type not in ("int64", "uint64", "string"): + raise ValueError("unsupported person identity logical type") + if self.logical_type == "string": + if not isinstance(self.value, str): + raise ValueError("string identity requires a string value") + # Copy the underlying string, bypassing mutable subclass methods. + value = str.__str__(self.value) + try: + value.encode("utf-8") + except UnicodeEncodeError as exc: + raise ValueError( + "string identity requires valid UTF-8" + ) from exc + object.__setattr__(self, "value", value) + return + value = _integer(self.value, "identity") + lower, upper = ( + (_INT64_MIN, _INT64_MAX) + if self.logical_type == "int64" + else (0, _UINT64_MAX) + ) + if not lower <= value <= upper: + raise ValueError(f"identity is outside {self.logical_type} range") + object.__setattr__(self, "value", value) + + @property + def canonical_value(self) -> str: + return str(self.value) + + +@dataclass(frozen=True) +class IdentityEntry: + identity: PersonIdentity + dynamics_person_key: int + + def __post_init__(self) -> None: + if type(self.identity) is not PersonIdentity: + raise ValueError("entry requires an explicit PersonIdentity") + key = _integer(self.dynamics_person_key, "dynamics person key") + if not 0 <= key <= _INT64_MAX: + raise ValueError( + "dynamics person key is outside nonnegative int64" + ) + object.__setattr__(self, "dynamics_person_key", key) + + +@dataclass(frozen=True) +class PersonIdentityMap: + """Immutable dense private keys; extensions preserve every prior entry. + + New identities are sorted by (logical type, canonical value) within each + admission batch. Sorting is lexical, not numeric. Batch order is therefore + irrelevant, while the sequence of admission batches is part of identity. + """ + + entries: tuple[IdentityEntry, ...] = () + _forward: Mapping[PersonIdentity, int] = field( + init=False, repr=False, compare=False, hash=False + ) + + def __init_subclass__(cls, **kwargs) -> None: + raise TypeError("PersonIdentityMap cannot be subclassed") + + def __post_init__(self) -> None: + entries = tuple(self.entries) + forward = {} + for position, entry in enumerate(entries): + if type(entry) is not IdentityEntry: + raise ValueError("map entries must be IdentityEntry values") + if entry.dynamics_person_key != position: + raise ValueError( + "private keys must be unique and dense from 0" + ) + if entry.identity in forward: + raise ValueError("duplicate person identity") + forward[entry.identity] = entry.dynamics_person_key + object.__setattr__(self, "entries", entries) + object.__setattr__(self, "_forward", MappingProxyType(forward)) + + def __reduce__(self): + # The read-only lookup proxy cannot be pickled. Entries are the only + # state, so pickle and deepcopy rebuild through the validating + # constructor, which derives a fresh read-only lookup. + return (type(self), (self.entries,)) + + @classmethod + def from_identities( + cls, identities: Iterable[PersonIdentity] + ) -> "PersonIdentityMap": + return cls().append(identities) + + def append( + self, identities: Iterable[PersonIdentity] + ) -> "PersonIdentityMap": + """Admit only new identities; duplicate or previously admitted IDs fail.""" + pending = list(identities) + seen: set[PersonIdentity] = set() + for identity in pending: + if type(identity) is not PersonIdentity: + raise ValueError("admission requires explicit PersonIdentity") + if identity in seen or identity in self._forward: + raise ValueError( + "duplicate or already admitted person identity" + ) + seen.add(identity) + if len(pending) > _INT64_MAX + 1 - len(self.entries): + raise ValueError("private int64 identity space exhausted") + if not pending: + return self + pending.sort( + key=lambda item: (item.logical_type, item.canonical_value) + ) + start = len(self.entries) + return type(self)( + self.entries + + tuple( + IdentityEntry(identity, start + offset) + for offset, identity in enumerate(pending) + ) + ) + + def map_rows( + self, identities: Iterable[PersonIdentity] + ) -> tuple[int, ...]: + """Map in caller row order; repeated observations of an ID are valid.""" + result = [] + for identity in identities: + if type(identity) is not PersonIdentity: + raise ValueError("mapping requires explicit PersonIdentity") + try: + result.append(self._forward[identity]) + except KeyError as exc: + raise ValueError( + "person identity has not been admitted" + ) from exc + return tuple(result) + + def reverse_rows(self, keys: Iterable[int]) -> tuple[PersonIdentity, ...]: + """Reverse private keys without changing order, type or source value.""" + result = [] + for value in keys: + key = _integer(value, "dynamics person key") + if not 0 <= key < len(self.entries): + raise ValueError("dynamics person key has not been admitted") + result.append(self.entries[key].identity) + return tuple(result) + + def require_extension_of(self, previous: "PersonIdentityMap") -> None: + """Reject a loaded manifest that deletes, reorders or remaps old IDs.""" + if type(previous) is not PersonIdentityMap: + raise ValueError("previous manifest must be a PersonIdentityMap") + if self.entries[: len(previous.entries)] != previous.entries: + raise ValueError( + "identity map does not preserve the prior manifest" + ) + + def to_json(self) -> str: + """Canonical UTF-8 JSON, with integer values encoded as decimal strings.""" + document = { + "schema": _SCHEMA, + "entries": [ + { + "logical_type": entry.identity.logical_type, + "value": entry.identity.canonical_value, + "dynamics_person_key": str(entry.dynamics_person_key), + } + for entry in self.entries + ], + } + return json.dumps( + document, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ) + + @property + def digest(self) -> str: + """SHA-256 of canonical JSON; this is integrity, not admission authority.""" + return sha256(self.to_json().encode("utf-8")).hexdigest() + + @classmethod + def from_json( + cls, + text: str, + *, + expected_digest: str | None = None, + previous: "PersonIdentityMap | None" = None, + ) -> "PersonIdentityMap": + def unique_object(pairs: list[tuple[str, object]]) -> dict: + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON field") + result[key] = value + return result + + def reject_constant(value: str) -> None: + raise ValueError(f"non-JSON numeric constant: {value}") + + document = json.loads( + text, + object_pairs_hook=unique_object, + parse_constant=reject_constant, + ) + if not isinstance(document, dict) or set(document) != { + "schema", + "entries", + }: + raise ValueError("invalid identity map document fields") + if document["schema"] != _SCHEMA: + raise ValueError("unsupported identity map schema") + if not isinstance(document["entries"], list): + raise ValueError("identity map entries must be an array") + entries = [] + for row in document["entries"]: + if not isinstance(row, dict) or set(row) != { + "logical_type", + "value", + "dynamics_person_key", + }: + raise ValueError("invalid identity entry fields") + value = row["value"] + if row["logical_type"] != "string": + value = _canonical_integer(value, "identity") + entries.append( + IdentityEntry( + PersonIdentity(row["logical_type"], value), + _canonical_integer(row["dynamics_person_key"], "key"), + ) + ) + result = cls(tuple(entries)) + if expected_digest is not None and result.digest != expected_digest: + raise ValueError("identity map digest mismatch") + if previous is not None: + result.require_extension_of(previous) + return result diff --git a/tests/README-tiers.md b/tests/README-tiers.md index a5ef9187..84c88dd1 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -38,9 +38,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 1,563 | -| `artifact` | 2,668 | +| `unit` | 2,298 | +| `artifact` | 2,672 | | `integration_psid` | 848 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 159 | -| **Total** | **5,758** | +| **Total** | **6,497** | diff --git a/tests/estimates/test_birth_evidence_artifact.py b/tests/estimates/test_birth_evidence_artifact.py index d4e838a1..e31d58bf 100644 --- a/tests/estimates/test_birth_evidence_artifact.py +++ b/tests/estimates/test_birth_evidence_artifact.py @@ -88,6 +88,26 @@ def test_post_review_sources_are_outside_historical_reducer_identity(): Path("src/populace_dynamics/estimates/anchor_context_registry.py"), Path("src/populace_dynamics/estimates/anchor_context_rehearsal.py"), Path("src/populace_dynamics/estimates/anchor_context_report.py"), + Path("src/populace_dynamics/graph/__init__.py"), + Path("src/populace_dynamics/graph/__main__.py"), + Path("src/populace_dynamics/graph/_compat.py"), + Path("src/populace_dynamics/graph/model.py"), + Path("src/populace_dynamics/graph/runtime.py"), + Path("src/populace_dynamics/graph/synthetic.py"), + Path("src/populace_dynamics/graph/trajectory.py"), + Path("src/populace_dynamics/graph/trajectory_accounting.py"), + Path("src/populace_dynamics/engine/accounting.py"), + Path("src/populace_dynamics/engine/claiming.py"), + Path("src/populace_dynamics/engine/entrant_schedule.py"), + Path("src/populace_dynamics/engine/entrant_domains.py"), + Path("src/populace_dynamics/engine/accounting_history.py"), + Path("src/populace_dynamics/person_identity.py"), + Path("src/populace_dynamics/forward_earnings_history.py"), + Path("src/populace_dynamics/covered_wage_history.py"), + Path("src/populace_dynamics/mortality_observer.py"), + Path("src/populace_dynamics/closed_cohort_history.py"), + Path("src/populace_dynamics/assembled_history_observer.py"), + Path("src/populace_dynamics/compact_cohort_history.py"), ) assert reducer.POST_REVIEW_SHARED_SOURCE_BLOBS == { Path( @@ -130,11 +150,7 @@ def _internal_imports( package_parts = module_parts if is_package else module_parts[:-1] for node in ast.walk(tree): if isinstance(node, ast.Import): - imports.update( - alias.name - for alias in node.names - if alias.name in module_paths - ) + imports.update(alias.name for alias in node.names) continue if not isinstance(node, ast.ImportFrom): continue @@ -148,16 +164,64 @@ def _internal_imports( base = ".".join(base_parts) else: base = node.module or "" - if base in module_paths: - imports.add(base) + imports.add(base) for alias in node.names: candidate = f"{base}.{alias.name}" if base else alias.name - if candidate in module_paths: - imports.add(candidate) - return imports + imports.add(candidate) + # Importing a leaf executes its parent package initializers too. Include + # those even when an imported leaf is external or not a tracked module. + internal = set() + for imported in imports: + parts = imported.split(".") + internal.update( + parent + for length in range(1, len(parts) + 1) + if (parent := ".".join(parts[:length])) in module_paths + ) + return internal + + +@pytest.mark.parametrize( + "statement", + [ + "import sample.leaf", + "from sample.leaf import function", + "from sample import leaf", + "import sample.untracked_extension", + ], +) +def test_source_reachability_includes_implicit_package_initializers( + tmp_path, statement +): + package = tmp_path / "__init__.py" + leaf = tmp_path / "leaf.py" + hidden = tmp_path / "hidden.py" + consumer = tmp_path / "consumer.py" + package.write_text("from . import hidden\n") + leaf.write_text("def function(): pass\n") + hidden.write_text("") + consumer.write_text(statement + "\n") + modules = { + "consumer": consumer, + "sample": package, + "sample.leaf": leaf, + "sample.hidden": hidden, + } + reachable = set() + pending = ["consumer"] + while pending: + name = pending.pop() + if name in reachable: + continue + reachable.add(name) + pending.extend( + _internal_imports(name, modules[name], modules) - reachable + ) + assert {"sample", "sample.hidden"}.issubset(reachable) + assert "sample.untracked_extension" not in reachable -def test_psid_identity_exclusions_are_unreachable_from_birth_evidence(): +def test_post_review_exclusions_are_unreachable_from_birth_evidence(): module_paths = _repository_module_paths() root_module = "scripts.first_estimates_birth_evidence" psid_exclusions = { @@ -170,6 +234,14 @@ def test_psid_identity_exclusions_are_unreachable_from_birth_evidence(): } assert root_module in module_paths assert psid_exclusions.issubset(module_paths) + graph_exclusions = { + name + for name in module_paths + if name == "populace_dynamics.graph" + or name.startswith("populace_dynamics.graph.") + } + assert graph_exclusions + assert "populace_dynamics.graph.trajectory_accounting" in graph_exclusions module_by_path = { path.resolve(): module_name for module_name, path in module_paths.items() @@ -205,6 +277,39 @@ def test_psid_identity_exclusions_are_unreachable_from_birth_evidence(): "historically excluded PSID modules became reachable from the " f"birth-evidence reducer: {sorted(psid_exclusions & reachable)}" ) + assert graph_exclusions.isdisjoint(reachable), ( + "opt-in graph modules became reachable from the birth-evidence " + f"reducer: {sorted(graph_exclusions & reachable)}" + ) + assert "populace_dynamics.engine.accounting" in module_paths + assert "populace_dynamics.engine.accounting" not in reachable + assert "populace_dynamics.engine.accounting_history" in module_paths + assert "populace_dynamics.engine.accounting_history" not in reachable + assert "populace_dynamics.engine" in reachable + successor = "populace_dynamics.engine.claiming" + assert successor in module_paths + assert successor not in reachable + entrant_modules = { + "populace_dynamics.engine.entrant_schedule", + "populace_dynamics.engine.entrant_domains", + } + assert entrant_modules.issubset(module_paths) + assert entrant_modules.isdisjoint(reachable) + history_modules = { + "populace_dynamics.person_identity", + "populace_dynamics.forward_earnings_history", + "populace_dynamics.covered_wage_history", + "populace_dynamics.mortality_observer", + "populace_dynamics.closed_cohort_history", + "populace_dynamics.assembled_history_observer", + "populace_dynamics.compact_cohort_history", + } + assert history_modules.issubset(module_paths) + assert history_modules.isdisjoint(reachable), ( + "opt-in identity and history modules became reachable from the " + f"birth-evidence reducer: {sorted(history_modules & reachable)}" + ) + assert "populace_dynamics.engine.steps" in reachable def test_reducer_accepts_explicit_unresolved_upstream_boundary(): diff --git a/tests/test_assembled_history_observer.py b/tests/test_assembled_history_observer.py new file mode 100644 index 00000000..dcafc205 --- /dev/null +++ b/tests/test_assembled_history_observer.py @@ -0,0 +1,920 @@ +"""Invented observations through the actual eight-step assembly callbacks.""" + +import copy +import json +import pickle +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal + +from populace_dynamics.assembled_history_observer import ( + AssembledHistoryObservation, + capture_fertility, + observe_assembled_history, +) +from populace_dynamics.data import disability, household_composition +from populace_dynamics.engine import assembly, steps +from populace_dynamics.engine.assembly import ( + M6_DRAW_OUTPUTS_KEY, + CertifiedEngineInputs, + assemble_period_modules, +) +from populace_dynamics.engine.loop import ( + SCHEDULED_ENTRIES_KEY, + MaritalStepResult, + ProjectionEngine, + SyntheticPersonIdAllocator, +) +from populace_dynamics.engine.steps import ( + AgeSexMortalityModel, + ClaimingSchedule, + FertilityDraws, +) +from populace_dynamics.engine.support import StartWaveWeightSnapshot +from populace_dynamics.mortality_observer import MortalityModelSnapshot +from populace_dynamics.person_identity import PersonIdentity, PersonIdentityMap +from tests.test_m6_engine_correlated_refresh import _correlated +from tests.test_m6_engine_forward_earnings import _generator + + +def assembled( + monkeypatch, + *, + birth=False, + entries=False, + correlated=False, + parent_id=1, + extinction=False, +): + """Reuse the assembly test's bounded fitted-core seams, actual callbacks.""" + empty = pd.DataFrame( + { + "parent_person_id": pd.Series(dtype="int64"), + "birth_year": pd.Series(dtype="Int64"), + } + ) + births = ( + pd.DataFrame({"parent_person_id": [parent_id], "birth_year": [2015]}) + if birth + else empty + ) + calls = [] + family = SimpleNamespace(name="invented family") + household = SimpleNamespace(family_transitions=family, male_gap=-2.0) + + def marital(*args, **kwargs): + calls.append("marital") + return MaritalStepResult( + pd.DataFrame( + { + "person_id": pd.Series(dtype="int64"), + "year": pd.Series(dtype="int64"), + } + ), + empty, + panel=object(), + ) + + def fertility(*args, **kwargs): + return FertilityDraws(births.copy(), empty.copy()) + + panel = disability.DisabilityPanel( + pd.DataFrame( + { + "person_id": pd.Series(dtype="int64"), + "period": pd.Series(dtype="int64"), + "sex": pd.Series(dtype="str"), + "age": pd.Series(dtype="int64"), + "weight": pd.Series(dtype="float64"), + **{ + c: pd.Series(dtype="bool") + for c in ("disabled", "retired", "di_converted") + }, + "status_code": pd.Series(dtype="int64"), + } + ), + pd.DataFrame(), + ) + + def simulate_disability(*args, **kwargs): + calls.append("disability") + return panel + + def composition(*args, **kwargs): + calls.append("household") + rows = pd.DataFrame( + { + "person_id": pd.Series(dtype="int64"), + "year": pd.Series(dtype="int64"), + **{ + c: pd.Series(dtype="float64") + for c in ( + "coresident_spouse", + "coresident_parent", + "coresident_child", + "coresident_grandchild", + "multigen", + "hh_size", + ) + }, + } + ) + return ( + household_composition.HouseholdCompositionPanel( + rows, pd.DataFrame() + ), + {}, + ) + + monkeypatch.setattr(assembly, "simulate_marital_step", marital) + monkeypatch.setattr(steps, "simulate_fertility", fertility) + monkeypatch.setattr(assembly, "simulate_fertility", fertility) + monkeypatch.setattr(assembly, "simulate_reproduction", simulate_disability) + monkeypatch.setattr(assembly, "simulate_candidate9_injected", composition) + model = AgeSexMortalityModel( + ((0, 120),), + {("0+", "female"): (1.0 if extinction else 0.0), ("0+", "male"): 1.0}, + ) + if extinction: + model = AgeSexMortalityModel( + ((0, 29), (30, 120)), + { + ("0-29", "female"): 0.0, + ("0-29", "male"): 0.0, + ("30+", "female"): 1.0, + ("30+", "male"): 1.0, + }, + ) + generator = _correlated(rho=-0.5) if correlated else _generator() + if parent_id != 1: + generator = replace( + generator, + **{ + name: { + parent_id if key == 1 else key: value + for key, value in getattr(generator, name).items() + } + for name in ( + "u_w_by_person", + "realized_earn_2014_by_person", + "realized_earn_2012_by_person", + ) + }, + ) + initial = pd.DataFrame( + { + "person_id": np.array([20, parent_id, 10], dtype="int64"), + "year": np.full(3, 2014, dtype="int64"), + "age": np.array([35, 30, 31], dtype="int64"), + "sex": ["male", "female", "female"], + "weight": np.ones(3), + } + ) + scheduled = ( + { + 2016: pd.DataFrame( + { + "person_id": np.array([2], dtype="int64"), + "year": np.array([2015], dtype="int64"), + "age": np.array([40], dtype="int64"), + "sex": ["male"], + "weight": [1.0], + } + ) + } + if entries + else {} + ) + if entries and extinction: + scheduled[2016]["age"] = np.array([10], dtype="int64") + scheduled[2016]["sex"] = "female" + scheduled[2016]["earnings_domain"] = False + scheduled[2016]["earnings"] = 0.0 + inputs = CertifiedEngineInputs( + family, + object(), + object(), + household, + model, + object(), + ClaimingSchedule( + {("female", 2014): {62: 1.0}, ("male", 2014): {62: 1.0}} + ), + generator, + lambda f, c: (object(), set(f.person_id)), + lambda f, c: (object(), set(f.person_id)), + panel, + set(), + StartWaveWeightSnapshot.from_frame( + initial[["person_id", "weight"]], boundary_period=2014 + ), + -2.0, + ) + modules = assemble_period_modules(inputs) + mapping = PersonIdentityMap.from_identities( + PersonIdentity("int64", x) for x in initial.person_id + ) + kwargs = dict( + mode="original_2014_view", + identity_map=mapping, + realization_id="invented-assembly-person-side-draw3", + generator_digest="a" * 64, + earnings_source_contract_digest="b" * 64, + mortality_snapshot=MortalityModelSnapshot.from_model(model), + mortality_snapshot_after=MortalityModelSnapshot.from_model(model), + mortality_source_contract_digest="c" * 64, + unit="XTS", + price_basis="nominal", + lineage_by_year={y: "d" * 64 for y in range(2014, 2023)}, + initial_native_ids=tuple(initial.person_id), + scheduled_entries_by_year=scheduled, + reserved_real_ids=frozenset([parent_id, 2, 10, 20]), + synthetic_id_start=100, + ) + + def run(*, capture=True, draw=3): + selected, boundary = ( + capture_fertility(modules) if capture else (modules, None) + ) + collector = {} + result = ProjectionEngine(selected).project( + initial, + end_year=2022, + draw_index=draw, + metadata={ + M6_DRAW_OUTPUTS_KEY: collector, + SCHEDULED_ENTRIES_KEY: scheduled, + "synthetic_id_allocator": SyntheticPersonIdAllocator( + 100, frozenset([parent_id, 2, 10, 20]) + ), + }, + ) + return result, collector, boundary + + run.modules = modules + return run, kwargs, calls, generator + + +@pytest.mark.parametrize("correlated", [False, True]) +@pytest.mark.parametrize("entries", [False, True]) +def test_actual_assembly_observation_preserves_every_frame_and_collector( + monkeypatch, correlated, entries +): + run, kwargs, calls, generator = assembled( + monkeypatch, correlated=correlated, entries=entries + ) + from populace_dynamics.engine.rng import ProjectionRNGRegistry + + streams = [] + for name in ("generator", "person_generator", "child_generator"): + original = getattr(ProjectionRNGRegistry, name) + + def track(self, *args, _method=original, _name=name, **kw): + rng = _method(self, *args, **kw) + streams.append((_name, args, kw, rng)) + return rng + + monkeypatch.setattr(ProjectionRNGRegistry, name, track) + control, control_outputs, _ = run(capture=False) + control_states = [ + (name, args, kw, pickle.dumps(r.bit_generator.state)) + for name, args, kw, r in streams + ] + streams.clear() + result, outputs, capture = run() + assert [ + (name, args, kw, pickle.dumps(r.bit_generator.state)) + for name, args, kw, r in streams + ] == control_states + assert pickle.dumps(outputs, protocol=5) == pickle.dumps( + control_outputs, protocol=5 + ) + for name in ("generator", "person_generator", "child_generator"): + monkeypatch.setattr( + ProjectionRNGRegistry, + name, + lambda *a, **k: pytest.fail("observer created RNG stream"), + ) + before = pickle.dumps((result, outputs, generator), protocol=5) + observed = observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + assert pickle.dumps((result, outputs, generator), protocol=5) == before + for left, right in zip(control.slices, result.slices, strict=True): + assert_frame_equal(left, right, check_exact=True) + for left, right in zip( + control_outputs["mortality_slices"], + outputs["mortality_slices"], + strict=True, + ): + assert_frame_equal(left, right, check_exact=True) + assert calls == ["marital", "disability", "household"] * 2 + assert observed.history.last_year == 2022 + assert len(observed.history.histories) == 3 + assert ( + observed.history.for_person(PersonIdentity("int64", 20)).last_year + == 2014 + ) + audit = json.loads(observed.audit_json) + if entries: + assert audit["excluded"] == [ + { + "native_id": "2", + "kind": "scheduled_entry", + "entry_year": 2016, + "death_year": 2016, + "parent_id": None, + } + ] + # Native numeric ordering includes entrant 2; lexical private keys do not. + native10 = next(x for x in audit["identity_bindings"] if x[0] == "10") + assert native10[2] == "2" + text = observed.to_json() + assert ( + AssembledHistoryObservation.from_json( + text, + baseline=observed.history.baseline, + expected_digest=observed.digest, + ) + == observed + ) + + +def test_births_require_matching_fertility_boundary(monkeypatch): + run, kwargs, _, _ = assembled(monkeypatch, birth=True, entries=True) + result, outputs, capture = run() + assert all( + t.authoritative_marital_state.births.empty for t in result.traces + ) + with pytest.raises(ValueError, match="capture"): + observe_assembled_history(result, outputs, **kwargs) + observed = observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + births = [ + x + for x in json.loads(observed.audit_json)["excluded"] + if x["kind"] == "native_synthetic_birth" + ] + assert len(births) == 1 and births[0]["parent_id"] == "1" + with pytest.raises(ValueError, match="strict"): + observe_assembled_history( + result, + outputs, + fertility_capture=capture, + **{**kwargs, "mode": "strict_full_roster"}, + ) + + +def test_capture_calls_original_once_by_identity_and_refuses_reuse( + monkeypatch, +): + run, kwargs, _, _ = assembled(monkeypatch) + result, outputs, capture = run() + observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + assert len(capture.records) == 8 + # Construction never evaluates a model or creates an RNG stream. + monkeypatch.setattr( + AgeSexMortalityModel, + "probabilities", + lambda *a: pytest.fail("model call"), + ) + monkeypatch.setattr( + MortalityModelSnapshot, + "to_model", + lambda *a: pytest.fail("model copy"), + ) + observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + + +@pytest.mark.parametrize( + "change", + [ + "missing_period", + "death_flag", + "baseline_age", + "entrant_schedule", + "disappearance", + ], +) +def test_refuses_incomplete_or_contradictory_full_evidence( + monkeypatch, change +): + run, kwargs, _, _ = assembled(monkeypatch, entries=True) + result, outputs, capture = run() + outputs = copy.deepcopy(outputs) + if change == "missing_period": + outputs["mortality_slices"].pop() + elif change == "death_flag": + outputs["mortality_slices"][0].loc[0, "death"] = False + elif change == "baseline_age": + result.slices[0].loc[0, "age"] += 1 + elif change == "entrant_schedule": + kwargs["scheduled_entries_by_year"] = {} + elif change == "disappearance": + result.slices[1].drop(result.slices[1].index[0], inplace=True) + with pytest.raises(ValueError): + observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + + +def test_serialized_selection_audit_is_bound_with_history(monkeypatch): + run, kwargs, _, _ = assembled(monkeypatch, entries=True) + result, outputs, capture = run() + observed = observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + data = json.loads(observed.to_json()) + data["audit"]["excluded"] = [] + with pytest.raises(ValueError): + AssembledHistoryObservation.from_json( + json.dumps(data), + baseline=observed.history.baseline, + expected_digest=observed.digest, + ) + + +@pytest.mark.parametrize("parent_id", [2**53, 2**53 + 1, 2**53 + 2]) +def test_native_float_parent_ambiguity_refuses_even_with_capture( + monkeypatch, parent_id +): + run, kwargs, _, _ = assembled(monkeypatch, birth=True, parent_id=parent_id) + result, outputs, capture = run() + # Native concat promotes the nullable parent column to float64. + assert result.slices[1].parent_person_id.dtype == np.dtype("float64") + with pytest.raises(ValueError, match="parent"): + observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + + +def test_large_native_ids_roundtrip_without_unsafe_parent_column(monkeypatch): + native = 2**53 + 1 + run, kwargs, _, _ = assembled(monkeypatch, parent_id=native) + result, outputs, capture = run() + observed = observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + assert ( + observed.history.for_person(PersonIdentity("int64", native)).last_year + == 2022 + ) + bindings = json.loads(observed.audit_json)["identity_bindings"] + assert str(native) in [row[0] for row in bindings] + + +def test_strict_mode_and_no_capture_work_for_unchanged_roster(monkeypatch): + run, kwargs, _, _ = assembled(monkeypatch) + result, outputs, _ = run(capture=False) + observed = observe_assembled_history( + result, outputs, **{**kwargs, "mode": "strict_full_roster"} + ) + assert not json.loads(observed.audit_json)["excluded"] + + +def test_pre_post_parameter_binding_must_match(monkeypatch): + run, kwargs, _, _ = assembled(monkeypatch) + result, outputs, _ = run(capture=False) + other = MortalityModelSnapshot(((0, 120, (0.1).hex(), (1.0).hex()),)) + with pytest.raises(ValueError, match="parameter"): + observe_assembled_history( + result, outputs, **{**kwargs, "mortality_snapshot_after": other} + ) + + +def test_capture_preserves_exact_callback_arguments_result_and_rng( + monkeypatch, +): + from populace_dynamics.engine.loop import PeriodContext + from populace_dynamics.engine.rng import ProjectionRNGRegistry + + run, _, _, _ = assembled(monkeypatch) + frame = pd.DataFrame({"x": [1]}) + context = PeriodContext(1, 2015, 3, {}, ProjectionRNGRegistry(3, 1)) + marital = object() + rng, control = np.random.default_rng(4), np.random.default_rng(4) + calls = [] + output = pd.DataFrame({"x": [2]}) + + def original(f, c, m, r): + assert f is frame and c is context and m is marital and r is rng + calls.append(r.random()) + return output + + modules = replace(run.modules, fertility=original) + wrapped, capture = capture_fertility(modules) + for name in ( + "mortality", + "aging", + "marital_core", + "disability", + "earnings", + "claiming", + "household_composition", + "initialize", + ): + assert getattr(wrapped, name) is getattr(modules, name) + assert wrapped.fertility(frame, context, marital, rng) is output + assert calls == [control.random()] + assert rng.bit_generator.state == control.bit_generator.state + saved = capture.records[0].after_json + archived = json.loads(capture.to_json()) + assert archived["schema"] == "assembled-fertility-boundaries/v1" + assert archived["records"][0][-1] == saved + output.loc[0, "x"] = 7 + assert capture.records[0].after_json == saved + with pytest.raises(ValueError, match="reused"): + wrapped.fertility(frame, context, marital, rng) + assert len(calls) == 1 + with pytest.raises(ValueError, match="failed"): + capture._require_complete(3, 1) + with pytest.raises(ValueError, match="failed"): + capture.to_json() + + +def test_failed_callback_cannot_yield_accepted_capture(monkeypatch): + from populace_dynamics.engine.loop import PeriodContext + from populace_dynamics.engine.rng import ProjectionRNGRegistry + + run, _, _, _ = assembled(monkeypatch) + + def fail(*args): + raise RuntimeError("callback failure") + + wrapped, capture = capture_fertility(replace(run.modules, fertility=fail)) + context = PeriodContext(1, 2015, 3, {}, ProjectionRNGRegistry(3, 1)) + with pytest.raises(RuntimeError, match="callback failure"): + wrapped.fertility( + pd.DataFrame({"x": [1]}), + context, + object(), + np.random.default_rng(1), + ) + with pytest.raises(ValueError, match="failed"): + capture._require_complete(3, 1) + assert capture.records == () + + +@pytest.mark.parametrize( + "change", + [ + "wrong_capture", + "incomplete_capture", + "float_id", + "age_jump", + "unknown_added", + "birth_parent", + "resurrection", + ], +) +def test_refuses_roster_or_capture_tampering(monkeypatch, change): + run, kwargs, _, _ = assembled(monkeypatch, birth=True, entries=True) + result, outputs, capture = run() + if change == "wrong_capture": + _, _, capture = run(draw=4) + elif change == "incomplete_capture": + capture._records.pop() + elif change == "float_id": + result.slices[1]["person_id"] = result.slices[1].person_id.astype( + float + ) + elif change == "age_jump": + result.slices[1].loc[result.slices[1].person_id == 1, "age"] += 1 + elif change == "unknown_added": + result.slices[1].loc[ + result.slices[1].person_id == 100, "person_id" + ] = 101 + elif change == "birth_parent": + result.slices[1].loc[ + result.slices[1].person_id == 100, "parent_person_id" + ] = 10.0 + elif change == "resurrection": + # Original 20 died in 2015. A changed label cannot resurrect that key. + result.slices[2].loc[result.slices[2].person_id == 1, "person_id"] = 20 + with pytest.raises(ValueError): + observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + + +@pytest.mark.parametrize( + "mutation", ["drop_exclusion", "ordinal", "extra_field", "mode"] +) +def test_canonical_audit_revalidates_internal_consistency( + monkeypatch, mutation +): + run, kwargs, _, _ = assembled(monkeypatch, entries=True) + result, outputs, capture = run() + observed = observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + audit = json.loads(observed.audit_json) + if mutation == "drop_exclusion": + audit["excluded"] = [] + elif mutation == "ordinal": + audit["identity_bindings"][1][2] = "0" + elif mutation == "extra_field": + audit["unrecognized"] = 1 + elif mutation == "mode": + audit["mode"] = "strict_full_roster" + with pytest.raises(ValueError): + AssembledHistoryObservation( + observed.history, + json.dumps(audit, sort_keys=True, separators=(",", ":")), + ) + + +def test_full_cohort_extinction_retains_each_history_and_empty_period( + monkeypatch, +): + run, kwargs, _, _ = assembled(monkeypatch, extinction=True) + result, outputs, capture = run() + assert all(frame.empty for frame in result.slices[1:]) + observed = observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + assert observed.history.last_year == 2022 + assert observed.history.active_keys == () + assert all(h.last_year == 2014 for h in observed.history.histories) + assert len(observed.history.transitions) == 8 + + +def test_original_cohort_definition_cannot_select_only_survivors(monkeypatch): + run, kwargs, _, _ = assembled(monkeypatch) + result, outputs, capture = run() + with pytest.raises(ValueError, match="initial cohort"): + observe_assembled_history( + result, + outputs, + fertility_capture=capture, + **{**kwargs, "initial_native_ids": (1, 10)}, + ) + + +def test_changed_initial_demographics_refuse_even_first_mortality_step( + monkeypatch, +): + run, kwargs, _, _ = assembled(monkeypatch) + result, outputs, capture = run() + outputs["mortality_slices"][0].loc[0, "sex"] = "female" + with pytest.raises(ValueError, match="demographic"): + observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + + +def test_validly_rehashed_bad_selection_still_refuses(monkeypatch): + import hashlib + + run, kwargs, _, _ = assembled(monkeypatch, entries=True) + result, outputs, capture = run() + observed = observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + data = json.loads(observed.to_json()) + data["audit"]["excluded"] = [] + text = json.dumps(data, sort_keys=True, separators=(",", ":")) + with pytest.raises(ValueError, match="excluded"): + AssembledHistoryObservation.from_json( + text, + baseline=observed.history.baseline, + expected_digest=hashlib.sha256(text.encode()).hexdigest(), + ) + + +@pytest.mark.parametrize("outside_int64", [-(2**63) - 1, 2**63]) +def test_rehashed_noncohort_identity_must_remain_native_int64( + monkeypatch, outside_int64 +): + import hashlib + + run, kwargs, _, _ = assembled(monkeypatch, birth=True) + result, outputs, capture = run() + observed = observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + data = json.loads(observed.to_json()) + + def replace_birth(value): + if isinstance(value, list): + return [replace_birth(x) for x in value] + if isinstance(value, dict): + return {k: replace_birth(v) for k, v in value.items()} + return str(outside_int64) if value == "100" else value + + data["audit"] = replace_birth(data["audit"]) + text = json.dumps(data, sort_keys=True, separators=(",", ":")) + with pytest.raises(ValueError, match="signed int64"): + AssembledHistoryObservation.from_json( + text, + baseline=observed.history.baseline, + expected_digest=hashlib.sha256(text.encode()).hexdigest(), + ) + + +def test_original_cohort_extinction_does_not_hide_remaining_entrant( + monkeypatch, +): + run, kwargs, _, _ = assembled(monkeypatch, entries=True, extinction=True) + result, outputs, capture = run() + observed = observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + assert observed.history.active_keys == () + assert result.slices[-1].person_id.tolist() == [2] + assert json.loads(observed.audit_json)["excluded"][0]["death_year"] is None + assert all(h.last_year == 2014 for h in observed.history.histories) + + +def test_safe_exact_range_float_parent_is_supported(monkeypatch): + native = 2**53 - 1 + run, kwargs, _, _ = assembled(monkeypatch, birth=True, parent_id=native) + result, outputs, capture = run() + observed = observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + birth = next( + x + for x in json.loads(observed.audit_json)["excluded"] + if x["kind"] == "native_synthetic_birth" + ) + assert birth["parent_id"] == str(native) + + +@pytest.mark.parametrize( + ("field", "message"), + [ + ("period_year", "audit year"), + ("entry_year", "excluded"), + ("death_year", "excluded"), + ], +) +def test_rehashed_audit_refuses_float_year_encoding( + monkeypatch, field, message +): + import hashlib + + run, kwargs, _, _ = assembled(monkeypatch, entries=True) + result, outputs, capture = run() + observed = observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + data = json.loads(observed.to_json()) + if field == "period_year": + period = data["audit"]["periods"][1] + period["year"] = float(period["year"]) + else: + (entrant,) = data["audit"]["excluded"] + entrant[field] = float(entrant[field]) + text = json.dumps(data, sort_keys=True, separators=(",", ":")) + # 2016.0 is itself canonical JSON and equals 2016 in Python, so only an + # exact type check keeps one observation from having a second encoding. + assert ( + json.dumps(json.loads(text), sort_keys=True, separators=(",", ":")) + == text + ) + with pytest.raises(ValueError, match=message): + AssembledHistoryObservation.from_json( + text, + baseline=observed.history.baseline, + expected_digest=hashlib.sha256(text.encode()).hexdigest(), + ) + + +@pytest.mark.parametrize("row", ["binding", "birth", "scheduled"]) +def test_rehashed_audit_refuses_non_array_row_encoding(monkeypatch, row): + import hashlib + + # parent_id=2 makes the birth row ["100", "2"], already in the key order + # a canonical JSON object would sort into. Native 2 is otherwise the + # scheduled entrant, so the scheduled case keeps the default parent. + scheduled = row == "scheduled" + run, kwargs, _, _ = assembled( + monkeypatch, + birth=True, + entries=scheduled, + parent_id=1 if scheduled else 2, + ) + result, outputs, capture = run() + observed = observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + data = json.loads(observed.to_json()) + audit = data["audit"] + if row == "binding": + # With one-digit fields, ["2", "1", "0"] and "210" both unpack to + # the same three strings. + binding = audit["identity_bindings"][0] + assert [len(x) for x in binding] == [1, 1, 1] + audit["identity_bindings"][0] = "".join(binding) + elif row == "birth": + (birth,) = audit["periods"][0]["births"] + assert birth == ["100", "2"] + audit["periods"][0]["births"] = [dict.fromkeys(birth)] + else: + audit["scheduled"] = [audit["scheduled"][0][0]] + text = json.dumps(data, sort_keys=True, separators=(",", ":")) + with pytest.raises(ValueError, match="fixed-width arrays"): + AssembledHistoryObservation.from_json( + text, + baseline=observed.history.baseline, + expected_digest=hashlib.sha256(text.encode()).hexdigest(), + ) + + +@pytest.mark.parametrize( + "field", ["draw_index", "scheduled_year", "trace_year"] +) +def test_numpy_integer_inputs_observe_exactly_like_python_ints( + monkeypatch, field +): + run, kwargs, _, _ = assembled(monkeypatch, birth=True, entries=True) + result, outputs, capture = run() + expected = observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + archive = capture.to_json() + if field == "draw_index": + # The engine hands a NumPy draw index through to every callback. + result, outputs, capture = run(draw=np.int64(3)) + elif field == "scheduled_year": + kwargs["scheduled_entries_by_year"] = { + np.int64(year): frame + for year, frame in kwargs["scheduled_entries_by_year"].items() + } + elif field == "trace_year": + result = replace( + result, + traces=tuple( + replace(trace, year=np.int64(trace.year)) + for trace in result.traces + ), + ) + assert capture.to_json() == archive + observed = observe_assembled_history( + result, outputs, fertility_capture=capture, **kwargs + ) + assert observed.to_json() == expected.to_json() + assert observed.digest == expected.digest + + +def test_capture_archives_callback_coordinates_as_exact_ints(monkeypatch): + from populace_dynamics.engine.loop import PeriodContext + from populace_dynamics.engine.rng import ProjectionRNGRegistry + + run, _, _, _ = assembled(monkeypatch) + + def identity(frame, context, marital, rng): + return frame + + archives = [] + # A NumPy end_year reaches the registry as a NumPy n_periods. + for integer in (int, np.int64): + wrapped, capture = capture_fertility( + replace(run.modules, fertility=identity) + ) + context = PeriodContext( + integer(1), + integer(2015), + integer(3), + {}, + ProjectionRNGRegistry(integer(3), integer(1)), + ) + wrapped.fertility( + pd.DataFrame({"x": [1]}), + context, + object(), + np.random.default_rng(4), + ) + (record,) = capture.records + assert [ + type(x) + for x in (record.draw_index, record.period_index, record.year) + ] == [int, int, int] + capture._require_complete(3, 1) + archives.append(capture.to_json()) + assert archives[0] == archives[1] + assert json.loads(archives[0])["n_periods"] == 1 + # A non-integer horizon is refused, not archived as 1.0. + wrapped, capture = capture_fertility( + replace(run.modules, fertility=identity) + ) + context = PeriodContext(1, 2015, 3, {}, ProjectionRNGRegistry(3, 1.0)) + with pytest.raises(ValueError, match="exact integer"): + wrapped.fertility( + pd.DataFrame({"x": [1]}), + context, + object(), + np.random.default_rng(4), + ) + with pytest.raises(ValueError, match="failed"): + capture._require_complete(3, 1) diff --git a/tests/test_closed_cohort_history.py b/tests/test_closed_cohort_history.py new file mode 100644 index 00000000..7d9211cf --- /dev/null +++ b/tests/test_closed_cohort_history.py @@ -0,0 +1,598 @@ +"""Invented closed-cohort records using actual mortality and earnings steps.""" + +import json +from dataclasses import FrozenInstanceError, replace + +import numpy as np +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal + +from populace_dynamics.closed_cohort_history import ClosedCohortEarningsHistory +from populace_dynamics.covered_wage_history import ( + CoveredWageHistory, + CoveredWageObservation, + SourceAmount, +) +from populace_dynamics.engine.earnings_domain import EarningsDomainAdapter +from populace_dynamics.engine.loop import PeriodContext +from populace_dynamics.engine.rng import ProjectionRNGRegistry +from populace_dynamics.engine.steps import ( + AgeSexMortalityModel, + advance_age, + apply_earnings, + apply_mortality, +) +from populace_dynamics.forward_earnings_history import ForwardEarningsHistory +from populace_dynamics.mortality_observer import observe_mortality +from populace_dynamics.person_identity import PersonIdentity, PersonIdentityMap +from tests.test_m6_engine_correlated_refresh import _correlated +from tests.test_m6_engine_forward_earnings import _generator + + +def setup(*, death_year=2016, correlated=False): + mapping = PersonIdentityMap.from_identities( + [ + PersonIdentity("int64", -(2**63)), + PersonIdentity("string", "001"), + *(PersonIdentity("uint64", 2**64 - 1 - i) for i in range(19)), + ] + ) + generator = EarningsDomainAdapter( + _correlated(rho=-0.5) if correlated else _generator() + ) + initial = pd.DataFrame( + { + "person_id": np.array([20, 10, 1, 0], dtype="int64"), + "year": np.full(4, 2014, dtype="int64"), + "age": np.array( + [32, 35 - ((death_year or 2016) - 2015), 31, 30], dtype="int64" + ), + "sex": ["female", "male", "female", "female"], + } + ) + frame = generator.materialize_initial_frame(initial) + history = ForwardEarningsHistory.start( + mapping, + frame, + realization_id="invented-cohort", + generator_digest="a" * 64, + source_contract_digest="b" * 64, + lineage_digest="c" * 64, + unit="XTS", + price_basis="nominal", + ) + mortality = AgeSexMortalityModel( + ((0, 34), (35, 120)), + { + ("0-34", "female"): 0.0, + ("0-34", "male"): 0.0, + ("35+", "female"): 0.0, + ("35+", "male"): float(death_year is not None), + }, + ) + return frame, generator, history, mortality + + +def step(frame, year, generator, baseline, model, *, registry=True): + context = PeriodContext( + year - 2014, + year, + 3, + {}, + rng_registry=ProjectionRNGRegistry(3, 8) if registry else None, + person_ordinals={0: 0, 1: 1, 10: 2, 20: 3}, + ) + rng = np.random.default_rng(year) + survivors, mortality = observe_mortality( + frame, + context, + rng, + model=model, + identity_map=baseline.identity_map, + realization_id=baseline.realization_id, + source_contract_digest="d" * 64, + ) + aged = advance_age(survivors, context, rng) + earnings = apply_earnings(aged, context, rng, model=generator) + return earnings, mortality, context, rng + + +@pytest.mark.parametrize("death_year", [2015, 2016, 2022, None]) +@pytest.mark.parametrize("correlated", [False, True]) +@pytest.mark.parametrize("registry", [False, True]) +def test_actual_steps_preserve_frames_rng_and_history( + death_year, correlated, registry +): + frame, generator, baseline, model = setup( + death_year=death_year, correlated=correlated + ) + cohort = ClosedCohortEarningsHistory.start(baseline, draw_index=3) + baseline_json = baseline.to_json() + control = frame.copy(deep=True) + snapshots = [cohort] + for year in range(2015, 2023): + frame, mortality, context, rng = step( + frame, year, generator, baseline, model, registry=registry + ) + direct_rng = np.random.default_rng(year) + control = apply_mortality(control, context, direct_rng, model=model) + control = advance_age(control, context, direct_rng) + control = apply_earnings(control, context, direct_rng, model=generator) + before = frame.copy(deep=True) + cohort = cohort.append( + mortality=mortality, earnings_frame=frame, lineage_digest="e" * 64 + ) + snapshots.append(cohort) + assert_frame_equal(frame, control) + assert_frame_equal(frame, before) + assert rng.bit_generator.state == direct_rng.bit_generator.state + assert cohort.active_keys == tuple(sorted(frame.person_id)) + cohort.require_extension_of(snapshots[-2]) + dead_identity = baseline.identity_map.reverse_rows([10])[0] + dead_history = cohort.for_person(dead_identity) + assert dead_history.last_year == (death_year - 1 if death_year else 2022) + assert ( + dead_history.observations[0] == baseline.for_person(dead_identity)[0] + ) + if death_year: + assert ( + cohort.amount_state(dead_identity, year=death_year) + == "not_generated_after_mortality_step" + ) + assert cohort.death_step(dead_identity).target_year == death_year + assert all(row.year < death_year for row in dead_history.observations) + else: + assert cohort.death_step(dead_identity) is None + assert baseline.to_json() == baseline_json + for key in (1, 20): + history = cohort.for_person( + baseline.identity_map.reverse_rows([key])[0] + ) + assert len(history.observations) == 9 + for prior, row in zip( + history.observations[:-1], history.observations[1:], strict=True + ): + if row.year % 2: + assert row.amount_hex == prior.amount_hex + outside = baseline.identity_map.reverse_rows([0])[0] + assert cohort.amount_state(outside, year=2022) == "unavailable" + restored = ClosedCohortEarningsHistory.from_json( + cohort.to_json(), + baseline=baseline, + expected_digest=cohort.digest, + previous=snapshots[3], + ) + assert restored == cohort + assert restored.to_json() == cohort.to_json() + + +def test_extinction_preserves_all_past_rows_and_observes_empty_years(): + frame, generator, baseline, model = setup() + model = replace(model, probability=dict.fromkeys(model.probability, 1.0)) + cohort = ClosedCohortEarningsHistory.start(baseline, draw_index=3) + initial_histories = cohort.histories + for year in range(2015, 2023): + frame, mortality, _, _ = step(frame, year, generator, baseline, model) + assert frame.empty + cohort = cohort.append( + mortality=mortality, earnings_frame=frame, lineage_digest="e" * 64 + ) + assert cohort.histories == initial_histories + assert cohort.active_keys == () + assert cohort.last_year == year + assert len(cohort.transitions) == 8 + assert ( + ClosedCohortEarningsHistory.from_json( + cohort.to_json(), baseline=baseline + ) + == cohort + ) + with pytest.raises(ValueError, match="2022"): + cohort.append( + mortality=mortality, earnings_frame=frame, lineage_digest="e" * 64 + ) + + +@pytest.fixture +def first(): + frame, generator, baseline, model = setup() + cohort = ClosedCohortEarningsHistory.start(baseline, draw_index=3) + frame, mortality, _, _ = step(frame, 2015, generator, baseline, model) + return cohort, frame, mortality, generator, model + + +@pytest.mark.parametrize( + "column,values", + [ + ("age", [32, 35, 33, 31]), + ("age", [31.0, 32.0, 35.0, 33.0]), + ("sex", ["male"] * 4), + ("year", [2016] * 4), + ("person_id", [0, 1, 10, 10]), + ("person_id", [0, 1, 10, 19]), + ("person_id", [0.0, 1.0, 10.0, 20.0]), + ("earnings_domain", [True] * 4), + ("earnings", [0.0] * 4), + ], +) +def test_bad_survivor_frame_refuses(first, column, values): + cohort, frame, mortality, _, _ = first + frame[column] = values + with pytest.raises(ValueError): + cohort.append( + mortality=mortality, earnings_frame=frame, lineage_digest="e" * 64 + ) + assert cohort.last_year == 2014 + + +def test_missing_survivor_and_dead_reintroduced_refuse(first): + cohort, frame, mortality, generator, model = first + with pytest.raises(ValueError, match="survivor"): + cohort.append( + mortality=mortality, + earnings_frame=frame.iloc[:-1], + lineage_digest="e" * 64, + ) + cohort = cohort.append( + mortality=mortality, earnings_frame=frame, lineage_digest="e" * 64 + ) + next_frame, next_mortality, _, _ = step( + frame, 2016, generator, cohort.baseline, model + ) + extra = frame.loc[frame.person_id == 10].copy() + extra["year"], extra["age"] = 2016, 36 + with pytest.raises(ValueError, match="survivor"): + cohort.append( + mortality=next_mortality, + earnings_frame=pd.concat([next_frame, extra]), + lineage_digest="e" * 64, + ) + fixed = cohort.baseline.append(frame, lineage_digest="e" * 64) + with pytest.raises(ValueError, match="roster"): + fixed.append(next_frame, lineage_digest="e" * 64) + + +@pytest.mark.parametrize( + "field,value", + [ + ("target_year", 2016), + ("period_index", 2), + ("draw_index", 4), + ("realization_id", "other"), + ], +) +def test_wrong_initial_mortality_binding_refuses(first, field, value): + cohort, frame, mortality, _, _ = first + with pytest.raises(ValueError): + cohort.append( + mortality=replace(mortality, **{field: value}), + earnings_frame=frame, + lineage_digest="e" * 64, + ) + + +@pytest.mark.parametrize( + "change", ["age", "sex", "ordinal", "source", "model", "horizon"] +) +def test_consecutive_mortality_drift_refuses(first, change): + cohort, frame, mortality, generator, model = first + cohort = cohort.append( + mortality=mortality, earnings_frame=frame, lineage_digest="e" * 64 + ) + frame, next_mortality, _, _ = step( + frame, 2016, generator, cohort.baseline, model + ) + rows = list(next_mortality.rows) + if change in ("age", "sex", "ordinal"): + row = rows[0] + field, value = { + "age": ("age", row.age + 2), + "sex": ("sex", "male"), + "ordinal": ("person_ordinal", 99), + }[change] + rows[0] = replace(row, **{field: value}) + next_mortality = replace(next_mortality, rows=tuple(rows)) + if change in ("age", "sex"): + # Even a matching altered output must not evade cross-step checks. + frame.loc[frame.person_id == row.dynamics_person_key, field] = ( + value + 1 if field == "age" else value + ) + elif change == "source": + next_mortality = replace( + next_mortality, source_contract_digest="f" * 64 + ) + elif change == "horizon": + next_mortality = replace(next_mortality, registry_n_periods=9) + else: + cells = list(next_mortality.model.cells) + cells[0] = (*cells[0][:2], (0.1).hex(), cells[0][3]) + next_mortality = replace( + next_mortality, + model=replace(next_mortality.model, cells=tuple(cells)), + ) + with pytest.raises(ValueError): + cohort.append( + mortality=next_mortality, + earnings_frame=frame, + lineage_digest="e" * 64, + ) + + +def test_frame_order_and_source_sidecar_remain_independent(first): + cohort, frame, mortality, _, _ = first + sidecar = CoveredWageHistory( + cohort.baseline, + "f" * 64, + tuple( + CoveredWageObservation( + key, 2014, SourceAmount("decimal", "7.00"), None, "a" * 64 + ) + for key in cohort.baseline.roster_keys + ), + ) + before = sidecar.to_json() + next_cohort = cohort.append( + mortality=mortality, earnings_frame=frame, lineage_digest="e" * 64 + ) + permuted = cohort.append( + mortality=mortality, + earnings_frame=frame.iloc[::-1], + lineage_digest="e" * 64, + ) + assert next_cohort == permuted + assert sidecar.to_json() == before + assert sidecar.history.digest == cohort.baseline.digest + assert sidecar.history.last_year == 2014 + with pytest.raises(FrozenInstanceError): + next_cohort.draw_index = 9 + + +@pytest.mark.parametrize( + "change", + [ + "drop_history", + "drop_transition", + "demographics", + "lineage", + "old_amount", + "extra", + "last_year", + ], +) +def test_persistence_refuses_inconsistent_composition(first, change): + cohort, frame, mortality, _, _ = first + cohort = cohort.append( + mortality=mortality, earnings_frame=frame, lineage_digest="e" * 64 + ) + doc = json.loads(cohort.to_json()) + if change == "drop_history": + doc["histories"].pop() + elif change == "drop_transition": + doc["transitions"].clear() + elif change == "demographics": + doc["transitions"][0]["survivor_demographics"][0][1] = "99" + elif change == "lineage": + doc["transitions"][0]["lineage_digest"] = "f" * 64 + elif change == "old_amount": + doc["histories"][1]["observations"][0]["amount_hex"] = (13.0).hex() + doc["histories"][1]["observations"][1]["amount_hex"] = (13.0).hex() + elif change == "last_year": + doc["last_year"] = "2016" + else: + doc["accepted"] = True + with pytest.raises(ValueError): + ClosedCohortEarningsHistory.from_json( + json.dumps(doc), baseline=cohort.baseline + ) + + +def test_external_baseline_digest_prefix_and_query_bounds(first): + cohort, frame, mortality, _, _ = first + next_cohort = cohort.append( + mortality=mortality, earnings_frame=frame, lineage_digest="e" * 64 + ) + with pytest.raises(ValueError, match="baseline"): + ClosedCohortEarningsHistory.from_json( + next_cohort.to_json(), + baseline=replace(cohort.baseline, realization_id="other"), + ) + with pytest.raises(ValueError, match="digest"): + ClosedCohortEarningsHistory.from_json( + next_cohort.to_json(), + baseline=cohort.baseline, + expected_digest="0" * 64, + ) + with pytest.raises(ValueError): + ClosedCohortEarningsHistory.from_json( + cohort.to_json(), baseline=cohort.baseline, previous=next_cohort + ) + duplicate = next_cohort.to_json().replace("{", '{"schema":"duplicate",', 1) + with pytest.raises(ValueError, match="duplicate"): + ClosedCohortEarningsHistory.from_json( + duplicate, baseline=cohort.baseline + ) + identity = cohort.baseline.identity_map.reverse_rows([1])[0] + for year in (2013, 2016, True, 2014.0): + with pytest.raises(ValueError): + next_cohort.amount_state(identity, year=year) + with pytest.raises(ValueError): + next_cohort.for_person( + cohort.baseline.identity_map.reverse_rows([2])[0] + ) + + +@pytest.mark.parametrize( + "column,dtype", + [ + ("person_id", "float64"), + ("age", "float64"), + ("year", "float64"), + ("earnings", "int64"), + ("earnings_domain", "float64"), + ("sex", "float64"), + ], +) +def test_empty_frame_still_requires_exact_types(first, column, dtype): + cohort, frame, mortality, _, _ = first + mortality = replace( + mortality, + rows=tuple(replace(row, survived=False) for row in mortality.rows), + ) + frame = frame.iloc[:0].copy() + frame[column] = frame[column].astype(dtype) + with pytest.raises(ValueError): + cohort.append( + mortality=mortality, earnings_frame=frame, lineage_digest="e" * 64 + ) + + +@pytest.mark.parametrize("change", ["age", "sex", "death"]) +def test_loaded_cross_step_demographics_and_death_extents_refuse( + first, change +): + cohort, frame, mortality, generator, model = first + cohort = cohort.append( + mortality=mortality, earnings_frame=frame, lineage_digest="e" * 64 + ) + frame, mortality, _, _ = step( + frame, 2016, generator, cohort.baseline, model + ) + cohort = cohort.append( + mortality=mortality, earnings_frame=frame, lineage_digest="e" * 64 + ) + document = json.loads(cohort.to_json()) + transition = document["transitions"][1] + if change == "death": + dead = next( + row + for row in transition["mortality"]["rows"] + if row["dynamics_person_key"] == "10" + ) + dead["survived"] = True + transition["survivor_demographics"].insert(2, ["10", "36", "male"]) + else: + row = transition["mortality"]["rows"][0] + if change == "age": + row["age"] = "70" + transition["survivor_demographics"][0][1] = "71" + else: + row["sex"] = "male" + transition["survivor_demographics"][0][2] = "male" + with pytest.raises(ValueError): + ClosedCohortEarningsHistory.from_json( + json.dumps(document), baseline=cohort.baseline + ) + + +def test_closed_roster_does_not_allow_new_or_resurrected_mortality_rows(first): + cohort, frame, mortality, generator, model = first + cohort = cohort.append( + mortality=mortality, earnings_frame=frame, lineage_digest="e" * 64 + ) + next_frame, next_mortality, _, _ = step( + frame, 2016, generator, cohort.baseline, model + ) + cohort = cohort.append( + mortality=next_mortality, + earnings_frame=next_frame, + lineage_digest="e" * 64, + ) + # Reusing the earlier cohort as a 2017 input resurrects key 10. + resurrected_frame = frame.copy() + resurrected_frame["year"] = 2016 + resurrected_frame["age"] += 1 + final_frame, resurrected, _, _ = step( + resurrected_frame, 2017, generator, cohort.baseline, model + ) + with pytest.raises(ValueError, match="roster"): + cohort.append( + mortality=resurrected, + earnings_frame=final_frame, + lineage_digest="e" * 64, + ) + # Even a previously known identity outside the original cohort is an entrant. + rows = list(mortality.rows) + rows.insert(2, replace(rows[0], dynamics_person_key=2, person_ordinal=99)) + entrant = replace(mortality, rows=tuple(rows)) + extra = frame.iloc[[0]].copy() + extra["person_id"] = 2 + with pytest.raises(ValueError, match="roster"): + ClosedCohortEarningsHistory.start( + cohort.baseline, draw_index=3 + ).append( + mortality=entrant, + earnings_frame=pd.concat([frame, extra]), + lineage_digest="e" * 64, + ) + + +def test_zero_missing_death_and_unobserved_are_distinct(first): + cohort, _, _, _, _ = first + baseline = replace( + cohort.baseline, + observations=tuple( + ( + replace(row, amount_hex=(0.0).hex()) + if row.dynamics_person_key == 1 + else row + ) + for row in cohort.baseline.observations + ), + ) + cohort = ClosedCohortEarningsHistory.start(baseline, draw_index=3) + zero, unknown = ( + baseline.identity_map.reverse_rows([key])[0] for key in (1, 0) + ) + assert cohort.amount_state(zero, year=2014) == "known_zero" + assert cohort.amount_state(unknown, year=2014) == "unavailable" + assert cohort.death_step(zero) is None + with pytest.raises(ValueError): + cohort.amount_state(zero, year=2015) + + +def test_append_is_detached_and_cannot_skip_or_repeat(first): + cohort, frame, mortality, _, _ = first + result = cohort.append( + mortality=mortality, earnings_frame=frame, lineage_digest="e" * 64 + ) + original_json = result.to_json() + frame.loc[:, "age"] += 10 + frame.loc[:, "earnings"] = 123.0 + assert result.to_json() == original_json + assert cohort.last_year == 2014 + with pytest.raises(ValueError): + result.append( + mortality=mortality, earnings_frame=frame, lineage_digest="e" * 64 + ) + with pytest.raises(ValueError, match="2014"): + ClosedCohortEarningsHistory.start(result.histories[0], draw_index=3) + + +def test_fallback_rng_records_are_supported_and_mode_switch_refuses(first): + cohort, frame, mortality, generator, model = first + mortality = replace( + mortality, + registry_n_periods=None, + rows=tuple( + replace(row, person_ordinal=None) for row in mortality.rows + ), + ) + cohort = cohort.append( + mortality=mortality, earnings_frame=frame, lineage_digest="e" * 64 + ) + assert ( + ClosedCohortEarningsHistory.from_json( + cohort.to_json(), baseline=cohort.baseline + ) + == cohort + ) + frame, registry_mortality, _, _ = step( + frame, 2016, generator, cohort.baseline, model + ) + with pytest.raises(ValueError, match="RNG mode"): + cohort.append( + mortality=registry_mortality, + earnings_frame=frame, + lineage_digest="e" * 64, + ) diff --git a/tests/test_compact_cohort_history.py b/tests/test_compact_cohort_history.py new file mode 100644 index 00000000..a9f58870 --- /dev/null +++ b/tests/test_compact_cohort_history.py @@ -0,0 +1,297 @@ +"""Invented-record tests for optional compact closed-cohort persistence.""" + +import hashlib +import json + +import numpy as np +import pandas as pd +import pytest + +from populace_dynamics.closed_cohort_history import ClosedCohortEarningsHistory +from populace_dynamics.compact_cohort_history import ( + compact_history_digest, + compact_history_from_json, + compact_history_to_json, +) +from populace_dynamics.engine.steps import AgeSexMortalityModel +from populace_dynamics.forward_earnings_history import ForwardEarningsHistory +from populace_dynamics.person_identity import PersonIdentity, PersonIdentityMap +from tests.test_closed_cohort_history import setup, step + + +def baseline(identities): + mapping = PersonIdentityMap.from_identities(identities) + count = len(mapping.entries) + frame = pd.DataFrame( + { + "person_id": np.arange(count, dtype="int64"), + "year": np.full(count, 2014, dtype="int64"), + "earnings": np.array( + [0.0 if i % 2 == 0 else i + 0.25 for i in range(count)], + dtype="float64", + ), + "earnings_domain": np.array( + [i % 3 != 0 for i in range(count)], dtype="bool" + ), + } + ) + frame.loc[~frame.earnings_domain, "earnings"] = 0.0 + return ForwardEarningsHistory.start( + mapping, + frame, + realization_id="invented-compact", + generator_digest="a" * 64, + source_contract_digest="b" * 64, + unit="XTS", + price_basis="nominal", + lineage_digest="c" * 64, + ) + + +def round_trip(history, *, baseline_history=None, previous=None): + text = compact_history_to_json(history) + restored = compact_history_from_json( + text, + baseline=baseline_history or history.baseline, + expected_digest=hashlib.sha256(text.encode()).hexdigest(), + previous=previous, + ) + assert restored == history + assert restored.digest == history.digest + assert compact_history_to_json(restored) == text + return text + + +def test_mixed_identity_missing_and_zero_round_trip_preserves_legacy_value(): + initial = baseline( + [ + PersonIdentity("int64", -(2**63)), + PersonIdentity("uint64", 2**64 - 1), + PersonIdentity("string", "001"), + ] + ) + history = ClosedCohortEarningsHistory.start(initial, draw_index=7) + legacy_text, legacy_digest = history.to_json(), history.digest + text = round_trip(history) + assert history.to_json() == legacy_text + assert history.digest == legacy_digest + assert ( + compact_history_digest(history) + == hashlib.sha256(text.encode()).hexdigest() + ) + assert "identity_map" not in json.loads(text)["histories"][0] + + +@pytest.mark.parametrize("death_year", [2015, 2016, None]) +def test_observed_partial_death_and_extension_round_trip(death_year): + frame, generator, initial, model = setup(death_year=death_year) + previous = ClosedCohortEarningsHistory.start(initial, draw_index=3) + history = previous + for year in range(2015, 2023): + frame, mortality, _, _ = step(frame, year, generator, initial, model) + history = history.append( + mortality=mortality, + earnings_frame=frame, + lineage_digest="e" * 64, + ) + round_trip(history, previous=previous) + + +def test_full_extinction_and_later_empty_periods_round_trip(): + frame, generator, initial, _ = setup(death_year=None) + model = AgeSexMortalityModel( + ((0, 120),), + {("0+", "female"): 1.0, ("0+", "male"): 1.0}, + ) + previous = ClosedCohortEarningsHistory.start(initial, draw_index=3) + history = previous + mortality_row_counts = [] + for year in range(2015, 2023): + frame, mortality, _, _ = step(frame, year, generator, initial, model) + mortality_row_counts.append(len(mortality.rows)) + history = history.append( + mortality=mortality, + earnings_frame=frame, + lineage_digest="e" * 64, + ) + round_trip(history, previous=previous) + assert mortality_row_counts == [4, 0, 0, 0, 0, 0, 0, 0] + assert history.active_keys == () + assert len(history.histories) == 4 + assert all(item.last_year == 2014 for item in history.histories) + + +def mutate(text, callback): + document = json.loads(text) + callback(document) + changed = json.dumps( + document, + sort_keys=True, + ensure_ascii=False, + separators=(",", ":"), + allow_nan=False, + ) + return changed, hashlib.sha256(changed.encode()).hexdigest() + + +@pytest.mark.parametrize( + "change,match", + [ + (lambda d: d.update(extra=True), "fields"), + (lambda d: d["histories"][0].update(extra=True), "fields"), + (lambda d: d["histories"][0].update(roster_keys=["00"]), "canonical"), + ( + lambda d: d["histories"][0]["observations"][0].update( + amount_hex="nan", earnings_domain=True + ), + "binary64", + ), + ( + lambda d: d["histories"][0].update(identity_map_digest="0" * 64), + "identity", + ), + (lambda d: d.update(last_year="02014"), "canonical"), + ], +) +def test_recomputed_digest_does_not_admit_invalid_internal_content( + change, match +): + initial = baseline([PersonIdentity("string", "invented")]) + history = ClosedCohortEarningsHistory.start(initial, draw_index=0) + changed, digest = mutate(compact_history_to_json(history), change) + with pytest.raises(ValueError, match=match): + compact_history_from_json( + changed, baseline=initial, expected_digest=digest + ) + + +def test_wrong_digest_baseline_duplicate_and_nonfinite_refuse(): + initial = baseline([PersonIdentity("string", "invented")]) + history = ClosedCohortEarningsHistory.start(initial, draw_index=0) + text = compact_history_to_json(history) + digest = compact_history_digest(history) + with pytest.raises(ValueError, match="digest mismatch"): + compact_history_from_json( + text, baseline=initial, expected_digest="0" * 64 + ) + other = baseline([PersonIdentity("string", "other")]) + with pytest.raises(ValueError, match="baseline binding"): + compact_history_from_json(text, baseline=other, expected_digest=digest) + duplicate = text.replace( + '{"baseline_digest":', '{"schema":"x","baseline_digest":' + ) + with pytest.raises(ValueError, match="duplicate JSON field"): + compact_history_from_json( + duplicate, + baseline=initial, + expected_digest=hashlib.sha256(duplicate.encode()).hexdigest(), + ) + nonfinite = text.replace('"draw_index":"0"', '"draw_index":NaN') + with pytest.raises(ValueError, match="invalid JSON constant"): + compact_history_from_json( + nonfinite, + baseline=initial, + expected_digest=hashlib.sha256(nonfinite.encode()).hexdigest(), + ) + + +def test_previous_rewrite_refuses_existing_extension_check(): + initial = baseline([PersonIdentity("string", "invented")]) + previous = ClosedCohortEarningsHistory.start(initial, draw_index=0) + replacement = baseline([PersonIdentity("string", "invented")]) + replacement = ForwardEarningsHistory( + replacement.identity_map, + "different-realization", + replacement.generator_digest, + replacement.source_contract_digest, + replacement.unit, + replacement.price_basis, + replacement.roster_keys, + replacement.last_year, + replacement.observations, + ) + changed = ClosedCohortEarningsHistory.start(replacement, draw_index=0) + with pytest.raises(ValueError, match="baseline or draw"): + round_trip(changed, previous=previous) + + +def test_previous_prefix_accepts_extension_and_rejects_rewritten_old_row(): + frame, generator, initial, model = setup(death_year=None) + previous = ClosedCohortEarningsHistory.start(initial, draw_index=3) + frame, mortality, _, _ = step(frame, 2015, generator, initial, model) + current = previous.append( + mortality=mortality, + earnings_frame=frame, + lineage_digest="e" * 64, + ) + text = round_trip(current, previous=previous) + changed, digest = mutate( + text, + lambda d: d["histories"][0]["observations"][0].update( + amount_hex=float(1).hex(), earnings_domain=True + ), + ) + with pytest.raises( + ValueError, match="fixed earnings domain|changed or removed prior rows" + ): + compact_history_from_json( + changed, + baseline=initial, + expected_digest=digest, + previous=previous, + ) + + +def test_bounded_storage_growth_removes_repeated_map_quadratic_term(): + sizes = {} + for count in (80, 160): + initial = baseline( + PersonIdentity("string", f"invented-{i:03d}") for i in range(count) + ) + history = ClosedCohortEarningsHistory.start(initial, draw_index=0) + compact = compact_history_to_json(history).encode() + legacy = history.to_json().encode() + sizes[count] = (len(compact), len(legacy)) + assert round_trip(history).encode() == compact + assert sizes[160][0] < 2.2 * sizes[80][0] + assert sizes[160][1] > 3.5 * sizes[80][1] + assert sizes[160][0] < sizes[160][1] / 5 + + +@pytest.mark.parametrize("count", [1, 160]) +def test_identity_map_serialization_is_constant_per_envelope( + monkeypatch, count +): + initial = baseline( + PersonIdentity("string", f"invented-{i:03d}") for i in range(count) + ) + history = ClosedCohortEarningsHistory.start(initial, draw_index=0) + original = PersonIdentityMap.digest.fget + calls = 0 + + def counted(mapping): + nonlocal calls + calls += 1 + return original(mapping) + + monkeypatch.setattr(PersonIdentityMap, "digest", property(counted)) + text = compact_history_to_json(history) + # One direct map digest plus the unchanged baseline digest, independent of N. + assert calls == 2 + calls = 0 + restored = compact_history_from_json( + text, + baseline=initial, + expected_digest=hashlib.sha256(text.encode()).hexdigest(), + ) + assert restored == history + # Loading, baseline binding and canonical reserialization remain constant. + assert calls == 4 + + +def test_nontext_input_refuses_with_value_error(): + initial = baseline([PersonIdentity("string", "invented")]) + with pytest.raises(ValueError, match="JSON text"): + compact_history_from_json( + b"{}", baseline=initial, expected_digest="0" * 64 + ) diff --git a/tests/test_covered_wage_history.py b/tests/test_covered_wage_history.py new file mode 100644 index 00000000..4526a791 --- /dev/null +++ b/tests/test_covered_wage_history.py @@ -0,0 +1,273 @@ +"""Invented source observations only; no benefit or population calculation.""" + +import json +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest + +from populace_dynamics.covered_wage_history import ( + CoveredWageHistory, + CoveredWageObservation, + SourceAmount, +) +from populace_dynamics.forward_earnings_history import ForwardEarningsHistory +from populace_dynamics.person_identity import PersonIdentity, PersonIdentityMap + + +def history(): + mapping = PersonIdentityMap.from_identities( + [PersonIdentity("uint64", 2**64 - 1), PersonIdentity("string", "001")] + ) + frame = pd.DataFrame( + { + "person_id": [0, 1], + "year": [2014, 2014], + "earnings": np.array([0.1, 0.0], dtype=np.float64), + "earnings_domain": [True, False], + } + ) + first = ForwardEarningsHistory.start( + mapping, + frame, + realization_id="invented-career-1", + generator_digest="a" * 64, + source_contract_digest="b" * 64, + lineage_digest="c" * 64, + unit="XTS", + price_basis="nominal", + ) + frame["year"] = 2015 + return first.append(frame, lineage_digest="d" * 64) + + +def bundle(forward=None): + return CoveredWageHistory( + history() if forward is None else forward, + "e" * 64, + ( + CoveredWageObservation( + 0, 2014, SourceAmount("decimal", "0.00"), None, "f" * 64 + ), + CoveredWageObservation( + 1, 2014, None, "coverage_unresolved", "1" * 64 + ), + CoveredWageObservation( + 0, 2015, None, "source_field_unavailable", "2" * 64 + ), + CoveredWageObservation( + 1, + 2015, + SourceAmount("uint64", "9007199254740993"), + None, + "3" * 64, + ), + ), + ) + + +def test_known_zero_unknown_and_independent_source_domain_remain_distinct(): + wages = bundle() + assert [r.amount_state for r in wages.observations] == [ + "known_zero", + "unavailable", + "unavailable", + "known_amount", + ] + # Independent covered-wage evidence is permitted for a person outside the + # forward generator's domain; its control zero is never copied or inferred. + person = PersonIdentity("uint64", 2**64 - 1) + assert wages.history.for_person(person)[1].amount_hex is None + assert ( + wages.for_person(person)[1].amount.serialization == "9007199254740993" + ) + assert wages.history.coverage_status == "not_materialized" + assert wages.history.source_registry_status == "registration_required" + + +@pytest.mark.parametrize( + "kind,value", + [ + ("uint64", "9007199254740993"), + ("decimal", "123456789012345678901234567890.0100"), + ("binary64", (0.1).hex()), + ("binary64", (-0.0).hex()), + ], +) +def test_source_representation_round_trips_without_rounding(kind, value): + wages = bundle() + row = replace(wages.observations[0], amount=SourceAmount(kind, value)) + wages = replace(wages, observations=(row,) + wages.observations[1:]) + restored = CoveredWageHistory.from_json( + wages.to_json(), history=wages.history, expected_digest=wages.digest + ) + assert restored == wages + assert restored.observations[0].amount.serialization == value + assert restored.to_json() == wages.to_json() + + +def test_decimal_scale_is_source_identity_and_row_order_is_not(): + wages = bundle() + assert ( + replace(wages, observations=wages.observations[::-1]).digest + == wages.digest + ) + row = replace(wages.observations[0], amount=SourceAmount("decimal", "0.0")) + assert ( + replace(wages, observations=(row,) + wages.observations[1:]).digest + != wages.digest + ) + + +@pytest.mark.parametrize( + "kind,value", + [ + ("uint64", True), + ("uint64", 1), + ("uint64", "01"), + ("uint64", "-1"), + ("decimal", "NaN"), + ("decimal", "Infinity"), + ("decimal", "1e3"), + ("decimal", "-0.01"), + ("decimal", " 1.00"), + ("decimal", "01.00"), + ("binary64", "nan"), + ("binary64", float("inf").hex()), + ("binary64", (-1.0).hex()), + ("binary64", "0x1p+0"), + ("unknown", "0"), + ], +) +def test_ambiguous_or_negative_source_amount_refuses(kind, value): + with pytest.raises(ValueError): + SourceAmount(kind, value) + + +@pytest.mark.parametrize( + "change", + [ + {"amount": None}, + {"missing_reason": "coverage_unresolved"}, + {"source_digest": ""}, + {"year": True}, + {"dynamics_person_key": 0.0}, + ], +) +def test_observation_requires_exact_identity_amount_state_and_provenance( + change, +): + with pytest.raises(ValueError): + replace(bundle().observations[0], **change) + + +def test_dense_envelope_cannot_shrink_or_expand_silently(): + wages = bundle() + invalid = [ + wages.observations[:-1], + wages.observations[:2], + wages.observations + (wages.observations[0],), + (replace(wages.observations[0], year=2016),) + wages.observations[1:], + (replace(wages.observations[0], dynamics_person_key=2),) + + wages.observations[1:], + ] + for rows in invalid: + with pytest.raises(ValueError): + replace(wages, observations=rows) + + +@pytest.mark.parametrize( + "field,value", + [ + ("realization_id", "another-draw"), + ("unit", "USD"), + ("generator_digest", "0" * 64), + ("source_contract_digest", "0" * 64), + ], +) +def test_loading_refuses_rebinding_to_different_forward_history(field, value): + wages = bundle() + with pytest.raises(ValueError, match="history"): + CoveredWageHistory.from_json( + wages.to_json(), history=replace(wages.history, **{field: value}) + ) + + +def test_loading_revalidates_envelope_metadata_and_expected_digest(): + wages = bundle() + for mutation in ( + "missing_row", + "unknown_field", + "row_unknown_field", + "history_digest", + "unit", + ): + doc = json.loads(wages.to_json()) + if mutation == "missing_row": + doc["observations"].pop() + elif mutation == "unknown_field": + doc["accepted"] = True + elif mutation == "row_unknown_field": + doc["observations"][0]["accepted"] = True + else: + doc[mutation] = "0" * 64 + with pytest.raises(ValueError): + CoveredWageHistory.from_json( + json.dumps(doc), history=wages.history + ) + with pytest.raises(ValueError, match="digest"): + CoveredWageHistory.from_json( + wages.to_json(), history=wages.history, expected_digest="0" * 64 + ) + + +def test_duplicate_json_members_and_nonfinite_constants_refuse(): + wages = bundle() + duplicate = wages.to_json().replace( + '"schema":', '"schema":"duplicate","schema":' + ) + nonfinite = wages.to_json().replace( + '"serialization":"0.00"', '"serialization":NaN' + ) + for text in (duplicate, nonfinite): + with pytest.raises(ValueError): + CoveredWageHistory.from_json(text, history=wages.history) + + +def test_attaching_wages_does_not_change_labor_history_or_produce_policy_input(): + forward = history() + before = forward.to_json() + wages = bundle(forward) + assert forward.to_json() == before + assert wages.history is forward + doc = json.loads(wages.to_json()) + assert doc["concept"] == "source_reported_uncapped_employee_wages_covered" + assert "creditable_earnings" not in doc + + +@pytest.mark.parametrize("kind,bound", [("int64", 2**63), ("uint64", 2**64)]) +def test_integer_source_range_is_preserved(kind, bound): + assert SourceAmount(kind, str(bound - 1)).serialization == str(bound - 1) + with pytest.raises(ValueError): + SourceAmount(kind, str(bound)) + + +def test_missing_coordinates_and_decimal_signed_zero_preserve_source_state(): + wages = bundle() + assert wages.missing_coordinates == ((1, 2014), (0, 2015)) + source = SourceAmount("decimal", "-0.00") + assert source.is_zero + assert source.serialization == "-0.00" + with pytest.raises(ValueError): + replace(wages.observations[1], missing_reason="guess") + + +def test_same_private_keys_do_not_permit_a_different_identity_map(): + wages = bundle() + another = PersonIdentityMap.from_identities( + [PersonIdentity("string", "002"), PersonIdentity("uint64", 2**64 - 1)] + ) + different = replace(wages.history, identity_map=another) + with pytest.raises(ValueError, match="history"): + CoveredWageHistory.from_json(wages.to_json(), history=different) diff --git a/tests/test_entrant_domains.py b/tests/test_entrant_domains.py new file mode 100644 index 00000000..3896b7a0 --- /dev/null +++ b/tests/test_entrant_domains.py @@ -0,0 +1,381 @@ +"""Tests for the entrant exclusion adapters. + +The load-bearing test is +``test_without_the_adapter_every_entrant_over_50_draws_a_claim_age``: it runs +the historical ``apply_claiming`` step directly on entrant rows and shows the +unconditional draw happening, which is the failure the adapter exists to +prevent. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from populace_dynamics.engine import entrant_domains as edm +from populace_dynamics.engine.earnings_domain import EARNINGS_DOMAIN_COLUMN +from populace_dynamics.engine.entrant_schedule import ( + ENTRY_KIND_BIRTH, + ENTRY_KIND_COLUMN, + ENTRY_KIND_IMMIGRANT, +) +from populace_dynamics.engine.loop import ( + PeriodContext, + SyntheticPersonIdAllocator, +) +from populace_dynamics.engine.rng import ProjectionRNGRegistry +from populace_dynamics.engine.steps import ( + ClaimingSchedule, + apply_claiming, + materialize_maternal_births, +) + + +def _roster() -> pd.DataFrame: + """Four incumbents and four entrants, spanning the claiming threshold.""" + return pd.DataFrame( + { + "person_id": [1, 2, 3, 4, 101, 102, 103, 104], + "year": [2026] * 8, + "age": [30, 55, 62, 70, 30, 55, 62, 70], + "sex": ["female", "male", "female", "male"] * 2, + "weight": [1.0] * 8, + ENTRY_KIND_COLUMN: [None] * 4 + [ENTRY_KIND_IMMIGRANT] * 4, + } + ) + + +def _claiming_schedule() -> ClaimingSchedule: + return ClaimingSchedule( + pmf={ + ("female", 2026): {62: 0.5, 67: 0.5}, + ("male", 2026): {62: 0.5, 67: 0.5}, + } + ) + + +def _context() -> PeriodContext: + return PeriodContext(period_index=1, year=2026, draw_index=0, metadata={}) + + +# -------------------------------------------------------------------------- +# Membership +# -------------------------------------------------------------------------- +def test_entrant_mask_reads_the_provenance_column(): + mask = edm.entrant_mask(_roster()) + assert mask.tolist() == [False] * 4 + [True] * 4 + + +def test_a_frame_without_the_column_has_no_entrants(): + """Safe to call on a closed-panel roster that never saw a schedule.""" + frame = _roster().drop(columns=[ENTRY_KIND_COLUMN]) + assert not edm.entrant_mask(frame).any() + + +def test_a_missing_value_is_an_incumbent(): + frame = _roster() + frame.loc[4, ENTRY_KIND_COLUMN] = pd.NA + assert edm.entrant_mask(frame).tolist() == [False] * 5 + [True] * 3 + + +def test_births_are_not_immigrant_entrants_by_default(): + frame = _roster() + frame.loc[4, ENTRY_KIND_COLUMN] = ENTRY_KIND_BIRTH + assert edm.entrant_mask(frame).sum() == 3 + assert ( + edm.entrant_mask( + frame, entry_kinds=(ENTRY_KIND_IMMIGRANT, ENTRY_KIND_BIRTH) + ).sum() + == 4 + ) + + +def test_the_three_exclusion_id_sets_are_the_entrant_ids(): + frame = _roster() + expected = {101, 102, 103, 104} + assert edm.excluded_fertility_ids(frame) == expected + assert edm.excluded_claiming_ids(frame) == expected + assert edm.excluded_disability_ids(frame) == expected + + +# -------------------------------------------------------------------------- +# Claiming: the hazard, and the adapter that removes it +# -------------------------------------------------------------------------- +def test_without_the_adapter_every_entrant_over_50_draws_a_claim_age(): + """The unconditional draw at ``steps.py:411-413``, demonstrated. + + This establishes behavioral claiming-age assignment only. It does not + calculate benefits or establish a direction for aggregate fiscal effects. + """ + out = apply_claiming( + _roster(), + _context(), + np.random.default_rng(0), + schedule=_claiming_schedule(), + ) + entrants = out[out[ENTRY_KIND_COLUMN] == ENTRY_KIND_IMMIGRANT] + over_50 = entrants[entrants["age"] >= 50] + assert len(over_50) == 3 + assert over_50["claim_age"].notna().all() + assert bool(out.loc[out["person_id"] == 104, "claimed"].iloc[0]) is True + + +def test_the_adapter_leaves_entrants_unclaimed_and_incumbents_untouched(): + adapter = edm.EntrantClaimingAdapter( + lambda frame, context, rng: apply_claiming( + frame, context, rng, schedule=_claiming_schedule() + ) + ) + out = adapter(_roster(), _context(), np.random.default_rng(0)) + entrants = out[out[ENTRY_KIND_COLUMN] == ENTRY_KIND_IMMIGRANT] + incumbents = out[out[ENTRY_KIND_COLUMN].isna()] + + assert entrants["claim_age"].isna().all() + assert not entrants["claimed"].any() + assert entrants["claim_year"].isna().all() + # The incumbents get exactly what the historical adapter gives them. + assert incumbents.loc[incumbents["age"] >= 50, "claim_age"].notna().all() + + +def test_the_adapter_is_a_passthrough_without_entrants(): + frame = _roster().drop(columns=[ENTRY_KIND_COLUMN]) + step = lambda f, c, r: apply_claiming( # noqa: E731 + f, c, r, schedule=_claiming_schedule() + ) + adapter = edm.EntrantClaimingAdapter(step) + direct = step(frame, _context(), np.random.default_rng(0)) + through = adapter(frame, _context(), np.random.default_rng(0)) + pd.testing.assert_frame_equal(direct, through) + + +def test_the_adapter_preserves_the_roster_and_person_sort(): + adapter = edm.EntrantClaimingAdapter( + lambda frame, context, rng: apply_claiming( + frame, context, rng, schedule=_claiming_schedule() + ) + ) + out = adapter(_roster(), _context(), np.random.default_rng(0)) + assert len(out) == 8 + assert out["person_id"].is_monotonic_increasing + + +# -------------------------------------------------------------------------- +# Earnings domain +# -------------------------------------------------------------------------- +def test_entrants_marked_inside_the_earnings_domain_are_rejected(): + frame = _roster() + frame[EARNINGS_DOMAIN_COLUMN] = [True] * 4 + [False, False, True, False] + with pytest.raises(ValueError, match="marked inside the fitted earnings"): + edm.assert_entrants_out_of_earnings_domain(frame) + + +def test_entrants_outside_the_earnings_domain_pass(): + frame = _roster() + frame[EARNINGS_DOMAIN_COLUMN] = [True] * 4 + [False] * 4 + assert edm.assert_entrants_out_of_earnings_domain(frame) == 4 + + +def test_a_frame_without_the_domain_column_still_counts_entrants(): + assert edm.assert_entrants_out_of_earnings_domain(_roster()) == 4 + + +# -------------------------------------------------------------------------- +# Benefit suppression +# -------------------------------------------------------------------------- +def test_entrant_benefit_outputs_are_missing_not_zero(): + frame = _roster() + frame["aime"] = 100.0 + frame["pia"] = 50.0 + out = edm.suppress_entrant_benefit_outputs(frame) + entrants = out[out[ENTRY_KIND_COLUMN] == ENTRY_KIND_IMMIGRANT] + assert entrants["aime"].isna().all() + assert entrants["pia"].isna().all() + # A zero would be a measurement; prior US covered earnings are censored. + assert not (entrants["aime"] == 0).any() + assert (out[out[ENTRY_KIND_COLUMN].isna()]["aime"] == 100.0).all() + + +# -------------------------------------------------------------------------- +# The report +# -------------------------------------------------------------------------- +def test_exclusion_report_covers_all_four_domains(): + report = edm.exclusion_report(_roster()) + assert report.n_entrants == 4 + assert set(report.excluded) == set(edm.EXCLUDED_DOMAINS) + assert ( + report.excluded["claiming_eligibility"][ + "n_would_have_drawn_a_claim_age" + ] + == 3 + ) + + +def test_exclusion_report_states_it_is_report_only_and_not_a_behaviour_model(): + record = edm.exclusion_report(_roster()).as_dict() + assert record["gated"] is False + assert record["execution_verified"] is False + assert record["status"] == "inventory_only" + assert "OUTSIDE the estimand" in record["interpretation"] + assert any( + "mortality" in entry + for entry in record["intended_demographic_domains"] + ) + + +def test_every_excluded_domain_names_the_code_that_makes_it_unfitted(): + for domain, reason in edm.EXCLUDED_DOMAINS.items(): + assert ".py:" in reason, domain + + +@pytest.mark.parametrize( + "column,value", + [ + ("claim_age", 62), + ("claim_year", 2020), + ("claimed", True), + ("claimed", "False"), + ("di_converted", True), + ], +) +def test_existing_entrant_claim_state_is_rejected_before_any_step( + column, value +): + frame = _roster() + frame[column] = pd.Series([pd.NA] * len(frame), dtype="object") + frame.loc[4, column] = value + original = frame.copy(deep=True) + rng = np.random.default_rng(9) + untouched_rng = np.random.default_rng(9) + + def forbidden(*args): + raise AssertionError("contradictory state reached the incumbent step") + + with pytest.raises(ValueError, match="excluded entrants have"): + edm.EntrantClaimingAdapter(forbidden)(frame, _context(), rng) + pd.testing.assert_frame_equal(frame, original) + assert rng.bytes(32) == untouched_rng.bytes(32) + + +def test_an_all_entrant_frame_never_calls_the_incumbent_step(): + frame = _roster().iloc[4:].copy() + + def forbidden(*args): + raise AssertionError("all-entrant frame invoked the incumbent step") + + result = edm.EntrantClaimingAdapter(forbidden)( + frame, _context(), np.random.default_rng(0) + ) + assert result["claim_age"].isna().all() + assert result["claim_year"].isna().all() + assert not result["claimed"].any() + assert result["person_id"].tolist() == frame["person_id"].tolist() + + +@pytest.mark.parametrize("kind", [None, "immgrant_cohort", 1]) +def test_synthetic_entrants_cannot_lose_their_provenance(kind): + frame = _roster().astype({ENTRY_KIND_COLUMN: "object"}) + frame["synthetic_entry"] = [False] * 4 + [True] * 4 + frame.loc[4, ENTRY_KIND_COLUMN] = kind + with pytest.raises(ValueError, match="entry_kind"): + edm.entrant_mask(frame) + + +def test_losing_the_entire_provenance_column_is_detected_for_synthetic_rows(): + frame = _roster().drop(columns=ENTRY_KIND_COLUMN) + frame["synthetic_entry"] = [False] * 4 + [True] * 4 + with pytest.raises(ValueError, match="require explicit entry_kind"): + edm.exclusion_report(frame) + + +def test_incumbent_claims_and_random_consumption_match_direct_subset(): + frame = _roster() + + def step(f, c, r): + return apply_claiming(f, c, r, schedule=_claiming_schedule()) + + first_rng = np.random.default_rng(11) + direct_rng = np.random.default_rng(11) + result = edm.EntrantClaimingAdapter(step)(frame, _context(), first_rng) + direct = step(frame.iloc[:4].copy(), _context(), direct_rng) + pd.testing.assert_frame_equal(result.iloc[:4], direct, check_dtype=False) + assert first_rng.bytes(32) == direct_rng.bytes(32) + + +def test_claiming_inventory_counts_missing_plans_without_claiming_execution(): + frame = _roster() + frame["claim_age"] = pd.array([pd.NA] * len(frame), dtype="Int64") + frame.loc[5, "claim_age"] = 62 + report = edm.exclusion_report(frame).as_dict() + assert ( + report["excluded_domains"]["claiming_eligibility"][ + "n_would_have_drawn_a_claim_age" + ] + == 2 + ) + assert not report["execution_verified"] + + +@pytest.mark.parametrize( + "selectors", [("immgrant_cohort",), "immigrant_cohort", ()] +) +def test_invalid_selectors_cannot_silently_bypass_the_claiming_guard( + selectors, +): + def forbidden(*args): + raise AssertionError("bad selectors reached the incumbent step") + + with pytest.raises(ValueError, match="entry_kind|entry kind"): + edm.EntrantClaimingAdapter(forbidden, entry_kinds=selectors)( + _roster(), _context(), np.random.default_rng(0) + ) + + +def test_birth_provenance_boundary_preserves_historical_materialization(): + frame = _roster() + frame["synthetic_entry"] = [False] * 4 + [True] * 4 + births = pd.DataFrame({"parent_person_id": [1], "birth_year": [2026]}) + context = PeriodContext( + 1, + 2026, + 0, + {"synthetic_id_allocator": SyntheticPersonIdAllocator(105)}, + rng_registry=ProjectionRNGRegistry(0, 2), + person_ordinals={ + pid: index for index, pid in enumerate(frame["person_id"]) + }, + ) + raw_context = PeriodContext( + 1, 2026, 0, {"synthetic_id_allocator": SyntheticPersonIdAllocator(105)} + ) + raw = materialize_maternal_births( + frame, births, raw_context, np.random.default_rng(4) + ) + with pytest.raises(ValueError, match="require explicit entry_kind"): + edm.entrant_mask(raw) + labeled = edm.materialize_births_with_provenance( + frame, births, context, np.random.default_rng(4) + ) + pd.testing.assert_frame_equal( + labeled.drop(columns=ENTRY_KIND_COLUMN), + raw.drop(columns=ENTRY_KIND_COLUMN), + ) + child = labeled.loc[labeled["person_id"] == 105].iloc[0] + assert child[ENTRY_KIND_COLUMN] == ENTRY_KIND_BIRTH + assert child["age"] == 0 + assert child["parent_person_id"] == 1 + assert context.synthetic_id_allocator.next_id == 106 + assert edm.excluded_claiming_ids(labeled) == {101, 102, 103, 104} + + def step(f, c, r): + return apply_claiming(f, c, r, schedule=_claiming_schedule()) + + result = edm.EntrantClaimingAdapter(step)( + labeled, context, np.random.default_rng(0) + ) + direct = step(frame.iloc[:4], context, np.random.default_rng(0)) + pd.testing.assert_frame_equal( + result.iloc[:4][direct.columns], direct, check_dtype=False + ) + assert not result.loc[result["person_id"] == 105, "claimed"].iloc[0] + assert edm.exclusion_report(result).n_entrants == 4 diff --git a/tests/test_entrant_id_allocation.py b/tests/test_entrant_id_allocation.py new file mode 100644 index 00000000..c4206906 --- /dev/null +++ b/tests/test_entrant_id_allocation.py @@ -0,0 +1,222 @@ +"""Projection metadata must not let an in-loop allocation reuse a known ID. + +Every fixture here is SYNTHETIC: the survival and birth events are scripted +by the test, not drawn from a fitted law. What is real is the projection +loop, the entrant schedule builder, and the maternal-birth materializer that +takes child IDs from ``context.synthetic_id_allocator``. + +The failure these tests pin: ``M6RealizedPopulation.projection_metadata`` +places a fresh ``SyntheticPersonIdAllocator(synthetic_id_start, reserved)`` in +the metadata on every call. A schedule built from any *other* allocator that +starts at the same ``synthetic_id_start`` holds the same IDs the loop will +later hand to newborns. When the entrant has already died, the birth takes +its ID silently, so one ``person_id`` names two different people (and shares +one stable RNG ordinal). + +``engine/loop.py`` sits inside the birth-evidence reducer's reviewed +implementation identity, so the guard lives in +``entrant_schedule.validate_projection_allocator`` and callers run it on the +exact metadata they project with. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from populace_dynamics.engine import entrant_schedule as esm +from populace_dynamics.engine.loop import ( + SCHEDULED_ENTRIES_KEY, + MaritalStepResult, + PeriodModules, + ProjectionEngine, + SyntheticPersonIdAllocator, +) +from populace_dynamics.engine.steps import ( + advance_age, + materialize_maternal_births, +) + +RESERVED = frozenset({1, 2, 3}) +SYNTHETIC_ID_START = max(RESERVED) + 1 # the m6_population.py rule + + +def _donor() -> pd.DataFrame: + """One SYNTHETIC 60-year-old male donor row.""" + return pd.DataFrame( + { + "person_id": [900], + "weight": [1.0], + "entry_age": [60], + "is_female": [False], + "source_year": [2024], + "peinusyr": [28], + "prcitshp": [4], + "penatvty": [300], + "foreign_born": [True], + } + ) + + +def _initial() -> pd.DataFrame: + return pd.DataFrame( + { + "person_id": [1, 2, 3], + "year": [2025, 2025, 2025], + "age": [30, 40, 50], + "sex": ["female", "male", "male"], + "weight": [1.0, 1.0, 1.0], + } + ) + + +def _modules(calls: list[int]) -> PeriodModules: + """SYNTHETIC events: the entrant dies in 2027; person 1 gives birth in + 2028. The birth itself goes through the real materializer.""" + + def mortality(frame, context, rng): + del rng + calls.append(context.year) + if context.year == 2027 and esm.ENTRY_KIND_COLUMN in frame: + frame = frame.loc[frame[esm.ENTRY_KIND_COLUMN].isna()] + return frame.sort_values("person_id").reset_index(drop=True) + + def marital(frame, context, rng): + del context, rng + births = pd.DataFrame( + { + "parent_person_id": [1], + "birth_year": pd.array([2028], dtype="Int64"), + } + ) + return MaritalStepResult(frame.copy(), births) + + def fertility(frame, context, marital_result, rng): + return materialize_maternal_births( + frame, marital_result.births, context, rng + ) + + def passthrough(frame, context, rng): + del context, rng + return frame.copy() + + return PeriodModules( + mortality=mortality, + aging=advance_age, + marital_core=marital, + fertility=fertility, + disability=passthrough, + earnings=passthrough, + claiming=passthrough, + household_composition=lambda frame, context, result, rng: frame.copy(), + ) + + +def _split_allocator_metadata(): + """The reviewer's scenario: a schedule built with one allocator, the + projection carrying a second allocator at the same start.""" + schedule = esm.build_entrant_schedule( + _donor(), + {2026: 1.0}, + allocator=SyntheticPersonIdAllocator(SYNTHETIC_ID_START, RESERVED), + ) + assert schedule.frames[2026]["person_id"].tolist() == [4] + projection_allocator = SyntheticPersonIdAllocator( + SYNTHETIC_ID_START, RESERVED + ) + return projection_allocator, { + "synthetic_id_allocator": projection_allocator, + SCHEDULED_ENTRIES_KEY: schedule.as_metadata(), + } + + +def test_a_second_allocator_at_the_same_start_is_refused(): + projection_allocator, metadata = _split_allocator_metadata() + with pytest.raises( + ValueError, + match=r"next_id 4 can reach initial or scheduled person_id \[4\]", + ): + esm.validate_projection_allocator(_initial()["person_id"], metadata) + assert projection_allocator.next_id == SYNTHETIC_ID_START + + +def test_the_unvalidated_loop_still_reuses_the_id(): + """Pins why validation is required: the sealed loop does not check, and + without the validator person 4 is both an immigrant and a newborn.""" + _, metadata = _split_allocator_metadata() + result = ProjectionEngine(_modules([])).project( + _initial(), end_year=2028, draw_index=0, metadata=metadata + ) + rows = result.panel.loc[result.panel["person_id"] == 4] + assert sorted(rows["year"].tolist()) == [2026, 2028] + + +def test_the_shared_allocator_gives_the_birth_a_fresh_id(): + """The supported path: build the schedule with the same allocator object + the projection uses, so the cursor is already past the entrant.""" + allocator = SyntheticPersonIdAllocator(SYNTHETIC_ID_START, RESERVED) + schedule = esm.build_entrant_schedule( + _donor(), {2026: 1.0}, allocator=allocator + ) + metadata = { + "synthetic_id_allocator": allocator, + SCHEDULED_ENTRIES_KEY: schedule.as_metadata(), + } + esm.validate_projection_allocator(_initial()["person_id"], metadata) + calls: list[int] = [] + result = ProjectionEngine(_modules(calls)).project( + _initial(), end_year=2028, draw_index=0, metadata=metadata + ) + assert calls == [2026, 2027, 2028] + panel = result.panel + entrant = panel.loc[panel["person_id"] == 4] + assert entrant["year"].tolist() == [2026] + assert entrant[esm.ENTRY_KIND_COLUMN].tolist() == [ + esm.ENTRY_KIND_IMMIGRANT + ] + newborn = result.slices[-1].loc[lambda f: f["parent_person_id"].notna()] + assert newborn["person_id"].tolist() == [5] + assert newborn["parent_person_id"].tolist() == [1] + assert allocator.next_id == 6 + + +def test_an_initial_roster_id_at_or_above_the_cursor_is_refused(): + """The same collision class without a schedule.""" + initial = _initial() + initial.loc[2, "person_id"] = 50 + with pytest.raises(ValueError, match=r"person_id \[50\]"): + esm.validate_projection_allocator( + initial["person_id"], + {"synthetic_id_allocator": SyntheticPersonIdAllocator(10)}, + ) + + +@pytest.mark.parametrize( + "allocator", + [ + SyntheticPersonIdAllocator(51), + SyntheticPersonIdAllocator(np.int64(51)), + # A real ID above the cursor is safe when it is reserved: allocate() + # raises rather than hand it out (the large-native-ID fixture shape). + SyntheticPersonIdAllocator(10, frozenset({1, 2, 50})), + ], + ids=["above", "numpy-above", "reserved-above-cursor"], +) +def test_a_cursor_that_cannot_reach_a_known_id_is_accepted(allocator): + initial = _initial() + initial.loc[2, "person_id"] = 50 + esm.validate_projection_allocator( + initial["person_id"], {"synthetic_id_allocator": allocator} + ) + + +def test_metadata_without_an_allocator_is_accepted(): + esm.validate_projection_allocator(_initial()["person_id"], {}) + + +def test_a_non_allocator_is_refused(): + with pytest.raises(TypeError, match="SyntheticPersonIdAllocator"): + esm.validate_projection_allocator( + _initial()["person_id"], {"synthetic_id_allocator": object()} + ) diff --git a/tests/test_entrant_schedule.py b/tests/test_entrant_schedule.py new file mode 100644 index 00000000..a50dcf46 --- /dev/null +++ b/tests/test_entrant_schedule.py @@ -0,0 +1,617 @@ +"""Tests for the entrant schedule builder and its seam contract. + +The final test runs a real :class:`ProjectionEngine` over a built schedule. +That is the point of the piece: the seam already exists and is tested, so a +schedule builder is only correct if the loop accepts what it produces. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping + +import numpy as np +import pandas as pd +import pytest + +from populace_dynamics.engine import entrant_schedule as esm +from populace_dynamics.engine.entrant_domains import ( + EntrantClaimingAdapter, + suppress_entrant_benefit_outputs, +) +from populace_dynamics.engine.loop import ( + SCHEDULED_ENTRIES_KEY, + MaritalStepResult, + PeriodModules, + ProjectionEngine, + SyntheticPersonIdAllocator, +) +from populace_dynamics.engine.steps import advance_age + + +def _donor(n: int = 40, seed: int = 2) -> pd.DataFrame: + """An explicit donor pool. + + The schedule builder's contract is the donor COLUMNS, not the frame it + came from, so the fixture is built directly rather than sampled out of a + synthetic frame -- that keeps these tests independent of how thinly a + random frame happens to populate the recent-arrival band. + The native frame-reader path remains outside this source-only slice. + """ + rng = np.random.default_rng(seed) + return pd.DataFrame( + { + "person_id": np.arange(n), + "weight": rng.uniform(500.0, 5000.0, size=n), + "entry_age": rng.integers(0, 80, size=n).astype(np.int64), + "is_female": rng.random(n) < 0.5, + "source_year": np.full(n, 2024, dtype=np.int64), + "peinusyr": np.full(n, 28, dtype=np.int64), + "prcitshp": rng.choice([4, 5], size=n).astype(np.int64), + "penatvty": rng.integers(100, 556, size=n).astype(np.int64), + "foreign_born": np.ones(n, dtype=bool), + } + ) + + +def _schedule(years=(2026, 2027), start_id: int = 90_000_000, **kwargs): + donor = _donor() + inflow = {year: 1000.0 + 10 * index for index, year in enumerate(years)} + allocator = SyntheticPersonIdAllocator(start_id) + return ( + esm.build_entrant_schedule( + donor, inflow, allocator=allocator, **kwargs + ), + donor, + allocator, + ) + + +# -------------------------------------------------------------------------- +# The seam contract, which loop.py enforces +# -------------------------------------------------------------------------- +def test_frames_carry_the_year_before_activation(): + schedule, _, _ = _schedule(years=(2026, 2027)) + for year, frame in schedule.frames.items(): + assert set(frame["year"].unique()) == {year - 1} + + +def test_person_ids_are_unique_across_every_activation_year(): + schedule, _, _ = _schedule(years=(2026, 2027, 2028)) + pooled = pd.concat(schedule.frames.values(), ignore_index=True) + assert not pooled["person_id"].duplicated().any() + + +def test_ids_come_from_the_allocator_and_advance_it(): + schedule, donor, allocator = _schedule(years=(2026, 2027)) + assert allocator.next_id == 90_000_000 + 2 * len(donor) + pooled = pd.concat(schedule.frames.values(), ignore_index=True) + assert pooled["person_id"].min() >= 90_000_000 + + +def test_the_allocator_still_guards_the_reserved_real_namespace(): + donor = _donor() + allocator = SyntheticPersonIdAllocator( + 10, reserved_real_ids=frozenset({12}) + ) + with pytest.raises(RuntimeError, match="overlap the reserved real-person"): + esm.build_entrant_schedule(donor, {2026: 1000.0}, allocator=allocator) + + +def test_rows_are_person_sorted(): + schedule, _, _ = _schedule() + for frame in schedule.frames.values(): + assert frame["person_id"].is_monotonic_increasing + + +def test_birth_year_is_consistent_with_the_frame_coordinate(): + schedule, _, _ = _schedule(years=(2026,)) + frame = schedule.frames[2026] + assert ( + frame["birth_year"].to_numpy() + == frame["year"].to_numpy() - frame["age"].to_numpy() + ).all() + + +def test_sex_uses_the_roster_string_encoding(): + schedule, _, _ = _schedule() + for frame in schedule.frames.values(): + assert set(frame["sex"].unique()) <= {"female", "male"} + + +# -------------------------------------------------------------------------- +# Sizing +# -------------------------------------------------------------------------- +def test_cohort_weight_equals_the_control_in_persons(): + schedule, _, _ = _schedule(years=(2026,)) + record = schedule.alignment[2026] + assert record["target_weighted_persons"] == 1000.0 * 1000.0 + assert record["scheduled_weighted_persons"] == pytest.approx( + 1_000_000.0, rel=1e-12 + ) + assert abs(record["relative_residual"]) < 1e-12 + + +def test_reweighting_preserves_the_donor_composition_exactly(): + schedule, donor, _ = _schedule(years=(2026,)) + frame = schedule.frames[2026] + donor_share = donor["weight"].to_numpy() / donor["weight"].sum() + merged = frame.sort_values("donor_person_id") + scheduled_share = merged["weight"].to_numpy() / merged["weight"].sum() + expected = ( + donor.sort_values("person_id")["weight"].to_numpy() + / donor["weight"].sum() + ) + assert np.allclose(np.sort(donor_share), np.sort(scheduled_share)) + assert np.allclose(scheduled_share, expected) + + +def test_no_rng_is_consumed(): + """Two builds from independent allocators agree row for row.""" + donor = _donor() + first = esm.build_entrant_schedule( + donor, {2026: 1340.0}, allocator=SyntheticPersonIdAllocator(1_000) + ) + second = esm.build_entrant_schedule( + donor, {2026: 1340.0}, allocator=SyntheticPersonIdAllocator(1_000) + ) + pd.testing.assert_frame_equal(first.frames[2026], second.frames[2026]) + + +def test_negative_or_nonfinite_control_is_rejected(): + donor = _donor() + with pytest.raises(ValueError, match="negative control inflow"): + esm.build_entrant_schedule( + donor, {2026: -1.0}, allocator=SyntheticPersonIdAllocator(10) + ) + + +def test_empty_donor_is_rejected(): + donor = _donor().iloc[0:0] + with pytest.raises(ValueError, match="donor pool is empty"): + esm.build_entrant_schedule( + donor, {2026: 1.0}, allocator=SyntheticPersonIdAllocator(10) + ) + + +def test_no_activation_years_is_rejected(): + with pytest.raises(ValueError, match="no activation years"): + esm.build_entrant_schedule( + _donor(), {}, allocator=SyntheticPersonIdAllocator(10) + ) + + +# -------------------------------------------------------------------------- +# Provenance counters +# -------------------------------------------------------------------------- +def test_counters_read_the_entry_kind_column_not_id_arithmetic(): + schedule, donor, _ = _schedule(years=(2026, 2027)) + counters = esm.entrant_provenance_counters(schedule.frames) + assert counters["immigrant_cohorts"] == 2 * len(donor) + assert counters["n_rows_by_entry_kind"] == { + esm.ENTRY_KIND_IMMIGRANT: 2 * len(donor) + } + assert "not ID arithmetic" in counters["counter_basis"] + + +def test_counters_reject_a_frame_without_the_provenance_column(): + schedule, _, _ = _schedule(years=(2026,)) + frame = schedule.frames[2026].drop(columns=[esm.ENTRY_KIND_COLUMN]) + with pytest.raises(ValueError, match="entrant provenance cannot be"): + esm.entrant_provenance_counters({2026: frame}) + + +def test_provenance_states_the_sizing_basis_and_exclusions(): + schedule, _, _ = _schedule() + provenance = schedule.provenance + assert provenance["sizing_basis"] == "trustees_va2_gross_positive_inflow" + assert provenance["report_only"] is True + assert provenance["gated"] is False + joined = " ".join(provenance["sizing_excludes"]) + assert "emigration" in joined and "reclassification" in joined + + +def test_provenance_refuses_to_call_the_control_an_arrival_count(): + """The control is a stock-accounting proxy, and must say so. + + V.A2's temporary-or-unlawfully-present inflow counts only those who + remain to year-end, so the gross total is not a count of physical + arrivals. Naming the basis without that qualification would invite the + cohort to be read as arrivals, which is the same class of error as + reading an ssa_area_proxy as resident-aligned. + """ + schedule, _, _ = _schedule() + disclosure = schedule.provenance["sizing_basis_disclosure"] + assert "NOT a count of physical" in disclosure + assert "remain to year-end" in disclosure + + +# -------------------------------------------------------------------------- +# The loop actually accepts it +# -------------------------------------------------------------------------- +def test_a_built_schedule_activates_through_the_real_projection_engine(): + donor = _donor(n=200, seed=4) + allocator = SyntheticPersonIdAllocator(90_000_000) + schedule = esm.build_entrant_schedule( + donor, {2026: 1340.0, 2027: 1350.0}, allocator=allocator + ) + + def mortality(frame, context, rng): + del context, rng + return frame.sort_values("person_id").reset_index(drop=True) + + def marital(frame, context, rng): + del frame, context, rng + return MaritalStepResult(pd.DataFrame(), pd.DataFrame()) + + def passthrough(frame, context, rng): + del context, rng + return frame.copy() + + modules = PeriodModules( + mortality=mortality, + aging=advance_age, + marital_core=marital, + fertility=lambda frame, context, result, rng: frame.copy(), + disability=passthrough, + earnings=passthrough, + claiming=passthrough, + household_composition=( + lambda frame, context, result, rng: frame.copy() + ), + ) + initial = pd.DataFrame( + { + "person_id": [1, 2], + "year": [2025, 2025], + "age": [40, 41], + "sex": ["female", "male"], + "weight": [1.0, 1.0], + } + ) + result = ProjectionEngine(modules).project( + initial, + end_year=2027, + draw_index=0, + metadata={SCHEDULED_ENTRIES_KEY: schedule.as_metadata()}, + ) + + # 2 incumbents, then + one cohort in 2026, then + another in 2027. + assert [len(frame) for frame in result.slices] == [ + 2, + 2 + len(donor), + 2 + 2 * len(donor), + ] + final = result.slices[-1] + assert set(final["year"].unique()) == {2027} + assert not final["person_id"].duplicated().any() + entrants = final[final[esm.ENTRY_KIND_COLUMN].notna()] + assert len(entrants) == 2 * len(donor) + + +def test_the_entrant_faces_mortality_at_its_entry_age_before_aging(): + """The seam's fixed convention, pinned rather than assumed. + + A row scheduled with ``age = entry_age`` at ``year = y - 1`` is seen by + the mortality step at ``entry_age``; only then does aging advance it. + """ + donor = _donor(n=120, seed=6) + schedule = esm.build_entrant_schedule( + donor, {2026: 1000.0}, allocator=SyntheticPersonIdAllocator(90_000_000) + ) + seen: list[np.ndarray] = [] + + def mortality(frame, context, rng): + del context, rng + ordered = frame.sort_values("person_id").reset_index(drop=True) + seen.append(ordered["age"].to_numpy(dtype=np.int64).copy()) + return ordered + + def marital(frame, context, rng): + del frame, context, rng + return MaritalStepResult(pd.DataFrame(), pd.DataFrame()) + + def passthrough(frame, context, rng): + del context, rng + return frame.copy() + + modules = PeriodModules( + mortality=mortality, + aging=advance_age, + marital_core=marital, + fertility=lambda frame, context, result, rng: frame.copy(), + disability=passthrough, + earnings=passthrough, + claiming=passthrough, + household_composition=( + lambda frame, context, result, rng: frame.copy() + ), + ) + initial = pd.DataFrame( + { + "person_id": [1], + "year": [2025], + "age": [30], + "sex": ["female"], + "weight": [1.0], + } + ) + result = ProjectionEngine(modules).project( + initial, + end_year=2026, + draw_index=0, + metadata={SCHEDULED_ENTRIES_KEY: schedule.as_metadata()}, + ) + scheduled_ages = np.sort( + schedule.frames[2026]["age"].to_numpy(dtype=np.int64) + ) + # The mortality step saw the entry ages themselves, alongside the one + # incumbent (age 30). Compare the full multiset so an entrant aged 0 is + # not silently confused with the incumbent by position. + expected_seen = np.sort(np.concatenate([scheduled_ages, [30]])) + assert np.array_equal(np.sort(seen[0]), expected_seen) + # ...and the activation-year slice carries entry_age + 1. + final = result.slices[-1] + entrant_ages = np.sort( + final.loc[final[esm.ENTRY_KIND_COLUMN].notna(), "age"].to_numpy( + dtype=np.int64 + ) + ) + assert np.array_equal(entrant_ages, scheduled_ages + 1) + + +def test_zero_inflow_has_an_alignment_record_without_rows_or_ids(): + allocator = SyntheticPersonIdAllocator(90_000_000) + schedule = esm.build_entrant_schedule( + _donor(), {2026: 0.0, 2027: 0.0}, allocator=allocator + ) + assert schedule.frames == schedule.as_metadata() == {} + assert schedule.total_rows() == schedule.total_weight() == 0 + assert allocator.next_id == 90_000_000 + assert set(schedule.alignment) == {2026, 2027} + assert all(record["n_rows"] == 0 for record in schedule.alignment.values()) + assert schedule.provenance["zero_inflow_years"] == [2026, 2027] + assert schedule.provenance["scheduled_activation_years"] == [] + + +def test_zero_years_and_donors_do_not_change_later_cohort_ids(): + donor = _donor() + donor.loc[0, "weight"] = 0.0 + first = esm.build_entrant_schedule( + donor, + {2026: 0.0, 2027: 1000.0}, + allocator=SyntheticPersonIdAllocator(90_000_000), + ) + direct = esm.build_entrant_schedule( + donor.iloc[1:], + {2027: 1000.0}, + allocator=SyntheticPersonIdAllocator(90_000_000), + ) + assert set(first.frames) == {2027} + pd.testing.assert_frame_equal(first.frames[2027], direct.frames[2027]) + assert first.total_rows() == len(donor) - 1 + assert (first.frames[2027]["weight"] > 0).all() + + +@pytest.mark.parametrize("control", [-1.0, np.nan, np.inf, 1e308]) +def test_all_controls_are_checked_before_allocating_any_ids(control): + allocator = SyntheticPersonIdAllocator(1000) + with pytest.raises(ValueError, match="control inflow"): + esm.build_entrant_schedule( + _donor(), {2026: 1.0, 2027: control}, allocator=allocator + ) + assert allocator.next_id == 1000 + + +@pytest.mark.parametrize("year", [2026.5, "2026", True]) +def test_activation_years_are_not_silently_coerced(year): + allocator = SyntheticPersonIdAllocator(1000) + with pytest.raises(ValueError, match="activation years must be integers"): + esm.build_entrant_schedule(_donor(), {year: 1.0}, allocator=allocator) + assert allocator.next_id == 1000 + + +@pytest.mark.parametrize("dtype", [np.int64, np.int32, np.uint16]) +def test_numpy_activation_years_become_python_ints_in_the_audit_record(dtype): + """Controls keyed by a numpy/pandas year column stay JSON-serializable.""" + keys = np.array([2027, 2026], dtype=dtype) + controls = dict(zip(keys, [0.0, 1.0], strict=True)) + schedule = esm.build_entrant_schedule( + _donor(n=2), controls, allocator=SyntheticPersonIdAllocator(1000) + ) + records = { + "frames": list(schedule.frames), + "alignment": list(schedule.alignment), + "activation_years": schedule.provenance["activation_years"], + "scheduled_activation_years": schedule.provenance[ + "scheduled_activation_years" + ], + "zero_inflow_years": schedule.provenance["zero_inflow_years"], + } + for name, years in records.items(): + assert all(type(year) is int for year in years), name + assert records["activation_years"] == [2026, 2027] + assert records["scheduled_activation_years"] == [2026] + assert records["zero_inflow_years"] == [2027] + assert json.loads(json.dumps(schedule.alignment)).keys() == { + "2026", + "2027", + } + assert json.loads(json.dumps(schedule.provenance))["activation_years"] == [ + 2026, + 2027, + ] + assert set(schedule.frames[2026]["year"]) == {2025} + assert set(schedule.frames[2026]["entry_year"]) == {2026} + + +def test_a_year_supplied_twice_after_normalization_is_refused(): + """A pairs-backed mapping can hold 2026 and np.int64(2026) separately; + normalizing must not silently keep only one of the two controls.""" + + class _Pairs(Mapping): + def __init__(self, pairs): + self._pairs = list(pairs) + + def __getitem__(self, key): + for candidate, value in self._pairs: + if candidate is key: + return value + raise KeyError(key) + + def __iter__(self): + return (key for key, _ in self._pairs) + + def __len__(self): + return len(self._pairs) + + allocator = SyntheticPersonIdAllocator(1000) + with pytest.raises(ValueError, match="activation year 2026 is supplied"): + esm.build_entrant_schedule( + _donor(), + _Pairs([(2026, 1.0), (np.int64(2026), 2.0)]), + allocator=allocator, + ) + assert allocator.next_id == 1000 + + +@pytest.mark.parametrize("age", [-1, 10.5, np.nan, np.inf]) +def test_invalid_donor_ages_do_not_become_integer_demographic_states(age): + donor = _donor().astype({"entry_age": "float64"}) + donor.loc[0, "entry_age"] = age + with pytest.raises(ValueError, match="entry_age must be nonnegative"): + esm.build_entrant_schedule( + donor, {2026: 1.0}, allocator=SyntheticPersonIdAllocator(1000) + ) + + +@pytest.mark.parametrize("column", ["is_female", "foreign_born"]) +@pytest.mark.parametrize("value", ["False", pd.NA, 2]) +def test_donor_flags_are_not_truthiness_coerced(column, value): + donor = _donor().astype({column: "object"}) + donor.loc[0, column] = value + with pytest.raises(ValueError, match=f"{column} must contain booleans"): + esm.build_entrant_schedule( + donor, {2026: 1.0}, allocator=SyntheticPersonIdAllocator(1000) + ) + + +def test_a_mislabeled_immigrant_schedule_is_rejected(): + with pytest.raises(ValueError, match="require immigrant_cohort kind"): + esm.build_entrant_schedule( + _donor(), + {2026: 1.0}, + allocator=SyntheticPersonIdAllocator(1000), + entry_kind=esm.ENTRY_KIND_BIRTH, + ) + + +@pytest.mark.parametrize("kind", [None, "immgrant_cohort"]) +def test_provenance_counts_reject_missing_or_unknown_kinds(kind): + schedule, _, _ = _schedule(years=(2026,)) + schedule.frames[2026].loc[0, esm.ENTRY_KIND_COLUMN] = kind + with pytest.raises(ValueError, match="missing or unknown entry_kind"): + esm.entrant_provenance_counters(schedule.frames) + + +@pytest.mark.parametrize("weight,inflow", [(1e-300, 1e300), (1e300, 1e-300)]) +def test_unrepresentable_scaling_fails_before_cohort_allocation( + weight, inflow +): + donor = _donor(n=1) + donor["weight"] = weight + allocator = SyntheticPersonIdAllocator(1000) + with pytest.raises(ValueError, match="finite positive donor weights"): + esm.build_entrant_schedule(donor, {2026: inflow}, allocator=allocator) + assert allocator.next_id == 1000 + + +def test_large_integer_donor_weights_use_the_validated_float_sum(): + donor = _donor(n=2) + donor["weight"] = np.array([2**62, 2**62], dtype=np.int64) + schedule = esm.build_entrant_schedule( + donor, {2026: 1.0}, allocator=SyntheticPersonIdAllocator(1000) + ) + assert schedule.frames[2026]["weight"].tolist() == [500.0, 500.0] + assert schedule.alignment[2026]["residual_persons"] == 0.0 + + +def test_positive_subnormal_scale_does_not_distort_the_control(): + donor = _donor(n=1) + donor["weight"] = 2.0**1023 + allocator = SyntheticPersonIdAllocator(1000) + schedule = esm.build_entrant_schedule( + donor, {2026: 1.0, 2027: 2.0**-60}, allocator=allocator + ) + assert schedule.frames[2026]["weight"].iloc[0] == 1000.0 + assert schedule.frames[2027]["weight"].iloc[0] == 1000.0 * 2.0**-60 + assert schedule.alignment[2027]["residual_persons"] == 0.0 + assert allocator.next_id == 1002 + + +def test_later_cohorts_activate_after_extinction_with_unknown_benefits(): + """Real loop mechanics with synthetic survival and benefit placeholders. + + The placeholder values test output suppression only; no statutory benefit + calculator or native fitted transition is exercised. + """ + donor = _donor(n=3) + allocator = SyntheticPersonIdAllocator(1000) + schedule = esm.build_entrant_schedule( + donor, {2026: 0.0, 2027: 1.0, 2028: 2.0}, allocator=allocator + ) + seen = {} + + def mortality(frame, context, rng): + seen[context.year] = frame.copy() + if context.year == 2026: + return frame.iloc[:0].copy() + # A later cohort must activate even when the previous cohort dies. + if context.year == 2028: + return frame.loc[frame["entry_year"] == 2028].copy() + return frame.copy() + + def forbidden_claiming(*args): + raise AssertionError("unsupported entrants reached claiming behavior") + + def passthrough(frame, context, rng): + return frame.copy() + + def marital(frame, context, rng): + return MaritalStepResult(frame.copy(), pd.DataFrame()) + + modules = PeriodModules( + mortality=mortality, + aging=advance_age, + marital_core=marital, + fertility=lambda f, c, m, r: f.copy(), + disability=passthrough, + earnings=lambda f, c, r: f.assign(aime=10.0, pia=20.0, benefit=30.0), + claiming=EntrantClaimingAdapter( + lambda f, c, r: f if f.empty else forbidden_claiming(f, c, r) + ), + household_composition=lambda f, c, m, r: suppress_entrant_benefit_outputs( + f + ), + ) + initial = pd.DataFrame( + {"person_id": [1], "year": [2025], "age": [30], "sex": ["female"]} + ) + result = ProjectionEngine(modules).project( + initial, + end_year=2028, + draw_index=0, + metadata={SCHEDULED_ENTRIES_KEY: schedule.as_metadata()}, + ) + assert [len(frame) for frame in result.slices] == [1, 0, 3, 3] + assert allocator.next_id == 1006 + for year in (2027, 2028): + observed = seen[year].loc[seen[year]["entry_year"] == year] + assert observed["age"].tolist() == donor["entry_age"].tolist() + frame = result.slices[year - 2025] + assert (frame["age"] == frame["entry_age"] + 1).all() + assert frame[["aime", "pia", "benefit"]].isna().all().all() + assert frame["claim_age"].isna().all() + assert not frame["claimed"].any() + assert frame["weight"].sum() == pytest.approx((year - 2026) * 1000.0) + assert set(result.slices[-1]["person_id"]) == {1003, 1004, 1005} diff --git a/tests/test_forward_earnings_history.py b/tests/test_forward_earnings_history.py new file mode 100644 index 00000000..ae3002ff --- /dev/null +++ b/tests/test_forward_earnings_history.py @@ -0,0 +1,303 @@ +"""Invented projection records; no fitting, benefit or population scoring.""" + +import copy +import json +import pickle +from dataclasses import FrozenInstanceError, replace + +import numpy as np +import pandas as pd +import pytest + +from populace_dynamics.forward_earnings_history import ForwardEarningsHistory +from populace_dynamics.person_identity import PersonIdentity, PersonIdentityMap + + +def mapping(): + return PersonIdentityMap.from_identities( + [PersonIdentity("uint64", 2**64 - 1), PersonIdentity("string", "001")] + ) + + +def frame(year=2014, values=(0.0, 0.0)): + return pd.DataFrame( + { + "person_id": [0, 1], + "year": [year, year], + "earnings": np.asarray(values, dtype=np.float64), + "earnings_domain": [True, False], + } + ) + + +def start(data=None, **kwargs): + return ForwardEarningsHistory.start( + mapping(), + frame() if data is None else data, + realization_id="invented-career-draw-1", + generator_digest="a" * 64, + source_contract_digest="b" * 64, + lineage_digest="c" * 64, + unit="XTS", + price_basis="nominal", + **kwargs, + ) + + +def test_zero_and_outside_domain_control_zero_remain_distinct(): + history = start() + known, missing = history.observations + assert known.amount_state == "known_zero" + assert known.amount_hex == float(0).hex() + assert missing.amount_state == "unavailable" + assert missing.amount_hex is None + assert missing.missing_reason == "outside_forward_earnings_domain" + assert history.identity_map.reverse_rows([1])[0].value == 2**64 - 1 + assert history.source_registry_status == "registration_required" + assert history.coverage_status == "not_materialized" + + +def test_dense_append_keeps_realization_original_values_and_input_frame(): + initial = frame(values=(0.1, 0.0)) + original = initial.copy(deep=True) + first = start(initial) + old_json = first.to_json() + second = first.append(frame(2015, (0.1, 0.0)), lineage_digest="d" * 64) + third = second.append(frame(2016, (0.2, 0.0)), lineage_digest="e" * 64) + pd.testing.assert_frame_equal(initial, original) + assert first.to_json() == old_json + assert third.observations[:4] == second.observations + rows = third.for_person(PersonIdentity("string", "001")) + assert [row.year for row in rows] == [2014, 2015, 2016] + assert [row.generation_method for row in rows] == [ + "boundary_method", + "odd_year_carry", + "biennial_draw", + ] + assert rows[0].amount_hex == rows[1].amount_hex == (0.1).hex() + assert rows[2].amount_hex == (0.2).hex() + assert third.realization_id == first.realization_id + restored = ForwardEarningsHistory.from_json( + third.to_json(), expected_digest=third.digest + ) + assert restored == third + assert restored.to_json() == third.to_json() + assert ( + ForwardEarningsHistory.from_json(third.to_json(), previous=second) + == third + ) + + +def test_row_order_and_unrelated_private_lag_do_not_change_record(): + data = frame(values=(100.0, 0.0)) + data["realized_earn_2012"] = [np.nan, np.nan] + assert start(data) == start(data.iloc[::-1].copy()) + assert start(data).observations[0].amount_hex == (100.0).hex() + + +@pytest.mark.parametrize("value", [np.nan, np.inf, -np.inf, -1.0]) +def test_invalid_supported_earnings_are_never_padded_or_floored(value): + with pytest.raises(ValueError): + start(frame(values=(value, 0.0))) + + +@pytest.mark.parametrize( + "column,values", + [ + ("person_id", [0.0, 1.0]), + ("person_id", [0, 0]), + ("person_id", [0, 2]), + ("year", [2014.0, 2014.0]), + ("year", [2014, 2015]), + ("earnings_domain", [True, None]), + ("earnings_domain", [1, 0]), + ("earnings", [0, 0]), + ], +) +def test_ambiguous_frame_contract_refuses(column, values): + data = frame() + data[column] = values + with pytest.raises(ValueError): + start(data) + + +def test_outside_domain_nonzero_is_not_discarded(): + with pytest.raises(ValueError, match="control zero"): + start(frame(values=(0.0, 4.0))) + + +@pytest.mark.parametrize("year", [2014, 2016, 2023]) +def test_append_requires_exact_next_year(year): + with pytest.raises(ValueError): + start().append(frame(year), lineage_digest="d" * 64) + + +def test_carry_must_preserve_exact_source_bits(): + first = start(frame(values=(-0.0, 0.0))) + with pytest.raises(ValueError, match="carry"): + first.append(frame(2015), lineage_digest="d" * 64) + assert first.observations[0].amount_state == "known_zero" + assert first.observations[0].amount_hex == (-0.0).hex() + + +def test_roster_or_domain_change_requires_a_separate_contract(): + first = start() + with pytest.raises(ValueError, match="roster"): + first.append(frame(2015).iloc[:1], lineage_digest="d" * 64) + changed = frame(2015) + changed["earnings_domain"] = [False, False] + with pytest.raises(ValueError, match="domain"): + first.append(changed, lineage_digest="d" * 64) + + +def test_projection_envelope_cannot_silently_extend_past_2022(): + history = start() + for year in range(2015, 2023): + history = history.append(frame(year), lineage_digest="d" * 64) + with pytest.raises(ValueError, match="2022"): + history.append(frame(2023), lineage_digest="d" * 64) + + +def test_loaded_missing_or_duplicate_years_refuse(): + history = start().append(frame(2015), lineage_digest="d" * 64) + with pytest.raises(ValueError): + replace(history, observations=history.observations[:-1]) + with pytest.raises(ValueError): + replace(history, observations=history.observations * 2) + with pytest.raises(ValueError): + replace(history, observations=history.observations[:2]) + with pytest.raises(ValueError): + replace(history, observations=history.observations[::2]) + + +def test_serialized_tampering_and_unknown_claims_refuse(): + history = start() + document = json.loads(history.to_json()) + document["coverage_status"] = "resolved" + with pytest.raises(ValueError): + ForwardEarningsHistory.from_json(json.dumps(document)) + document = json.loads(history.to_json()) + document["observations"][0]["amount_hex"] = (100.0).hex() + with pytest.raises(ValueError, match="digest"): + ForwardEarningsHistory.from_json( + json.dumps(document), expected_digest=history.digest + ) + with pytest.raises(ValueError, match="duplicate"): + ForwardEarningsHistory.from_json('{"schema":"a","schema":"b"}') + + +def test_history_is_frozen_and_observation_inputs_are_snapshotted(): + history = start() + with pytest.raises(FrozenInstanceError): + history.realization_id = "different-career" + with pytest.raises(FrozenInstanceError): + history.observations[0].amount_hex = (100.0).hex() + + +@pytest.mark.parametrize( + "copier", + [ + pytest.param( + lambda value: pickle.loads(pickle.dumps(value, protocol=5)), + id="pickle", + ), + pytest.param(copy.deepcopy, id="deepcopy"), + ], +) +def test_history_round_trips_through_pickle_and_deepcopy(copier): + history = start(frame(values=(0.1, 0.0))).append( + frame(2015, (0.1, 0.0)), lineage_digest="d" * 64 + ) + restored = copier(history) + assert restored == history + assert restored.to_json() == history.to_json() + assert restored.digest == history.digest + assert restored.identity_map.digest == history.identity_map.digest + person = PersonIdentity("uint64", 2**64 - 1) + assert restored.for_person(person) == history.for_person(person) + restored.require_extension_of(history) + successor = restored.append( + frame(2016, (0.2, 0.0)), lineage_digest="e" * 64 + ) + successor.require_extension_of(history) + + +def test_person_id_column_is_read_as_private_keys_not_native_ids(): + """Invented native IDs 0..10: lexical admission gives native 10 key 2. + + The recorder reads ``person_id`` as this map's private key, so a caller + holding native IDs must translate them with ``map_rows`` first. + """ + identities = [PersonIdentity("int64", x) for x in range(11)] + identity_map = PersonIdentityMap.from_identities(identities) + assert identity_map.map_rows([PersonIdentity("int64", 10)]) == (2,) + data = pd.DataFrame( + { + "person_id": np.asarray( + identity_map.map_rows(identities), dtype=np.int64 + ), + "year": np.full(11, 2014, dtype=np.int64), + "earnings": np.asarray([1000.0 * x for x in range(11)]), + "earnings_domain": np.ones(11, dtype=bool), + } + ) + history = ForwardEarningsHistory.start( + identity_map, + data, + realization_id="invented-native-translation", + generator_digest="a" * 64, + source_contract_digest="b" * 64, + lineage_digest="c" * 64, + unit="XTS", + price_basis="nominal", + ) + for native, identity in enumerate(identities): + (row,) = history.for_person(identity) + assert float.fromhex(row.amount_hex) == 1000.0 * native + + +def test_loaded_successor_cannot_change_realization_or_erase_old_rows(): + first = start() + second = first.append(frame(2015), lineage_digest="d" * 64) + with pytest.raises(ValueError, match="provenance"): + ForwardEarningsHistory.from_json( + replace(second, realization_id="another-draw").to_json(), + previous=first, + ) + with pytest.raises(ValueError, match="prior rows"): + first.require_extension_of(second) + + +def test_identity_digest_cannot_be_null_in_document(): + document = json.loads(start().to_json()) + document["identity_map_digest"] = None + with pytest.raises(ValueError, match="identity map digest"): + ForwardEarningsHistory.from_json(json.dumps(document)) + + +def test_loaded_annual_snapshot_cannot_mix_lineage_digests(): + document = json.loads(start().to_json()) + document["observations"][1]["lineage_digest"] = "f" * 64 + with pytest.raises(ValueError, match="one lineage digest"): + ForwardEarningsHistory.from_json(json.dumps(document)) + + +def test_duplicate_frame_columns_refuse(): + data = frame() + with pytest.raises(ValueError, match="columns"): + start(pd.concat([data, data[["earnings"]]], axis=1)) + + +@pytest.mark.parametrize( + "field,value", + [ + ("generator_digest", "unknown"), + ("source_contract_digest", ""), + ("realization_id", ""), + ("unit", ""), + ("price_basis", "unknown"), + ], +) +def test_metadata_is_explicit(field, value): + with pytest.raises(ValueError): + replace(start(), **{field: value}) diff --git a/tests/test_graph_mortality.py b/tests/test_graph_mortality.py new file mode 100644 index 00000000..16f9d25a --- /dev/null +++ b/tests/test_graph_mortality.py @@ -0,0 +1,649 @@ +"""Synthetic engineering tests for the optional population graph.""" + +import copy +import json +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from populace_dynamics.graph.model import MortalityArtifact, fit_mortality +from populace_dynamics.graph.synthetic import write_synthetic_inputs + + +@pytest.fixture +def inputs(tmp_path): + return write_synthetic_inputs(tmp_path / "inputs") + + +@pytest.fixture +def runtime(): + from populace_dynamics.graph._compat import require_graph + + try: + require_graph() + except ImportError as error: + pytest.skip(str(error)) + from populace_dynamics.graph import runtime + + return runtime + + +def _read(path): + return json.loads(path.read_text()) + + +def _write(path, value): + path.write_text(json.dumps(value)) + + +def _fit(inputs): + return fit_mortality( + pd.DataFrame(_read(inputs["training"])), + pd.DataFrame(_read(inputs["rates"])), + boundary_year=2014, + external_vintage_year=2014, + ) + + +def _run(runtime, inputs, tmp_path, **kwargs): + return runtime.run_mortality_graph( + **inputs, output_dir=tmp_path / "output", **kwargs + ) + + +def test_existing_mortality_fit_and_json_roundtrip(inputs): + artifact = _fit(inputs) + assert artifact.fit_rows == 8 + assert artifact.model.probability[("0+", "female")] == pytest.approx( + -np.expm1(-0.25) + ) + restored = MortalityArtifact.from_bytes(artifact.to_bytes()) + assert restored.to_bytes() == artifact.to_bytes() + assert restored.model == artifact.model + + +def test_future_exposure_and_interview_information_are_excluded(inputs): + baseline = _fit(inputs) + training = _read(inputs["training"]) + training[-1]["death"] = 0.0 + _write(inputs["training"], training) + assert _fit(inputs).to_bytes() == baseline.to_bytes() + with pytest.raises(ValueError, match="vintage"): + fit_mortality( + pd.DataFrame(training), + pd.DataFrame(_read(inputs["rates"])), + boundary_year=2013, + external_vintage_year=2014, + ) + + +@pytest.mark.parametrize("mutation", ["schema", "duplicate", "missing", "nan"]) +def test_model_payload_fails_closed(inputs, mutation): + raw = json.loads(_fit(inputs).to_bytes()) + if mutation == "schema": + raw["schema_version"] = 99 + elif mutation == "duplicate": + raw["probabilities"].append(copy.deepcopy(raw["probabilities"][0])) + elif mutation == "missing": + raw["probabilities"].pop() + else: + raw["probabilities"][0]["probability"] = float("nan") + with pytest.raises(ValueError): + MortalityArtifact.from_bytes(json.dumps(raw).encode()) + + +def test_model_rejects_duplicate_json_members(inputs): + payload = ( + _fit(inputs) + .to_bytes() + .replace( + b'"schema_version":1', b'"schema_version":1,"schema_version":1' + ) + ) + with pytest.raises(ValueError, match="duplicate"): + MortalityArtifact.from_bytes(payload) + + +def test_graph_reuses_fit_and_reports_period_mass(runtime, inputs, tmp_path): + cold = _run(runtime, inputs, tmp_path) + warm = _run(runtime, inputs, tmp_path) + assert cold.report["scope"] == "synthetic_engineering" + assert cold.report["fixture_verdict"] == "pass" + assert cold.report["engineering_verdict"] == "pass" + assert all(node.hit for node in warm.manifest.nodes.values()) + assert cold.model_payload == warm.model_payload + assert cold.report["mass"] == warm.report["mass"] + mass = cold.report["mass"] + assert mass["after"] == mass["before"] + mass["next_period"] + assert mass["partition"]["stratum_before"]["2014"] == ( + mass["partition"]["stratum_after"]["2014"] + ) + assert (tmp_path / "output" / "report.json").is_file() + assert (tmp_path / "output" / "manifest.json").is_file() + + +def test_graph_matches_existing_steps_with_explicit_uniforms( + runtime, inputs, tmp_path +): + from populace_dynamics.engine.steps import ( + advance_age, + apply_mortality, + ) + + result = _run(runtime, inputs, tmp_path) + initial = pd.DataFrame(_read(inputs["initial"])).sort_values("person_id") + initial["year"] = 2014 + uniforms = runtime.mortality_uniforms(initial.person_id.tolist()) + + class FixedUniforms: + def random(self, n): + assert n == len(uniforms) + return uniforms.copy() + + context = SimpleNamespace(rng_registry=None, year=2015, metadata={}) + model = MortalityArtifact.from_bytes(result.model_payload).model + survived = apply_mortality(initial, context, FixedUniforms(), model=model) + assert ( + survived.person_id.tolist() + == initial.loc[ + uniforms >= model.probabilities(initial), "person_id" + ].tolist() + ) + expected = advance_age(survived, context, np.random.default_rng(0)) + actual = result.next_slice.sort_values("person_id") + pd.testing.assert_frame_equal( + actual[["person_id", "age", "year"]].reset_index(drop=True), + expected[["person_id", "age", "year"]].reset_index(drop=True), + ) + + +def test_holdout_changes_only_evaluation_and_can_fail_fixture( + runtime, inputs, tmp_path +): + baseline = _run(runtime, inputs, tmp_path) + holdout = _read(inputs["holdout"]) + for row in holdout["outcomes"]: + row["death"] = 1 + _write(inputs["holdout"], holdout) + changed = _run(runtime, inputs, tmp_path) + assert changed.report["fixture_verdict"] == "fail" + assert changed.report["engineering_verdict"] == "pass" + assert changed.model_payload == baseline.model_payload + for node_id in ( + "training", + "fit", + "initial", + "apply", + "advance", + "age", + "outcomes", + ): + assert changed.manifest.nodes[node_id].hit + assert not changed.manifest.nodes["evaluate"].hit + pd.testing.assert_frame_equal(baseline.next_slice, changed.next_slice) + + +def test_recipient_edit_keeps_model_fit(runtime, inputs, tmp_path): + baseline = _run(runtime, inputs, tmp_path) + initial = _read(inputs["initial"]) + initial[0]["age"] += 1 + _write(inputs["initial"], initial) + changed = _run(runtime, inputs, tmp_path) + assert changed.manifest.nodes["fit"].hit + assert changed.model_payload == baseline.model_payload + assert not changed.manifest.nodes["apply"].hit + + +def test_training_weight_edit_invalidates_fit_and_application( + runtime, inputs, tmp_path +): + baseline = _run(runtime, inputs, tmp_path) + training = _read(inputs["training"]) + training[0]["start_weight"] = 8.0 + _write(inputs["training"], training) + changed = _run(runtime, inputs, tmp_path) + assert not changed.manifest.nodes["fit"].hit + assert not changed.manifest.nodes["apply"].hit + assert changed.model_payload != baseline.model_payload + + +def test_stable_draws_are_row_chunk_and_unrelated_person_invariant(runtime): + first = runtime.mortality_uniforms([11, 21, 31]) + np.testing.assert_array_equal( + runtime.mortality_uniforms([31, 11, 21]), first[[2, 0, 1]] + ) + np.testing.assert_array_equal( + runtime.mortality_uniforms([1, 11, 21, 31])[1:], first + ) + np.testing.assert_array_equal( + np.concatenate( + [ + runtime.mortality_uniforms([11]), + runtime.mortality_uniforms([21, 31]), + ] + ), + first, + ) + + +@pytest.mark.parametrize("identity", [None, 1.5, "1", True]) +def test_mortality_draws_reject_noninteger_person_ids(runtime, identity): + with pytest.raises(ValueError, match="identities"): + runtime.mortality_uniforms([identity]) + + +def test_population_reorder_and_unrelated_person_preserve_survivors( + runtime, inputs, tmp_path +): + baseline = _run(runtime, inputs, tmp_path) + initial = _read(inputs["initial"]) + _write(inputs["initial"], initial[::-1]) + reordered = _run(runtime, inputs, tmp_path) + pd.testing.assert_frame_equal(baseline.next_slice, reordered.next_slice) + initial.append({"person_id": 1, "age": 44, "sex": "male", "weight": 2.0}) + _write(inputs["initial"], initial) + holdout = _read(inputs["holdout"]) + holdout["outcomes"].append( + {"person_id": 1, "year": 2015, "age": 45, "death": 0} + ) + _write(inputs["holdout"], holdout) + extended = _run(runtime, inputs, tmp_path) + pd.testing.assert_frame_equal( + baseline.next_slice, + extended.next_slice.query("person_id != 1").reset_index(drop=True), + ) + + +@pytest.mark.parametrize("all_die", [False, True]) +def test_empty_or_complete_survivor_expansion( + runtime, inputs, tmp_path, all_die +): + training = _read(inputs["training"]) + for row in training: + row["death"] = 1.0 if all_die else 0.0 + row["exposure"] = 1e-9 if all_die else 1.0 + _write(inputs["training"], training) + result = _run(runtime, inputs, tmp_path) + warm = _run(runtime, inputs, tmp_path) + assert warm.manifest.nodes["advance"].hit + assert len(result.next_slice) == (0 if all_die else 20) + assert result.report["mass"]["next_period"] == ( + 0.0 if all_die else result.report["mass"]["before"] + ) + if all_die: + assert ( + "2015" not in result.report["mass"]["partition"]["stratum_after"] + ) + + +def test_household_accounting_is_explicitly_unsupported( + runtime, inputs, tmp_path +): + with pytest.raises(ValueError, match="household"): + _run(runtime, inputs, tmp_path, household_accounting=True) + + +def test_optional_entrypoint_has_actionable_missing_capability(monkeypatch): + from populace_dynamics.graph import _compat + + monkeypatch.setattr(_compat, "_python_version", lambda: (3, 12)) + with pytest.raises(ImportError, match="Python >=3.13"): + _compat.require_graph() + + +def test_optional_entrypoint_reports_old_core_capabilities(monkeypatch): + from populace_dynamics.graph import _compat + + monkeypatch.setattr(_compat, "_python_version", lambda: (3, 14)) + monkeypatch.setattr( + _compat.importlib, "import_module", lambda name: SimpleNamespace() + ) + with pytest.raises(ImportError, match="typed model-artifact"): + _compat.require_graph() + + +def test_cli_requires_output_directory(): + from populace_dynamics.graph.__main__ import parser + + with pytest.raises(SystemExit): + parser().parse_args(["--synthetic"]) + + +def test_keyed_kernel_hashes_random_coordinate_encoding(runtime, monkeypatch): + from microcosm.graph import canonical + + _, registry = runtime.build_graph() + kernel = registry.get("dynamics.mortality.apply@1") + assert kernel.capabilities.dependencies == ("numpy", "pandas") + before = kernel.implementation_hash() + encoding_source = Path(canonical.__file__).resolve() + original = Path.read_bytes + + def changed_source(path): + payload = original(path) + if path.resolve() == encoding_source: + payload += b"\n# coordinate encoding change\n" + return payload + + monkeypatch.setattr(Path, "read_bytes", changed_source) + assert kernel.implementation_hash() != before + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + ("missing_person", "held-out identities must match"), + ("wrong_period", "held-out outcomes must be binary deaths"), + ("invalid_json", "invalid mortality JSON"), + # 2**64 - 1 once wrapped to year -1 and failed only by accident. + ("uint64_year", "signed 64-bit"), + ], +) +def test_failed_holdout_exports_named_gate_evidence_cold_and_warm( + runtime, inputs, tmp_path, mutation, message, capsys +): + from populace_dynamics.graph.__main__ import main + + holdout = _read(inputs["holdout"]) + if mutation == "missing_person": + holdout["outcomes"].pop() + elif mutation == "wrong_period": + holdout["outcomes"][0]["year"] = 2014 + elif mutation == "uint64_year": + holdout["outcomes"][0]["year"] = 2**64 - 1 + _write(inputs["holdout"], holdout) + if mutation == "invalid_json": + inputs["holdout"].write_text("{invalid JSON") + + for warm in (False, True): + result = _run(runtime, inputs, tmp_path) + gate = result.manifest.nodes["evaluate"] + assert gate.hit is warm + assert gate.receipt["outcome"] == "fail" + assert "report" not in gate.opaque_artifacts + diagnostic = result.report["evaluation_gate"] + assert diagnostic["node_id"] == "evaluate" + assert diagnostic["kernel_ref"] == "dynamics.mortality.evaluate@1" + assert diagnostic["outcome"] == "fail" + assert diagnostic["evidence"]["exception_type"] == "ValueError" + assert message in diagnostic["evidence"]["message"] + assert result.report["engineering_verdict"] == "not_evaluated" + assert result.report["fixture_verdict"] == "not_evaluated" + assert "expected_deaths" not in result.report + assert "mass" not in result.report + assert _read(tmp_path / "output" / "report.json") == result.report + assert _read(tmp_path / "output" / "manifest.json") == json.loads( + result.manifest.to_json() + ) + + arguments = ["--output-dir", str(tmp_path / "output")] + for name, path in inputs.items(): + arguments.extend([f"--{name}", str(path)]) + assert main(arguments) == 1 + assert "not_evaluated" in capsys.readouterr().out + assert ( + message + in _read(tmp_path / "output" / "report.json")["evaluation_gate"][ + "evidence" + ]["message"] + ) + + +@pytest.mark.parametrize("excluded_person_id", [1009, 1010]) +def test_graph_fit_matches_direct_cutoff_and_ignores_excluded_outcomes( + runtime, inputs, tmp_path, excluded_person_id +): + """Future events and late interviews cannot leak through graph mapping.""" + baseline = _run(runtime, inputs, tmp_path) + direct = _fit(inputs) + assert baseline.model_payload == direct.to_bytes() + assert direct.fit_rows == 8 + training = _read(inputs["training"]) + excluded = next( + row for row in training if row["person_id"] == excluded_person_id + ) + assert ( + excluded["event_year"] > 2014 + or excluded["required_interview_year"] > 2014 + ) + excluded["death"] = 1.0 - excluded["death"] + excluded["start_weight"] = 50.0 + excluded["exposure"] = 0.25 + _write(inputs["training"], training) + changed = _run(runtime, inputs, tmp_path) + assert not changed.manifest.nodes["training"].hit + assert not changed.manifest.nodes["fit"].hit + assert changed.model_payload == _fit(inputs).to_bytes() + assert changed.model_payload == baseline.model_payload + pd.testing.assert_frame_equal(changed.next_slice, baseline.next_slice) + + +@pytest.mark.parametrize( + "coordinates", + [ + {"experiment_id": "independent-stream"}, + {"replicate": 7}, + {"base_seed": 831}, + ], +) +def test_nondefault_stream_matches_direct_steps_and_reuses_fit( + runtime, inputs, tmp_path, coordinates +): + """An accidentally hard-coded default cannot pass both sides of parity.""" + from microcosm.graph.randomness import keyed_uniform + + from populace_dynamics.engine.steps import advance_age, apply_mortality + + baseline = _run(runtime, inputs, tmp_path) + changed = _run(runtime, inputs, tmp_path, **coordinates) + assert changed.manifest.nodes["fit"].hit + assert ( + changed.manifest.nodes["fit"].key == baseline.manifest.nodes["fit"].key + ) + assert changed.model_payload == baseline.model_payload + assert not changed.manifest.nodes["apply"].hit + assert ( + changed.manifest.nodes["apply"].key + != baseline.manifest.nodes["apply"].key + ) + assert changed.report["engineering_verdict"] == "pass" + + initial = pd.DataFrame(_read(inputs["initial"])).sort_values("person_id") + initial["year"] = 2014 + # Derive the oracle directly from the registered coordinate contract, + # independently of both mortality_uniforms and _GraphPeriodContext. + stream = ( + "sha256-u53-v1", + coordinates.get("experiment_id", "mortality"), + coordinates.get("replicate", 0), + coordinates.get("base_seed", 0), + ) + keys = [(int(pid), "mortality", 2015, 0) for pid in initial.person_id] + uniforms = keyed_uniform(stream=stream, keys=keys) + default = keyed_uniform( + stream=("sha256-u53-v1", "mortality", 0, 0), keys=keys + ) + assert not np.array_equal(uniforms, default) + + class FixedUniforms: + def random(self, n): + assert n == len(uniforms) + return uniforms.copy() + + context = SimpleNamespace(rng_registry=None, year=2015, metadata={}) + # Use the independent direct model, not the graph's returned artifact. + model = _fit(inputs).model + survivors = apply_mortality(initial, context, FixedUniforms(), model=model) + expected = advance_age(survivors, context, np.random.default_rng(0)) + pd.testing.assert_frame_equal( + changed.next_slice[["person_id", "age", "year"]].reset_index( + drop=True + ), + expected[["person_id", "age", "year"]].reset_index(drop=True), + ) + assert ( + changed.next_slice.person_id.tolist() + != baseline.next_slice.person_id.tolist() + ) + + +def test_next_period_rows_carry_no_undrawn_mortality_outcome( + runtime, inputs, tmp_path +): + """Admitted survivor rows are not drawn; their outcome cells stay null. + + Copying the parent's ``survives=True`` and period-B probability onto the + period-B+1 rows once overstated pooled survival (30/35 instead of 15/20) + and attached a stale age band's probability to anyone who aged across it. + """ + result = _run(runtime, inputs, tmp_path) + assert result.report["engineering_verdict"] == "pass" + model = MortalityArtifact.from_bytes(result.model_payload).model + exported = pd.read_csv( + tmp_path / "output" / "person_period.csv", + float_precision="round_trip", + ) + in_memory = result.manifest.population("advance").table(runtime.OBS) + for table in (exported, in_memory): + start = table.loc[table[runtime.PERIOD_ID] == 2014] + future = table.loc[table[runtime.PERIOD_ID] == 2015] + assert len(start) == result.report["initial_records"] == 20 + assert len(future) == result.report["survivor_records"] == 15 + assert start.survives.notna().all() + np.testing.assert_array_equal( + start.death_probability.to_numpy(dtype=np.float64), + model.probabilities(start), + ) + assert future.survives.isna().all() + assert future.death_probability.isna().all() + drawn = table.survives.dropna().astype(bool) + assert len(drawn) == 20 + assert int(drawn.sum()) == result.report["survivor_records"] + + +def test_integer_column_refuses_rather_than_wraps_uint64(runtime): + frame = pd.DataFrame({"person_id": [1, 2**64 - 1]}) + assert frame.person_id.dtype == np.uint64 + with pytest.raises(ValueError, match="signed 64-bit"): + runtime._integer_column(frame, "person_id") + with pytest.raises(ValueError, match="signed 64-bit"): + runtime._integer_column( + pd.DataFrame({"year": [-(2**63) - 1, 2014]}), "year" + ) + bounds = pd.DataFrame({"person_id": [-(2**63), 1, 2**63 - 1]}) + runtime._integer_column(bounds, "person_id") + assert bounds.person_id.dtype == np.int64 + assert bounds.person_id.tolist() == [-(2**63), 1, 2**63 - 1] + + +@pytest.mark.parametrize( + ("source", "row", "column", "value"), + [ + # Row 8 is the 2015-dated event the fit cutoff must exclude; wrapped + # to -1 it passed the <= boundary check and entered the fit. + ("training", 8, "event_year", 2**64 - 1), + ("training", 0, "person_id", 2**63), + ("training", 0, "required_interview_year", -(2**63) - 1), + ("initial", 0, "person_id", 2**64 - 1), + ("initial", 0, "age", 2**64 - 1), + ], +) +def test_source_integers_outside_int64_are_refused( + runtime, inputs, tmp_path, source, row, column, value +): + from microcosm.graph.errors import NodeRejectedError + + rows = _read(inputs[source]) + if column == "event_year": + assert rows[row]["event_year"] == 2015 + rows[row][column] = value + _write(inputs[source], rows) + with pytest.raises(NodeRejectedError, match="signed 64-bit"): + _run(runtime, inputs, tmp_path) + + +@pytest.mark.parametrize( + "value", ["7.5", True, np.True_, None, [1.0], {"weight": 1.0}] +) +def test_weights_refuse_json_strings_booleans_and_nonnumbers(runtime, value): + with pytest.raises(ValueError, match="JSON numbers"): + runtime._weights([1.0, value]) + with pytest.raises(ValueError, match="JSON numbers"): + runtime._weights(pd.Series([value, value], dtype=object)) + + +def test_weights_accept_real_numbers_and_refuse_overflow(runtime): + weights = runtime._weights( + [1, 2.5, np.float64(3.0), np.int64(4), np.float32(0.5)] + ) + np.testing.assert_array_equal(weights.values, [1.0, 2.5, 3.0, 4.0, 0.5]) + assert weights.values.dtype == np.float64 + with pytest.raises(ValueError, match="finite"): + runtime._weights([1.0, 10**400]) + + +@pytest.mark.parametrize( + ("source", "column", "value", "every_row"), + [ + ("initial", "weight", "7.5", False), + ("initial", "weight", True, False), + ("training", "start_weight", "3", True), + ("training", "start_weight", True, False), + ("training", "exposure", "1.0", False), + ("training", "exposure", True, False), + ("training", "death", True, False), + ("training", "death", "1", False), + ], +) +def test_source_numbers_refuse_json_strings_and_booleans( + runtime, inputs, tmp_path, source, column, value, every_row +): + from microcosm.graph.errors import NodeRejectedError + + rows = _read(inputs[source]) + for row in rows if every_row else rows[:1]: + row[column] = value + _write(inputs[source], rows) + with pytest.raises(NodeRejectedError, match="JSON numbers"): + _run(runtime, inputs, tmp_path) + + +def test_integer_json_weights_remain_valid_numbers(runtime, inputs, tmp_path): + initial = _read(inputs["initial"]) + initial[0]["weight"] = 2 + _write(inputs["initial"], initial) + training = _read(inputs["training"]) + training[0]["exposure"] = 1 + training[1]["death"] = 0 + _write(inputs["training"], training) + result = _run(runtime, inputs, tmp_path) + assert result.report["engineering_verdict"] == "pass" + population = result.manifest.population("initial") + weights = population.weights_for(runtime.OBS).values + assert weights.dtype == np.float64 + assert weights[0] == 2.0 + + +def test_evaluation_fails_if_admitted_rows_claim_a_mortality_outcome( + runtime, inputs, tmp_path, monkeypatch +): + """The engineering verdict guards the undrawn next-period outcome.""" + claim = runtime._outcome_claim + + def inherited_claim(context): + result = claim(context) + survives = result.columns[(runtime.OBS, "survives")] + probability = result.columns[(runtime.OBS, "death_probability")] + return runtime.KernelResult( + columns={ + (runtime.OBS, "survives"): survives.fillna(True), + (runtime.OBS, "death_probability"): probability.fillna(0.25), + } + ) + + monkeypatch.setattr(runtime, "_outcome_claim", inherited_claim) + result = _run(runtime, inputs, tmp_path) + assert result.report["engineering_verdict"] == "fail" diff --git a/tests/test_graph_mortality_trajectory.py b/tests/test_graph_mortality_trajectory.py new file mode 100644 index 00000000..fb087c45 --- /dev/null +++ b/tests/test_graph_mortality_trajectory.py @@ -0,0 +1,691 @@ +"""Independent synthetic engineering checks for the annual mortality DAG. + +All sources are generated in the test directory. Aggregate fixtures below +are deliberately hand specified and are neither native population evidence +nor scientific acceptance thresholds. +""" + +import json +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from populace_dynamics.graph.model import fit_mortality +from populace_dynamics.graph.synthetic import write_synthetic_inputs + +TRAJECTORY_COLUMNS = ["person_id", "age", "year", "weight"] + + +@pytest.fixture +def runtime(): + from populace_dynamics.graph._compat import require_graph + + try: + require_graph() + except ImportError as error: + pytest.skip(str(error)) + from populace_dynamics.graph import run_mortality_trajectory + + return run_mortality_trajectory + + +def _read(path): + return json.loads(path.read_text()) + + +def _write(path, value): + path.write_text(json.dumps(value)) + + +@pytest.fixture +def inputs(tmp_path): + sources = write_synthetic_inputs(tmp_path / "inputs") + sources.pop("holdout") + holdouts = {} + for year in range(2015, 2019): + path = tmp_path / "inputs" / f"aggregate-{year}.json" + _write( + path, + { + "scope": "synthetic_engineering", + "year": year, + "expected_death_rate": 0.2, + "fixture_max_abs_death_rate_gap": 0.25, + }, + ) + holdouts[year] = path + return {**sources, "holdouts": holdouts} + + +def _run(runtime, inputs, tmp_path, *, end_year=2017, **kwargs): + sources = { + **inputs, + "holdouts": { + year: path + for year, path in inputs["holdouts"].items() + if year <= end_year + }, + } + return runtime( + **sources, + end_year=end_year, + output_dir=tmp_path / "output", + **kwargs, + ) + + +def _ordered(frame): + return ( + frame[TRAJECTORY_COLUMNS] + .sort_values(["year", "person_id"]) + .reset_index(drop=True) + ) + + +def _direct_projection(inputs, *, end_year=2017, **coordinates): + """Use the original fit/steps and an independently assembled RNG key.""" + from microcosm.graph.randomness import keyed_uniform + + from populace_dynamics.engine.steps import advance_age, apply_mortality + + artifact = fit_mortality( + pd.DataFrame(_read(inputs["training"])), + pd.DataFrame(_read(inputs["rates"])), + boundary_year=2014, + external_vintage_year=2014, + ) + stream = ( + "sha256-u53-v1", + coordinates.get("experiment_id", "mortality"), + coordinates.get("replicate", 0), + coordinates.get("base_seed", 0), + ) + current = pd.DataFrame(_read(inputs["initial"])).sort_values("person_id") + current["year"] = 2014 + history = [current[TRAJECTORY_COLUMNS].copy()] + diagnostics = {} + for year in range(2015, end_year + 1): + current = current.sort_values("person_id").reset_index(drop=True) + uniforms = keyed_uniform( + stream=stream, + keys=[ + (int(pid), "mortality", year, 0) for pid in current.person_id + ], + ) + + class FixedUniforms: + def __init__(self, values): + self.values = values + + def random(self, n): + assert n == len(self.values) + return self.values.copy() + + context = SimpleNamespace(rng_registry=None, year=year, metadata={}) + survived = apply_mortality( + current, context, FixedUniforms(uniforms), model=artifact.model + ) + future = advance_age(survived, context, np.random.default_rng(0)) + probability = artifact.model.probabilities(current) + diagnostics[str(year)] = { + "from_year": year - 1, + "year": year, + "initial_records": len(current), + "survivor_records": len(future), + "expected_deaths": float(np.dot(current.weight, probability)), + "generated_deaths": float( + current.loc[ + ~current.person_id.isin(future.person_id), "weight" + ].sum() + ), + "start_mass": float(current.weight.sum()), + "next_period_mass": float(future.weight.sum()), + } + history.append(future[TRAJECTORY_COLUMNS].copy()) + current = future + return artifact.to_bytes(), _ordered(pd.concat(history)), diagnostics + + +def _set_death_regime(inputs, *, all_die): + training = _read(inputs["training"]) + for row in training: + row["death"] = 1.0 if all_die else 0.0 + row["exposure"] = 1e-9 if all_die else 1.0 + _write(inputs["training"], training) + for path in inputs["holdouts"].values(): + holdout = _read(path) + holdout["expected_death_rate"] = 1.0 if all_die else 0.0 + _write(path, holdout) + + +@pytest.mark.parametrize( + "coordinates", + [ + {}, + {"experiment_id": "trajectory-alternative"}, + {"replicate": 7}, + {"base_seed": 831}, + ], +) +def test_annual_projection_matches_independent_steps_and_weighted_diagnostics( + runtime, inputs, tmp_path, coordinates +): + result = _run(runtime, inputs, tmp_path, **coordinates) + payload, expected, diagnostics = _direct_projection(inputs, **coordinates) + assert result.model_payload == payload + pd.testing.assert_frame_equal(_ordered(result.trajectory), expected) + assert result.report["scope"] == "synthetic_engineering" + assert result.report["engineering_verdict"] == "pass" + assert result.report["fixture_verdict"] == "pass" + assert set(result.report["periods"]) == {"2015", "2016", "2017"} + for year, expected_period in diagnostics.items(): + actual = result.report["periods"][year] + for field, value in expected_period.items(): + assert actual[field] == pytest.approx(value), (year, field) + assert actual["engineering_verdict"] == "pass" + assert actual["fixture_verdict"] == "pass" + assert actual["start_mass"] == pytest.approx( + actual["generated_deaths"] + actual["next_period_mass"] + ) + + +def test_expansion_preserves_every_historical_row_and_person_link( + runtime, inputs, tmp_path +): + result = _run(runtime, inputs, tmp_path) + previous = result.manifest.population("initial").table("person_period") + for year in range(2015, 2018): + population = result.manifest.population(f"advance_{year}") + observations = population.table("person_period") + assert observations.person_period_id.is_unique + assert not observations.duplicated( + ["person_period_person_id", "person_period_period_id"] + ).any() + retained = observations.loc[ + observations.person_period_id.isin(previous.person_period_id), + previous.columns, + ] + pd.testing.assert_frame_equal( + retained.sort_values("person_period_id").reset_index(drop=True), + previous.sort_values("person_period_id").reset_index(drop=True), + ) + entrants = observations.loc[ + observations.person_period_period_id == year + ] + at_risk = previous.loc[ + previous.person_period_period_id == year - 1 + ].set_index("person_period_person_id") + assert set(entrants.person_period_person_id) <= set(at_risk.index) + for row in entrants.itertuples(index=False): + parent = at_risk.loc[row.person_period_person_id] + assert row.age == parent.age + 1 + assert row.sex == parent.sex + previous = observations + assert set(previous.person_period_period_id) == {2014, 2015, 2016, 2017} + + +def test_cold_warm_cache_and_exported_artifacts(runtime, inputs, tmp_path): + cold = _run(runtime, inputs, tmp_path) + warm = _run(runtime, inputs, tmp_path) + assert not any(node.hit for node in cold.manifest.nodes.values()) + assert all(node.hit for node in warm.manifest.nodes.values()) + assert type(warm.manifest).__module__.startswith("microcosm.graph") + assert cold.model_payload == warm.model_payload + assert cold.report["periods"] == warm.report["periods"] + pd.testing.assert_frame_equal(cold.trajectory, warm.trajectory) + output = tmp_path / "output" + assert _read(output / "report.json") == warm.report + assert _read(output / "manifest.json") == json.loads( + warm.manifest.to_json() + ) + assert (output / "model.json").read_bytes() == warm.model_payload + pd.testing.assert_frame_equal( + _ordered(pd.read_csv(output / "trajectory.csv")), + _ordered(warm.trajectory), + ) + + +def test_horizon_extension_reuses_fit_and_existing_annual_nodes( + runtime, inputs, tmp_path +): + short = _run(runtime, inputs, tmp_path, end_year=2015) + extended = _run(runtime, inputs, tmp_path, end_year=2018) + for name, node in short.manifest.nodes.items(): + assert extended.manifest.nodes[name].hit, name + assert extended.manifest.nodes[name].key == node.key, name + for year in range(2016, 2019): + for prefix in ("apply", "advance", "age", "evaluate"): + assert not extended.manifest.nodes[f"{prefix}_{year}"].hit + assert short.model_payload == extended.model_payload + pd.testing.assert_frame_equal( + _ordered(short.trajectory), + _ordered(extended.trajectory.query("year <= 2015")), + ) + + +@pytest.mark.parametrize( + "coordinates", + [ + {"experiment_id": "trajectory-alternative"}, + {"replicate": 7}, + {"base_seed": 831}, + ], +) +def test_stream_change_reuses_fit_but_invalidates_each_application( + runtime, inputs, tmp_path, coordinates +): + original = _run(runtime, inputs, tmp_path) + changed = _run(runtime, inputs, tmp_path, **coordinates) + for name in ("training", "fit", "initial"): + assert changed.manifest.nodes[name].hit + assert ( + changed.manifest.nodes[name].key + == original.manifest.nodes[name].key + ) + for year in range(2015, 2018): + name = f"apply_{year}" + assert not changed.manifest.nodes[name].hit + assert ( + changed.manifest.nodes[name].key + != original.manifest.nodes[name].key + ) + assert changed.model_payload == original.model_payload + _, expected, _ = _direct_projection(inputs, **coordinates) + pd.testing.assert_frame_equal(_ordered(changed.trajectory), expected) + + +def test_holdout_change_invalidates_only_its_own_evaluation( + runtime, inputs, tmp_path +): + original = _run(runtime, inputs, tmp_path) + holdout = _read(inputs["holdouts"][2016]) + holdout["expected_death_rate"] = 1.0 + holdout["fixture_max_abs_death_rate_gap"] = 0.0 + _write(inputs["holdouts"][2016], holdout) + changed = _run(runtime, inputs, tmp_path) + for name, node in changed.manifest.nodes.items(): + if name == "evaluate_2016": + assert not node.hit + assert node.key != original.manifest.nodes[name].key + else: + assert node.hit, name + assert node.key == original.manifest.nodes[name].key, name + assert changed.report["engineering_verdict"] == "pass" + assert changed.report["fixture_verdict"] == "fail" + assert changed.report["periods"]["2016"]["fixture_verdict"] == "fail" + for year in ("2015", "2017"): + assert ( + changed.report["periods"][year] == original.report["periods"][year] + ) + assert changed.model_payload == original.model_payload + pd.testing.assert_frame_equal(changed.trajectory, original.trajectory) + + +def test_recipient_change_reuses_the_fitted_model(runtime, inputs, tmp_path): + original = _run(runtime, inputs, tmp_path) + initial = _read(inputs["initial"]) + initial[0]["age"] += 1 + initial[0]["weight"] *= 2 + _write(inputs["initial"], initial) + changed = _run(runtime, inputs, tmp_path) + assert changed.manifest.nodes["fit"].hit + assert changed.model_payload == original.model_payload + assert not changed.manifest.nodes["initial"].hit + assert not changed.manifest.nodes["apply_2015"].hit + _, expected, _ = _direct_projection(inputs) + pd.testing.assert_frame_equal(_ordered(changed.trajectory), expected) + + +def test_training_change_refits_and_reapplies(runtime, inputs, tmp_path): + original = _run(runtime, inputs, tmp_path) + training = _read(inputs["training"]) + training[0]["start_weight"] = 8.0 + _write(inputs["training"], training) + changed = _run(runtime, inputs, tmp_path) + assert not changed.manifest.nodes["fit"].hit + assert not changed.manifest.nodes["apply_2015"].hit + assert changed.model_payload != original.model_payload + payload, expected, _ = _direct_projection(inputs) + assert changed.model_payload == payload + pd.testing.assert_frame_equal(_ordered(changed.trajectory), expected) + + +def test_complete_extinction_leaves_no_future_period_groups( + runtime, inputs, tmp_path +): + _set_death_regime(inputs, all_die=True) + result = _run(runtime, inputs, tmp_path) + warm = _run(runtime, inputs, tmp_path) + assert all(node.hit for node in warm.manifest.nodes.values()) + assert set(result.trajectory.year) == {2014} + first = result.report["periods"]["2015"] + assert first["generated_deaths"] == first["start_mass"] + assert first["survivor_records"] == 0 + assert first["next_period_mass"] == 0 + for year in (2016, 2017): + period = result.report["periods"][str(year)] + for field in ( + "initial_records", + "survivor_records", + "expected_deaths", + "generated_deaths", + "start_mass", + "next_period_mass", + ): + assert period[field] == 0, (year, field) + assert period["engineering_verdict"] == "not_applicable" + assert period["fixture_verdict"] == "not_applicable" + for year in range(2015, 2018): + population = result.manifest.population(f"advance_{year}") + assert population.table("period").period.tolist() == [2014] + assert set( + population.table("person_period").person_period_period_id + ) == {2014} + + +def test_zero_mortality_keeps_all_people_and_each_periods_mass( + runtime, inputs, tmp_path +): + _set_death_regime(inputs, all_die=False) + result = _run(runtime, inputs, tmp_path) + initial = pd.DataFrame(_read(inputs["initial"])) + for year in range(2014, 2018): + period = result.trajectory.loc[result.trajectory.year == year] + assert set(period.person_id) == set(initial.person_id) + assert period.weight.sum() == initial.weight.sum() + _, expected, _ = _direct_projection(inputs) + pd.testing.assert_frame_equal(_ordered(result.trajectory), expected) + for period in result.report["periods"].values(): + assert period["expected_deaths"] == 0 + assert period["generated_deaths"] == 0 + assert period["fixture_verdict"] == "pass" + + +def test_missing_annual_holdout_fails_closed(runtime, inputs, tmp_path): + inputs["holdouts"].pop(2016) + with pytest.raises(ValueError, match="holdout"): + _run(runtime, inputs, tmp_path) + + +@pytest.mark.parametrize("mutation", ["year", "scope", "rate", "json"]) +def test_bad_holdout_retains_gate_diagnostics_without_changing_simulation( + runtime, inputs, tmp_path, mutation +): + original = _run(runtime, inputs, tmp_path) + path = inputs["holdouts"][2016] + holdout = _read(path) + if mutation == "year": + holdout["year"] = 2015 + elif mutation == "scope": + holdout["scope"] = "scientific_acceptance" + elif mutation == "rate": + holdout["expected_death_rate"] = float("nan") + _write(path, holdout) + if mutation == "json": + path.write_text("{broken JSON") + for cached_failure in (False, True): + result = _run(runtime, inputs, tmp_path) + gate = result.manifest.nodes["evaluate_2016"] + assert gate.hit is cached_failure + assert gate.receipt["outcome"] == "fail" + diagnostic = result.report["periods"]["2016"]["evaluation_gate"] + assert diagnostic["node_id"] == "evaluate_2016" + assert diagnostic["outcome"] == "fail" + assert diagnostic["evidence"]["exception_type"] == "ValueError" + assert diagnostic["evidence"]["message"] + assert result.report["fixture_verdict"] == "not_evaluated" + for name, node in result.manifest.nodes.items(): + if name != "evaluate_2016": + assert node.hit, name + assert node.key == original.manifest.nodes[name].key + pd.testing.assert_frame_equal(result.trajectory, original.trajectory) + assert _read(tmp_path / "output" / "report.json") == result.report + assert _read(tmp_path / "output" / "manifest.json") == json.loads( + result.manifest.to_json() + ) + + +def test_unsupported_age_stops_future_application_and_preserves_evidence( + runtime, inputs, tmp_path +): + _set_death_regime(inputs, all_die=False) + initial = _read(inputs["initial"]) + initial[0]["age"] = 120 + _write(inputs["initial"], initial) + short = _run(runtime, inputs, tmp_path, end_year=2015) + assert short.trajectory.query("year == 2015").age.max() == 121 + for cached_failure in (False, True): + result = _run(runtime, inputs, tmp_path) + gate = result.manifest.nodes["apply_2016"] + assert gate.hit is cached_failure + assert gate.receipt["outcome"] == "fail" + assert set(gate.opaque_artifacts) == {"transition"} + assert gate.receipt["application_status"] == "failed" + diagnostic = result.report["periods"]["2016"]["application_gate"] + assert diagnostic["node_id"] == "apply_2016" + assert diagnostic["outcome"] == "fail" + assert diagnostic["evidence"]["exception_type"] == "ValueError" + assert "age" in diagnostic["evidence"]["message"].lower() + # The exact pinned core executes guarded descendants. Their native + # receipts stay honest; application-level blocked is not unreached. + for name in ("advance_2016", "apply_2017", "advance_2017"): + assert ( + result.manifest.nodes[name].receipt["application_status"] + == "blocked" + ) + for year in (2016, 2017): + assert ( + result.manifest.nodes[f"evaluate_{year}"].receipt["outcome"] + == "evidence_absent" + ) + period = result.report["periods"][str(year)] + assert period["completed_year"] == 2015 + assert period["engineering_verdict"] == "not_evaluated" + population = result.manifest.population(f"advance_{year}") + pd.testing.assert_frame_equal( + population.table("person_period"), + short.manifest.population("advance_2015").table( + "person_period" + ), + ) + assert result.report["periods"]["2017"]["application_status"] == ( + "blocked" + ) + assert result.report["completed_year"] == 2015 + assert result.report["engineering_verdict"] == "not_evaluated" + assert result.model_payload == short.model_payload + pd.testing.assert_frame_equal(result.trajectory, short.trajectory) + assert _read(tmp_path / "output" / "report.json") == result.report + assert _read(tmp_path / "output" / "manifest.json") == json.loads( + result.manifest.to_json() + ) + + +def test_household_accounting_remains_explicitly_unsupported( + runtime, inputs, tmp_path +): + with pytest.raises(ValueError, match="household"): + _run(runtime, inputs, tmp_path, household_accounting=True) + + +def test_known_fixture_failure_survives_later_missing_evaluation( + runtime, inputs, tmp_path +): + failed_fixture = _read(inputs["holdouts"][2015]) + failed_fixture["expected_death_rate"] = 1.0 + failed_fixture["fixture_max_abs_death_rate_gap"] = 0.0 + _write(inputs["holdouts"][2015], failed_fixture) + inputs["holdouts"][2016].write_text("{broken JSON") + result = _run(runtime, inputs, tmp_path) + assert result.report["periods"]["2015"]["fixture_verdict"] == "fail" + assert result.report["periods"]["2016"]["fixture_verdict"] == ( + "not_evaluated" + ) + assert result.report["periods"]["2017"]["engineering_verdict"] == "pass" + assert result.report["fixture_verdict"] == "fail" + assert result.report["execution_status"] == "failed" + assert result.report["completed_year"] == 2017 + + +def test_zero_risk_set_cannot_certify_an_unexpected_future_observation( + runtime, inputs, tmp_path, monkeypatch +): + from dataclasses import replace + + from populace_dynamics.graph import trajectory + + _set_death_regime(inputs, all_die=True) + original_snapshot = trajectory._snapshot + + def snapshot_with_unexpected_future_row(context): + result = original_snapshot(context) + if context.params["year"] != 2016: + return result + payload = json.loads(result.artifacts["snapshot"]) + record = dict(payload["observations"][0]) + record["person_period_id"] = 999 + record["person_period_period_id"] = 2016 + record["age"] += 2 + payload["observations"].append(record) + payload["periods"].append({"period_id": 2016, "period": 2016}) + payload["weights"].append(1.0) + return replace( + result, artifacts={"snapshot": json.dumps(payload).encode()} + ) + + monkeypatch.setattr( + trajectory, "_snapshot", snapshot_with_unexpected_future_row + ) + result = _run(runtime, inputs, tmp_path) + period = result.report["periods"]["2016"] + assert period["initial_records"] == 0 + assert period["survivor_records"] == 1 + assert period["engineering_verdict"] == "fail" + assert period["fixture_verdict"] == "not_applicable" + assert result.manifest.nodes["evaluate_2016"].receipt["outcome"] == "fail" + assert result.report["engineering_verdict"] == "fail" + + +@pytest.mark.parametrize( + "payload_kind", ["malformed_json", "unknown_status", "before_boundary"] +) +def test_malformed_prerequisite_does_not_invent_completed_year( + runtime, monkeypatch, payload_kind +): + from populace_dynamics.graph import trajectory + + def unexpected_application(context): + raise AssertionError("invalid prerequisite reached model application") + + monkeypatch.setattr(trajectory, "_apply_complete", unexpected_application) + payload = b"{broken JSON" + if payload_kind != "malformed_json": + payload = json.dumps( + { + "format": "populace-dynamics.mortality-transition", + "schema_version": 1, + "from_year": 2015, + "year": 2016, + "status": ( + "unknown" if payload_kind == "unknown_status" else "failed" + ), + "completed_year": ( + 2013 if payload_kind == "before_boundary" else 2015 + ), + "records": [], + "diagnostic": {"message": "prior failure"}, + } + ).encode() + context = SimpleNamespace( + params={"year": 2017, "boundary_year": 2014}, + artifacts={"previous_transition": SimpleNamespace(payload=payload)}, + ) + result = trajectory._apply(context) + outcome = json.loads(result.artifacts["transition"]) + assert outcome["status"] == "failed" + assert outcome["completed_year"] == 2014 + assert outcome["records"] == [] + assert outcome["diagnostic"]["exception_type"] == "ValueError" + assert result.receipt["outcome"] == "fail" + + +def test_blocked_years_do_not_apply_laws_or_parse_holdouts( + runtime, inputs, tmp_path, monkeypatch +): + from populace_dynamics.graph import trajectory + + _set_death_regime(inputs, all_die=False) + initial = _read(inputs["initial"]) + initial[0]["age"] = 120 + _write(inputs["initial"], initial) + for year in (2016, 2017): + # The executor still hashes these bytes, but guarded evaluations + # must not parse them after application fails in 2016. + inputs["holdouts"][year].write_text("{unparseable held-out fixture") + + application_years, ageing_years, holdout_years = [], [], [] + apply_complete = trajectory._apply_complete + advance_age = trajectory.rt.advance_age + holdout = trajectory._holdout + + def tracked_application(context): + application_years.append(context.params["year"]) + assert context.params["year"] <= 2016 + return apply_complete(context) + + def tracked_ageing(frame, context, rng): + ageing_years.append(context.year) + assert context.year == 2015 + return advance_age(frame, context, rng) + + def tracked_holdout(context): + holdout_years.append(context.params["year"]) + assert context.params["year"] == 2015 + return holdout(context) + + monkeypatch.setattr(trajectory, "_apply_complete", tracked_application) + monkeypatch.setattr(trajectory.rt, "advance_age", tracked_ageing) + monkeypatch.setattr(trajectory, "_holdout", tracked_holdout) + result = _run(runtime, inputs, tmp_path) + assert application_years == [2015, 2016] + assert ageing_years == [2015] + assert holdout_years == [2015] + assert result.report["completed_year"] == 2015 + assert ( + result.report["periods"]["2016"]["application_gate"]["evidence"][ + "exception_type" + ] + == "ValueError" + ) + assert result.report["periods"]["2017"]["application_status"] == ( + "blocked" + ) + + +@pytest.mark.parametrize( + ("source", "column", "value", "every_row", "message"), + [ + # Once wrapped to -1 and exported under that identity. + ("initial", "person_id", 2**64 - 1, False, "signed 64-bit"), + # Once parsed as 7.5 and 1.0 population weights. + ("initial", "weight", "7.5", False, "JSON numbers"), + ("initial", "weight", True, False, "JSON numbers"), + # Once parsed into the fitted law's exposure weights. + ("training", "start_weight", "3", True, "JSON numbers"), + ("training", "exposure", "1.0", False, "JSON numbers"), + ], +) +def test_malformed_source_numbers_are_refused_not_coerced( + runtime, inputs, tmp_path, source, column, value, every_row, message +): + from microcosm.graph.errors import NodeRejectedError + + rows = _read(inputs[source]) + for row in rows if every_row else rows[:1]: + row[column] = value + _write(inputs[source], rows) + with pytest.raises(NodeRejectedError, match=message): + _run(runtime, inputs, tmp_path, end_year=2015) + assert not (tmp_path / "output" / "trajectory.csv").exists() diff --git a/tests/test_graph_trajectory_accounting.py b/tests/test_graph_trajectory_accounting.py new file mode 100644 index 00000000..47d4e9ab --- /dev/null +++ b/tests/test_graph_trajectory_accounting.py @@ -0,0 +1,641 @@ +"""Invented-source checks of accounting on the actual optional graph.""" + +import copy +import json +import math +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import pandas as pd +import pytest + +from populace_dynamics.graph.synthetic import write_synthetic_inputs + + +@pytest.fixture +def runtime(): + from populace_dynamics.graph._compat import require_graph + + try: + require_graph() + except ImportError as error: + pytest.skip(str(error)) + from populace_dynamics.graph import trajectory_accounting + + return trajectory_accounting + + +def _read(path): + return json.loads(path.read_text()) + + +def _write(path, value): + path.write_text(json.dumps(value)) + + +@pytest.fixture +def inputs(tmp_path): + sources = write_synthetic_inputs(tmp_path / "inputs") + sources.pop("holdout") + holdouts = {} + for year in range(2015, 2019): + path = tmp_path / "inputs" / f"annual-{year}.json" + _write( + path, + { + "scope": "synthetic_engineering", + "year": year, + "expected_death_rate": 0.2, + "fixture_max_abs_death_rate_gap": 0.25, + }, + ) + holdouts[year] = path + return {**sources, "holdouts": holdouts} + + +def _sources(inputs, end_year): + return { + **inputs, + "holdouts": { + year: path + for year, path in inputs["holdouts"].items() + if year <= end_year + }, + } + + +def _run(runtime, inputs, tmp_path, *, end_year=2017, **kwargs): + return runtime.run_accounted_mortality_trajectory( + **_sources(inputs, end_year), + end_year=end_year, + output_dir=tmp_path / "output", + **kwargs, + ) + + +def _regime(inputs, *, all_die): + training = _read(inputs["training"]) + for row in training: + row["death"] = 1.0 if all_die else 0.0 + row["exposure"] = 1e-9 if all_die else 1.0 + _write(inputs["training"], training) + + +def _artifact(runtime, result, tmp_path, node, name): + store = runtime.ContentStore(tmp_path / "output" / "store") + return runtime.parse_json( + store.load_bytes(result.manifest.nodes[node].opaque_artifacts[name]) + ) + + +def _context(runtime, result, tmp_path, year=2015): + return SimpleNamespace( + params={"year": year, "boundary_year": 2014}, + sources={}, + artifacts={ + name: SimpleNamespace( + payload=runtime.json_bytes( + _artifact(runtime, result, tmp_path, node, name) + ) + ) + for name, node in ( + ("transition", f"apply_{year}"), + ("snapshot", f"snapshot_{year}"), + ) + }, + ) + + +def test_actual_graph_preserves_original_keys_populations_and_evaluations( + runtime, inputs, tmp_path +): + original = runtime.trajectory.run_mortality_trajectory( + **_sources(inputs, 2017), + end_year=2017, + output_dir=tmp_path / "output", + ) + result = _run(runtime, inputs, tmp_path) + for name, node in original.manifest.nodes.items(): + actual = result.manifest.nodes[name] + assert actual.hit, name + assert actual.key == node.key, name + assert actual.receipt == node.receipt, name + assert result.report["accounting_status"] == "complete" + for year in range(2015, 2018): + name = f"advance_{year}" + old = original.manifest.population(name) + new = result.manifest.population(name) + pd.testing.assert_frame_equal( + old.table("person_period"), new.table("person_period") + ) + assert not result.manifest.nodes[f"account_{year}"].hit + period = result.report["periods"][str(year)] + assert period["application_status"] == "complete" + assert period["completed_year"] == year + account = period["account"] + assert account["status"] == runtime.accounting.ENGINEERING_STATUS + transition = _artifact( + runtime, result, tmp_path, f"apply_{year}", "transition" + ) + deaths = [row for row in transition["records"] if not row["survives"]] + assert account["counts"]["exits_total"] == len(deaths) + assert account["counts"]["exits_by_kind"]["death"] == len(deaths) + assert account["counts"]["additions_total"] == 0 + weights = account["weights"] + assert weights["revaluation"]["total"] == 0 + assert weights["closing"] == math.fsum( + (weights["opening"], -weights["exits_total"]) + ) + assert account["residuals"]["count"] == 0 + assert account["residuals"]["weight"] == 0 + assert _read(tmp_path / "output" / "accounting-report.json") == ( + result.report + ) + assert _read(tmp_path / "output" / "manifest.json") == json.loads( + result.manifest.to_json() + ) + # Existing outputs from the original runner were not overwritten. + assert _read(tmp_path / "output" / "report.json") == original.report + + +def test_cold_warm_and_horizon_extension(runtime, inputs, tmp_path): + cold = _run(runtime, inputs, tmp_path, end_year=2015) + warm = _run(runtime, inputs, tmp_path, end_year=2015) + assert not any(node.hit for node in cold.manifest.nodes.values()) + assert all(node.hit for node in warm.manifest.nodes.values()) + assert cold.report == warm.report + extended = _run(runtime, inputs, tmp_path, end_year=2018) + for name, node in warm.manifest.nodes.items(): + assert extended.manifest.nodes[name].hit, name + assert extended.manifest.nodes[name].key == node.key, name + assert extended.report["periods"]["2015"] == cold.report["periods"]["2015"] + assert not extended.manifest.nodes["account_2018"].hit + + +@pytest.mark.parametrize("malformed", [False, True]) +def test_holdout_changes_only_its_evaluation( + runtime, inputs, tmp_path, malformed +): + original = _run(runtime, inputs, tmp_path) + path = inputs["holdouts"][2016] + if malformed: + path.write_text("{invalid fixture JSON") + else: + holdout = _read(path) + holdout["expected_death_rate"] = 1.0 + holdout["fixture_max_abs_death_rate_gap"] = 0.0 + _write(path, holdout) + changed = _run(runtime, inputs, tmp_path) + for name, node in changed.manifest.nodes.items(): + if name == "evaluate_2016": + assert not node.hit + assert node.receipt["outcome"] == "fail" + else: + assert node.hit, name + assert node.key == original.manifest.nodes[name].key, name + assert changed.report == original.report + + +@pytest.mark.parametrize("all_die", [False, True]) +def test_extinction_and_zero_mortality(runtime, inputs, tmp_path, all_die): + _regime(inputs, all_die=all_die) + result = _run(runtime, inputs, tmp_path) + for year in range(2015, 2018): + period = result.report["periods"][str(year)] + assert period["accounting_status"] == "complete" + account = period["account"] + counts = account["counts"] + expected_opening = 0 if all_die and year > 2015 else 20 + assert counts["opening"] == expected_opening + assert counts["closing"] == (0 if all_die else 20) + assert counts["exits_total"] == (expected_opening if all_die else 0) + assert account["opening_year"] == year - 1 + assert account["closing_year"] == year + assert account["residuals"]["count"] == 0 + if all_die: + population = result.manifest.population("advance_2017") + assert population.table("period").period.tolist() == [2014] + + +def test_failed_and_blocked_application_never_reconciles_stale_snapshots( + runtime, inputs, tmp_path, monkeypatch +): + _regime(inputs, all_die=False) + initial = _read(inputs["initial"]) + initial[0]["age"] = 120 + _write(inputs["initial"], initial) + original = runtime.accounting.reconcile_period + calls = [] + + def tracked(*args, **kwargs): + calls.append(kwargs["closing_year"]) + return original(*args, **kwargs) + + monkeypatch.setattr(runtime.accounting, "reconcile_period", tracked) + cold = _run(runtime, inputs, tmp_path) + warm = _run(runtime, inputs, tmp_path) + assert calls == [2015] + assert cold.report == warm.report + assert all(node.hit for node in warm.manifest.nodes.values()) + assert cold.report["accounting_status"] == "not_evaluated" + for year, status in ((2016, "failed"), (2017, "blocked")): + period = cold.report["periods"][str(year)] + assert period["application_status"] == status + assert period["completed_year"] == 2015 + assert period["accounting_status"] == "not_evaluated" + assert period["account"] is None + transition = _artifact( + runtime, cold, tmp_path, f"apply_{year}", "transition" + ) + assert period["diagnostic"] == transition["diagnostic"] + + +def test_accounting_error_does_not_change_mortality_or_later_application( + runtime, inputs, tmp_path, monkeypatch +): + def refuse(*args, **kwargs): + raise runtime.accounting.PopulationAccountingInputError( + "invented accounting refusal" + ) + + monkeypatch.setattr(runtime.accounting, "reconcile_period", refuse) + result = _run(runtime, inputs, tmp_path) + assert result.report["accounting_status"] == "failed" + for year in range(2015, 2018): + period = result.report["periods"][str(year)] + assert period["account"] is None + assert period["application_status"] == "complete" + assert period["completed_year"] == year + assert period["diagnostic"]["message"] == "invented accounting refusal" + assert ( + result.manifest.nodes[f"apply_{year}"].receipt["outcome"] == "pass" + ) + assert ( + result.manifest.nodes[f"evaluate_{year}"].receipt["outcome"] + == "pass" + ) + + +@pytest.mark.parametrize( + "mutation", + ["person", "observation", "boolean", "probability", "missing_record"], +) +def test_transition_records_must_bind_to_snapshot( + runtime, inputs, tmp_path, mutation +): + result = _run(runtime, inputs, tmp_path, end_year=2015) + context = _context(runtime, result, tmp_path) + value = runtime.parse_json(context.artifacts["transition"].payload) + row = value["records"][0] + if mutation == "person": + row["person_id"] += 9999 + elif mutation == "observation": + row["observation_id"] += 9999 + elif mutation == "boolean": + row["survives"] = 1 + elif mutation == "probability": + row["death_probability"] = 1.1 + else: + value["records"].pop() + context.artifacts["transition"].payload = runtime.json_bytes(value) + report = runtime.parse_json(runtime._account(context).artifacts["account"]) + assert report["accounting_status"] == "failed" + assert report["account"] is None + assert report["diagnostic"]["exception_type"] == "ValueError" + + +@pytest.mark.parametrize("mutation", ["omit_survivor", "retain_death"]) +def test_endpoint_differences_do_not_infer_deaths( + runtime, inputs, tmp_path, mutation +): + result = _run(runtime, inputs, tmp_path, end_year=2015) + context = _context(runtime, result, tmp_path) + snapshot = runtime.parse_json(context.artifacts["snapshot"].payload) + transition = runtime.parse_json(context.artifacts["transition"].payload) + rows = snapshot["observations"] + if mutation == "omit_survivor": + index = next( + i + for i, row in enumerate(rows) + if row["person_period_period_id"] == 2015 + ) + rows.pop(index) + snapshot["weights"].pop(index) + expected = "undeclared_exit" + else: + dead = next( + row for row in transition["records"] if not row["survives"] + ) + index = next( + i + for i, row in enumerate(rows) + if row["person_period_id"] == dead["observation_id"] + ) + child = { + **rows[index], + "person_period_id": 9999, + "person_period_period_id": 2015, + } + rows.append(child) + snapshot["weights"].append(snapshot["weights"][index]) + expected = "exit_contradicted_by_closing" + context.artifacts["snapshot"].payload = runtime.json_bytes(snapshot) + report = runtime.parse_json(runtime._account(context).artifacts["account"]) + assert report["accounting_status"] == "failed" + assert report["account"] is None + discrepancies = report["diagnostic"]["reconciliation"]["discrepancies"] + assert expected in {item["kind"] for item in discrepancies} + + +def test_accounting_copies_rows_without_mutating_snapshot_or_columns( + runtime, inputs, tmp_path +): + result = _run(runtime, inputs, tmp_path, end_year=2015) + context = _context(runtime, result, tmp_path) + before = copy.deepcopy(context) + frozen = runtime.trajectory._evaluation_context(context) + opening, closing = runtime._period_frames(frozen) + for frame in (opening, closing): + assert {"age", "sex", "person_period_id"} <= set(frame.columns) + opening.loc[:, "age"] = -99 + assert (frozen.tables["person_period"].age >= 0).all() + runtime._account(context) + assert ( + context.artifacts["snapshot"].payload + == before.artifacts["snapshot"].payload + ) + assert ( + context.artifacts["transition"].payload + == before.artifacts["transition"].payload + ) + + +@pytest.mark.parametrize("column", ["atomic_location_id", "household_id"]) +def test_initial_reader_refuses_unsupported_location_and_household_columns( + runtime, inputs, tmp_path, column +): + initial = _read(inputs["initial"]) + for row in initial: + row[column] = "invented-anchor" + _write(inputs["initial"], initial) + with pytest.raises(Exception, match="unsupported fields"): + _run(runtime, inputs, tmp_path) + assert _read(inputs["initial"]) == initial + + +def test_snapshot_refuses_unsupported_location_column( + runtime, inputs, tmp_path +): + result = _run(runtime, inputs, tmp_path, end_year=2015) + context = _context(runtime, result, tmp_path) + snapshot = runtime.parse_json(context.artifacts["snapshot"].payload) + for row in snapshot["observations"]: + row["atomic_location_id"] = "invented-anchor" + context.artifacts["snapshot"].payload = runtime.json_bytes(snapshot) + report = runtime.parse_json(runtime._account(context).artifacts["account"]) + assert report["accounting_status"] == "failed" + assert report["account"] is None + assert "snapshot row binding" in report["diagnostic"]["message"] + + +@pytest.mark.parametrize( + "mutation", + ["period_alias", "duplicate_history", "weight_length", "future_period"], +) +def test_snapshot_binding_and_period_history_fail_closed( + runtime, inputs, tmp_path, mutation +): + result = _run(runtime, inputs, tmp_path, end_year=2015) + context = _context(runtime, result, tmp_path) + snapshot = runtime.parse_json(context.artifacts["snapshot"].payload) + if mutation == "period_alias": + snapshot["periods"].append({"period_id": 9999, "period": 2015}) + elif mutation == "duplicate_history": + snapshot["observations"].append( + {**snapshot["observations"][0], "person_period_id": 9999} + ) + snapshot["weights"].append(snapshot["weights"][0]) + elif mutation == "weight_length": + snapshot["weights"].pop() + else: + snapshot["periods"].append({"period_id": 2016, "period": 2016}) + context.artifacts["snapshot"].payload = runtime.json_bytes(snapshot) + report = runtime.parse_json(runtime._account(context).artifacts["account"]) + assert report["accounting_status"] == "failed" + assert report["account"] is None + assert report["diagnostic"]["exception_type"] == "ValueError" + + +def test_changed_snapshot_survivor_weight_is_reported_separately_from_evaluation( + runtime, inputs, tmp_path, monkeypatch +): + original = runtime.trajectory._snapshot + + def changed_weight(context): + result = original(context) + snapshot = runtime.parse_json(result.artifacts["snapshot"]) + if context.params["year"] == 2016: + index = next( + i + for i, row in enumerate(snapshot["observations"]) + if row["person_period_period_id"] == 2016 + ) + snapshot["weights"][index] += 3.0 + return replace( + result, artifacts={"snapshot": runtime.json_bytes(snapshot)} + ) + + monkeypatch.setattr(runtime.trajectory, "_snapshot", changed_weight) + result = _run(runtime, inputs, tmp_path) + account = result.report["periods"]["2016"]["account"] + assert account["weights"]["revaluation"]["carried"] == 3.0 + assert account["residuals"]["weight"] == 0.0 + assert result.report["accounting_status"] == "complete" + assert result.manifest.nodes["evaluate_2016"].receipt["outcome"] == "fail" + assert result.manifest.nodes["apply_2017"].receipt["outcome"] == "pass" + assert result.manifest.nodes["evaluate_2017"].receipt["outcome"] == "pass" + + +def test_incomplete_application_does_not_parse_snapshot(runtime): + outcome = { + "format": runtime.trajectory.TRANSITION_TYPE.name, + "schema_version": 1, + "from_year": 2015, + "year": 2016, + "status": "failed", + "completed_year": 2015, + "records": [], + "diagnostic": {"message": "synthetic prior refusal"}, + } + context = SimpleNamespace( + params={"year": 2016, "boundary_year": 2014}, + artifacts={ + "transition": SimpleNamespace(payload=runtime.json_bytes(outcome)), + "snapshot": SimpleNamespace(payload=b"{invalid snapshot JSON"), + }, + ) + report = runtime.parse_json(runtime._account(context).artifacts["account"]) + assert report["accounting_status"] == "not_evaluated" + assert report["account"] is None + assert report["diagnostic"] == outcome["diagnostic"] + + +@pytest.mark.parametrize( + "mutation", ["json", "completed_year", "unknown_status"] +) +def test_malformed_transition_does_not_invent_application_completion( + runtime, mutation +): + value = { + "format": runtime.trajectory.TRANSITION_TYPE.name, + "schema_version": 1, + "from_year": 2015, + "year": 2016, + "status": "failed", + "completed_year": 2015, + "records": [], + "diagnostic": {"message": "synthetic prior refusal"}, + } + if mutation == "completed_year": + value["completed_year"] = 2013 + elif mutation == "unknown_status": + value["status"] = "unknown" + context = SimpleNamespace( + params={"year": 2016, "boundary_year": 2014}, + artifacts={ + "transition": SimpleNamespace( + payload=( + b"{invalid" + if mutation == "json" + else runtime.json_bytes(value) + ) + ) + }, + ) + report = runtime.parse_json(runtime._account(context).artifacts["account"]) + assert report["accounting_status"] == "failed" + assert report["account"] is None + assert report["application_status"] is None + assert report["completed_year"] is None + + +def test_runner_requires_exact_annual_holdouts(runtime, inputs, tmp_path): + inputs["holdouts"].pop(2016) + with pytest.raises(ValueError, match="exactly one source per year"): + _run(runtime, inputs, tmp_path) + + +@pytest.mark.parametrize("value", ["1.0", True]) +def test_raw_snapshot_weights_are_checked_before_float_conversion( + runtime, inputs, tmp_path, value +): + result = _run(runtime, inputs, tmp_path, end_year=2015) + context = _context(runtime, result, tmp_path) + snapshot = runtime.parse_json(context.artifacts["snapshot"].payload) + snapshot["weights"][0] = value + context.artifacts["snapshot"].payload = runtime.json_bytes(snapshot) + report = runtime.parse_json(runtime._account(context).artifacts["account"]) + assert report["accounting_status"] == "failed" + assert report["account"] is None + assert ( + report["diagnostic"]["exception_type"] + == "PopulationAccountingInputError" + ) + assert "real number" in report["diagnostic"]["message"] + + +@pytest.mark.parametrize("identifier", ["person", "observation"]) +def test_oversized_raw_identifier_cannot_wrap_into_transition_binding( + runtime, inputs, tmp_path, identifier +): + result = _run(runtime, inputs, tmp_path, end_year=2015) + context = _context(runtime, result, tmp_path) + snapshot = runtime.parse_json(context.artifacts["snapshot"].payload) + transition = runtime.parse_json(context.artifacts["transition"].payload) + if identifier == "person": + column, field = "person_period_person_id", "person_id" + else: + column, field = "person_period_id", "observation_id" + original_id = transition["records"][0][field] + for row in snapshot["observations"]: + if row[column] == original_id: + row[column] = 2**64 - 1 + transition["records"][0][field] = -1 + context.artifacts["snapshot"].payload = runtime.json_bytes(snapshot) + context.artifacts["transition"].payload = runtime.json_bytes(transition) + report = runtime.parse_json(runtime._account(context).artifacts["account"]) + assert report["accounting_status"] == "failed" + assert report["account"] is None + assert ( + report["diagnostic"]["exception_type"] + == "PopulationAccountingInputError" + ) + assert "signed int64" in report["diagnostic"]["message"] + + +@pytest.mark.parametrize("status", ["failed", "blocked"]) +def test_overflowing_diagnostic_becomes_a_serializable_accounting_refusal( + runtime, status +): + payload = ( + '{"format":"populace-dynamics.mortality-transition",' + '"schema_version":1,"from_year":2015,"year":2016,' + f'"status":"{status}","completed_year":2015,"records":[], ' + '"diagnostic":{"message":"synthetic refusal","nested":[1e400]}}' + ).encode() + context = SimpleNamespace( + params={"year": 2016, "boundary_year": 2014}, + artifacts={"transition": SimpleNamespace(payload=payload)}, + ) + report = runtime.parse_json(runtime._account(context).artifacts["account"]) + assert report["accounting_status"] == "failed" + assert report["account"] is None + assert report["application_status"] is None + assert report["completed_year"] is None + assert report["diagnostic"]["exception_type"] == "ValueError" + json.dumps(report, allow_nan=False) + + +@pytest.mark.parametrize( + "module", ["accounting", "trajectory", "runtime", "model"] +) +def test_accounting_hash_includes_reused_source_modules( + runtime, monkeypatch, module +): + _, registry = runtime.build_accounted_trajectory_graph(end_year=2015) + kernel = registry.get(runtime.ACCOUNT_KERNEL) + modules = { + "accounting": runtime.accounting, + "trajectory": runtime.trajectory, + "runtime": runtime.trajectory.rt, + "model": runtime.trajectory.rt.model_module, + } + target = Path(modules[module].__file__).resolve() + before = kernel.implementation_hash() + original = Path.read_bytes + + def changed(path): + payload = original(path) + return ( + payload + b"\n# synthetic source change\n" + if path.resolve() == target + else payload + ) + + monkeypatch.setattr(Path, "read_bytes", changed) + assert kernel.implementation_hash() != before + + +def test_account_nodes_have_only_artifact_dependencies(runtime): + graph, _ = runtime.build_accounted_trajectory_graph(end_year=2017) + for node in graph.nodes: + if not node.id.startswith("account_"): + continue + assert node.population == "training" + assert node.inputs == () + assert node.sources == () + assert {item.name for item in node.artifact_inputs} == { + "transition", + "snapshot", + } diff --git a/tests/test_m6_accounting_history.py b/tests/test_m6_accounting_history.py new file mode 100644 index 00000000..04db480c --- /dev/null +++ b/tests/test_m6_accounting_history.py @@ -0,0 +1,546 @@ +"""Synthetic supplied-history accounting; no data or fitted models.""" + +import copy +import json +import multiprocessing +import pickle +from concurrent.futures import ProcessPoolExecutor +from dataclasses import FrozenInstanceError + +import numpy as np +import pandas as pd +import pytest + +from populace_dynamics.engine import accounting +from populace_dynamics.engine.accounting_history import ( + SINGLE_PRESENCE_EPISODE, + AnnualTransition, + HistoryAccountingError, + HistoryErrorKind, + reconcile_history, +) + + +def _frame(year, rows=()): + return pd.DataFrame( + { + "person_id": [person_id for person_id, _ in rows], + "year": [year] * len(rows), + "weight": [weight for _, weight in rows], + } + ) + + +def _steady(year, rows=((1, 1.0),)): + return AnnualTransition( + year, year + 1, _frame(year, rows), _frame(year + 1, rows) + ) + + +def _event(person_id, kind, year, weight=None): + return accounting.PopulationEvent( + person_id, + kind, + year, + weight=weight, + reason="supplied synthetic event", + source="synthetic fixture", + ) + + +def _reconcile(transitions): + return reconcile_history( + transitions, identity_contract=SINGLE_PRESENCE_EPISODE + ) + + +def _history(): + first = AnnualTransition( + 2020, + 2021, + _frame(2020, [(1, 0), (2, 2), (3, 3)]), + _frame(2021, [(1, 1), (2, 2), (4, 4)]), + additions=[ + _event(4, "birth", 2021, 5), + _event(8, "scheduled_entry", 2021, 2), + ], + exits=[ + _event(3, "death", 2021), + _event(8, "other_exit", 2021, 1), + ], + ) + second = AnnualTransition( + 2021, + 2022, + _frame(2021, [(4, 4), (2, 2), (1, 1)]), + _frame(2022, [(2, 2), (4, 4)]), + additions=[_event(9, "other_entry", 2022, 3)], + exits=[ + _event(1, "emigration", 2022), + _event(9, "death", 2022, 3), + ], + ) + return [first, second, _steady(2022, [(4, 4), (2, 2)])] + + +def test_accounts_are_unchanged_annual_results_in_supplied_order(monkeypatch): + transitions = _history() + expected = tuple( + accounting.reconcile_period( + item.opening, + item.closing, + opening_year=item.opening_year, + closing_year=item.closing_year, + additions=item.additions, + exits=item.exits, + ) + for item in transitions + ) + calls = [] + original = accounting.reconcile_period + + def record(*args, **kwargs): + calls.append(kwargs["closing_year"]) + return original(*args, **kwargs) + + monkeypatch.setattr(accounting, "reconcile_period", record) + result = _reconcile(transitions) + assert calls == [2021, 2022, 2023] + assert result.periods == expected + assert result.seen_person_ids == (1, 2, 3, 4, 8, 9) + assert result.retired_person_ids == (1, 3, 8, 9) + assert result.periods[0].counts.transient == 1 + assert result.periods[0].weights.revaluation.transient == -1.0 + assert result.provenance["event_log_completeness_verified"] is False + assert result.provenance["event_truth_verified"] is False + + +def test_contract_keyword_is_required(): + with pytest.raises(TypeError, match="identity_contract"): + reconcile_history([_steady(2020)]) + + +@pytest.mark.parametrize("contract", [None, "", "general_reentry", True, []]) +def test_unsupported_identity_contract_is_a_typed_refusal(contract): + with pytest.raises(HistoryAccountingError) as caught: + reconcile_history([_steady(2020)], identity_contract=contract) + assert caught.value.kind is HistoryErrorKind.INPUT + assert caught.value.period_index is None + json.dumps(caught.value.to_dict(), allow_nan=False) + + +@pytest.mark.parametrize("transitions", [[], (), None, "history", iter(())]) +def test_nonempty_sequence_is_required(transitions): + with pytest.raises(HistoryAccountingError) as caught: + _reconcile(transitions) + assert caught.value.kind is HistoryErrorKind.INPUT + + +@pytest.mark.parametrize("second_year", [2019, 2020, 2022]) +def test_reversal_duplicate_and_gap_are_rejected(second_year): + with pytest.raises(HistoryAccountingError) as caught: + _reconcile([_steady(2020), _steady(second_year)]) + assert caught.value.kind is HistoryErrorKind.NONCONTIGUOUS + assert caught.value.period_index == 1 + assert caught.value.opening_year == second_year + assert caught.value.closing_year == second_year + 1 + + +def test_nonannual_period_preserves_accountant_input_error(): + transition = AnnualTransition(2020, 2022, _frame(2020), _frame(2022)) + with pytest.raises(HistoryAccountingError) as caught: + _reconcile([transition]) + error = caught.value + assert error.kind is HistoryErrorKind.PERIOD + assert error.period_index == 0 + assert (error.opening_year, error.closing_year) == (2020, 2022) + assert isinstance( + error.__cause__, accounting.PopulationAccountingInputError + ) + assert "annual" in str(error.__cause__) + + +def test_invalid_descriptor_preserves_period_index_and_cause(): + with pytest.raises(HistoryAccountingError) as caught: + _reconcile([_steady(2020), object()]) + error = caught.value + assert error.period_index == 1 + assert error.opening_year is None + assert error.closing_year is None + assert isinstance( + error.__cause__, accounting.PopulationAccountingInputError + ) + + +def test_annual_discrepancies_remain_available_on_original_cause(): + broken = AnnualTransition(2021, 2022, _frame(2021, [(1, 1)]), _frame(2022)) + with pytest.raises(HistoryAccountingError) as caught: + _reconcile([_steady(2020), broken]) + error = caught.value + assert error.kind is HistoryErrorKind.PERIOD + assert error.period_index == 1 + assert (error.opening_year, error.closing_year) == (2021, 2022) + assert isinstance( + error.__cause__, accounting.PopulationReconciliationError + ) + assert ( + error.__cause__.discrepancies[0].kind + is accounting.DiscrepancyKind.UNDECLARED_EXIT + ) + payload = error.to_dict() + assert payload["cause"]["type"] == "PopulationReconciliationError" + assert ( + payload["cause"]["details"]["discrepancies"][0]["kind"] + == "undeclared_exit" + ) + assert json.loads(json.dumps(payload, allow_nan=False)) == payload + payload["person_ids"].append(999) + payload["cause"]["details"]["discrepancies"].clear() + assert error.to_dict()["person_ids"] == [] + assert len(error.to_dict()["cause"]["details"]["discrepancies"]) == 1 + + +HISTORY_ROUND_TRIPS = [ + pytest.param( + lambda value: pickle.loads(pickle.dumps(value, protocol=0)), + id="pickle-protocol-0", + ), + pytest.param( + lambda value: pickle.loads( + pickle.dumps(value, protocol=pickle.HIGHEST_PROTOCOL) + ), + id="pickle-highest-protocol", + ), + pytest.param(copy.copy, id="copy"), + pytest.param(copy.deepcopy, id="deepcopy"), +] + + +def _broken_second_period(): + return [ + _steady(2020), + AnnualTransition(2021, 2022, _frame(2021, [(1, 1)]), _frame(2022)), + ] + + +@pytest.mark.parametrize("round_trip", HISTORY_ROUND_TRIPS) +@pytest.mark.parametrize( + ("refuse", "kind", "period_index", "years", "person_ids", "cause_type"), + [ + pytest.param( + lambda: _reconcile(_broken_second_period()), + HistoryErrorKind.PERIOD, + 1, + (2021, 2022), + (), + accounting.PopulationReconciliationError, + id="annual-period", + ), + pytest.param( + lambda: _reconcile([_steady(2020), _steady(2021, [(2, 1)])]), + HistoryErrorKind.BOUNDARY_PERSON_IDS, + 1, + (2021, 2022), + (1, 2), + None, + id="boundary-person-ids", + ), + pytest.param( + lambda: reconcile_history( + [_steady(2020)], identity_contract="general_reentry" + ), + HistoryErrorKind.INPUT, + None, + (None, None), + (), + None, + id="history-input", + ), + ], +) +def test_history_refusal_survives_pickle_and_copy( + round_trip, refuse, kind, period_index, years, person_ids, cause_type +): + with pytest.raises(HistoryAccountingError) as caught: + refuse() + error = caught.value + + restored = round_trip(error) + + assert type(restored) is HistoryAccountingError + assert restored is not error + # The rebuilt refusal must not prefix its message a second time. + assert restored.args == error.args + assert str(restored) == str(error) + assert restored.kind is kind + assert restored.period_index == period_index + assert (restored.opening_year, restored.closing_year) == years + assert restored.person_ids == person_ids + assert restored.to_dict() == error.to_dict() + if cause_type is None: + assert restored.__cause__ is None + else: + assert type(restored.__cause__) is cause_type + assert restored.__suppress_context__ is True + assert restored.__cause__.discrepancies == ( + error.__cause__.discrepancies + ) + assert ( + restored.__cause__.discrepancies[0].kind + is accounting.DiscrepancyKind.UNDECLARED_EXIT + ) + + +def test_history_refusal_crosses_a_process_pool(): + context = multiprocessing.get_context("spawn") + with ProcessPoolExecutor(max_workers=1, mp_context=context) as pool: + future = pool.submit( + reconcile_history, + _broken_second_period(), + identity_contract=SINGLE_PRESENCE_EPISODE, + ) + with pytest.raises(HistoryAccountingError) as caught: + future.result(timeout=120) + error = caught.value + assert error.kind is HistoryErrorKind.PERIOD + assert error.period_index == 1 + assert (error.opening_year, error.closing_year) == (2021, 2022) + assert str(error).startswith("period[1]: ") + # concurrent.futures replaces __cause__ with its remote-traceback text + # after unpickling, so the typed annual cause is asserted by the direct + # pickle and copy test above rather than here. + + +def test_boundary_person_sets_must_match_even_at_equal_counts_and_mass(): + with pytest.raises(HistoryAccountingError) as caught: + _reconcile([_steady(2020), _steady(2021, [(2, 1)])]) + assert caught.value.kind is HistoryErrorKind.BOUNDARY_PERSON_IDS + assert caught.value.person_ids == (1, 2) + assert caught.value.period_index == 1 + + +def test_boundary_weights_are_compared_per_person_not_in_aggregate(): + with pytest.raises(HistoryAccountingError) as caught: + _reconcile( + [_steady(2020, [(1, 2), (2, 3)]), _steady(2021, [(1, 3), (2, 2)])] + ) + assert caught.value.kind is HistoryErrorKind.BOUNDARY_WEIGHTS + assert caught.value.person_ids == (1, 2) + + +def test_boundary_weight_comparison_has_no_tolerance(): + with pytest.raises(HistoryAccountingError) as caught: + _reconcile( + [_steady(2020), _steady(2021, [(1, np.nextafter(1.0, 2.0))])] + ) + assert caught.value.kind is HistoryErrorKind.BOUNDARY_WEIGHTS + + +def test_boundary_comparison_uses_accountant_binary64_domain(): + first = _steady(2020, [(1, 2**53 + 1)]) + second = _steady(2021, [(1, float(2**53))]) + result = _reconcile([first, second]) + assert ( + result.periods[0].weights.closing == result.periods[1].weights.opening + ) + + +def test_boundary_order_and_opaque_columns_do_not_imply_other_continuity(): + first = _steady(2020, [(1, 0), (2, 3)]) + second = _steady(2021, [(2, 3), (1, 0)]) + first.closing["opaque"] = ["earlier", "earlier"] + second.opening["opaque"] = ["later", "later"] + before = [ + frame.copy(deep=True) for frame in (first.closing, second.opening) + ] + result = _reconcile([first, second]) + assert result.periods[0].counts.closing == 2 + assert result.provenance["other_column_continuity_verified"] is False + for frame, original in zip( + (first.closing, second.opening), before, strict=True + ): + pd.testing.assert_frame_equal(frame, original) + + +@pytest.mark.parametrize("exit_kind", ["death", "emigration", "other_exit"]) +@pytest.mark.parametrize("return_is_transient", [False, True]) +def test_departed_id_return_is_unsupported_under_selected_contract( + exit_kind, return_is_transient +): + departure = AnnualTransition( + 2020, + 2021, + _frame(2020, [(1, 1)]), + _frame(2021), + exits=[_event(1, exit_kind, 2021)], + ) + returning = AnnualTransition( + 2022, + 2023, + _frame(2022), + _frame(2023, [] if return_is_transient else [(1, 1)]), + additions=[_event(1, "scheduled_entry", 2023, 1)], + exits=[_event(1, "death", 2023, 1)] if return_is_transient else (), + ) + with pytest.raises( + HistoryAccountingError, match="same-person return" + ) as caught: + _reconcile([departure, _steady(2021, []), returning]) + assert caught.value.kind is HistoryErrorKind.RETIRED_PERSON_ID + assert caught.value.period_index == 2 + assert caught.value.person_ids == (1,) + + +def test_transient_id_is_seen_and_retired_before_any_endpoint_presence(): + first = AnnualTransition( + 2020, + 2021, + _frame(2020), + _frame(2021), + additions=[_event(10, "other_entry", 2021, 0)], + exits=[_event(10, "other_exit", 2021, 0)], + ) + result = _reconcile([first]) + assert result.seen_person_ids == result.retired_person_ids == (10,) + second = AnnualTransition( + 2021, + 2022, + _frame(2021), + _frame(2022, [(10, 0)]), + additions=[_event(10, "birth", 2022)], + ) + with pytest.raises(HistoryAccountingError) as caught: + _reconcile([first, second]) + assert caught.value.kind is HistoryErrorKind.RETIRED_PERSON_ID + + +def test_extinction_empty_years_and_fresh_ids_remain_supported(): + first = AnnualTransition( + 2020, + 2021, + _frame(2020, [(1, 0)]), + _frame(2021), + exits=[_event(1, "death", 2021)], + ) + third = AnnualTransition( + 2022, + 2023, + _frame(2022), + _frame(2023, [(2, 0)]), + additions=[_event(2, "scheduled_entry", 2023)], + ) + result = _reconcile([first, _steady(2021, []), third]) + assert ( + result.periods[1].counts.opening + == result.periods[1].counts.closing + == 0 + ) + assert result.seen_person_ids == (1, 2) + assert result.retired_person_ids == (1,) + + +def test_omitting_both_transient_declarations_remains_undetectable(): + result = _reconcile([_steady(2020, []), _steady(2021, [])]) + assert result.seen_person_ids == result.retired_person_ids == () + assert result.provenance["event_log_completeness_verified"] is False + assert all(period.counts.transient == 0 for period in result.periods) + + +def test_first_opening_has_no_inferred_prehistory(): + result = _reconcile([_steady(2020, [(100, 1)])]) + assert result.seen_person_ids == (100,) + assert result.retired_person_ids == () + assert "no prehistory" in result.provenance["scope"] + + +@pytest.mark.parametrize( + ("column", "value"), + [ + ("person_id", True), + ("person_id", "1"), + ("person_id", 2**64 - 1), + ("year", 2021.0), + ("weight", True), + ("weight", "1.0"), + ("weight", float("inf")), + ], +) +def test_malformed_boundary_values_are_not_coerced_before_validation( + column, value +): + second = _steady(2021) + second.opening[column] = pd.Series([value], dtype=object) + with pytest.raises(HistoryAccountingError) as caught: + _reconcile([_steady(2020), second]) + assert caught.value.kind is HistoryErrorKind.PERIOD + assert caught.value.period_index == 1 + assert isinstance( + caught.value.__cause__, accounting.PopulationAccountingInputError + ) + json.dumps(caught.value.to_dict(), allow_nan=False) + + +@pytest.mark.parametrize("year", [True, "2020", 2020.0, 2**64 - 1]) +def test_malformed_period_years_are_wrapped_without_unsafe_coordinates(year): + transition = AnnualTransition(year, 2021, _frame(2020), _frame(2021)) + with pytest.raises(HistoryAccountingError) as caught: + _reconcile([transition]) + assert caught.value.opening_year is None + assert caught.value.closing_year is None + json.dumps(caught.value.to_dict(), allow_nan=False) + + +def test_ids_at_signed_int64_limits_remain_exact(): + rows = [(np.int64(-(2**63)), 0), (np.int64(2**63 - 1), 1)] + result = _reconcile([_steady(2020, rows), _steady(2021, rows[::-1])]) + assert result.seen_person_ids == (-(2**63), 2**63 - 1) + assert "seen_person_ids" not in result.to_dict() + + +def test_result_and_serialization_are_isolated_from_caller_mutation(): + transitions = _history() + result = _reconcile(transitions) + expected = result.to_dict() + for transition in transitions: + transition.opening["weight"] = 999 + transition.closing["person_id"] = 999 + transitions[0].additions.clear() + transitions.clear() + assert result.to_dict() == expected + with pytest.raises(FrozenInstanceError): + result.identity_contract = "changed" + with pytest.raises(FrozenInstanceError): + result.periods[0].opening_year = 1999 + with pytest.raises(TypeError): + result.provenance["scope"] = "changed" + with pytest.raises(TypeError): + result.periods[0].counts.additions_by_kind["birth"] = 100 + payload = result.to_dict() + payload["provenance"]["scope"] = "changed" + payload["periods"][0]["provenance"]["declaration_sources"].append( + "changed" + ) + payload["periods"][0]["counts"]["opening"] = 999 + assert result.to_dict() == expected + json.dumps(result.to_dict(), allow_nan=False) + + +def test_descriptor_freezes_fields_but_does_not_own_input_frames(): + transition = _steady(2020) + with pytest.raises(FrozenInstanceError): + transition.opening_year = 2000 + transition.opening.loc[0, "weight"] = 2 + assert transition.opening.loc[0, "weight"] == 2 + + +@pytest.mark.parametrize("events", ["events", iter(()), [object()]]) +def test_malformed_declaration_containers_stay_accountant_errors(events): + transition = AnnualTransition( + 2020, 2021, _frame(2020), _frame(2021), additions=events + ) + with pytest.raises(HistoryAccountingError) as caught: + _reconcile([transition]) + assert caught.value.kind is HistoryErrorKind.PERIOD + assert isinstance( + caught.value.__cause__, accounting.PopulationAccountingInputError + ) diff --git a/tests/test_m6_claiming_successor.py b/tests/test_m6_claiming_successor.py new file mode 100644 index 00000000..d0cb6a65 --- /dev/null +++ b/tests/test_m6_claiming_successor.py @@ -0,0 +1,371 @@ +"""Synthetic opt-in claiming checks; no fitted objects or external inputs.""" + +from __future__ import annotations + +from functools import partial + +import numpy as np +import pandas as pd +import pytest + +from populace_dynamics.engine.claiming import apply_claiming +from populace_dynamics.engine.loop import ( + MaritalStepResult, + PeriodContext, + PeriodModules, + ProjectionEngine, +) +from populace_dynamics.engine.rng import ( + ProjectionModule, + ProjectionRNGRegistry, +) +from populace_dynamics.engine.steps import ClaimingSchedule, advance_age +from populace_dynamics.engine.steps import ( + apply_claiming as historical_apply_claiming, +) + + +class NoDraw: + def choice(self, *args, **kwargs): + raise AssertionError("ineligible people must not consume a draw") + + +class NoDistribution: + def distribution(self, sex, year): + raise AssertionError("ineligible people must not query a PMF") + + +def _context(person_ids=None, *, year=2020, period=1): + if person_ids is None: + return PeriodContext(period, year, 0, {}) + return PeriodContext( + period, + year, + 0, + {}, + rng_registry=ProjectionRNGRegistry(0, 2), + person_ordinals={ + person_id: ordinal + for ordinal, person_id in enumerate(sorted(person_ids)) + }, + ) + + +def _schedule(): + return ClaimingSchedule( + { + ("female", 2014): {62: 0.3, 65: 0.2, 70: 0.5}, + ("male", 2014): {62: 0.2, 66: 0.6, 70: 0.2}, + } + ) + + +@pytest.mark.parametrize("keyed", [False, True]) +def test_converted_and_previously_claimed_people_never_draw(keyed): + frame = pd.DataFrame( + { + "person_id": [10, 20], + "age": [67, 68], + "sex": ["male", "female"], + "di_converted": [True, False], + "claimed": [False, True], + "claim_year": pd.array([pd.NA, 2018], dtype="Int64"), + } + ) + original = frame.copy(deep=True) + # An empty ordinal map rejects even constructing a keyed person stream. + context = _context([] if keyed else None) + result = apply_claiming( + frame, context, NoDraw(), schedule=NoDistribution() + ) + + assert result["claim_age"].isna().all() + assert str(result["claim_age"].dtype) == "Int64" + assert result["claimed"].tolist() == [True, True] + assert result["claim_year"].tolist() == [2020, 2018] + pd.testing.assert_frame_equal(frame, original) + + +@pytest.mark.parametrize("keyed", [False, True]) +def test_conversion_event_reset_does_not_cause_a_later_plan(keyed): + frame = pd.DataFrame( + {"age": [67], "sex": ["male"], "di_converted": [True]} + ) + first = apply_claiming( + frame, + _context([] if keyed else None), + NoDraw(), + schedule=NoDistribution(), + ) + second_input = first.assign(age=68, di_converted=False) + second = apply_claiming( + second_input, + _context([] if keyed else None, year=2021, period=2), + NoDraw(), + schedule=NoDistribution(), + ) + + assert second["claim_age"].isna().all() + assert second["claimed"].tolist() == [True] + assert second["claim_year"].tolist() == [2020] + + +def test_supplied_plans_and_previously_claimed_years_are_preserved(): + frame = pd.DataFrame( + { + "age": [67, 68, 68, 63, 63, 70], + "sex": ["male"] * 6, + "di_converted": [True, True, False, False, False, False], + "claimed": pd.array( + [False, True, True, False, False, True], dtype="boolean" + ), + "claim_age": pd.array( + [70, pd.NA, pd.NA, 64, 62, 60], dtype="Int64" + ), + "claim_year": pd.array( + [pd.NA, 2018, 2017, pd.NA, pd.NA, 2010], dtype="Int64" + ), + }, + index=[9, 7, 5, 3, 1, 0], + ) + original = frame.copy(deep=True) + result = apply_claiming( + frame, _context([]), NoDraw(), schedule=NoDistribution() + ) + + pd.testing.assert_series_equal(result["claim_age"], frame["claim_age"]) + assert result["claimed"].tolist() == [True, True, True, False, True, True] + pd.testing.assert_series_equal( + result["claim_year"], + pd.Series( + pd.array([2020, 2018, 2017, pd.NA, 2020, 2010], dtype="Int64"), + index=frame.index, + name="claim_year", + ), + ) + pd.testing.assert_frame_equal(frame, original) + + +@pytest.mark.parametrize( + "year, expected_age", [(2019, 62), (2020, 62), (2021, 70)] +) +def test_age_50_threshold_and_nearest_year_selection(year, expected_age): + schedule = ClaimingSchedule( + { + ("female", 2018): {62: 1.0}, + ("female", 2022): {70: 1.0}, + } + ) + frame = pd.DataFrame( + {"person_id": [10, 20], "age": [49, 50], "sex": ["male", "female"]} + ) + # No male PMF or ordinal is needed for the under-50 person. + result = apply_claiming( + frame, _context([20], year=year), NoDraw(), schedule=schedule + ) + + assert pd.isna(result.loc[0, "claim_age"]) + assert result.loc[1, "claim_age"] == expected_age + assert not result["claimed"].any() + assert result["claim_year"].isna().all() + + +def test_converted_only_sex_needs_no_pmf_or_person_ordinal(): + frame = pd.DataFrame( + { + "person_id": [10, 20, 30], + "age": [67, 62, 68], + "sex": ["male", "female", "male"], + "di_converted": [True, False, False], + "claimed": [False, False, True], + } + ) + result = apply_claiming( + frame, + _context([20]), + NoDraw(), + schedule=ClaimingSchedule({("female", 2014): {62: 1.0}}), + ) + + assert result["claim_age"].isna().tolist() == [True, False, True] + assert result.loc[1, "claim_age"] == 62 + assert result["claimed"].all() + assert pd.isna(result.loc[result["person_id"] == 30, "claim_year"].iloc[0]) + + +@pytest.mark.parametrize("flags", ["absent", "null"]) +def test_missing_flags_retain_historical_nonconversion_behavior(flags): + frame = pd.DataFrame( + { + "person_id": [10, 20, 30, 40], + "age": [62, 68, 51, 49], + "sex": ["female", "male", "female", "male"], + } + ) + if flags == "null": + frame["di_converted"] = pd.array([pd.NA] * 4, dtype="boolean") + frame["claimed"] = pd.array([pd.NA] * 4, dtype="boolean") + context = _context(frame["person_id"]) + successor = apply_claiming(frame, context, NoDraw(), schedule=_schedule()) + historical = historical_apply_claiming( + frame, context, NoDraw(), schedule=_schedule() + ) + + pd.testing.assert_frame_equal(successor, historical) + + +def test_keyed_nonentrant_draws_match_history_and_ignore_row_order(): + frame = pd.DataFrame( + { + "person_id": [60, 10, 40, 30, 20, 50], + "age": [62, 67, 69, 66, 52, 63], + "sex": ["female", "female", "male", "male", "female", "male"], + "di_converted": [False, True, False, True, False, False], + "claimed": [False, False, True, False, False, False], + } + ) + context = _context(frame["person_id"]) + successor = apply_claiming(frame, context, NoDraw(), schedule=_schedule()) + historical = historical_apply_claiming( + frame, context, NoDraw(), schedule=_schedule() + ) + eligible = ~frame["di_converted"] & ~frame["claimed"] + pd.testing.assert_frame_equal( + successor.loc[eligible], historical.loc[eligible] + ) + for subset in (frame.iloc[[2, 5, 0, 3, 1, 4]], frame.loc[eligible]): + replay = apply_claiming( + subset, context, NoDraw(), schedule=_schedule() + ) + pd.testing.assert_frame_equal( + replay.sort_index(), successor.loc[subset.index].sort_index() + ) + + +def test_batch_fallback_is_reproducible_but_same_seed_can_shift_draws(): + frame = pd.DataFrame( + { + "age": [67, 51, 51, 51, 67, 51], + "sex": ["female", "female", "male", "female", "male", "male"], + "di_converted": [True, False, False, False, True, False], + } + ) + eligible = ~frame["di_converted"] + first_rng = np.random.default_rng(21) + replay_rng = np.random.default_rng(21) + first = apply_claiming(frame, _context(), first_rng, schedule=_schedule()) + replay = apply_claiming( + frame, _context(), replay_rng, schedule=_schedule() + ) + pd.testing.assert_frame_equal(first, replay) + assert first_rng.bytes(32) == replay_rng.bytes(32) + + # Exactly the eligible sex groups consume draws; there is no draw burning. + eligible_rng = np.random.default_rng(21) + eligible_only = apply_claiming( + frame.loc[eligible], _context(), eligible_rng, schedule=_schedule() + ) + pd.testing.assert_frame_equal(first.loc[eligible], eligible_only) + # The preceding state comparison consumed 32 bytes from first_rng. + eligible_rng.bytes(32) + assert first_rng.bytes(32) == eligible_rng.bytes(32) + + historical = historical_apply_claiming( + frame, _context(), np.random.default_rng(21), schedule=_schedule() + ) + assert not first.loc[eligible, "claim_age"].equals( + historical.loc[eligible, "claim_age"] + ) + + +def test_empty_frame_retains_nullable_claim_state_without_draws(): + frame = pd.DataFrame( + {"age": pd.Series(dtype="int64"), "sex": pd.Series(dtype="str")} + ) + result = apply_claiming( + frame, _context([]), NoDraw(), schedule=NoDistribution() + ) + + assert result.empty + assert result.dtypes.astype(str).to_dict() == { + "age": "int64", + "sex": str(frame["sex"].dtype), + "claim_age": "Int64", + "claimed": "bool", + "claim_year": "Int64", + } + + +@pytest.mark.parametrize("missing", ["age", "sex"]) +def test_required_columns_are_validated_even_for_conversions(missing): + frame = pd.DataFrame( + {"age": [67], "sex": ["male"], "di_converted": [True]} + ).drop(columns=missing) + with pytest.raises(ValueError, match=f"missing columns.*{missing}"): + apply_claiming(frame, _context(), NoDraw(), schedule=NoDistribution()) + + +def test_successor_in_real_projection_loop_preserves_conversion_history(): + initial = pd.DataFrame( + { + "person_id": [30, 10, 20], + "year": [2019] * 3, + "age": [60, 66, 60], + "sex": ["female", "male", "female"], + } + ) + schedule = ClaimingSchedule({("female", 2014): {62: 1.0}}) + + def passthrough(frame, context, rng): + return frame + + def marital(frame, context, rng): + return MaritalStepResult(frame.copy(), pd.DataFrame()) + + def marital_reader(frame, context, marital, rng): + return frame + + def disability(frame, context, rng): + return frame.assign( + di_converted=(frame["person_id"] == 10) & (context.year == 2020) + ) + + modules = PeriodModules( + mortality=passthrough, + aging=advance_age, + marital_core=marital, + fertility=marital_reader, + disability=disability, + earnings=passthrough, + claiming=partial(apply_claiming, schedule=schedule), + household_composition=marital_reader, + ) + result = ProjectionEngine(modules).project( + initial, end_year=2021, draw_index=0 + ) + first, second = result.slices[1:] + for projected in (first, second): + converter = projected.set_index("person_id").loc[10] + assert pd.isna(converter["claim_age"]) + assert converter["claimed"] + assert converter["claim_year"] == 2020 + assert first.set_index("person_id").loc[10, "di_converted"] + assert not second["di_converted"].any() + assert not first.loc[first["person_id"] != 10, "claimed"].any() + assert second["claimed"].all() + assert second.loc[second["person_id"] != 10, "claim_year"].tolist() == [ + 2021, + 2021, + ] + assert result.traces[0].steps == tuple( + module.value for module in ProjectionModule + ) + reordered = ProjectionEngine(modules).project( + initial.iloc[::-1], end_year=2021, draw_index=0 + ) + pd.testing.assert_frame_equal( + result.panel.sort_values(["year", "person_id"]).reset_index(drop=True), + reordered.panel.sort_values(["year", "person_id"]).reset_index( + drop=True + ), + ) diff --git a/tests/test_m6_engine_forward_earnings.py b/tests/test_m6_engine_forward_earnings.py index 9bdac9de..94105a84 100644 --- a/tests/test_m6_engine_forward_earnings.py +++ b/tests/test_m6_engine_forward_earnings.py @@ -189,6 +189,59 @@ def values( ) +def test_actual_generator_output_can_be_recorded_without_changing_draws(): + from populace_dynamics.engine.earnings_domain import EarningsDomainAdapter + from populace_dynamics.engine.loop import PeriodContext + from populace_dynamics.engine.steps import apply_earnings + from populace_dynamics.forward_earnings_history import ( + ForwardEarningsHistory, + ) + from populace_dynamics.person_identity import ( + PersonIdentity, + PersonIdentityMap, + ) + + identities = PersonIdentityMap.from_identities( + [PersonIdentity("uint64", 2**63 + i) for i in range(21)] + ) + generator = EarningsDomainAdapter(_generator()) + initial = _frame((0, 1, 10, 20)) + initial["year"] = np.full(len(initial), 2014, dtype=np.int64) + projected = generator.materialize_initial_frame(initial) + control = projected.copy(deep=True) + history = ForwardEarningsHistory.start( + identities, + projected, + realization_id="invented-generator-draw", + generator_digest="a" * 64, + source_contract_digest="b" * 64, + unit="XTS", + price_basis="nominal", + lineage_digest="c" * 64, + ) + for year in (2015, 2016, 2017): + context = PeriodContext(year - 2014, year, 0, {}) + projected = apply_earnings( + projected, context, np.random.default_rng(year), model=generator + ) + control = apply_earnings( + control, context, np.random.default_rng(year), model=generator + ) + projected["year"] = control["year"] = year + history = history.append(projected, lineage_digest="d" * 64) + pd.testing.assert_frame_equal(projected, control) + assert len(history.observations) == 16 + assert all( + row.amount_state == "unavailable" + for row in history.for_person(identities.reverse_rows([0])[0]) + ) + for key in (1, 10, 20): + records = history.for_person(identities.reverse_rows([key])[0]) + assert records[0].amount_hex == records[1].amount_hex + assert records[2].amount_hex == records[3].amount_hex + assert records[2].generation_method == "biennial_draw" + + def test_age_grid_has_eight_bins_and_clips_both_outer_ranges(): ages = np.asarray( [0, 24.999, 25, 29.999, 30, 34.999, 35, 40, 45, 50, 55, 60, 64, 99] diff --git a/tests/test_m6_stock_flow.py b/tests/test_m6_stock_flow.py new file mode 100644 index 00000000..1feb41fe --- /dev/null +++ b/tests/test_m6_stock_flow.py @@ -0,0 +1,1660 @@ +"""Tests for the experimental annual stock-flow accounting interface. + +Every input here is built in memory. The conservation tests state the +closing population and the expected identity components independently, +as hand-checked literals, rather than recomputing the module's own +formula and comparing it to itself. The projection-engine tests drive +the real :class:`populace_dynamics.engine.loop.ProjectionEngine` with +synthetic adapters that record their own event declarations, so the +accountant is exercised against frames the engine actually produced. + +No fitted model, survey extract, benefit calculation or benchmark value +is involved anywhere in this module. +""" + +from __future__ import annotations + +import copy +import json +import math +import multiprocessing +import pickle +from concurrent.futures import ProcessPoolExecutor +from dataclasses import FrozenInstanceError, dataclass, field +from inspect import signature + +import numpy as np +import pandas as pd +import pytest + +import populace_dynamics.engine.accounting as accounting +from populace_dynamics.engine.accounting import ( + ACCOUNTING_INTERFACE_VERSION, + ENGINEERING_STATUS, + AccountingDiscrepancy, + DiscrepancyKind, + PopulationAccountingInputError, + PopulationEvent, + PopulationEventKind, + PopulationReconciliationError, + reconcile_period, +) +from populace_dynamics.engine.loop import ( + SCHEDULED_ENTRIES_KEY, + MaritalStepResult, + PeriodModules, + ProjectionEngine, +) + +# --------------------------------------------------------------------- +# in-memory frame helpers +# --------------------------------------------------------------------- + + +def frame(year: int, rows: dict[int, float], **columns) -> pd.DataFrame: + """Build a population frame from an ``{id: weight}`` mapping.""" + person_ids = list(rows) + built = pd.DataFrame( + { + "person_id": np.asarray(person_ids, dtype=np.int64), + "year": np.full(len(person_ids), year, dtype=np.int64), + "weight": np.asarray( + [rows[key] for key in person_ids], dtype=np.float64 + ), + } + ) + for name, values in columns.items(): + built[name] = values + return built + + +def empty_frame() -> pd.DataFrame: + """Build the naive spelling of an empty population.""" + return pd.DataFrame({"person_id": [], "year": [], "weight": []}) + + +def death(person_id: int, year: int, **kwargs) -> PopulationEvent: + return PopulationEvent(person_id, "death", year, **kwargs) + + +def birth(person_id: int, year: int, **kwargs) -> PopulationEvent: + return PopulationEvent(person_id, "birth", year, **kwargs) + + +def entry(person_id: int, year: int, **kwargs) -> PopulationEvent: + return PopulationEvent(person_id, "scheduled_entry", year, **kwargs) + + +def kinds(error: PopulationReconciliationError) -> list[DiscrepancyKind]: + return [item.kind for item in error.discrepancies] + + +# --------------------------------------------------------------------- +# conservation, with independently specified expectations +# --------------------------------------------------------------------- + + +def test_hand_checked_year_reconciles_to_stated_totals(): + """A worked year whose every component is stated, not derived. + + Opening 2020 holds five people weighing 100, 200, 300, 400 and 500, + so the opening stock is 1500. Over the period person 5 dies, person + 3's weight rises from 300 to 350, and one child is born weighing + 100. The closing frame is written out independently and sums to + 1150. The identity a reader can check by hand is + ``1500 - 500 + 100 + 50 = 1150``. + """ + opening = frame(2020, {1: 100.0, 2: 200.0, 3: 300.0, 4: 400.0, 5: 500.0}) + closing = frame( + 2021, {1: 100.0, 2: 200.0, 3: 350.0, 4: 400.0, 1001: 100.0} + ) + + account = reconcile_period( + opening, + closing, + opening_year=2020, + closing_year=2021, + additions=[birth(1001, 2021)], + exits=[death(5, 2021)], + ) + + assert account.weights.opening == 1500.0 + assert account.weights.closing == 1150.0 + assert account.weights.exits_total == 500.0 + assert account.weights.additions_total == 100.0 + assert account.weights.revaluation.carried == 50.0 + assert account.weights.revaluation.total == 50.0 + assert account.reconstructed_closing_weight == 1150.0 + assert account.weight_residual == 0.0 + assert account.count_residual == 0 + assert account.counts.opening == 5 + assert account.counts.closing == 5 + assert account.counts.carried == 4 + assert account.counts.entered == 1 + assert account.counts.exited == 1 + assert account.counts.transient == 0 + assert account.carried_person_ids == (1, 2, 3, 4) + assert account.entered_person_ids == (1001,) + assert account.exited_person_ids == (5,) + + +def test_flows_split_by_declared_kind(): + """Two arrivals and two departures of different kinds stay apart. + + Opening 10 + 20 = 30. A birth of 4 and a scheduled entry of 6 + arrive; a death of 10 and an emigration of 20 depart. The closing + frame holds only the two arrivals and weighs 10. + """ + opening = frame(2030, {1: 10.0, 2: 20.0}) + closing = frame(2031, {50: 4.0, 60: 6.0}) + + account = reconcile_period( + opening, + closing, + opening_year=2030, + closing_year=2031, + additions=[birth(50, 2031), entry(60, 2031)], + exits=[ + death(1, 2031), + PopulationEvent(2, "emigration", 2031), + ], + ) + + assert account.weights.closing == 10.0 + assert dict(account.weights.additions_by_kind) == { + "birth": 4.0, + "scheduled_entry": 6.0, + "other_entry": 0.0, + } + assert dict(account.weights.exits_by_kind) == { + "death": 10.0, + "emigration": 20.0, + "other_exit": 0.0, + } + assert dict(account.counts.additions_by_kind) == { + "birth": 1, + "scheduled_entry": 1, + "other_entry": 0, + } + assert dict(account.counts.exits_by_kind) == { + "death": 1, + "emigration": 1, + "other_exit": 0, + } + assert account.weight_residual == 0.0 + + +def test_unchanged_survivors_produce_a_flat_account(): + """Nothing happens: every flow and every revaluation is zero.""" + rows = {7: 1.5, 8: 2.5, 9: 3.5} + account = reconcile_period( + frame(2040, rows), + frame(2041, rows), + opening_year=2040, + closing_year=2041, + ) + + assert account.weights.opening == 7.5 + assert account.weights.closing == 7.5 + assert account.weights.additions_total == 0.0 + assert account.weights.exits_total == 0.0 + assert account.weights.revaluation.to_dict() == { + "carried": 0.0, + "entrant": 0.0, + "exiting": 0.0, + "transient": 0.0, + "total": 0.0, + } + assert account.counts.carried == 3 + assert account.weight_residual == 0.0 + + +def test_carried_weight_change_is_its_own_component(): + """A pure revaluation moves no person and is never a flow. + + Both people survive. One weight rises by 3 and the other falls by + 1, so the stock moves from 30 to 32 with no arrival or departure. + """ + account = reconcile_period( + frame(2040, {1: 10.0, 2: 20.0}), + frame(2041, {1: 13.0, 2: 19.0}), + opening_year=2040, + closing_year=2041, + ) + + assert account.weights.opening == 30.0 + assert account.weights.closing == 32.0 + assert account.weights.additions_total == 0.0 + assert account.weights.exits_total == 0.0 + assert account.weights.revaluation.carried == 2.0 + assert account.counts.additions_total == 0 + assert account.counts.exits_total == 0 + assert account.weight_residual == 0.0 + + +def test_within_period_entry_and_exit_nets_out(): + """Someone who arrives and leaves inside the period holds no stock. + + Persons 1 and 2 carry 10 and 20 through unchanged. Person 7 + scheduled-enters weighing 5 and emigrates the same year weighing 5. + They appear in neither frame, yet both flows are booked and the + closing stock is still 30. + """ + account = reconcile_period( + frame(2050, {1: 10.0, 2: 20.0}), + frame(2051, {1: 10.0, 2: 20.0}), + opening_year=2050, + closing_year=2051, + additions=[entry(7, 2051, weight=5.0)], + exits=[PopulationEvent(7, "emigration", 2051, weight=5.0)], + ) + + assert account.counts.opening == 2 + assert account.counts.closing == 2 + assert account.counts.carried == 2 + assert account.counts.entered == 0 + assert account.counts.exited == 0 + assert account.counts.transient == 1 + assert account.counts.additions_total == 1 + assert account.counts.exits_total == 1 + assert account.transient_person_ids == (7,) + assert account.added_person_ids == (7,) + assert account.departed_person_ids == (7,) + assert account.weights.additions_total == 5.0 + assert account.weights.exits_total == 5.0 + assert account.weights.revaluation.total == 0.0 + assert account.weights.closing == 30.0 + assert account.weight_residual == 0.0 + + +def test_transient_weight_change_is_reported_not_absorbed(): + """A transient whose weight moved keeps the identity honest. + + Person 7 enters weighing 5 and leaves weighing 8. The 3 is a + transient revaluation, not a silent gap in the closing stock. + """ + account = reconcile_period( + frame(2050, {1: 10.0}), + frame(2051, {1: 10.0}), + opening_year=2050, + closing_year=2051, + additions=[entry(7, 2051, weight=5.0)], + exits=[PopulationEvent(7, "emigration", 2051, weight=8.0)], + ) + + assert account.weights.additions_total == 5.0 + assert account.weights.exits_total == 8.0 + assert account.weights.revaluation.transient == 3.0 + assert account.weights.revaluation.carried == 0.0 + assert account.weights.closing == 10.0 + assert account.weight_residual == 0.0 + + +def test_declared_exit_weight_differing_from_opening_is_a_component(): + """A departure priced away from its opening weight is visible. + + Person 2 opens at 20 but is declared to leave at 12. The 8 lands + in the ``exiting`` revaluation rather than vanishing. + """ + account = reconcile_period( + frame(2060, {1: 10.0, 2: 20.0}), + frame(2061, {1: 10.0}), + opening_year=2060, + closing_year=2061, + exits=[death(2, 2061, weight=12.0)], + ) + + assert account.weights.exits_total == 12.0 + assert account.weights.revaluation.exiting == -8.0 + assert account.weights.revaluation.carried == 0.0 + assert account.weights.closing == 10.0 + assert account.weight_residual == 0.0 + + +def test_declared_entry_weight_differing_from_closing_is_a_component(): + """An arrival repriced after entry is visible too.""" + account = reconcile_period( + frame(2060, {1: 10.0}), + frame(2061, {1: 10.0, 9: 6.0}), + opening_year=2060, + closing_year=2061, + additions=[entry(9, 2061, weight=4.0)], + ) + + assert account.weights.additions_total == 4.0 + assert account.weights.revaluation.entrant == 2.0 + assert account.weights.closing == 16.0 + assert account.weight_residual == 0.0 + + +def test_complete_extinction_reconciles(): + """Everyone dies: the closing frame is empty and the stock is zero.""" + account = reconcile_period( + frame(2070, {1: 10.0, 2: 20.0, 3: 30.0}), + empty_frame(), + opening_year=2070, + closing_year=2071, + exits=[death(person, 2071) for person in (1, 2, 3)], + ) + + assert account.counts.opening == 3 + assert account.counts.closing == 0 + assert account.counts.exited == 3 + assert account.weights.opening == 60.0 + assert account.weights.closing == 0.0 + assert account.weights.exits_total == 60.0 + assert account.weight_residual == 0.0 + + +def test_empty_to_empty_reconciles(): + """An empty population that stays empty is a valid, flat account.""" + account = reconcile_period( + empty_frame(), + empty_frame(), + opening_year=2080, + closing_year=2081, + ) + + assert account.counts.to_dict()["opening"] == 0 + assert account.counts.closing == 0 + assert account.weights.opening == 0.0 + assert account.weights.closing == 0.0 + assert account.weights.revaluation.total == 0.0 + assert account.weight_residual == 0.0 + assert account.carried_person_ids == () + + +def test_population_may_start_empty_and_be_repopulated(): + """Additions in a later period do not need an earlier stock.""" + account = reconcile_period( + empty_frame(), + frame(2081, {4: 2.0, 5: 3.0}), + opening_year=2080, + closing_year=2081, + additions=[entry(4, 2081), entry(5, 2081)], + ) + + assert account.counts.opening == 0 + assert account.counts.entered == 2 + assert account.weights.additions_total == 5.0 + assert account.weights.closing == 5.0 + assert account.weight_residual == 0.0 + + +def test_a_quiet_period_may_precede_one_with_additions(): + """Chaining periods: the closing frame becomes the next opening.""" + first_year = frame(2090, {1: 10.0, 2: 20.0}) + second_year = frame(2091, {1: 10.0, 2: 20.0}) + third_year = frame(2092, {1: 10.0, 2: 20.0, 30: 5.0}) + + quiet = reconcile_period( + first_year, + second_year, + opening_year=2090, + closing_year=2091, + ) + active = reconcile_period( + second_year, + third_year, + opening_year=2091, + closing_year=2092, + additions=[entry(30, 2092)], + ) + + assert quiet.counts.additions_total == 0 + assert active.counts.additions_total == 1 + assert quiet.weights.closing == active.weights.opening == 30.0 + assert active.weights.closing == 35.0 + assert active.weight_residual == 0.0 + + +def test_zero_weight_rows_stay_visible_as_people(): + """A zero weight is a person, not an absence. + + Three rows open, one of them weighing nothing. The zero-weight + person then dies. Counts move by one; the stock does not move. + """ + account = reconcile_period( + frame(2100, {1: 0.0, 2: 5.0, 3: 7.0}), + frame(2101, {2: 5.0, 3: 7.0}), + opening_year=2100, + closing_year=2101, + exits=[death(1, 2101)], + ) + + assert account.counts.opening == 3 + assert account.counts.closing == 2 + assert account.counts.exited == 1 + assert account.exited_person_ids == (1,) + assert account.provenance["opening_zero_weight_rows"] == 1 + assert account.provenance["closing_zero_weight_rows"] == 0 + assert account.weights.exits_total == 0.0 + assert account.weights.opening == 12.0 + assert account.weights.closing == 12.0 + assert account.weight_residual == 0.0 + + +def test_a_zero_weight_arrival_is_still_an_arrival(): + """Arriving with no weight still books a person.""" + account = reconcile_period( + frame(2100, {1: 5.0}), + frame(2101, {1: 5.0, 2: 0.0}), + opening_year=2100, + closing_year=2101, + additions=[birth(2, 2101)], + ) + + assert account.counts.entered == 1 + assert account.counts.closing == 2 + assert account.weights.additions_total == 0.0 + assert account.provenance["closing_zero_weight_rows"] == 1 + + +# --------------------------------------------------------------------- +# stable summation and residual reporting +# --------------------------------------------------------------------- + + +def test_stable_summation_preserves_small_terms_in_this_example(): + """The stock is the exactly-rounded sum, which naive addition misses. + + ``1.0 + 1e16 + 1.0`` accumulated left to right loses both ones, and + 2.0 is representable at that magnitude, so the exactly-rounded + answer is 10000000000000002.0. + """ + weights = [1.0, 1e16, 1.0] + naive = 0.0 + for value in weights: + naive += value + assert naive == 1e16 + + rows = dict(zip((1, 2, 3), weights, strict=True)) + account = reconcile_period( + frame(2110, rows), + frame(2111, rows), + opening_year=2110, + closing_year=2111, + ) + + assert account.weights.opening == 10000000000000002.0 + assert account.weights.opening != naive + assert account.weight_residual == 0.0 + + +def test_totals_do_not_depend_on_row_order(): + """Reordering rows and declarations changes no reported number.""" + rows = {1: 0.1, 2: 0.2, 3: 0.3, 4: 1e15, 5: 0.7} + closing_rows = {1: 0.1, 2: 0.2, 4: 1e15, 5: 0.7} + forward = reconcile_period( + frame(2120, rows), + frame(2121, closing_rows), + opening_year=2120, + closing_year=2121, + exits=[death(3, 2121)], + ) + reversed_rows = dict(reversed(list(rows.items()))) + reversed_closing = dict(reversed(list(closing_rows.items()))) + backward = reconcile_period( + frame(2120, reversed_rows), + frame(2121, reversed_closing), + opening_year=2120, + closing_year=2121, + exits=[death(3, 2121)], + ) + + assert forward.to_dict() == backward.to_dict() + + +def test_a_representable_residual_is_reported_and_not_gated(): + """The residual is a reported number, never a pass/fail verdict. + + Nothing in the module compares it to a tolerance, so the field is + simply present and finite on a reconciled account. + """ + account = reconcile_period( + frame(2130, {1: 0.1, 2: 0.2}), + frame(2131, {1: 0.1}), + opening_year=2130, + closing_year=2131, + exits=[death(2, 2131)], + ) + + assert isinstance(account.weight_residual, float) + assert math.isfinite(account.weight_residual) + assert account.weight_residual != 0.0 + assert account.status == ENGINEERING_STATUS + assert account.count_residual == 0 + + +def test_the_module_defines_no_tolerance_or_acceptance_knob(): + """No threshold, tolerance or gate may creep into this interface.""" + banned = ("toler", "threshold", "atol", "rtol", "epsilon", "accept") + offenders = [ + name + for name in dir(accounting) + if not name.startswith("_") + and any(token in name.lower() for token in banned) + ] + assert offenders == [] + assert not any( + any(token in name.lower() for token in banned) + for name in signature(reconcile_period).parameters + ) + + +# --------------------------------------------------------------------- +# refusal to infer +# --------------------------------------------------------------------- + + +def test_a_disappearance_is_not_assumed_to_be_a_death(): + """An undeclared disappearance is refused, not booked as mortality.""" + with pytest.raises(PopulationReconciliationError) as caught: + reconcile_period( + frame(2140, {1: 10.0, 2: 20.0}), + frame(2141, {1: 10.0}), + opening_year=2140, + closing_year=2141, + ) + + assert kinds(caught.value) == [DiscrepancyKind.UNDECLARED_EXIT] + assert caught.value.discrepancies[0].person_id == 2 + assert "will not assume the person died" in ( + caught.value.discrepancies[0].detail + ) + + +def test_a_new_identifier_is_not_assumed_to_be_an_immigrant(): + """A new identifier flagged synthetic is still refused. + + The closing frame marks the row ``synthetic_entry`` exactly as + :func:`populace_dynamics.engine.steps.materialize_maternal_births` + would. The accountant ignores the flag: only a declaration counts. + """ + closing = frame(2141, {1: 10.0, 999: 3.0}, synthetic_entry=[False, True]) + with pytest.raises(PopulationReconciliationError) as caught: + reconcile_period( + frame(2140, {1: 10.0}), + closing, + opening_year=2140, + closing_year=2141, + ) + + assert kinds(caught.value) == [DiscrepancyKind.UNDECLARED_ADDITION] + assert caught.value.discrepancies[0].person_id == 999 + + +def test_both_unexplained_directions_are_reported_together(): + """One refusal carries every finding, ordered deterministically.""" + with pytest.raises(PopulationReconciliationError) as caught: + reconcile_period( + frame(2140, {1: 10.0, 2: 20.0}), + frame(2141, {1: 10.0, 3: 30.0}), + opening_year=2140, + closing_year=2141, + ) + + assert kinds(caught.value) == [ + DiscrepancyKind.UNDECLARED_ADDITION, + DiscrepancyKind.UNDECLARED_EXIT, + ] + assert [item.person_id for item in caught.value.discrepancies] == [3, 2] + assert "undeclared_addition=1" in str(caught.value) + assert "undeclared_exit=1" in str(caught.value) + + +def test_an_explicit_other_exit_is_accepted_when_the_caller_says_why(): + """The escape hatch is declaration, never inference.""" + account = reconcile_period( + frame(2140, {1: 10.0, 2: 20.0}), + frame(2141, {1: 10.0}), + opening_year=2140, + closing_year=2141, + exits=[ + PopulationEvent( + 2, + "other_exit", + 2141, + reason="removed by the caller's own roster surgery", + ) + ], + ) + + assert account.counts.exits_by_kind["other_exit"] == 1 + assert account.weights.exits_by_kind["other_exit"] == 20.0 + assert account.weight_residual == 0.0 + + +def test_refusal_payload_is_serializable(): + """A refusal can be written down without parsing its message.""" + with pytest.raises(PopulationReconciliationError) as caught: + reconcile_period( + frame(2140, {1: 10.0}), + frame(2141, {2: 10.0}), + opening_year=2140, + closing_year=2141, + ) + + payload = json.loads(json.dumps(caught.value.to_dict())) + assert payload["error"] == "population_reconciliation_error" + assert payload["interface_version"] == ACCOUNTING_INTERFACE_VERSION + assert {item["kind"] for item in payload["discrepancies"]} == { + "undeclared_addition", + "undeclared_exit", + } + + +# --------------------------------------------------------------------- +# refusals survive pickling, copying and process boundaries +# --------------------------------------------------------------------- + +REFUSAL_ROUND_TRIPS = [ + pytest.param( + lambda value: pickle.loads(pickle.dumps(value, protocol=0)), + id="pickle-protocol-0", + ), + pytest.param( + lambda value: pickle.loads( + pickle.dumps(value, protocol=pickle.HIGHEST_PROTOCOL) + ), + id="pickle-highest-protocol", + ), + pytest.param(copy.copy, id="copy"), + pytest.param(copy.deepcopy, id="deepcopy"), +] + + +def _undeclared_exit_refusal() -> PopulationReconciliationError: + with pytest.raises(PopulationReconciliationError) as caught: + reconcile_period( + frame(2140, {1: 10.0, 2: 20.0}), + frame(2141, {1: 10.0}), + opening_year=2140, + closing_year=2141, + ) + return caught.value + + +@pytest.mark.parametrize("round_trip", REFUSAL_ROUND_TRIPS) +def test_reconciliation_refusal_survives_pickle_and_copy(round_trip): + """The typed findings are rebuilt, not dropped or re-parsed.""" + error = _undeclared_exit_refusal() + error.add_note("synthetic caller note") + + restored = round_trip(error) + + assert type(restored) is PopulationReconciliationError + assert restored is not error + assert restored.args == error.args + assert str(restored) == str(error) + assert isinstance(restored.discrepancies, tuple) + assert restored.discrepancies == error.discrepancies + assert kinds(restored) == [DiscrepancyKind.UNDECLARED_EXIT] + assert restored.discrepancies[0].person_id == 2 + assert restored.to_dict() == error.to_dict() + assert restored.__notes__ == ["synthetic caller note"] + + +def _unknown_kind_refusal() -> None: + PopulationEvent(1, "abduction", 2141) + + +def _unpriced_transient_refusal() -> None: + reconcile_period( + frame(2150, {1: 10.0}), + frame(2151, {1: 10.0}), + opening_year=2150, + closing_year=2151, + additions=[entry(7, 2151)], + exits=[death(7, 2151, weight=1.0)], + ) + + +@pytest.mark.parametrize("round_trip", REFUSAL_ROUND_TRIPS) +@pytest.mark.parametrize( + ("refuse", "cause_type"), + [ + pytest.param(_unknown_kind_refusal, ValueError, id="raise-from"), + pytest.param(_unpriced_transient_refusal, None, id="raise-from-none"), + ], +) +def test_input_refusal_keeps_its_explicit_cause_chain( + round_trip, refuse, cause_type +): + """``raise ... from`` metadata is rebuilt with the refusal.""" + with pytest.raises(PopulationAccountingInputError) as caught: + refuse() + error = caught.value + + restored = round_trip(error) + + assert type(restored) is PopulationAccountingInputError + assert str(restored) == str(error) + assert restored.__suppress_context__ is True + if cause_type is None: + assert restored.__cause__ is None + else: + assert type(restored.__cause__) is cause_type + assert str(restored.__cause__) == str(error.__cause__) + + +def test_reconciliation_refusal_crosses_a_process_pool(): + """A worker's refusal reaches the parent typed, not as a broken pool.""" + context = multiprocessing.get_context("spawn") + with ProcessPoolExecutor(max_workers=1, mp_context=context) as pool: + future = pool.submit( + reconcile_period, + frame(2140, {1: 10.0, 2: 20.0}), + frame(2141, {1: 10.0}), + opening_year=2140, + closing_year=2141, + ) + with pytest.raises(PopulationReconciliationError) as caught: + future.result(timeout=120) + + assert kinds(caught.value) == [DiscrepancyKind.UNDECLARED_EXIT] + assert caught.value.discrepancies[0].person_id == 2 + assert caught.value.to_dict()["discrepancies"] == [ + { + "kind": "undeclared_exit", + "person_id": 2, + "detail": caught.value.discrepancies[0].detail, + } + ] + + +# --------------------------------------------------------------------- +# rejecting colliding, duplicated and impossible declarations +# --------------------------------------------------------------------- + + +def test_duplicate_addition_declarations_are_rejected(): + with pytest.raises(PopulationReconciliationError) as caught: + reconcile_period( + frame(2150, {1: 10.0}), + frame(2151, {1: 10.0, 2: 5.0}), + opening_year=2150, + closing_year=2151, + additions=[birth(2, 2151), entry(2, 2151)], + ) + + assert DiscrepancyKind.DUPLICATE_ADDITION in kinds(caught.value) + + +def test_duplicate_exit_declarations_are_rejected(): + with pytest.raises(PopulationReconciliationError) as caught: + reconcile_period( + frame(2150, {1: 10.0, 2: 5.0}), + frame(2151, {1: 10.0}), + opening_year=2150, + closing_year=2151, + exits=[ + death(2, 2151), + PopulationEvent(2, "emigration", 2151), + ], + ) + + assert DiscrepancyKind.DUPLICATE_EXIT in kinds(caught.value) + + +def test_an_addition_colliding_with_the_opening_roster_is_rejected(): + """Declaring an arrival for somebody already present is a collision.""" + with pytest.raises(PopulationReconciliationError) as caught: + reconcile_period( + frame(2150, {1: 10.0, 2: 5.0}), + frame(2151, {1: 10.0, 2: 5.0}), + opening_year=2150, + closing_year=2151, + additions=[entry(2, 2151)], + ) + + assert kinds(caught.value) == [ + DiscrepancyKind.ADDITION_COLLIDES_WITH_OPENING + ] + + +def test_an_exit_for_someone_never_present_is_rejected(): + """Leaving requires having been here: no exit before entry.""" + with pytest.raises(PopulationReconciliationError) as caught: + reconcile_period( + frame(2150, {1: 10.0}), + frame(2151, {1: 10.0}), + opening_year=2150, + closing_year=2151, + exits=[death(404, 2151)], + ) + + assert kinds(caught.value) == [DiscrepancyKind.EXIT_WITHOUT_PRESENCE] + assert caught.value.discrepancies[0].person_id == 404 + + +def test_an_exit_contradicted_by_the_closing_frame_is_rejected(): + """Declaring a death for somebody still on the roster is refused.""" + with pytest.raises(PopulationReconciliationError) as caught: + reconcile_period( + frame(2150, {1: 10.0, 2: 5.0}), + frame(2151, {1: 10.0, 2: 5.0}), + opening_year=2150, + closing_year=2151, + exits=[death(2, 2151)], + ) + + assert kinds(caught.value) == [ + DiscrepancyKind.EXIT_CONTRADICTED_BY_CLOSING + ] + + +def test_an_arrival_that_never_lands_is_rejected(): + """A declared arrival absent at the close needs a declared exit.""" + with pytest.raises(PopulationReconciliationError) as caught: + reconcile_period( + frame(2150, {1: 10.0}), + frame(2151, {1: 10.0}), + opening_year=2150, + closing_year=2151, + additions=[birth(77, 2151, weight=1.0)], + ) + + assert kinds(caught.value) == [DiscrepancyKind.ADDITION_ABSENT_AT_CLOSE] + assert caught.value.discrepancies[0].person_id == 77 + + +def test_a_transient_without_declared_weights_is_rejected(): + """A person in neither frame must price both of their own events.""" + with pytest.raises(PopulationAccountingInputError, match="explicit"): + reconcile_period( + frame(2150, {1: 10.0}), + frame(2151, {1: 10.0}), + opening_year=2150, + closing_year=2151, + additions=[entry(7, 2151)], + exits=[death(7, 2151, weight=1.0)], + ) + + +# --------------------------------------------------------------------- +# malformed declarations +# --------------------------------------------------------------------- + + +def test_an_unknown_event_kind_is_rejected(): + with pytest.raises(PopulationAccountingInputError, match="unknown"): + PopulationEvent(1, "abduction", 2020) + + +def test_an_other_kind_without_a_reason_is_rejected(): + with pytest.raises(PopulationAccountingInputError, match="reason"): + PopulationEvent(1, "other_exit", 2020) + with pytest.raises(PopulationAccountingInputError, match="reason"): + PopulationEvent(1, "other_entry", 2020, reason=" ") + + +def test_a_boolean_person_identifier_is_rejected(): + with pytest.raises(PopulationAccountingInputError, match="integer"): + PopulationEvent(True, "death", 2020) + + +def test_a_float_person_identifier_is_rejected(): + with pytest.raises(PopulationAccountingInputError, match="integer"): + PopulationEvent(1.0, "death", 2020) + + +@pytest.mark.parametrize("value", [-1.0, float("nan"), float("inf")]) +def test_an_invalid_declared_weight_is_rejected(value): + with pytest.raises(PopulationAccountingInputError, match="non-negative"): + PopulationEvent(1, "death", 2020, weight=value) + + +def test_a_non_string_reason_is_rejected(): + with pytest.raises(PopulationAccountingInputError, match="string"): + PopulationEvent(1, "death", 2020, reason=7) + + +def test_a_departure_declared_as_an_arrival_is_rejected(): + with pytest.raises(PopulationAccountingInputError, match="separately"): + reconcile_period( + frame(2160, {1: 10.0}), + frame(2161, {1: 10.0}), + opening_year=2160, + closing_year=2161, + additions=[death(1, 2161)], + ) + + +def test_an_arrival_declared_as_a_departure_is_rejected(): + with pytest.raises(PopulationAccountingInputError, match="separately"): + reconcile_period( + frame(2160, {1: 10.0}), + frame(2161, {1: 10.0}), + opening_year=2160, + closing_year=2161, + exits=[birth(1, 2161)], + ) + + +def test_an_event_booked_to_the_wrong_year_is_rejected(): + with pytest.raises(PopulationAccountingInputError, match="closing year"): + reconcile_period( + frame(2160, {1: 10.0, 2: 1.0}), + frame(2161, {1: 10.0}), + opening_year=2160, + closing_year=2161, + exits=[death(2, 2160)], + ) + + +def test_a_non_event_in_a_declaration_sequence_is_rejected(): + with pytest.raises( + PopulationAccountingInputError, match="PopulationEvent" + ): + reconcile_period( + frame(2160, {1: 10.0}), + frame(2161, {1: 10.0}), + opening_year=2160, + closing_year=2161, + exits=[{"person_id": 1, "kind": "death"}], + ) + + +def test_a_generator_of_declarations_is_rejected(): + """Declarations must be a re-readable sequence, not a one-shot stream.""" + with pytest.raises(PopulationAccountingInputError, match="sequence"): + reconcile_period( + frame(2160, {1: 10.0}), + frame(2161, {1: 10.0}), + opening_year=2160, + closing_year=2161, + exits=(event for event in ()), + ) + + +# --------------------------------------------------------------------- +# malformed frames and period coordinates +# --------------------------------------------------------------------- + + +def test_a_missing_column_is_rejected(): + opening = frame(2170, {1: 10.0}).drop(columns=["weight"]) + with pytest.raises(PopulationAccountingInputError, match="missing"): + reconcile_period( + opening, + frame(2171, {1: 10.0}), + opening_year=2170, + closing_year=2171, + ) + + +def test_a_duplicate_person_row_is_rejected(): + opening = pd.DataFrame( + { + "person_id": [1, 1], + "year": [2170, 2170], + "weight": [10.0, 10.0], + } + ) + with pytest.raises(PopulationAccountingInputError, match="duplicate"): + reconcile_period( + opening, + frame(2171, {1: 10.0}), + opening_year=2170, + closing_year=2171, + ) + + +def test_a_float_identifier_column_is_rejected(): + opening = frame(2170, {1: 10.0}) + opening["person_id"] = opening["person_id"].astype(np.float64) + with pytest.raises(PopulationAccountingInputError, match="integer dtype"): + reconcile_period( + opening, + frame(2171, {1: 10.0}), + opening_year=2170, + closing_year=2171, + ) + + +def test_a_null_identifier_is_rejected(): + opening = frame(2170, {1: 10.0}) + opening["person_id"] = pd.array([pd.NA], dtype="Int64") + with pytest.raises(PopulationAccountingInputError, match="null"): + reconcile_period( + opening, + frame(2171, {1: 10.0}), + opening_year=2170, + closing_year=2171, + ) + + +@pytest.mark.parametrize( + ("value", "message"), + [ + (float("nan"), "null"), + (float("inf"), "non-finite"), + (-0.5, "negative"), + ], +) +def test_an_invalid_frame_weight_is_rejected(value, message): + closing = frame(2171, {1: value}) + with pytest.raises(PopulationAccountingInputError, match=message): + reconcile_period( + frame(2170, {1: 10.0}), + closing, + opening_year=2170, + closing_year=2171, + ) + + +def test_a_row_carrying_the_wrong_year_is_rejected(): + closing = frame(2171, {1: 10.0, 2: 5.0}) + closing.loc[1, "year"] = 2172 + with pytest.raises(PopulationAccountingInputError, match="must carry"): + reconcile_period( + frame(2170, {1: 10.0, 2: 5.0}), + closing, + opening_year=2170, + closing_year=2171, + ) + + +def test_a_non_annual_period_is_rejected(): + with pytest.raises(PopulationAccountingInputError, match="annual"): + reconcile_period( + frame(2170, {1: 10.0}), + frame(2172, {1: 10.0}), + opening_year=2170, + closing_year=2172, + ) + + +def test_a_non_frame_input_is_rejected(): + with pytest.raises(PopulationAccountingInputError, match="DataFrame"): + reconcile_period( + {"person_id": [1]}, + frame(2171, {1: 10.0}), + opening_year=2170, + closing_year=2171, + ) + + +def test_a_float_year_coordinate_is_rejected(): + with pytest.raises(PopulationAccountingInputError, match="integer year"): + reconcile_period( + frame(2170, {1: 10.0}), + frame(2171, {1: 10.0}), + opening_year=2170.0, + closing_year=2171, + ) + + +# --------------------------------------------------------------------- +# purity, immutability and the serializable payload +# --------------------------------------------------------------------- + + +def test_input_frames_are_not_mutated(): + """The accountant reads; it never writes back. + + The rows are supplied deliberately out of identifier order and with + a non-default index, so an in-place sort, a reindex or an added + bookkeeping column would all show up here. + """ + opening = frame(2180, {2: 20.0, 1: 10.0}, note=["b", "a"]) + closing = frame(2181, {3: 1.0, 2: 25.0}, note=["c", "b"]) + opening.index = pd.Index([11, 10], name="row") + closing.index = pd.Index([13, 12], name="row") + opening_before = opening.copy(deep=True) + closing_before = closing.copy(deep=True) + + reconcile_period( + opening, + closing, + opening_year=2180, + closing_year=2181, + additions=[birth(3, 2181)], + exits=[death(1, 2181)], + ) + + pd.testing.assert_frame_equal(opening, opening_before) + pd.testing.assert_frame_equal(closing, closing_before) + assert list(opening["person_id"]) == [2, 1] + assert list(closing["person_id"]) == [3, 2] + assert list(opening.columns) == ["person_id", "year", "weight", "note"] + assert list(closing.columns) == ["person_id", "year", "weight", "note"] + assert list(opening.index) == [11, 10] + assert list(closing.index) == [13, 12] + + +def test_the_account_is_immutable(): + account = reconcile_period( + frame(2180, {1: 10.0}), + frame(2181, {1: 10.0}), + opening_year=2180, + closing_year=2181, + ) + + with pytest.raises(FrozenInstanceError): + account.weight_residual = 1.0 + with pytest.raises(TypeError): + account.provenance["opening_rows"] = 99 + with pytest.raises(TypeError): + account.counts.additions_by_kind["birth"] = 99 + + +def test_the_account_payload_is_json_serializable_and_flat(): + account = reconcile_period( + frame(2180, {1: 10.0, 2: 20.0}), + frame(2181, {1: 10.0, 3: 4.0}), + opening_year=2180, + closing_year=2181, + additions=[birth(3, 2181, source="synthetic.fertility")], + exits=[death(2, 2181, source="synthetic.mortality")], + ) + + payload = account.to_dict() + round_tripped = json.loads(json.dumps(payload)) + assert round_tripped == payload + + def leaves(value): + if isinstance(value, dict): + for item in value.values(): + yield from leaves(item) + elif isinstance(value, list): + for item in value: + yield from leaves(item) + else: + yield value + + assert all( + isinstance(leaf, (int, float, str, bool)) or leaf is None + for leaf in leaves(payload) + ) + assert payload["provenance"]["declaration_sources"] == [ + "synthetic.fertility", + "synthetic.mortality", + ] + assert payload["provenance"]["inference"].startswith("none:") + + +def test_the_status_is_explicitly_engineering_only(): + account = reconcile_period( + frame(2180, {1: 10.0}), + frame(2181, {1: 10.0}), + opening_year=2180, + closing_year=2181, + ) + + assert account.status == ENGINEERING_STATUS + assert account.status == "engineering-accounting-coherence-only" + note = account.status_note.lower() + assert "not scientific acceptance" in note + assert "not a benchmark comparison" in note + assert "not a gate outcome" in note + + +def test_a_discrepancy_record_is_serializable(): + record = AccountingDiscrepancy( + kind=DiscrepancyKind.UNDECLARED_EXIT, + person_id=5, + detail="example", + ) + assert json.loads(json.dumps(record.to_dict())) == { + "kind": "undeclared_exit", + "person_id": 5, + "detail": "example", + } + + +def test_an_event_record_is_serializable(): + event = birth(3, 2181, source="synthetic.fertility") + assert json.loads(json.dumps(event.to_dict())) == { + "person_id": 3, + "kind": "birth", + "year": 2181, + "weight": None, + "reason": "", + "source": "synthetic.fertility", + } + assert event.is_addition is True + assert death(3, 2181).is_addition is False + assert set(PopulationEventKind) == ( + accounting.ADDITION_KINDS | accounting.EXIT_KINDS + ) + + +# --------------------------------------------------------------------- +# the real ProjectionEngine, driven by recording synthetic adapters +# --------------------------------------------------------------------- + + +@dataclass +class EventLog: + """Declarations captured from the synthetic adapters themselves.""" + + additions: list[PopulationEvent] = field(default_factory=list) + exits: list[PopulationEvent] = field(default_factory=list) + + def for_year(self, year: int) -> tuple[list, list]: + return ( + [item for item in self.additions if item.year == year], + [item for item in self.exits if item.year == year], + ) + + +def _recording_modules( + log: EventLog, + deaths_by_year: dict[int, tuple[int, ...]], + births_by_year: dict[int, tuple[int, ...]], +) -> PeriodModules: + """Build eight adapters that record every presence change they make. + + These are deliberately trivial: no fitted component, no draw, no + demography. Their only job is to move people in and out of the + roster through the engine's real seams and write down what they + did. + """ + + def mortality(current, context, rng): + del rng + doomed = set(deaths_by_year.get(context.year, ())) + leaving = current["person_id"].isin(doomed) + for row in current.loc[leaving].to_dict("records"): + log.exits.append( + PopulationEvent( + person_id=int(row["person_id"]), + kind=PopulationEventKind.DEATH, + year=context.year, + weight=float(row["weight"]), + source="synthetic.mortality", + ) + ) + return current.loc[~leaving].reset_index(drop=True) + + def aging(current, context, rng): + del rng + out = current.copy() + out["year"] = context.year + out["age"] = out["age"].to_numpy(dtype=np.int64) + 1 + return out + + def marital_core(current, context, rng): + del current, context, rng + return MaritalStepResult( + sim_years=pd.DataFrame(), births=pd.DataFrame() + ) + + def fertility(current, context, marital, rng): + del marital, rng + parents = births_by_year.get(context.year, ()) + if not parents: + return current + weight_of = dict( + zip( + current["person_id"].tolist(), + current["weight"].tolist(), + strict=True, + ) + ) + child_ids = context.synthetic_id_allocator.allocate(len(parents)) + children = pd.DataFrame( + { + "person_id": child_ids, + "year": np.full(len(parents), context.year, dtype=np.int64), + "age": np.zeros(len(parents), dtype=np.int64), + "weight": np.asarray( + [weight_of[parent] for parent in parents], + dtype=np.float64, + ), + } + ) + for child_id in child_ids.tolist(): + log.additions.append( + PopulationEvent( + person_id=int(child_id), + kind=PopulationEventKind.BIRTH, + year=context.year, + source="synthetic.fertility", + ) + ) + return pd.concat([current, children], ignore_index=True) + + def unchanged(current, context, rng): + del context, rng + return current + + def unchanged_reader(current, context, marital, rng): + del context, marital, rng + return current + + return PeriodModules( + mortality=mortality, + aging=aging, + marital_core=marital_core, + fertility=fertility, + disability=unchanged, + earnings=unchanged, + claiming=unchanged, + household_composition=unchanged_reader, + ) + + +def _run_projection() -> tuple[object, EventLog]: + """Project 2020-2023 with a birth, a death and a transient entrant. + + The 2022 scheduled entrant is registered on the metadata seam the + loop reads, joins the roster before mortality, and dies in the same + wave -- so they appear in no projected slice at all. + """ + log = EventLog() + initial = frame(2020, {1: 10.0, 2: 20.0, 3: 30.0}, age=[40, 41, 42]) + entrants_2022 = frame(2021, {100: 7.0}, age=[50]) + schedule = {2022: entrants_2022} + for year, entrant_frame in schedule.items(): + for row in entrant_frame.to_dict("records"): + log.additions.append( + PopulationEvent( + person_id=int(row["person_id"]), + kind=PopulationEventKind.SCHEDULED_ENTRY, + year=year, + weight=float(row["weight"]), + source="loop.m6_scheduled_entries_by_year", + ) + ) + engine = ProjectionEngine( + _recording_modules( + log, + deaths_by_year={2021: (3,), 2022: (100,)}, + births_by_year={2021: (1,)}, + ) + ) + result = engine.project( + initial, + end_year=2023, + draw_index=0, + metadata={SCHEDULED_ENTRIES_KEY: schedule}, + ) + return result, log + + +def test_projection_slices_reconcile_year_by_year(): + """Every wave of a real projection balances against its records. + + The projection is small enough to state outright: 2021 loses person + 3 (weight 30) and gains one child carrying the mother's weight 10, + so the stock goes 60 -> 40. 2022 admits and then buries person 100 + (weight 7), leaving the stock at 40. 2023 does nothing. + """ + result, log = _run_projection() + assert [int(slice_.iloc[0]["year"]) for slice_ in result.slices] == [ + 2020, + 2021, + 2022, + 2023, + ] + + accounts = [] + for index in range(len(result.slices) - 1): + opening_year = 2020 + index + additions, exits = log.for_year(opening_year + 1) + accounts.append( + reconcile_period( + result.slices[index], + result.slices[index + 1], + opening_year=opening_year, + closing_year=opening_year + 1, + additions=additions, + exits=exits, + ) + ) + + first, second, third = accounts + + assert first.counts.opening == 3 + assert first.counts.closing == 3 + assert first.counts.entered == 1 + assert first.counts.exited == 1 + assert first.counts.transient == 0 + assert first.counts.additions_by_kind["birth"] == 1 + assert first.counts.exits_by_kind["death"] == 1 + assert first.weights.opening == 60.0 + assert first.weights.closing == 40.0 + assert first.weights.additions_total == 10.0 + assert first.weights.exits_total == 30.0 + assert first.weights.revaluation.total == 0.0 + assert first.exited_person_ids == (3,) + + assert second.counts.opening == 3 + assert second.counts.closing == 3 + assert second.counts.carried == 3 + assert second.counts.transient == 1 + assert second.transient_person_ids == (100,) + assert second.counts.additions_by_kind["scheduled_entry"] == 1 + assert second.counts.exits_by_kind["death"] == 1 + assert second.weights.opening == 40.0 + assert second.weights.closing == 40.0 + assert second.weights.additions_total == 7.0 + assert second.weights.exits_total == 7.0 + assert second.weights.revaluation.total == 0.0 + + assert third.counts.carried == 3 + assert third.counts.additions_total == 0 + assert third.counts.exits_total == 0 + assert third.weights.closing == 40.0 + + assert [account.count_residual for account in accounts] == [0, 0, 0] + assert [account.weight_residual for account in accounts] == [ + 0.0, + 0.0, + 0.0, + ] + assert {account.status for account in accounts} == {ENGINEERING_STATUS} + + +def test_the_transient_entrant_never_appears_in_a_projected_slice(): + """The within-period arrival is invisible to the frames alone. + + Only the captured declarations show that person 100 was ever in the + population. Endpoint frames cannot expose omission of both events; + the accountant does not verify event-log completeness. + """ + result, log = _run_projection() + for slice_ in result.slices: + assert 100 not in set(slice_["person_id"].tolist()) + assert [item.person_id for item in log.additions if item.year == 2022] == [ + 100 + ] + assert [item.person_id for item in log.exits if item.year == 2022] == [100] + + +def test_dropping_one_captured_death_makes_the_projection_refuse(): + """Omitting a real event is caught against real engine output.""" + result, log = _run_projection() + additions, exits = log.for_year(2021) + withheld = [item for item in exits if item.person_id != 3] + assert len(withheld) == len(exits) - 1 + + with pytest.raises(PopulationReconciliationError) as caught: + reconcile_period( + result.slices[0], + result.slices[1], + opening_year=2020, + closing_year=2021, + additions=additions, + exits=withheld, + ) + + assert kinds(caught.value) == [DiscrepancyKind.UNDECLARED_EXIT] + assert caught.value.discrepancies[0].person_id == 3 + + +def test_dropping_one_captured_birth_makes_the_projection_refuse(): + """A synthetic identifier is never quietly accepted as a birth.""" + result, log = _run_projection() + _, exits = log.for_year(2021) + + with pytest.raises(PopulationReconciliationError) as caught: + reconcile_period( + result.slices[0], + result.slices[1], + opening_year=2020, + closing_year=2021, + additions=[], + exits=exits, + ) + + assert kinds(caught.value) == [DiscrepancyKind.UNDECLARED_ADDITION] + + +# Root regression cases to append after builder releases source ownership. + + +@pytest.mark.parametrize("column", ["person_id", "year"]) +@pytest.mark.parametrize( + "dtype,value", + [(np.uint64, 2**64 - 1), (object, 2**63), (object, -(2**63) - 1)], +) +def test_frame_integer_domain_cannot_wrap(column, dtype, value): + opening = frame(2020, {1: 1.0}) + opening[column] = pd.Series([value], dtype=dtype) + with pytest.raises(PopulationAccountingInputError): + reconcile_period( + opening, + frame(2021, {-1: 1.0}), + opening_year=2020, + closing_year=2021, + ) + + +@pytest.mark.parametrize("field_name", ["person_id", "year"]) +@pytest.mark.parametrize("value", [2**63, -(2**63) - 1, np.uint64(2**64 - 1)]) +def test_event_integer_domain_matches_frames(field_name, value): + kwargs = {"person_id": 1, "kind": "death", "year": 2021} + kwargs[field_name] = value + with pytest.raises(PopulationAccountingInputError): + PopulationEvent(**kwargs) + + +@pytest.mark.parametrize( + "value,dtype", + [ + ("2.0", None), + (2 + 3j, None), + (True, object), + (np.bool_(True), object), + (1 + 0j, object), + (10**400, object), + ], +) +def test_frame_weights_refuse_lossy_or_non_numeric_values(value, dtype): + opening = frame(2020, {1: 1.0}) + opening["weight"] = pd.Series([value], dtype=dtype) + with pytest.raises(PopulationAccountingInputError): + reconcile_period( + opening, + frame(2021, {1: 1.0}), + opening_year=2020, + closing_year=2021, + ) + + +def test_unrepresentable_weight_total_has_typed_refusal(): + with pytest.raises( + PopulationAccountingInputError, match="represent|overflow|finite" + ): + reconcile_period( + frame(2020, {1: 1e308, 2: 1e308}), + frame(2021, {1: 1e308, 2: 1e308}), + opening_year=2020, + closing_year=2021, + ) + + +def test_unrepresentable_event_weight_has_typed_refusal(): + with pytest.raises(PopulationAccountingInputError): + PopulationEvent(1, "death", 2021, weight=10**400) + + +def test_serialized_provenance_cannot_mutate_original_or_other_payload(): + account = reconcile_period( + empty_frame(), + frame(2021, {1: 1.0}), + opening_year=2020, + closing_year=2021, + additions=[birth(1, 2021, source="synthetic.birth")], + ) + first, second = account.to_dict(), account.to_dict() + first["provenance"]["declaration_sources"].append("invented") + assert second["provenance"]["declaration_sources"] == ["synthetic.birth"] + assert account.to_dict()["provenance"]["declaration_sources"] == [ + "synthetic.birth" + ] + assert account.provenance["declaration_sources"] == ("synthetic.birth",) + + +def test_omitting_both_transient_events_is_not_observable_at_endpoints(): + result, log = _run_projection() + additions, exits = log.for_year(2022) + assert additions and exits + account = reconcile_period( + result.slices[1], + result.slices[2], + opening_year=2021, + closing_year=2022, + ) + assert account.counts.transient == 0 + assert account.count_residual == 0 + assert account.provenance["event_log_completeness_verified"] is False + + +@pytest.mark.parametrize("person_id", [-(2**63), 2**63 - 1]) +def test_signed_int64_boundary_ids_remain_exact(person_id): + account = reconcile_period( + frame(2020, {person_id: 1.0}), + empty_frame(), + opening_year=2020, + closing_year=2021, + exits=[death(person_id, 2021)], + ) + assert account.exited_person_ids == (person_id,) + assert account.weights.exits_total == 1.0 + + +def test_nullable_integer_columns_and_real_object_weights_are_supported(): + opening = pd.DataFrame( + { + "person_id": pd.Series([1], dtype="Int64"), + "year": pd.Series([2020], dtype="Int64"), + "weight": pd.Series([np.float64(2.5)], dtype=object), + } + ) + account = reconcile_period( + opening, + frame(2021, {1: 2.5}), + opening_year=2020, + closing_year=2021, + ) + assert account.carried_person_ids == (1,) + assert account.weights.opening == 2.5 + + +@pytest.mark.skipif( + np.finfo(np.longdouble).minexp >= np.finfo(np.float64).minexp, + reason="platform longdouble has no wider exponent range than binary64", +) +@pytest.mark.parametrize("sign", [-1, 1]) +@pytest.mark.parametrize("via", ["frame", "event"]) +def test_extended_weight_cannot_lose_sign_or_mass_on_conversion(sign, via): + weight = sign * np.nextafter(np.longdouble(0), np.longdouble(1)) + assert weight != 0 + assert float(weight) == 0.0 + with pytest.raises(PopulationAccountingInputError): + if via == "event": + death(1, 2021, weight=weight) + else: + opening = frame(2020, {1: 1.0}) + opening["weight"] = np.array([weight], dtype=np.longdouble) + reconcile_period( + opening, + frame(2021, {1: 0.0}), + opening_year=2020, + closing_year=2021, + ) diff --git a/tests/test_mortality_observer.py b/tests/test_mortality_observer.py new file mode 100644 index 00000000..daf7f7a7 --- /dev/null +++ b/tests/test_mortality_observer.py @@ -0,0 +1,366 @@ +"""Invented mortality steps; no native population or fitted-data execution.""" + +import json +from dataclasses import FrozenInstanceError, replace +from unittest.mock import patch + +import numpy as np +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal + +from populace_dynamics.engine.loop import PeriodContext +from populace_dynamics.engine.rng import ProjectionRNGRegistry +from populace_dynamics.engine.steps import ( + AgeSexMortalityModel, + apply_mortality, +) +from populace_dynamics.mortality_observer import ( + MortalityStepObservation, + observe_mortality, +) +from populace_dynamics.person_identity import PersonIdentity, PersonIdentityMap + + +@pytest.fixture +def inputs(): + identities = [ + PersonIdentity("int64", 2**53 + 1), + PersonIdentity("uint64", 2**64 - 1), + PersonIdentity("string", "01"), + PersonIdentity("string", "1"), + ] + mapping = PersonIdentityMap.from_identities(identities) + frame = pd.DataFrame( + { + "person_id": np.array([3, 1, 0, 2], dtype="int64"), + "age": np.array([120, 0, 30, 65], dtype="int64"), + "year": np.full(4, 2014, dtype="int64"), + "sex": ["male", "female", "female", "male"], + "untouched": pd.array([1, None, 3, 4], dtype="Int64"), + }, + index=[11, 12, 13, 14], + ) + model = AgeSexMortalityModel( + ((0, 64), (65, 120)), + { + ("0-64", "female"): 0.3, + ("0-64", "male"): 0.7, + ("65+", "female"): 0.2, + ("65+", "male"): 0.8, + }, + ) + context = PeriodContext(1, 2015, 3, {}) + kwargs = dict( + model=model, + identity_map=mapping, + realization_id="invented-draw-3", + source_contract_digest="a" * 64, + ) + return frame, context, kwargs + + +@pytest.mark.parametrize("registry", [False, True]) +@pytest.mark.parametrize("probability", [None, 0.0, 1.0]) +@pytest.mark.parametrize("empty", [False, True]) +def test_actual_step_matches_output_and_rng( + inputs, registry, probability, empty +): + frame, context, kwargs = inputs + if empty: + frame = frame.iloc[:0] + if probability is not None: + kwargs["model"] = AgeSexMortalityModel( + kwargs["model"].bands, + dict.fromkeys(kwargs["model"].probability, probability), + ) + if registry: + context = replace( + context, + rng_registry=ProjectionRNGRegistry(3, 5), + person_ordinals={0: 8, 1: 6, 2: 9, 3: 7}, + ) + direct_rng, observed_rng = np.random.default_rng( + 81 + ), np.random.default_rng(81) + original = frame.copy(deep=True) + expected = apply_mortality( + frame, context, direct_rng, model=kwargs["model"] + ) + with patch( + "populace_dynamics.mortality_observer.apply_mortality", + wraps=apply_mortality, + ) as called: + actual, record = observe_mortality( + frame, context, observed_rng, **kwargs + ) + assert called.call_count == 1 + assert_frame_equal(actual, expected) + assert_frame_equal(frame, original) + assert direct_rng.bit_generator.state == observed_rng.bit_generator.state + assert record.pre_keys == tuple(sorted(frame.person_id)) + assert record.post_keys == tuple(actual.person_id) + assert record.target_year == 2015 + assert record.identity_map.reverse_rows(record.pre_keys) == kwargs[ + "identity_map" + ].reverse_rows(record.pre_keys) + restored = MortalityStepObservation.from_json( + record.to_json(), + identity_map=kwargs["identity_map"], + expected_digest=record.digest, + ) + assert restored == record + assert restored.to_json() == record.to_json() + + +@pytest.mark.parametrize( + "column,values", + [ + ("age", [-1, 0, 30, 65]), + ("age", [121, 0, 30, 65]), + ("age", [120.0, 0.0, 30.0, 65.0]), + ("age", [120, 0, 30.9, 65]), + ("age", [120, 0, None, 65]), + ("sex", ["other"] * 4), + ("sex", [None] * 4), + ("person_id", [0, 0, 1, 2]), + ("person_id", [0, 1, 2, 99]), + ("person_id", [0.0, 1.0, 2.0, 3.0]), + ("person_id", [True, False, True, False]), + ("year", [2013] * 4), + ], +) +def test_invalid_frame_fails_before_rng(inputs, column, values): + frame, context, kwargs = inputs + frame[column] = values + rng = np.random.default_rng(22) + state = rng.bit_generator.state + with patch( + "populace_dynamics.mortality_observer.apply_mortality" + ) as called: + with pytest.raises(ValueError): + observe_mortality(frame, context, rng, **kwargs) + called.assert_not_called() + assert state == rng.bit_generator.state + + +@pytest.mark.parametrize( + "changes", + [ + {"period_index": 0}, + {"period_index": 1.0}, + {"draw_index": -1}, + {"year": True}, + {"rng_registry": ProjectionRNGRegistry(2, 5)}, + {"rng_registry": ProjectionRNGRegistry(3, 0)}, + {"person_ordinals": {0: 0, 1: 0, 2: 2, 3: 3}}, + {"person_ordinals": {0: 0, 1: 1}}, + {"person_ordinals": {0: 0, 1: 1, 2: 2, 3: -1}}, + {"person_ordinals": {0: 0, 1: 1, 2: 2, 3: 3.0}}, + ], +) +def test_invalid_context_fails_without_generators(inputs, changes): + frame, context, kwargs = inputs + context = replace( + context, + rng_registry=ProjectionRNGRegistry(3, 5), + person_ordinals={i: i for i in range(4)}, + ) + context = replace(context, **changes) + rng = np.random.default_rng(22) + state = rng.bit_generator.state + with patch.object(ProjectionRNGRegistry, "person_generator") as generator: + with patch( + "populace_dynamics.mortality_observer.apply_mortality" + ) as called: + with pytest.raises(ValueError): + observe_mortality(frame, context, rng, **kwargs) + called.assert_not_called() + generator.assert_not_called() + assert state == rng.bit_generator.state + + +def test_snapshot_death_identity_and_permutation(inputs): + frame, context, kwargs = inputs + kwargs["model"].probability.update( + dict.fromkeys(kwargs["model"].probability, 1.0) + ) + _, record = observe_mortality( + frame, context, np.random.default_rng(2), **kwargs + ) + _, reordered = observe_mortality( + frame.iloc[::-1], context, np.random.default_rng(2), **kwargs + ) + assert record == reordered + before = record.to_json(), record.digest + assert record.post_keys == () + assert len(record.identity_map.reverse_rows(record.pre_keys)) == 4 + kwargs["model"].probability[("0-64", "female")] = 0.0 + frame.loc[:, "age"] = 1 + assert (record.to_json(), record.digest) == before + with pytest.raises(FrozenInstanceError): + record.target_year = 2000 + with pytest.raises(FrozenInstanceError): + record.rows[0].survived = True + + +@pytest.mark.parametrize( + "mutation", + [ + lambda d: d.update(target_year="2016"), + lambda d: d.update(extra=True), + lambda d: d["rows"][0].update(survived="false"), + lambda d: d["rows"][0].update(age="030"), + lambda d: d["rows"].append(d["rows"][0]), + lambda d: d["model"][0].__setitem__(2, "nan"), + lambda d: d["model"][0].__setitem__(2, "0x1.0000000000000p-1"), + ], +) +def test_json_tampering_is_rejected(inputs, mutation): + frame, context, kwargs = inputs + _, record = observe_mortality( + frame, context, np.random.default_rng(2), **kwargs + ) + document = json.loads(record.to_json()) + mutation(document) + with pytest.raises(ValueError): + MortalityStepObservation.from_json( + json.dumps(document), + identity_map=kwargs["identity_map"], + expected_digest=record.digest, + ) + + +def test_json_external_map_and_duplicate_fields(inputs): + frame, context, kwargs = inputs + _, record = observe_mortality( + frame, context, np.random.default_rng(2), **kwargs + ) + other = PersonIdentityMap.from_identities( + [PersonIdentity("string", str(i)) for i in range(4)] + ) + with pytest.raises(ValueError, match="identity map"): + MortalityStepObservation.from_json( + record.to_json(), identity_map=other + ) + duplicate = record.to_json().replace("{", '{"schema":"duplicate",', 1) + with pytest.raises(ValueError, match="duplicate"): + MortalityStepObservation.from_json( + duplicate, identity_map=kwargs["identity_map"] + ) + + +def test_mutated_model_fails_before_call(inputs): + frame, context, kwargs = inputs + kwargs["model"].probability[("0-64", "female")] = float("nan") + with patch( + "populace_dynamics.mortality_observer.apply_mortality" + ) as called: + with pytest.raises(ValueError): + observe_mortality( + frame, context, np.random.default_rng(2), **kwargs + ) + called.assert_not_called() + + +@pytest.mark.parametrize( + "field,value", + [ + ("realization_id", ""), + ("source_contract_digest", "invalid"), + ("identity_map", {}), + ], +) +def test_bad_provenance_is_rejected_before_call(inputs, field, value): + frame, context, kwargs = inputs + kwargs[field] = value + with patch( + "populace_dynamics.mortality_observer.apply_mortality" + ) as called: + with pytest.raises(ValueError): + observe_mortality( + frame, context, np.random.default_rng(2), **kwargs + ) + called.assert_not_called() + + +def test_one_registry_draw_per_input_row(inputs): + frame, context, kwargs = inputs + context = replace( + context, + rng_registry=ProjectionRNGRegistry(3, 5), + person_ordinals={0: 5, 1: 8, 2: 13, 3: 9}, + ) + real_generator = ProjectionRNGRegistry.person_generator + with patch.object( + ProjectionRNGRegistry, + "person_generator", + autospec=True, + side_effect=real_generator, + ) as generators: + _, record = observe_mortality( + frame, context, np.random.default_rng(2), **kwargs + ) + assert generators.call_count == len(frame) + assert [call.args[-1] for call in generators.call_args_list] == [ + 5, + 8, + 13, + 9, + ] + assert [row.person_ordinal for row in record.rows] == [5, 8, 13, 9] + + +@pytest.mark.parametrize( + "change", + [ + lambda d: d.update(schema="unsupported"), + lambda d: d.update(model_digest="0" * 64), + lambda d: d["rows"][0].update(extra=0), + lambda d: d["rows"][0].update(dynamics_person_key="99"), + lambda d: d["rows"][0].update(person_ordinal="0"), + lambda d: d["model"][0].__setitem__(2, "0x1p999999999"), + lambda d: d["model"][0].__setitem__(2, "0x1.0p-1"), + lambda d: d["model"].__setitem__( + 0, ["0", "60", "0x0.0p+0", "0x0.0p+0"] + ), + ], +) +def test_invalid_json_rejected_without_expected_digest(inputs, change): + frame, context, kwargs = inputs + _, record = observe_mortality( + frame, context, np.random.default_rng(2), **kwargs + ) + document = json.loads(record.to_json()) + change(document) + with pytest.raises(ValueError): + MortalityStepObservation.from_json( + json.dumps(document), identity_map=kwargs["identity_map"] + ) + + +def test_detached_parameters_cannot_mutate_and_preserve_effective_hex(inputs): + frame, context, kwargs = inputs + kwargs["model"].probability[("0-64", "female")] = np.float32(0.3) + _, record = observe_mortality( + frame, context, np.random.default_rng(2), **kwargs + ) + assert record.model.cells[0][2] == float(np.float32(0.3)).hex() + detached = record.model.to_model() + with pytest.raises(TypeError): + detached.probability[("0-64", "female")] = 0.0 + + +def test_float_band_endpoints_refused_even_if_native_model_accepts(inputs): + frame, context, kwargs = inputs + kwargs["model"] = AgeSexMortalityModel( + ((0.0, 120.0),), {("0.0+", "female"): 0.5, ("0.0+", "male"): 0.5} + ) + with patch( + "populace_dynamics.mortality_observer.apply_mortality" + ) as called: + with pytest.raises(ValueError): + observe_mortality( + frame, context, np.random.default_rng(2), **kwargs + ) + called.assert_not_called() diff --git a/tests/test_person_identity.py b/tests/test_person_identity.py new file mode 100644 index 00000000..a89b4b4b --- /dev/null +++ b/tests/test_person_identity.py @@ -0,0 +1,347 @@ +"""Invented identity cases; no population, historical earnings or policy run.""" + +import copy +import json +import pickle +from dataclasses import FrozenInstanceError + +import pytest + +from populace_dynamics.person_identity import ( + IdentityEntry, + PersonIdentity, + PersonIdentityMap, +) + + +def test_lossless_mixed_identities_and_repeated_row_order(): + identities = ( + PersonIdentity("int64", -(2**63)), + PersonIdentity("int64", 2**53 + 1), + PersonIdentity("int64", 2**63 - 1), + PersonIdentity("uint64", 2**63), + PersonIdentity("uint64", 2**64 - 1), + PersonIdentity("string", "1"), + PersonIdentity("int64", 1), + PersonIdentity("uint64", 1), + PersonIdentity("string", ""), + PersonIdentity("string", "é"), + PersonIdentity("string", "e\u0301"), + ) + mapping = PersonIdentityMap.from_identities(identities) + reordered = identities[::-1] + (identities[0], identities[0]) + keys = mapping.map_rows(reordered) + assert mapping.reverse_rows(keys) == reordered + assert len(set(mapping.map_rows(identities))) == len(identities) + assert all(type(key) is int and 0 <= key < 2**63 for key in keys) + restored = PersonIdentityMap.from_json( + mapping.to_json(), expected_digest=mapping.digest + ) + assert restored == mapping + assert restored.to_json() == mapping.to_json() + assert restored.reverse_rows(keys) == reordered + assert '"18446744073709551615"' in mapping.to_json() + + +def test_batch_order_is_irrelevant_and_appending_never_renumbers(): + people = [PersonIdentity("string", s) for s in ["z", "b", "a"]] + original = PersonIdentityMap.from_identities(people) + assert original == PersonIdentityMap.from_identities(reversed(people)) + old_json, old_digest = original.to_json(), original.digest + entrants = [PersonIdentity("string", "0"), PersonIdentity("uint64", 0)] + extended = original.append(entrants) + assert extended == original.append(reversed(entrants)) + assert extended.entries[:3] == original.entries + assert extended.map_rows(people) == original.map_rows(people) + assert extended.reverse_rows(extended.map_rows(entrants)) == tuple( + entrants + ) + assert original.to_json() == old_json and original.digest == old_digest + assert extended.digest != original.digest + assert original.append([]) is original + restored = PersonIdentityMap.from_json( + extended.to_json(), previous=original, expected_digest=extended.digest + ) + assert restored == extended + + +def test_empty_manifest_round_trip(): + empty = PersonIdentityMap() + assert empty.map_rows([]) == empty.reverse_rows([]) == () + assert PersonIdentityMap.from_json(empty.to_json()) == empty + empty.require_extension_of(empty) + + +def test_string_subclasses_are_snapshotted_before_admission(): + class MutableString(str): + def __new__(cls, raw): + result = super().__new__(cls, raw) + result.rendered = raw + return result + + def __str__(self): + return self.rendered + + logical_type = MutableString("string") + source = MutableString("person-A") + source.rendered = "different-before-admission" + identity = PersonIdentity(logical_type, source) + mapping = PersonIdentityMap.from_identities([identity]) + before_json, before_digest = mapping.to_json(), mapping.digest + before_hash = hash(identity) + source.rendered = "person-B" + logical_type.rendered = "uint64" + + assert type(identity.value) is str + assert type(identity.logical_type) is str + assert identity == PersonIdentity("string", "person-A") + assert hash(identity) == before_hash + assert mapping.to_json() == before_json + assert mapping.digest == before_digest + assert mapping.map_rows([PersonIdentity("string", "person-A")]) == (0,) + assert mapping.reverse_rows([0]) == (identity,) + + +def test_identity_subclasses_cannot_supply_mutable_serialization(): + rendered = ["person-A"] + + class MutableIdentity(PersonIdentity): + @property + def canonical_value(self): + return rendered[0] + + identity = MutableIdentity("string", "person-A") + with pytest.raises(ValueError, match="explicit PersonIdentity"): + PersonIdentityMap.from_identities([identity]) + with pytest.raises(ValueError, match="explicit PersonIdentity"): + IdentityEntry(identity, 0) + mapping = PersonIdentityMap.from_identities( + [PersonIdentity("string", "person-A")] + ) + rendered[0] = "person-B" + with pytest.raises(ValueError, match="explicit PersonIdentity"): + mapping.map_rows([identity]) + + +def test_manifest_value_subclasses_refuse(): + class CustomEntry(IdentityEntry): + pass + + with pytest.raises(ValueError, match="IdentityEntry values"): + PersonIdentityMap((CustomEntry(PersonIdentity("string", "a"), 0),)) + with pytest.raises(ValueError, match="previous manifest"): + PersonIdentityMap().require_extension_of(object()) + + +def test_map_subclasses_cannot_override_digest_verification(): + with pytest.raises(TypeError, match="cannot be subclassed"): + + class ForgedDigestMap(PersonIdentityMap): + @property + def digest(self): + return "forged expected digest" + + +@pytest.mark.parametrize("value", [True, False, None, 1.0, float("nan"), "1"]) +@pytest.mark.parametrize("logical_type", ["int64", "uint64"]) +def test_numeric_identity_never_coerces_ambiguous_values(value, logical_type): + with pytest.raises(ValueError): + PersonIdentity(logical_type, value) + + +@pytest.mark.parametrize( + "logical_type,value", + [ + ("int64", -(2**63) - 1), + ("int64", 2**63), + ("uint64", -1), + ("uint64", 2**64), + ("string", 1), + ("string", None), + ("string", "\ud800"), + ("float64", 1), + ("Int64", 1), + ], +) +def test_invalid_type_or_range_refuses(logical_type, value): + with pytest.raises(ValueError): + PersonIdentity(logical_type, value) + + +def test_duplicate_admission_refuses_without_changing_original(): + identity = PersonIdentity("int64", 10) + with pytest.raises(ValueError, match="duplicate"): + PersonIdentityMap.from_identities([identity, identity]) + mapping = PersonIdentityMap.from_identities([identity]) + before = mapping.to_json() + with pytest.raises(ValueError, match="already admitted"): + mapping.append([PersonIdentity("int64", 11), identity]) + assert mapping.to_json() == before + + +@pytest.mark.parametrize("keys", [[0, 0], [1, 0], [0, 2], [-1, 0]]) +def test_duplicate_non_dense_or_reordered_keys_refuse(keys): + with pytest.raises(ValueError): + PersonIdentityMap( + tuple( + IdentityEntry(PersonIdentity("string", str(i)), key) + for i, key in enumerate(keys) + ) + ) + + +def test_duplicate_identity_in_loaded_entries_refuses(): + identity = PersonIdentity("string", "one") + with pytest.raises(ValueError, match="duplicate person identity"): + PersonIdentityMap( + (IdentityEntry(identity, 0), IdentityEntry(identity, 1)) + ) + + +def test_manifest_and_entries_are_immutable(): + identity = PersonIdentity("int64", 1) + entries = [IdentityEntry(identity, 0)] + mapping = PersonIdentityMap(entries) + entries.clear() + assert mapping.map_rows([identity]) == (0,) + with pytest.raises(FrozenInstanceError): + identity.value = 2 + with pytest.raises(FrozenInstanceError): + mapping.entries = () + with pytest.raises(TypeError): + mapping._forward[identity] = 8 + + +@pytest.mark.parametrize("key", [True, 0.0, "0", None, -1, 1, 2**64]) +def test_reverse_mapping_rejects_unknown_or_coerced_keys(key): + mapping = PersonIdentityMap.from_identities([PersonIdentity("int64", 1)]) + with pytest.raises(ValueError): + mapping.reverse_rows([key]) + + +def test_unknown_person_and_untyped_inputs_refuse(): + mapping = PersonIdentityMap.from_identities([PersonIdentity("int64", 1)]) + for identity in [PersonIdentity("string", "1"), 1, None]: + with pytest.raises(ValueError): + mapping.map_rows([identity]) + with pytest.raises(ValueError): + mapping.append(["1"]) + + +@pytest.mark.parametrize("value", ["01", "+1", " 1", "1.0", "-0", "١", 1]) +def test_noncanonical_serialized_integer_refuses(value): + mapping = PersonIdentityMap.from_identities([PersonIdentity("int64", 1)]) + document = json.loads(mapping.to_json()) + document["entries"][0]["value"] = value + with pytest.raises(ValueError, match="canonical decimal string"): + PersonIdentityMap.from_json(json.dumps(document)) + + +@pytest.mark.parametrize( + "text", + [ + "null", + "[]", + '{"schema":"unknown","entries":[]}', + '{"schema":"x","schema":"x","entries":[]}', + '{"schema":NaN,"entries":[]}', + ], +) +def test_invalid_json_document_refuses(text): + with pytest.raises(ValueError): + PersonIdentityMap.from_json(text) + + +def test_unknown_fields_and_duplicate_entry_fields_refuse(): + mapping = PersonIdentityMap.from_identities( + [PersonIdentity("string", "a")] + ) + document = json.loads(mapping.to_json()) + document["entries"][0]["household_id"] = "not-a-person-key" + with pytest.raises(ValueError, match="entry fields"): + PersonIdentityMap.from_json(json.dumps(document)) + duplicated = mapping.to_json().replace( + '"value":"a"', '"value":"a","value":"b"' + ) + with pytest.raises(ValueError, match="duplicate JSON field"): + PersonIdentityMap.from_json(duplicated) + + +def test_digest_and_prior_manifest_reject_remapping_or_deletion(): + a, b = PersonIdentity("string", "a"), PersonIdentity("string", "b") + old = PersonIdentityMap.from_identities([a, b]) + remapped = PersonIdentityMap((IdentityEntry(b, 0), IdentityEntry(a, 1))) + with pytest.raises(ValueError, match="digest mismatch"): + PersonIdentityMap.from_json( + remapped.to_json(), expected_digest=old.digest + ) + with pytest.raises(ValueError, match="prior manifest"): + PersonIdentityMap.from_json(remapped.to_json(), previous=old) + with pytest.raises(ValueError, match="prior manifest"): + PersonIdentityMap.from_json( + PersonIdentityMap().to_json(), previous=old + ) + + +def _pickled(value, protocol=pickle.HIGHEST_PROTOCOL): + return pickle.loads(pickle.dumps(value, protocol=protocol)) + + +@pytest.mark.parametrize( + "copier", + [ + *( + pytest.param( + lambda value, p=p: _pickled(value, p), id=f"pickle-{p}" + ) + for p in range(pickle.HIGHEST_PROTOCOL + 1) + ), + pytest.param(copy.deepcopy, id="deepcopy"), + pytest.param(copy.copy, id="copy"), + ], +) +def test_map_round_trips_through_pickle_and_copy(copier): + identities = ( + PersonIdentity("uint64", 2**64 - 1), + PersonIdentity("int64", -(2**63)), + PersonIdentity("string", "é"), + PersonIdentity("string", ""), + ) + mapping = PersonIdentityMap.from_identities(identities).append( + [PersonIdentity("string", "0")] + ) + restored = copier(mapping) + assert type(restored) is PersonIdentityMap + assert restored == mapping + assert restored.to_json() == mapping.to_json() + assert restored.digest == mapping.digest + assert restored.map_rows(identities) == mapping.map_rows(identities) + assert restored.reverse_rows([4]) == (PersonIdentity("string", "0"),) + with pytest.raises(ValueError, match="already admitted"): + restored.append([PersonIdentity("string", "é")]) + with pytest.raises(TypeError): + restored._forward[identities[0]] = 8 + assert copier(PersonIdentityMap()) == PersonIdentityMap() + + +def test_map_reduces_to_its_entries_so_loading_revalidates(): + mapping = PersonIdentityMap.from_identities( + [PersonIdentity("string", "a"), PersonIdentity("string", "b")] + ) + constructor, arguments = mapping.__reduce__() + assert constructor is PersonIdentityMap + assert arguments == (mapping.entries,) + swapped = (IdentityEntry(PersonIdentity("string", "a"), 1),) + with pytest.raises(ValueError, match="dense"): + constructor(swapped) + + +def test_json_whitespace_does_not_change_canonical_digest(): + mapping = PersonIdentityMap.from_identities( + [PersonIdentity("uint64", 2**64 - 1)] + ) + pretty = json.dumps(json.loads(mapping.to_json()), indent=2) + assert ( + PersonIdentityMap.from_json(pretty, expected_digest=mapping.digest) + == mapping + ) diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 71769153..d5a41003 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,8 +1,8 @@ { "schema_version": 1, "counts": { - "unit": 1563, - "artifact": 2668, + "unit": 2298, + "artifact": 2672, "integration_psid": 848, "reproduction_legacy": 520, "oracle_policyengine": 159