From 310c53a5d4643f534807fe2ca421abfcf371ed8b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 24 Sep 2026 05:42:40 -0400 Subject: [PATCH 1/9] Continue Track U for DynaSim exercise 2: invented data, dry run, U-inst, U6, F17, registered entry Continues the partial Track U build of #456 (plan critical-path-uniform-cut-20260923.md, items U5, U9 and U10, rows U6 and U-inst, field F17). Nothing here computes a poverty statistic on PSID data; the registered entry point refuses to run (the specification is a draft, d189 and d194 are open, the 2005/2007 wealth supplements are not staged, and no #42 registration exists). - cohorts/age67.py: institution_income_rule for row U-inst (builder default family_of_record, pending the referee; alternative excluded): an institutionalized member takes the family unit whose interview number its record carries, as an OFUM with no co-resident spouse. The PSID collects no income for persons in institutions (ER34137-ER34143 are inapplicable for them) and attaches their record to the family they left (2011 User Guide 2.4). The builder also records an "invented" provenance kind for frames that hash to the digest their invented provenance carries. - estimates/adjusted_poverty.py: cut_start_year (default None, every observation cut) builds row U6; with 2004 an observation is cut only when its member turns 67 in or after 2004, so U1's 1936 birth year is uncut and gets no SSI response. birth_year is now a required column. The start year is pending plan decision 8. - uniform_cut_track_u/ (new package): rows.py (the registered rows as spec overrides, held equal to the specification block), invented.py (U5: a seeded INVENTED age-67 population in the reader shapes whose income and WEALTH1 identities hold exactly, an INVENTED threshold table, and re-generation checks), runner.py (the row pipeline with provenance guards and the F17 official-concept diagnostic), diagnostics.py (F17 Social Security, SSI and WEALTH1 summaries against the committed SSA Supplement 2025 Table 5.A4 snapshot and the committed federal benefit rates; imports neither the income concept nor the tabulation). - scripts/track_u_dry_run.py (U9), scripts/run_track_u_registered.py (U10 entry point: #42 comment pointer, registered commit on a clean tree, ratified block with nothing awaiting or blocking, pinned Census capture, every wave's WEALTH1; exclusive artifact creation), and scripts/track_u_component_diagnostics.py (F17 components on the staged PSID, no threshold). - docs/design/boomers2004_uniform_cut_comparison.md: u1-draft-3. - New modules registered in POST_REVIEW_SOURCE_EXCLUSIONS, the pinned tuple and the reachability guard; tier counts recollected (7,414). Co-Authored-By: Claude Opus 5.5 --- .../boomers2004_uniform_cut_comparison.md | 209 ++- scripts/first_estimates_birth_evidence.py | 8 + scripts/run_track_u_registered.py | 319 ++++ scripts/track_u_component_diagnostics.py | 244 +++ scripts/track_u_dry_run.py | 408 +++++ src/populace_dynamics/cohorts/age67.py | 182 +- .../estimates/adjusted_poverty.py | 60 +- .../uniform_cut_track_u/__init__.py | 43 + .../uniform_cut_track_u/diagnostics.py | 718 ++++++++ .../uniform_cut_track_u/invented.py | 1477 +++++++++++++++++ .../uniform_cut_track_u/rows.py | 280 ++++ .../uniform_cut_track_u/runner.py | 522 ++++++ tests/README-tiers.md | 8 +- tests/cohorts/test_age67.py | 107 +- tests/estimates/test_adjusted_poverty.py | 57 + .../estimates/test_birth_evidence_artifact.py | 10 + tests/test_boomers2004_uniform_cut_spec.py | 24 +- tests/tier_counts.json | 6 +- .../test_component_diagnostics_integration.py | 72 + tests/track_u/test_diagnostics.py | 199 +++ tests/track_u/test_diagnostics_published.py | 85 + tests/track_u/test_dry_run_script.py | 88 + tests/track_u/test_invented_generator.py | 254 +++ tests/track_u/test_registered_script.py | 236 +++ tests/track_u/test_rows.py | 126 ++ tests/track_u/test_runner.py | 272 +++ 26 files changed, 5940 insertions(+), 74 deletions(-) create mode 100644 scripts/run_track_u_registered.py create mode 100644 scripts/track_u_component_diagnostics.py create mode 100644 scripts/track_u_dry_run.py create mode 100644 src/populace_dynamics/uniform_cut_track_u/__init__.py create mode 100644 src/populace_dynamics/uniform_cut_track_u/diagnostics.py create mode 100644 src/populace_dynamics/uniform_cut_track_u/invented.py create mode 100644 src/populace_dynamics/uniform_cut_track_u/rows.py create mode 100644 src/populace_dynamics/uniform_cut_track_u/runner.py create mode 100644 tests/track_u/test_component_diagnostics_integration.py create mode 100644 tests/track_u/test_diagnostics.py create mode 100644 tests/track_u/test_diagnostics_published.py create mode 100644 tests/track_u/test_dry_run_script.py create mode 100644 tests/track_u/test_invented_generator.py create mode 100644 tests/track_u/test_registered_script.py create mode 100644 tests/track_u/test_rows.py create mode 100644 tests/track_u/test_runner.py diff --git a/docs/design/boomers2004_uniform_cut_comparison.md b/docs/design/boomers2004_uniform_cut_comparison.md index d8292a5c..39779bdd 100644 --- a/docs/design/boomers2004_uniform_cut_comparison.md +++ b/docs/design/boomers2004_uniform_cut_comparison.md @@ -6,7 +6,7 @@ that awaits him or the specification freeze is an explicit parameter in the code with the plan's recommended default, and §16 lists them. - **Specification:** `boomers2004_uniform_cut_exercise2`, version - `u1-draft-2`, drafted 2026-09-24. §19 is the changelog. + `u1-draft-3`, drafted 2026-09-24. §19 is the changelog. - **Plan item:** U1 of the Track U plan, `EVID/critical-path-uniform-cut-20260923.md` (§7 fields F1–F17, §8 work items), where `EVID` = @@ -70,8 +70,11 @@ has not seen them and does not know which cells they cover. | PSID marriage history (`mh85_23`) and earnings panel | Birth-year law and marital status | Existing readers | | NCHS United States Life Tables, 2000 | Annuity mortality (F8) | Committed, `data/external/nchs_life_tables_2000.json`, SHA-256 pinned | | SSA period life table for 2004 (2008 vintage) | Row U9 | Committed, `data/external/tr2008/ssa_2008_vintage.json` | -| SSI federal benefit rates 2004–2012, exclusions, resource limits | SSI response (F13) | Captured from policyengine-us revision `a03e82e503`: `data/external/track_u_ssi_parameters.json`, SHA-256 `79e641a1…3c523` | -| Census poverty thresholds 2004–2012 | Threshold (F10) | **Not captured** (§6) | +| SSI federal benefit rates 2004–2012, exclusions, resource limits | SSI response (F13); F17 SSI diagnostic | Captured from policyengine-us revision `a03e82e503`: `data/external/track_u_ssi_parameters.json`, SHA-256 `79e641a1…3c523` | +| Census poverty thresholds 2004–2012 | Threshold (F10) | **Not captured** (§6; cos decision d194) | +| SSA, *Annual Statistical Supplement, 2025*, Table 5.A4 (December beneficiaries and total monthly benefits by type, 1940–2024) | F17 Social Security diagnostic | Committed snapshot `data/external/snapshots/ssa_level_anchors_vintage1/supplement2025_5a.html`, SHA-256 `d61e9484…aa8e` (the capture manifest's) | +| Survey of Consumer Finances wealth tables | F17 WEALTH1 comparator (plan §11) | **Not committed or saved**: WEALTH1 is summarized without a published comparison | +| PSID 2011 User Guide §2.4; individual-file codebook (ER34102, ER34103, ER34137–ER34143); family-file codebook (`# IN FU`, the individual-record count) | The institution income rule (§3) | Staged documentation, read for the rule | ## 3. Population @@ -99,10 +102,33 @@ choice: the plan says only "1936 from 2004 only". positive core/immigrant cross-section weight. The cross-section weight is also positive for movers-out (71–80) and decedents (81–89); neither option admits them and the dispositions count them. Registered option -U-inst adds sequence 51–59 (institutions); institutionalized persons have -no family-file income and the plan gives them no income rule, so the -builder marks those observations `income_rule_missing` and no income -concept can be computed for them. +U-inst adds sequence 51–59 (institutions). + +**Institution income rule (U-inst; builder default, pending the +referee).** The plan gives institutionalized persons no income rule, and +the PSID collects no income for them: the individual-file Social +Security items are "Inap.: … in an institution" (codebook, ER34137– +ER34143 for 2011). The PSID does associate them with a family: when a +sample member moves to an institution it "attaches an institutional +status data record to the family they left" (2011 User Guide, §2.4), and +the family file's record-count variable counts "any institutionalized +individuals associated with the family" among the records "having the +same family-level data". The default `institution_income_rule = +family_of_record` therefore gives an institutionalized observation the +family unit whose interview number its record carries: that family's +income concept, wealth, size, children and threshold, with the member's +own age and sex for a single-life annuity (it has no co-resident +spouse). Its role is OFUM: the relationship code of a person in an +institution is to the previous wave's head (codebook note on ER34103), +so it is neither the current head nor the wife. The member's own income +is missing from the family's, and `# IN FU` ("the actual number of +persons currently in the FU") is read as not counting them; the F17 +counts in §14 check that reading on the staged files. Both are named +deltas (§12). An institutionalized person whose interview number has no +family-file record that wave is a disposition +(`institution_family_of_record_missing`). The alternative `excluded` +keeps institutionalized persons out of the universe (disposition +`institution_excluded_by_rule`), which makes U-inst's population U0's. **Birth year.** `estimates.career.derive_birth_years` (the first-estimates §3.1 law, as Track A uses), total over the union of the five waves' @@ -260,6 +286,16 @@ row: row U6 (on U1) leaves 1936 uncut if the cut starts in 2004. The scorecard's "from 2004" has no source this builder could read (plan §10, decision 8). +**Row U6 (built; the start year awaits decision 8).** +`AdjustedPovertySpec.cut_start_year` (default `None`: every observation +is cut) is the row's parameter. With a start year, an observation is cut +only when its member turns 67 (birth year + 67) in or after it: the +observation stands for the member's status at 67, so the rule keys on +the age-67 year, not on the income year (U1 observes 1936 at 68, in +2004). With 2004, only the 1936 birth year (67 in 2003) is uncut; its +reform income equals its baseline, and with no fall in countable income +the SSI rules give it no offset and no new take-up. + ## 8. SSI response **Primary (F13, pending Max, d189): offset for existing recipients.** For @@ -326,6 +362,30 @@ present in the sealed comparator are scored; the others are reported as **Undefined cells.** An empty cell or one with zero total weight has no statistic; it is reported with its reason and never imputed. +**F17 diagnostics (not scored).** Two kinds, both reported for U0: + +- *Official-concept poverty rate* for the same sample: money income + (TOTAL FAMILY INCOME, reported asset income kept, no annuity, no cut) + against the row's threshold, weighted per cell + (`uniform_cut_track_u.runner.official_concept_rates`). It involves the + threshold, so it runs only inside the registered run (and on invented + data in the dry run), never on real data before registration. +- *Component summaries* (`uniform_cut_track_u.diagnostics`): per income + year, the weighted share of head and wife members with their own + Social Security, their weighted mean monthly amount set against SSA's + December average retired-worker benefit (Annual Statistical Supplement + 2025, Table 5.A4, committed snapshot; total monthly benefits over + number, a division made here; every age, retired workers only, December + against a calendar-year total over twelve, unadjusted); SSI receipt, + the mean among recipients and the recipients above twelve times the + federal benefit rate for their unit; WEALTH1 weighted quantiles (the + smallest value whose cumulative weight share reaches the level) and the + shares at or below zero and imputed, with no published comparison (no + SCF table is committed or saved); and counts of `# IN FU` against the + individual records. These are component aggregates with no threshold, + which plan §8 allows before registration; the module imports neither + the income concept nor the tabulation. + ## 10. Uncertainty - **Draws:** K = 1 (deterministic). @@ -360,12 +420,12 @@ results exist. | U3 | SSI | Full static recomputation (upper bound) | Yes | | U4 | Income unit | Head and wife only | Yes | | U5 | Asset income | Keep reported asset income and add the annuity | Yes | -| U6 | Cut (on U1) | 1936 uncut if the cut starts in 2004 | Not built (needs decision 8) | +| U6 | Cut (on U1) | 1936 uncut if the cut starts in 2004 (`cut_start_year = 2004`) | Yes; the start year awaits decision 8 | | U7 | Financial assets | Plus employer DC balances | Not built (label investigation) | | U8 | Threshold | PSID `CENSUS NEEDS STANDARD` | Yes | | U9 | Mortality | SSA period life table 2004 | Yes | | U10 | Threshold | Census size-by-children matrix | Yes (needs the capture) | -| U-inst | Universe | Add institutions (sequence 51–59) | Counted only; no income rule | +| U-inst | Universe | Add institutions (sequence 51–59), family-of-record income rule (§3) | Yes; the rule awaits the referee | ## 12. Named deltas @@ -375,7 +435,9 @@ Carried on every output (plan §7), plus those found while building: - realized 2004–2012 history versus DYNASIM's 1992-based projection, including the 2008–09 asset shock for the 1941–45 cohorts at 67; - realized COLAs versus 2002 Trustees assumptions; -- immigrant under-coverage; institutionalized persons; attrition; +- immigrant under-coverage; institutionalized persons (under U-inst the + family-of-record rule assigns the family's income and threshold without + the member's own income, which the PSID does not collect); attrition; - OFUM-owned assets inside family wealth; - imputed rent excluded; - self-reported Social Security, possibly net of Medicare premiums; @@ -415,14 +477,23 @@ Invented life table (ages 0–4): male qx 0, 0, 0.2, 0.5, 1; female qx 0, | SSI, SS under the exclusion (SS 200) | 0 − 0 | 0 | | U3 new take-up (SS 900, no other income) | before 660 ≥ 600; after 543 < 600 | +57 | +| U6, birth year 1936 (T = 1,000, SS 1,050) | primary: R = 1,050 − 136.5 = 913.5 (poor); `cut_start_year = 2004`: 1936 + 67 = 2003 < 2004, no cut, R = 1,050 | uncut | +| U6, SSI recipient born 1936 (SS 1,000, SSI 100) | primary offset 130; uncut: no fall | 0 | +| U-inst, member in an institution attached to family 11 | family 11's income 1,000 and WEALTH1 500; role OFUM; single life | assigned | + Invented tabulation (five observations, weights 1, 1, 2, 4, 2): P_B = 20, P_R = 70, Δ = 50 for `all`; design SE = 100·√0.1 with two strata of two clusters each. +F17 weighted quantile (invented values 10, 20, 30, 40 with weights 1, 1, +1, 1): the median is the smallest value whose cumulative weight share +reaches 0.5, 20; p90 is 40. + ## 14. What is built and what is blocked -Built on branch `dynamics-ex2-track-u-20260924` (all opt-in; registered -in `POST_REVIEW_SOURCE_EXCLUSIONS` and the reachability guard): +Built on branch `dynamics-ex2-track-u-20260924` (#456) and continued on +`dynamics-ex2-track-u-2-20260924` (all opt-in; registered in +`POST_REVIEW_SOURCE_EXCLUSIONS` and the reachability guard): - `src/populace_dynamics/data/family_income.py`: label-verified family income for waves 2005–2013 (81 items a wave, 85 in 2013, including the @@ -439,6 +510,39 @@ in `POST_REVIEW_SOURCE_EXCLUSIONS` and the reachability guard): registration pointer. - `scripts/capture_track_u_parameters.py` (SSI capture written; Census parser ready), `scripts/track_u_structure.py` (counts only). +- Continued (`u1-draft-3`): the institution income rule and row U6's + `cut_start_year` (above); the package + `src/populace_dynamics/uniform_cut_track_u/`: + - `rows.py`: the registered rows as `Age67Spec` and + `AdjustedPovertySpec` overrides, held equal to §15's rows; + - `invented.py` (U5): a seeded INVENTED age-67 population in the + reader shapes (retired, SSI, near-threshold, OFUM-parent, + adult-child, working, negative-wealth, cohabiting, separated, + institutionalized, moved-out, deceased and zero-weight families; + income and WEALTH1 identities exact), the INVENTED threshold table, + and the re-generation checks that keep invented data from passing as + PSID (the builder records the `invented` kind only for frames that + hash to the generator's digest); + - `runner.py`: the row pipeline with its provenance guards and the + official-concept diagnostic; + - `diagnostics.py`: the F17 component summaries. +- `scripts/track_u_dry_run.py` (U9): every row on the invented cohort + with invented thresholds and the committed life tables and SSI + capture; evidence `EVID/track-u-dry-run-20260924/`, headed "INVENTED + DATA - NOT A COMPARISON". +- `scripts/run_track_u_registered.py` (U10 entry point): refuses unless + the pointer is an issue #42 comment, `HEAD` is the registered commit + on a clean tree, §15 is ratified with nothing awaiting and nothing + blocking, the Census capture is pinned and every wave has WEALTH1; + writes `runs/replication_boomers2004_uniform_cut_v1.json` and its + `.env.json` exclusively. +- `scripts/track_u_component_diagnostics.py`: the F17 component + summaries on the staged PSID (evidence + `EVID/track-u-diagnostics-20260924/`). On the staged files every + family's `# IN FU` equals its number of in-family records (sequence + 1–20) in all five waves, and none of the 389–426 families per wave + with an attached institution record counts it in `# IN FU`: the + reading the institution rule relies on. Blocked: @@ -446,8 +550,8 @@ Blocked: and 1939 until staged, adjudicated and read. A fallback row on 1941–45 only would need registering in advance (plan §13); `income_rows(..., allow_blocked=True)` supports it. -2. **Census thresholds** (download approval or Max): no primary threshold - until captured. +2. **Census thresholds** (download approval, cos decision d194): no + primary threshold until captured; the dry run uses an INVENTED table. 3. **Registration** on issue #42 after ratification; no real-data poverty statistic may be computed before it. 4. **Comparator coverage** of the frozen cells (comparator side, values @@ -461,7 +565,7 @@ holds it to the code's defaults. ```json { "specification": "boomers2004_uniform_cut_exercise2", - "version": "u1-draft-2", + "version": "u1-draft-3", "status": "draft_for_referee", "claim_class": { "proposed": "track_u_psid_realized_measurement_not_a_projection", @@ -493,6 +597,7 @@ holds it to the code's defaults. "seed_wave_rule": "earliest_presence_wave", "separated_is_married": true, "unresolved_marital_status": "non_married", + "institution_income_rule": "family_of_record", "design": {"stratum": "ER31996", "cluster": "ER31997"} }, "income_concept": { @@ -523,7 +628,13 @@ holds it to the code's defaults. "capture_status": "not_captured", "poor_if": "income_below_threshold" }, - "cut": {"rate": 0.13, "base": "all_social_security_of_the_unit", "behavior": "none"}, + "cut": { + "rate": 0.13, + "base": "all_social_security_of_the_unit", + "behavior": "none", + "start_year": null, + "start_year_rule": "cut_when_birth_year_plus_67_at_or_after_start" + }, "ssi": { "rule": "offset_existing_recipients", "awaiting": "d189", @@ -563,12 +674,25 @@ holds it to the code's defaults. "U3": {"ssi_rule": "full_static_recomputation"}, "U4": {"income_unit": "head_wife"}, "U5": {"asset_income_rule": "keep"}, - "U6": {"on": "U1", "cut_start_year": 2004, "status": "not_built"}, + "U6": {"on": "U1", "cut_start_year": 2004, "awaiting": "plan section 10 decision 8"}, "U7": {"financial_assets": "wealth1_plus_employer_dc", "status": "not_built"}, "U8": {"threshold_rule": "psid_census_needs_standard"}, "U9": {"mortality_basis": "ssa_period_2004"}, "U10": {"threshold_rule": "census_matrix_65plus"}, - "U-inst": {"presence": "in_family_or_institution", "status": "counted_only_no_income_rule"} + "U-inst": {"presence": "in_family_or_institution", "institution_income_rule": "family_of_record", "awaiting": "referee"} + }, + "diagnostics_f17": { + "official_concept_poverty_rate": "registered_run_only", + "components": ["social_security", "ssi", "wealth1", "fu_size_record_counts"], + "social_security_published": "ssa_supplement_2025_table_5a4_december_retired_workers", + "ssi_published": "federal_benefit_rate_capture_only", + "wealth1_published": null + }, + "entry_points": { + "dry_run": "scripts/track_u_dry_run.py", + "registered_run": "scripts/run_track_u_registered.py", + "registered_artifact": "runs/replication_boomers2004_uniform_cut_v1.json", + "component_diagnostics": "scripts/track_u_component_diagnostics.py" }, "acceptance_rule": null, "labels": [ @@ -606,7 +730,13 @@ values-redacted definitions extract by the comparator side), decision 4 (Python SSI arithmetic labelled "not Axiom"), decision 6 (acceptance rule; default none, report gaps), decision 7 (ratify this specification by merge; post the #42 registration), decision 8 (the scorecard's "from -2004"), and approval to download the Census threshold files. +2004": row U6 is built with `cut_start_year = 2004`, and the primary's +`cut_start_year = None` cuts every observation), and approval to +download the Census threshold files (cos decision d194). + +**Awaiting the referee:** the institution income rule +(`institution_income_rule = family_of_record`; alternative `excluded`), +which only row U-inst reads. **Awaiting the specification freeze (defaults shown):** income unit (family unit), asset-income rule (replace), annuitized share (0.8), real @@ -617,7 +747,11 @@ threshold rule (weighted average, 65-and-over), SSI deeming (spouse's Social Security counted), OFUM SSI unit (one individual unit), row (U0), presence (in family), separated is married (true), unresolved marital status (non-married), U1 1936 weight (1), seed wave (earliest presence -wave). +wave), cut start year (none). + +A registered run refuses a §15 block that still names anything as +`awaiting`, so ratification must resolve each of these (and the code's +rows must be updated to match, `uniform_cut_track_u.rows`). ## 17. Questions for the referee @@ -643,6 +777,14 @@ wave). wave, which the repository's birth-year law equates with the income-year age; should it use the spouse's derived birth year (income year minus birth year) instead, as the member's age does? +9. Is `family_of_record` the right income rule for institutionalized + members under U-inst (their family's income and threshold, without + their own income, which the PSID does not collect), or should they be + `excluded` from the universe, or get a rule not built here (for + example a one-person unit)? +10. Row U6 keys the cut on the year the member turns 67, not on the + income year observed (U1 observes 1936 at 68, in 2004). Is that the + reading the plan's "the 1936 cohort unaffected" intends? ## 18. What this draft read and did not verify @@ -659,8 +801,19 @@ unweighted frequency counts and ranges, which were not used); the family-file documentation on wealth; the policyengine-us SSI parameter files; the Census historical thresholds page's link list. +`u1-draft-3` also read: the PSID 2011 User Guide §2.4 (sample +following rules), the individual-file codebook entries for ER34102, +ER34103 and ER34137–ER34143, the 2011 family codebook entry for `# IN +FU`, the family-file record-count text quoted in +`data/psid_unit_predicate_authority.py`, the cos record of d194 (its +text only), and Table 5.A4 of the committed SSA supplement snapshot. +The codebook entries print whole-file unweighted counts, which were not +used. + **Ran on staged PSID:** label verification, the component-identity -reconciliation counts and the structural counts of §3. No income concept, +reconciliation counts and the structural counts of §3, and (`u1-draft-3`) +the F17 component summaries of §9 for row U0 (Social Security, SSI and +WEALTH1 weighted summaries; `# IN FU` record counts). No income concept, annuity, threshold assignment, poverty status or poverty rate was computed on PSID data. @@ -673,10 +826,22 @@ and contents of the 2005 and 2007 wealth supplements; whether PSID Social Security amounts are net of Medicare premiums; whether SSI and Social Security are kept apart by respondents; the P-section DC items for U7; the coverage of the sealed comparator; anything in the Report beyond the -plan's reading of its methods. +plan's reading of its methods; whether an institutionalized member's +income enters any family-file item (the builder found no documentation +that it does); any SCF wealth aggregate (none is committed or saved). ## 19. Changelog +- `u1-draft-3` (2026-09-24, continuation on + `dynamics-ex2-track-u-2-20260924`): row U-inst gets an income rule + (`family_of_record`, pending the referee) and row U6 is built + (`cut_start_year`, the start year pending decision 8); the invented + generator (U5), the dry run (U9), the registered entry point (U10) and + the F17 component diagnostics are added; §15 gains + `institution_income_rule`, the cut's `start_year`, the F17 block and + the entry points, and U6 and U-inst record what they await; referee + questions 9 and 10 are added. + - `u1-draft-2` (2026-09-24, independent review): the half-split floor splits on family units linked through shared persons, so it is person-disjoint under U1 as plan F15 requires (U0 is unchanged); the diff --git a/scripts/first_estimates_birth_evidence.py b/scripts/first_estimates_birth_evidence.py index 42856cef..82ae84e3 100644 --- a/scripts/first_estimates_birth_evidence.py +++ b/scripts/first_estimates_birth_evidence.py @@ -230,6 +230,14 @@ Path("src/populace_dynamics/cohorts/age67.py"), Path("src/populace_dynamics/estimates/adjusted_poverty.py"), Path("src/populace_dynamics/estimates/uniform_cut_tabulation.py"), + # The opt-in Track U assembly (rows, invented generator, row runner and + # F17 component diagnostics) composes the Track U layer; nothing + # historical imports it. + Path("src/populace_dynamics/uniform_cut_track_u/__init__.py"), + Path("src/populace_dynamics/uniform_cut_track_u/diagnostics.py"), + Path("src/populace_dynamics/uniform_cut_track_u/invented.py"), + Path("src/populace_dynamics/uniform_cut_track_u/rows.py"), + Path("src/populace_dynamics/uniform_cut_track_u/runner.py"), ) POST_REVIEW_SHARED_SOURCE_BLOBS = { Path( diff --git a/scripts/run_track_u_registered.py b/scripts/run_track_u_registered.py new file mode 100644 index 00000000..6d54b2c4 --- /dev/null +++ b/scripts/run_track_u_registered.py @@ -0,0 +1,319 @@ +"""Track U registered one-shot run on the real PSID age-67 cohort (U10). + +This is the only entry point that computes the exercise-2 statistic (the +change in the adjusted poverty rate at 67 under a uniform 13 percent +cut) on real data. It runs once, after the issue #42 registration +comment exists, at exactly the commit that comment registers: + +* the registration pointer must be a comment on issue #42 + (``.../issues/42#issuecomment-``); +* the working tree must be clean and ``HEAD`` must equal + ``--registered-commit``; +* the U1 specification (``docs/design/boomers2004_uniform_cut_ + comparison.md``, section 15 block) must be ratified: its status and + version must each say "ratified" without a draft, candidate or other + negating marker (the exercise-1 test, + ``cola_age_profile.specification_unratified_fields``); no key of the + block may still be ``awaiting`` a decision; ``blocked_by`` must be + empty; and the threshold capture must be recorded as captured; +* the output artifact and its sidecar must not exist yet (one shot, no + overwrite; both are created exclusively, so a file that appears during + the run is not overwritten either); +* the rows are :data:`populace_dynamics.uniform_cut_track_u.rows. + REGISTERED_ROWS`, which must equal the block's rows; no flag changes + them; +* the Census thresholds must be the committed, hash-pinned capture + (:func:`populace_dynamics.estimates.adjusted_poverty. + load_poverty_thresholds` refuses until it exists), and every PSID wave + must have WEALTH1 (the 2005 and 2007 wealth supplements staged and + adjudicated); both are checked before any PSID file is read for the + statistic. + +The artifact publishes regardless of outcome. It never reads the sealed +comparator; the seal is opened only after this artifact is committed. + +As of this script's writing none of these preconditions holds: the +specification is a draft (``u1-draft-3``), Max has not ruled on exercise +2 (cos decision d189), the Census thresholds are not captured (d194) and +the 2005/2007 wealth supplements are not staged. + +Usage:: + + python scripts/run_track_u_registered.py \\ + --registration-pointer \\ + --registered-commit \\ + [--output runs/replication_boomers2004_uniform_cut_v1.json] + +Writes the artifact and a ``.env.json`` sidecar next to it. +""" + +from __future__ import annotations + +import argparse +import datetime +import hashlib +import importlib.metadata +import json +import platform +import re +import subprocess +import sys +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT / "src") not in sys.path: + sys.path.insert(0, str(ROOT / "src")) + +import populace_dynamics # noqa: E402 +from populace_dynamics.cohorts import age67 # noqa: E402 +from populace_dynamics.estimates import adjusted_poverty as ap # noqa: E402 +from populace_dynamics.estimates import cola_age_profile # noqa: E402 +from populace_dynamics.uniform_cut_track_u import ( # noqa: E402 + REGISTERED_HEADER, + rows, + runner, +) + +DEFAULT_OUTPUT = ROOT / "runs" / "replication_boomers2004_uniform_cut_v1.json" +#: A comment on issue #42, the registration issue; nothing else is a +#: registration pointer. +REGISTRATION_POINTER = runner.REGISTRATION_POINTER +#: The exercise-1 markers of an unratified status or version. +UNRATIFIED_MARKERS = cola_age_profile.UNRATIFIED_MARKERS +#: Import names whose installed distributions the sidecar records. +ENV_IMPORTS = ("numpy", "pandas", "populace_dynamics") + + +def _git(*args: str) -> str: + return subprocess.run( + ["git", "-C", str(ROOT), *args], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def _sha256(path: Path) -> str: + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def _awaiting(value: Any, path: str = "") -> list[str]: + """Every ``awaiting`` key in the block that still names a decision.""" + + found: list[str] = [] + if isinstance(value, Mapping): + for key, item in value.items(): + here = f"{path}.{key}" if path else str(key) + if key == "awaiting" and item not in (None, "", []): + found.append(here) + else: + found.extend(_awaiting(item, here)) + elif isinstance(value, list): + for index, item in enumerate(value): + found.extend(_awaiting(item, f"{path}[{index}]")) + return found + + +def check_specification_ratified(block: Mapping[str, Any]) -> None: + """Refuse a specification block that is not ratified and complete.""" + + for field in cola_age_profile.specification_unratified_fields(block): + raise ValueError( + f"the U1 specification {field} is {block.get(field)!r}: U1 " + "authorizes no run until Max ratifies it by merging (plan " + "section 10 item 7), and the ratified text must say so in its " + "section 15 block" + ) + pending = _awaiting(block) + if pending: + raise ValueError( + "the U1 specification authorizes no run while decisions are " + f"awaited: {pending}" + ) + if block.get("blocked_by"): + raise ValueError( + "the U1 specification authorizes no run while it is blocked " + f"by {block.get('blocked_by')}" + ) + status = (block.get("threshold") or {}).get("capture_status") + if status != "captured": + raise ValueError( + "the U1 specification authorizes no run until the Census " + f"thresholds are captured (capture_status {status!r})" + ) + + +def preflight( + *, + registration_pointer: str, + registered_commit: str, + output: Path, + git: Any = _git, + specification: Mapping[str, Any] | None = None, +) -> dict[str, str]: + """Refuse to run unless this is the registered one-shot state.""" + + if not REGISTRATION_POINTER.fullmatch(registration_pointer): + raise ValueError( + "the registration pointer must be an issue #42 comment URL " + "(https://github.com/PolicyEngine/microcosm-dynamics/issues/42" + "#issuecomment-)" + ) + if not re.fullmatch(r"[0-9a-f]{40}", registered_commit): + raise ValueError("--registered-commit must be a full 40-hex SHA") + head = git("rev-parse", "HEAD") + if head != registered_commit: + raise ValueError( + f"HEAD {head} is not the registered commit {registered_commit}" + ) + if git("status", "--porcelain") != "": + raise ValueError("the working tree must be clean for a registered run") + check_specification_ratified( + rows.specification_block() if specification is None else specification + ) + if output.exists() or output.with_suffix(".env.json").exists(): + raise ValueError( + f"{output} already exists: the registered run is one-shot" + ) + return {"head": head} + + +def _write_new(path: Path, text: str) -> None: + """Create ``path`` exclusively: never overwrite (one shot).""" + + with path.open("x", encoding="utf-8") as handle: + handle.write(text) + + +def distributions_for_import( + import_name: str, + packages: Mapping[str, Sequence[str]] | None = None, +) -> dict[str, str]: + """The installed distributions that provide ``import_name``, by version.""" + + if packages is None: + packages = importlib.metadata.packages_distributions() + names = dict.fromkeys(packages.get(import_name, ())) + return {name: importlib.metadata.version(name) for name in names} + + +def _environment() -> dict[str, Any]: + import tomllib + + with (ROOT / "pyproject.toml").open("rb") as handle: + project = tomllib.load(handle)["project"] + packages = importlib.metadata.packages_distributions() + by_import = { + name: distributions_for_import(name, packages) for name in ENV_IMPORTS + } + versions: dict[str, str] = {} + for distributions in by_import.values(): + versions.update(distributions) + source = Path(populace_dynamics.__file__).resolve().parent + return { + "python": platform.python_version(), + "platform": platform.platform(), + "packages": dict(sorted(versions.items())), + "distributions_by_import": { + name: sorted(distributions) + for name, distributions in by_import.items() + }, + "project": {"name": project["name"], "version": project["version"]}, + "populace_dynamics_source": ( + str(source.relative_to(ROOT)) + if source.is_relative_to(ROOT) + else str(source) + ), + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--registration-pointer", required=True) + parser.add_argument("--registered-commit", required=True) + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + args = parser.parse_args(argv) + + started = datetime.datetime.now(datetime.timezone.utc).isoformat() + state = preflight( + registration_pointer=args.registration_pointer, + registered_commit=args.registered_commit, + output=args.output, + ) + block = rows.specification_block() + row_check = rows.check_rows_against_block(block) + # Refused before any PSID file is read: no pinned Census capture, no + # run. + thresholds = ap.load_poverty_thresholds() + params = runner.committed_parameters(thresholds) + environment = _environment() + inputs = age67.load_age67_inputs() + if inputs.wealth_refusals: + raise ValueError( + "WEALTH1 is refused for waves " + f"{sorted(inputs.wealth_refusals)}; the registered run needs " + "every wave (stage and adjudicate the wealth supplements)" + ) + result = runner.run_track_u( + inputs, + params, + data_provenance=ap.REGISTERED_REAL, + registration_pointer=args.registration_pointer, + progress=lambda message: print(message, file=sys.stderr), + ) + artifact = { + "header": REGISTERED_HEADER, + "publishes_regardless": True, + "comparator_seal_opened_before_commit": False, + **result, + "specification": { + "path": str(rows.SPECIFICATION_PATH.relative_to(ROOT)), + "sha256": _sha256(rows.SPECIFICATION_PATH), + "version": block.get("version"), + "status": block.get("status"), + }, + "checks": {"specification_rows": row_check}, + "run": { + "started": started, + "finished": datetime.datetime.now( + datetime.timezone.utc + ).isoformat(), + "registration_pointer": args.registration_pointer, + "registered_commit": args.registered_commit, + "git_head": state["head"], + "git_clean": True, + "command": " ".join( + [ + "python", + "scripts/run_track_u_registered.py", + *(sys.argv[1:] if argv is None else argv), + ] + ), + }, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + _write_new( + args.output, + json.dumps(artifact, indent=2, sort_keys=False, allow_nan=False) + + "\n", + ) + _write_new( + args.output.with_suffix(".env.json"), + json.dumps( + { + "artifact": args.output.name, + "artifact_sha256": _sha256(args.output), + "environment": environment, + }, + indent=2, + ) + + "\n", + ) + print(args.output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/track_u_component_diagnostics.py b/scripts/track_u_component_diagnostics.py new file mode 100644 index 00000000..da414aa4 --- /dev/null +++ b/scripts/track_u_component_diagnostics.py @@ -0,0 +1,244 @@ +"""F17 component diagnostics of the Track U age-67 population (no statistic). + +Builds the exercise-2 age-67 observations of row U0 from the staged PSID +(:mod:`populace_dynamics.cohorts.age67`) and writes +``track-u-component-diagnostics.json`` and ``SUMMARY.md``: weighted +Social Security, SSI and WEALTH1 summaries per income year, the Social +Security summaries set against the committed SSA Annual Statistical +Supplement 2025 Table 5.A4 cells, the SSI amounts against the committed +federal benefit rates, and counts of ``# IN FU`` against the individual +records (the U-inst rule's reading). WEALTH1 has no committed or saved +published comparator, so it is reported without one. + +It computes **no** income concept, annuity, threshold, poverty status or +poverty rate, and no official-concept poverty rate (plan +``critical-path-uniform-cut-20260923.md`` section 8 limits real-file +work before the issue #42 registration to label verification, +structural counts and component aggregates that involve no threshold). +It never imports the income concept, the tabulation, the Track U runner +or the invented generator, and checks that none was loaded. + +Usage:: + + python scripts/track_u_component_diagnostics.py --output-dir DIR +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT / "src") not in sys.path: + sys.path.insert(0, str(ROOT / "src")) + +from populace_dynamics.cohorts import age67 # noqa: E402 +from populace_dynamics.uniform_cut_track_u import diagnostics # noqa: E402 + +FORBIDDEN_MODULES = ( + "populace_dynamics.estimates.adjusted_poverty", + "populace_dynamics.estimates.uniform_cut_tabulation", + "populace_dynamics.uniform_cut_track_u.runner", + "populace_dynamics.uniform_cut_track_u.invented", +) +LABELS = ( + "PSID-realized outcomes (not a projection)", + "component aggregates only: no income concept, threshold or poverty " + "rate", +) + + +def _git(*args: str) -> str: + return subprocess.run( + ["git", *args], cwd=ROOT, capture_output=True, text=True, check=True + ).stdout.strip() + + +def _leaked() -> list[str]: + return [name for name in FORBIDDEN_MODULES if name in sys.modules] + + +def build(data_dir: Path | None = None) -> dict[str, Any]: + leaked = _leaked() + if leaked: + raise RuntimeError(f"forbidden modules already imported: {leaked}") + inputs = age67.load_age67_inputs(data_dir=data_dir) + cohort = age67.build_age67_cohort(inputs, age67.Age67Spec()) + result = diagnostics.component_diagnostics(cohort, inputs) + leaked = _leaked() + if leaked: + raise RuntimeError(f"income-concept modules were imported: {leaked}") + return { + "description": ( + "Track U (DynaSim exercise 2) F17 component diagnostics for the " + "row U0 age-67 population: Social Security, SSI and WEALTH1 " + "summaries against committed published aggregates. No income " + "concept, annuity, threshold, poverty status or poverty rate " + "was computed." + ), + "labels": list(LABELS), + "code_commit": _git("rev-parse", "HEAD"), + "code_clean": _git("status", "--porcelain") == "", + "provenance": { + key: value + for key, value in cohort.provenance.items() + if key != "psid_files_sha256" + }, + "psid_files_sha256": dict(cohort.provenance["psid_files_sha256"]), + "wealth_refusals": { + str(k): v for k, v in sorted(inputs.wealth_refusals.items()) + }, + "diagnostics": result, + "forbidden_modules_loaded": _leaked(), + } + + +def _sentence(text: str) -> str: + return text[:1].upper() + text[1:] + "." + + +def _fmt(value: Any, digits: int) -> str: + return "n/a" if value is None else f"{value:,.{digits}f}" + + +def summary_markdown(result: dict[str, Any]) -> str: + diag = result["diagnostics"] + lines = [ + "# Track U component diagnostics (plan field F17)", + "", + "Labels: " + "; ".join(f"*{label}*" for label in result["labels"]), + "", + f"Row U0 population: {diag['population']['n_observations']} " + f"observations of {diag['population']['n_persons']} persons, income " + f"years {diag['population']['income_years']}; weighted by the " + "observation weights. Code " + f"`{result['code_commit']}` (clean: {result['code_clean']}). No " + "income concept, annuity, threshold, poverty status or poverty " + "rate was computed.", + "", + "## Social Security (head and wife members' own amounts)", + "", + "PSID: weighted share of head and wife members reporting their own " + "Social Security and the weighted mean annual amount among them, " + "divided by 12. SSA: Annual Statistical Supplement 2025 Table 5.A4, " + "December average monthly benefit of retired workers (total monthly " + "benefits over number; the division is ours), all ages.", + "", + "| Income year | Obs. | Own receipt share | Family receipt share | " + "PSID mean monthly | SSA Dec (prior) | SSA Dec (income year) | " + "PSID ÷ SSA prior Dec | PSID ÷ SSA Dec |", + "|---|---|---|---|---|---|---|---|---|", + ] + for year, ss in diag["social_security"].items(): + prior = ss["published"]["december_prior_year"] + same = ss["published"]["december_income_year"] + lines.append( + f"| {year} | {ss['n_observations']} " + f"| {_fmt(ss['own_receipt_share'], 3)} " + f"| {_fmt(ss['family_receipt_share'], 3)} " + f"| {_fmt(ss['own_mean_monthly_among_recipients'], 0)} " + f"| {_fmt(prior['ssa_retired_worker_average_monthly'], 0)} " + f"| {_fmt(same['ssa_retired_worker_average_monthly'], 0)} " + f"| {_fmt(prior['psid_over_ssa_retired_worker'], 3)} " + f"| {_fmt(same['psid_over_ssa_retired_worker'], 3)} |" + ) + lines += [ + "", + "Concept differences (not adjusted): " + + "; ".join( + diag["published_sources"]["social_security"]["concept_differences"] + ) + + ".", + "", + "## SSI", + "", + "| Income year | Own receipt share | Family receipt share | Own " + "recipients | Mean annual among recipients | Federal maximum " + "(individual, annual) | Recipients above the federal maximum |", + "|---|---|---|---|---|---|---|", + ] + for year, ssi in diag["ssi"].items(): + lines.append( + f"| {year} | {_fmt(ssi['own_receipt_share'], 3)} " + f"| {_fmt(ssi['family_receipt_share'], 3)} " + f"| {ssi['n_own_recipients']} " + f"| {_fmt(ssi['own_mean_annual_among_recipients'], 0)} " + f"| {_fmt(ssi['federal_maximum_annual']['individual'], 0)} " + f"| {ssi['n_recipients_above_federal_maximum']} |" + ) + lines += [ + "", + "No published SSI recipient or payment aggregate is committed or " + "saved; the federal benefit rates are the committed capture " + "(policyengine-us, citing SSA's SSI Federal Payment Amounts). An " + "amount above the federal maximum is consistent with a state " + "supplement or a reporting difference; which is not determined.", + "", + "## WEALTH1 (family imputed wealth excluding home equity)", + "", + "| Income year | Obs. | p10 | p25 | p50 | p75 | p90 | Share ≤ 0 | " + "Share imputed |", + "|---|---|---|---|---|---|---|---|---|", + ] + for year, wealth in diag["wealth1"].items(): + q = wealth["quantiles"] or {} + lines.append( + f"| {year} | {wealth['n_observations']} " + + " ".join( + f"| {_fmt(q.get(key), 0)}" + for key in ("p10", "p25", "p50", "p75", "p90") + ) + + f" | {_fmt(wealth['share_at_or_below_zero'], 3)} " + f"| {_fmt(wealth['share_imputed'], 3)} |" + ) + lines += [ + "", + _sentence(diag["published_sources"]["wealth1"]["not_compared"]) + + " Waves 2005 and 2007 have no WEALTH1 (supplements not staged).", + "", + "## `# IN FU` against the individual records (U-inst rule)", + "", + "| Wave | Families | # IN FU = in-family records | Institution " + "records | Families with them | … # IN FU = in-family only | … " + "# IN FU = in-family + institution |", + "|---|---|---|---|---|---|---|", + ] + for wave, counts in diag["institution_record_counts"].items(): + lines.append( + f"| {wave} | {counts['n_families']} " + f"| {counts['n_fu_size_equals_in_family_records']} " + f"| {counts['n_institution_records']} " + f"| {counts['n_families_with_institution_records']} " + f"| {counts['of_which_fu_size_equals_in_family_only']} " + f"| {counts['of_which_fu_size_equals_in_family_plus_institution']}" + " |" + ) + lines.append("") + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--data-dir", type=Path, default=None) + args = parser.parse_args(argv) + args.output_dir.mkdir(parents=True, exist_ok=True) + result = build(args.data_dir) + path = args.output_dir / "track-u-component-diagnostics.json" + path.write_text( + json.dumps(result, indent=2, allow_nan=False) + "\n", + encoding="utf-8", + ) + (args.output_dir / "SUMMARY.md").write_text( + summary_markdown(result), encoding="utf-8" + ) + print(path) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/track_u_dry_run.py b/scripts/track_u_dry_run.py new file mode 100644 index 00000000..886009b4 --- /dev/null +++ b/scripts/track_u_dry_run.py @@ -0,0 +1,408 @@ +"""Track U end-to-end DRY RUN on an INVENTED cohort (plan item U9). + +INVENTED DATA - NOT A COMPARISON. The cohort is the invented population +of :mod:`populace_dynamics.uniform_cut_track_u.invented` (persons, +families, income, Social Security, SSI, wealth, weights and design +variables all invented), run through the real age-67 builder, income +rows, income concept and tabulation for every registered row, with the +F17 diagnostics. The poverty thresholds are INVENTED as well (round +numbers in the Census table shape): the Census thresholds are not +captured (cos decision d194). The other parameters are the committed +ones: the NCHS 2000 and SSA 2004 life tables and the SSI capture. No +PSID file is opened and no comparator value is read. + +The invented generator gives waves 2005 and 2007 invented wealth, as if +the PSID wealth supplements were staged, so U0's five birth years run +end to end; the real supplements are not staged (cos decision d189). The +checks record what the real staging does today (the income rows refuse +the blocked waves) and that the registered-run guards refuse invented +inputs and an uncaptured threshold table. + +Usage:: + + python scripts/track_u_dry_run.py --output-dir [--seed N] + +Writes ``result.json`` and ``RESULTS.md`` into ``--output-dir``. +""" + +from __future__ import annotations + +import argparse +import datetime +import json +import math +import platform +import subprocess +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT / "src") not in sys.path: + sys.path.insert(0, str(ROOT / "src")) + +import pandas as pd # noqa: E402 + +from populace_dynamics.cohorts import age67 # noqa: E402 +from populace_dynamics.data import family_income # noqa: E402 +from populace_dynamics.estimates import adjusted_poverty as ap # noqa: E402 +from populace_dynamics.uniform_cut_track_u import ( # noqa: E402 + DRY_RUN_HEADER, + invented, + rows, + runner, +) + +#: A syntactically valid registration pointer used only to show that the +#: registered path refuses invented inputs (no such comment is implied). +_GUARD_POINTER = ( + "https://github.com/PolicyEngine/microcosm-dynamics/issues/42" + "#issuecomment-0" +) + + +def _git(*args: str) -> str: + return subprocess.run( + ["git", *args], cwd=ROOT, capture_output=True, text=True, check=True + ).stdout.strip() + + +def _refusal(call: Any) -> dict[str, Any]: + try: + call() + except Exception as error: # recorded, not swallowed + return { + "refused": True, + "error": type(error).__name__, + "message": str(error), + } + return {"refused": False} + + +def checks(seed: int, params: runner.TrackUParameters) -> dict[str, Any]: + """The dry run's checks, each recorded (none computes a PSID value).""" + + staged = invented.invented_age67_inputs( + seed=seed, supplement_waves_staged=True + ) + incomes = pd.concat( + [staged.family_income[wave] for wave in age67.WAVES], + ignore_index=True, + ) + wealth = pd.concat( + [staged.family_wealth[wave] for wave in family_income.WEALTH_WAVES], + ignore_index=True, + ) + blocked = invented.invented_age67_inputs(seed=seed) + cohort = age67.build_age67_cohort(blocked) + left = age67.income_rows(cohort, blocked, allow_blocked=True) + return { + "specification_rows": rows.check_rows_against_block( + rows.specification_block() + ), + "invented_family_income_reconciliation": ( + family_income.reconcile_family_income(incomes) + ), + "invented_wealth1_reconciliation": family_income.reconcile_wealth1( + wealth + ), + "blocked_waves_as_staged_today": { + "wealth_refusals": { + str(k): v for k, v in sorted(blocked.wealth_refusals.items()) + }, + "income_rows_without_allow_blocked": _refusal( + lambda: age67.income_rows(cohort, blocked) + ), + "left_out_with_allow_blocked": dict(left.attrs["left_out"]), + "observations_kept_with_allow_blocked": int(len(left)), + }, + "registered_guard_refuses_invented_inputs": _refusal( + lambda: runner.run_track_u( + staged, + params, + data_provenance=ap.REGISTERED_REAL, + registration_pointer=_GUARD_POINTER, + ) + ), + "census_thresholds_not_captured": _refusal(ap.load_poverty_thresholds), + } + + +def _fmt(value: Any, digits: int = 2) -> str: + if value is None or ( + isinstance(value, float) and not math.isfinite(value) + ): + return "undefined" + return f"{value:.{digits}f}" + + +def _cell(tabulation: dict[str, Any], name: str) -> dict[str, Any]: + return next(c for c in tabulation["cells"] if c["cell"] == name) + + +def _cell_line(label: str, cell: dict[str, Any]) -> str: + if not cell["defined"]: + return ( + f"| {label} | {cell['n_observations']} | undefined " + f"({cell['undefined_reason']}) | | | | |" + ) + floor = cell["floor"]["delta"] + se = cell.get("design_se", {}).get("delta", {}).get("se") + return ( + f"| {label} | {cell['n_observations']} " + f"| {_fmt(cell['baseline_rate'])} | {_fmt(cell['reform_rate'])} " + f"| {_fmt(cell['delta'])} " + f"| {_fmt(floor['mean']) if floor['defined'] else 'undefined'} " + f"| {_fmt(se)} |" + ) + + +def results_markdown(result: dict[str, Any]) -> str: + run = result["run"] + provenance = result["cohort_provenance"] + lines = [ + f"# {DRY_RUN_HEADER}", + "", + f"Track U (DynaSim exercise 2, plan item U9) end-to-end dry run, " + f"{run['date']}. Every person, family, income amount, Social " + "Security and SSI amount, wealth component, weight and sampling " + "stratum and cluster is **invented** " + "(`populace_dynamics.uniform_cut_track_u.invented`, seed " + f"{provenance['seed']}). The poverty thresholds are **invented** " + "too: round numbers in the Census table shape, labelled " + f"\"{result['parameters']['thresholds']['label']}\". No PSID file " + "was opened, no comparator value was read, and nothing below is a " + "result or a comparison with DYNASIM.", + "", + "Labels on every output: " + + "; ".join(f"*{label}*" for label in result["labels"]) + + ".", + "", + "## What ran", + "", + "- Invented cohort through the real builder " + "(`cohorts.age67.build_age67_cohort`), income rows, income concept " + "(`estimates.adjusted_poverty.adjusted_incomes`) and tabulation " + "(`estimates.uniform_cut_tabulation.tabulate_uniform_cut`), once " + "per registered row. Waves 2005 and 2007 carry invented wealth as " + "if the PSID wealth supplements were staged; the real ones are " + "not (see Checks).", + "- Parameters: the committed NCHS 2000 and SSA 2004 life tables " + "and the committed SSI capture; **invented** thresholds (the Census " + "capture does not exist, cos decision d194).", + "- Rows: " + + ", ".join( + f"{row_id} ({entry['status'].replace('_', ' ')})" + for row_id, entry in result["rows"].items() + ) + + ".", + "", + "## Invented-data tabulation (not a comparison)", + "", + "Cell `all` of each row: observations, baseline rate P_B, reform " + "rate P_R and Δ = P_R − P_B in percentage points, the mean " + "half-split floor of Δ over five seeds (split units: family units " + "linked by persons) and the design-based standard error of Δ. " + "These numbers describe the invented cohort only.", + "", + "| Row | Observations | P_B | P_R | Δ | Δ floor | Δ design SE |", + "|---|---|---|---|---|---|---|", + ] + for row_id, entry in result["rows"].items(): + if entry["status"] != "computed": + lines.append( + f"| {row_id} | not built: {entry['reason']} | | | | | |" + ) + continue + lines.append(_cell_line(row_id, _cell(entry["tabulation"], "all"))) + primary = result["rows"]["U0"]["tabulation"] + lines += [ + "", + "### Row U0 by cell (invented)", + "", + "| Cell | Observations | P_B | P_R | Δ | Δ floor | Δ design SE |", + "|---|---|---|---|---|---|---|", + ] + for cell in primary["cells"]: + lines.append(_cell_line(cell["cell"], cell)) + undefined = [ + f"- {row_id}, {item['cell']}: {item['reason']}." + for row_id, entry in result["rows"].items() + if entry["status"] == "computed" + for item in entry["tabulation"]["undefined_cells"] + ] + lines += ["", "Undefined cells, each with its reason:", ""] + lines += undefined or ["- none in this run."] + f17 = result["f17_diagnostics"] + lines += [ + "", + "## F17 diagnostics on invented data (not scored)", + "", + "Official-concept poverty rate (money income, reported asset " + "income kept, no annuity, no cut, against the same invented " + "thresholds), row U0: " + + "; ".join( + f"{name} {_fmt(cell.get('rate'))}" + for name, cell in f17["official_concept_poverty_rate"][ + "cells" + ].items() + ) + + ".", + "", + "| Income year | Own SS receipt share | Own SS mean monthly | " + "÷ SSA Dec retired-worker average (prior Dec) | Own SSI receipt " + "share | SSI above federal maximum | WEALTH1 p50 |", + "|---|---|---|---|---|---|---|", + ] + components = f17["components"] + for year, ss in components["social_security"].items(): + ssi = components["ssi"][year] + wealth = components["wealth1"].get(year) + ratio = ss["published"]["december_prior_year"][ + "psid_over_ssa_retired_worker" + ] + lines.append( + f"| {year} | {_fmt(ss['own_receipt_share'], 3)} " + f"| {_fmt(ss['own_mean_monthly_among_recipients'], 0)} " + f"| {_fmt(ratio, 3)} | {_fmt(ssi['own_receipt_share'], 3)} " + f"| {ssi['n_recipients_above_federal_maximum']} " + f"| {_fmt(wealth['quantiles']['p50'], 0) if wealth else 'n/a'} |" + ) + lines += [ + "", + "The SSA averages are the committed Annual Statistical Supplement " + "2025 Table 5.A4 cells (a real published source); set against " + "invented amounts, their ratio says nothing about PSID.", + "", + "## Checks", + "", + ] + check = result["checks"] + blocked = check["blocked_waves_as_staged_today"] + reconciliation = check["invented_family_income_reconciliation"] + exact = all( + counts["n_exact"] == counts["n_families"] + for identities in reconciliation.values() + for counts in identities.values() + ) + wealth_exact = all( + counts["n_exact"] == counts["n_families"] + for identities in check["invented_wealth1_reconciliation"].values() + for counts in identities.values() + ) + lines += [ + "- Every registered row equals the specification block " + f"({check['specification_rows']['specification_version']}): " + f"{check['specification_rows']['rows_equal_the_block']}.", + "- Invented family income adds up exactly under the codebook " + f"identities in every wave: {exact}; WEALTH1 and WEALTH2 " + f"identities exact: {wealth_exact}.", + "- The inputs and the U0 cohort re-generate from the invented " + "generator's seed (`check_invented_inputs`, " + "`check_invented_cohort`).", + "- As the real staging is today (no 2005/2007 wealth), the income " + "rows refuse without `allow_blocked`: " + f"{blocked['income_rows_without_allow_blocked']['refused']}; with " + "it, " + f"{blocked['left_out_with_allow_blocked']['wealth_supplement_not_staged']}" + " observations (waves 2005 and 2007) are left out and " + f"{blocked['observations_kept_with_allow_blocked']} kept.", + "- The registered path refuses these invented inputs: " + f"{check['registered_guard_refuses_invented_inputs']['refused']} " + f"(`{check['registered_guard_refuses_invented_inputs'].get('error')}`).", + "- The committed Census threshold capture does not exist, so the " + "registered run cannot start: " + f"{check['census_thresholds_not_captured']['refused']} " + f"(`{check['census_thresholds_not_captured'].get('error')}`).", + "", + "## Pending decisions", + "", + "Every choice below is a parameter at its default; none is ratified.", + "", + ] + for item in ( + result["pending_decisions"]["age67"] + + result["pending_decisions"]["income_concept"] + ): + lines.append( + f"- `{item['field']}` = `{item['default']}` (awaiting " + f"{item['awaiting']})." + ) + for row_id, awaiting in result["pending_decisions"]["rows"].items(): + lines.append(f"- Row {row_id}: awaiting {awaiting}.") + lines += ["", "## Named deltas", ""] + lines += [f"- {delta}." for delta in result["named_deltas"]] + thresholds = result["parameters"]["thresholds"] + lines += [ + "", + "## Provenance", + "", + f"- Code: `{run['git_head']}` (worktree clean: {run['git_clean']}).", + f"- Invented inputs: generator `{provenance['generator']}`, seed " + f"{provenance['seed']}, frames SHA-256 " + f"`{provenance['input_frames_sha256']}`.", + f"- Thresholds: `{thresholds['kind']}` ({thresholds['generator']}).", + f"- SSI: sha256 `{result['parameters']['ssi']['sha256']}`.", + "- Life tables: " + + ", ".join( + f"`{name}` sha256 `{entry.get('sha256')}`" + for name, entry in result["parameters"]["life_tables"].items() + ) + + ".", + f"- Python {run['python']}; command `{run['command']}`.", + "", + ] + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--seed", type=int, default=invented.DEFAULT_SEED) + args = parser.parse_args(argv) + + params = runner.committed_parameters( + invented.invented_poverty_thresholds() + ) + inputs = invented.invented_age67_inputs( + seed=args.seed, supplement_waves_staged=True + ) + result = runner.run_track_u( + inputs, + params, + data_provenance=ap.INVENTED, + progress=lambda message: print(message, file=sys.stderr), + ) + result = { + "header": DRY_RUN_HEADER, + **result, + "checks": checks(args.seed, params), + "run": { + "date": datetime.date.today().isoformat(), + "invented_seed": args.seed, + "git_head": _git("rev-parse", "HEAD"), + "git_clean": _git("status", "--porcelain") == "", + "python": platform.python_version(), + "command": " ".join( + [ + "python", + "scripts/track_u_dry_run.py", + *(sys.argv[1:] if argv is None else argv), + ] + ), + }, + } + args.output_dir.mkdir(parents=True, exist_ok=True) + (args.output_dir / "result.json").write_text( + json.dumps(result, indent=2, allow_nan=False) + "\n", + encoding="utf-8", + ) + (args.output_dir / "RESULTS.md").write_text( + results_markdown(result), encoding="utf-8" + ) + print(args.output_dir / "result.json") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/populace_dynamics/cohorts/age67.py b/src/populace_dynamics/cohorts/age67.py index e49d7c75..e28429e7 100644 --- a/src/populace_dynamics/cohorts/age67.py +++ b/src/populace_dynamics/cohorts/age67.py @@ -23,12 +23,33 @@ Universe per wave (F2): sequence 1-20 (in a responding family) with a positive cross-section weight; the registered option U-inst adds -sequence 51-59 (institution), which this builder counts but for which no -income rule exists (institutionalized persons have no family-file -income; the plan does not say what income they get), so their -observations are marked ``income_rule_missing``. The cross-section -weight is also positive for movers-out (71-80) and decedents (81-89), -whom neither option admits; the dispositions count them. +sequence 51-59 (institution). The PSID collects no income for an +institutionalized person (the individual-file Social Security items are +"Inap.: ... in an institution", codebook ER34137-ER34143 for 2011) and +the plan gives them no income rule, so ``institution_income_rule`` (a +builder default pending the referee) supplies one: + +* ``family_of_record`` (default): the PSID attaches an institutionalized + sample member's record to the family they left (2011 User Guide, + section 2.4) and counts it among the individual records "having the + same family-level data" as that family (family-file codebook text for + the record-count variable). The observation takes that family unit's + income, wealth, size and children through its interview number, with + the member's own age and sex; its role is ``ofum`` (neither the + current head nor the wife: the relationship code of a person in an + institution is to the previous wave's head, codebook note on ER34103) + and it has no co-resident spouse. The member's own income is not in + the family's income and ``# IN FU`` ("the actual number of persons + currently in the FU") is read as not counting them: a named delta. + An institutionalized person whose interview number has no family-file + record in that wave is a disposition, not an observation. +* ``excluded``: institutionalized persons stay out of the poverty + universe (a disposition, as under the primary's universe), so U-inst + equals U0 in population and only its counts differ. + +The cross-section weight is also positive for movers-out (71-80) and +decedents (81-89), whom neither option admits; the dispositions count +them. Birth year: :func:`populace_dynamics.estimates.career.derive_birth_years` (first-estimates section 3.1), total over the union of the five waves' @@ -55,7 +76,12 @@ Provenance: :func:`load_age67_inputs` records the SHA-256 of every PSID file it read and seals the returned inputs; the builder marks a cohort -``psid_files`` only for sealed inputs and ``caller_frames`` otherwise. +``psid_files`` only for sealed inputs, ``invented`` for inputs whose +frames hash to the digest their recorded ``invented`` provenance carries +(the invented generator, +:mod:`populace_dynamics.uniform_cut_track_u.invented`, which this module +cannot import, re-generates and checks them), and ``caller_frames`` +otherwise. :func:`income_rows` carries the kind to the income concept, whose guard refuses PSID-built rows without a registration. This module computes no poverty status, threshold, annuity or statistic. @@ -89,6 +115,8 @@ "ALL_BIRTH_YEARS", "ANCHOR_LAYOUTS", "CALLER_FRAMES", + "INSTITUTION_INCOME_RULES", + "INVENTED", "PRIMARY_BIRTH_YEARS", "PSID_FILES", "ROWS", @@ -114,7 +142,22 @@ ALL_BIRTH_YEARS: tuple[int, ...] = tuple(range(1936, 1946)) ROWS: tuple[str, ...] = ("U0", "U1") PSID_FILES = "psid_files" +INVENTED = "invented" CALLER_FRAMES = "caller_frames" +#: The income rules for an institutionalized observation (row U-inst). +INSTITUTION_INCOME_RULES: dict[str, str] = { + "family_of_record": ( + "the family unit whose interview number the institutionalized " + "member's record carries (the family they left): its income, " + "wealth, size and children, with the member's own age and sex, " + "role ofum and no co-resident spouse (builder default, pending the " + "referee)" + ), + "excluded": ( + "institutionalized persons stay out of the poverty universe (a " + "disposition); U-inst then equals U0 in population" + ), +} #: Each wave's label-verified individual-file anchor variables (labels #: checked 2026-09-24 against IND2023ER.sps; 2009 and 2011 are the @@ -201,12 +244,18 @@ class Age67Spec: u1_single_observation_weight: float = 1.0 seed_wave_rule: str = "earliest_presence_wave" unresolved_marital_status: str = "non_married" + institution_income_rule: str = "family_of_record" def __post_init__(self) -> None: if self.row not in ROWS: raise ValueError(f"row must be one of {ROWS}") if self.presence not in ("in_family", "in_family_or_institution"): raise ValueError("presence must be in_family[_or_institution]") + if self.institution_income_rule not in INSTITUTION_INCOME_RULES: + raise ValueError( + "institution_income_rule must be one of " + f"{sorted(INSTITUTION_INCOME_RULES)}" + ) if self.seed_wave_rule != "earliest_presence_wave": raise ValueError("seed_wave_rule must be earliest_presence_wave") if self.unresolved_marital_status != "non_married": @@ -279,6 +328,21 @@ def pending_decisions() -> tuple[psid2010.PendingDecision, ...]: "non_married cell, a single-life annuity)", freeze, ), + psid2010.PendingDecision( + "institution_income_rule", + spec.institution_income_rule, + ("excluded",), + "builder default, pending the referee: plan F2 adds " + "institutions under U-inst but gives them no income rule and " + "the PSID collects none for them; the PSID attaches an " + "institutionalized member's record to the family they left " + "(2011 User Guide section 2.4), so the observation takes that " + "family's income concept and threshold (the member's own " + "income is missing and the family's size does not count them: " + "a named delta). Applies only under presence " + "in_family_or_institution", + freeze, + ), ) @@ -537,6 +601,22 @@ def _input_provenance(inputs: Age67Inputs) -> dict[str, Any]: recorded = dict(inputs.provenance or {}) frames = input_frames_sha256(inputs) files = recorded.get("psid_files_sha256") + if recorded.get("kind") == INVENTED: + if recorded.get("input_frames_sha256") != frames: + raise ValueError( + "inputs claim invented provenance but their frames differ " + "from the digest the invented generator recorded " + f"({recorded.get('input_frames_sha256')!r} != {frames!r}); " + "invented data cannot be mixed with other frames" + ) + return { + "kind": INVENTED, + "generator": recorded.get("generator"), + "seed": recorded.get("seed"), + "supplement_waves_staged": recorded.get("supplement_waves_staged"), + "label": recorded.get("data"), + "input_frames_sha256": frames, + } if recorded.get("kind") == PSID_FILES: if inputs.loader_seal is None or not isinstance(files, Mapping): raise ValueError( @@ -664,8 +744,10 @@ def build_age67_cohort( ``sequence``, ``relationship``, ``member_role``, ``stratum``, ``cluster``, ``marital_status``, ``married``, ``spouse_person_id``, ``member_married_coresident``, ``spouse_age``, ``spouse_sex``, - ``fu_legal_wife_present``, ``wife_sex``, ``wealth_status`` and - ``income_status``. + ``fu_legal_wife_present``, ``wife_sex``, ``wealth_status``, + ``in_institution`` and ``income_status`` (``family_file`` for a member + of the family unit, ``family_of_record`` for an institutionalized + member under that rule). """ spec = Age67Spec() if spec is None else spec @@ -701,6 +783,11 @@ def build_age67_cohort( by_family = { int(k): v for k, v in in_family.groupby("interview", sort=False) } + family_records = ( + set(inputs.family_income[wave]["interview"].astype(int)) + if wave in inputs.family_income + else set() + ) cohort_ids = sorted( pid for pid in targets if births[pid].birth_year == birth_year ) @@ -742,10 +829,31 @@ def build_age67_cohort( {**base, "disposition": "excluded_sex_unknown"} ) continue - disposition_rows.append({**base, "disposition": "observation"}) interview = int(record["interview"]) sequence = int(record["sequence"]) relationship = int(record["relationship"]) + in_institution = ( + _SEQUENCE_INSTITUTION[0] + <= sequence + <= _SEQUENCE_INSTITUTION[1] + ) + if in_institution: + if spec.institution_income_rule == "excluded": + disposition_rows.append( + {**base, "disposition": "institution_excluded_by_rule"} + ) + continue + if interview not in family_records: + disposition_rows.append( + { + **base, + "disposition": ( + "institution_family_of_record_missing" + ), + } + ) + continue + disposition_rows.append({**base, "disposition": "observation"}) family_rows = by_family.get(interview) if pid in episodes: state = psid2010.marital_state_at( @@ -799,16 +907,22 @@ def build_age67_cohort( wife_sex = str( sex.get(int(wives["person_id"].iloc[0]), "na") ) + # An institutionalized member is neither the current head nor + # the wife of its family of record: its relationship code is to + # the previous wave's head (codebook note on ER34103). role = ( - "head" - if relationship == _HEAD + "ofum" + if in_institution else ( - "wife" - if relationship in (_LEGAL_WIFE, _PARTNER) - else "ofum" + "head" + if relationship == _HEAD + else ( + "wife" + if relationship in (_LEGAL_WIFE, _PARTNER) + else "ofum" + ) ) ) - in_institution = sequence >= _SEQUENCE_INSTITUTION[0] observation_rows.append( { "observation_id": f"{pid}:{wave}", @@ -852,10 +966,9 @@ def build_age67_cohort( if wave in inputs.family_wealth else "blocked_wealth_supplement_not_staged" ), + "in_institution": bool(in_institution), "income_status": ( - "income_rule_missing" - if in_institution - else "family_file" + "family_of_record" if in_institution else "family_file" ), } ) @@ -903,8 +1016,10 @@ def _check_family_records( if income is None: raise ValueError(f"inputs lack family income for {wave}") known = set(income["interview"].astype(int)) - in_family = rows["income_status"].eq("family_file") - absent = rows.loc[in_family & ~rows["interview"].isin(known)] + from_family = rows["income_status"].isin( + ["family_file", "family_of_record"] + ) + absent = rows.loc[from_family & ~rows["interview"].isin(known)] missing.extend(absent["observation_id"].tolist()) if missing: raise ValueError( @@ -922,24 +1037,24 @@ def income_rows( """Observations merged with their family's income and wealth. The rows :func:`populace_dynamics.estimates.adjusted_poverty. - adjusted_incomes` consumes. Observations whose wave has no staged - wealth, or whose income rule is missing, are refused unless - ``allow_blocked`` (then they are left out and counted in - ``attrs["left_out"]``). ``attrs["provenance_kind"]`` carries the + adjusted_incomes` consumes. An institutionalized observation under + ``family_of_record`` merges its family of record's income and wealth + like any member. Observations whose wave has no staged wealth are + refused unless ``allow_blocked`` (then they are left out and counted + in ``attrs["left_out"]``). ``attrs["provenance_kind"]`` carries the cohort's provenance kind to the income concept's guard. """ obs = cohort.observations - blocked = obs["wealth_status"].ne("family_file") | obs["income_status"].ne( - "family_file" - ) + blocked = obs["wealth_status"].ne("family_file") | ~obs[ + "income_status" + ].isin(["family_file", "family_of_record"]) if blocked.any() and not allow_blocked: waves = sorted(set(obs.loc[blocked, "wave"].astype(int))) raise ValueError( f"{int(blocked.sum())} observations (waves {waves}) cannot get " - "the income concept: wealth supplement not staged or no income " - "rule; pass allow_blocked=True only for a registered fallback " - "row" + "the income concept: wealth supplement not staged; pass " + "allow_blocked=True only for a registered fallback row" ) kept = obs.loc[~blocked].copy() frames = [] @@ -969,9 +1084,6 @@ def income_rows( "wealth_supplement_not_staged": int( obs["wealth_status"].ne("family_file").sum() ), - "income_rule_missing": int( - obs["income_status"].ne("family_file").sum() - ), } return out @@ -1068,7 +1180,7 @@ def structural_summary( summary["family_units_with_two_or_more_members"] = int((shared > 1).sum()) computable = obs["wealth_status"].eq("family_file") & obs[ "income_status" - ].eq("family_file") + ].isin(["family_file", "family_of_record"]) summary["n_observations_computable_now"] = int(computable.sum()) summary["n_observations_blocked"] = int((~computable).sum()) receipt = {} diff --git a/src/populace_dynamics/estimates/adjusted_poverty.py b/src/populace_dynamics/estimates/adjusted_poverty.py index d216377d..6033e385 100644 --- a/src/populace_dynamics/estimates/adjusted_poverty.py +++ b/src/populace_dynamics/estimates/adjusted_poverty.py @@ -50,6 +50,13 @@ :func:`load_poverty_thresholds`). * **Cut** (F11). Reform income subtracts ``cut_rate`` (0.13) times all Social Security of the unit; there is no behavioural response. + ``cut_start_year`` (default ``None``: every observation is cut, the F11 + primary) is row U6's parameter: with a start year, an observation is + cut only when the year its member turns 67 (birth year + 67) is at or + after it, so with 2004 the 1936 birth year (67 in 2003, observed at 68 + under U1) is uncut. The plan defines U6 on U1 with the scorecard's + "from 2004", which has no source the builder could read (plan section + 10, decision 8); the start year is pending that ruling. * **SSI response** (F13, pending Max, decision record d189). ``offset_existing_recipients`` (proposed primary): for each SSI unit with baseline SSI, SSI rises by the fall in countable Social Security @@ -126,6 +133,7 @@ "annuity_factor_joint", "annuity_factor_single", "countable_income", + "cut_applies", "load_life_table", "load_nchs_2000_life_table", "load_poverty_thresholds", @@ -309,6 +317,7 @@ class AdjustedPovertySpec: """Every income-concept choice; defaults are the plan's primary.""" cut_rate: float = 0.13 + cut_start_year: int | None = None annuitized_share: float = 0.8 real_interest_rate: float = 0.03 annuity_timing: str = "immediate" @@ -340,6 +349,16 @@ def __post_init__(self) -> None: rate = self.real_interest_rate if isinstance(rate, bool) or not (-0.5 < float(rate) < 1.0): raise AdjustedPovertyError("real_interest_rate out of range") + start = self.cut_start_year + if start is not None and ( + isinstance(start, bool) + or not isinstance(start, int) + or not 1990 <= start <= 2030 + ): + raise AdjustedPovertyError( + "cut_start_year must be None or an integer year in " + "1990-2030" + ) _choice(self.annuity_timing, ANNUITY_TIMINGS, "annuity_timing") _choice(self.mortality_basis, MORTALITY_BASES, "mortality_basis") _choice(self.terminal_closure, TERMINAL_CLOSURES, "terminal_closure") @@ -413,6 +432,19 @@ def pending_decisions() -> tuple[PendingDecision, ...]: "(plan section 10 decisions 2 and 8)", _FREEZE, ), + PendingDecision( + "cut_start_year", + spec.cut_start_year, + (2004,), + "plan F11: the primary cuts every observation (None); row U6 " + "(on U1) leaves an observation uncut when its member turned 67 " + "before the start year, 2004 being the scorecard's 'from 2004', " + "which has no source the builder could read; under U0 every " + "age-67 year is 2004 or later, so only U1's 1936 birth year " + "moves", + "Max (plan section 10 decision 8: the scenario's start year; " + "not yet on a decision card)", + ), PendingDecision( "income_unit", spec.income_unit, @@ -958,6 +990,7 @@ def _countable_ss_fall(ss: float, cut_rate: float, general: float) -> float: "observation_id", "family_unit_id", "income_year", + "birth_year", "member_role", "member_age", "member_sex", @@ -1115,11 +1148,26 @@ def _int_or_none(value: Any) -> int | None: return int(value) +def cut_applies(rows: pd.DataFrame, spec: AdjustedPovertySpec) -> np.ndarray: + """Whether the cut reaches each observation (``cut_start_year``). + + Every observation when ``cut_start_year`` is ``None``; otherwise + those whose member turns 67 (``birth_year + 67``) in or after the + start year. + """ + + if spec.cut_start_year is None: + return np.ones(len(rows), dtype=bool) + age67_year = rows["birth_year"].to_numpy(dtype=np.int64) + 67 + return age67_year >= int(spec.cut_start_year) + + def _ssi_response( rows: pd.DataFrame, spec: AdjustedPovertySpec, units: dict, ssi: SsiParameters | None, + cut_rates: np.ndarray, ) -> tuple[np.ndarray, np.ndarray]: n = len(rows) offset = np.zeros(n, dtype=np.float64) @@ -1129,8 +1177,8 @@ def _ssi_response( if ssi is None: raise AdjustedPovertyError(f"ssi_rule {spec.ssi_rule} needs SSI data") general = _MONTHS * ssi.general_income_exclusion_monthly - cut = spec.cut_rate for i, row in enumerate(rows.itertuples(index=False)): + cut = float(cut_rates[i]) year = int(row.income_year) head_ssi, wife_ssi = float(row.head_ssi), float(row.wife_ssi) hw_ssi = head_ssi + wife_ssi @@ -1198,7 +1246,8 @@ def adjusted_incomes( (``family_unit`` or ``head_wife``), ``money_income``, ``asset_income_reported``, ``asset_income_removed``, ``financial_assets``, ``annuity_basis``, ``annuity_factor``, - ``annuity``, ``baseline_income``, ``social_security``, ``cut``, + ``annuity``, ``baseline_income``, ``social_security``, + ``cut_applies``, ``cut``, ``ssi_offset``, ``ssi_new``, ``reform_income``, ``unit_size``, ``threshold``, ``threshold_cell``, ``poor_baseline`` and ``poor_reform``. @@ -1254,8 +1303,10 @@ def adjusted_incomes( else np.zeros(len(rows)) ) baseline = units["money"] - removed + annuity - cut = spec.cut_rate * units["social_security"] - offset, new = _ssi_response(rows, spec, units, ssi) + applies = cut_applies(rows, spec) + cut_rates = np.where(applies, spec.cut_rate, 0.0) + cut = cut_rates * units["social_security"] + offset, new = _ssi_response(rows, spec, units, ssi, cut_rates) reform = baseline - cut + offset + new thresholds_out = np.zeros(len(rows), dtype=np.float64) cells: list[str] = [] @@ -1292,6 +1343,7 @@ def adjusted_incomes( "annuity": annuity, "baseline_income": baseline, "social_security": units["social_security"], + "cut_applies": applies, "cut": cut, "ssi_offset": offset, "ssi_new": new, diff --git a/src/populace_dynamics/uniform_cut_track_u/__init__.py b/src/populace_dynamics/uniform_cut_track_u/__init__.py new file mode 100644 index 00000000..8541901a --- /dev/null +++ b/src/populace_dynamics/uniform_cut_track_u/__init__.py @@ -0,0 +1,43 @@ +"""Track U assembly for DynaSim scorecard exercise 2 (uniform 13% cut). + +Plan ``critical-path-uniform-cut-20260923.md`` (work items U5, U9, U10 +and field F17). The Track U layer itself lives in +:mod:`populace_dynamics.data.family_income`, +:mod:`populace_dynamics.cohorts.age67`, +:mod:`populace_dynamics.estimates.adjusted_poverty` and +:mod:`populace_dynamics.estimates.uniform_cut_tabulation`; this package +composes it: + +* :mod:`.rows` -- the registered rows (U0-U10, U-inst) as builder and + income-concept parameters, checked against the specification block; +* :mod:`.invented` -- the INVENTED age-67 cohort generator and the + INVENTED threshold table (dry runs and tests only); +* :mod:`.runner` -- the row pipeline (cohort, income rows, income + concept, tabulation) and the official-concept diagnostic, with the + provenance guards; +* :mod:`.diagnostics` -- the F17 component summaries (Social Security, + SSI, WEALTH1) against committed published aggregates. It imports + neither the income concept nor the tabulation, so the pre-registration + diagnostics script can prove it computed no poverty statistic. + +Submodules are imported explicitly, never re-exported here, so importing +:mod:`.diagnostics` does not load the income concept. + +Every output is a *PSID-realized outcome (not a projection)* computed +with a *Python income concept (not Axiom)* under *mechanical +incidence*. +""" + +from __future__ import annotations + +#: The first line of every invented-data dry-run output. +DRY_RUN_HEADER = "INVENTED DATA - NOT A COMPARISON" +#: The first line of the registered one-shot artifact. +REGISTERED_HEADER = ( + "REGISTERED ONE-SHOT RUN - Track U, DynaSim exercise 2 (uniform 13% " + "cut, adjusted poverty at 67, 1936-45 cohort). PSID-realized outcomes " + "(not a projection); Python income concept (not Axiom); mechanical " + "incidence. Publishes regardless of outcome." +) + +__all__ = ["DRY_RUN_HEADER", "REGISTERED_HEADER"] diff --git a/src/populace_dynamics/uniform_cut_track_u/diagnostics.py b/src/populace_dynamics/uniform_cut_track_u/diagnostics.py new file mode 100644 index 00000000..fda4294c --- /dev/null +++ b/src/populace_dynamics/uniform_cut_track_u/diagnostics.py @@ -0,0 +1,718 @@ +"""F17 component diagnostics for Track U: Social Security, SSI and WEALTH1. + +Plan ``critical-path-uniform-cut-20260923.md`` field F17 lists two kinds +of diagnostic row. The official-concept poverty rate involves the +threshold and runs only inside a registered run +(:func:`populace_dynamics.uniform_cut_track_u.runner. +official_concept_rates`). The other kind -- PSID Social Security, SSI +and WEALTH1 summaries for the age-67 population against published +aggregates -- is a component aggregate with no threshold, which plan +section 8 allows before registration. This module computes that kind. +It imports neither the income concept +(:mod:`populace_dynamics.estimates.adjusted_poverty`) nor the tabulation, +computes no annuity, threshold, poverty status or rate, and reads the +SSI parameters through its own hash pin. + +Population: the observations of an age-67 cohort +(:func:`populace_dynamics.cohorts.age67.build_age67_cohort`, row U0 by +default), weighted by their observation weights, each merged with its +family unit's income (every wave) and WEALTH1 (waves with staged wealth). + +Summaries per income year: + +* **Social Security.** A member's own amount is the family file's head + or wife amount by the member's role; an OFUM's own amount is not + identified (the family file reports one OFUM total), so OFUM members + enter only the family-level share. Reported: the weighted share of + head and wife members with their own Social Security, the weighted mean + own annual amount among them and its twelfth, and the weighted share of + all members whose family unit reports any Social Security. +* **SSI.** The weighted share of head and wife members reporting their + own SSI and of all members whose family reports any; the weighted mean + own annual SSI among recipients; and, against the federal benefit rate + in force on January 1 of the income year (the committed capture, + :data:`SSI_PARAMETERS_PATH`), the number and weighted share of head and + wife SSI units reporting more than twelve times the federal maximum for + their unit (individual, or couple when both receive SSI). An amount + above the federal maximum is consistent with a state supplement or a + reporting difference; this module does not say which. +* **WEALTH1.** Weighted quantiles (the smallest value whose cumulative + weight share reaches the level; :func:`weighted_quantile`) at 10, 25, + 50, 75 and 90 percent, and the weighted shares at or below zero, below + zero and imputed (accuracy code 1). + +Published aggregates (committed sources only; nothing downloaded): + +* Social Security: SSA, *Annual Statistical Supplement, 2025*, Table + 5.A4 ("Number of beneficiaries and total monthly benefits, by trust + fund and type of benefit, December 1940-2024, selected years"), read + from the committed, hash-verified snapshot + ``data/external/snapshots/ssa_level_anchors_vintage1/ + supplement2025_5a.html``. For December of the year before and of the + income year it gives the number of retired-worker beneficiaries and + their total monthly benefits; the average is their quotient (the + division is this module's, not a published cell). Concept + differences, all unadjusted: every age against the members at 66-68; + retired workers only against any benefit type (spouse and widow(er) + benefits included); December monthly benefits against a calendar-year + total divided by twelve; SSA's benefits in current-payment status + against a self-report possibly net of the Medicare premium. +* SSI: only the federal benefit rate above; no published SSI recipient + or payment aggregate is committed or saved. +* WEALTH1: none. No Survey of Consumer Finances table (the plan's + comparator, wealth percentiles by age) is committed or saved, so the + WEALTH1 summaries are not compared; capturing one is a gap. +""" + +from __future__ import annotations + +import hashlib +import html +import json +import math +import re +from collections.abc import Iterable, Mapping +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +from populace_dynamics.cohorts import age67 +from populace_dynamics.data import family_income + +__all__ = [ + "QUANTILES", + "SSA_5A_SNAPSHOT_PATH", + "SSA_5A_SNAPSHOT_SHA256", + "SSA_5A_URL", + "SSI_PARAMETERS_PATH", + "SSI_PARAMETERS_SHA256", + "component_diagnostics", + "component_rows", + "federal_benefit_rates", + "institution_record_counts", + "social_security_summary", + "ssa_table_5a4", + "ssi_summary", + "wealth1_summary", + "weighted_quantile", +] + +_ROOT = Path(__file__).resolve().parents[3] +_EXTERNAL = _ROOT / "data" / "external" +#: The committed SSI capture; the same pin as +#: ``adjusted_poverty.SSI_PARAMETERS_SHA256`` (a test holds them equal), +#: kept here so this module need not import the income concept. +SSI_PARAMETERS_PATH = _EXTERNAL / "track_u_ssi_parameters.json" +SSI_PARAMETERS_SHA256 = ( + "79e641a10d95afba81c8d831852fb52ef0a801e7175e06df17e6cc7cc3e3c523" +) +#: The committed snapshot of the SSA Annual Statistical Supplement, 2025, +#: section 5.A (captured 2026-07-27 for the SSA level anchors; its +#: SHA-256 is in the snapshot directory's capture manifest). +SSA_5A_SNAPSHOT_PATH = ( + _EXTERNAL + / "snapshots" + / "ssa_level_anchors_vintage1" + / "supplement2025_5a.html" +) +SSA_5A_SNAPSHOT_SHA256 = ( + "d61e9484d271aec0126d8897a780668adf55f5d86f5b440a0f69782de968aa8e" +) +SSA_5A_URL = ( + "https://www.ssa.gov/policy/docs/statcomps/supplement/2025/5a.html" +) +_TABLE_5A4_CAPTION = ( + "Number of beneficiaries and total monthly benefits, by trust fund and " + "type of benefit, December" +) +_TABLE_5A4_COLUMNS = ( + "Year", + "Total", + "OASI Trust Fund", + "DI Trust Fund", + "Retired workers", + "Disabled workers", + "Wives and husbands", + "Children", + "Widowed mothers and fathers", + "Widow(er)s", + "Parents of deceased workers", + "Special age-72 beneficiaries", +) +#: Table 5.A4 columns read, by key. +_TABLE_5A4_READ = { + "retired_workers": "Retired workers", + "wives_and_husbands": "Wives and husbands", + "widowers": "Widow(er)s", +} +QUANTILES: tuple[float, ...] = (0.10, 0.25, 0.50, 0.75, 0.90) +_MONTHS = 12 + + +def _sha256(path: Path) -> str: + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +# ========================================================================= +# Published and committed sources +# ========================================================================= +def federal_benefit_rates( + path: Path = SSI_PARAMETERS_PATH, + *, + expected_sha256: str = SSI_PARAMETERS_SHA256, +) -> dict[str, Any]: + """The committed federal benefit rates (monthly), hash-verified.""" + + observed = _sha256(path) + if observed != expected_sha256: + raise ValueError( + f"{path} sha256 {observed} != pinned {expected_sha256}" + ) + data = json.loads(Path(path).read_text(encoding="utf-8")) + rates = data["federal_benefit_rate_monthly"] + return { + "individual": { + int(y): float(v) for y, v in rates["individual"].items() + }, + "couple": {int(y): float(v) for y, v in rates["couple"].items()}, + "rule": data["source"]["rule"], + "source": { + "file": ( + str(Path(path).relative_to(_ROOT)) + if Path(path).is_relative_to(_ROOT) + else str(path) + ), + "sha256": observed, + "policyengine_us_revision": data["source"][ + "policyengine_us_revision" + ], + "references": data["source"]["references"]["fbr_individual"], + }, + } + + +def _cells(row: str) -> list[str]: + cells = re.findall(r"]*>(.*?)", row, re.S) + return [ + " ".join(html.unescape(re.sub(r"<[^>]+>", "", c)).split()) + for c in cells + ] + + +def ssa_table_5a4( + years: Iterable[int], + path: Path = SSA_5A_SNAPSHOT_PATH, + *, + expected_sha256: str = SSA_5A_SNAPSHOT_SHA256, +) -> dict[int, dict[str, Any]]: + """Retired-worker, spouse and widow(er) cells of Table 5.A4 by year. + + Returns, per December ``year``, ``{key: {"number", "total_monthly_ + thousands", "average_monthly"}}`` for the columns of + :data:`_TABLE_5A4_READ`, plus ``aged_types_combined`` (their summed + totals over their summed numbers). ``number`` and ``total_monthly_ + thousands`` are the published cells ("Number" and "Total monthly + benefits (thousands of dollars)" panels) as integers; the averages are + divisions made here. The layout is checked by its printed caption and + column headers; any other layout is refused. + """ + + observed = _sha256(path) + if observed != expected_sha256: + raise ValueError( + f"{path} sha256 {observed} != pinned {expected_sha256}" + ) + text = Path(path).read_text(encoding="utf-8") + start = text.find("Table 5.A4") + if start < 0: + raise ValueError("Table 5.A4 not found in the snapshot") + end = text.find("", start) + table = text[start:end] + caption = " ".join( + html.unescape(re.sub(r"<[^>]+>", " ", table[:600])).split() + ) + if _TABLE_5A4_CAPTION not in caption: + raise ValueError(f"unexpected Table 5.A4 caption: {caption[:200]!r}") + rows = re.findall(r"]*>(.*?)", table, re.S) + headers = [cell for row in rows[:2] for cell in _cells(row)] + expected = [h for h in _TABLE_5A4_COLUMNS if h != "OASDI"] + flat = [h for h in headers if h != "OASDI"] + if sorted(flat) != sorted(expected): + raise ValueError(f"unexpected Table 5.A4 headers {headers}") + columns = list(_TABLE_5A4_COLUMNS) + panels: dict[str, dict[int, list[str]]] = {} + panel = None + for row in rows[2:]: + cells = _cells(row) + if len(cells) == 2 and cells[0] == "": + panel = cells[1] + panels[panel] = {} + continue + if panel is None or not cells or not cells[0].isdigit(): + continue + if len(cells) != len(columns): + raise ValueError(f"row {cells[0]} has {len(cells)} cells") + panels[panel][int(cells[0])] = cells + number = panels.get("Number") + totals = panels.get("Total monthly benefits (thousands of dollars)") + if number is None or totals is None: + raise ValueError(f"unexpected Table 5.A4 panels {sorted(panels)}") + out: dict[int, dict[str, Any]] = {} + for year in sorted({int(y) for y in years}): + if year not in number or year not in totals: + raise ValueError(f"Table 5.A4 has no December {year} row") + entry: dict[str, Any] = {} + sum_number = 0 + sum_total = 0 + for key, header in _TABLE_5A4_READ.items(): + index = columns.index(header) + count = int(number[year][index].replace(",", "")) + total = int(totals[year][index].replace(",", "")) + sum_number += count + sum_total += total + entry[key] = { + "number": count, + "total_monthly_thousands": total, + "average_monthly": 1000.0 * total / count, + } + entry["aged_types_combined"] = { + "number": sum_number, + "total_monthly_thousands": sum_total, + "average_monthly": 1000.0 * sum_total / sum_number, + } + out[year] = entry + return out + + +# ========================================================================= +# Rows and summaries +# ========================================================================= +def component_rows( + cohort: age67.Age67Cohort, inputs: age67.Age67Inputs +) -> pd.DataFrame: + """One row per observation with its Social Security, SSI and WEALTH1. + + Columns: ``observation_id``, ``person_id``, ``birth_year``, + ``income_year``, ``wave``, ``weight``, ``member_role``, ``sex``, + ``married``, ``own_identified`` (head or wife member), ``own_ss``, + ``family_ss``, ``own_ssi``, ``head_ssi``, ``wife_ssi``, ``ofum_ssi``, + ``family_ssi``, ``wealth_available``, ``wealth1`` and ``wealth1_acc`` + (NaN where the wave has no staged wealth). No income concept is + formed: the amounts are the family file's items as read. + """ + + obs = cohort.observations + frames = [] + for wave, rows in obs.groupby("wave", sort=True): + income = inputs.family_income[int(wave)] + items = [ + "interview", + *family_income.SOCIAL_SECURITY_CONCEPTS, + *family_income.SSI_CONCEPTS, + ] + merged = rows.merge( + income[items], on="interview", how="left", validate="many_to_one" + ) + wealth = inputs.family_wealth.get(int(wave)) + if wealth is not None: + merged = merged.merge( + wealth[["interview", "wealth1", "wealth1_acc"]], + on="interview", + how="left", + validate="many_to_one", + ) + else: + merged["wealth1"] = np.nan + merged["wealth1_acc"] = np.nan + frames.append(merged) + if not frames: + raise ValueError("the cohort has no observations") + data = pd.concat(frames, ignore_index=True) + if data["head_ss"].isna().any(): + raise ValueError("an observation has no family-file income record") + role = data["member_role"].astype(str) + own_identified = role.isin(["head", "wife"]) + own_ss = np.where( + role.eq("head"), + data["head_ss"], + np.where(role.eq("wife"), data["wife_ss"], np.nan), + ) + own_ssi = np.where( + role.eq("head"), + data["head_ssi"], + np.where(role.eq("wife"), data["wife_ssi"], np.nan), + ) + out = pd.DataFrame( + { + "observation_id": data["observation_id"], + "person_id": data["person_id"], + "birth_year": data["birth_year"].astype("int64"), + "income_year": data["income_year"].astype("int64"), + "wave": data["wave"].astype("int64"), + "weight": data["weight"].astype("float64"), + "member_role": role, + "sex": data["sex"].astype(str), + "married": data["married"].astype(bool), + "own_identified": own_identified.to_numpy(), + "own_ss": own_ss.astype("float64"), + "family_ss": data[list(family_income.SOCIAL_SECURITY_CONCEPTS)] + .sum(axis=1) + .astype("float64"), + "own_ssi": own_ssi.astype("float64"), + "head_ssi": data["head_ssi"].astype("float64"), + "wife_ssi": data["wife_ssi"].astype("float64"), + "ofum_ssi": data["ofum_ssi"].astype("float64"), + "family_ssi": data[list(family_income.SSI_CONCEPTS)] + .sum(axis=1) + .astype("float64"), + "wealth_available": data["wealth1"].notna().to_numpy(), + "wealth1": data["wealth1"].astype("float64"), + "wealth1_acc": data["wealth1_acc"].astype("float64"), + } + ) + return out + + +def weighted_quantile( + values: Iterable[float], weights: Iterable[float], level: float +) -> float: + """The smallest value whose cumulative weight share reaches ``level``. + + Values are sorted ascending; the answer is the first value ``v_k`` + with ``sum_{j<=k} w_j >= level * sum w`` (no interpolation). + """ + + values = np.asarray(list(values), dtype=np.float64) + weights = np.asarray(list(weights), dtype=np.float64) + if values.size == 0 or values.size != weights.size: + raise ValueError("weighted_quantile needs matching, non-empty inputs") + if not 0.0 < float(level) <= 1.0: + raise ValueError("level must lie in (0, 1]") + if (weights < 0).any() or weights.sum() <= 0: + raise ValueError("weights must be >= 0 with a positive total") + order = np.argsort(values, kind="stable") + cumulative = np.cumsum(weights[order]) + target = float(level) * cumulative[-1] + index = int(np.searchsorted(cumulative, target - 1e-9 * cumulative[-1])) + return float(values[order][min(index, values.size - 1)]) + + +def _share(mask: np.ndarray, weights: np.ndarray) -> float | None: + total = float(weights.sum()) + return None if total <= 0 else float(weights[mask].sum() / total) + + +def _mean(values: np.ndarray, weights: np.ndarray) -> float | None: + total = float(weights.sum()) + return None if total <= 0 else float(np.sum(values * weights) / total) + + +def _by_year(rows: pd.DataFrame) -> dict[int, pd.DataFrame]: + return { + int(year): part.reset_index(drop=True) + for year, part in rows.groupby("income_year", sort=True) + } + + +def social_security_summary( + rows: pd.DataFrame, + published: Mapping[int, Mapping[str, Any]] | None = None, +) -> dict[str, Any]: + """Weighted Social Security summaries per income year (see module). + + ``published`` is :func:`ssa_table_5a4` output covering December of + each income year and the year before; when given, each year carries + the PSID monthly mean over each December average. + """ + + out: dict[str, Any] = {} + for year, part in _by_year(rows).items(): + w = part["weight"].to_numpy() + own = part["own_identified"].to_numpy() + own_ss = part["own_ss"].to_numpy() + recipient = own & (np.nan_to_num(own_ss) > 0) + entry: dict[str, Any] = { + "n_observations": int(len(part)), + "n_own_identified": int(own.sum()), + "n_own_recipients": int(recipient.sum()), + "weight_total": float(w.sum()), + "own_receipt_share": _share(recipient[own], w[own]), + "own_mean_annual_among_recipients": _mean( + own_ss[recipient], w[recipient] + ), + "family_receipt_share": _share( + part["family_ss"].to_numpy() > 0, w + ), + } + annual = entry["own_mean_annual_among_recipients"] + entry["own_mean_monthly_among_recipients"] = ( + None if annual is None else annual / _MONTHS + ) + if published is not None: + comparisons = {} + for label, december in ( + ("december_prior_year", year - 1), + ("december_income_year", year), + ): + cells = published[december] + comparisons[label] = { + "december": december, + "ssa_retired_worker_average_monthly": cells[ + "retired_workers" + ]["average_monthly"], + "ssa_aged_types_average_monthly": cells[ + "aged_types_combined" + ]["average_monthly"], + "psid_over_ssa_retired_worker": ( + None + if entry["own_mean_monthly_among_recipients"] is None + else entry["own_mean_monthly_among_recipients"] + / cells["retired_workers"]["average_monthly"] + ), + "psid_over_ssa_aged_types": ( + None + if entry["own_mean_monthly_among_recipients"] is None + else entry["own_mean_monthly_among_recipients"] + / cells["aged_types_combined"]["average_monthly"] + ), + } + entry["published"] = comparisons + out[str(year)] = entry + return out + + +def ssi_summary( + rows: pd.DataFrame, rates: Mapping[str, Mapping[int, float]] +) -> dict[str, Any]: + """Weighted SSI summaries per income year against the federal maximum. + + A head/wife SSI unit is a couple when both the head and the wife + report SSI (its combined SSI is set against twelve times the couple + rate), else an individual (the member's own SSI against twelve times + the individual rate). Rates are those in force on January 1 of the + income year. + """ + + out: dict[str, Any] = {} + for year, part in _by_year(rows).items(): + w = part["weight"].to_numpy() + own = part["own_identified"].to_numpy() + own_ssi = np.nan_to_num(part["own_ssi"].to_numpy()) + recipient = own & (own_ssi > 0) + couple = (part["head_ssi"].to_numpy() > 0) & ( + part["wife_ssi"].to_numpy() > 0 + ) + unit_ssi = np.where( + couple, + part["head_ssi"].to_numpy() + part["wife_ssi"].to_numpy(), + own_ssi, + ) + maximum = np.where( + couple, + _MONTHS * float(rates["couple"][year]), + _MONTHS * float(rates["individual"][year]), + ) + above = recipient & (unit_ssi > maximum) + out[str(year)] = { + "n_observations": int(len(part)), + "n_own_recipients": int(recipient.sum()), + "own_receipt_share": _share(recipient[own], w[own]), + "family_receipt_share": _share( + part["family_ssi"].to_numpy() > 0, w + ), + "own_mean_annual_among_recipients": _mean( + own_ssi[recipient], w[recipient] + ), + "federal_maximum_annual": { + "individual": _MONTHS * float(rates["individual"][year]), + "couple": _MONTHS * float(rates["couple"][year]), + }, + "n_recipients_above_federal_maximum": int(above.sum()), + "recipient_share_above_federal_maximum": _share( + above[recipient], w[recipient] + ), + } + return out + + +def wealth1_summary(rows: pd.DataFrame) -> dict[str, Any]: + """Weighted WEALTH1 quantiles and shares per income year with wealth.""" + + out: dict[str, Any] = {} + available = rows[rows["wealth_available"]] + for year, part in _by_year(available).items(): + w = part["weight"].to_numpy() + wealth = part["wealth1"].to_numpy() + entry: dict[str, Any] = { + "n_observations": int(len(part)), + "weight_total": float(w.sum()), + "quantiles": ( + { + f"p{int(round(100 * q))}": weighted_quantile(wealth, w, q) + for q in QUANTILES + } + if w.sum() > 0 + else None + ), + "share_at_or_below_zero": _share(wealth <= 0, w), + "share_below_zero": _share(wealth < 0, w), + "share_imputed": _share(part["wealth1_acc"].to_numpy() == 1, w), + } + out[str(year)] = entry + return out + + +def institution_record_counts(inputs: age67.Age67Inputs) -> dict[str, Any]: + """Counts only: ``# IN FU`` against the individual records per wave. + + For each wave: the families whose ``# IN FU`` equals their number of + in-family individual records (sequence 1-20), the institution records + (51-59) attached to a responding family, and among the families with + at least one such record, how many have ``# IN FU`` equal to the + in-family count alone and how many equal to the in-family count plus + the institution records. This checks the U-inst rule's reading that + ``# IN FU`` does not count institutionalized members. + """ + + out: dict[str, Any] = {} + for wave in age67.WAVES: + anchor = inputs.anchors[wave] + income = inputs.family_income[wave].set_index("interview")["fu_size"] + in_family = ( + anchor[anchor["sequence"].between(1, 20)] + .groupby("interview") + .size() + ) + institution = ( + anchor[anchor["sequence"].between(51, 59)] + .groupby("interview") + .size() + ) + counts = pd.DataFrame({"fu_size": income}) + counts["in_family"] = in_family.reindex(counts.index).fillna(0) + counts["institution"] = institution.reindex(counts.index).fillna(0) + attached = counts[counts["institution"] > 0] + out[str(wave)] = { + "n_families": int(len(counts)), + "n_fu_size_equals_in_family_records": int( + (counts["fu_size"] == counts["in_family"]).sum() + ), + "n_institution_records": int(institution.sum()), + "n_institution_records_with_family_record": int( + institution[institution.index.isin(counts.index)].sum() + ), + "n_families_with_institution_records": int(len(attached)), + "of_which_fu_size_equals_in_family_only": int( + (attached["fu_size"] == attached["in_family"]).sum() + ), + "of_which_fu_size_equals_in_family_plus_institution": int( + ( + attached["fu_size"] + == attached["in_family"] + attached["institution"] + ).sum() + ), + } + return out + + +def _clean(value: Any) -> Any: + """JSON-ready: numpy scalars to Python, non-finite floats to None.""" + + if isinstance(value, Mapping): + return {str(key): _clean(item) for key, item in value.items()} + if isinstance(value, list | tuple): + return [_clean(item) for item in value] + if isinstance(value, np.generic): + value = value.item() + if isinstance(value, float) and not math.isfinite(value): + return None + return value + + +def component_diagnostics( + cohort: age67.Age67Cohort, + inputs: age67.Age67Inputs, + *, + rates: Mapping[str, Any] | None = None, + published: Mapping[int, Mapping[str, Any]] | None = None, +) -> dict[str, Any]: + """Every F17 component summary for ``cohort`` (no poverty statistic). + + ``rates`` defaults to :func:`federal_benefit_rates` and ``published`` + to :func:`ssa_table_5a4` for the Decembers the income years need. + """ + + rows = component_rows(cohort, inputs) + years = sorted(set(rows["income_year"].astype(int))) + rates = federal_benefit_rates() if rates is None else rates + if published is None: + decembers = sorted({y for year in years for y in (year - 1, year)}) + published = ssa_table_5a4(decembers) + result = { + "population": { + "row": cohort.spec.row, + "provenance_kind": cohort.provenance.get("kind"), + "n_observations": int(len(rows)), + "n_persons": int(rows["person_id"].nunique()), + "income_years": years, + "weights": "observation weights (cross-section weight of the " + "wave reporting the income year, times the row multiplier)", + }, + "social_security": social_security_summary(rows, published), + "ssi": ssi_summary(rows, rates), + "wealth1": wealth1_summary(rows), + "institution_record_counts": institution_record_counts(inputs), + "published_sources": { + "social_security": { + "publication": "SSA, Annual Statistical Supplement, 2025", + "table": "5.A4", + "title": ( + "Number of beneficiaries and total monthly benefits, by " + "trust fund and type of benefit, December 1940-2024, " + "selected years" + ), + "url": SSA_5A_URL, + "snapshot": str(SSA_5A_SNAPSHOT_PATH.relative_to(_ROOT)), + "snapshot_sha256": SSA_5A_SNAPSHOT_SHA256, + "cells": {str(k): v for k, v in sorted(published.items())}, + "derived": ( + "average_monthly = 1000 * total monthly benefits " + "(thousands) / number (a division made here)" + ), + "concept_differences": [ + "every age against members observed at 66-68", + "retired workers (or retired workers, spouses and " + "widow(er)s combined) against any benefit type", + "December monthly benefit against a calendar-year total " + "divided by twelve", + "benefits in current-payment status against a " + "self-report possibly net of the Medicare premium", + ], + }, + "ssi": { + "federal_benefit_rates": rates["source"], + "rule": rates["rule"], + "published_aggregates": ( + "none committed or saved (no SSI recipient or payment " + "table)" + ), + }, + "wealth1": { + "published_aggregates": None, + "not_compared": ( + "no Survey of Consumer Finances table (the plan's " + "comparator: wealth percentiles by age) is committed or " + "saved; the WEALTH1 summaries are reported without a " + "published comparison" + ), + }, + }, + "not_computed": [ + "no income concept, annuity, threshold, poverty status or " + "poverty rate (the official-concept poverty rate of plan F17 " + "runs only in the registered run)", + ], + } + return _clean(result) diff --git a/src/populace_dynamics/uniform_cut_track_u/invented.py b/src/populace_dynamics/uniform_cut_track_u/invented.py new file mode 100644 index 00000000..00f00a28 --- /dev/null +++ b/src/populace_dynamics/uniform_cut_track_u/invented.py @@ -0,0 +1,1477 @@ +"""INVENTED age-67 cohort inputs and thresholds (dry runs and tests only). + +Every person, family, income amount, Social Security and SSI amount, +wealth component, weight, sampling stratum and cluster produced here is +INVENTED by a seeded generator. Nothing is a PSID observation, an SSA or +Census statistic, or a comparator value. The frames have the shapes of +the Track U readers (:class:`populace_dynamics.cohorts.age67.Age67Inputs`: +the individual-file anchors of waves 2005-2013, the design variables, the +death and marriage records, and the family income and wealth frames of +:mod:`populace_dynamics.data.family_income`), so the invented population +runs through the real age-67 builder, the income concept and the +tabulation without touching PSID. + +The family mix is chosen to exercise every Track U path, not to resemble +any real population: + +* retired couples (both members sometimes in the target cohort, so a + family unit holds two members), single retirees (widowed, divorced, + never married, and an undated marriage whose status is ``unknown``), + low-income singles near the invented threshold, SSI recipients (some + above the federal maximum, a stand-in for a state supplement) and SSI + couples; +* a parent in the target cohort living as an OFUM in an adult child's + family with a grandchild under 18 (U4 and U10), some without a + marriage-history record (birth year from the seed coordinate); and + couples in the target cohort with an earning adult child as an OFUM + (U4 drops the child's income for them); +* couples still working at 67 with business, farm and rental losses + (negative items the codebooks allow) and large wealth; singles with + negative WEALTH1; cohabiting couples (relationship 22, not married; + the partner's birth year inferred from invented earnings); separated + members whose spouse lives elsewhere; +* members who enter an institution (sequence 51) in the first wave whose + income year is at or after their 66th birthday year and stay, attached + to the family they left, where the spouse becomes head (row U-inst); + members who move out (71) or die (81) by that wave's interview; and + members whose weight is zero from that wave. + +Every family unit's income adds up exactly under the codebook identities +(:func:`populace_dynamics.data.family_income.reconcile_family_income`), +and WEALTH1 equals its assets less its debts in every wave. Waves 2005 +and 2007 have no invented wealth unless ``supplement_waves_staged`` (the +real supplements are not staged, cos decision d189); with it they carry +invented wealth in the 2009 component layout. + +The threshold table (:func:`invented_poverty_thresholds`) is INVENTED +too: round numbers in the shape of the Census tables, not Census values. +The Census thresholds are not captured (cos decision d194). + +Provenance: the inputs record ``kind="invented"``, the seed, the +supplement flag and the SHA-256 of their frames, which the age-67 builder +checks. A recorded digest proves nothing on its own, so +:func:`check_invented_inputs` and :func:`check_invented_cohort` +re-generate the frames from the seed and compare. +""" + +from __future__ import annotations + +import dataclasses +from collections.abc import Iterable, Mapping +from typing import Any + +import numpy as np +import pandas as pd + +from populace_dynamics.cohorts import age67 +from populace_dynamics.data import family_income +from populace_dynamics.estimates import adjusted_poverty as ap + +__all__ = [ + "DEFAULT_SEED", + "INVENTED_FAMILY_COUNTS", + "INVENTED_INPUTS_LABEL", + "INVENTED_THRESHOLDS_LABEL", + "INVENTED_THRESHOLD_BASE_2004", + "check_invented_cohort", + "check_invented_inputs", + "invented_age67_inputs", + "invented_frames_sha256", + "invented_needs_standard", + "invented_poverty_thresholds", +] + +INVENTED_INPUTS_LABEL = ( + "INVENTED DATA: synthetic persons and families in the Track U age-67 " + "reader shapes (PSID individual and family files, waves 2005-2013); " + "not PSID observations" +) +INVENTED_THRESHOLDS_LABEL = ( + "INVENTED THRESHOLDS: made-up round numbers in the shape of the Census " + "poverty-threshold tables, not Census values; the Census thresholds " + "are not captured (cos decision d194)" +) +DEFAULT_SEED = 20260924 +_GENERATOR = ( + "populace_dynamics.uniform_cut_track_u.invented.invented_age67_inputs" +) + +#: Families per invented type. +INVENTED_FAMILY_COUNTS: dict[str, int] = { + "retired_couple": 36, + "retired_single": 30, + "ssi_single": 12, + "ssi_couple": 4, + "near_threshold_single": 24, + "elder_with_child": 10, + "couple_with_adult_child": 6, + "working_couple": 10, + "negative_wealth_single": 8, + "cohabiting_couple": 8, + "separated": 4, + "institution_spouse": 6, + "mover_out": 3, + "decedent": 3, + "zero_weight": 2, +} + +#: INVENTED threshold rows for income year 2004 (round numbers, not the +#: Census values); later years grow by :data:`_THRESHOLD_GROWTH`. +INVENTED_THRESHOLD_BASE_2004: dict[str, float] = { + "one_under_65": 9_800.0, + "one_65_plus": 9_000.0, + "two_under_65": 12_600.0, + "two_65_plus": 11_400.0, + "three": 15_000.0, + "four": 19_300.0, + "five": 22_800.0, + "six": 25_800.0, + "seven": 29_300.0, + "eight": 32_600.0, + "nine_plus": 39_000.0, +} +_THRESHOLD_SIZES: dict[str, int] = { + "one_under_65": 1, + "one_65_plus": 1, + "two_under_65": 2, + "two_65_plus": 2, + "three": 3, + "four": 4, + "five": 5, + "six": 6, + "seven": 7, + "eight": 8, + "nine_plus": 9, +} +#: INVENTED annual growth of thresholds and of income amounts. +_THRESHOLD_GROWTH = 0.028 +_PRICE_GROWTH = 0.028 +#: INVENTED annual growth of wealth components. +_WEALTH_GROWTH = 0.03 +#: An invented monthly SSI payment level (not the federal benefit rate). +_INVENTED_SSI_MONTHLY_2004 = 560.0 +_YEARS = tuple(range(2004, 2013)) +_WAVES = age67.WAVES +_TARGET_YEARS = age67.ALL_BIRTH_YEARS +_FAMILY_UNIT_OFFSET = 1_000 +_PERSON_BASE = 500_000 +_N_STRATA = 8 +_SEQ_INSTITUTION, _SEQ_MOVED, _SEQ_DIED = 51, 71, 81 +_HEAD, _WIFE, _PARTNER, _PARENT, _GRANDCHILD = 10, 20, 22, 50, 60 +#: Individual-income items of an invented person, in 2004 dollars. +_ITEMS = ( + "labor", + "farm", + "business_labor", + "business_asset", + "rent", + "dividends", + "interest", + "trusts", + "ss", + "ssi", + "pension", + "annuity", + "va", + "other_retirement", + "ira", +) +_WEALTH_ITEMS = ( + "farm_business", + "checking_saving", + "other_real_estate", + "stocks", + "vehicles", + "other_assets", + "ira_annuity", + "unsecured_debt", + "home_equity", +) + + +# ========================================================================= +# Thresholds +# ========================================================================= +def _threshold(row: str, year: int) -> float: + base = INVENTED_THRESHOLD_BASE_2004[row] + value = base * (1.0 + _THRESHOLD_GROWTH) ** (int(year) - 2004) + return float(round(value / 10.0) * 10) + + +def _matrix_value(row: str, year: int, children: int) -> float: + value = _threshold(row, year) * (1.02 - 0.015 * int(children)) + return float(round(value / 10.0) * 10) + + +def invented_poverty_thresholds( + years: Iterable[int] = _YEARS, +) -> ap.PovertyThresholds: + """INVENTED thresholds in the Census table shapes (not Census values). + + Weighted averages: :data:`INVENTED_THRESHOLD_BASE_2004` grown by 2.8 + percent a year and rounded to $10. Matrix: the weighted average times + ``1.02 - 0.015 * children`` for children 0 to ``min(size - 1, 8)``, + rounded to $10. Its provenance kind is ``invented``, which a + registered run refuses. + """ + + years = tuple(int(year) for year in years) + weighted = { + year: {row: _threshold(row, year) for row in ap.THRESHOLD_ROW_KEYS} + for year in years + } + matrix = { + year: { + row: { + children: _matrix_value(row, year, children) + for children in range(min(_THRESHOLD_SIZES[row] - 1, 8) + 1) + } + for row in ap.THRESHOLD_ROW_KEYS + } + for year in years + } + return ap.PovertyThresholds( + weighted_average=weighted, + matrix=matrix, + provenance={ + "kind": ap.INVENTED, + "label": INVENTED_THRESHOLDS_LABEL, + "generator": ( + "populace_dynamics.uniform_cut_track_u.invented." + "invented_poverty_thresholds" + ), + "base_2004": dict(INVENTED_THRESHOLD_BASE_2004), + "annual_growth": _THRESHOLD_GROWTH, + }, + ) + + +def invented_needs_standard( + year: int, size: int, children: int, head_age: int | None +) -> int: + """An INVENTED stand-in for PSID's CENSUS NEEDS STANDARD (row U8). + + The invented matrix cell by unit size, children and the householder's + age (the under-65 rows for a head under 65), not the 65-and-over rule. + """ + + size = int(size) + if size <= 2: + older = head_age is not None and int(head_age) >= 65 + row = ("one" if size == 1 else "two") + ( + "_65_plus" if older else "_under_65" + ) + else: + row = {3: "three", 4: "four", 5: "five", 6: "six", 7: "seven"}.get( + size, "eight" if size == 8 else "nine_plus" + ) + column = min(int(children), size - 1, 8) + return int(_matrix_value(row, year, column)) + + +# ========================================================================= +# Families +# ========================================================================= +def _first_wave(birth_year: int) -> int: + """The first wave whose income year is at or after age 66.""" + + return next(w for w in _WAVES if w - 1 >= birth_year + 66) + + +class _Builder: + """Accumulates invented families (one pass, fixed draw order).""" + + def __init__(self, rng: np.random.Generator) -> None: + self.rng = rng + self.families: list[dict[str, Any]] = [] + self.target_counter = 0 + + # -- draws ------------------------------------------------------------- + def uniform(self, low: float, high: float) -> float: + return float(self.rng.uniform(low, high)) + + def chance(self, p: float) -> bool: + return bool(self.rng.random() < p) + + def target_year(self) -> int: + year = _TARGET_YEARS[self.target_counter % len(_TARGET_YEARS)] + self.target_counter += 1 + return int(year) + + def sex(self) -> str: + return str(self.rng.choice(["female", "male"])) + + # -- persons ----------------------------------------------------------- + def person( + self, + family: dict[str, Any], + slot: int, + *, + sex: str, + birth_year: int, + marital: str, + spouse_slot: int | None = None, + **income: float, + ) -> dict[str, Any]: + roll = self.rng.random() + reported: int | None = birth_year + if roll < 0.08: + reported = None + elif roll < 0.10: + reported = birth_year + 1 + person = { + "person_id": _PERSON_BASE + 100 * family["index"] + slot, + "slot": slot, + "sex": sex, + "birth_year": int(birth_year), + "marital": marital, + "spouse_slot": spouse_slot, + "reported_birth_year": reported, + "ss_start_age": int(self.rng.integers(62, 67)), + "income": {item: float(income.get(item, 0.0)) for item in _ITEMS}, + "noise": { + wave: float(np.exp(self.rng.normal(0.0, 0.04))) + for wave in _WAVES + }, + "earnings_rows": False, + } + family["persons"].append(person) + return person + + def family(self, family_type: str) -> dict[str, Any]: + index = len(self.families) + 1 + family = { + "type": family_type, + "index": index, + "persons": [], + "roster": {}, + "weight": round(self.uniform(1_500.0, 15_000.0), 2), + "zero_weight": {}, + "stratum": 1 + index % _N_STRATA, + "cluster": 1 + (index // _N_STRATA) % 2, + "wealth": {item: 0.0 for item in _WEALTH_ITEMS}, + "wealth_imputed": self.chance(0.1), + "ss_imputed": self.chance(0.05), + "death_year": {}, + } + self.families.append(family) + return family + + def steady_roster( + self, family: dict[str, Any], members: list[tuple[int, int, int]] + ) -> None: + """Every wave: ``(slot, sequence, relationship)`` for each member.""" + + for wave in _WAVES: + family["roster"][wave] = list(members) + + def wealth(self, family: dict[str, Any], **values: float) -> None: + for key, value in values.items(): + family["wealth"][key] = float(value) + + # -- family types ------------------------------------------------------ + def retired_couple(self) -> None: + family = self.family("retired_couple") + head_birth = self.target_year() + wife_birth = int( + np.clip(head_birth + self.rng.integers(-4, 6), 1930, 1955) + ) + modest = self.chance(0.25) + self.person( + family, + 1, + sex="male", + birth_year=head_birth, + marital="married", + spouse_slot=2, + ss=( + self.uniform(8_000, 11_000) + if modest + else self.uniform(11_000, 22_000) + ), + pension=( + 0.0 + if modest or self.chance(0.5) + else self.uniform(3_000, 25_000) + ), + interest=self.uniform(0, 3_000) if self.chance(0.6) else 0.0, + dividends=self.uniform(0, 5_000) if self.chance(0.3) else 0.0, + rent=self.uniform(-3_000, 8_000) if self.chance(0.1) else 0.0, + ira=self.uniform(1_000, 8_000) if self.chance(0.2) else 0.0, + ) + self.person( + family, + 2, + sex="female", + birth_year=wife_birth, + marital="married", + spouse_slot=1, + ss=( + self.uniform(3_000, 6_000) + if modest + else self.uniform(5_000, 13_000) + ), + pension=self.uniform(2_000, 10_000) if self.chance(0.3) else 0.0, + interest=self.uniform(0, 1_500) if self.chance(0.4) else 0.0, + ) + self.steady_roster(family, [(1, 1, _HEAD), (2, 2, _WIFE)]) + if modest: + self.wealth( + family, + checking_saving=self.uniform(0, 4_000), + vehicles=self.uniform(0, 6_000), + unsecured_debt=self.uniform(0, 3_000), + home_equity=self.uniform(0, 60_000), + ) + else: + self.wealth( + family, + checking_saving=self.uniform(2_000, 40_000), + stocks=( + self.uniform(5_000, 200_000) if self.chance(0.4) else 0.0 + ), + ira_annuity=( + self.uniform(10_000, 300_000) if self.chance(0.5) else 0.0 + ), + vehicles=self.uniform(3_000, 25_000), + other_real_estate=( + self.uniform(20_000, 150_000) if self.chance(0.1) else 0.0 + ), + other_assets=( + self.uniform(1_000, 30_000) if self.chance(0.1) else 0.0 + ), + unsecured_debt=( + self.uniform(500, 15_000) if self.chance(0.3) else 0.0 + ), + home_equity=( + self.uniform(30_000, 250_000) if self.chance(0.8) else 0.0 + ), + ) + + def retired_single(self) -> None: + family = self.family("retired_single") + birth = self.target_year() + roll = self.rng.random() + marital = ( + "widowed" + if roll < 0.5 + else ( + "divorced" + if roll < 0.75 + else "never_married" if roll < 0.9 else "unknown" + ) + ) + self.person( + family, + 1, + sex=self.sex(), + birth_year=birth, + marital=marital, + ss=self.uniform(7_000, 17_000), + pension=self.uniform(2_000, 12_000) if self.chance(0.3) else 0.0, + interest=self.uniform(0, 1_200) if self.chance(0.5) else 0.0, + ) + self.steady_roster(family, [(1, 1, _HEAD)]) + self.wealth( + family, + checking_saving=self.uniform(500, 20_000), + ira_annuity=( + self.uniform(5_000, 80_000) if self.chance(0.3) else 0.0 + ), + vehicles=self.uniform(1_000, 12_000) if self.chance(0.5) else 0.0, + unsecured_debt=( + self.uniform(200, 5_000) if self.chance(0.2) else 0.0 + ), + home_equity=( + self.uniform(20_000, 150_000) if self.chance(0.5) else 0.0 + ), + ) + + def _ssi_amount(self, ss: float, above_federal: bool) -> float: + base = 12 * _INVENTED_SSI_MONTHLY_2004 - max(0.0, ss - 240.0) + amount = max(0.0, base) * self.uniform(0.8, 1.0) + return amount + (1_200.0 if above_federal else 0.0) + + def ssi_single(self) -> None: + family = self.family("ssi_single") + ss = self.uniform(2_400, 6_000) + self.person( + family, + 1, + sex=self.sex(), + birth_year=self.target_year(), + marital="never_married" if self.chance(0.5) else "divorced", + ss=ss, + ssi=self._ssi_amount(ss, above_federal=self.chance(0.25)), + ) + self.steady_roster(family, [(1, 1, _HEAD)]) + self.wealth( + family, + checking_saving=self.uniform(0, 1_500), + vehicles=self.uniform(0, 2_500) if self.chance(0.5) else 0.0, + ) + + def ssi_couple(self) -> None: + family = self.family("ssi_couple") + head_birth = self.target_year() + for slot, sex, spouse in ((1, "male", 2), (2, "female", 1)): + ss = self.uniform(2_000, 5_000) + self.person( + family, + slot, + sex=sex, + # Both members share a birth year, so under U0 one family + # unit holds two observations. + birth_year=head_birth, + marital="married", + spouse_slot=spouse, + ss=ss, + ssi=self._ssi_amount(ss, above_federal=False) * 0.75, + ) + self.steady_roster(family, [(1, 1, _HEAD), (2, 2, _WIFE)]) + self.wealth( + family, + checking_saving=self.uniform(0, 2_000), + vehicles=self.uniform(0, 3_000), + ) + + def near_threshold_single(self) -> None: + family = self.family("near_threshold_single") + target = INVENTED_THRESHOLD_BASE_2004["one_65_plus"] + self.person( + family, + 1, + sex=self.sex(), + birth_year=self.target_year(), + marital="widowed" if self.chance(0.6) else "divorced", + ss=target * self.uniform(0.95, 1.18), + interest=self.uniform(50, 400) if self.chance(0.4) else 0.0, + ) + self.steady_roster(family, [(1, 1, _HEAD)]) + self.wealth( + family, + checking_saving=self.uniform(0, 3_000), + vehicles=self.uniform(0, 4_000) if self.chance(0.4) else 0.0, + ) + + def elder_with_child(self) -> None: + family = self.family("elder_with_child") + elder_birth = self.target_year() + child_birth = int(elder_birth + 28 + self.rng.integers(0, 6)) + self.person( + family, + 1, + sex="male", + birth_year=child_birth, + marital="married", + spouse_slot=2, + labor=self.uniform(25_000, 70_000), + ) + self.person( + family, + 2, + sex="female", + birth_year=int(child_birth + self.rng.integers(-3, 4)), + marital="married", + spouse_slot=1, + labor=self.uniform(10_000, 40_000) if self.chance(0.6) else 0.0, + ) + self.person( + family, + 3, + sex=self.sex(), + birth_year=elder_birth, + marital="widowed" if self.chance(0.5) else "none", + ss=self.uniform(6_000, 14_000), + ssi=self.uniform(1_000, 3_000) if self.chance(0.2) else 0.0, + interest=self.uniform(0, 1_500) if self.chance(0.3) else 0.0, + pension=self.uniform(1_000, 6_000) if self.chance(0.2) else 0.0, + ) + self.person( + family, + 4, + sex=self.sex(), + birth_year=int(1995 + self.rng.integers(0, 6)), + marital="none", + ) + self.steady_roster( + family, + [ + (1, 1, _HEAD), + (2, 2, _WIFE), + (3, 3, _PARENT), + (4, 4, _GRANDCHILD), + ], + ) + self.wealth( + family, + checking_saving=self.uniform(1_000, 15_000), + ira_annuity=( + self.uniform(5_000, 60_000) if self.chance(0.5) else 0.0 + ), + vehicles=self.uniform(5_000, 25_000), + unsecured_debt=self.uniform(1_000, 12_000), + home_equity=self.uniform(20_000, 120_000), + ) + + def couple_with_adult_child(self) -> None: + family = self.family("couple_with_adult_child") + head_birth = self.target_year() + self.person( + family, + 1, + sex="male", + birth_year=head_birth, + marital="married", + spouse_slot=2, + ss=self.uniform(7_000, 14_000), + ) + self.person( + family, + 2, + sex="female", + birth_year=int(head_birth + self.rng.integers(-2, 4)), + marital="married", + spouse_slot=1, + ss=self.uniform(3_000, 8_000), + ) + self.person( + family, + 3, + sex=self.sex(), + birth_year=int(head_birth + 30 + self.rng.integers(0, 5)), + marital="never_married", + labor=self.uniform(12_000, 40_000), + interest=self.uniform(0, 500) if self.chance(0.3) else 0.0, + ) + self.steady_roster(family, [(1, 1, _HEAD), (2, 2, _WIFE), (3, 3, 30)]) + self.wealth( + family, + checking_saving=self.uniform(500, 6_000), + vehicles=self.uniform(2_000, 12_000), + unsecured_debt=self.uniform(0, 4_000), + home_equity=self.uniform(0, 90_000), + ) + + def working_couple(self) -> None: + family = self.family("working_couple") + owner = self.chance(0.5) + head_birth = self.target_year() + self.person( + family, + 1, + sex="male", + birth_year=head_birth, + marital="married", + spouse_slot=2, + labor=self.uniform(20_000, 90_000), + business_labor=self.uniform(5_000, 40_000) if owner else 0.0, + business_asset=self.uniform(-15_000, 30_000) if owner else 0.0, + farm=self.uniform(-5_000, 20_000) if self.chance(0.2) else 0.0, + rent=self.uniform(-6_000, 12_000) if self.chance(0.4) else 0.0, + dividends=self.uniform(0, 20_000), + interest=self.uniform(0, 6_000), + trusts=self.uniform(0, 5_000) if self.chance(0.2) else 0.0, + ss=0.0 if self.chance(0.5) else self.uniform(15_000, 25_000), + ) + family["persons"][-1]["earnings_rows"] = True + self.person( + family, + 2, + sex="female", + birth_year=int(head_birth + self.rng.integers(-3, 6)), + marital="married", + spouse_slot=1, + labor=self.uniform(10_000, 50_000) if self.chance(0.5) else 0.0, + business_labor=self.uniform(0, 10_000) if owner else 0.0, + business_asset=self.uniform(-3_000, 8_000) if owner else 0.0, + ss=self.uniform(6_000, 14_000) if self.chance(0.5) else 0.0, + ) + self.steady_roster(family, [(1, 1, _HEAD), (2, 2, _WIFE)]) + self.wealth( + family, + farm_business=self.uniform(50_000, 500_000) if owner else 0.0, + checking_saving=self.uniform(10_000, 80_000), + stocks=self.uniform(20_000, 400_000), + ira_annuity=self.uniform(50_000, 600_000), + vehicles=self.uniform(10_000, 40_000), + other_real_estate=( + self.uniform(50_000, 300_000) if self.chance(0.4) else 0.0 + ), + home_equity=self.uniform(100_000, 500_000), + ) + + def negative_wealth_single(self) -> None: + family = self.family("negative_wealth_single") + self.person( + family, + 1, + sex=self.sex(), + birth_year=self.target_year(), + marital="divorced", + ss=self.uniform(8_000, 14_000), + ) + self.steady_roster(family, [(1, 1, _HEAD)]) + self.wealth( + family, + checking_saving=self.uniform(0, 1_000), + vehicles=self.uniform(0, 3_000), + unsecured_debt=self.uniform(5_000, 40_000), + ) + + def cohabiting_couple(self) -> None: + family = self.family("cohabiting_couple") + head_birth = self.target_year() + self.person( + family, + 1, + sex="male", + birth_year=head_birth, + marital="divorced", + ss=self.uniform(9_000, 16_000), + ) + partner = self.person( + family, + 2, + sex="female", + birth_year=int(head_birth + self.rng.integers(-3, 5)), + marital="none", + ss=self.uniform(4_000, 10_000), + labor=self.uniform(0, 8_000) if self.chance(0.3) else 0.0, + ) + partner["earnings_rows"] = True + self.steady_roster(family, [(1, 1, _HEAD), (2, 2, _PARTNER)]) + self.wealth( + family, + checking_saving=self.uniform(500, 8_000), + vehicles=self.uniform(1_000, 10_000), + home_equity=self.uniform(0, 80_000), + ) + + def separated(self) -> None: + family = self.family("separated") + self.person( + family, + 1, + sex="female", + birth_year=self.target_year(), + marital="separated", + ss=self.uniform(6_000, 12_000), + pension=self.uniform(1_000, 5_000) if self.chance(0.5) else 0.0, + ) + self.steady_roster(family, [(1, 1, _HEAD)]) + self.wealth( + family, + checking_saving=self.uniform(200, 6_000), + vehicles=self.uniform(0, 8_000), + ) + + def institution_spouse(self) -> None: + family = self.family("institution_spouse") + birth = self.target_year() + self.person( + family, + 1, + sex="male", + birth_year=birth, + marital="married", + spouse_slot=2, + ss=self.uniform(10_000, 18_000), + ) + self.person( + family, + 2, + sex="female", + birth_year=int(birth + self.rng.integers(-2, 5)), + marital="married", + spouse_slot=1, + ss=self.uniform(5_000, 11_000), + ) + start = _first_wave(birth) + for wave in _WAVES: + if wave < start: + family["roster"][wave] = [(1, 1, _HEAD), (2, 2, _WIFE)] + else: + # The husband enters an institution; the wife becomes head + # and his record (relationship to the previous head, 10) + # stays attached to her family. + family["roster"][wave] = [ + (2, 1, _HEAD), + (1, _SEQ_INSTITUTION, _HEAD), + ] + self.wealth( + family, + checking_saving=self.uniform(1_000, 20_000), + vehicles=self.uniform(0, 10_000), + home_equity=self.uniform(0, 100_000), + ) + + def exact_age_year(self) -> int: + """An odd (U0) birth year from 1939, so the member is in a family + in a wave before the first one at or after age 66.""" + + return int(self.rng.choice([1939, 1941, 1943, 1945])) + + def _leaver(self, family_type: str, sequence: int) -> None: + family = self.family(family_type) + birth = self.exact_age_year() + self.person( + family, + 1, + sex=self.sex(), + birth_year=birth, + marital="widowed", + ss=self.uniform(8_000, 15_000), + ) + start = _first_wave(birth) + for wave in _WAVES: + if wave < start: + family["roster"][wave] = [(1, 1, _HEAD)] + elif wave == start: + family["roster"][wave] = [(1, sequence, _HEAD)] + else: + family["roster"][wave] = [] + if sequence == _SEQ_DIED: + family["death_year"][1] = start - 1 + self.wealth(family, checking_saving=self.uniform(500, 5_000)) + + def mover_out(self) -> None: + self._leaver("mover_out", _SEQ_MOVED) + + def decedent(self) -> None: + self._leaver("decedent", _SEQ_DIED) + + def zero_weight(self) -> None: + family = self.family("zero_weight") + birth = self.exact_age_year() + self.person( + family, + 1, + sex=self.sex(), + birth_year=birth, + marital="never_married", + ss=self.uniform(8_000, 15_000), + ) + self.steady_roster(family, [(1, 1, _HEAD)]) + start = _first_wave(birth) + family["zero_weight"] = {1: start} + self.wealth(family, checking_saving=self.uniform(500, 5_000)) + + +def _families(rng: np.random.Generator) -> list[dict[str, Any]]: + builder = _Builder(rng) + for family_type, count in INVENTED_FAMILY_COUNTS.items(): + make = getattr(builder, family_type) + for _ in range(count): + make() + return builder.families + + +# ========================================================================= +# Frames +# ========================================================================= +def _interview(family: Mapping[str, Any], wave: int) -> int: + return (wave - 2000) * _FAMILY_UNIT_OFFSET + int(family["index"]) + + +def _grow(amount: float, year: int, rate: float = _PRICE_GROWTH) -> float: + return float(amount) * (1.0 + rate) ** (int(year) - 2004) + + +def _person_amounts(person: Mapping[str, Any], wave: int) -> dict[str, int]: + """One person's invented items in ``wave``'s income year (dollars).""" + + year = wave - 1 + age = year - int(person["birth_year"]) + factor = person["noise"][wave] + out = {} + for item in _ITEMS: + amount = person["income"][item] + if item == "ss" and age < int(person["ss_start_age"]): + amount = 0.0 + out[item] = int(round(_grow(amount, year) * factor)) + return out + + +def _marriage_rows(family: Mapping[str, Any]) -> list[dict[str, Any]]: + rows = [] + by_slot = {person["slot"]: person for person in family["persons"]} + for person in family["persons"]: + marital = person["marital"] + if marital == "none": + continue + base = { + "person_id": person["person_id"], + "sex": person["sex"], + "birth_year": person["birth_year"], + "birth_month": pd.NA, + "marriage_order": pd.NA, + "spouse_person_id": pd.NA, + "start_year": pd.NA, + "start_month": pd.NA, + "end_year": pd.NA, + "end_month": pd.NA, + "separation_year": pd.NA, + "separation_month": pd.NA, + "how_ended": "never_married", + "last_known_status": "never_married", + "most_recent_report_year": 2023, + "n_marriages": 0, + "n_records": 1, + "is_marriage": False, + } + married = { + "is_marriage": True, + "marriage_order": 1, + "n_marriages": 1, + } + birth = int(person["birth_year"]) + if marital == "married": + spouse = by_slot[person["spouse_slot"]] + base.update( + married, + spouse_person_id=spouse["person_id"], + start_year=max(birth, int(spouse["birth_year"])) + 24, + how_ended="intact", + last_known_status="married", + ) + elif marital == "widowed": + base.update( + married, + spouse_person_id=person["person_id"] + 90_000_000, + start_year=birth + 23, + end_year=1990 + int(person["person_id"]) % 13, + how_ended="widowhood", + last_known_status="widowed", + ) + elif marital == "divorced": + base.update( + married, + start_year=birth + 25, + end_year=birth + 33, + how_ended="divorce", + last_known_status="divorced", + ) + elif marital == "separated": + base.update( + married, + spouse_person_id=person["person_id"] + 80_000_000, + start_year=birth + 22, + separation_year=1998, + how_ended="separated", + last_known_status="separated", + ) + elif marital == "unknown": + # A marriage the history cannot date: the state is "unknown". + base.update( + married, + how_ended="other", + last_known_status="married", + ) + rows.append(base) + return rows + + +def _typed_marriage_frame(rows: list[dict[str, Any]]) -> pd.DataFrame: + frame = pd.DataFrame(rows) + for column in ( + "birth_year", + "birth_month", + "marriage_order", + "spouse_person_id", + "start_year", + "start_month", + "end_year", + "end_month", + "separation_year", + "separation_month", + "most_recent_report_year", + "n_marriages", + ): + frame[column] = frame[column].astype("Int64") + for column in ("sex", "how_ended", "last_known_status"): + frame[column] = frame[column].astype("string") + frame["n_records"] = frame["n_records"].astype("int64") + frame["is_marriage"] = frame["is_marriage"].astype(bool) + return frame + + +def _anchor_frames( + families: list[dict[str, Any]], +) -> dict[int, pd.DataFrame]: + out = {} + for index, wave in enumerate(_WAVES): + rows = [] + for family in families: + placed = { + slot: (sequence, relationship) + for slot, sequence, relationship in family["roster"][wave] + } + for person in family["persons"]: + slot = person["slot"] + if slot not in placed: + rows.append( + { + "person_id": person["person_id"], + "interview": 0, + "sequence": 0, + "relationship": 0, + "age": 0, + "reported_birth_year": pd.NA, + "weight": 0.0, + } + ) + continue + sequence, relationship = placed[slot] + zero_from = family["zero_weight"].get(slot) + weight = round(family["weight"] * (1.0 + 0.01 * index), 2) + if zero_from is not None and wave >= zero_from: + weight = 0.0 + reported = person["reported_birth_year"] + rows.append( + { + "person_id": person["person_id"], + "interview": _interview(family, wave), + "sequence": sequence, + "relationship": relationship, + "age": (wave - 1) - int(person["birth_year"]), + "reported_birth_year": ( + pd.NA if reported is None else int(reported) + ), + "weight": float(weight), + } + ) + frame = pd.DataFrame(rows) + for column in ("interview", "sequence", "relationship", "age"): + frame[column] = frame[column].astype("int64") + frame["weight"] = frame["weight"].astype("float64") + frame["reported_birth_year"] = frame["reported_birth_year"].astype( + "Int64" + ) + out[wave] = frame + return out + + +def _in_family( + family: Mapping[str, Any], wave: int +) -> list[tuple[dict[str, Any], int]]: + by_slot = {person["slot"]: person for person in family["persons"]} + return [ + (by_slot[slot], relationship) + for slot, sequence, relationship in family["roster"][wave] + if 1 <= sequence <= 20 + ] + + +def _income_row(family: Mapping[str, Any], wave: int) -> dict[str, Any]: + """One family's invented income record in the reader's decoded shape.""" + + table = family_income.income_variables(wave) + row: dict[str, Any] = {concept: 0 for concept in table} + members = _in_family(family, wave) + head = next(person for person, rel in members if rel == _HEAD) + wives = [person for person, rel in members if rel in (_WIFE, _PARTNER)] + wife = wives[0] if wives else None + ofums = [ + person + for person, rel in members + if person is not head and person is not wife + ] + year = wave - 1 + for role, person in (("head", head), ("wife", wife)): + if person is None: + continue + a = _person_amounts(person, wave) + row[f"{role}_labor"] += a["labor"] + row[f"{role}_business_labor"] += a["business_labor"] + row[f"{role}_business_asset"] += a["business_asset"] + for item in ("rent", "dividends", "interest", "trusts"): + row[f"{role}_{item}"] += a[item] + row[f"{role}_ss"] += a["ss"] + row[f"{role}_ssi"] += a["ssi"] + # The codebooks' farm income is the head's and wife's together. + row["head_farm"] += a["farm"] + if role == "head": + row["head_retirement_pensions"] += a["pension"] + row["head_annuities"] += a["annuity"] + row["head_va_pension"] += a["va"] + if wave == 2013: + row["head_other_retirement"] += a["other_retirement"] + row["head_iras"] += a["ira"] + else: + row["head_other_retirement"] += ( + a["other_retirement"] + a["ira"] + ) + elif wave == 2013: + row["wife_retirement_pensions"] += a["pension"] + a["va"] + row["wife_annuities"] += a["annuity"] + row["wife_iras"] += a["ira"] + row["wife_other_retirement"] += a["other_retirement"] + else: + row["wife_retirement_annuities"] += ( + a["pension"] + + a["annuity"] + + a["va"] + + a["other_retirement"] + + a["ira"] + ) + for person in ofums: + a = _person_amounts(person, wave) + row["ofum_labor"] += a["labor"] + a["business_labor"] + a["farm"] + row["ofum_asset"] += ( + a["business_asset"] + + a["rent"] + + a["dividends"] + + a["interest"] + + a["trusts"] + ) + row["ofum_ss"] += a["ss"] + row["ofum_ssi"] += a["ssi"] + row["ofum_va_pension"] += a["va"] + row["ofum_retirement_annuities"] += ( + a["pension"] + a["annuity"] + a["other_retirement"] + a["ira"] + ) + hw_assets = [ + c for c in family_income.ASSET_INCOME_CONCEPTS if c != "ofum_asset" + ] + row["hw_taxable"] = sum( + row[c] for c in (*family_income.HW_EARNED_CONCEPTS, *hw_assets) + ) + row["hw_transfer"] = sum( + row[c] for c in family_income.HW_TRANSFER_COMPONENTS[wave] + ) + row["ofum_taxable"] = row["ofum_labor"] + row["ofum_asset"] + row["ofum_transfer"] = sum( + row[c] for c in family_income.OFUM_TRANSFER_COMPONENTS + ) + row["total_family_income"] = sum( + row[c] for c in family_income.FAMILY_INCOME_AGGREGATES + ) + size = len(members) + children = sum( + 1 for person in ofums if year - int(person["birth_year"]) < 18 + ) + head_age = year - int(head["birth_year"]) + row["interview"] = _interview(family, wave) + row["fu_size"] = size + row["n_children"] = children + row["head_age"] = head_age + row["head_sex"] = head["sex"] + row["wife_age"] = None if wife is None else year - int(wife["birth_year"]) + row["census_needs_standard"] = invented_needs_standard( + year, size, children, head_age + ) + if family["ss_imputed"] and row["head_ss"] > 0: + row["head_ss_acc"] = 1 + return row + + +def _income_frame(families: list[dict[str, Any]], wave: int) -> pd.DataFrame: + rows = [ + _income_row(family, wave) + for family in families + if _in_family(family, wave) + ] + table = family_income.income_variables(wave) + frame = pd.DataFrame(rows)[list(table)] + for concept in table: + if concept not in ("head_sex", "head_age", "wife_age"): + frame[concept] = frame[concept].astype("int64") + frame["head_sex"] = frame["head_sex"].astype("string") + frame["head_age"] = frame["head_age"].astype("Int64") + frame["wife_present"] = frame["wife_age"].notna().astype(bool) + frame["wife_age"] = pd.array( + [ + pd.NA if pd.isna(value) else int(value) + for value in frame["wife_age"] + ], + dtype="Int64", + ) + frame.insert(0, "income_year", wave - 1) + frame.insert(0, "wave", wave) + return frame.reset_index(drop=True) + + +def _wealth_concepts(wave: int) -> list[str]: + layout = 2009 if wave in family_income.WEALTH_SUPPLEMENT_WAVES else wave + return list(family_income.wealth_variables(layout)) + + +def _wealth_row(family: Mapping[str, Any], wave: int) -> dict[str, Any]: + growth = (1.0 + _WEALTH_GROWTH) ** (wave - 2005) + values = { + key: int(round(value * growth)) + for key, value in family["wealth"].items() + } + layout = 2009 if wave in family_income.WEALTH_SUPPLEMENT_WAVES else wave + row: dict[str, Any] = dict.fromkeys(_wealth_concepts(wave), 0) + for concept in ("checking_saving", "stocks", "vehicles", "other_assets"): + row[concept] = values[concept] + row["ira_annuity"] = values["ira_annuity"] + if layout == 2013: + row["farm_business_asset"] = values["farm_business"] + row["other_real_estate_asset"] = values["other_real_estate"] + else: + row["farm_business"] = values["farm_business"] + row["other_real_estate"] = values["other_real_estate"] + debt = values["unsecured_debt"] + if layout == 2011 or layout == 2013: + card = int(round(0.6 * debt)) + medical = int(round(0.2 * debt)) + row["credit_card_debt"] = card + row["medical_debt"] = medical + row["family_loan_debt"] = debt - card - medical + else: + row["other_debt"] = debt + assets = sum(row[c] for c in family_income.WEALTH1_ASSETS[layout]) + debts = sum(row[c] for c in family_income.WEALTH1_DEBTS[layout]) + row["wealth1"] = assets - debts + row["home_equity"] = values["home_equity"] + row["wealth2"] = row["wealth1"] + row["home_equity"] + row["wealth1_acc"] = 1 if family["wealth_imputed"] else 0 + row["interview"] = _interview(family, wave) + return row + + +def _wealth_frame(families: list[dict[str, Any]], wave: int) -> pd.DataFrame: + rows = [ + _wealth_row(family, wave) + for family in families + if _in_family(family, wave) + ] + frame = pd.DataFrame(rows)[["interview", *_wealth_concepts(wave)]] + frame = frame.astype("int64") + frame.insert(0, "wave", wave) + return frame.reset_index(drop=True) + + +def _earnings_frame(families: list[dict[str, Any]]) -> pd.DataFrame: + rows = [] + for family in families: + for person in family["persons"]: + if not person["earnings_rows"]: + continue + birth = int(person["birth_year"]) + for period in range(1990, 2003, 2): + rows.append( + { + "person_id": person["person_id"], + "period": period, + "earnings": float(20_000 + 500 * (period - 1990)), + "role": "head" if person["slot"] == 1 else "spouse", + "age": period - birth, + "weight": 1.0, + "earnings_acc": 0, + } + ) + frame = pd.DataFrame( + rows, + columns=[ + "person_id", + "period", + "earnings", + "role", + "age", + "weight", + "earnings_acc", + ], + ) + return frame.astype( + { + "person_id": "int64", + "period": "int64", + "earnings": "float64", + "age": "int64", + "weight": "float64", + "earnings_acc": "int64", + } + ) + + +def invented_age67_inputs( + *, + seed: int = DEFAULT_SEED, + supplement_waves_staged: bool = False, +) -> age67.Age67Inputs: + """INVENTED :class:`~populace_dynamics.cohorts.age67.Age67Inputs`. + + ``supplement_waves_staged`` adds invented wealth for waves 2005 and + 2007 (as if the PSID wealth supplements were staged, in the 2009 + component layout); without it those waves carry the reader's + refusal, as the real staging does today. + """ + + if not isinstance(seed, int) or isinstance(seed, bool): + raise ValueError("seed must be an integer") + rng = np.random.default_rng(seed) + families = _families(rng) + anchors = _anchor_frames(families) + persons = [p for family in families for p in family["persons"]] + design = pd.DataFrame( + { + "person_id": [p["person_id"] for p in persons], + "stratum": [ + family["stratum"] + for family in families + for _ in family["persons"] + ], + "cluster": [ + family["cluster"] + for family in families + for _ in family["persons"] + ], + } + ).astype("int64") + deaths = [] + for family in families: + for person in family["persons"]: + death = family["death_year"].get(person["slot"]) + deaths.append( + { + "person_id": person["person_id"], + "sex": person["sex"], + "death_status": ( + "not_deceased" if death is None else "exact" + ), + "death_year": pd.NA if death is None else int(death), + } + ) + death_frame = pd.DataFrame(deaths) + death_frame["death_year"] = death_frame["death_year"].astype("Int64") + marriages = [row for family in families for row in _marriage_rows(family)] + wealth_waves = ( + age67.WAVES if supplement_waves_staged else family_income.WEALTH_WAVES + ) + refusals = ( + {} + if supplement_waves_staged + else { + wave: ( + "WealthSupplementNotStagedError: INVENTED inputs mirror the " + f"real staging: the PSID {wave} wealth supplement is not " + "staged" + ) + for wave in family_income.WEALTH_SUPPLEMENT_WAVES + } + ) + inputs = age67.Age67Inputs( + anchors=anchors, + design=design, + death_records=death_frame, + marriage_history=_typed_marriage_frame(marriages), + observed_earnings=_earnings_frame(families), + family_income={ + wave: _income_frame(families, wave) for wave in age67.WAVES + }, + family_wealth={ + wave: _wealth_frame(families, wave) for wave in wealth_waves + }, + wealth_refusals=refusals, + ) + return dataclasses.replace( + inputs, + provenance={ + "kind": age67.INVENTED, + "data": INVENTED_INPUTS_LABEL, + "generator": _GENERATOR, + "seed": int(seed), + "supplement_waves_staged": bool(supplement_waves_staged), + "family_counts": dict(INVENTED_FAMILY_COUNTS), + "input_frames_sha256": age67.input_frames_sha256(inputs), + }, + ) + + +def invented_frames_sha256(*, seed: int, supplement_waves_staged: bool) -> str: + """The frame digest this generator produces for ``seed``.""" + + return age67.input_frames_sha256( + invented_age67_inputs( + seed=seed, supplement_waves_staged=supplement_waves_staged + ) + ) + + +def _recorded_invented(provenance: Mapping[str, Any]) -> tuple[int, bool]: + if provenance.get("kind") != age67.INVENTED: + raise ValueError( + f"the provenance kind is {provenance.get('kind')!r}, not " + "invented: only the invented generator's output may run under " + "the invented label" + ) + seed = provenance.get("seed") + staged = provenance.get("supplement_waves_staged") + if not isinstance(seed, int) or isinstance(seed, bool): + raise ValueError(f"an invented input records seed {seed!r}") + if not isinstance(staged, bool): + raise ValueError( + f"an invented input records supplement flag {staged!r}" + ) + if provenance.get("generator") != _GENERATOR: + raise ValueError( + f"generator {provenance.get('generator')!r} is not {_GENERATOR}" + ) + return seed, staged + + +def check_invented_inputs(inputs: age67.Age67Inputs) -> dict[str, Any]: + """Refuse inputs that are not this generator's output for their seed. + + Re-generates the frames from the recorded seed and supplement flag + and compares their digest with the inputs' own frames. + """ + + seed, staged = _recorded_invented(dict(inputs.provenance or {})) + expected = invented_frames_sha256( + seed=seed, supplement_waves_staged=staged + ) + observed = age67.input_frames_sha256(inputs) + if observed != expected: + raise ValueError( + "the inputs are labelled invented but their frames are not the " + f"invented generator's for seed {seed} (digest {observed} != " + f"{expected})" + ) + return { + "seed": seed, + "supplement_waves_staged": staged, + "input_frames_sha256": observed, + "regenerated": True, + } + + +def _observations_digest(frame: pd.DataFrame) -> str: + import hashlib + + digest = hashlib.sha256() + digest.update(frame.to_csv(index=False, lineterminator="\n").encode()) + return digest.hexdigest() + + +def check_invented_cohort(cohort: age67.Age67Cohort) -> dict[str, Any]: + """Refuse a cohort that is not the generator's cohort under its spec. + + The cohort's builder-recorded provenance must be ``invented``; the + generator's inputs for the recorded seed are rebuilt with the + cohort's spec and the observations compared. + """ + + provenance = dict(cohort.provenance or {}) + seed, staged = _recorded_invented(provenance) + inputs = invented_age67_inputs(seed=seed, supplement_waves_staged=staged) + if provenance.get("input_frames_sha256") != age67.input_frames_sha256( + inputs + ): + raise ValueError( + "the cohort's recorded input digest is not the invented " + f"generator's for seed {seed}" + ) + rebuilt = age67.build_age67_cohort(inputs, cohort.spec) + if _observations_digest(rebuilt.observations) != _observations_digest( + cohort.observations + ): + raise ValueError( + "the cohort's observations differ from the invented generator's " + f"cohort for seed {seed}" + ) + return {"seed": seed, "supplement_waves_staged": staged, "rebuilt": True} diff --git a/src/populace_dynamics/uniform_cut_track_u/rows.py b/src/populace_dynamics/uniform_cut_track_u/rows.py new file mode 100644 index 00000000..3adaa975 --- /dev/null +++ b/src/populace_dynamics/uniform_cut_track_u/rows.py @@ -0,0 +1,280 @@ +"""The registered Track U rows as builder and income-concept parameters. + +Each row of the exercise-2 specification +(``docs/design/boomers2004_uniform_cut_comparison.md``, sections 11 and +15) differs from the primary U0 in one field, except U6, which is defined +on U1 (plan ``critical-path-uniform-cut-20260923.md`` section 7). A row +here is a set of overrides of :class:`populace_dynamics.cohorts.age67. +Age67Spec` and :class:`populace_dynamics.estimates.adjusted_poverty. +AdjustedPovertySpec`; the defaults of those two classes are U0. + +:func:`check_rows_against_block` holds :data:`REGISTERED_ROWS` to the +specification's machine-readable block, so a row a run computes is the +row the specification registers. U7 (employer DC balances) is not +built: it needs a label investigation of the PSID P-section items. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Mapping +from dataclasses import dataclass, field, fields +from pathlib import Path +from typing import Any + +from populace_dynamics.cohorts import age67 +from populace_dynamics.estimates import adjusted_poverty as ap + +__all__ = [ + "PRIMARY_ROW", + "REGISTERED_ROWS", + "SPECIFICATION_PATH", + "TrackURow", + "check_rows_against_block", + "row_from_block", + "specification_block", +] + +_ROOT = Path(__file__).resolve().parents[3] +SPECIFICATION_PATH = ( + _ROOT / "docs" / "design" / "boomers2004_uniform_cut_comparison.md" +) +PRIMARY_ROW = "U0" +_AGE67_FIELDS = frozenset(f.name for f in fields(age67.Age67Spec)) +_INCOME_FIELDS = frozenset(f.name for f in fields(ap.AdjustedPovertySpec)) +#: Block keys that describe a row without being a parameter. +_METADATA_KEYS = frozenset({"status", "awaiting", "financial_assets"}) + + +@dataclass(frozen=True) +class TrackURow: + """One registered row: its overrides of the two specs, and status.""" + + row_id: str + field_changed: str + description: str + age67: Mapping[str, Any] = field(default_factory=dict) + income: Mapping[str, Any] = field(default_factory=dict) + built: bool = True + not_built_reason: str | None = None + awaiting: str | None = None + + def __post_init__(self) -> None: + unknown = (set(self.age67) - _AGE67_FIELDS) | ( + set(self.income) - _INCOME_FIELDS + ) + if unknown: + raise ValueError(f"{self.row_id}: unknown fields {unknown}") + if self.built: + self.age67_spec() + self.income_spec() + elif not self.not_built_reason: + raise ValueError(f"{self.row_id}: a row not built needs a reason") + + def age67_spec(self) -> age67.Age67Spec: + return age67.Age67Spec(**dict(self.age67)) + + def income_spec(self) -> ap.AdjustedPovertySpec: + return ap.AdjustedPovertySpec(**dict(self.income)) + + def as_dict(self) -> dict[str, Any]: + return { + "row": self.row_id, + "field": self.field_changed, + "description": self.description, + "age67_overrides": dict(self.age67), + "income_overrides": dict(self.income), + "built": self.built, + "not_built_reason": self.not_built_reason, + "awaiting": self.awaiting, + } + + +REGISTERED_ROWS: dict[str, TrackURow] = { + row.row_id: row + for row in ( + TrackURow( + "U0", + "-", + "primary: exact age (odd birth years 1937-1945 at 67), family " + "unit, reported asset income replaced by the annuity, Census " + "weighted-average 65+ thresholds, SSI offset for existing " + "recipients", + ), + TrackURow( + "U1", + "population", + "all ten birth years: even birth years at 66 and 68 with half " + "weight each, 1936 at 68 only", + age67={"row": "U1"}, + ), + TrackURow( + "U2", + "SSI", + "no SSI response", + income={"ssi_rule": "none"}, + ), + TrackURow( + "U3", + "SSI", + "full static SSI recomputation (upper bound)", + income={"ssi_rule": "full_static_recomputation"}, + ), + TrackURow( + "U4", + "income unit", + "head and wife income only", + income={"income_unit": "head_wife"}, + ), + TrackURow( + "U5", + "asset income", + "keep reported asset income and add the annuity", + income={"asset_income_rule": "keep"}, + ), + TrackURow( + "U6", + "cut (on U1)", + "U1 with the 1936 birth year uncut if the cut starts in 2004 " + "(the cut reaches an observation only when its member turns 67 " + "in or after 2004)", + age67={"row": "U1"}, + income={"cut_start_year": 2004}, + awaiting=( + "Max, plan section 10 decision 8 (the scenario's start " + "year; the scorecard's 'from 2004' has no readable source)" + ), + ), + TrackURow( + "U7", + "financial assets", + "WEALTH1 plus employer DC balances", + built=False, + not_built_reason=( + "needs a label investigation of the PSID P-section pension " + "account items (plan section 4); not built" + ), + ), + TrackURow( + "U8", + "threshold", + "PSID CENSUS NEEDS STANDARD", + income={"threshold_rule": "psid_census_needs_standard"}, + ), + TrackURow( + "U9", + "mortality", + "SSA period life table for 2004", + income={"mortality_basis": "ssa_period_2004"}, + ), + TrackURow( + "U10", + "threshold", + "Census size-by-related-children matrix (65+ rows for sizes 1 " + "and 2)", + income={"threshold_rule": "census_matrix_65plus"}, + ), + TrackURow( + "U-inst", + "universe", + "add persons in institutions (sequence 51-59) with the " + "family-of-record income rule", + age67={ + "presence": "in_family_or_institution", + "institution_income_rule": "family_of_record", + }, + awaiting=( + "the referee (the institution income rule is a builder " + "default)" + ), + ), + ) +} + + +def specification_block(path: Path = SPECIFICATION_PATH) -> dict[str, Any]: + """The specification's machine-readable JSON block (section 15).""" + + text = Path(path).read_text(encoding="utf-8") + blocks = re.findall(r"```json\n(.*?)\n```", text, re.DOTALL) + if len(blocks) != 1: + raise ValueError( + f"{path} must hold exactly one JSON block, found {len(blocks)}" + ) + return json.loads(blocks[0]) + + +def row_from_block(row_id: str, entry: Mapping[str, Any]) -> dict[str, Any]: + """A block row's overrides: ``age67``, ``income``, ``built``, ``awaiting``. + + ``population: all_ten_birth_years`` and ``on: U1`` select the U1 + population; keys naming an :class:`~populace_dynamics.cohorts.age67. + Age67Spec` or :class:`~populace_dynamics.estimates.adjusted_poverty. + AdjustedPovertySpec` field are overrides; ``status: not_built`` marks + a row not built. Any other key is refused. + """ + + age67_overrides: dict[str, Any] = {} + income: dict[str, Any] = {} + for key, value in entry.items(): + if key in ("population", "on"): + expected = "all_ten_birth_years" if key == "population" else "U1" + if value != expected: + raise ValueError(f"{row_id}: {key}={value!r} is not known") + age67_overrides["row"] = "U1" + elif key in _AGE67_FIELDS: + age67_overrides[key] = value + elif key in _INCOME_FIELDS: + income[key] = value + elif key not in _METADATA_KEYS: + raise ValueError(f"{row_id}: unknown block key {key!r}") + status = entry.get("status") + if status not in (None, "not_built"): + raise ValueError(f"{row_id}: unknown status {status!r}") + return { + "age67": age67_overrides, + "income": income, + "built": status != "not_built", + "awaiting": entry.get("awaiting"), + } + + +def check_rows_against_block( + block: Mapping[str, Any], + rows: Mapping[str, TrackURow] | None = None, +) -> dict[str, Any]: + """Refuse rows that differ from the specification block's rows. + + Every block row must be a row here with the same overrides, built + status and awaiting note, and no row here may be missing from the + block. + """ + + rows = REGISTERED_ROWS if rows is None else rows + registered = dict(block.get("rows") or {}) + if set(registered) != set(rows): + raise ValueError( + f"rows {sorted(rows)} differ from the block's " + f"{sorted(registered)}" + ) + for row_id, entry in registered.items(): + parsed = row_from_block(row_id, entry) + row = rows[row_id] + mine = { + "age67": dict(row.age67), + "income": dict(row.income), + "built": row.built, + "awaiting": row.awaiting is not None, + } + theirs = {**parsed, "awaiting": parsed["awaiting"] is not None} + if mine != theirs: + raise ValueError( + f"row {row_id} differs from the specification block: " + f"code {mine} != block {theirs}" + ) + return { + "specification_version": block.get("version"), + "rows_checked": sorted(registered), + "rows_equal_the_block": True, + } diff --git a/src/populace_dynamics/uniform_cut_track_u/runner.py b/src/populace_dynamics/uniform_cut_track_u/runner.py new file mode 100644 index 00000000..8f20266e --- /dev/null +++ b/src/populace_dynamics/uniform_cut_track_u/runner.py @@ -0,0 +1,522 @@ +"""The Track U row pipeline for DynaSim scorecard exercise 2. + +For each registered row (:data:`populace_dynamics.uniform_cut_track_u. +rows.REGISTERED_ROWS`): build the age-67 observations +(:func:`populace_dynamics.cohorts.age67.build_age67_cohort`), attach the +family income and wealth (:func:`~populace_dynamics.cohorts.age67. +income_rows`), form baseline and reform adjusted income and poverty +status (:func:`populace_dynamics.estimates.adjusted_poverty. +adjusted_incomes`) and tabulate the frozen statistic +(:func:`populace_dynamics.estimates.uniform_cut_tabulation. +tabulate_uniform_cut`). For the primary row it also computes the two F17 +diagnostics: the component summaries +(:mod:`populace_dynamics.uniform_cut_track_u.diagnostics`) and the +official-concept poverty rate (:func:`official_concept_rates`: money +income against the same thresholds, reported asset income kept, no +annuity, no cut). + +Provenance guards, before anything is computed: + +* ``data_provenance="invented"``: the inputs must be the invented + generator's output for their recorded seed (re-generated and compared, + :func:`populace_dynamics.uniform_cut_track_u.invented. + check_invented_inputs`), and the built cohort must record the + ``invented`` kind. +* ``data_provenance="registered_real"``: the registration pointer must be + an issue #42 comment URL; the cohort must record ``psid_files`` (inputs + returned and sealed by :func:`~populace_dynamics.cohorts.age67. + load_age67_inputs`); the thresholds must be the committed Census + capture, the SSI parameters and life tables the committed ones; no + wealth wave may be refused and no observation left out. + +The income concept and the tabulation apply their own guards as well. +""" + +from __future__ import annotations + +import re +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Any + +import numpy as np +import pandas as pd + +from populace_dynamics.cohorts import age67 +from populace_dynamics.estimates import adjusted_poverty as ap +from populace_dynamics.estimates import uniform_cut_tabulation as ut +from populace_dynamics.uniform_cut_track_u import diagnostics, invented +from populace_dynamics.uniform_cut_track_u.rows import ( + PRIMARY_ROW, + REGISTERED_ROWS, + TrackURow, +) + +__all__ = [ + "NAMED_DELTAS", + "OFFICIAL_CONCEPT_CELLS", + "REGISTRATION_POINTER", + "RUN_SCHEMA_VERSION", + "TrackUParameters", + "TrackURunError", + "committed_parameters", + "official_concept_rates", + "run_track_u", +] + +RUN_SCHEMA_VERSION = "populace_dynamics.track_u_run.v1" +#: A comment on issue #42, the registration issue; nothing else is a +#: registration pointer (the Track A convention). +REGISTRATION_POINTER = re.compile( + r"https://github\.com/PolicyEngine/microcosm-dynamics/issues/42" + r"#issuecomment-[0-9]+" +) +OFFICIAL_CONCEPT_CELLS: tuple[str, ...] = ut.DEFAULT_CELLS +#: The named deltas every output carries (specification section 12). +NAMED_DELTAS: tuple[str, ...] = ( + "PSID versus SIPP wealth measurement", + "realized 2004-2012 history versus DYNASIM's 1992-based projection, " + "including the 2008-09 asset shock for the 1941-45 cohorts at 67", + "realized COLAs versus 2002 Trustees assumptions", + "immigrant under-coverage; institutionalized persons (row U-inst: " + "the family-of-record rule assigns the family's income and threshold " + "without the member's own income, which the PSID does not collect); " + "attrition", + "OFUM-owned assets inside family wealth", + "imputed rent excluded", + "self-reported Social Security, possibly net of Medicare premiums", + "pension income that may include account withdrawals", + "exact-age sampling of alternate birth years (mean birth year 1941 " + "against 1940.5)", + "farm income's asset portion is not separable and stays in income; " + "business income is split 50/50 between labor and asset parts by PSID " + "convention for working owners", + "SSI units approximated from head, wife and OFUM totals (deeming by " + "full attribution; OFUMs as one unit)", + "reported SSI may include state supplementary payments, so capping " + "the offset at the federal benefit rate less reported SSI can " + "understate the offset", +) + + +class TrackURunError(ValueError): + """The inputs, parameters or provenance cannot run under the label.""" + + +@dataclass(frozen=True) +class TrackUParameters: + """The parameters a run reads: thresholds, SSI and life tables.""" + + thresholds: ap.PovertyThresholds | None + ssi: ap.SsiParameters + life_tables: Mapping[str, ap.LifeTable] + + def provenance(self) -> dict[str, Any]: + return { + "thresholds": ( + None + if self.thresholds is None + else dict(self.thresholds.provenance) + ), + "ssi": dict(self.ssi.provenance), + "life_tables": { + name: {"name": table.name, **dict(table.source)} + for name, table in sorted(self.life_tables.items()) + }, + } + + +def committed_parameters( + thresholds: ap.PovertyThresholds | None, +) -> TrackUParameters: + """The committed SSI capture and life tables, with ``thresholds``. + + The dry run passes the invented threshold table (the Census capture + does not exist yet, cos decision d194); the registered run passes + :func:`populace_dynamics.estimates.adjusted_poverty. + load_poverty_thresholds`, which refuses until the capture is pinned. + """ + + return TrackUParameters( + thresholds=thresholds, + ssi=ap.load_ssi_parameters(), + life_tables={ + "nchs_2000": ap.load_nchs_2000_life_table(), + "ssa_period_2004": ap.load_ssa_period_2004_life_table(), + }, + ) + + +def _check_parameters(params: TrackUParameters, data_provenance: str) -> None: + for basis, table in params.life_tables.items(): + invented_table = ( + data_provenance == ap.INVENTED + and table.name == ap.INVENTED_LIFE_TABLE + ) + if table.name != basis and not invented_table: + raise TrackURunError( + f"life table {table.name!r} is filed under {basis!r}" + ) + if data_provenance != ap.REGISTERED_REAL: + return + from populace_dynamics.data import tr2008 + + thresholds = ( + {} if params.thresholds is None else dict(params.thresholds.provenance) + ) + if thresholds.get("kind") != "census_capture" or ( + thresholds.get("sha256") != ap.THRESHOLDS_SHA256 + ): + raise TrackURunError( + "a registered run needs the committed, pinned Census threshold " + f"capture; the thresholds record {thresholds!r}" + ) + if params.ssi.provenance.get("kind") != "policyengine_us_capture" or ( + params.ssi.provenance.get("sha256") != ap.SSI_PARAMETERS_SHA256 + ): + raise TrackURunError( + "a registered run needs the committed SSI capture; the SSI " + f"parameters record {dict(params.ssi.provenance)!r}" + ) + pins = { + "nchs_2000": ap.NCHS_2000_SHA256, + "ssa_period_2004": tr2008.FILE_SHA256["ssa_2008_vintage.json"], + } + if set(params.life_tables) != set(pins) or any( + params.life_tables[basis].source.get("sha256") != pin + for basis, pin in pins.items() + ): + raise TrackURunError( + "a registered run needs the committed life tables " + f"{sorted(pins)} with their pinned SHA-256" + ) + + +def _check_inputs( + inputs: age67.Age67Inputs, + data_provenance: str, + registration_pointer: str | None, + allow_blocked: bool, +) -> dict[str, Any]: + if data_provenance == ap.INVENTED: + if registration_pointer is not None: + raise TrackURunError( + "an invented run carries no registration pointer" + ) + return {"invented_inputs": invented.check_invented_inputs(inputs)} + if data_provenance != ap.REGISTERED_REAL: + raise TrackURunError( + f"data_provenance must be one of {ap.DATA_PROVENANCES}" + ) + if not isinstance(registration_pointer, str) or not ( + REGISTRATION_POINTER.fullmatch(registration_pointer) + ): + raise TrackURunError( + "a registered run needs an issue #42 comment URL as its " + "registration pointer (https://github.com/PolicyEngine/" + "microcosm-dynamics/issues/42#issuecomment-)" + ) + if (inputs.provenance or {}).get("kind") != age67.PSID_FILES: + raise TrackURunError( + "a registered run reads the staged PSID through " + "age67.load_age67_inputs; these inputs record " + f"{(inputs.provenance or {}).get('kind')!r}" + ) + if inputs.wealth_refusals: + raise TrackURunError( + "a registered run needs WEALTH1 for every wave; refused: " + f"{sorted(inputs.wealth_refusals)}" + ) + if allow_blocked: + raise TrackURunError( + "a registered run may not leave blocked observations out" + ) + return {} + + +def _counts(series: pd.Series) -> dict[str, int]: + return { + str(key): int(value) + for key, value in series.value_counts(dropna=False) + .sort_index() + .items() + } + + +def _cell_masks(members: pd.DataFrame) -> dict[str, np.ndarray]: + male = members["sex"].astype(str).eq("male").to_numpy() + married = members["married"].astype(bool).to_numpy() + return { + "all": np.ones(len(members), dtype=bool), + "men": male, + "women": ~male, + "married": married, + "non_married": ~married, + } + + +def official_concept_rates( + members: pd.DataFrame, adjusted: pd.DataFrame +) -> dict[str, Any]: + """F17: the official-concept poverty rate for the same sample. + + Poor when the unit's money income (``money_income``: TOTAL FAMILY + INCOME under the family-unit basis, reported asset income kept, no + annuity, no cut) is below the row's threshold (the same threshold as + the adjusted concept, so the 65-and-over rule, not the Census + householder-age convention). Weighted rate per cell of + :data:`OFFICIAL_CONCEPT_CELLS`, in percent; an empty or zero-weight + cell is undefined. Not scored. + """ + + joined = members[["observation_id", "weight", "sex", "married"]].merge( + adjusted[["observation_id", "money_income", "threshold"]], + on="observation_id", + how="inner", + validate="1:1", + ) + if len(joined) != len(members) or len(joined) != len(adjusted): + raise TrackURunError("members and adjusted incomes do not match") + poor = (joined["money_income"] < joined["threshold"]).to_numpy() + weight = joined["weight"].to_numpy(dtype=np.float64) + cells = {} + for name, mask in _cell_masks(joined).items(): + total = float(weight[mask].sum()) + if not mask.any() or total <= 0: + cells[name] = { + "defined": False, + "undefined_reason": ( + "empty cell" if not mask.any() else "zero total weight" + ), + "n_observations": int(mask.sum()), + } + continue + cells[name] = { + "defined": True, + "rate": 100.0 * float(weight[mask & poor].sum()) / total, + "n_observations": int(mask.sum()), + "weight_total": total, + } + return { + "definition": ( + "100 * sum w 1{money income < threshold} / sum w; money income " + "is TOTAL FAMILY INCOME (family-unit basis) with reported asset " + "income kept, no annuity and no cut; the threshold is the " + "row's (not scored; plan field F17)" + ), + "cells": cells, + } + + +def _income_counts(adjusted: pd.DataFrame) -> dict[str, Any]: + entering = adjusted["poor_reform"] & ~adjusted["poor_baseline"] + leaving = adjusted["poor_baseline"] & ~adjusted["poor_reform"] + return { + "n_observations": int(len(adjusted)), + "n_cut_applies": int(adjusted["cut_applies"].sum()), + "n_social_security_positive": int( + (adjusted["social_security"] > 0).sum() + ), + "n_annuity_positive": int((adjusted["annuity"] > 0).sum()), + "n_financial_assets_negative": int( + (adjusted["financial_assets"] < 0).sum() + ), + "n_asset_income_removed_nonzero": int( + (adjusted["asset_income_removed"] != 0).sum() + ), + "n_ssi_offset_positive": int((adjusted["ssi_offset"] > 0).sum()), + "n_ssi_new_positive": int((adjusted["ssi_new"] > 0).sum()), + "n_poor_baseline": int(adjusted["poor_baseline"].sum()), + "n_poor_reform": int(adjusted["poor_reform"].sum()), + "n_entering_poverty": int(entering.sum()), + "n_leaving_poverty": int(leaving.sum()), + "income_basis": _counts(adjusted["income_basis"]), + "threshold_cells": _counts(adjusted["threshold_cell"]), + "annuity_basis_kind": _counts( + adjusted["annuity_basis"].str.split(":").str[0] + ), + } + + +def _pending(row: TrackURow) -> list[dict[str, Any]]: + items = [item.as_dict() for item in age67.pending_decisions()] + items += [item.as_dict() for item in ap.pending_decisions()] + if row.awaiting: + items.append( + { + "field": f"row {row.row_id}", + "default": None, + "alternatives": [], + "default_basis": row.description, + "awaiting": row.awaiting, + } + ) + return items + + +def _compute_row( + row: TrackURow, + inputs: age67.Age67Inputs, + params: TrackUParameters, + *, + data_provenance: str, + registration_pointer: str | None, + allow_blocked: bool, + tabulation_config: ut.TabulationConfig | None, +) -> tuple[dict[str, Any], pd.DataFrame, pd.DataFrame, age67.Age67Cohort]: + age67_spec = row.age67_spec() + income_spec = row.income_spec() + cohort = age67.build_age67_cohort(inputs, age67_spec) + kind = cohort.provenance.get("kind") + expected = ( + age67.INVENTED if data_provenance == ap.INVENTED else age67.PSID_FILES + ) + if kind != expected: + raise TrackURunError( + f"row {row.row_id}: the cohort records {kind!r}, not {expected!r}" + ) + members = age67.income_rows(cohort, inputs, allow_blocked=allow_blocked) + if data_provenance == ap.REGISTERED_REAL and any( + members.attrs.get("left_out", {}).values() + ): + raise TrackURunError(f"row {row.row_id}: observations were left out") + adjusted = ap.adjusted_incomes( + members, + income_spec, + data_provenance=data_provenance, + life_table=params.life_tables[income_spec.mortality_basis], + thresholds=params.thresholds, + ssi=params.ssi, + registration_pointer=registration_pointer, + ) + joined = ut.tabulation_rows(members, adjusted) + pending = _pending(row) + tabulation = ut.tabulate_uniform_cut( + joined, + data_provenance=data_provenance, + config=tabulation_config, + registration_pointer=registration_pointer, + upstream_spec=adjusted.attrs["spec"], + pending_decisions=pending, + ) + obs = cohort.observations + result = { + "row": row.as_dict(), + "status": "computed", + "age67_spec": age67_spec.as_dict(), + "income_spec": income_spec.as_dict(), + "life_table": adjusted.attrs["life_table"], + "population": { + "n_observations_built": int(len(obs)), + "n_observations_tabulated": int(len(members)), + "n_persons": int(members["person_id"].nunique()), + "n_family_units": int(members["family_unit_id"].nunique()), + "n_in_institution": int(members["in_institution"].sum()), + "left_out": dict(members.attrs.get("left_out", {})), + "observations_by_birth_year": _counts(members["birth_year"]), + "dispositions": ( + _counts(cohort.dispositions["disposition"]) + if not cohort.dispositions.empty + else {} + ), + }, + "income_concept_counts": _income_counts(adjusted), + "tabulation": tabulation, + } + return result, members, adjusted, cohort + + +def run_track_u( + inputs: age67.Age67Inputs, + params: TrackUParameters, + *, + data_provenance: str, + registration_pointer: str | None = None, + rows: Mapping[str, TrackURow] | None = None, + allow_blocked: bool = False, + tabulation_config: ut.TabulationConfig | None = None, + progress: Callable[[str], None] | None = None, +) -> dict[str, Any]: + """Every registered row, the F17 diagnostics and the provenance. + + Rows not built (U7) are reported with their reason. An undefined + cell stays undefined with its reason (the tabulation's rule); nothing + is imputed or dropped silently. + """ + + rows = REGISTERED_ROWS if rows is None else rows + if PRIMARY_ROW not in rows: + raise TrackURunError("the primary row U0 must run") + checks = _check_inputs( + inputs, data_provenance, registration_pointer, allow_blocked + ) + _check_parameters(params, data_provenance) + say = progress or (lambda message: None) + results: dict[str, Any] = {} + primary: tuple[pd.DataFrame, pd.DataFrame, age67.Age67Cohort] | None = None + for row_id, row in rows.items(): + if not row.built: + results[row_id] = { + "row": row.as_dict(), + "status": "not_built", + "reason": row.not_built_reason, + "tabulation": None, + } + continue + say(f"Track U row {row_id}") + result, members, adjusted, cohort = _compute_row( + row, + inputs, + params, + data_provenance=data_provenance, + registration_pointer=registration_pointer, + allow_blocked=allow_blocked, + tabulation_config=tabulation_config, + ) + results[row_id] = result + if row_id == PRIMARY_ROW: + primary = (members, adjusted, cohort) + assert primary is not None + members, adjusted, cohort = primary + if data_provenance == ap.INVENTED: + checks["invented_cohort"] = invented.check_invented_cohort(cohort) + labels = list(ap.OUTPUT_LABELS) + if data_provenance == ap.INVENTED: + labels.insert(0, ut.INVENTED_DATA_LABEL) + return { + "schema_version": RUN_SCHEMA_VERSION, + "statistic_id": ut.STATISTIC_ID, + "data_provenance": data_provenance, + "registration_pointer": registration_pointer, + "labels": labels, + "cohort_provenance": { + key: value + for key, value in cohort.provenance.items() + if key != "psid_files_sha256" + }, + "psid_files_sha256": dict( + cohort.provenance.get("psid_files_sha256") or {} + ), + "parameters": params.provenance(), + "input_checks": checks, + "rows": results, + "f17_diagnostics": { + "row": PRIMARY_ROW, + "official_concept_poverty_rate": official_concept_rates( + members, adjusted + ), + "components": diagnostics.component_diagnostics(cohort, inputs), + }, + "named_deltas": list(NAMED_DELTAS), + "pending_decisions": { + "age67": [item.as_dict() for item in age67.pending_decisions()], + "income_concept": [ + item.as_dict() for item in ap.pending_decisions() + ], + "rows": { + row_id: row.awaiting + for row_id, row in rows.items() + if row.awaiting + }, + }, + } diff --git a/tests/README-tiers.md b/tests/README-tiers.md index 18e72bbe..5ef7342c 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -38,9 +38,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 2,993 | -| `artifact` | 2,801 | -| `integration_psid` | 862 | +| `unit` | 3,027 | +| `artifact` | 2,841 | +| `integration_psid` | 863 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 163 | -| **Total** | **7,339** | +| **Total** | **7,414** | diff --git a/tests/cohorts/test_age67.py b/tests/cohorts/test_age67.py index 808e129d..f4f045c4 100644 --- a/tests/cohorts/test_age67.py +++ b/tests/cohorts/test_age67.py @@ -287,12 +287,111 @@ def test_u1_adds_even_birth_years_at_half_weight(): assert obs.loc["5001:2005", "weight_multiplier"] == 1.0 -def test_institution_option_marks_the_missing_income_rule(): - cohort = age67.build_age67_cohort( - _inputs(), age67.Age67Spec(presence="in_family_or_institution") +def _with_institution_member() -> age67.Age67Inputs: + """Invented: person 9001 (no marriage history, female) is in an + institution in 2009 (sequence 52), attached to family 11.""" + + anchors = _anchors() + for wave, frame in anchors.items(): + row = ( + (9001, 11, 52, 30, 67, 70.0) + if wave == 2009 + else (9001, 0, 0, 0, 0, 0.0) + ) + extra = pd.DataFrame( + [row], + columns=[ + "person_id", + "interview", + "sequence", + "relationship", + "age", + "weight", + ], + ) + extra["reported_birth_year"] = pd.array([pd.NA], dtype="Int64") + anchors[wave] = pd.concat([frame, extra], ignore_index=True) + base = _inputs() + return _inputs( + anchors=anchors, + design=pd.concat( + [ + base.design, + pd.DataFrame( + {"person_id": [9001], "stratum": [1], "cluster": [2]} + ), + ], + ignore_index=True, + ), + death_records=pd.concat( + [ + base.death_records, + pd.DataFrame({"person_id": [9001], "sex": ["female"]}), + ], + ignore_index=True, + ), ) + + +def test_institution_rule_family_of_record_is_the_default(): + """U-inst: the institutionalized member takes its family of record. + + Invented 9001 is in an institution in 2009 attached to family 11 + (person 1001's family): the observation is an OFUM of that family, + with no co-resident spouse, and merges family 11's income and wealth. + """ + + assert age67.Age67Spec().institution_income_rule == "family_of_record" + inputs = _with_institution_member() + spec = age67.Age67Spec(presence="in_family_or_institution") + cohort = age67.build_age67_cohort(inputs, spec) obs = cohort.observations.set_index("observation_id") - assert obs.loc["7001:2007", "income_status"] == "income_rule_missing" + member = obs.loc["9001:2009"] + assert member["in_institution"] + assert member["income_status"] == "family_of_record" + assert member["member_role"] == "ofum" + assert member["relationship"] == 30 + assert not member["member_married_coresident"] + assert member["family_unit_id"] == 2009 * 100_000 + 11 + assert member["birth_year"] == 1941 + assert not obs.loc["1001:2009", "in_institution"] + rows = age67.income_rows(cohort, inputs, allow_blocked=True) + merged = rows.set_index("observation_id") + # family 11 is the first 2009 family record: income 1,000, wealth 500 + assert merged.loc["9001:2009", "total_family_income"] == 1000 + assert merged.loc["9001:2009", "wealth1"] == 500 + assert merged.loc["9001:2009", "fu_size"] == 1 + # the default population (in family only) never sees the member: it + # is in no wave's in-family universe + default = age67.build_age67_cohort(inputs) + assert "9001:2009" not in set(default.observations["observation_id"]) + assert (9001, 2009) not in _dispositions(default) + + +def test_institution_rule_excluded_and_missing_family_of_record(): + inputs = _with_institution_member() + excluded = age67.build_age67_cohort( + inputs, + age67.Age67Spec( + presence="in_family_or_institution", + institution_income_rule="excluded", + ), + ) + assert "9001:2009" not in set(excluded.observations["observation_id"]) + assert _dispositions(excluded)[(9001, 2009)] == ( + "institution_excluded_by_rule" + ) + # 7001 is in an institution in 2007 attached to interview 7, which has + # no 2007 family-file record in the invented frames + cohort = age67.build_age67_cohort( + inputs, age67.Age67Spec(presence="in_family_or_institution") + ) + assert "7001:2007" not in set(cohort.observations["observation_id"]) + assert _dispositions(cohort)[(7001, 2007)] == ( + "institution_family_of_record_missing" + ) + with pytest.raises(ValueError, match="institution_income_rule"): + age67.Age67Spec(institution_income_rule="own_income") def test_income_rows_refuse_blocked_observations_unless_allowed(): diff --git a/tests/estimates/test_adjusted_poverty.py b/tests/estimates/test_adjusted_poverty.py index 8f446abd..b529fd6f 100644 --- a/tests/estimates/test_adjusted_poverty.py +++ b/tests/estimates/test_adjusted_poverty.py @@ -412,6 +412,63 @@ def test_ssi_offset_for_existing_recipients(): assert (none["ssi_new"] == 0).all() +def test_cut_start_year_leaves_earlier_age67_years_uncut(): + """Row U6: with a 2004 start, the 1936 birth year (67 in 2003) is uncut. + + INVENTED rows, threshold T = 1,000 (one person, 65+ row): + + * "1936": B = 1,050 (not poor). Primary: cut 0.13 * 1,050 = 136.5, + R = 913.5 (poor). U6: 1936 + 67 = 2003 < 2004, no cut, R = 1,050. + * "1937": 1937 + 67 = 2004 >= 2004, cut under both: R = 913.5. + * "ssi": birth year 1936, SSI recipient with SS 1,000, SSI 100: + primary offset (1,000 - 240) - (870 - 240) = 130 (cap 500); U6 has + no cut, so no fall in countable income and no offset. + """ + + rows = [ + _row("1936", birth_year=1936, head_ss=1050, total_family_income=1050), + _row("1937", birth_year=1937, head_ss=1050, total_family_income=1050), + _row( + "ssi", + birth_year=1936, + head_ss=1000, + head_ssi=100, + hw_transfer=100, + total_family_income=1100, + ), + ] + primary = _run(*rows) + assert primary["cut_applies"].all() + assert primary.loc["1936", "cut"] == pytest.approx(136.5) + assert primary.loc["1936", "reform_income"] == pytest.approx(913.5) + assert primary.loc["1936", "poor_reform"] + assert primary.loc["ssi", "ssi_offset"] == pytest.approx(130.0) + u6 = _run(*rows, cut_start_year=2004) + assert u6["cut_applies"].to_dict() == { + "1936": False, + "1937": True, + "ssi": False, + } + assert u6.loc["1936", "cut"] == 0.0 + assert u6.loc["1936", "reform_income"] == pytest.approx(1050.0) + assert not u6.loc["1936", "poor_reform"] + assert u6.loc["1937", "reform_income"] == pytest.approx(913.5) + assert u6.loc["1937", "poor_reform"] + assert u6.loc["ssi", "ssi_offset"] == 0.0 + assert u6.loc["ssi", "reform_income"] == pytest.approx( + u6.loc["ssi", "baseline_income"] + ) + # full static recomputation: an uncut unit cannot become newly eligible + u3 = _run(*rows, cut_start_year=2004, ssi_rule="full_static_recomputation") + assert (u3.loc[["1936", "ssi"], "ssi_new"] == 0).all() + + +@pytest.mark.parametrize("value", [True, "2004", 1989, 2031, 2004.0]) +def test_cut_start_year_must_be_a_year_or_none(value): + with pytest.raises(ap.AdjustedPovertyError, match="cut_start_year"): + ap.AdjustedPovertySpec(cut_start_year=value) + + def test_full_static_recomputation_enrols_newly_eligible_units(): rows = [ # before 900-240 = 660 >= 600; after 783-240 = 543 < 600: SSI 57 diff --git a/tests/estimates/test_birth_evidence_artifact.py b/tests/estimates/test_birth_evidence_artifact.py index cd8724a4..8b64c40d 100644 --- a/tests/estimates/test_birth_evidence_artifact.py +++ b/tests/estimates/test_birth_evidence_artifact.py @@ -130,6 +130,11 @@ def test_post_review_sources_are_outside_historical_reducer_identity(): Path("src/populace_dynamics/cohorts/age67.py"), Path("src/populace_dynamics/estimates/adjusted_poverty.py"), Path("src/populace_dynamics/estimates/uniform_cut_tabulation.py"), + Path("src/populace_dynamics/uniform_cut_track_u/__init__.py"), + Path("src/populace_dynamics/uniform_cut_track_u/diagnostics.py"), + Path("src/populace_dynamics/uniform_cut_track_u/invented.py"), + Path("src/populace_dynamics/uniform_cut_track_u/rows.py"), + Path("src/populace_dynamics/uniform_cut_track_u/runner.py"), ) assert reducer.POST_REVIEW_SHARED_SOURCE_BLOBS == { Path( @@ -400,6 +405,11 @@ def test_post_review_exclusions_are_unreachable_from_birth_evidence(): "populace_dynamics.cohorts.age67", "populace_dynamics.estimates.adjusted_poverty", "populace_dynamics.estimates.uniform_cut_tabulation", + "populace_dynamics.uniform_cut_track_u", + "populace_dynamics.uniform_cut_track_u.diagnostics", + "populace_dynamics.uniform_cut_track_u.invented", + "populace_dynamics.uniform_cut_track_u.rows", + "populace_dynamics.uniform_cut_track_u.runner", } assert track_u_modules.issubset(module_paths) assert track_u_modules.isdisjoint(reachable), ( diff --git a/tests/test_boomers2004_uniform_cut_spec.py b/tests/test_boomers2004_uniform_cut_spec.py index b79dff62..ae6adefb 100644 --- a/tests/test_boomers2004_uniform_cut_spec.py +++ b/tests/test_boomers2004_uniform_cut_spec.py @@ -70,6 +70,12 @@ def test_population_matches_the_builder(block): assert population["unresolved_marital_status"] == ( spec.unresolved_marital_status ) + assert population["institution_income_rule"] == ( + spec.institution_income_rule + ) + assert population["institution_income_rule"] in ( + age67.INSTITUTION_INCOME_RULES + ) assert population["u1_single_observation_weight"] == ( spec.u1_single_observation_weight ) @@ -115,6 +121,7 @@ def test_income_concept_matches_the_spec_defaults(block): assert block["threshold"]["capture_status"] == "not_captured" assert ap.THRESHOLDS_SHA256 is None assert block["cut"]["rate"] == spec.cut_rate == block["target"]["cut_rate"] + assert block["cut"]["start_year"] == spec.cut_start_year is None ssi = block["ssi"] assert ssi["rule"] == spec.ssi_rule assert ssi["deeming"] == spec.ssi_deeming @@ -139,7 +146,7 @@ def test_rows_name_real_alternatives(block): "U-inst", } for name, row in rows.items(): - if row.get("status") in ("not_built", "counted_only_no_income_rule"): + if row.get("status") == "not_built": continue overrides = { key: value @@ -150,7 +157,17 @@ def test_rows_name_real_alternatives(block): if "population" in row: assert row["population"] == "all_ten_birth_years", name assert rows["U-inst"]["presence"] == "in_family_or_institution" - age67.Age67Spec(presence=rows["U-inst"]["presence"]) + age67.Age67Spec( + presence=rows["U-inst"]["presence"], + institution_income_rule=rows["U-inst"]["institution_income_rule"], + ) + assert rows["U6"]["on"] == "U1" + ap.AdjustedPovertySpec(cut_start_year=rows["U6"]["cut_start_year"]) + # every row but U7 is built, and the code's rows equal the block's + assert [name for name, row in rows.items() if "status" in row] == ["U7"] + from populace_dynamics.uniform_cut_track_u import rows as track_u_rows + + assert track_u_rows.check_rows_against_block(block)["rows_equal_the_block"] def test_cells_and_uncertainty_match_the_tabulation(block): @@ -170,6 +187,9 @@ def test_cells_and_uncertainty_match_the_tabulation(block): def test_pending_decisions_are_listed_in_the_text(text): section = text.split("## 16. Pending decisions")[1].split("## 17.")[0] assert "d189" in section + assert "decision 8" in section and "cut_start_year" in section + assert "institution_income_rule = family_of_record" in section + assert "d194" in section ssi = next( item for item in ap.pending_decisions() if item.field == "ssi_rule" ) diff --git a/tests/tier_counts.json b/tests/tier_counts.json index f04c88f4..18d659c1 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,9 +1,9 @@ { "schema_version": 1, "counts": { - "unit": 2993, - "artifact": 2801, - "integration_psid": 862, + "unit": 3027, + "artifact": 2841, + "integration_psid": 863, "reproduction_legacy": 520, "oracle_policyengine": 163 } diff --git a/tests/track_u/test_component_diagnostics_integration.py b/tests/track_u/test_component_diagnostics_integration.py new file mode 100644 index 00000000..d766ae96 --- /dev/null +++ b/tests/track_u/test_component_diagnostics_integration.py @@ -0,0 +1,72 @@ +"""F17 component diagnostics on the staged PSID (no poverty statistic). + +Skipped when the staged PSID products under ``~/PolicyEngine/psid-data`` +are absent. Runs ``scripts/track_u_component_diagnostics.py`` in a fresh +interpreter (the script refuses to run once the income concept is +imported, which other tests in this process do) and checks its structure: +the population, that no income-concept module was loaded, and the +``# IN FU`` record counts the institution rule relies on. It asserts no +Social Security, SSI or WEALTH1 value. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from populace_dynamics.cohorts import age67 + +ROOT = Path(__file__).resolve().parents[2] +REAL_DATA = Path("~/PolicyEngine/psid-data").expanduser() + +needs_real_psid = pytest.mark.skipif( + not (REAL_DATA / "ind2023er").is_dir() + or not all( + (REAL_DATA / "family" / str(wave)).is_dir() for wave in age67.WAVES + ), + reason="staged PSID individual and family files not present", +) + + +@needs_real_psid +def test_component_diagnostics_on_the_staged_psid(tmp_path): + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "track_u_component_diagnostics.py"), + "--output-dir", + str(tmp_path), + "--data-dir", + str(REAL_DATA), + ], + check=True, + capture_output=True, + text=True, + ) + result = json.loads( + (tmp_path / "track-u-component-diagnostics.json").read_text() + ) + assert result["forbidden_modules_loaded"] == [] + assert result["provenance"]["kind"] == age67.PSID_FILES + diag = result["diagnostics"] + # structure found on the staged files 2026-09-24 (counts only) + assert diag["population"]["n_observations"] == 483 + assert diag["population"]["income_years"] == [2004, 2006, 2008, 2010, 2012] + assert set(diag["wealth1"]) == {"2008", "2010", "2012"} + assert set(result["wealth_refusals"]) == {"2005", "2007"} + for counts in diag["institution_record_counts"].values(): + assert counts["n_fu_size_equals_in_family_records"] == ( + counts["n_families"] + ) + assert ( + counts["of_which_fu_size_equals_in_family_plus_institution"] == 0 + ) + assert counts["n_families_with_institution_records"] > 0 + assert "not_compared" in diag["published_sources"]["wealth1"] + text = (tmp_path / "SUMMARY.md").read_text() + for forbidden in ("poverty rate:", "threshold:", "annuity:"): + assert forbidden not in text diff --git a/tests/track_u/test_diagnostics.py b/tests/track_u/test_diagnostics.py new file mode 100644 index 00000000..a0360a0a --- /dev/null +++ b/tests/track_u/test_diagnostics.py @@ -0,0 +1,199 @@ +"""Hand-computed cases for the F17 component diagnostics. + +Every frame, rate and published figure here is INVENTED (the arithmetic is +written next to each assertion); none is a PSID value, an SSA statistic or +a comparator value. The published-source readers are tested against the +committed files in ``test_diagnostics_published.py``. +""" + +from __future__ import annotations + +import subprocess +import sys + +import numpy as np +import pandas as pd +import pytest + +from populace_dynamics.cohorts import age67 +from populace_dynamics.uniform_cut_track_u import diagnostics as dg +from populace_dynamics.uniform_cut_track_u import invented + + +def test_weighted_quantile_by_hand(): + # equal weights: cumulative shares .25 .5 .75 1 + assert dg.weighted_quantile([40, 10, 30, 20], [1, 1, 1, 1], 0.5) == 20 + assert dg.weighted_quantile([10, 20, 30, 40], [1, 1, 1, 1], 0.25) == 10 + assert dg.weighted_quantile([10, 20, 30, 40], [1, 1, 1, 1], 0.9) == 40 + # weights 1, 1, 1, 5: cumulative 1, 2, 3, 8; half of 8 is 4 -> 40 + assert dg.weighted_quantile([10, 20, 30, 40], [1, 1, 1, 5], 0.5) == 40 + # a zero weight never decides: cumulative 0, 1, 2 -> the median is 10 + assert dg.weighted_quantile([5, 10, 20], [0, 1, 1], 0.5) == 10 + with pytest.raises(ValueError): + dg.weighted_quantile([], [], 0.5) + with pytest.raises(ValueError): + dg.weighted_quantile([1], [1], 0.0) + with pytest.raises(ValueError): + dg.weighted_quantile([1, 2], [0, 0], 0.5) + + +def _rows(**columns) -> pd.DataFrame: + frame = pd.DataFrame(columns) + n = len(frame) + defaults = { + "observation_id": [f"o{i}" for i in range(n)], + "person_id": list(range(n)), + "birth_year": [1943] * n, + "income_year": [2010] * n, + "wave": [2011] * n, + "sex": ["male"] * n, + "married": [False] * n, + "own_ssi": [0.0] * n, + "head_ssi": [0.0] * n, + "wife_ssi": [0.0] * n, + "ofum_ssi": [0.0] * n, + "family_ssi": [0.0] * n, + "wealth_available": [True] * n, + "wealth1": [0.0] * n, + "wealth1_acc": [0.0] * n, + } + for key, value in defaults.items(): + if key not in frame: + frame[key] = value + return frame + + +def test_social_security_summary_by_hand(): + rows = _rows( + member_role=["head", "wife", "ofum", "head"], + own_identified=[True, True, False, True], + own_ss=[12_000.0, 6_000.0, np.nan, 0.0], + family_ss=[12_000.0, 6_000.0, 5_000.0, 0.0], + weight=[1.0, 3.0, 2.0, 4.0], + ) + published = { + 2009: { + "retired_workers": {"average_monthly": 500.0}, + "aged_types_combined": {"average_monthly": 400.0}, + }, + 2010: { + "retired_workers": {"average_monthly": 625.0}, + "aged_types_combined": {"average_monthly": 500.0}, + }, + } + entry = dg.social_security_summary(rows, published)["2010"] + # own: recipients weigh 1 + 3 of the 8 head/wife weight + assert entry["own_receipt_share"] == pytest.approx(0.5) + # (12,000 * 1 + 6,000 * 3) / 4 = 7,500 a year, 625 a month + assert entry["own_mean_annual_among_recipients"] == pytest.approx(7_500) + assert entry["own_mean_monthly_among_recipients"] == pytest.approx(625) + # family: 1 + 3 + 2 of the total weight 10 + assert entry["family_receipt_share"] == pytest.approx(0.6) + prior = entry["published"]["december_prior_year"] + assert prior["december"] == 2009 + assert prior["psid_over_ssa_retired_worker"] == pytest.approx(1.25) + assert prior["psid_over_ssa_aged_types"] == pytest.approx(1.5625) + same = entry["published"]["december_income_year"] + assert same["psid_over_ssa_retired_worker"] == pytest.approx(1.0) + assert entry["n_own_recipients"] == 2 + + +def test_ssi_summary_by_hand(): + rows = _rows( + member_role=["head", "head", "wife", "head", "ofum"], + own_identified=[True, True, True, True, False], + own_ss=[0.0] * 5, + family_ss=[0.0] * 5, + # 0: individual 8,000 > 7,200 (above); 1-2: a couple unit whose + # head and wife each report 5,000: 10,000 < 10,800 (not above); + # 3: no SSI; 4: OFUM, own amount not identified + own_ssi=[8_000.0, 5_000.0, 5_000.0, 0.0, np.nan], + head_ssi=[8_000.0, 5_000.0, 5_000.0, 0.0, 0.0], + wife_ssi=[0.0, 5_000.0, 5_000.0, 0.0, 0.0], + family_ssi=[8_000.0, 10_000.0, 10_000.0, 0.0, 700.0], + weight=[1.0, 1.0, 1.0, 1.0, 2.0], + ) + rates = {"individual": {2010: 600.0}, "couple": {2010: 900.0}} + entry = dg.ssi_summary(rows, rates)["2010"] + assert entry["federal_maximum_annual"] == { + "individual": 7_200.0, + "couple": 10_800.0, + } + assert entry["n_own_recipients"] == 3 + assert entry["own_receipt_share"] == pytest.approx(3 / 4) + # family: 1 + 1 + 1 + 2 of 6 + assert entry["family_receipt_share"] == pytest.approx(5 / 6) + assert entry["own_mean_annual_among_recipients"] == pytest.approx(6_000) + assert entry["n_recipients_above_federal_maximum"] == 1 + assert entry["recipient_share_above_federal_maximum"] == pytest.approx( + 1 / 3 + ) + + +def test_wealth1_summary_by_hand(): + rows = _rows( + member_role=["head"] * 5, + own_identified=[True] * 5, + own_ss=[0.0] * 5, + family_ss=[0.0] * 5, + wealth1=[-500.0, 0.0, 1_000.0, 5_000.0, 0.0], + wealth1_acc=[0.0, 1.0, 0.0, 0.0, 0.0], + wealth_available=[True, True, True, True, False], + weight=[1.0, 1.0, 1.0, 1.0, 9.0], + ) + entry = dg.wealth1_summary(rows)["2010"] + # the unavailable row (weight 9) is left out + assert entry["n_observations"] == 4 + assert entry["share_at_or_below_zero"] == pytest.approx(0.5) + assert entry["share_below_zero"] == pytest.approx(0.25) + assert entry["share_imputed"] == pytest.approx(0.25) + assert entry["quantiles"] == { + "p10": -500.0, + "p25": -500.0, + "p50": 0.0, + "p75": 1_000.0, + "p90": 5_000.0, + } + + +def test_component_rows_on_invented_inputs(): + inputs = invented.invented_age67_inputs() + cohort = age67.build_age67_cohort(inputs) + rows = dg.component_rows(cohort, inputs) + assert len(rows) == len(cohort.observations) + heads = rows["member_role"].eq("head") + assert rows.loc[rows["member_role"].eq("ofum"), "own_ss"].isna().all() + assert rows.loc[heads, "own_ss"].notna().all() + # waves 2005 and 2007 have no invented wealth unless staged + blocked = rows["wave"].isin([2005, 2007]) + assert not rows.loc[blocked, "wealth_available"].any() + assert rows.loc[~blocked, "wealth_available"].all() + + +def test_institution_record_counts_on_invented_inputs(): + inputs = invented.invented_age67_inputs() + counts = dg.institution_record_counts(inputs) + for entry in counts.values(): + assert entry["n_fu_size_equals_in_family_records"] == ( + entry["n_families"] + ) + assert entry["of_which_fu_size_equals_in_family_plus_institution"] == 0 + assert counts["2013"]["n_institution_records"] > 0 + + +def test_the_diagnostics_module_loads_no_income_concept(): + """Importing the diagnostics loads neither the income concept, the + tabulation, the runner nor the invented generator.""" + + code = ( + "import sys\n" + "import populace_dynamics.uniform_cut_track_u.diagnostics\n" + "bad = [m for m in (\n" + " 'populace_dynamics.estimates.adjusted_poverty',\n" + " 'populace_dynamics.estimates.uniform_cut_tabulation',\n" + " 'populace_dynamics.uniform_cut_track_u.runner',\n" + " 'populace_dynamics.uniform_cut_track_u.invented',\n" + ") if m in sys.modules]\n" + "assert not bad, bad\n" + ) + subprocess.run([sys.executable, "-c", code], check=True) diff --git a/tests/track_u/test_diagnostics_published.py b/tests/track_u/test_diagnostics_published.py new file mode 100644 index 00000000..ee3a8085 --- /dev/null +++ b/tests/track_u/test_diagnostics_published.py @@ -0,0 +1,85 @@ +"""The F17 published-source readers against the committed files. + +Reads two committed files under ``data/external``: the SSI capture and the +SSA Annual Statistical Supplement 2025 section 5.A snapshot. The Table +5.A4 cells below were transcribed by hand from that snapshot (they are +published SSA figures, not PSID values or comparator values); the test +checks that the parser returns exactly them. +""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import pytest + +from populace_dynamics.estimates import adjusted_poverty as ap +from populace_dynamics.uniform_cut_track_u import diagnostics as dg + +ROOT = Path(__file__).resolve().parents[2] +SNAPSHOT_MANIFEST = ( + ROOT + / "data" + / "external" + / "snapshots" + / "ssa_level_anchors_vintage1" + / "capture_manifest.txt" +) + + +def test_the_snapshot_pin_is_the_capture_manifest_hash(): + manifest = SNAPSHOT_MANIFEST.read_text(encoding="utf-8") + line = next( + entry + for entry in manifest.splitlines() + if entry.endswith("supplement2025_5a.html") + ) + assert line.split()[1] == dg.SSA_5A_SNAPSHOT_SHA256 + + +def test_table_5a4_cells_as_published(): + cells = dg.ssa_table_5a4([2003, 2004, 2010, 2012]) + # Retired workers, December: number and total monthly benefits + # (thousands of dollars), as printed. + assert cells[2004]["retired_workers"]["number"] == 29_952_465 + assert cells[2004]["retired_workers"]["total_monthly_thousands"] == ( + 28_601_329 + ) + assert cells[2010]["retired_workers"]["number"] == 34_593_080 + assert cells[2010]["retired_workers"]["total_monthly_thousands"] == ( + 40_662_492 + ) + assert cells[2003]["widowers"]["number"] == 4_707_215 + assert cells[2012]["wives_and_husbands"]["number"] == 2_443_212 + # the average is the division 1000 * total / number + assert cells[2004]["retired_workers"]["average_monthly"] == ( + pytest.approx(1000 * 28_601_329 / 29_952_465) + ) + combined = cells[2003]["aged_types_combined"] + assert combined["number"] == 29_531_611 + 2_772_577 + 4_707_215 + assert combined["total_monthly_thousands"] == ( + 27_230_634 + 1_247_504 + 4_110_963 + ) + + +def test_table_5a4_refuses_a_changed_snapshot(tmp_path): + copy = tmp_path / "supplement2025_5a.html" + shutil.copy(dg.SSA_5A_SNAPSHOT_PATH, copy) + with copy.open("a", encoding="utf-8") as handle: + handle.write("\n") + with pytest.raises(ValueError, match="sha256"): + dg.ssa_table_5a4([2004], copy) + with pytest.raises(ValueError, match="December 1939"): + dg.ssa_table_5a4([1939]) + + +def test_federal_benefit_rates_equal_the_income_concepts_capture(): + assert dg.SSI_PARAMETERS_SHA256 == ap.SSI_PARAMETERS_SHA256 + assert dg.SSI_PARAMETERS_PATH == ap.SSI_PARAMETERS_PATH + rates = dg.federal_benefit_rates() + ssi = ap.load_ssi_parameters() + assert rates["individual"] == dict(ssi.fbr_individual_monthly) + assert rates["couple"] == dict(ssi.fbr_couple_monthly) + assert rates["rule"] == "value in force on January 1 of the income year" + assert rates["source"]["sha256"] == ap.SSI_PARAMETERS_SHA256 diff --git a/tests/track_u/test_dry_run_script.py b/tests/track_u/test_dry_run_script.py new file mode 100644 index 00000000..78a3eacc --- /dev/null +++ b/tests/track_u/test_dry_run_script.py @@ -0,0 +1,88 @@ +"""The Track U dry run (plan item U9) on INVENTED data. + +Runs ``scripts/track_u_dry_run.py`` into a temporary directory. The +cohort and the thresholds are INVENTED; the life tables, the SSI capture +and the SSA snapshot the diagnostics read are the committed files under +``data/external``. No PSID file and no comparator value is read. +""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +import pytest + +from populace_dynamics.estimates import adjusted_poverty as ap +from populace_dynamics.uniform_cut_track_u import DRY_RUN_HEADER + +ROOT = Path(__file__).resolve().parents[2] +COMMITTED = ROOT / "data" / "external" + + +def _script(): + path = ROOT / "scripts" / "track_u_dry_run.py" + spec = importlib.util.spec_from_file_location("_track_u_dry_run", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def output(tmp_path_factory) -> Path: + directory = tmp_path_factory.mktemp("track_u_dry_run") + assert _script().main(["--output-dir", str(directory)]) == 0 + return directory + + +def test_the_dry_run_is_headed_invented(output): + result = json.loads((output / "result.json").read_text()) + assert ( + result["header"] + == DRY_RUN_HEADER + == ("INVENTED DATA - NOT A COMPARISON") + ) + text = (output / "RESULTS.md").read_text() + assert text.splitlines()[0] == f"# {DRY_RUN_HEADER}" + assert result["data_provenance"] == ap.INVENTED + assert result["labels"][0].startswith("INVENTED DATA") + assert result["cohort_provenance"]["kind"] == "invented" + assert result["parameters"]["thresholds"]["kind"] == "invented" + assert "not Census values" in result["parameters"]["thresholds"]["label"] + assert "psid_files" not in json.dumps(result["cohort_provenance"]) + assert result["psid_files_sha256"] == {} + + +def test_the_dry_run_uses_the_committed_parameters(output): + result = json.loads((output / "result.json").read_text()) + assert (COMMITTED / "track_u_ssi_parameters.json").is_file() + assert result["parameters"]["ssi"]["sha256"] == ap.SSI_PARAMETERS_SHA256 + assert result["parameters"]["life_tables"]["nchs_2000"]["sha256"] == ( + ap.NCHS_2000_SHA256 + ) + + +def test_the_dry_run_checks_hold(output): + checks = json.loads((output / "result.json").read_text())["checks"] + assert checks["specification_rows"]["rows_equal_the_block"] + blocked = checks["blocked_waves_as_staged_today"] + assert blocked["income_rows_without_allow_blocked"]["refused"] + assert ( + blocked["left_out_with_allow_blocked"]["wealth_supplement_not_staged"] + > 0 + ) + assert checks["registered_guard_refuses_invented_inputs"]["refused"] + assert checks["census_thresholds_not_captured"]["error"] == ( + "ThresholdsNotCapturedError" + ) + for identities in checks["invented_family_income_reconciliation"].values(): + for counts in identities.values(): + assert counts["n_exact"] == counts["n_families"] + + +def test_every_built_row_is_tabulated(output): + result = json.loads((output / "result.json").read_text()) + statuses = {row: entry["status"] for row, entry in result["rows"].items()} + assert statuses.pop("U7") == "not_built" + assert set(statuses.values()) == {"computed"} diff --git a/tests/track_u/test_invented_generator.py b/tests/track_u/test_invented_generator.py new file mode 100644 index 00000000..1e3a8762 --- /dev/null +++ b/tests/track_u/test_invented_generator.py @@ -0,0 +1,254 @@ +"""The INVENTED Track U generator (plan item U5). + +Everything here runs on the invented generator's output: no PSID file, no +Census threshold, no SSA value and no comparator value is read. The tests +check that the invented frames have the reader shapes, satisfy the +codebook identities, exercise every Track U path, and cannot pass as PSID. +""" + +from __future__ import annotations + +import dataclasses + +import pandas as pd +import pytest + +from populace_dynamics.cohorts import age67 +from populace_dynamics.data import family_income +from populace_dynamics.estimates import adjusted_poverty as ap +from populace_dynamics.uniform_cut_track_u import invented +from tests.cohorts.test_age67 import _inputs + + +@pytest.fixture(scope="module") +def blocked() -> age67.Age67Inputs: + return invented.invented_age67_inputs() + + +@pytest.fixture(scope="module") +def staged() -> age67.Age67Inputs: + return invented.invented_age67_inputs(supplement_waves_staged=True) + + +def test_the_generator_is_deterministic_in_its_seed(blocked): + again = invented.invented_age67_inputs() + assert age67.input_frames_sha256(again) == age67.input_frames_sha256( + blocked + ) + other = invented.invented_age67_inputs(seed=invented.DEFAULT_SEED + 1) + assert age67.input_frames_sha256(other) != age67.input_frames_sha256( + blocked + ) + with pytest.raises(ValueError, match="integer"): + invented.invented_age67_inputs(seed=True) + + +def test_provenance_is_labelled_invented(blocked): + provenance = blocked.provenance + assert provenance["kind"] == age67.INVENTED == "invented" + assert provenance["data"].startswith("INVENTED DATA") + assert "not PSID observations" in provenance["data"] + assert provenance["seed"] == invented.DEFAULT_SEED + assert provenance["supplement_waves_staged"] is False + assert provenance["input_frames_sha256"] == age67.input_frames_sha256( + blocked + ) + cohort = age67.build_age67_cohort(blocked) + assert cohort.provenance["kind"] == age67.INVENTED + assert cohort.provenance["label"] == provenance["data"] + + +def test_frames_have_the_reader_shapes(staged): + for wave in age67.WAVES: + income = staged.family_income[wave] + assert list(income.columns) == [ + "wave", + "income_year", + *family_income.income_variables(wave), + "wife_present", + ] + assert (income["income_year"] == wave - 1).all() + assert str(income["head_sex"].dtype) == "string" + assert str(income["head_age"].dtype) == "Int64" + assert str(income["wife_age"].dtype) == "Int64" + assert (income["wife_present"] == income["wife_age"].notna()).all() + assert income["interview"].is_unique + assert (income["census_needs_standard"] > 0).all() + assert (income["n_children"] < income["fu_size"]).all() + # negative amounts only where the codebooks document a loss + for concept in family_income.income_variables(wave): + if concept in family_income.MAY_BE_NEGATIVE or concept in ( + "head_sex", + "head_age", + "wife_age", + ): + continue + assert (income[concept] >= 0).all(), (wave, concept) + layout = 2009 if wave in (2005, 2007) else wave + wealth = staged.family_wealth[wave] + assert list(wealth.columns) == [ + "wave", + "interview", + *family_income.wealth_variables(layout), + ] + assert set(wealth["interview"]) == set(income["interview"]) + anchor = staged.anchors[2009] + assert list(anchor.columns) == [ + "person_id", + "interview", + "sequence", + "relationship", + "age", + "reported_birth_year", + "weight", + ] + assert anchor["person_id"].is_unique + + +def test_income_and_wealth_identities_hold_exactly(staged): + incomes = pd.concat( + [staged.family_income[wave] for wave in age67.WAVES], + ignore_index=True, + ) + for identities in family_income.reconcile_family_income(incomes).values(): + for counts in identities.values(): + assert counts["n_exact"] == counts["n_families"] + wealth = pd.concat( + [staged.family_wealth[wave] for wave in family_income.WEALTH_WAVES], + ignore_index=True, + ) + for identities in family_income.reconcile_wealth1(wealth).values(): + for counts in identities.values(): + assert counts["n_exact"] == counts["n_families"] + + +def test_fu_size_counts_the_in_family_records_only(staged): + for wave in age67.WAVES: + anchor = staged.anchors[wave] + in_family = ( + anchor[anchor["sequence"].between(1, 20)] + .groupby("interview") + .size() + ) + income = staged.family_income[wave].set_index("interview") + assert (income["fu_size"] == in_family.reindex(income.index)).all() + + +def test_supplement_waves_follow_the_flag(blocked, staged): + assert set(blocked.family_wealth) == set(family_income.WEALTH_WAVES) + assert set(blocked.wealth_refusals) == {2005, 2007} + assert "INVENTED" in blocked.wealth_refusals[2005] + assert set(staged.family_wealth) == set(age67.WAVES) + assert staged.wealth_refusals == {} + + +def test_every_track_u_path_is_exercised(staged): + u0 = age67.build_age67_cohort(staged) + obs = u0.observations + assert set(obs["birth_year"]) == set(age67.PRIMARY_BIRTH_YEARS) + assert set(obs["member_role"]) == {"head", "wife", "ofum"} + assert { + "married", + "widowed", + "divorced", + "never_married", + "unknown", + "no_marriage_history", + } <= set(obs["marital_status"]) + assert { + "exact_marriage", + "inferred_period_age", + "derived_projection_age", + } <= set(obs["birth_source"]) + assert obs["member_married_coresident"].any() + assert (obs["married"] & ~obs["member_married_coresident"]).any() + assert (obs["relationship"] == 22).any() + assert obs.groupby("family_unit_id")["person_id"].nunique().max() >= 2 + assert { + "not_present:moved_out", + "not_present:died", + "not_present:institution", + "zero_weight", + } <= set(u0.dispositions["disposition"]) + rows = age67.income_rows(u0, staged) + assert (rows["wealth1"] < 0).any() + assert (rows[list(family_income.SSI_CONCEPTS)].sum(axis=1) > 0).any() + assert (rows["n_children"] > 0).any() + u_inst = age67.build_age67_cohort( + staged, age67.Age67Spec(presence="in_family_or_institution") + ) + institution = u_inst.observations[u_inst.observations["in_institution"]] + assert len(institution) > 0 + assert set(institution["income_status"]) == {"family_of_record"} + assert set(institution["member_role"]) == {"ofum"} + u1 = age67.build_age67_cohort(staged, age67.Age67Spec(row="U1")) + assert set(u1.observations["birth_year"]) == set(age67.ALL_BIRTH_YEARS) + # losses the codebooks allow reach cohort members' families + losses = age67.income_rows(u1, staged)[ + ["head_business_asset", "head_rent", "head_farm"] + ] + assert (losses < 0).any().any() + + +def test_invented_thresholds_by_hand(): + thresholds = invented.invented_poverty_thresholds() + assert thresholds.provenance["kind"] == ap.INVENTED + assert thresholds.provenance["label"].startswith("INVENTED THRESHOLDS") + assert "not Census values" in thresholds.provenance["label"] + # 2004 is the base; 2005: 9,000 * 1.028 = 9,252 -> $10 rounding 9,250 + assert thresholds.weighted_average[2004]["one_65_plus"] == 9_000.0 + assert thresholds.weighted_average[2005]["one_65_plus"] == 9_250.0 + # matrix, three persons, two children: 15,000 * (1.02 - 0.03) = 14,850 + assert thresholds.matrix[2004]["three"][2] == 14_850.0 + assert set(thresholds.matrix[2004]["one_65_plus"]) == {0} + assert set(thresholds.matrix[2004]["nine_plus"]) == set(range(9)) + # the needs standard stand-in uses the householder's age: + # 65+: 9,000 * 1.02 = 9,180; under 65: 9,800 * 1.02 = 9,996 -> 10,000 + assert invented.invented_needs_standard(2004, 1, 0, 70) == 9_180 + assert invented.invented_needs_standard(2004, 1, 0, 60) == 10_000 + + +def test_invented_inputs_are_regenerated_not_trusted(blocked): + assert invented.check_invented_inputs(blocked)["regenerated"] + # edit one amount and re-seal the digest: the builder accepts the + # recorded digest, but re-generation from the seed refuses it + income = dict(blocked.family_income) + edited = income[2009].copy() + edited.loc[0, "head_ss"] += 1 + income[2009] = edited + forged = dataclasses.replace(blocked, family_income=income) + forged = dataclasses.replace( + forged, + provenance={ + **blocked.provenance, + "input_frames_sha256": age67.input_frames_sha256(forged), + }, + ) + age67.build_age67_cohort(forged) + with pytest.raises(ValueError, match="not the invented generator's"): + invented.check_invented_inputs(forged) + # an edit without re-sealing is refused by the builder itself + stale = dataclasses.replace(blocked, family_income=income) + with pytest.raises(ValueError, match="invented provenance"): + age67.build_age67_cohort(stale) + + +def test_other_frames_cannot_claim_the_invented_label(): + caller = _inputs() + with pytest.raises(ValueError, match="not invented"): + invented.check_invented_inputs(caller) + cohort = age67.build_age67_cohort(caller) + assert cohort.provenance["kind"] == age67.CALLER_FRAMES + with pytest.raises(ValueError, match="not invented"): + invented.check_invented_cohort(cohort) + + +def test_the_invented_cohort_is_checked_by_rebuilding(staged): + cohort = age67.build_age67_cohort(staged) + assert invented.check_invented_cohort(cohort)["rebuilt"] + edited = cohort.observations.copy() + edited.loc[0, "weight"] = edited.loc[0, "weight"] + 1.0 + forged = dataclasses.replace(cohort, observations=edited) + object.__setattr__(forged, "provenance", cohort.provenance) + with pytest.raises(ValueError, match="observations differ"): + invented.check_invented_cohort(forged) diff --git a/tests/track_u/test_registered_script.py b/tests/track_u/test_registered_script.py new file mode 100644 index 00000000..41ae99ea --- /dev/null +++ b/tests/track_u/test_registered_script.py @@ -0,0 +1,236 @@ +"""The Track U registered one-shot entry point refuses every other state. + +No PSID file is read and no statistic is computed: only the preflight +guards of ``scripts/run_track_u_registered.py`` run, with a fake ``git`` +and hand-built specification headers, plus one run of ``main`` whose +preflight is replaced to show the threshold refusal comes before any PSID +read. +""" + +from __future__ import annotations + +import copy +import importlib.util +from pathlib import Path + +import pytest + +from populace_dynamics.estimates import adjusted_poverty as ap +from populace_dynamics.uniform_cut_track_u import rows + +ROOT = Path(__file__).resolve().parents[2] +POINTER = ( + "https://github.com/PolicyEngine/microcosm-dynamics/issues/42" + "#issuecomment-1" +) +COMMIT = "a" * 40 +#: A hand-built header of a ratified, complete block (not the committed +#: specification, which is a draft). +RATIFIED = { + "status": "ratified_frozen", + "version": "u1-ratified-1", + "threshold": {"capture_status": "captured"}, + "blocked_by": [], + "claim_class": {"accepted": "track_u", "decision": "d189"}, +} + + +def _script(): + path = ROOT / "scripts" / "run_track_u_registered.py" + spec = importlib.util.spec_from_file_location("_track_u_registered", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _git(head: str = COMMIT, porcelain: str = ""): + def fake(*args: str) -> str: + if args == ("rev-parse", "HEAD"): + return head + if args == ("status", "--porcelain"): + return porcelain + raise AssertionError(args) + + return fake + + +def _arguments(tmp_path, **overrides): + arguments = { + "registration_pointer": POINTER, + "registered_commit": COMMIT, + "output": tmp_path / "run.json", + "git": _git(), + "specification": RATIFIED, + } + arguments.update(overrides) + return arguments + + +def test_the_registered_state_passes(tmp_path): + assert _script().preflight(**_arguments(tmp_path)) == {"head": COMMIT} + + +@pytest.mark.parametrize( + ("overrides", "match"), + [ + ({"registration_pointer": "https://example.org/x"}, "issue #42"), + ( + { + "registration_pointer": ( + "https://github.com/PolicyEngine/microcosm-dynamics/" + "issues/420#issuecomment-1" + ) + }, + "issue #42", + ), + ( + { + "registration_pointer": ( + "https://github.com/PolicyEngine/microcosm-dynamics/" + "issues/42" + ) + }, + "issue #42", + ), + ({"registered_commit": "abc123"}, "full 40-hex"), + ({"git": _git(head="b" * 40)}, "is not the registered commit"), + ({"git": _git(porcelain=" M src/x.py")}, "clean"), + ( + {"specification": {**RATIFIED, "status": "draft_for_referee"}}, + "authorizes no run", + ), + ( + {"specification": {**RATIFIED, "version": "u1-draft-3"}}, + "authorizes no run", + ), + ( + {"specification": {**RATIFIED, "status": "not yet ratified"}}, + "authorizes no run", + ), + ( + { + "specification": { + **RATIFIED, + "rows": {"U6": {"on": "U1", "awaiting": "decision 8"}}, + } + }, + "awaited", + ), + ( + { + "specification": { + **RATIFIED, + "claim_class": {"awaiting": "d189"}, + } + }, + "awaited", + ), + ( + { + "specification": { + **RATIFIED, + "blocked_by": ["census_thresholds_not_captured"], + } + }, + "blocked", + ), + ( + { + "specification": { + **RATIFIED, + "threshold": {"capture_status": "not_captured"}, + } + }, + "captured", + ), + ], + ids=[ + "pointer", + "other-issue", + "issue-not-comment", + "short-sha", + "wrong-head", + "dirty-tree", + "draft-status", + "draft-version", + "negated-status", + "row-awaiting", + "claim-awaiting", + "blocked-by", + "thresholds-not-captured", + ], +) +def test_non_registered_states_are_refused(tmp_path, overrides, match): + with pytest.raises(ValueError, match=match): + _script().preflight(**_arguments(tmp_path, **overrides)) + + +def test_the_committed_specification_authorizes_no_run(tmp_path): + block = rows.specification_block() + with pytest.raises(ValueError, match="authorizes no run"): + _script().check_specification_ratified(block) + # even with a ratified header, the committed draft still awaits + # decisions and is blocked + header = copy.deepcopy(block) + header.update(status="ratified_frozen", version="u1-ratified-1") + with pytest.raises(ValueError, match="awaited"): + _script().check_specification_ratified(header) + + +@pytest.mark.parametrize("existing", ["run.json", "run.env.json"]) +def test_an_existing_artifact_refuses_a_second_shot(tmp_path, existing): + (tmp_path / existing).write_text("{}") + with pytest.raises(ValueError, match="one-shot"): + _script().preflight(**_arguments(tmp_path)) + + +def test_the_artifact_is_created_exclusively(tmp_path): + target = tmp_path / "run.json" + _script()._write_new(target, "first\n") + with pytest.raises(FileExistsError): + _script()._write_new(target, "second\n") + assert target.read_text() == "first\n" + + +def test_the_registration_pointer_and_commit_are_required(): + with pytest.raises(SystemExit): + _script().main([]) + + +def test_uncaptured_thresholds_stop_the_run_before_any_psid_read( + tmp_path, monkeypatch +): + script = _script() + monkeypatch.setattr(script, "preflight", lambda **_: {"head": COMMIT}) + + def no_psid(**_): + raise AssertionError("PSID read before the threshold check") + + monkeypatch.setattr(script.age67, "load_age67_inputs", no_psid) + output = tmp_path / "run.json" + with pytest.raises(ap.ThresholdsNotCapturedError): + script.main( + [ + "--registration-pointer", + POINTER, + "--registered-commit", + COMMIT, + "--output", + str(output), + ] + ) + assert not output.exists() + assert not output.with_suffix(".env.json").exists() + + +def test_the_script_shares_the_exercise_1_ratification_test(): + from populace_dynamics.estimates import cola_age_profile + from populace_dynamics.uniform_cut_track_u import runner + + script = _script() + assert script.UNRATIFIED_MARKERS is cola_age_profile.UNRATIFIED_MARKERS + assert script.REGISTRATION_POINTER is runner.REGISTRATION_POINTER + assert script.DEFAULT_OUTPUT == ( + ROOT / "runs" / "replication_boomers2004_uniform_cut_v1.json" + ) + assert not script.DEFAULT_OUTPUT.exists() diff --git a/tests/track_u/test_rows.py b/tests/track_u/test_rows.py new file mode 100644 index 00000000..ea7c7a5a --- /dev/null +++ b/tests/track_u/test_rows.py @@ -0,0 +1,126 @@ +"""The registered Track U rows equal the specification block. + +Reads only the specification document and the code: no PSID value, no +model output and no comparator value. +""" + +from __future__ import annotations + +import copy + +import pytest + +from populace_dynamics.cohorts import age67 +from populace_dynamics.estimates import adjusted_poverty as ap +from populace_dynamics.uniform_cut_track_u import rows + + +@pytest.fixture(scope="module") +def block() -> dict: + return rows.specification_block() + + +def test_the_code_rows_equal_the_committed_block(block): + check = rows.check_rows_against_block(block) + assert check["rows_equal_the_block"] + assert check["specification_version"] == block["version"] == "u1-draft-3" + assert check["rows_checked"] == sorted(rows.REGISTERED_ROWS) + + +def test_each_row_changes_the_field_the_plan_names(): + registered = rows.REGISTERED_ROWS + assert registered["U0"].age67_spec() == age67.Age67Spec() + assert registered["U0"].income_spec() == ap.AdjustedPovertySpec() + assert registered["U1"].age67_spec().row == "U1" + assert registered["U2"].income_spec().ssi_rule == "none" + assert registered["U3"].income_spec().ssi_rule == ( + "full_static_recomputation" + ) + assert registered["U4"].income_spec().income_unit == "head_wife" + assert registered["U5"].income_spec().asset_income_rule == "keep" + # U6 is defined on U1 and differs in the cut's start year only + assert registered["U6"].age67_spec() == registered["U1"].age67_spec() + assert registered["U6"].income_spec().cut_start_year == 2004 + assert registered["U6"].awaiting and "decision 8" in ( + registered["U6"].awaiting + ) + assert not registered["U7"].built + assert "label investigation" in registered["U7"].not_built_reason + assert registered["U8"].income_spec().threshold_rule == ( + "psid_census_needs_standard" + ) + assert registered["U9"].income_spec().mortality_basis == ( + "ssa_period_2004" + ) + assert registered["U10"].income_spec().threshold_rule == ( + "census_matrix_65plus" + ) + inst = registered["U-inst"].age67_spec() + assert inst.presence == "in_family_or_institution" + assert inst.institution_income_rule == "family_of_record" + assert "referee" in registered["U-inst"].awaiting + + +def test_row_from_block_reads_populations_and_overrides(): + assert rows.row_from_block( + "U1", {"population": "all_ten_birth_years"} + ) == { + "age67": {"row": "U1"}, + "income": {}, + "built": True, + "awaiting": None, + } + parsed = rows.row_from_block( + "U6", {"on": "U1", "cut_start_year": 2004, "awaiting": "x"} + ) + assert parsed["age67"] == {"row": "U1"} + assert parsed["income"] == {"cut_start_year": 2004} + assert parsed["awaiting"] == "x" + assert not rows.row_from_block( + "U7", {"financial_assets": "y", "status": "not_built"} + )["built"] + with pytest.raises(ValueError, match="unknown block key"): + rows.row_from_block("U9", {"mortality": "nchs_2000"}) + with pytest.raises(ValueError, match="not known"): + rows.row_from_block("U1", {"on": "U0"}) + with pytest.raises(ValueError, match="unknown status"): + rows.row_from_block("U-inst", {"status": "counted_only"}) + + +@pytest.mark.parametrize( + ("row_id", "edit"), + [ + ("U2", {"ssi_rule": "full_static_recomputation"}), + ("U6", {"cut_start_year": 2005}), + ("U-inst", {"institution_income_rule": "excluded"}), + ("U7", {"status": None}), + ], +) +def test_a_block_that_differs_from_the_code_is_refused(block, row_id, edit): + changed = copy.deepcopy(block) + changed["rows"][row_id].update(edit) + if edit.get("status", "keep") is None: + del changed["rows"][row_id]["status"] + del changed["rows"][row_id]["financial_assets"] + with pytest.raises(ValueError, match=f"row {row_id} differs"): + rows.check_rows_against_block(changed) + + +def test_a_resolved_awaiting_note_must_be_resolved_in_both_places(block): + changed = copy.deepcopy(block) + del changed["rows"]["U6"]["awaiting"] + with pytest.raises(ValueError, match="row U6 differs"): + rows.check_rows_against_block(changed) + missing = copy.deepcopy(block) + del missing["rows"]["U10"] + with pytest.raises(ValueError, match="differ from the block"): + rows.check_rows_against_block(missing) + + +def test_rows_validate_their_fields(): + with pytest.raises(ValueError, match="unknown fields"): + rows.TrackURow("X", "-", "x", income={"cut": 0.2}) + with pytest.raises(ValueError, match="needs a reason"): + rows.TrackURow("X", "-", "x", built=False) + with pytest.raises(ap.AdjustedPovertyError): + rows.TrackURow("X", "-", "x", income={"cut_start_year": 1900}) diff --git a/tests/track_u/test_runner.py b/tests/track_u/test_runner.py new file mode 100644 index 00000000..27c602c8 --- /dev/null +++ b/tests/track_u/test_runner.py @@ -0,0 +1,272 @@ +"""The Track U row pipeline on INVENTED data, and its guards. + +The cohort, the thresholds and every amount are INVENTED (the generator of +:mod:`populace_dynamics.uniform_cut_track_u.invented`); the life tables and +the SSI parameters are the committed files under ``data/external``, and +the F17 component diagnostics read the committed SSA snapshot there. No +PSID file and no comparator value is read. +""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path + +import pandas as pd +import pytest + +from populace_dynamics.cohorts import age67 +from populace_dynamics.estimates import adjusted_poverty as ap +from populace_dynamics.estimates import uniform_cut_tabulation as ut +from populace_dynamics.uniform_cut_track_u import invented, rows, runner +from tests.cohorts.test_age67 import _inputs + +ROOT = Path(__file__).resolve().parents[2] +COMMITTED = ROOT / "data" / "external" +POINTER = ( + "https://github.com/PolicyEngine/microcosm-dynamics/issues/42" + "#issuecomment-1" +) + + +@pytest.fixture(scope="module") +def params() -> runner.TrackUParameters: + assert (COMMITTED / "track_u_ssi_parameters.json").is_file() + return runner.committed_parameters(invented.invented_poverty_thresholds()) + + +@pytest.fixture(scope="module") +def staged() -> age67.Age67Inputs: + return invented.invented_age67_inputs(supplement_waves_staged=True) + + +@pytest.fixture(scope="module") +def result(staged, params) -> dict: + return runner.run_track_u(staged, params, data_provenance=ap.INVENTED) + + +def _all(entry: dict) -> dict: + return next( + cell for cell in entry["tabulation"]["cells"] if cell["cell"] == "all" + ) + + +def test_every_row_runs_and_u7_is_reported_not_built(result): + assert list(result["rows"]) == list(rows.REGISTERED_ROWS) + assert result["rows"]["U7"]["status"] == "not_built" + assert result["rows"]["U7"]["tabulation"] is None + for row_id, entry in result["rows"].items(): + if row_id == "U7": + continue + assert entry["status"] == "computed" + assert entry["tabulation"]["data_provenance"] == ap.INVENTED + assert entry["tabulation"]["labels"][0] == ut.INVENTED_DATA_LABEL + cell = _all(entry) + assert cell["defined"] + # nobody leaves poverty under a cut (plan section 1) + assert entry["income_concept_counts"]["n_leaving_poverty"] == 0 + assert cell["delta"] >= 0 + assert result["labels"][0] == ut.INVENTED_DATA_LABEL + assert result["labels"][1:] == list(ap.OUTPUT_LABELS) + assert result["cohort_provenance"]["kind"] == age67.INVENTED + assert result["psid_files_sha256"] == {} + assert result["input_checks"]["invented_inputs"]["regenerated"] + assert result["input_checks"]["invented_cohort"]["rebuilt"] + + +def test_rows_change_only_what_they_register(result): + rows_ = result["rows"] + u0, u1, u6 = rows_["U0"], rows_["U1"], rows_["U6"] + # U2: no SSI response at all + assert rows_["U2"]["income_concept_counts"]["n_ssi_offset_positive"] == 0 + assert rows_["U2"]["income_concept_counts"]["n_ssi_new_positive"] == 0 + # U0 never enrols anyone new + assert u0["income_concept_counts"]["n_ssi_new_positive"] == 0 + # U4 puts the head and wife members on the head/wife basis + assert "head_wife" in rows_["U4"]["income_concept_counts"]["income_basis"] + # U5 removes no reported asset income + counts = rows_["U5"]["income_concept_counts"] + assert counts["n_asset_income_removed_nonzero"] == 0 + # U6 is U1 with the 1936 birth year uncut + n_1936 = u1["population"]["observations_by_birth_year"]["1936"] + assert u6["population"] == u1["population"] + assert u6["income_concept_counts"]["n_cut_applies"] == ( + u1["income_concept_counts"]["n_cut_applies"] - n_1936 + ) + assert _all(u6)["baseline_rate"] == pytest.approx( + _all(u1)["baseline_rate"] + ) + assert _all(u6)["reform_rate"] <= _all(u1)["reform_rate"] + cells = {cell["cell"]: cell for cell in u6["tabulation"]["cells"]} + assert cells["birth_year_1936"]["delta"] == 0.0 + # U-inst adds the institutionalized members to U0's population + inst = rows_["U-inst"]["population"] + assert inst["n_in_institution"] > 0 + assert inst["n_observations_tabulated"] == ( + u0["population"]["n_observations_tabulated"] + inst["n_in_institution"] + ) + # U9 prices on the SSA 2004 table + assert rows_["U9"]["life_table"] == "ssa_period_2004" + assert u0["life_table"] == "nchs_2000" + + +def test_f17_diagnostics_are_reported_for_u0(result): + f17 = result["f17_diagnostics"] + assert f17["row"] == "U0" + official = f17["official_concept_poverty_rate"]["cells"] + assert set(official) == set(runner.OFFICIAL_CONCEPT_CELLS) + assert all(cell["defined"] for cell in official.values()) + components = f17["components"] + assert components["population"]["provenance_kind"] == age67.INVENTED + assert set(components["social_security"]) == { + "2004", + "2006", + "2008", + "2010", + "2012", + } + + +def test_official_concept_rates_by_hand(): + # INVENTED: weights 1, 1, 2, 4; poor under money income: rows a and d + members = pd.DataFrame( + { + "observation_id": ["a", "b", "c", "d"], + "weight": [1.0, 1.0, 2.0, 4.0], + "sex": ["male", "female", "female", "male"], + "married": [True, False, True, False], + } + ) + adjusted = pd.DataFrame( + { + "observation_id": ["a", "b", "c", "d"], + "money_income": [900.0, 1_000.0, 2_000.0, 500.0], + "threshold": [1_000.0, 1_000.0, 1_000.0, 1_000.0], + } + ) + cells = runner.official_concept_rates(members, adjusted)["cells"] + # all: (1 + 4) / 8; b at the threshold is not poor + assert cells["all"]["rate"] == pytest.approx(62.5) + assert cells["men"]["rate"] == pytest.approx(100.0) + assert cells["women"]["rate"] == pytest.approx(0.0) + assert cells["married"]["rate"] == pytest.approx(100 / 3) + assert cells["non_married"]["rate"] == pytest.approx(80.0) + with pytest.raises(runner.TrackURunError, match="do not match"): + runner.official_concept_rates(members, adjusted.iloc[:3]) + + +def test_the_invented_label_needs_the_generators_inputs(params, staged): + with pytest.raises(ValueError, match="not invented"): + runner.run_track_u(_inputs(), params, data_provenance=ap.INVENTED) + income = dict(staged.family_income) + edited = income[2011].copy() + edited.loc[0, "total_family_income"] += 5 + income[2011] = edited + forged = dataclasses.replace(staged, family_income=income) + forged = dataclasses.replace( + forged, + provenance={ + **staged.provenance, + "input_frames_sha256": age67.input_frames_sha256(forged), + }, + ) + with pytest.raises(ValueError, match="not the invented generator's"): + runner.run_track_u(forged, params, data_provenance=ap.INVENTED) + with pytest.raises(runner.TrackURunError, match="no registration"): + runner.run_track_u( + staged, + params, + data_provenance=ap.INVENTED, + registration_pointer=POINTER, + ) + + +@pytest.mark.parametrize( + "pointer", + [ + None, + "https://example.org/x", + "https://github.com/PolicyEngine/microcosm-dynamics/issues/42", + "https://github.com/PolicyEngine/microcosm-dynamics/issues/420" + "#issuecomment-1", + ], +) +def test_a_registered_run_needs_an_issue_42_comment(params, staged, pointer): + with pytest.raises(runner.TrackURunError, match="issue #42"): + runner.run_track_u( + staged, + params, + data_provenance=ap.REGISTERED_REAL, + registration_pointer=pointer, + ) + + +def test_a_registered_run_refuses_invented_or_blocked_inputs(params, staged): + with pytest.raises(runner.TrackURunError, match="load_age67_inputs"): + runner.run_track_u( + staged, + params, + data_provenance=ap.REGISTERED_REAL, + registration_pointer=POINTER, + ) + claimed = dataclasses.replace( + staged, provenance={**staged.provenance, "kind": age67.PSID_FILES} + ) + with pytest.raises(runner.TrackURunError, match="blocked observations"): + runner.run_track_u( + claimed, + params, + data_provenance=ap.REGISTERED_REAL, + registration_pointer=POINTER, + allow_blocked=True, + ) + blocked = dataclasses.replace( + invented.invented_age67_inputs(), + provenance={**staged.provenance, "kind": age67.PSID_FILES}, + ) + with pytest.raises(runner.TrackURunError, match="WEALTH1"): + runner.run_track_u( + blocked, + params, + data_provenance=ap.REGISTERED_REAL, + registration_pointer=POINTER, + ) + + +def test_a_registered_run_needs_the_committed_pinned_parameters(params): + with pytest.raises(runner.TrackURunError, match="Census threshold"): + runner._check_parameters(params, ap.REGISTERED_REAL) + fake_capture = dataclasses.replace( + params, + thresholds=dataclasses.replace( + params.thresholds, + provenance={"kind": "census_capture", "sha256": "0" * 64}, + ), + ) + # the capture is not pinned yet (THRESHOLDS_SHA256 is None), so no + # threshold table can pass + assert ap.THRESHOLDS_SHA256 is None + with pytest.raises(runner.TrackURunError, match="pinned"): + runner._check_parameters(fake_capture, ap.REGISTERED_REAL) + invented_table = ap.LifeTable( + name=ap.INVENTED_LIFE_TABLE, + qx={sex: (0.1,) * 110 + (1.0,) for sex in ("male", "female")}, + ) + with pytest.raises(runner.TrackURunError, match="filed under"): + runner._check_parameters( + dataclasses.replace( + params, + life_tables={ + **params.life_tables, + "nchs_2000": invented_table, + }, + ), + ap.REGISTERED_REAL, + ) + runner._check_parameters( + dataclasses.replace( + params, + life_tables={**params.life_tables, "nchs_2000": invented_table}, + ), + ap.INVENTED, + ) From 09ae4bcb8d6a407a6bfcd860eb093839630b5900 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 24 Sep 2026 06:52:23 -0400 Subject: [PATCH 2/9] Apply the first Track U referee pass to the exercise-2 specification (u1-draft-4) Applies the independent referee report EVID/boomers2004-referee-20260924.md (SHA-256 b083fb2638787c204d09324a16fbbc5fa09cfc191e0450bcadb7d44f1647b2b6), which refereed u1-draft-2 at 8d7e7431, to the specification, code and tests. Each required change was checked against its source before it was applied: the Report's lists of tables and printed pp. 19-24 (the range RESTRICTED-FILES.md clears), the PSID codebooks and formats, the eCFR text of 20 CFR 416.1112, 416.1121, 416.1124, 416.1163 and 416.1205, the nine SSA COLA notices in the Federal Register, the committed life tables, and a design count on the staged individual file. No income concept, threshold or poverty statistic was computed on PSID data; the specification stays a draft (d189, the fallback rule and ratification pending). - adjusted_poverty: retirement_account_income_rule (F4a, default remove_head: HEAD ANNUITIES, "Head's Income from Annuities and IRAs" in 2005-2011, plus HEAD IRAS in 2013, removed under replace); farm_asset_share (F4b, default 0.5 of positive farm income and the whole loss); annuity_lives defaults to fu_head_rule, priced on the head and the head's co-resident legal spouse (code 20 or 90); 0.02 is the real-rate sensitivity; pending decisions updated. - age67: unresolved_marital_status defaults to relationship_code (R9) and annuitant_age_source to derived_birth_year, with the birth-year law extended to in-family heads and spouses outside the universe (R8); row U0-F (1941, 1943, 1945); head and head-spouse annuitant columns, marital_resolution and age sources; threshold-free counts of farm and head retirement-account income in the structural summary. - uniform_cut_tabulation: the design SE is a domain estimator on the full sample design (design_se_domain; z = 0 outside the cell; R10). - uniform_cut_track_u: row U0-F and the fallback headline rule (rows that need a wave without WEALTH1 are reported blocked with counts); the runner's design frame; NAMED_DELTAS held equal to section 12; the invented generator adds relationship-code couples, code-90 husbands, farm, annuity and IRA income, and interview ages. - run_track_u_registered.py: --headline-row must equal the fallback rule's row for the staged PSID; the dry run reports the fallback as staged today. family_income notes the 2005-2011 HEAD ANNUITIES label. - Specification u1-draft-4 (the referee asked for draft-3, already used on this branch): sections 1-19 revised, new 10a (comparison and acceptance); section 17 records the pass: R1-R17 applied, R11 in part (row U-inst kept, since u1-draft-3 gave it an income rule), O1 not adopted with reasons, open questions 9-14. - Tier counts recollected (7,434). Co-Authored-By: Claude Opus 5.5 --- .../boomers2004_uniform_cut_comparison.md | 989 ++++++++++++------ scripts/run_track_u_registered.py | 49 +- scripts/track_u_dry_run.py | 66 +- scripts/track_u_structure.py | 9 +- src/populace_dynamics/cohorts/age67.py | 448 +++++++- src/populace_dynamics/data/family_income.py | 6 +- .../estimates/adjusted_poverty.py | 244 ++++- .../estimates/uniform_cut_tabulation.py | 119 ++- .../uniform_cut_track_u/invented.py | 130 ++- .../uniform_cut_track_u/rows.py | 58 +- .../uniform_cut_track_u/runner.py | 201 +++- tests/README-tiers.md | 6 +- tests/cohorts/test_age67.py | 270 ++++- tests/estimates/test_adjusted_poverty.py | 250 ++++- .../estimates/test_uniform_cut_tabulation.py | 132 ++- tests/test_boomers2004_uniform_cut_spec.py | 208 +++- tests/tier_counts.json | 4 +- tests/track_u/test_dry_run_script.py | 12 + tests/track_u/test_invented_generator.py | 28 + tests/track_u/test_invented_pipeline.py | 1 + tests/track_u/test_registered_script.py | 39 + tests/track_u/test_rows.py | 29 +- tests/track_u/test_runner.py | 76 +- 23 files changed, 2813 insertions(+), 561 deletions(-) diff --git a/docs/design/boomers2004_uniform_cut_comparison.md b/docs/design/boomers2004_uniform_cut_comparison.md index 39779bdd..6e5220bc 100644 --- a/docs/design/boomers2004_uniform_cut_comparison.md +++ b/docs/design/boomers2004_uniform_cut_comparison.md @@ -1,12 +1,15 @@ # Boomers 2004 uniform cut comparison: specification for DynaSim scorecard exercise 2 -- **Status:** draft for the referee. Nothing here is ratified. Max has not - ruled on exercise 2 (cos decision d189, open, default "yes to Track U - with the proposed SSI rule; downloads needed from Max"). Every choice - that awaits him or the specification freeze is an explicit parameter in - the code with the plan's recommended default, and §16 lists them. +- **Status:** draft for the referee. Nothing here is ratified. One + referee pass is recorded (§17); this draft applies it and awaits a + second pass on what the first did not see. Max has not ruled on + exercise 2 (cos decision d189, open, default "yes to Track U with the + proposed SSI rule; downloads needed from Max") or on the fallback rule + (§11). Every choice that awaits him or the specification freeze is an + explicit parameter in the code with the recommended default, and §16 + lists them. - **Specification:** `boomers2004_uniform_cut_exercise2`, version - `u1-draft-3`, drafted 2026-09-24. §19 is the changelog. + `u1-draft-4`, drafted 2026-09-24. §19 is the changelog. - **Plan item:** U1 of the Track U plan, `EVID/critical-path-uniform-cut-20260923.md` (§7 fields F1–F17, §8 work items), where `EVID` = @@ -20,41 +23,73 @@ projection)*; *Python income concept (not Axiom)*; *mechanical incidence* (`adjusted_poverty.OUTPUT_LABELS`; the tabulation refuses labels without them). -- **Builder boundary:** a model-builder lane wrote this draft. It did not - open the Report PDF (`REFS/900767-how-will-boomers-fare.pdf`), any - results table of the Report, the exercise-2 or COLA comparator - directories, any comparator seal or reconciliation file, the Track A - comparison memo, the issue #42 result comments, or page 3 / Figure 2 of - the Urban 2010 report. Every statement below about the Report is the - plan's reading of its methods section (plan §2), cited as "plan §2". - §18 lists what this draft read and what it did not verify. +- **Builder boundary:** model-builder lanes wrote this draft. Drafts 1–3 + did not open the Report PDF (`REFS/900767-how-will-boomers-fare.pdf`) + and cited the plan's reading of its methods (plan §2). For + `u1-draft-4`, to verify the referee's changes, the builder read the + Report's table of contents, list of tables and list of appendix tables + (PDF pages 5–7) and Section III, printed pages 19–24 (PDF pages 20–25), + the range `EVID/RESTRICTED-FILES.md` clears; the page mapping (printed + *n* = PDF *n* + 1) was checked from the footer digits of PDF pages 20 + and 25 alone. Statements checked there now cite the Report page. No + lane opened printed page 25 or later for this draft, any results table + of the Report, the exercise-2 or COLA comparator directories, any + comparator seal or reconciliation file, the Track A comparison memo, + the issue #42 result comments, or page 3 or later of the Urban 2010 + report. Through the plan, this draft inherits two things the plan + disclosed or used: a qualitative fragment of printed page 41 (late + boomers; no number; no exercise-2 cell), shown when the plan mapped + page numbers (plan §14), and the plan's reading of printed page 25 + (reporting splits, 2003 dollars). `RESTRICTED-FILES.md` names printed + pp. 19–24 as the readable methods range, so page-25 readings are cited + here only where another readable source confirms them (the cells now + cite the list of appendix tables). §18 lists what this draft read and + what it did not verify. ## 1. Target The target is the adjusted poverty rate at age 67 in Butrica and Uccello, *How Will Boomers Fare at Retirement?* (AARP Public Policy -Institute #2004-05, May 2004), as the plan reads its methods and list of -tables (plan §2): +Institute #2004-05, May 2004): - **Tables:** 19, "Adjusted Poverty Rates at Age 67", and 21, "Adjusted Poverty Rates at Age 67, Assuming 13 Percent Reduction in Social - Security Benefits". The comparator is the change from Table 19 to - Table 21. Which rows those tables carry is unknown to this builder - (plan §2 and §10, decision 2(b)). -- **Cohort:** persons born 1936–1945 ("near-retirees"), each analysed in - the year they reach 67. -- **Model:** DYNASIM, from the 1990–93 SIPP, with the intermediate - assumptions of the 2002 Trustees Report. -- **Income concept:** the Report's concept differs from Census money - income: it imputes income from financial assets as the real annuity - that 80 percent of financial assets would buy, with a 50 percent - survivor annuity for couples (plan §2, citing Report p. 24 and fn. 10). -- **Threshold:** official Census poverty thresholds, "65-and-over" - (plan §2, citing Report p. 24). -- **Cut:** a 13 percent reduction in Social Security benefits (plan §2, - citing Report p. 22). Its start year, coverage, SSI treatment and any - behavioural response are not in the methods section (plan §2 and §10, - decisions 2 and 8). + Security Benefits" (list of tables, printed p. 5). The comparator is + the change from Table 19 to Table 21. Which rows those tables carry is + unknown to this builder (plan §10, decision 2(b)). +- **Cohort:** persons born 1936–1945 ("near-retirees"), each analysed + when they reach 67 (p. 24). +- **Model:** DYNASIM, from a sample of about 100,000 individuals from the + 1990–93 SIPP, aged in yearly steps to 2050 (p. 20), with OCACT + projections from the intermediate scenario of the 2002 Trustees Report + for mortality, wage growth and prices (p. 20, fn. 4). +- **Unit and income concept:** results are reported for individuals, but + each individual's income and wealth are the household's, including the + spouse's if married (p. 24). The income resources projected at 67 + include earnings, SSI and co-resident income, meaning income of + non-spouse co-resident family members (p. 19). Whether co-residents + count in family size is not stated in pp. 19–24. The Report's income + differs from Census money income by imputing income from financial + assets as the real (price-indexed) annuity that 80 percent of those + assets would buy (p. 24), recalculated each year, with a 50 percent + survivor annuity for married couples (fn. 10). The Census measure, by + contrast, counts interest and dividends, and counts retirement-account + income only when annuitized or otherwise withdrawn (p. 24; fn. 11 + lists its sources). +- **Financial assets:** non-pension wealth (vehicle, other real estate, + farm and business equity, stock, mutual fund and bond values, and + checking, savings, money market and certificate of deposit balances, + less unsecured debt; p. 22) plus IRA, Keogh and 401(k) balances + (p. 24). Home equity is projected separately (p. 23) and is not among + the financial assets (pp. 22, 24). +- **Threshold:** "the official poverty thresholds of the U.S. Census + Bureau", which "vary with family size and age"; "we use the + 65-and-over poverty threshold" (p. 24). +- **Cut:** a 13 percent reduction in Social Security benefits: OCACT + "estimates that benefits would need to be reduced immediately by 13 + percent" (p. 22, citing the 2003 Trustees Report). Its start year, + coverage, SSI treatment and any behavioural response are not in + pp. 19–24 (plan §10, decisions 2 and 8). The comparator values are sealed on the comparator side. This builder has not seen them and does not know which cells they cover. @@ -63,16 +98,18 @@ has not seen them and does not know which cells they cover. | Source | Use | Status | |---|---|---| -| Plan, `critical-path-uniform-cut-20260923.md` | Every field proposal (§7), the Report's methods as the plan read them (§2) | Read in full | -| PSID family files 2005–2013 (`psid-data/family//FAMER.*`) | Family income components, Social Security, SSI, asset income, FU composition, PSID's needs standard; WEALTH1 in 2009–2013 | Staged; labels verified by `data/family_income.py` | +| Plan, `critical-path-uniform-cut-20260923.md` | Every field proposal (§7) | Read in full | +| The Report: table of contents, lists of tables and appendix tables (PDF pp. 5–7); Section III, printed pp. 19–24 (PDF pp. 20–25) | §1, §4, §5 citations (u1-draft-4) | Read; nothing later | +| Referee report `EVID/boomers2004-referee-20260924.md` | §17 | Read in full; every required change checked against its source | +| PSID family files 2005–2013 (`psid-data/family//FAMER.*`) | Family income components, Social Security, SSI, asset income, the head's annuity and IRA income, farm income, FU composition, PSID's needs standard; WEALTH1 in 2009–2013 | Staged; labels verified by `data/family_income.py`; codebook entries for `HEAD ANNUITIES`, `HEAD IRAS`, `WIFE RETIREMENT/ANNUITIES`, `OTHER FU MEMBR RETIREMENT/ANNUITIES`, ER52214, ER52216, ER52368 and W21, W33 read | | PSID wealth supplements 2005 and 2007 | WEALTH1 for the 1937 and 1939 birth years (U0) and six U1 observation cells | **Not staged.** The reader refuses with the missing files named (§3) | -| PSID cross-year individual file (`ind2023er`) | Anchors per wave, cross-section weights, relationship to head, age, sampling-error stratum and cluster | Staged; labels verified by `cohorts/age67.py` | +| PSID cross-year individual file (`ind2023er`) | Anchors per wave, cross-section weights, relationship to head, age, sampling-error stratum and cluster | Staged; labels verified by `cohorts/age67.py`, including relationship code 90 ("Legal husband of Head") | | PSID marriage history (`mh85_23`) and earnings panel | Birth-year law and marital status | Existing readers | | NCHS United States Life Tables, 2000 | Annuity mortality (F8) | Committed, `data/external/nchs_life_tables_2000.json`, SHA-256 pinned | | SSA period life table for 2004 (2008 vintage) | Row U9 | Committed, `data/external/tr2008/ssa_2008_vintage.json` | -| SSI federal benefit rates 2004–2012, exclusions, resource limits | SSI response (F13); F17 SSI diagnostic | Captured from policyengine-us revision `a03e82e503`: `data/external/track_u_ssi_parameters.json`, SHA-256 `79e641a1…3c523` | +| SSI federal benefit rates 2004–2012, exclusions, resource limits | SSI response (F13); F17 SSI diagnostic | Captured from policyengine-us revision `a03e82e503`: `data/external/track_u_ssi_parameters.json`, SHA-256 `79e641a1…3c523`; checked against the Federal Register and eCFR (§8) | | Census poverty thresholds 2004–2012 | Threshold (F10) | **Not captured** (§6; cos decision d194) | -| SSA, *Annual Statistical Supplement, 2025*, Table 5.A4 (December beneficiaries and total monthly benefits by type, 1940–2024) | F17 Social Security diagnostic | Committed snapshot `data/external/snapshots/ssa_level_anchors_vintage1/supplement2025_5a.html`, SHA-256 `d61e9484…aa8e` (the capture manifest's) | +| SSA, *Annual Statistical Supplement, 2025*, Table 5.A4 | F17 Social Security diagnostic | Committed snapshot `data/external/snapshots/ssa_level_anchors_vintage1/supplement2025_5a.html`, SHA-256 `d61e9484…aa8e` | | Survey of Consumer Finances wealth tables | F17 WEALTH1 comparator (plan §11) | **Not committed or saved**: WEALTH1 is summarized without a published comparison | | PSID 2011 User Guide §2.4; individual-file codebook (ER34102, ER34103, ER34137–ER34143); family-file codebook (`# IN FU`, the individual-record count) | The institution income rule (§3) | Staged documentation, read for the rule | @@ -95,37 +132,47 @@ even income years exist; an odd birth year turns 67 in an even year. even birth year 1938–1944 observed at 66 and at 68 (income years *b* + 66 and *b* + 68) with half weight each; 1936 observed at 68 only (income year 2004; its age-66 year, 2002, is in the 2003 wave, which carries -aggregates only, plan §3) with weight 1. The weight for 1936 is a builder -choice: the plan says only "1936 from 2004 only". +aggregates only, plan §3) with weight 1. With weight 1 every birth year +enters U1 with one cross-section's worth of weight: even birth years from +two half-weighted observations, the odd years and 1936 from one +full-weighted observation; 0.5 would halve 1936's share of the cohort +(referee Q4). The 1936 observation is at 68, so it omits 1936-born +persons who died between 67 and 68, a survivor effect inside the +exact-age named delta. + +**Fallback (U0-F): 1941, 1943 and 1945.** U0 restricted to the birth +years whose waves carry WEALTH1 in the family file; the blocked 1937 and +1939 are left out and counted. §11's fallback rule decides whether U0 or +U0-F is the headline, by staging status only. **Universe per wave (F2).** Sequence 1–20 (in a responding family) and a positive core/immigrant cross-section weight. The cross-section weight -is also positive for movers-out (71–80) and decedents (81–89); neither -option admits them and the dispositions count them. Registered option -U-inst adds sequence 51–59 (institutions). - -**Institution income rule (U-inst; builder default, pending the -referee).** The plan gives institutionalized persons no income rule, and -the PSID collects no income for them: the individual-file Social -Security items are "Inap.: … in an institution" (codebook, ER34137– -ER34143 for 2011). The PSID does associate them with a family: when a -sample member moves to an institution it "attaches an institutional -status data record to the family they left" (2011 User Guide, §2.4), and -the family file's record-count variable counts "any institutionalized -individuals associated with the family" among the records "having the -same family-level data". The default `institution_income_rule = +is also positive for movers-out (71–80) and decedents (81–89); no row +admits them and the dispositions count them. Registered row U-inst adds +sequence 51–59 (institutions). + +**Institution income rule (U-inst; builder default of `u1-draft-3`, +pending the freeze, not yet refereed).** The plan gives +institutionalized persons no income rule, and the PSID collects no +income for them: the individual-file Social Security items are "Inap.: +… in an institution" (codebook, ER34137–ER34143 for 2011). The PSID does +associate them with a family: when a sample member moves to an +institution it "attaches an institutional status data record to the +family they left" (2011 User Guide, §2.4), and the family file's +record-count variable counts "any institutionalized individuals +associated with the family" among the records "having the same +family-level data". The default `institution_income_rule = family_of_record` therefore gives an institutionalized observation the family unit whose interview number its record carries: that family's -income concept, wealth, size, children and threshold, with the member's -own age and sex for a single-life annuity (it has no co-resident -spouse). Its role is OFUM: the relationship code of a person in an -institution is to the previous wave's head (codebook note on ER34103), -so it is neither the current head nor the wife. The member's own income -is missing from the family's, and `# IN FU` ("the actual number of -persons currently in the FU") is read as not counting them; the F17 -counts in §14 check that reading on the staged files. Both are named -deltas (§12). An institutionalized person whose interview number has no -family-file record that wave is a disposition +income concept, wealth, size, children and threshold, and (under +`fu_head_rule`, §5) that family's annuity. Its role is OFUM: the +relationship code of a person in an institution is to the previous +wave's head (codebook note on ER34103), so it is neither the current +head nor the wife, and it has no co-resident spouse. The member's own +income is missing from the family's, and `# IN FU` ("the actual number +of persons currently in the FU") does not count them (§14). Both are +named deltas (§12). An institutionalized person whose interview number +has no family-file record that wave is a disposition (`institution_family_of_record_missing`). The alternative `excluded` keeps institutionalized persons out of the universe (disposition `institution_excluded_by_rule`), which makes U-inst's population U0's. @@ -144,19 +191,29 @@ year, times the U1 multiplier. No reweighting or alignment. splits on family units linked through shared persons (§10). **Attached per observation** (`cohorts/age67.build_age67_cohort`): sex -(ER32000 through the death-record reader), relationship to head (10 -head, 20 legal wife, 22 cohabiting "wife", otherwise OFUM; the codes are -verified in `IND2023ER_formats.sas` for each wave), marital status at the -end of the income year from the marriage history (F12; separated counts -as married; a member whose history cannot date the state, `unknown`, or -who has no marriage-history record, `no_marriage_history`, counts as -non-married, `unresolved_marital_status`, a pending builder choice), -whether a legal spouse lives in the same family unit, the -spouse's age and sex, whether the family has a legal wife, and the -sampling-error stratum and cluster (ER31996, ER31997). +(ER32000 through the death-record reader); relationship to head (10 +head, 20 legal wife, 22 cohabiting "wife", 90 legal husband of head, +otherwise OFUM; the codes are verified in `IND2023ER_formats.sas` for +each wave); marital status at the end of the income year from the +marriage history (F12; separated counts as married); whether a legal +spouse lives in the same family unit, and the spouse's age and sex; the +family head and the head's co-resident legal spouse (code 20 or 90), +with their ages and sexes (the annuitants of §5); and the sampling-error +stratum and cluster (ER31996, ER31997). A member whose history cannot +date the state (`unknown`), or who has no marriage-history record +(`no_marriage_history`), is classified by +`unresolved_marital_status="relationship_code"` (pending the freeze; +referee Q7): married and co-resident when the member is head (10) with a +co-resident legal spouse (20 or 90), or is the head's legal wife (20) or +legal husband (90); otherwise non-married. The alternative `non_married` +classifies them all as non-married (the `u1-draft-3` rule). Each +resolution path is counted (`marital_resolution`). The marital pairs +indicator of the individual file is not used. An institutionalized +member is never resolved by code: its relationship code is to the +previous wave's head. **Structural counts** (staged PSID, counts only; `scripts/track_u_structure.py`, -evidence `EVID/track-u-structure-20260924/`): +evidence `EVID/track-u-structure-20260924/`, code of `u1-draft-1`): | Row | Observations | Persons | Computable now | Blocked by the 2005/2007 wealth supplements | |---|---|---|---|---| @@ -164,10 +221,16 @@ evidence `EVID/track-u-structure-20260924/`): | U1 | 1,349 | 970 | 857 | 492 | U0 by birth year: 1937 81, 1939 82, 1941 93, 1943 116, 1945 111 -observations. The rest of each birth year's universe persons were not -present at the observation wave (sequence 0, moved out or died); the -evidence file lists every disposition. No income, threshold or poverty -status was computed. +observations; U0-F is the last three, 320. The rest of each birth year's +universe persons were not present at the observation wave (sequence 0, +moved out or died); the evidence file lists every disposition. Of the +483 U0 observations, 35 are OFUMs (95 of 1,349 under U1), 457 have +marriage-history birth years, and 35 have an unresolved marital state +(26 `no_marriage_history`, 9 `unknown`), which `u1-draft-1` counted as +non-married. The counts under this draft's rules (resolution paths, +annuitant age sources, farm and head retirement-account income) are +recollected by the same script; §14 records them. No income, threshold +or poverty status was computed. ## 4. Income concept @@ -187,19 +250,48 @@ and wife rent, dividends, interest, trusts/royalties and the asset part of unincorporated-business income, and, under the family-unit basis, the OFUM total asset income. Rent, business asset income and the OFUM total can be negative (losses, per the codebooks); removing a loss raises -income. Row U5: keep reported asset income and add the annuity. - -Finding while building the reader: `LABOR INCOME OF HEAD` and `OF WIFE` -exclude farm income and the labor part of business income. Head and wife -taxable income equals labor income plus farm income plus the head's and -wife's labor part of business income plus the ten asset items, exactly -for every family in 2007, 2011 and 2013, within $10 in 2009, and for all -but two 2005 families. The codebook says farm income "includes both labor -and asset portions"; its asset portion is not separated, so the primary -does not replace it (named delta, §12). The PSID splits a working owner's -business income equally between labor and asset parts and codes a whole -business loss in the asset part (codebook text for ER52216); the asset -part is what F4 removes. +income. Row U5: keep reported asset income (and the F4a and F4b items) +and add the annuity. + +**Retirement-account income (F4a, pending the freeze).** +`retirement_account_income_rule="remove_head"` (default): under +`replace`, the head's annuity and IRA income is removed with the asset +income. The item is `HEAD ANNUITIES`, "Head's Income from Annuities and +IRAs" in the 2005, 2007, 2009 and 2011 codebooks (ER27964, ER40954, +ER46862, ER52270); the 2013 file splits it into `HEAD ANNUITIES` +(ER58071, "Head's income from annuities") and `HEAD IRAS` (ER58073), and +both are removed. The balances that produce this income are in WEALTH1 +(W21/W22, "private annuities or Individual Retirement Accounts") and +annuitized under F5–F6, and the Report contrasts its annuity with Census +income, which counts retirement-account income only when annuitized or +withdrawn (p. 24). The wife's item before 2013 (`WIFE +RETIREMENT/ANNUITIES`, "Income from Pensions and Annuities") and the +OFUMs' (`OTHER FU MEMBR RETIREMENT/ANNUITIES`, "Other Retirement, +Pensions, and Annuities") combine pensions with annuity income and are +kept in every wave, including 2013, so the rule is the same across waves +(named delta, §12). Alternative `keep`: no removal (the `u1-draft-3` +behaviour). + +**Farm income (F4b, pending the freeze).** `LABOR INCOME OF HEAD` and +`OF WIFE` exclude farm income and the labor part of business income. +Head and wife taxable income equals labor income plus farm income plus +the head's and wife's labor part of business income plus the ten asset +items, exactly for every family in 2007, 2011 and 2013, within $10 in +2009, and for all but two 2005 families. Farm income (ER52214 in 2011, +"Head's and Wife's Income from Farming") "includes both labor and asset +portions of income"; its asset portion is not separated. F4 applies the +PSID's own business convention to it (`farm_asset_share`, default 0.5): +half of positive farm income, and the whole of a farm loss, count as +asset income and are removed under `replace`. The convention is the +codebook's for business income: "Total business income of the Head is +equally split between labor and asset income when the Head put in actual +work hours", and "If total farm or business income represents a loss …, +then the labor portion equals 0 and the loss is coded in the asset +portion" (ER52216). The Report's financial assets include farm equity +(p. 22), so its income is income from financial assets. Alternative 0: +farm income stays whole in income (the `u1-draft-3` behaviour). The +number of cohort observations with nonzero farm income is recollected +with the structural counts (§14). **Financial assets (F5).** The family unit's WEALTH1, "IMP WEALTH W/O EQUITY": the codebooks define it as seven asset values (farm or business, @@ -207,72 +299,119 @@ checking and savings, other real estate, stocks, vehicles, other assets, IRAs and annuities) net of debts (2009: one "other debt" item; 2011: five debt types; 2013: farm/business and real-estate debt split out plus six debt types). On the staged files the component identity holds exactly -for every family in 2009, 2011 and 2013. Row U7 (employer DC balances -outside IRAs) needs a label investigation and is not built. +for every family in 2009, 2011 and 2013. The Report's financial assets +include 401(k) balances (p. 24); WEALTH1's seven asset types do not +include employer DC balances held outside IRAs, so the primary +understates financial assets for families that hold them (named delta, +§12). Row U7 adds them where the PSID observes them (P-section items, +label investigation pending). U7 must be built, or removed from §11, +before registration; if the investigation shows the items are reliable +for 66–68-year-olds before the freeze, the freeze should consider making +WEALTH1 plus employer DC balances the primary (the Report's definition) +and WEALTH1 alone the alternative. ## 5. Annuity - **Amount:** `0.8 × max(WEALTH1, 0) / price` (F6, F9). - **Price** of a real level annuity of 1 per year, annuity-immediate, - 3 percent real, no load (F7): single life - `a_x = Σ_{t≥1} v^t ₜp_x`; joint and 50 percent survivor, paying 1 + 3 percent real, no load (F7). The 3 percent is the real return on + government bonds that DYNASIM assumes after 2001 for DC and IRA + accumulation (p. 22); the same paragraph subtracts 1 percentage point + from stock and bond real returns for administrative costs, so the net + government-bond return there is 2 percent. Neither is stated to be + DYNASIM's annuity-pricing rate. The primary uses the gross 3 percent; + `real_interest_rate=0.02` is a reported, unscored sensitivity. Single + life `a_x = Σ_{t≥1} v^t ₜp_x`; joint and 50 percent survivor, paying 1 while both live and 0.5 to the survivor, with independent lives: `Σ_{t≥1} v^t (s·ₜp_x + s·ₜp_y + (1 − 2s)·ₜp_x·ₜp_y)`, which is `0.5·(a_x + a_y)` at s = 0.5. -- **Lives (F6 as written, `annuity_lives="member_rule"`):** joint on the - member and a co-resident legal spouse if the member is married; single - life on the member's age and sex otherwise. The plan does not say whose - lives price an OFUM member's family wealth; the alternative - `fu_head_rule` prices on the family head and a co-resident legal wife. - Ages: the member's income-year age (income year − birth year); the - spouse's individual-file age at the wave, which the repository's - birth-year law treats as the income-year age. +- **Lives (F6, `annuity_lives="fu_head_rule"`, pending the freeze; + referee Q1):** the family unit's WEALTH1 is priced on the lives of its + main owners: joint on the family head and the head's co-resident legal + spouse (a legal wife, code 20, or a legal husband, code 90), single + life on the head otherwise. For heads and legal spouses this equals + pricing on the member and a co-resident legal spouse; it differs for + OFUM members and cohabiting partners (code 22), whose family's wealth + would otherwise be annuitized over their own lives (alternative + `member_rule`, F6 as the plan wrote it). For example (NCHS 2000, 3 + percent), a 67-year-old OFUM woman's single-life price is 12.5868, + while a family headed by a couple aged 40 (man) and 38 (woman) has a + joint price of 22.0299, so `member_rule` would give that family's + wealth an annuity 1.75 times as large. +- **Ages (`annuitant_age_source="derived_birth_year"`, pending the + freeze; referee Q8):** every annuitant's income-year age, income year + minus derived birth year, from `estimates.career.derive_birth_years` + extended beyond the universe to in-family heads, legal wives, + cohabitors (22, 88), legal husbands and marriage-history spouses of + cohort members (typically zero-weight nonsample spouses; their seed + coordinate is the earliest of the five waves in which they are in a + family). Where the law leaves a person unresolved (including + conflicting marriage-history birth years, which the extension leaves + out rather than failing the build), the individual-file age at the + wave is used and counted (`wave_age`). The law gives the + marriage-history birth year precedence and treats `(wave − 1) − age` + as the birth year only in its last clause; for people dated by + marriage history, the wave age can exceed the income-year age by one + year (NCHS 2000, 3 percent: m67 + f67 costs 11.7610, m67 + f68 + 11.5487, a 1.8 percent larger annuity). Alternative `wave_age`: every + annuitant at the individual-file age of the wave. - **Mortality (F8):** NCHS 2000 by sex. Survival stops with the table: the open interval "100 years and over" (qx = 1) is treated as death within the year at 100 (builder choice). Row U9: the SSA period life table for 2004 (ages 0–119). - **Negative WEALTH1 (F9):** buys no annuity and is not subtracted. -Annuity prices computed by this draft from the committed tables (3 -percent, immediate; not PSID data, not comparator values): +Annuity prices computed from the committed tables (immediate; not PSID +data, not comparator values; the 3 percent table recomputed by the +referee in independent code): -| Age | NCHS 2000 male | NCHS 2000 female | SSA 2004 male | SSA 2004 female | -|---|---|---|---|---| -| 66 | 11.3375 | 13.0092 | 11.6068 | 13.2336 | -| 67 | 10.9353 | 12.5868 | 11.1936 | 12.8042 | -| 68 | 10.5344 | 12.1622 | 10.7802 | 12.3725 | +| Age | Rate | NCHS 2000 male | NCHS 2000 female | SSA 2004 male | SSA 2004 female | +|---|---|---|---|---|---| +| 66 | 3% | 11.3375 | 13.0092 | 11.6068 | 13.2336 | +| 67 | 3% | 10.9353 | 12.5868 | 11.1936 | 12.8042 | +| 68 | 3% | 10.5344 | 12.1622 | 10.7802 | 12.3725 | +| 67 | 2% | 11.9261 | 13.8562 | 12.2165 | 14.1120 | Joint and 50 percent survivor, male 67 and female 67: 11.7610 (NCHS -2000), 11.9989 (SSA 2004). +2000, 3%), 11.9989 (SSA 2004, 3%), 12.8912 (NCHS 2000, 2%), 13.1643 (SSA +2004, 2%). ## 6. Thresholds **Primary (F10):** the Census weighted-average poverty threshold of the income year for the unit's size, using the "65 years and over" rows for -sizes 1 and 2 whatever the householder's age (the Report uses the -65-and-over threshold, plan §2); sizes 3 to 8 by size; 9 or more share one -row. Poor means income below the threshold. +sizes 1 and 2 whatever the householder's age (the Report uses "the +65-and-over poverty threshold", p. 24); sizes 3 to 8 by size; 9 or more +share one row. Poor means income below the threshold. The capture uses +the weighted averages as printed in the historical workbooks +(`threshYY.xlsx`); the Census historical-thresholds page notes that +these may differ from Historical Poverty Table 1 because of later +weight-control updates (the referee's reading of the page; not re-read +for this draft). **Row U10:** the Census size-by-related-children matrix, with the householder-65-and-over rows for sizes 1 and 2; PSID `# CHILDREN IN FU` -(persons under 18 other than head and wife) stands in for related -children. **Row U8:** PSID's `CENSUS NEEDS STANDARD-`, which -the codebook describes as the Census matrix threshold by family size, -number under 18 and householder age, adjusted for composition changes -during the year (householder age, not the 65-and-over rule). +stands in for related children, although it counts all persons under 18 +other than head and wife, related or not. **Row U8:** PSID's `CENSUS +NEEDS STANDARD-`, which the codebook describes as the Census +matrix threshold by family size, number under 18 and householder age, +adjusted for composition changes during the year (householder age, not +the 65-and-over rule). **Capture status: not captured.** The Census historical thresholds page lists one spreadsheet per year, `thresh04.xlsx` … `thresh12.xlsx`, under `https://www2.census.gov/programs-surveys/cps/tables/time-series/historical-poverty-thresholds/`. -This builder read that page's link list and did not download the files -(downloads need Max's approval in this environment). Once the nine files -are staged in a directory, `scripts/capture_track_u_parameters.py ---census-dir DIR` parses them by their printed labels, checks that the -65-and-over rows sit below the under-65 rows and that thresholds rise with -size, records each file's URL, bytes and SHA-256, and writes +The files have not been downloaded (downloads need Max's approval in +this environment, cos decision d194). Once the nine files are staged in +a directory, `scripts/capture_track_u_parameters.py --census-dir DIR` +parses them by their printed labels, checks that the 65-and-over rows +sit below the under-65 rows and that thresholds rise with size, records +each file's URL, bytes and SHA-256, and writes `data/external/census_poverty_thresholds_2004_2012.json`; the pin `adjusted_poverty.THRESHOLDS_SHA256` must then be set. The parser has run -only on invented workbooks; the first real capture must be checked by eye. +only on invented workbooks, and the labels it expects ("65 years and +over", "Householder 65 years and over") have not been checked against +the real 2004–2012 files; the first real capture must be checked by eye. Until then `load_poverty_thresholds` refuses and no primary threshold can be assigned. @@ -283,8 +422,9 @@ Security income of the unit (head, wife and OFUM amounts under the family basis) in the income year. No behavioural response. Every U0 income year is 2004 or later, so the unread start year matters only for U1's 1936 row: row U6 (on U1) leaves 1936 uncut if the cut starts in 2004. The -scorecard's "from 2004" has no source this builder could read (plan §10, -decision 8). +Report's methods say "reduced immediately" (p. 22); the scorecard's +"from 2004" has no source this builder could read (plan §10, decision +8). **Row U6 (built; the start year awaits decision 8).** `AdjustedPovertySpec.cut_start_year` (default `None`: every observation @@ -316,9 +456,18 @@ is understated (named delta, §12). otherwise; its Social Security includes the spouse's (deeming approximated by full attribution; alternative `recipients_only`). All OFUMs together form one individual unit (PSID reports one OFUM total). + Under 20 CFR 416.1163(d), nothing is deemed when the ineligible + spouse's income after allocations is at most the couple-minus-individual + FBR (2004: $846 − $564 = $282 a month), and otherwise the couple + computation applies, capped (416.1163(e)(2)) at the benefit without + deeming. Full attribution can therefore only overstate the offset; + `recipients_only` errs the other way; the two bracket the rule (referee + Q3). The cap `12·FBR_individual − SSI` is looser than the regulation's + individual-computation cap but binds only in the no-deeming case. - **Row U2:** no SSI response. -- **Row U3 (upper bound):** the offset, plus take-up by every head/wife - unit that the cut makes newly income-eligible and whose resource proxy +- **Row U3 (the largest SSI response of the three registered rules, so + the smallest Δ):** the offset, plus take-up by every head/wife unit + that the cut makes newly income-eligible and whose resource proxy `max(0, WEALTH1 − vehicles)` is within the resource limit. Countable income follows 20 CFR 416.1112 and 416.1124 in order: the $20 exclusion applies to unearned income first, then the $65 earned exclusion and half @@ -327,22 +476,35 @@ is understated (named delta, §12). asset income is left out; earned income is labor, farm and business labor income. A unit with a wife present is treated as a couple. OFUMs are never newly enrolled. Deterministic (every eligible unit takes up), - so K stays 1. - -SSI parameters (captured from policyengine-us; the files cite SSA's "SSI -Federal Payment Amounts" and 20 CFR 416.1112 and 416.1205): FBR in force -on January 1, monthly, individual / couple — 2004 564 / 846; 2005 579 / -869; 2006 603 / 904; 2007 623 / 934; 2008 637 / 956; 2009–2011 674 / -1,011; 2012 698 / 1,048. General exclusion $20, earned exclusion $65, -half of remaining earnings excluded, resource limits $2,000 / $3,000. -This builder did not check these against ssa.gov (the plan records that -ssa.gov refuses programmatic fetches). + so K stays 1. It is not a bound on DYNASIM's simulation: the referee + notes that the resource proxy nets debts and drops every vehicle where + 20 CFR 416.1218 excludes one, and that SSI treats unmarried cohabitors + as a couple only when they hold themselves out as married (416.1806); + those two sections were not re-read for this draft. + +SSI parameters (captured from policyengine-us revision `a03e82e503`, +`data/external/track_u_ssi_parameters.json`): FBR in force on January 1, +monthly, individual / couple — 2004 564 / 846; 2005 579 / 869; 2006 603 +/ 904; 2007 623 / 934; 2008 637 / 956; 2009–2011 674 / 1,011; 2012 698 / +1,048. These equal the SSA notices "Cost-of-Living Increase and Other +Determinations" for each year: 68 FR 60437, 69 FR 62497, 70 FR 61677, +71 FR 62636, 72 FR 60703, 73 FR 64651, 74 FR 55614 and 75 FR 65696 (no +increase in 2010 or 2011), 76 FR 66111 (checked by the referee and again +for this draft on govinfo.gov, 2026-09-24; ssa.gov refuses programmatic +fetches). The $20 general exclusion applies to unearned income other +than income based on need (20 CFR 416.1124(c)(12), current text), with +any remainder applied to earnings (416.1112(c)(4), whose current text +cross-references the exclusion as "§ 416.1124(c)(10)"); then $65 and one +half of the rest of earnings (416.1112(c)(5), (c)(7)). Interest and +dividends on countable resources are excluded (416.1124(c)(22), current +text). Resource limits $2,000 / $3,000 from January 1989 (416.1205(c)). +Deeming from an ineligible spouse: 416.1163. ## 9. Statistic For cell c, with observation weights w, baseline adjusted income B, -reform income R and threshold T (all family-level, assigned to each -member observation): +reform income R and threshold T (family-level under `fu_head_rule`: +every member of a family unit gets the same B, R and T): ```text P_B[c] = 100 × Σ_{i∈c} w_i·1{B_i < T_i} / Σ_{i∈c} w_i @@ -351,18 +513,24 @@ P_R[c] = 100 × Σ_{i∈c} w_i·1{R_i < T_i} / Σ_{i∈c} w_i ``` Δ is the headline; P_B and P_R are the secondary rows (the Table 19 and -Table 21 analogues). +Table 21 analogues). Under row U4, head and wife members use the +head-and-wife basis and OFUM members the family basis, so B and T differ +by role there. **Cells.** `all` (headline); `men`, `women`, `married`, `non_married` -(the Report's stated splits, plan §7); the four sex-by-marital cells -(optional); one diagnostic cell per birth year (not scored). Only cells -present in the sealed comparator are scored; the others are reported as -"not in comparator". The cell list freezes at registration. +(the Report's list of appendix tables includes Appendix Table 17, +"Adjusted Poverty Rates at Age 67, by Gender and Marital Status"; which +of these splits Tables 19 and 21 carry is unknown, plan §10 decision +2(b)); the four sex-by-marital cells (optional); one diagnostic cell per +birth year (not scored). Only cells present in the sealed comparator are +scored; the others are reported as "not in comparator". The cell list +freezes at registration. **Undefined cells.** An empty cell or one with zero total weight has no statistic; it is reported with its reason and never imputed. -**F17 diagnostics (not scored).** Two kinds, both reported for U0: +**F17 diagnostics (not scored).** Two kinds, both reported for the +headline row (§11): - *Official-concept poverty rate* for the same sample: money income (TOTAL FAMILY INCOME, reported asset income kept, no annuity, no cut) @@ -402,65 +570,117 @@ statistic; it is reported with its reason and never imputed. both halves are defined; with fewer than two usable seeds it is undefined, never zero. Floors are at half sample and not rescaled. - **Design-based SE:** Taylor linearization of each weighted ratio with - the PSID stratum and cluster; the subpopulation estimator relative to - the tabulated rows (clusters with no tabulated row do not enter); - single-cluster strata are left out and counted. + the PSID sampling-error stratum and cluster (ER31996, ER31997), as a + domain estimator on the full sample design (referee Q5): every + (stratum, cluster) pair among persons with a positive cross-section + weight in the row's observation waves enters, with z = 0 for clusters + holding no observation of the cell (`TabulationConfig.design_se_domain + = "full_sample_design"`, the design frame from + `uniform_cut_track_u.runner.design_frame`). On the staged individual + file each of the five waves has 63 strata of exactly two clusters + among persons with a positive cross-section weight (a structural count + made for this draft, as the referee's), so no stratum is a singleton; + if one arises it is left out, counted and listed, and the run records + it. The `u1-draft-3` estimator, relative to the tabulated rows only, + dropped every stratum with one cluster present (13 of 51 under U0's + structural counts, 16 of 59 under U1) and so understated the + variance; it remains a code option (`tabulated_rows`), not registered. + +## 10a. Comparison and acceptance + +- **Comparator:** for each scored cell, the Report's change from Table 19 + to Table 21 (and, as secondary rows, the two levels), sealed on the + comparator side and opened only after the run artifact is committed. +- **Gap:** our Δ minus the Report's Δ, in percentage points, per cell + and registered row; likewise for P_B and P_R. +- **Comparator interval:** the Report prints rates; if each is printed to + d decimals, each level carries ±0.5·10^−d and the printed difference + ±10^−d. The interval appears only in the comparison memo, never in the + run artifact. +- **Acceptance:** none by default (plan §10 decision 6, pending Max): the + memo reports our value, floor, design SE, the comparator with its + interval, and the gap, and declares no pass or fail. A rule, if any, + is set by Max before registration; none may be set after. No tuning, + no re-run; a change is a new registered version. ## 11. Registered rows -Each alternative differs from U0 in one field, except U6, which is -defined on U1. U0 is the headline row; no row may be promoted after -results exist. +Each alternative differs from U0 in one field, except U6 (defined on U1) +and U0-F. U0 is the headline row unless the fallback rule below makes +U0-F the headline; no row may be promoted after results exist. Every +registered row is computed in the same one-shot run and published with +the others. A row that is not built when the registration is posted is +removed from this table and listed as a named omission. + +**Fallback rule (pending Max, with plan §10 decision 3).** If the 2005 +and 2007 wealth supplements are staged, adjudicated and read before the +#42 registration is posted, U0 is the headline and U0-F an alternative; +otherwise U0-F is the headline, and U0 and U1 are reported as blocked +with their counts. The rule depends on staging status only. In code +(`uniform_cut_track_u.runner.headline_row`), every row whose observation +waves include a wave without WEALTH1 is reported as blocked with its +counts, which under the fallback is every row except U0-F and U7: the +one-field alternatives are defined on U0 and need 1937 and 1939 too. +Whether they should instead follow the headline to U0-F's population is +not decided (§16, §17). The registration comment states the headline +row, and the registered run refuses when the staged PSID gives a +different one (`scripts/run_track_u_registered.py --headline-row`). | Row | Field | Value | Built | |---|---|---|---| | **U0** | — | §§3–10 | Yes; 1937 and 1939 blocked by the wealth supplements | | U1 | Population | All ten birth years (§3) | Yes; six observation cells (waves 2005 and 2007) blocked | | U2 | SSI | No response | Yes | -| U3 | SSI | Full static recomputation (upper bound) | Yes | +| U3 | SSI | Full static recomputation (the largest SSI response of the three) | Yes | | U4 | Income unit | Head and wife only | Yes | | U5 | Asset income | Keep reported asset income and add the annuity | Yes | | U6 | Cut (on U1) | 1936 uncut if the cut starts in 2004 (`cut_start_year = 2004`) | Yes; the start year awaits decision 8 | -| U7 | Financial assets | Plus employer DC balances | Not built (label investigation) | +| U0-F | Population | Birth years 1941, 1943, 1945 only (the blocked 1937 and 1939 left out and counted) | Yes; the fallback rule awaits Max | +| U7 | Financial assets | Plus employer DC balances | **No** (label investigation); must be built before registration or removed | | U8 | Threshold | PSID `CENSUS NEEDS STANDARD` | Yes | | U9 | Mortality | SSA period life table 2004 | Yes | | U10 | Threshold | Census size-by-children matrix | Yes (needs the capture) | -| U-inst | Universe | Add institutions (sequence 51–59), family-of-record income rule (§3) | Yes; the rule awaits the referee | +| U-inst | Universe | Add institutions (sequence 51–59), family-of-record income rule (§3) | Yes; the rule awaits the freeze and a referee | ## 12. Named deltas -Carried on every output (plan §7), plus those found while building: +Carried on every output (plan §7, the referee's additions and those +found while building). Each bullet is the text of +`uniform_cut_track_u.runner.NAMED_DELTAS`, held equal by +`tests/test_boomers2004_uniform_cut_spec.py`: - PSID versus SIPP wealth measurement; -- realized 2004–2012 history versus DYNASIM's 1992-based projection, - including the 2008–09 asset shock for the 1941–45 cohorts at 67; +- realized 2004-2012 history versus DYNASIM's 1992-based projection, including the 2008-09 asset shock for the 1941-45 cohorts at 67; - realized COLAs versus 2002 Trustees assumptions; -- immigrant under-coverage; institutionalized persons (under U-inst the - family-of-record rule assigns the family's income and threshold without - the member's own income, which the PSID does not collect); attrition; +- immigrant under-coverage; institutionalized persons (row U-inst: the family-of-record rule assigns the family's income and threshold without the member's own income, which the PSID does not collect); attrition; - OFUM-owned assets inside family wealth; +- the family's wealth stands in for an OFUM cohort member's own wealth (the Report's unit is the individual plus spouse, p. 24); - imputed rent excluded; -- self-reported Social Security, possibly net of Medicare premiums; -- pension income that may include account withdrawals (the 2013 file adds - explicit head and wife IRA items; earlier files fold withdrawals into - retirement income); -- exact-age sampling of alternate birth years (mean birth year 1941 - against 1940.5); -- **found:** farm income's asset portion is not separable and stays in - income; business income is split 50/50 between labor and asset parts by - PSID convention for working owners; -- **found:** SSI units are approximated from head, wife and OFUM totals - (deeming by full attribution; OFUMs as one unit); -- **found (review):** reported SSI may include state supplementary - payments, so capping the offset at the federal benefit rate less - reported SSI can understate the offset. +- self-reported Social Security, possibly net of Medicare Part B premiums, so the 13 percent cut applies to a smaller base than the gross benefit (baseline income is lower too; the net direction on the change is not established); +- retirement-account income beside annuitized balances: the head's income from annuities and IRAs is removed (F4a), but the wife's (before 2013) and the OFUMs' items combine pensions with annuity income and stay, so any IRA or annuity income in them is counted twice; an annuity already in payment may have no balance in WEALTH1, so removing its income understates income; +- employer DC (401(k)) balances outside IRAs are not in WEALTH1 though the Report counts them (p. 24): the primary annuity is understated for their holders (row U7); +- PSID other assets (W34) include cash value of life insurance, collections and rights in a trust or estate, which the Report's list (p. 22) does not name; +- the annuity is priced on population period life tables by age and sex (NCHS 2000; U9 SSA 2004), while DYNASIM's mortality follows the 2002 Trustees projections (p. 20, fn. 4) and the Report ties the annuity to family life expectancy (p. 24): with falling mortality, period tables overstate the annuity; +- exact-age sampling of alternate birth years (mean birth year 1941 against 1940.5); +- U1 averages ages 66 and 68, and claiming between those ages is not linear in age; +- the universe is alive at the interview after the income year; +- farm income's asset portion is imputed by the PSID business convention (farm_asset_share); business income is split 50/50 between labor and asset parts by PSID convention for working owners; +- SSI units approximated from head, wife and OFUM totals (deeming by full attribution; OFUMs as one unit); +- SSI deeming by full attribution can only overstate the offset (so understate the change): under 20 CFR 416.1163 nothing is deemed when the ineligible spouse's income is at most the couple-minus-individual FBR; +- the SSI test uses annual amounts against 12 times the January FBR (SSI accounting is monthly), and U3 omits rent and royalties from countable income although 20 CFR 416.1121 counts them; +- reported SSI may include state supplementary payments, so capping the offset at the federal benefit rate less reported SSI can understate the offset; +- U0-F, if it is the headline: mean birth year 1943 against 1940.5, and every observation year at or after the 2008-09 asset shock. ## 13. Invented worked cases These cases are **invented** for the unit tests (`tests/estimates/test_adjusted_poverty.py`, -`tests/estimates/test_uniform_cut_tabulation.py`). They use no PSID -data, no Census threshold, no SSA parameter and no comparator value. +`tests/estimates/test_uniform_cut_tabulation.py`, +`tests/cohorts/test_age67.py`). They use no PSID data, no Census +threshold and no comparator value. The life table, interest rate, +thresholds, federal benefit rates and family amounts are invented; the +SSI exclusions ($20 and $65 a month, one half of remaining earnings) and +resource limits ($2,000 / $3,000) are the statutory values. Invented life table (ages 0–4): male qx 0, 0, 0.2, 0.5, 1; female qx 0, 0, 0.1, 0.25, 1; invented rate 25 percent (v = 0.8). @@ -476,14 +696,26 @@ Invented life table (ages 0–4): male qx 0, 0, 0.2, 0.5, 1; female qx 0, | SSI offset, capped (SSI 550) | min(130, 600 − 550) | +50 | | SSI, SS under the exclusion (SS 200) | 0 − 0 | 0 | | U3 new take-up (SS 900, no other income) | before 660 ≥ 600; after 543 < 600 | +57 | - | U6, birth year 1936 (T = 1,000, SS 1,050) | primary: R = 1,050 − 136.5 = 913.5 (poor); `cut_start_year = 2004`: 1936 + 67 = 2003 < 2004, no cut, R = 1,050 | uncut | | U6, SSI recipient born 1936 (SS 1,000, SSI 100) | primary offset 130; uncut: no fall | 0 | -| U-inst, member in an institution attached to family 11 | family 11's income 1,000 and WEALTH1 500; role OFUM; single life | assigned | +| U-inst, member in an institution attached to family 11 | family 11's income 1,000 and WEALTH1 500; role OFUM; no co-resident spouse | assigned | +| F4a, head annuity and IRA income 200 (money income 1,200) | `remove_head`: B = 1,200 − 200 = 1,000; `keep`: B = 1,200 | 1,000 / 1,200 | +| F4a, 2013 file: `HEAD ANNUITIES` 150 and `HEAD IRAS` 50 | both removed | 200 removed | +| F4b, farm income 400 | share 0.5: remove 200; share 0: remove 0 | 200 / 0 | +| F4b, farm loss −300 | share 0.5: remove −300 (income rises by 300); share 0: 0 | −300 / 0 | +| Lives, OFUM woman aged 3 in a family headed by a man aged 2 and his legal wife aged 2 (WEALTH1 1,024) | `fu_head_rule`: joint 1.024, annuity 800; `member_rule`: single female 3, 0.75·0.8 = 0.6, annuity 1,365.3 | 800 / 1,365.3 | +| Lives, female head aged 2 with a legal husband (code 90) aged 2 | joint on the head and her legal husband | 1.024 | +| Unresolved marital state, head with a legal wife in the family | `relationship_code`: married, co-resident with the wife; `non_married`: non-married, no spouse | married / non-married | +| Spouse age, birth year 1943 by marriage history, individual-file age 66 at the 2009 interview | income year 2008: `derived_birth_year` 65; `wave_age` 66 | 65 / 66 | Invented tabulation (five observations, weights 1, 1, 2, 4, 2): P_B = 20, -P_R = 70, Δ = 50 for `all`; design SE = 100·√0.1 with two strata of two -clusters each. +P_R = 70, Δ = 50 for `all`. Design SE of Δ for `all` with the design +the rows occupy (two strata of two clusters each; stratum contributions +0.01 and 0.09): 100·√0.1. On a full design that adds a third cluster to +stratum 2 and a third stratum of two clusters, none holding an +observation: stratum 2's cluster totals become 0.2, −0.1 and 0 (mean +1/30, contribution 1.5 × 0.04667 = 0.07), stratum 3 contributes 0, and +the SE is 100·√0.08. F17 weighted quantile (invented values 10, 20, 30, 40 with weights 1, 1, 1, 1): the median is the smallest value whose cumulative weight share @@ -491,51 +723,51 @@ reaches 0.5, 20; p90 is 40. ## 14. What is built and what is blocked -Built on branch `dynamics-ex2-track-u-20260924` (#456) and continued on -`dynamics-ex2-track-u-2-20260924` (all opt-in; registered in -`POST_REVIEW_SOURCE_EXCLUSIONS` and the reachability guard): +On master since #456 (merge `8d7e7431`; built on branch +`dynamics-ex2-track-u-20260924`), continued on +`dynamics-ex2-track-u-2-20260924` (`u1-draft-3` and this draft; not +merged); all opt-in; registered in `POST_REVIEW_SOURCE_EXCLUSIONS` and +the reachability guard: - `src/populace_dynamics/data/family_income.py`: label-verified family income for waves 2005–2013 (81 items a wave, 85 in 2013, including the raw accuracy flags of the Social Security, SSI and asset items) and WEALTH1 with its components for 2009–2013; refusal naming the missing 2005 and 2007 wealth supplements; reconciliation counts. -- `src/populace_dynamics/cohorts/age67.py`: the age-67 observations for U0 - and U1, dispositions, provenance seal, structural summary. +- `src/populace_dynamics/cohorts/age67.py`: the age-67 observations for + U0, U1 and U0-F, dispositions, the relationship-code resolution of + unresolved marital states, the head and head's legal spouse with their + income-year ages (the birth-year law extended to in-family heads and + spouses), provenance seal, structural summary. - `src/populace_dynamics/estimates/adjusted_poverty.py`: annuity prices, - threshold lookup, SSI rules, baseline and reform income, poverty status; - refuses PSID-built rows without a registration. + threshold lookup, SSI rules, baseline and reform income (with F4a and + F4b), poverty status; refuses PSID-built rows without a registration. - `src/populace_dynamics/estimates/uniform_cut_tabulation.py`: cells, - Δ/P_B/P_R, half-split floors, design SE; refuses real data without a - registration pointer. + Δ/P_B/P_R, half-split floors, design SE on the full sample design; + refuses real data without a registration pointer. - `scripts/capture_track_u_parameters.py` (SSI capture written; Census parser ready), `scripts/track_u_structure.py` (counts only). -- Continued (`u1-draft-3`): the institution income rule and row U6's - `cut_start_year` (above); the package - `src/populace_dynamics/uniform_cut_track_u/`: - - `rows.py`: the registered rows as `Age67Spec` and - `AdjustedPovertySpec` overrides, held equal to §15's rows; - - `invented.py` (U5): a seeded INVENTED age-67 population in the - reader shapes (retired, SSI, near-threshold, OFUM-parent, - adult-child, working, negative-wealth, cohabiting, separated, - institutionalized, moved-out, deceased and zero-weight families; - income and WEALTH1 identities exact), the INVENTED threshold table, - and the re-generation checks that keep invented data from passing as - PSID (the builder records the `invented` kind only for frames that - hash to the generator's digest); - - `runner.py`: the row pipeline with its provenance guards and the - official-concept diagnostic; - - `diagnostics.py`: the F17 component summaries. +- `src/populace_dynamics/uniform_cut_track_u/` (`u1-draft-3`, extended + here): `rows.py` (the registered rows, held equal to §15's rows, and + the headline rule); `invented.py` (U5: a seeded INVENTED age-67 + population in the reader shapes, the INVENTED threshold table and the + re-generation checks; this draft adds couples resolved by relationship + code, legal husbands, farm, annuity and IRA income, and interview ages + that differ from income-year ages); `runner.py` (the row pipeline with + its provenance guards, the fallback headline, the full-design SE and + the official-concept diagnostic); `diagnostics.py` (the F17 component + summaries). - `scripts/track_u_dry_run.py` (U9): every row on the invented cohort with invented thresholds and the committed life tables and SSI - capture; evidence `EVID/track-u-dry-run-20260924/`, headed "INVENTED - DATA - NOT A COMPARISON". + capture, and the fallback rule on the invented cohort as staged today; + evidence headed "INVENTED DATA - NOT A COMPARISON". - `scripts/run_track_u_registered.py` (U10 entry point): refuses unless the pointer is an issue #42 comment, `HEAD` is the registered commit on a clean tree, §15 is ratified with nothing awaiting and nothing - blocking, the Census capture is pinned and every wave has WEALTH1; - writes `runs/replication_boomers2004_uniform_cut_v1.json` and its - `.env.json` exclusively. + blocking, the Census capture is pinned, and `--headline-row` equals the + fallback rule's row for the staged PSID; writes + `runs/replication_boomers2004_uniform_cut_v1.json` and its `.env.json` + exclusively. - `scripts/track_u_component_diagnostics.py`: the F17 component summaries on the staged PSID (evidence `EVID/track-u-diagnostics-20260924/`). On the staged files every @@ -547,15 +779,17 @@ Built on branch `dynamics-ex2-track-u-20260924` (#456) and continued on Blocked: 1. **2005 and 2007 wealth supplements** (Max, simba login): U0 loses 1937 - and 1939 until staged, adjudicated and read. A fallback row on 1941–45 - only would need registering in advance (plan §13); `income_rows(..., - allow_blocked=True)` supports it. + and 1939 until staged, adjudicated and read; under the fallback rule + U0-F is then the headline and the rows defined on U0 or U1 are + reported as blocked. 2. **Census thresholds** (download approval, cos decision d194): no primary threshold until captured; the dry run uses an INVENTED table. 3. **Registration** on issue #42 after ratification; no real-data poverty statistic may be computed before it. 4. **Comparator coverage** of the frozen cells (comparator side, values redacted). +5. **Row U7** is not built (label investigation of the P-section + employer DC items); it must be built or removed before registration. ## 15. Machine-readable parameter block @@ -565,7 +799,7 @@ holds it to the code's defaults. ```json { "specification": "boomers2004_uniform_cut_exercise2", - "version": "u1-draft-3", + "version": "u1-draft-4", "status": "draft_for_referee", "claim_class": { "proposed": "track_u_psid_realized_measurement_not_a_projection", @@ -577,11 +811,19 @@ holds it to the code's defaults. "birth_years": [1936, 1945], "age": 67, "cut_rate": 0.13, + "unit": "individual_with_household_income_and_wealth_including_spouse", + "financial_assets": "non_pension_wealth_plus_ira_keogh_401k_balances", "comparator_values": "sealed_comparator_side_not_opened_by_builder" }, "population": { "primary_row": "U0", + "headline": { + "rule": "u0_if_2005_2007_wealth_staged_before_registration_else_u0f", + "fallback_row": "U0-F", + "awaiting": "Max (fallback rule, with plan section 10 decision 3)" + }, "primary_birth_years": [1937, 1939, 1941, 1943, 1945], + "fallback_birth_years": [1941, 1943, 1945], "u1_birth_years": [1936, 1945], "u1_even_birth_year_weight": 0.5, "u1_single_observation_weight": 1.0, @@ -596,7 +838,8 @@ holds it to the code's defaults. "birth_year_law": "estimates.career.derive_birth_years", "seed_wave_rule": "earliest_presence_wave", "separated_is_married": true, - "unresolved_marital_status": "non_married", + "unresolved_marital_status": "relationship_code", + "annuitant_age_source": "derived_birth_year", "institution_income_rule": "family_of_record", "design": {"stratum": "ER31996", "cluster": "ER31997"} }, @@ -609,6 +852,10 @@ holds it to the code's defaults. "head_business_asset", "wife_rent", "wife_dividends", "wife_interest", "wife_trusts", "wife_business_asset", "ofum_asset" ], + "retirement_account_income_rule": "remove_head", + "retirement_account_income_items": ["head_annuities", "head_iras"], + "farm_asset_share": 0.5, + "farm_loss": "removed_whole_when_share_positive", "financial_assets": "WEALTH1", "annuitized_share": 0.8, "annuity": { @@ -618,13 +865,15 @@ holds it to the code's defaults. "survivor_share": 0.5, "mortality_basis": "nchs_2000", "terminal_closure": "table_end", - "lives": "member_rule", + "lives": "fu_head_rule", "negative_wealth": "annuity_floored_at_zero" - } + }, + "sensitivities_unscored": {"real_interest_rate": [0.02]} }, "threshold": { "rule": "census_weighted_average_65plus", "years": [2004, 2012], + "values": "weighted_averages_as_printed_in_thresh_workbooks", "capture_status": "not_captured", "poor_if": "income_below_threshold" }, @@ -642,7 +891,8 @@ holds it to the code's defaults. "ofum_unit": "single_individual", "parameters": { "file": "data/external/track_u_ssi_parameters.json", - "sha256": "79e641a10d95afba81c8d831852fb52ef0a801e7175e06df17e6cc7cc3e3c523" + "sha256": "79e641a10d95afba81c8d831852fb52ef0a801e7175e06df17e6cc7cc3e3c523", + "checked_against": "federal_register_cola_notices_2003_2011" } }, "statistic": { @@ -654,6 +904,7 @@ holds it to the code's defaults. "scored_candidates": ["all", "men", "women", "married", "non_married"], "optional": ["men_married", "men_non_married", "women_married", "women_non_married"], "diagnostic": "birth_year", + "source": "appendix_table_17_title_by_gender_and_marital_status", "scored_only_if_in_comparator": true }, "uncertainty": { @@ -665,7 +916,17 @@ holds it to the code's defaults. "summary": ["mean", "sample_sd"], "min_usable_seeds": 2 }, - "design_se": {"method": "taylor_linearization", "singleton_strata": "excluded_and_counted"} + "design_se": { + "method": "taylor_linearization", + "domain": "full_sample_design", + "outside_cell": "zero", + "singleton_strata": "excluded_counted_and_listed" + } + }, + "comparison": { + "gap": "model_minus_report", + "comparator_interval": "display_rounding", + "acceptance": {"rule": null, "awaiting": "Max (plan section 10 decision 6)"} }, "rows": { "U0": {}, @@ -675,13 +936,15 @@ holds it to the code's defaults. "U4": {"income_unit": "head_wife"}, "U5": {"asset_income_rule": "keep"}, "U6": {"on": "U1", "cut_start_year": 2004, "awaiting": "plan section 10 decision 8"}, + "U0-F": {"population": "birth_years_1941_1943_1945", "on": "U0", "awaiting": "Max (fallback rule, with plan section 10 decision 3)"}, "U7": {"financial_assets": "wealth1_plus_employer_dc", "status": "not_built"}, "U8": {"threshold_rule": "psid_census_needs_standard"}, "U9": {"mortality_basis": "ssa_period_2004"}, "U10": {"threshold_rule": "census_matrix_65plus"}, - "U-inst": {"presence": "in_family_or_institution", "institution_income_rule": "family_of_record", "awaiting": "referee"} + "U-inst": {"presence": "in_family_or_institution", "institution_income_rule": "family_of_record", "awaiting": "specification freeze (rule not yet refereed)"} }, "diagnostics_f17": { + "row": "headline", "official_concept_poverty_rate": "registered_run_only", "components": ["social_security", "ssi", "wealth1", "fu_size_record_counts"], "social_security_published": "ssa_supplement_2025_table_5a4_december_retired_workers", @@ -700,11 +963,23 @@ holds it to the code's defaults. "Python income concept (not Axiom)", "mechanical incidence" ], + "referee_passes": [ + { + "report": "EVID/boomers2004-referee-20260924.md", + "sha256": "b083fb2638787c204d09324a16fbbc5fa09cfc191e0450bcadb7d44f1647b2b6", + "object_version": "u1-draft-2", + "object_commit": "8d7e7431", + "verdict": "revise_before_ratification", + "required_changes": 17, + "applied_in": "u1-draft-4" + } + ], "blocked_by": [ "psid_2005_2007_wealth_supplements_not_staged", "census_thresholds_not_captured", "issue_42_registration_absent", - "max_ruling_d189_open" + "max_ruling_d189_open", + "row_u7_not_built" ] } ``` @@ -712,9 +987,9 @@ holds it to the code's defaults. ## 16. Pending decisions None of these is ratified. Each is a code parameter whose default is the -plan's proposal (or a builder choice where the plan is silent); -`adjusted_poverty.pending_decisions()` and `age67.pending_decisions()` -return them with their basis. +plan's proposal, the referee's recommendation or a builder choice where +both are silent; `adjusted_poverty.pending_decisions()` and +`age67.pending_decisions()` return them with their basis. **Awaiting Max (d189, open):** @@ -725,81 +1000,151 @@ return them with their basis. 3. Download the 2005 and 2007 PSID wealth supplements (plan §10, decision 3). +d189's text describes Track U as a "Python income concept, not Axiom", +so a yes on d189 may be read as covering plan decision 4; confirm when +ruling. + **Awaiting Max, not yet on a decision card (plan §10):** decision 2 (a values-redacted definitions extract by the comparator side), decision 4 (Python SSI arithmetic labelled "not Axiom"), decision 6 (acceptance rule; -default none, report gaps), decision 7 (ratify this specification by -merge; post the #42 registration), decision 8 (the scorecard's "from +default none, report gaps, §10a), decision 7 (ratify this specification +by merge; post the #42 registration), decision 8 (the scorecard's "from 2004": row U6 is built with `cut_start_year = 2004`, and the primary's -`cut_start_year = None` cuts every observation), and approval to -download the Census threshold files (cos decision d194). - -**Awaiting the referee:** the institution income rule -(`institution_income_rule = family_of_record`; alternative `excluded`), -which only row U-inst reads. - -**Awaiting the specification freeze (defaults shown):** income unit -(family unit), asset-income rule (replace), annuitized share (0.8), real -rate (3 percent), timing (immediate), load (0), survivor share (0.5, -reduced on either death), mortality (NCHS 2000), terminal closure (table -end), annuity lives (member rule), negative wealth (floor at zero), -threshold rule (weighted average, 65-and-over), SSI deeming (spouse's -Social Security counted), OFUM SSI unit (one individual unit), row (U0), +`cut_start_year = None` cuts every observation), approval to download the +Census threshold files (cos decision d194), and the fallback rule of §11 +(with decision 3), including whether the one-field alternatives follow +the headline to U0-F's population when the supplements are not staged +(default: they stay on U0 and are reported as blocked). + +**Awaiting the specification freeze (defaults shown):** cut rate (0.13; +the code lists it), cut base (all Social Security of the unit; not a code +parameter, decision 2(a)), cut start year (none), income unit (family +unit), asset-income rule (replace), retirement-account income (remove +the head's), farm asset share (0.5 of positive income, whole loss), +annuitized share (0.8), real rate (3 percent; 2 percent sensitivity), +timing (immediate), load (0), survivor share (0.5, reduced on either +death), mortality (NCHS 2000), terminal closure (table end), annuity +lives (FU head rule), annuitant ages (derived birth year), negative +wealth (floor at zero), financial assets (WEALTH1; U7 pending), threshold +rule (weighted average, 65-and-over; weighted averages as printed in +`threshYY.xlsx`), SSI deeming (spouse's Social Security counted), OFUM +SSI unit (one individual unit), row (U0, with the U0-F fallback rule), presence (in family), separated is married (true), unresolved marital -status (non-married), U1 1936 weight (1), seed wave (earliest presence -wave), cut start year (none). +status (relationship code), U1 1936 weight (1: each birth year then +carries one cross-section's weight), seed wave (earliest presence wave), +design SE (full-design domain), cells (after decision 2(b)), and the +institution income rule (`institution_income_rule = family_of_record`; +alternative `excluded`; only row U-inst reads it, and no referee has +reviewed it yet). A registered run refuses a §15 block that still names anything as `awaiting`, so ratification must resolve each of these (and the code's rows must be updated to match, `uniform_cut_track_u.rows`). -## 17. Questions for the referee - -1. Is `member_rule` right for OFUM members (annuitize the family's wealth - on the OFUM member's own life), or should the family head's lives price - it (`fu_head_rule`)? -2. Should the reported asset income removed under F4 include an imputed - asset share of farm income? -3. Is full attribution of the spouse's Social Security an acceptable - stand-in for SSI deeming, given the offset rule touches only baseline - recipients? -4. Should U1's 1936 observation carry weight 1 or 0.5? -5. Should the design SE include every PSID cluster of the stratum (zeros - outside the cohort) rather than only the clusters in the tabulated - rows? -6. Is a fallback row on 1941–45 worth registering now, in case the - wealth supplements are late? -7. Should a member whose marriage history cannot resolve the state - (`unknown`) or who has no record count as non-married (the default), - or should the relationship code (a legal wife, code 20, in the family) - decide? -8. The joint annuity prices the spouse at the individual-file age of the - wave, which the repository's birth-year law equates with the - income-year age; should it use the spouse's derived birth year - (income year minus birth year) instead, as the member's age does? +## 17. Referee record and open questions + +### 17.1 First pass (applied in `u1-draft-4`) + +- **Report:** `EVID/boomers2004-referee-20260924.md`, SHA-256 + `b083fb2638787c204d09324a16fbbc5fa09cfc191e0450bcadb7d44f1647b2b6` + (independent referee lane, Claude Opus 5.5, 2026-09-24). +- **Object:** `u1-draft-2` on `origin/master` at `8d7e7431`, blob + SHA-256 `3c0fc80d…6004f3d` (checked for this draft). The referee did + not see `u1-draft-3` (row U-inst's income rule, row U6, the invented + generator, the dry run and the registered entry point), which the + continuation branch had added the same day; its R11 and R16 are read + against that. +- **Verdict:** revise before ratification; 17 required changes (R1–R17), + 8 optional suggestions (O1–O8), answers to the draft's questions 1–8. +- **Version:** the referee asked for `u1-draft-3`; that number was + already taken on the continuation branch, so this pass is `u1-draft-4`. + +Each required change was checked against its source before it was +applied: the Report's pp. 19–24 and lists of tables; the PSID codebooks +and setup files; the eCFR current text of 20 CFR 416.1112, 416.1121, +416.1124, 416.1163 and 416.1205; the nine Federal Register notices; +the committed life tables (every price recomputed with the code); a +design-structure count on the staged individual file; and the +structural-count evidence. + +| Change | Disposition | +|---|---| +| R1 (§1 unit, financial assets; cells) | Applied. Verified on pp. 19, 22–24 and the list of appendix tables. Two wording corrections: the retirement-account sentence is in p. 24's text, with fn. 11 listing the Census sources; home equity is "projected separately" (p. 23) and not among the financial assets (pp. 22, 24), rather than "excluded" by p. 23 | +| R2 (builder boundary) | Applied, and extended: this draft read the Report's cleared pages itself, and says so | +| R3 (head's annuity and IRA income) | Applied as F4a (`retirement_account_income_rule`, default `remove_head`) with code, JSON and tests. Verified: `HEAD ANNUITIES` is "Head's Income from Annuities and IRAs" in the 2005–2011 codebooks, split in 2013. Corrected: the wife's pre-2013 item is labelled "Income from Pensions and Annuities" and the OFUMs' "Other Retirement, Pensions, and Annuities"; neither label names IRAs, so where a wife's IRA income is recorded before 2013 is not established. Added: an annuity in payment may have no WEALTH1 balance, so removing its income can understate income (named delta) | +| R4 (farm asset share) | Applied as F4b (`farm_asset_share`, default 0.5; a loss is removed whole when the share is positive). Verified on ER52214, ER52216 and p. 22 | +| R5 (401(k), U7) | Applied (text, named deltas, §11: U7 must be built or removed before registration; `blocked_by`) | +| R6 (real rate, administrative costs) | Applied. Verified on p. 22; the 2 percent prices recomputed with the code and the SSA 2004 prices added; `real_interest_rate` lists 0.02 | +| R7 (annuity lives) | Applied: `fu_head_rule` is the default, B, R and T are family-level. Extended: the head's legal spouse includes a legal husband (code 90, "Legal husband of Head", verified in the 2005–2013 formats; 10–15 in-family records a wave on the whole staged file, a structural count), who R7's text ("legal wife") would have left out | +| R8 (annuitant ages) | Applied (`annuitant_age_source`, default `derived_birth_year`). Verified in `estimates.career.derive_birth_years`. The extension runs the law without a required population, so a conflicting marriage-history birth year leaves the person unresolved (wave age, counted) rather than failing the build | +| R9 (unresolved marital state) | Applied (`unresolved_marital_status`, default `relationship_code`), symmetric for a head with a legal husband; institutionalized members are not resolved by code; the marital pairs indicator is not used | +| R10 (design SE) | Applied (`design_se_domain = "full_sample_design"`, the runner's design frame). Verified: 63 strata of exactly two clusters in each of the five waves among positive-weight persons | +| R11 (rows) | Applied in part. U0-F and the fallback rule are registered and built; U6 is built (since `u1-draft-3`); U7 must be built before registration or removed. **Declined:** replacing row U-inst by a count and `"institutionalized": "counted_only_not_a_row"`. The referee's reason was that U-inst had no income rule; `u1-draft-3` gave it one (`family_of_record`, §3), so it is a computable row. It stays registered, pending the freeze, with the rule flagged as not refereed (question 9). The fallback leaves the one-field alternatives blocked with U0; whether they should follow the headline is put to Max (§16) | +| R12 (comparison and acceptance) | Applied as §10a. The JSON nests the pending acceptance rule as `comparison.acceptance.awaiting`, which the registered run's `awaiting` scan sees, instead of a flat `acceptance_rule_awaiting` key, which it would not | +| R13 (named deltas) | Applied, all five | +| R14 (SSI sources) | Applied. The nine FBR notices and the CFR paragraphs re-checked for this draft (govinfo.gov, eCFR); 416.1112(c)(4)'s current text cross-references the exclusion as 416.1124(c)(10), which now sits at (c)(12); the resource limits are 416.1205(c) | +| R15 (§13) | Applied, with a full-design case that adds clusters holding no observation | +| R16 (§14 status and blockers) | Applied as adapted: the registered runner and dry run exist since `u1-draft-3`, so the referee's blocker 5 no longer applies; U7 is the remaining unbuilt row | +| R17 (pending decisions, tests, version) | Applied; the spec test now holds `cut`, `statistic`, `threshold.years`, `comparison`, the new fields, `rows.U0-F` and the named deltas to the code | + +| Suggestion | Disposition | +|---|---| +| O1 (416.1163 deeming threshold) | Not adopted in this draft. 416.1163(a)–(d) deems the ineligible spouse's earned and unearned income after the exclusions of 416.1161(a) and allocations for ineligible children; the suggested test counts only Social Security and other unearned income, and 416.1161(a) was not read. Full attribution and `recipients_only` still bracket the rule (§8) | +| O2 (weighted-average vintage) | Applied (§6), as the referee's reading of the Census page | +| O3 (U10 related children) | Applied (§6) | +| O4 (U3 wording) | Applied (§8, §11), citing 416.1218 and 416.1806 as the referee's reading | +| O5 (co-resident family size) | Not adopted, as the referee advised, until the definitions extract settles it | +| O6 (further named deltas) | Applied: alive at the interview, U1's 66/68 average, and the Part B base (with the caveat that the net direction on the change is not established) | +| O7 (page 25 governance note) | For the orchestrator, not the builder: the table of contents lists Section IV at p. 25 and the Sample Criteria subsection continues onto p. 25 (checked); `RESTRICTED-FILES.md` should say whether p. 25 is readable | +| O8 (brief names the version) | Noted: this pass's brief named `u1-draft-2` as the version to bump to, though the branch carried `u1-draft-3` | + +The referee's answers to questions 1–8 are adopted: Q1 `fu_head_rule` +(R7); Q2 yes, as F4b (R4); Q3 full attribution as the default with its +direction stated (§8, §12); Q4 weight 1 (§3); Q5 the full design (R10); +Q6 register U0-F (R11); Q7 the relationship code (R9); Q8 derived +birth years (R8). + +### 17.2 Questions for the next pass + 9. Is `family_of_record` the right income rule for institutionalized - members under U-inst (their family's income and threshold, without - their own income, which the PSID does not collect), or should they be - `excluded` from the universe, or get a rule not built here (for - example a one-person unit)? + members under U-inst (their family's income, annuity and threshold, + without their own income, which the PSID does not collect), or should + they be `excluded` from the universe, or get a rule not built here + (for example a one-person unit)? 10. Row U6 keys the cut on the year the member turns 67, not on the income year observed (U1 observes 1936 at 68, in 2004). Is that the reading the plan's "the 1936 cohort unaffected" intends? +11. Is extending the head's legal spouse to code 90 (legal husband), in + the annuity and in the relationship-code resolution, right, given + that the family file's "wife" items' treatment of a code-90 husband + was not checked? +12. Under the fallback rule, should the one-field alternatives follow the + headline to U0-F's population (so that U2–U5 and U8–U10 are computed + when the supplements are not staged), or stay on U0 and be reported + as blocked (the default)? +13. The birth-year law extension leaves a person with conflicting + marriage-history birth years unresolved and falls back to the wave + age. Is that acceptable, or should the build fail closed as the law + does for its required population? +14. F4a removes the head's annuity income, including annuities already + in payment that may carry no WEALTH1 balance. Should the rule remove + only `HEAD IRAS` in 2013 and keep `HEAD ANNUITIES` (not possible + before 2013, where the item combines both)? ## 18. What this draft read and did not verify **Read:** the plan in full; the cos records of d188 and d189 (their -text only); the -repository code it builds on (`data/social_security_income.py`, -`data/family.py`, `data/psid.py`, `cohorts/psid2010.py`, -`estimates/cola_age_profile.py`, `cola_track_a/statutory.py`, the -exercise-1 specification and its test, the birth-evidence guard); PSID -setup-file labels for the 2005–2013 family files and `IND2023ER.sps`; the -family codebook entries for the variables it reads (these print PSID's -unweighted frequency counts and ranges, which were not used); the -`IND2023ER_formats.sas` relationship codes; the 2005, 2007 and 2009 -family-file documentation on wealth; the policyengine-us SSI parameter -files; the Census historical thresholds page's link list. +text only); the repository code it builds on +(`data/social_security_income.py`, `data/family.py`, `data/psid.py`, +`cohorts/psid2010.py`, `estimates/cola_age_profile.py`, +`cola_track_a/statutory.py`, the exercise-1 specification and its test, +the birth-evidence guard); PSID setup-file labels for the 2005–2013 +family files and `IND2023ER.sps`; the family codebook entries for the +variables it reads (these print PSID's unweighted frequency counts and +ranges, which were not used); the `IND2023ER_formats.sas` relationship +codes; the 2005, 2007 and 2009 family-file documentation on wealth; the +policyengine-us SSI parameter files; the Census historical thresholds +page's link list. `u1-draft-3` also read: the PSID 2011 User Guide §2.4 (sample following rules), the individual-file codebook entries for ER34102, @@ -807,13 +1152,25 @@ ER34103 and ER34137–ER34143, the 2011 family codebook entry for `# IN FU`, the family-file record-count text quoted in `data/psid_unit_predicate_authority.py`, the cos record of d194 (its text only), and Table 5.A4 of the committed SSA supplement snapshot. -The codebook entries print whole-file unweighted counts, which were not -used. + +`u1-draft-4` also read: the referee report in full; the plan's §§2, 3, +7, 10, 13 and 14 again; `estimates.career.derive_birth_years`; relationship +code 90 in `IND2023ER_formats.sas` for 2005–2013; the Report's PDF +pages 5–7 and 20–25 (printed pp. 4–6 and 19–24); the family codebook +entries for `HEAD ANNUITIES` (2005–2013), `HEAD IRAS`, `WIFE ANNUITIES` +and `WIFE IRAS` (2013), `WIFE RETIREMENT/ANNUITIES` (2005, 2009, 2011), +`OTHER FU MEMBR RETIREMENT/ANNUITIES` (2005, 2011, 2013), ER52214, +ER52216, ER52217, ER52368, ER52392, and the W21 and W33 question text +(2011; their printed whole-sample counts were seen and not used); the +eCFR current text of 20 CFR 416.1112, 416.1121, 416.1124, 416.1163 and +416.1205; and the nine Federal Register notices of §8, fetched from +govinfo.gov into the session scratch directory to be read (not committed). **Ran on staged PSID:** label verification, the component-identity -reconciliation counts and the structural counts of §3, and (`u1-draft-3`) -the F17 component summaries of §9 for row U0 (Social Security, SSI and -WEALTH1 weighted summaries; `# IN FU` record counts). No income concept, +reconciliation counts and the structural counts of §3, the F17 component +summaries of §9 for row U0 (`u1-draft-3`), and (`u1-draft-4`) a design +count (strata and clusters among positive-weight persons per wave) and a +count of in-family code-90 records per wave. No income concept, annuity, threshold assignment, poverty status or poverty rate was computed on PSID data. @@ -821,27 +1178,47 @@ computed on PSID data. plus asset income by more than $10 in 21 families (2005), 34 (2007) and 4 (2013) (the OFUM items enter the primary only through TOTAL FAMILY INCOME and the OFUM asset total); the Census threshold values and the -real layout of the Census spreadsheets; the SSI parameters against ssa.gov; the names, IDs -and contents of the 2005 and 2007 wealth supplements; whether PSID Social -Security amounts are net of Medicare premiums; whether SSI and Social -Security are kept apart by respondents; the P-section DC items for U7; the -coverage of the sealed comparator; anything in the Report beyond the -plan's reading of its methods; whether an institutionalized member's -income enters any family-file item (the builder found no documentation -that it does); any SCF wealth aggregate (none is committed or saved). +real layout of the Census spreadsheets; the effective date of 20 CFR +416.1124(c)(22) for 2004–2012; 20 CFR 416.1161, 416.1218 and 416.1806; +the names, IDs and contents of the 2005 and 2007 wealth supplements; +whether PSID Social Security amounts are net of Medicare premiums; +whether SSI and Social Security are kept apart by respondents; where a +wife's IRA income is recorded before 2013; how the family file records a +code-90 legal husband's income; the P-section DC items for U7; the +coverage of the sealed comparator; anything in the Report on printed +page 25 or later; whether an institutionalized member's income enters +any family-file item (no documentation found that it does); DYNASIM's +actual annuity rate, mortality and survivor conventions beyond pp. 19–24; +the domain-estimation citation the referee gives (West, Berglund and +Heeringa 2008); any SCF wealth aggregate (none is committed or saved). ## 19. Changelog +- `u1-draft-4` (2026-09-24, first referee pass applied, §17): §1 states + the Report's unit and financial-asset definition from pp. 19–24 and + re-sources the cells to Appendix Table 17's title; F4a removes the + head's annuity and IRA income and F4b imputes farm income's asset part + (new parameters); the annuity prices the family head and the head's + legal spouse (`fu_head_rule`, code 90 included) at income-year ages + from derived birth years (`annuitant_age_source`); unresolved marital + states are resolved by relationship code; the design SE is a domain + estimator on the full sample design; row U0-F and the fallback rule are + registered and built, with blocked rows reported with their counts and + a `--headline-row` guard on the registered run; §10a (comparison and + acceptance) is new; the named deltas gain the referee's and are held + equal to the code; the SSI sources cite the Federal Register and eCFR; + the invented generator adds couples resolved by relationship code, + legal husbands, farm, annuity and IRA income, and interview ages; §15 + gains `referee_passes`, `comparison`, the headline rule and the new + fields; §17 records the pass and the open questions. - `u1-draft-3` (2026-09-24, continuation on `dynamics-ex2-track-u-2-20260924`): row U-inst gets an income rule - (`family_of_record`, pending the referee) and row U6 is built - (`cut_start_year`, the start year pending decision 8); the invented - generator (U5), the dry run (U9), the registered entry point (U10) and - the F17 component diagnostics are added; §15 gains - `institution_income_rule`, the cut's `start_year`, the F17 block and - the entry points, and U6 and U-inst record what they await; referee - questions 9 and 10 are added. - + (`family_of_record`) and row U6 is built (`cut_start_year`, the start + year pending decision 8); the invented generator (U5), the dry run + (U9), the registered entry point (U10) and the F17 component + diagnostics are added; §15 gains `institution_income_rule`, the cut's + `start_year`, the F17 block and the entry points, and U6 and U-inst + record what they await; referee questions 9 and 10 are added. - `u1-draft-2` (2026-09-24, independent review): the half-split floor splits on family units linked through shared persons, so it is person-disjoint under U1 as plan F15 requires (U0 is unchanged); the diff --git a/scripts/run_track_u_registered.py b/scripts/run_track_u_registered.py index 6d54b2c4..94c47fc4 100644 --- a/scripts/run_track_u_registered.py +++ b/scripts/run_track_u_registered.py @@ -24,24 +24,31 @@ them; * the Census thresholds must be the committed, hash-pinned capture (:func:`populace_dynamics.estimates.adjusted_poverty. - load_poverty_thresholds` refuses until it exists), and every PSID wave - must have WEALTH1 (the 2005 and 2007 wealth supplements staged and - adjudicated); both are checked before any PSID file is read for the - statistic. + load_poverty_thresholds` refuses until it exists), checked before any + PSID file is read; +* ``--headline-row`` must name the headline row the #42 registration + comment states, and it must equal the row the specification's fallback + rule gives for the staged PSID (:func:`populace_dynamics. + uniform_cut_track_u.runner.headline_row`: U0 when the 2005 and 2007 + wealth supplements are staged, adjudicated and read, U0-F otherwise); + a mismatch means the staging changed after registration and the run + refuses (a new registration is needed). Rows that need a wave without + WEALTH1 are reported as blocked with their counts. The artifact publishes regardless of outcome. It never reads the sealed comparator; the seal is opened only after this artifact is committed. As of this script's writing none of these preconditions holds: the -specification is a draft (``u1-draft-3``), Max has not ruled on exercise -2 (cos decision d189), the Census thresholds are not captured (d194) and -the 2005/2007 wealth supplements are not staged. +specification is a draft (``u1-draft-4``), Max has not ruled on exercise +2 (cos decision d189) or on the fallback rule, the Census thresholds are +not captured (d194) and the 2005/2007 wealth supplements are not staged. Usage:: python scripts/run_track_u_registered.py \\ --registration-pointer \\ --registered-commit \\ + --headline-row {U0,U0-F} \\ [--output runs/replication_boomers2004_uniform_cut_v1.json] Writes the artifact and a ``.env.json`` sidecar next to it. @@ -180,6 +187,21 @@ def preflight( return {"head": head} +def check_headline(registered: str, inputs: Any) -> str: + """Refuse unless the registered headline is the fallback rule's row.""" + + staged = runner.headline_row(inputs) + if registered != staged: + raise ValueError( + f"the registration names headline row {registered}, but the " + f"staged PSID gives {staged} under the fallback rule (WEALTH1 " + f"refused for waves {sorted(inputs.wealth_refusals)}): the " + "staging changed after registration, so this is not the " + "registered run" + ) + return staged + + def _write_new(path: Path, text: str) -> None: """Create ``path`` exclusively: never overwrite (one shot).""" @@ -233,6 +255,12 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--registration-pointer", required=True) parser.add_argument("--registered-commit", required=True) + parser.add_argument( + "--headline-row", + required=True, + choices=(rows.PRIMARY_ROW, rows.FALLBACK_ROW), + help="the headline row the #42 registration comment states", + ) parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) args = parser.parse_args(argv) @@ -250,12 +278,7 @@ def main(argv: list[str] | None = None) -> int: params = runner.committed_parameters(thresholds) environment = _environment() inputs = age67.load_age67_inputs() - if inputs.wealth_refusals: - raise ValueError( - "WEALTH1 is refused for waves " - f"{sorted(inputs.wealth_refusals)}; the registered run needs " - "every wave (stage and adjudicate the wealth supplements)" - ) + check_headline(args.headline_row, inputs) result = runner.run_track_u( inputs, params, diff --git a/scripts/track_u_dry_run.py b/scripts/track_u_dry_run.py index 886009b4..c74fbc91 100644 --- a/scripts/track_u_dry_run.py +++ b/scripts/track_u_dry_run.py @@ -15,8 +15,10 @@ the PSID wealth supplements were staged, so U0's five birth years run end to end; the real supplements are not staged (cos decision d189). The checks record what the real staging does today (the income rows refuse -the blocked waves) and that the registered-run guards refuse invented -inputs and an uncaptured threshold table. +the blocked waves, and under the fallback rule U0-F becomes the headline +while the rows that need 2005 or 2007 are reported as blocked with their +counts) and that the registered-run guards refuse invented inputs and an +uncaptured threshold table. Usage:: @@ -96,6 +98,7 @@ def checks(seed: int, params: runner.TrackUParameters) -> dict[str, Any]: blocked = invented.invented_age67_inputs(seed=seed) cohort = age67.build_age67_cohort(blocked) left = age67.income_rows(cohort, blocked, allow_blocked=True) + fallback = runner.run_track_u(blocked, params, data_provenance=ap.INVENTED) return { "specification_rows": rows.check_rows_against_block( rows.specification_block() @@ -115,6 +118,33 @@ def checks(seed: int, params: runner.TrackUParameters) -> dict[str, Any]: ), "left_out_with_allow_blocked": dict(left.attrs["left_out"]), "observations_kept_with_allow_blocked": int(len(left)), + "fallback_rule": { + "headline": fallback["headline"], + "row_status": { + row_id: entry["status"] + for row_id, entry in fallback["rows"].items() + }, + "blocked_counts": { + row_id: entry["population"] + for row_id, entry in fallback["rows"].items() + if entry["status"] == "blocked" + }, + "headline_all_cell": next( + { + key: cell[key] + for key in ( + "n_observations", + "baseline_rate", + "reform_rate", + "delta", + ) + } + for cell in fallback["rows"][fallback["headline"]["row"]][ + "tabulation" + ]["cells"] + if cell["cell"] == "all" + ), + }, }, "registered_guard_refuses_invented_inputs": _refusal( lambda: runner.run_track_u( @@ -210,15 +240,25 @@ def results_markdown(result: dict[str, Any]) -> str: ] for row_id, entry in result["rows"].items(): if entry["status"] != "computed": + status = entry["status"].replace("_", " ") lines.append( - f"| {row_id} | not built: {entry['reason']} | | | | | |" + f"| {row_id} | {status}: {entry['reason']} | | | | | |" ) continue lines.append(_cell_line(row_id, _cell(entry["tabulation"], "all"))) - primary = result["rows"]["U0"]["tabulation"] + headline = result["headline"]["row"] + primary = result["rows"][headline]["tabulation"] + design = primary["design"] lines += [ "", - "### Row U0 by cell (invented)", + f"Headline row {headline} (fallback rule " + f"`{result['headline']['rule']}`). Design-based standard errors " + f"use the {design['domain'].replace('_', ' ')}: " + f"{design.get('n_strata')} invented strata, " + f"{design.get('n_clusters')} clusters, singleton strata " + f"{design.get('singleton_strata')}.", + "", + f"### Row {headline} by cell (invented)", "", "| Cell | Observations | P_B | P_R | Δ | Δ floor | Δ design SE |", "|---|---|---|---|---|---|---|", @@ -240,7 +280,7 @@ def results_markdown(result: dict[str, Any]) -> str: "", "Official-concept poverty rate (money income, reported asset " "income kept, no annuity, no cut, against the same invented " - "thresholds), row U0: " + f"thresholds), row {f17['row']}: " + "; ".join( f"{name} {_fmt(cell.get('rate'))}" for name, cell in f17["official_concept_poverty_rate"][ @@ -307,6 +347,20 @@ def results_markdown(result: dict[str, Any]) -> str: f"{blocked['left_out_with_allow_blocked']['wealth_supplement_not_staged']}" " observations (waves 2005 and 2007) are left out and " f"{blocked['observations_kept_with_allow_blocked']} kept.", + "- Under the fallback rule with that staging, the headline is " + f"{blocked['fallback_rule']['headline']['row']}; rows reported " + "blocked with their counts: " + + ", ".join( + row_id + for row_id, status in blocked["fallback_rule"][ + "row_status" + ].items() + if status == "blocked" + ) + + f"; {blocked['fallback_rule']['headline']['row']} cell `all` " + f"(invented): {blocked['fallback_rule']['headline_all_cell']['n_observations']} " + "observations, Δ " + f"{_fmt(blocked['fallback_rule']['headline_all_cell']['delta'])}.", "- The registered path refuses these invented inputs: " f"{check['registered_guard_refuses_invented_inputs']['refused']} " f"(`{check['registered_guard_refuses_invented_inputs'].get('error')}`).", diff --git a/scripts/track_u_structure.py b/scripts/track_u_structure.py index 0e459159..4a280893 100644 --- a/scripts/track_u_structure.py +++ b/scripts/track_u_structure.py @@ -1,10 +1,13 @@ """Structural counts of the Track U age-67 population (counts only). -Builds the exercise-2 age-67 observations (rows U0 and U1) from the +Builds the exercise-2 age-67 observations (every builder row of +:data:`populace_dynamics.cohorts.age67.ROWS`: U0, U1 and U0-F) from the staged PSID with :mod:`populace_dynamics.cohorts.age67` and writes ``track-u-structure.json``: persons, observations, dispositions, family -units, design strata, the wealth blockers, threshold-free receipt counts -and the family-income and WEALTH1 reconciliation counts. +units, design strata, the wealth blockers, the marital-resolution paths +and annuitant age sources, threshold-free receipt counts (including farm +and the head's annuity and IRA income) and the family-income and WEALTH1 +reconciliation counts. It computes **no** income concept, annuity, threshold, poverty status or poverty rate: plan ``critical-path-uniform-cut-20260923.md`` section 8 diff --git a/src/populace_dynamics/cohorts/age67.py b/src/populace_dynamics/cohorts/age67.py index e28429e7..c1be8da9 100644 --- a/src/populace_dynamics/cohorts/age67.py +++ b/src/populace_dynamics/cohorts/age67.py @@ -18,8 +18,14 @@ as U0; each even birth year 1938-1944 (odd age-67 year) observed at 66 and at 68 with half weight each; 1936 observed at 68 only (income year 2004; its age-66 year 2002 is in the 2003 wave, which has aggregates - only) with weight ``u1_single_observation_weight`` (1, a builder - choice: the plan says only "1936 from 2004 only"). + only) with weight ``u1_single_observation_weight`` (1: each birth year + then carries one cross-section's weight; the plan says only "1936 from + 2004 only"). +* **U0-F (registered fallback): U0 on birth years 1941, 1943 and 1945 + only**, the U0 birth years whose waves (2009-2013) carry WEALTH1 in the + family file. The specification's fallback rule makes it the headline + row when the 2005 and 2007 wealth supplements are not staged before the + #42 registration (a rule on staging status, never on results). Universe per wave (F2): sequence 1-20 (in a responding family) with a positive cross-section weight; the registered option U-inst adds @@ -27,7 +33,8 @@ institutionalized person (the individual-file Social Security items are "Inap.: ... in an institution", codebook ER34137-ER34143 for 2011) and the plan gives them no income rule, so ``institution_income_rule`` (a -builder default pending the referee) supplies one: +builder default of u1-draft-3, pending the freeze and not yet refereed) +supplies one: * ``family_of_record`` (default): the PSID attaches an institutionalized sample member's record to the family they left (2011 User Guide, @@ -60,19 +67,40 @@ Attached to each observation: the family unit of that wave (``family_unit_id = wave * 100000 + interview``), the member's role (head 10, legal wife 20, cohabiting "wife" 22, otherwise OFUM; the codes -are verified against ``IND2023ER_formats.sas`` for each wave), sex, the -PSID sampling-error stratum and cluster (ER31996, ER31997), marital -status at the end of the income year from the marriage history +are verified against ``IND2023ER_formats.sas`` for each wave, with 90, +"Legal husband of Head"), sex, the PSID sampling-error stratum and +cluster (ER31996, ER31997), marital status at the end of the income year +from the marriage history (:func:`populace_dynamics.cohorts.psid2010.marital_state_at`, separated -counts as married by default, F12; a state the history cannot resolve, -``unknown`` or ``no_marriage_history``, counts as non-married under -``unresolved_marital_status``), whether a legal spouse lives in the -same family unit, the spouse's age and sex (for the joint annuity), and -the family's income (:func:`populace_dynamics.data.family_income. -read_family_income`) and wealth (:func:`populace_dynamics.data. -family_income.read_family_wealth`). Waves 2005 and 2007 have no staged -wealth, so their observations are marked -``blocked_wealth_supplement_not_staged``. +counts as married by default, F12), whether a legal spouse lives in the +same family unit, the spouse's age and sex, the family head's and the +head's co-resident legal spouse's ages and sexes (the annuitants under +the income concept's ``fu_head_rule``), and the family's income +(:func:`populace_dynamics.data.family_income.read_family_income`) and +wealth (:func:`populace_dynamics.data.family_income.read_family_wealth`). +Waves 2005 and 2007 have no staged wealth, so their observations are +marked ``blocked_wealth_supplement_not_staged``. + +Unresolved marital states (``unresolved_marital_status``, pending the +specification freeze): a member whose marriage history cannot date the +state (``unknown``) or who has no record (``no_marriage_history``) is, +under ``relationship_code`` (default, the referee's Q7 answer), married +and co-resident with that spouse when the member is the head with a +co-resident legal spouse (code 20 or 90) or is the head's legal wife (20) +or legal husband (90); otherwise non-married. ``non_married`` classifies +them all as non-married. ``marital_resolution`` records the path taken +for every observation. + +Annuitant ages (``annuitant_age_source``, pending the freeze): under +``derived_birth_year`` (default, the referee's Q8 answer) every +annuitant's age is the income year minus the birth year the birth-year +law derives, the law extended beyond the universe to in-family heads, +legal and cohabiting spouses and marriage-history spouses; where the law +leaves a person unresolved the individual-file age at the wave is used +and the source says so (``wave_age``). The law gives the +marriage-history birth year precedence, so for those people the wave age +(age at the interview) can exceed the income-year age by one year. +``wave_age`` uses the individual-file age at the wave throughout. Provenance: :func:`load_age67_inputs` records the SHA-256 of every PSID file it read and seals the returned inputs; the builder marks a cohort @@ -114,13 +142,16 @@ __all__ = [ "ALL_BIRTH_YEARS", "ANCHOR_LAYOUTS", + "ANNUITANT_AGE_SOURCES", "CALLER_FRAMES", + "FALLBACK_BIRTH_YEARS", "INSTITUTION_INCOME_RULES", "INVENTED", "PRIMARY_BIRTH_YEARS", "PSID_FILES", "ROWS", "TARGET_AGE", + "UNRESOLVED_MARITAL_RULES", "WAVES", "Age67Cohort", "Age67Inputs", @@ -140,7 +171,10 @@ WAVES: tuple[int, ...] = family_income.INCOME_WAVES PRIMARY_BIRTH_YEARS: tuple[int, ...] = (1937, 1939, 1941, 1943, 1945) ALL_BIRTH_YEARS: tuple[int, ...] = tuple(range(1936, 1946)) -ROWS: tuple[str, ...] = ("U0", "U1") +#: The U0 birth years whose observation waves carry WEALTH1 in the family +#: file (row U0-F, the registered fallback). +FALLBACK_BIRTH_YEARS: tuple[int, ...] = (1941, 1943, 1945) +ROWS: tuple[str, ...] = ("U0", "U1", "U0-F") PSID_FILES = "psid_files" INVENTED = "invented" CALLER_FRAMES = "caller_frames" @@ -150,14 +184,35 @@ "the family unit whose interview number the institutionalized " "member's record carries (the family they left): its income, " "wealth, size and children, with the member's own age and sex, " - "role ofum and no co-resident spouse (builder default, pending the " - "referee)" + "role ofum and no co-resident spouse (builder default, pending " + "the freeze; not yet refereed)" ), "excluded": ( "institutionalized persons stay out of the poverty universe (a " "disposition); U-inst then equals U0 in population" ), } +#: How a member whose marriage history cannot resolve the state is +#: classified. +UNRESOLVED_MARITAL_RULES: dict[str, str] = { + "relationship_code": ( + "married and co-resident when the member is the head with a " + "co-resident legal spouse (code 20 or 90) or is the head's legal " + "wife (20) or legal husband (90); otherwise non-married (the " + "referee's Q7 answer)" + ), + "non_married": "every unresolved member is non-married", +} +#: Where an annuitant's age comes from. +ANNUITANT_AGE_SOURCES: dict[str, str] = { + "derived_birth_year": ( + "income year minus the birth year the birth-year law derives " + "(extended to in-family heads and spouses); the individual-file " + "age at the wave where the law leaves the person unresolved (the " + "referee's Q8 answer)" + ), + "wave_age": "the individual-file age at the wave (age at interview)", +} #: Each wave's label-verified individual-file anchor variables (labels #: checked 2026-09-24 against IND2023ER.sps; 2009 and 2011 are the @@ -208,6 +263,18 @@ "ER30002": "PERSON NUMBER 68", } _HEAD, _LEGAL_WIFE, _PARTNER = 10, 20, 22 +_FIRST_YEAR_COHABITOR, _LEGAL_HUSBAND = 88, 90 +#: Relationship codes of the head's co-resident legal spouse. +_LEGAL_SPOUSE_CODES = (_LEGAL_WIFE, _LEGAL_HUSBAND) +#: Relationship codes whose in-family persons get a birth year for the +#: annuity even outside the universe (heads and spouses or partners). +_ANNUITANT_CODES = ( + _HEAD, + _LEGAL_WIFE, + _PARTNER, + _FIRST_YEAR_COHABITOR, + _LEGAL_HUSBAND, +) _SEQUENCE_IN_FAMILY = (1, 20) _SEQUENCE_INSTITUTION = (51, 59) _SEQUENCE_GROUPS: tuple[tuple[str, int, int], ...] = ( @@ -231,6 +298,7 @@ _HEAD: "head in {wave}", _LEGAL_WIFE: "legal wife in {wave}", _PARTNER: '"wife"', + _LEGAL_HUSBAND: "legal husband of head", } @@ -243,7 +311,8 @@ class Age67Spec: separated_is_married: bool = True u1_single_observation_weight: float = 1.0 seed_wave_rule: str = "earliest_presence_wave" - unresolved_marital_status: str = "non_married" + unresolved_marital_status: str = "relationship_code" + annuitant_age_source: str = "derived_birth_year" institution_income_rule: str = "family_of_record" def __post_init__(self) -> None: @@ -258,10 +327,15 @@ def __post_init__(self) -> None: ) if self.seed_wave_rule != "earliest_presence_wave": raise ValueError("seed_wave_rule must be earliest_presence_wave") - if self.unresolved_marital_status != "non_married": + if self.unresolved_marital_status not in UNRESOLVED_MARITAL_RULES: raise ValueError( - "unresolved_marital_status must be non_married (no other " - "rule is built)" + "unresolved_marital_status must be one of " + f"{sorted(UNRESOLVED_MARITAL_RULES)}" + ) + if self.annuitant_age_source not in ANNUITANT_AGE_SOURCES: + raise ValueError( + "annuitant_age_source must be one of " + f"{sorted(ANNUITANT_AGE_SOURCES)}" ) weight = float(self.u1_single_observation_weight) if not 0.0 < weight <= 1.0: @@ -280,9 +354,13 @@ def pending_decisions() -> tuple[psid2010.PendingDecision, ...]: psid2010.PendingDecision( "row", spec.row, - ("U1",), + ("U1", "U0-F"), "plan F1: exact age (U0) is the primary; U1 pools all ten " - "birth years", + "birth years; U0-F (1941, 1943, 1945) is the registered " + "fallback, the headline when the 2005 and 2007 wealth " + "supplements are not staged before the #42 registration " + "(specification section 11 fallback rule, with plan section 10 " + "decision 3)", freeze, ), psid2010.PendingDecision( @@ -305,7 +383,10 @@ def pending_decisions() -> tuple[psid2010.PendingDecision, ...]: "u1_single_observation_weight", spec.u1_single_observation_weight, (0.5,), - "builder choice: the plan says only '1936 from 2004 only'", + "builder choice (the plan says only '1936 from 2004 only'), " + "endorsed by the referee (Q4): with weight 1 every birth year " + "enters U1 with one cross-section's worth of weight; 0.5 would " + "halve 1936's share of the cohort", freeze, ), psid2010.PendingDecision( @@ -320,21 +401,39 @@ def pending_decisions() -> tuple[psid2010.PendingDecision, ...]: psid2010.PendingDecision( "unresolved_marital_status", spec.unresolved_marital_status, - (), - "builder choice: plan F12 ('legally married per MH85_23') " - "does not say how to classify a member whose marriage history " - "cannot date the state ('unknown') or who has no record " - "('no_marriage_history'); they count as non-married (the " - "non_married cell, a single-life annuity)", + ("non_married",), + "referee answer to Q7 (boomers2004-referee-20260924.md, R9): " + "plan F12 ('legally married per MH85_23') does not say how to " + "classify a member whose marriage history cannot date the " + "state ('unknown') or who has no record ('no_marriage_" + "history'); the relationship code decides (married and " + "co-resident as head with a co-resident legal spouse, 20 or " + "90, or as the head's legal wife, 20, or legal husband, 90), " + "otherwise non-married; the draft-2 rule classified them all " + "as non-married", + freeze, + ), + psid2010.PendingDecision( + "annuitant_age_source", + spec.annuitant_age_source, + ("wave_age",), + "referee answer to Q8 (R8): every annuitant's age is the " + "income year minus the derived birth year (the birth-year law " + "extended to in-family heads and spouses), the wave age only " + "where the law leaves the person unresolved (counted); the " + "draft-2 rule priced the spouse at the individual-file age of " + "the wave, which exceeds the income-year age by one year when " + "the birthday falls before the interview", freeze, ), psid2010.PendingDecision( "institution_income_rule", spec.institution_income_rule, ("excluded",), - "builder default, pending the referee: plan F2 adds " - "institutions under U-inst but gives them no income rule and " - "the PSID collects none for them; the PSID attaches an " + "builder default (u1-draft-3), not yet refereed (the first " + "referee pass reviewed u1-draft-2, which had no rule): plan F2 " + "adds institutions under U-inst but gives them no income rule " + "and the PSID collects none for them; the PSID attaches an " "institutionalized member's record to the family they left " "(2011 User Guide section 2.4), so the observation takes that " "family's income concept and threshold (the member's own " @@ -351,7 +450,10 @@ def observation_plan( ) -> tuple[tuple[int, int, int, float], ...]: """``(birth_year, wave, income_year, weight_multiplier)`` per row.""" - plan = [(b, b + 68, b + 67, 1.0) for b in PRIMARY_BIRTH_YEARS] + primary = ( + FALLBACK_BIRTH_YEARS if spec.row == "U0-F" else PRIMARY_BIRTH_YEARS + ) + plan = [(b, b + 68, b + 67, 1.0) for b in primary] if spec.row == "U1": for b in range(1938, 1945, 2): plan.append((b, b + 67, b + 66, 0.5)) @@ -723,6 +825,89 @@ def _birth_years( return births, universe +def _annuitant_birth_years( + inputs: Age67Inputs, + universe: set[int], + targets: set[int], +) -> dict[int, career.BirthYearRecord]: + """Birth years of in-family heads and spouses outside the universe. + + The annuity prices the family head and the head's legal spouse (or the + member's co-resident spouse) at their income-year ages, so the + birth-year law is extended to every in-family person (sequence 1-20) + of the five waves coded head, legal wife, cohabiting "wife", first-year + cohabitor or legal husband, and to every marriage-history spouse of a + target member, who is not in the universe (typically a zero-weight + nonsample spouse). The seed coordinate is the earliest of the five + waves in which the person is in a family, whatever the weight. The + law runs without a required population, so a person with conflicting + marriage-history birth years is left out (unresolved) instead of + failing the build; the caller then uses the wave age and counts it. + """ + + history = inputs.marriage_history + spouses = set() + if "spouse_person_id" in history.columns: + mine = history[history["person_id"].isin(targets)] + spouses = { + int(value) for value in mine["spouse_person_id"].dropna().tolist() + } + seeds = [] + for wave in WAVES: + anchor = inputs.anchors[wave] + in_family = anchor[ + anchor["sequence"].between(*_SEQUENCE_IN_FAMILY) + & ( + anchor["relationship"].isin(_ANNUITANT_CODES) + | anchor["person_id"].isin(spouses) + ) + & ~anchor["person_id"].isin(universe) + ] + # Seed ages the law accepts (1-125 or the 999 sentinel) and whose + # seed birth year stays inside the law's derived range. + ages = in_family["age"].astype("int64") + seed_birth = (wave - 1) - ages + in_family = in_family[ + ( + ages.between(1, _MAX_AGE_CODE) + & seed_birth.between( + career.DERIVED_BIRTH_MIN, career.DERIVED_BIRTH_MAX + ) + ) + | ages.eq(999) + ] + seeds.append( + pd.DataFrame( + { + "person_id": in_family["person_id"].astype("int64"), + "year": wave - 1, + "anchor_wave": wave, + "age": in_family["age"].astype("int64"), + } + ) + ) + seed = ( + pd.concat(seeds, ignore_index=True) + .sort_values(["person_id", "anchor_wave"]) + .drop_duplicates("person_id", keep="first") + .reset_index(drop=True) + ) + if seed.empty: + return {} + extra = set(int(pid) for pid in seed["person_id"]) + earnings = inputs.observed_earnings + records = career.derive_birth_years( + history[history["person_id"].isin(extra)], + earnings[earnings["person_id"].isin(extra)], + seed_coordinates=seed, + ) + return { + record.person_id: record + for record in records + if record.person_id in extra + } + + def _episodes(history: pd.DataFrame, person_ids: set[int]) -> dict: subset = history[history["person_id"].isin(person_ids)] if subset.empty: @@ -731,6 +916,89 @@ def _episodes(history: pd.DataFrame, person_ids: set[int]) -> dict: return {int(pid): rows for pid, rows in episodes.groupby("person_id")} +#: Marriage-history states the history cannot resolve. +_UNRESOLVED_STATES = ("unknown", "no_marriage_history") + + +def _head_and_legal_spouse( + family_rows: pd.DataFrame | None, +) -> tuple[int | None, int | None, int | None]: + """The family's head, the head's legal spouse and that spouse's code. + + From the family unit's in-family records: the one person coded head + (10) and the one coded legal wife (20) or legal husband (90); ``None`` + where there is not exactly one. + """ + + if family_rows is None: + return None, None, None + heads = family_rows.loc[family_rows["relationship"].eq(_HEAD)] + head = int(heads["person_id"].iloc[0]) if len(heads) == 1 else None + spouses = family_rows.loc[ + family_rows["relationship"].isin(_LEGAL_SPOUSE_CODES) + ] + if len(spouses) != 1: + return head, None, None + return ( + head, + int(spouses["person_id"].iloc[0]), + int(spouses["relationship"].iloc[0]), + ) + + +def _resolve_by_relationship( + spec: Age67Spec, + in_institution: bool, + pid: int, + relationship: int, + head: int | None, + head_spouse: int | None, +) -> tuple[str, Any]: + """``(marital_resolution, spouse)`` for an unresolved marital state.""" + + if spec.unresolved_marital_status == "relationship_code" and not ( + in_institution + ): + if relationship == _HEAD and head == pid and head_spouse is not None: + return "relationship_code_head_with_legal_spouse", head_spouse + if head is not None and head != pid and head_spouse == pid: + if relationship == _LEGAL_WIFE: + return "relationship_code_legal_wife", head + if relationship == _LEGAL_HUSBAND: + return "relationship_code_legal_husband", head + return "unresolved_non_married", pd.NA + + +def _annuitant_age( + pid: int | None, + income_year: int, + family_rows: pd.DataFrame | None, + births: Mapping[int, career.BirthYearRecord], + spec: Age67Spec, +) -> tuple[Any, str]: + """``(age, source)`` of one annuitant in the income year. + + ``derived_birth_year``: income year minus the law's birth year where + the law resolves the person; otherwise, and always under ``wave_age``, + the individual-file age at the wave (codes 1-125). ``absent`` when + there is no such person, ``missing`` when neither source has an age. + """ + + if pid is None: + return pd.NA, "absent" + if spec.annuitant_age_source == "derived_birth_year": + record = births.get(int(pid)) + if record is not None and record.birth_year is not None: + return int(income_year) - int(record.birth_year), ( + "derived_birth_year" + ) + if family_rows is not None: + ages = family_rows.loc[family_rows["person_id"].eq(int(pid)), "age"] + if len(ages) == 1 and 1 <= int(ages.iloc[0]) <= _MAX_AGE_CODE: + return int(ages.iloc[0]), "wave_age" + return pd.NA, "missing" + + def build_age67_cohort( inputs: Age67Inputs, spec: Age67Spec | None = None ) -> Age67Cohort: @@ -742,9 +1010,16 @@ def build_age67_cohort( year), ``reported_birth_year``, ``weight_raw``, ``weight_multiplier``, ``weight``, ``interview``, ``family_unit_id``, ``sequence``, ``relationship``, ``member_role``, ``stratum``, - ``cluster``, ``marital_status``, ``married``, ``spouse_person_id``, - ``member_married_coresident``, ``spouse_age``, ``spouse_sex``, - ``fu_legal_wife_present``, ``wife_sex``, ``wealth_status``, + ``cluster``, ``marital_status``, ``marital_resolution`` + (``marriage_history`` or the path an unresolved state took), + ``married``, ``spouse_person_id``, ``member_married_coresident``, + ``spouse_age``, ``spouse_age_source``, ``spouse_sex``, + ``fu_legal_wife_present``, ``wife_sex``, the head and the head's legal + spouse (``fu_head_person_id``, ``fu_head_age``, + ``fu_head_age_source``, ``fu_head_sex``, ``fu_head_spouse_present``, + ``fu_head_spouse_person_id``, ``fu_head_spouse_age``, + ``fu_head_spouse_age_source``, ``fu_head_spouse_sex``; ages are + income-year ages under ``annuitant_age_source``), ``wealth_status``, ``in_institution`` and ``income_status`` (``family_file`` for a member of the family unit, ``family_of_record`` for an institutionalized member under that rule). @@ -766,6 +1041,10 @@ def build_age67_cohort( if record.birth_year in target_years } episodes = _episodes(inputs.marriage_history, targets) + all_births = { + **_annuitant_birth_years(inputs, universe, targets), + **births, + } with_history = set( int(pid) for pid in inputs.marriage_history["person_id"] ) @@ -855,6 +1134,7 @@ def build_age67_cohort( continue disposition_rows.append({**base, "disposition": "observation"}) family_rows = by_family.get(interview) + head, head_spouse, _ = _head_and_legal_spouse(family_rows) if pid in episodes: state = psid2010.marital_state_at( episodes[pid], @@ -871,10 +1151,14 @@ def build_age67_cohort( "spouse_person_id": pd.NA, } spouse = state.get("spouse_person_id") - # An unresolved state ("unknown", "no_marriage_history") is - # classified by spec.unresolved_marital_status; its only built - # value, "non_married", leaves it out of "married". married = state["status"] == "married" + resolution = "marriage_history" + if state["status"] in _UNRESOLVED_STATES: + # spec.unresolved_marital_status decides (referee Q7). + resolution, spouse = _resolve_by_relationship( + spec, in_institution, pid, relationship, head, head_spouse + ) + married = resolution != "unresolved_non_married" coresident = ( married and not pd.isna(spouse) @@ -882,16 +1166,11 @@ def build_age67_cohort( and int(spouse) in set(family_rows["person_id"].astype(int)) and sequence <= _SEQUENCE_IN_FAMILY[1] ) - spouse_age = pd.NA + spouse_age, spouse_age_source = pd.NA, "absent" spouse_sex = pd.NA if coresident: - spouse_row = family_rows.set_index("person_id").loc[ - int(spouse) - ] - raw_age = int(spouse_row["age"]) - # Individual-file age codes 999 (NA) and 0 carry no age. - spouse_age = ( - raw_age if 1 <= raw_age <= _MAX_AGE_CODE else pd.NA + spouse_age, spouse_age_source = _annuitant_age( + int(spouse), income_year, family_rows, all_births, spec ) spouse_sex = str(sex.get(int(spouse), "na")) wife_sex = pd.NA @@ -907,6 +1186,14 @@ def build_age67_cohort( wife_sex = str( sex.get(int(wives["person_id"].iloc[0]), "na") ) + # The annuitants under fu_head_rule: the family head and the + # head's co-resident legal spouse (code 20 or 90). + head_age, head_age_source = _annuitant_age( + head, income_year, family_rows, all_births, spec + ) + head_spouse_age, head_spouse_age_source = _annuitant_age( + head_spouse, income_year, family_rows, all_births, spec + ) # An institutionalized member is neither the current head nor # the wife of its family of record: its relationship code is to # the previous wave's head (codebook note on ER34103). @@ -954,13 +1241,32 @@ def build_age67_cohort( else pd.NA ), "marital_status": state["status"], + "marital_resolution": resolution, "married": bool(married), "spouse_person_id": spouse, "member_married_coresident": bool(coresident), "spouse_age": spouse_age, + "spouse_age_source": spouse_age_source, "spouse_sex": spouse_sex, "fu_legal_wife_present": legal_wife, "wife_sex": wife_sex, + "fu_head_person_id": pd.NA if head is None else head, + "fu_head_age": head_age, + "fu_head_age_source": head_age_source, + "fu_head_sex": ( + pd.NA if head is None else str(sex.get(head, "na")) + ), + "fu_head_spouse_present": head_spouse is not None, + "fu_head_spouse_person_id": ( + pd.NA if head_spouse is None else head_spouse + ), + "fu_head_spouse_age": head_spouse_age, + "fu_head_spouse_age_source": head_spouse_age_source, + "fu_head_spouse_sex": ( + pd.NA + if head_spouse is None + else str(sex.get(head_spouse, "na")) + ), "wealth_status": ( "family_file" if wave in inputs.family_wealth @@ -972,16 +1278,31 @@ def build_age67_cohort( ), } ) + observations = pd.DataFrame(observation_rows) dispositions = pd.DataFrame(disposition_rows) if not observations.empty: _check_family_records(observations, inputs) - for column in ("spouse_person_id", "spouse_age", "stratum", "cluster"): + for column in ( + "spouse_person_id", + "spouse_age", + "stratum", + "cluster", + "fu_head_person_id", + "fu_head_age", + "fu_head_spouse_person_id", + "fu_head_spouse_age", + ): observations[column] = observations[column].astype("Int64") observations["reported_birth_year"] = observations[ "reported_birth_year" ].astype("Int64") - for column in ("spouse_sex", "wife_sex"): + for column in ( + "spouse_sex", + "wife_sex", + "fu_head_sex", + "fu_head_spouse_sex", + ): observations[column] = observations[column].astype("string") cohort = Age67Cohort( observations=observations, @@ -1074,7 +1395,7 @@ def income_rows( ) frames.append(merged) out = ( - pd.concat(frames, ignore_index=True) + pd.concat(frames, ignore_index=True).copy() if frames else pd.DataFrame(columns=obs.columns) ) @@ -1155,6 +1476,13 @@ def structural_summary( ).sum() ), "marital_status": _counts(rows["marital_status"]), + "marital_resolution": _counts(rows["marital_resolution"]), + "spouse_age_source": _counts(rows["spouse_age_source"]), + "fu_head_age_source": _counts(rows["fu_head_age_source"]), + "fu_head_spouse_present": _counts(rows["fu_head_spouse_present"]), + "fu_head_spouse_age_source": _counts( + rows["fu_head_spouse_age_source"] + ), "wealth_status": _counts(rows["wealth_status"]), "income_status": _counts(rows["income_status"]), "birth_source": _counts(rows["birth_source"]), @@ -1208,6 +1536,22 @@ def structural_summary( != 0 ).sum() ), + # Specification section 4 (F4a, F4b): the families whose head + # has annuity or IRA income and whose farm income is nonzero. + "family_head_retirement_account_income_nonzero": int( + ( + fam[ + [ + c + for c in ("head_annuities", "head_iras") + if c in fam + ] + ].sum(axis=1) + != 0 + ).sum() + ), + "family_farm_income_nonzero": int((fam["head_farm"] != 0).sum()), + "family_farm_income_negative": int((fam["head_farm"] < 0).sum()), "social_security_accuracy_nonzero": ( int( (fam[["head_ss_acc", "wife_ss_acc", "ofum_ss_acc"]] != 0) diff --git a/src/populace_dynamics/data/family_income.py b/src/populace_dynamics/data/family_income.py index 87411343..ebc496a7 100644 --- a/src/populace_dynamics/data/family_income.py +++ b/src/populace_dynamics/data/family_income.py @@ -40,8 +40,10 @@ (ER52315), OFUM transfer income (ER52336), and head, wife and OFUM Social Security (ER52337, ER52339, ER52341). The 2013 file splits the wife's retirement income into pensions, annuities, IRAs and other retirement and -adds head IRAs (``HEAD IRAS-2012``); earlier files carry one ``WIFE -RETIREMENT/ANNUITIES`` item. Head and wife here are the PSID's; "wife" +splits the head's ``HEAD ANNUITIES`` into annuities and ``HEAD +IRAS-2012``; earlier files carry one ``WIFE RETIREMENT/ANNUITIES`` item +("Income from Pensions and Annuities"), and their ``HEAD ANNUITIES`` is +"Head's Income from Annuities and IRAs" (2005-2011 codebooks). Head and wife here are the PSID's; "wife" includes a cohabiting female partner ("Wife"). The accuracy flag of each Social Security, SSI and asset item is read raw (:data:`ACCURACY_CONCEPTS`; 0 is "Actual value"). diff --git a/src/populace_dynamics/estimates/adjusted_poverty.py b/src/populace_dynamics/estimates/adjusted_poverty.py index 6033e385..8a292c13 100644 --- a/src/populace_dynamics/estimates/adjusted_poverty.py +++ b/src/populace_dynamics/estimates/adjusted_poverty.py @@ -23,21 +23,41 @@ unit (head and wife rent, dividends, interest, trusts/royalties and the asset part of business income; the OFUM asset total under the FU basis) is removed and the annuity added. ``keep`` (row U5): the annuity is - added and reported asset income kept. + added and reported asset income kept, and nothing below is removed. + Under ``replace`` two more items go (specification section 4): + ``retirement_account_income_rule="remove_head"`` (F4a) removes the + head's income from annuities and IRAs (``HEAD ANNUITIES``, "Head's + Income from Annuities and IRAs" in the 2005-2011 codebooks; ``HEAD + ANNUITIES`` plus ``HEAD IRAS`` in 2013), because WEALTH1 holds the + balances that produce it (W22, "private annuities or IRAs") and they + are annuitized; ``keep`` leaves it in income. The wife's and the + OFUMs' combined retirement items stay in every wave. + ``farm_asset_share`` (F4b, 0.5) imputes the asset part of farm income + by the PSID's business convention: that share of positive farm income, + and the whole of a farm loss when the share is positive, is removed; + 0 keeps farm income whole. * **Annuity** (F5-F9). ``annuitized_share`` (0.8) times the FU's imputed wealth excluding home equity (WEALTH1), floored at zero (F9), divided by the price of a real, level annuity of 1 per year: annuity-immediate (payments at the end of each year the annuitant survives) at ``real_interest_rate`` (3 percent: the real return on - government bonds after 2001 that the plan reads on the Report's p. 22, - a choice, not a known DYNASIM annuity parameter), no load, on the NCHS - 2000 life tables by sex (``mortality_basis``; row U9 the SSA period - life table for 2004). - A married member with a co-resident spouse gets a joint annuity that - pays 1 while both live and ``survivor_share`` (0.5) to the survivor; - under independent lives its price is + government bonds after 2001 that the Report assumes for DC and IRA + accumulation, p. 22; the same paragraph subtracts 1 percentage point + for administrative costs, so 2 percent is a reported sensitivity; + neither is stated to be DYNASIM's annuity-pricing rate), no load, on + the NCHS 2000 life tables by sex (``mortality_basis``; row U9 the SSA + period life table for 2004). + ``annuity_lives="fu_head_rule"`` (default, the referee's Q1 answer) + prices the family's WEALTH1 on the lives of its main owners: a joint + annuity on the family head and the head's co-resident legal spouse + (code 20 or 90) that pays 1 while both live and ``survivor_share`` + (0.5) to the survivor, a single life on the head otherwise, so every + member of a family unit gets the same annuity. ``member_rule`` prices + it on the member and a co-resident legal spouse (joint) or the member + alone. Under independent lives the joint price is ``s*a_x + s*a_y + (1-2s)*a_xy`` (``0.5*(a_x + a_y)`` at s = 0.5). - Anyone else gets a single-life annuity on the member's age and sex. + Ages are the builder's income-year ages + (:mod:`populace_dynamics.cohorts.age67`, ``annuitant_age_source``). * **Threshold** (F10). ``census_weighted_average_65plus``: the Census weighted-average poverty threshold of the income year for the unit's size, using the "65 years and over" column for sizes 1 and 2 (the @@ -106,6 +126,10 @@ __all__ = [ "ANNUITY_LIVES", "ASSET_INCOME_RULES", + "CUT_START_YEAR_RULE", + "FARM_LOSS_RULE", + "RETIREMENT_ACCOUNT_INCOME_CONCEPTS", + "RETIREMENT_ACCOUNT_INCOME_RULES", "DATA_PROVENANCES", "INCOME_UNITS", "INVENTED", @@ -197,15 +221,42 @@ ), } ANNUITY_LIVES: dict[str, str] = { + "fu_head_rule": ( + "joint-and-survivor on the FU head and the head's co-resident legal " + "spouse (code 20 or 90) if present, else single life on the head: " + "the family's wealth priced on its main owners (referee Q1)" + ), "member_rule": ( "joint-and-survivor on the member and a co-resident legal spouse " - "if married, else single life on the member (F6 as written)" + "if married, else single life on the member (F6 as the plan wrote " + "it)" ), - "fu_head_rule": ( - "joint-and-survivor on the FU head and a co-resident legal wife if " - "present, else single life on the head (not registered)" +} +RETIREMENT_ACCOUNT_INCOME_RULES: dict[str, str] = { + "remove_head": ( + "under asset_income_rule 'replace', remove the head's income from " + "annuities and IRAs (HEAD ANNUITIES; plus HEAD IRAS in 2013), whose " + "balances WEALTH1 annuitizes (F4a, referee R3)" ), + "keep": "keep the head's annuity and IRA income (u1-draft-3 behaviour)", } +#: The head's annuity and IRA income items (``head_iras`` exists in the +#: 2013 family file only; earlier files fold IRA income into +#: ``head_annuities``, "Head's Income from Annuities and IRAs"). +RETIREMENT_ACCOUNT_INCOME_CONCEPTS: tuple[str, ...] = ( + "head_annuities", + "head_iras", +) +#: How a farm loss is treated under ``farm_asset_share`` (F4b). +FARM_LOSS_RULE = "removed_whole_when_share_positive" +#: The rule ``cut_start_year`` applies (row U6). +CUT_START_YEAR_RULE = "cut_when_birth_year_plus_67_at_or_after_start" +#: Income years whose family file carries ``head_iras``. +_HEAD_IRA_INCOME_YEARS: frozenset[int] = frozenset( + wave - 1 + for wave in family_income.INCOME_WAVES + if "head_iras" in family_income.income_variables(wave) +) MORTALITY_BASES: dict[str, str] = { "nchs_2000": "NCHS United States Life Tables, 2000, by sex (F8)", "ssa_period_2004": ( @@ -327,7 +378,9 @@ class AdjustedPovertySpec: terminal_closure: str = "table_end" asset_income_rule: str = "replace" income_unit: str = "family_unit" - annuity_lives: str = "member_rule" + retirement_account_income_rule: str = "remove_head" + farm_asset_share: float = 0.5 + annuity_lives: str = "fu_head_rule" negative_wealth_rule: str = "annuity_floored_at_zero" threshold_rule: str = "census_weighted_average_65plus" ssi_rule: str = "offset_existing_recipients" @@ -340,6 +393,7 @@ def __post_init__(self) -> None: ("annuitized_share", 0.0, 1.0), ("survivor_share", 0.0, 1.0), ("annuity_load", 0.0, 1.0), + ("farm_asset_share", 0.0, 1.0), ): value = getattr(self, name) if isinstance(value, bool) or not (low <= float(value) <= high): @@ -366,6 +420,11 @@ def __post_init__(self) -> None: self.asset_income_rule, ASSET_INCOME_RULES, "asset_income_rule" ) _choice(self.income_unit, INCOME_UNITS, "income_unit") + _choice( + self.retirement_account_income_rule, + RETIREMENT_ACCOUNT_INCOME_RULES, + "retirement_account_income_rule", + ) _choice(self.annuity_lives, ANNUITY_LIVES, "annuity_lives") _choice( self.negative_wealth_rule, @@ -403,6 +462,7 @@ def as_dict(self) -> dict[str, Any]: _PLAN = "plan proposal (critical-path-uniform-cut-20260923.md section 7)" _BUILDER = "builder choice; the plan is silent" +_REFEREE = "referee (boomers2004-referee-20260924.md)" _MAX_D189 = "Max (cos decision d189, open)" _FREEZE = "U1 specification freeze (Max's ratification by merge)" @@ -426,10 +486,10 @@ def pending_decisions() -> tuple[PendingDecision, ...]: "cut_rate", spec.cut_rate, (), - "plan section 2, citing Report p. 22 (OCACT: benefits reduced " - "immediately by 13 " - "percent); the scenario's start year and coverage are unread " - "(plan section 10 decisions 2 and 8)", + "Report p. 22 (OCACT estimates 'that benefits would need to be " + "reduced immediately by 13 percent'); the scenario's start " + "year and coverage are not in the methods (plan section 10 " + "decisions 2 and 8)", _FREEZE, ), PendingDecision( @@ -460,6 +520,32 @@ def pending_decisions() -> tuple[PendingDecision, ...]: f"{_PLAN} F4; U5", _FREEZE, ), + PendingDecision( + "retirement_account_income_rule", + spec.retirement_account_income_rule, + ("keep",), + f"{_REFEREE} R3: HEAD ANNUITIES is 'Head's Income from " + "Annuities and IRAs' in the 2005-2011 codebooks (HEAD " + "ANNUITIES plus HEAD IRAS in 2013); WEALTH1 includes the " + "balances ('private annuities or IRAs', W22) and annuitizes " + "them, and the Report contrasts its annuity with Census " + "income, which counts withdrawals (p. 24); applies only under " + "asset_income_rule 'replace'", + _FREEZE, + ), + PendingDecision( + "farm_asset_share", + spec.farm_asset_share, + (0.0,), + f"{_REFEREE} Q2/R4: the Report's financial assets include farm " + "equity (p. 22); the PSID reports one farm item with labor and " + "asset portions and splits a working owner's business income " + "equally, coding a loss wholly as asset income (codebook text " + "for ER52216); the same convention imputes the asset part of " + "farm income (0 keeps farm income whole, the u1-draft-3 " + "behaviour); applies only under asset_income_rule 'replace'", + _FREEZE, + ), PendingDecision( "annuitized_share", spec.annuitized_share, @@ -471,11 +557,14 @@ def pending_decisions() -> tuple[PendingDecision, ...]: PendingDecision( "real_interest_rate", spec.real_interest_rate, - (), - f"{_PLAN} F7 (the plan cites Report p. 22: 3.0 percent real " - "return on " - "government bonds after 2001); not a known DYNASIM annuity " - "parameter", + (0.02,), + f"{_PLAN} F7: Report p. 22 assumes a 3.0 percent real return " + "on government bonds after 2001 for DC and IRA accumulation, " + "and the same paragraph subtracts 1 percentage point from " + "stock and bond returns for administrative costs (net 2 " + "percent); neither is stated to be DYNASIM's annuity-pricing " + f"rate; 0.02 is a reported, unscored sensitivity ({_REFEREE} " + "R6)", _FREEZE, ), PendingDecision( @@ -518,10 +607,13 @@ def pending_decisions() -> tuple[PendingDecision, ...]: PendingDecision( "annuity_lives", spec.annuity_lives, - ("fu_head_rule",), - f"{_PLAN} F6 as written (member's age and sex if non-married; " - "joint if married); the plan does not say whose lives price " - "an OFUM member's FU wealth", + ("member_rule",), + f"{_REFEREE} Q1/R7: WEALTH1 is the family unit's wealth, " + "which the head and the head's spouse mostly own, so it is " + "priced on their lives (joint on the head and a co-resident " + "legal spouse, code 20 or 90, else the head); the rules " + "coincide for heads and legal wives and differ for OFUMs and " + "cohabiting partners; member_rule is plan F6 as written", _FREEZE, ), PendingDecision( @@ -544,7 +636,13 @@ def pending_decisions() -> tuple[PendingDecision, ...]: spec.ssi_deeming, ("recipients_only",), f"{_BUILDER} (PSID reports SSI and Social Security by head, " - "wife and OFUM total, not by SSI unit)", + "wife and OFUM total, not by SSI unit); endorsed as the " + f"default by the {_REFEREE} (Q3): under 20 CFR 416.1163 " + "nothing is deemed when the ineligible spouse's income after " + "allocations is at most the couple-minus-individual FBR, so " + "full attribution can only overstate the offset; " + "recipients_only errs the other way, and the two bracket the " + "rule", _FREEZE, ), PendingDecision( @@ -997,11 +1095,11 @@ def _countable_ss_fall(ss: float, cut_rate: float, general: float) -> float: "member_married_coresident", "spouse_age", "spouse_sex", - "head_age", - "head_sex", - "wife_age", - "wife_sex", - "fu_legal_wife_present", + "fu_head_age", + "fu_head_sex", + "fu_head_spouse_present", + "fu_head_spouse_age", + "fu_head_spouse_sex", "wife_present", "fu_size", "n_children", @@ -1016,6 +1114,7 @@ def _countable_ss_fall(ss: float, cut_rate: float, general: float) -> float: "wife_tanf", "head_other_welfare", "wife_other_welfare", + "head_annuities", "census_needs_standard", "wealth1", "vehicles", @@ -1056,6 +1155,56 @@ def _validate_provenance( ) +def _head_retirement_account_income(rows: pd.DataFrame) -> np.ndarray: + """The head's annuity and IRA income per row (F4a). + + ``head_annuities`` in every wave plus ``head_iras`` where the wave's + family file carries it (2013, income year 2012); a row of such a year + without it, or a row of another year with it nonzero, is refused. + """ + + total = rows["head_annuities"].to_numpy(dtype=np.float64) + years = rows["income_year"].astype("int64").to_numpy() + with_iras = np.isin(years, sorted(_HEAD_IRA_INCOME_YEARS)) + if "head_iras" in rows.columns: + iras = pd.to_numeric(rows["head_iras"], errors="raise") + missing = iras.isna().to_numpy() + if (missing & with_iras).any(): + raise AdjustedPovertyError( + "head_iras missing for an income year whose family file " + "carries HEAD IRAS" + ) + iras = iras.fillna(0.0).to_numpy(dtype=np.float64) + if ((iras != 0) & ~with_iras).any(): + raise AdjustedPovertyError( + "head_iras nonzero for an income year whose family file " + "has no HEAD IRAS item" + ) + total = total + iras + elif with_iras.any(): + raise AdjustedPovertyError( + "rows of income year 2012 need head_iras (HEAD IRAS-2012)" + ) + return total + + +def _farm_asset_income( + rows: pd.DataFrame, spec: AdjustedPovertySpec +) -> np.ndarray: + """The imputed asset part of head and wife farm income (F4b). + + ``farm_asset_share`` of positive farm income; the whole of a farm + loss when the share is positive (the PSID codes a business loss wholly + in the asset part, codebook text for ER52216); nothing at share 0. + """ + + farm = rows["head_farm"].to_numpy(dtype=np.float64) + share = float(spec.farm_asset_share) + if share == 0.0: + return np.zeros(len(rows), dtype=np.float64) + return np.where(farm > 0, share * farm, farm) + + def _unit_frame(rows: pd.DataFrame, spec: AdjustedPovertySpec) -> dict: ofum_member = rows["member_role"].eq("ofum").to_numpy() fu = np.ones(len(rows), dtype=bool) @@ -1104,10 +1253,13 @@ def _annuity_factors( (str(row.spouse_sex), _int_or_none(row.spouse_age)), ) else: - joint = bool(row.fu_legal_wife_present) + joint = bool(row.fu_head_spouse_present) lives = ( - (str(row.head_sex), _int_or_none(row.head_age)), - (str(row.wife_sex), _int_or_none(row.wife_age)), + (str(row.fu_head_sex), _int_or_none(row.fu_head_age)), + ( + str(row.fu_head_spouse_sex), + _int_or_none(row.fu_head_spouse_age), + ), ) (sex_1, age_1), (sex_2, age_2) = lives if age_1 is None or (joint and age_2 is None): @@ -1244,7 +1396,9 @@ def adjusted_incomes( and the annuitant and threshold attributes. Returns one row per input row with ``observation_id``, ``family_unit_id``, ``income_basis`` (``family_unit`` or ``head_wife``), ``money_income``, - ``asset_income_reported``, ``asset_income_removed``, + ``asset_income_reported``, ``retirement_account_income_removed``, + ``farm_asset_income_removed``, ``asset_income_removed`` (their sum + with the reported asset income under ``replace``), ``financial_assets``, ``annuity_basis``, ``annuity_factor``, ``annuity``, ``baseline_income``, ``social_security``, ``cut_applies``, ``cut``, @@ -1297,10 +1451,18 @@ def adjusted_incomes( if not np.all(factors > 0): raise AdjustedPovertyError("non-positive annuity factor") annuity = annuitized / factors + replace = spec.asset_income_rule == "replace" + zeros = np.zeros(len(rows), dtype=np.float64) + retirement_removed = ( + _head_retirement_account_income(rows) + if replace and spec.retirement_account_income_rule == "remove_head" + else zeros + ) + farm_removed = _farm_asset_income(rows, spec) if replace else zeros removed = ( - units["asset_income"] - if spec.asset_income_rule == "replace" - else np.zeros(len(rows)) + (units["asset_income"] if replace else zeros) + + retirement_removed + + farm_removed ) baseline = units["money"] - removed + annuity applies = cut_applies(rows, spec) @@ -1336,6 +1498,8 @@ def adjusted_incomes( "income_basis": np.where(units["fu"], "family_unit", "head_wife"), "money_income": units["money"], "asset_income_reported": units["asset_income"], + "retirement_account_income_removed": retirement_removed, + "farm_asset_income_removed": farm_removed, "asset_income_removed": removed, "financial_assets": assets, "annuity_basis": bases, diff --git a/src/populace_dynamics/estimates/uniform_cut_tabulation.py b/src/populace_dynamics/estimates/uniform_cut_tabulation.py index f05913e8..72770626 100644 --- a/src/populace_dynamics/estimates/uniform_cut_tabulation.py +++ b/src/populace_dynamics/estimates/uniform_cut_tabulation.py @@ -48,10 +48,16 @@ ratio with the PSID sampling-error stratum and cluster (ER31996, ER31997): ``z_i = w_i (y_i - r) / sum w`` inside the cell and 0 outside it, summed by cluster; ``var = sum_h n_h/(n_h - 1) sum_c (z_hc - - mean_h)^2`` over the strata and clusters present in the input rows. - Strata with a single cluster are left out and counted. This is the - subpopulation estimator relative to the tabulated rows, not to the whole - PSID sample; clusters with no tabulated row do not enter. + mean_h)^2``. ``design_se_domain="full_sample_design"`` (default, the + specification's section 10 after the referee's Q5): a domain estimator + on the full sample design, where every (stratum, cluster) pair of the + ``design`` frame (the pairs of persons with a positive cross-section + weight in the observation waves) enters, with ``z = 0`` for a cluster + that holds no observation of the cell; a row whose pair is not in the + frame is refused. ``tabulated_rows`` (the u1-draft-3 estimator) uses + only the pairs present in the input rows. Strata with a single + cluster cannot contribute a variance term: they are left out, counted + and listed. Provenance: ``data_provenance="invented"`` prefixes the invented-data label; ``"registered_real"`` requires the issue #42 registration pointer, @@ -82,6 +88,7 @@ __all__ = [ "CELL_DEFINITIONS", "DEFAULT_CELLS", + "DESIGN_SE_DOMAINS", "DEFAULT_FLOOR_SEEDS", "DIAGNOSTIC_BIRTH_YEAR_CELLS", "FLOOR_FRACTION", @@ -160,6 +167,18 @@ #: The half-split unit: family units merged through shared persons #: (:func:`floor_split_units`). FLOOR_SPLIT_UNIT = "family_unit_id_linked_by_person_id" +#: The population of clusters the design-based standard error sums over. +DESIGN_SE_DOMAINS: dict[str, str] = { + "full_sample_design": ( + "domain estimation on the full sample design: every (stratum, " + "cluster) pair of the design frame enters, z = 0 for clusters " + "without an observation of the cell (referee Q5)" + ), + "tabulated_rows": ( + "the (stratum, cluster) pairs present in the tabulated rows only " + "(u1-draft-3)" + ), +} class UniformCutTabulationError(ValueError): @@ -174,6 +193,7 @@ class TabulationConfig: birth_year_diagnostics: bool = True floor_seeds: tuple[int, ...] = DEFAULT_FLOOR_SEEDS design_standard_errors: bool = True + design_se_domain: str = "full_sample_design" notes: tuple[str, ...] = field(default=()) def __post_init__(self) -> None: @@ -189,6 +209,10 @@ def __post_init__(self) -> None: raise UniformCutTabulationError( "floor seeds must be distinct >= 0" ) + if self.design_se_domain not in DESIGN_SE_DOMAINS: + raise UniformCutTabulationError( + f"design_se_domain must be one of {sorted(DESIGN_SE_DOMAINS)}" + ) def as_dict(self) -> dict[str, Any]: return { @@ -199,6 +223,7 @@ def as_dict(self) -> dict[str, Any]: "floor_split_unit": FLOOR_SPLIT_UNIT, "min_floor_seeds": MIN_FLOOR_SEEDS, "design_standard_errors": self.design_standard_errors, + "design_se_domain": self.design_se_domain, "draws": 1, "notes": list(self.notes), } @@ -245,6 +270,8 @@ def _normalize(rows: pd.DataFrame) -> pd.DataFrame: "person_id, stratum, cluster and family_unit_id must be present" ) out["birth_year"] = out["birth_year"].astype("int64") + for column in ("stratum", "cluster"): + out[column] = out[column].astype("int64") return out @@ -293,8 +320,30 @@ def _rates(rows: pd.DataFrame, mask: np.ndarray) -> dict[str, Any]: } +def _design_clusters(design: pd.DataFrame | None) -> pd.MultiIndex | None: + """The distinct (stratum, cluster) pairs of a design frame.""" + + if design is None: + return None + if not isinstance(design, pd.DataFrame): + raise UniformCutTabulationError("design must be a DataFrame") + missing = [c for c in ("stratum", "cluster") if c not in design.columns] + if missing: + raise UniformCutTabulationError(f"design lacks columns {missing}") + pairs = design[["stratum", "cluster"]] + if pairs.isna().any().any(): + raise UniformCutTabulationError("design stratum/cluster missing") + pairs = pairs.astype("int64").drop_duplicates() + if pairs.empty: + raise UniformCutTabulationError("the design frame has no clusters") + return pd.MultiIndex.from_frame(pairs.sort_values(["stratum", "cluster"])) + + def _design_se( - rows: pd.DataFrame, mask: np.ndarray, indicator: np.ndarray + rows: pd.DataFrame, + mask: np.ndarray, + indicator: np.ndarray, + clusters_all: pd.MultiIndex | None = None, ) -> dict[str, Any]: weight = rows["weight"].to_numpy() in_cell = weight * mask @@ -307,6 +356,8 @@ def _design_se( {"stratum": rows["stratum"], "cluster": rows["cluster"], "z": z} ) clusters = frame.groupby(["stratum", "cluster"], sort=True)["z"].sum() + if clusters_all is not None: + clusters = clusters.reindex(clusters_all, fill_value=0.0) variance = 0.0 used = 0 singleton = 0 @@ -326,7 +377,11 @@ def _design_se( def _cell_entry( - rows: pd.DataFrame, name: str, mask: np.ndarray, config: TabulationConfig + rows: pd.DataFrame, + name: str, + mask: np.ndarray, + config: TabulationConfig, + clusters_all: pd.MultiIndex | None = None, ) -> dict[str, Any]: entry: dict[str, Any] = { "cell": name, @@ -348,9 +403,9 @@ def _cell_entry( base = rows["poor_baseline"].to_numpy().astype(np.float64) reform = rows["poor_reform"].to_numpy().astype(np.float64) entry["design_se"] = { - "delta": _design_se(rows, mask, reform - base), - "baseline_rate": _design_se(rows, mask, base), - "reform_rate": _design_se(rows, mask, reform), + "delta": _design_se(rows, mask, reform - base, clusters_all), + "baseline_rate": _design_se(rows, mask, base, clusters_all), + "reform_rate": _design_se(rows, mask, reform, clusters_all), } return entry @@ -508,14 +563,18 @@ def tabulate_uniform_cut( labels: Sequence[str] = OUTPUT_LABELS, upstream_spec: dict[str, Any] | None = None, pending_decisions: Sequence[dict[str, Any]] = (), + design: pd.DataFrame | None = None, ) -> dict[str, Any]: """Tabulate the exercise-2 statistic from per-observation rows. ``rows`` has the columns of :data:`REQUIRED_COLUMNS` (extra columns are ignored). ``upstream_spec`` (the income-concept specification that produced the poverty flags) and ``pending_decisions`` are recorded - verbatim. Returns a JSON-serializable mapping; an undefined cell is - reported with its reason, never raised. + verbatim. ``design`` (columns ``stratum`` and ``cluster``) is the + sample design the ``full_sample_design`` standard error sums over; it + is required when design standard errors use that domain. Returns a + JSON-serializable mapping; an undefined cell is reported with its + reason, never raised. """ config = TabulationConfig() if config is None else config @@ -569,8 +628,43 @@ def tabulate_uniform_cut( ) normalized = _normalize(rows) masks = _cell_masks(normalized, config) + clusters_all = None + design_summary: dict[str, Any] = {"domain": config.design_se_domain} + if config.design_standard_errors: + if config.design_se_domain == "full_sample_design": + clusters_all = _design_clusters(design) + if clusters_all is None: + raise UniformCutTabulationError( + "the full_sample_design standard error needs the " + "design frame (every stratum and cluster of the " + "observation waves' positive-weight persons)" + ) + present = pd.MultiIndex.from_frame( + normalized[["stratum", "cluster"]].astype("int64") + ) + outside = present[~present.isin(clusters_all)].unique() + if len(outside): + raise UniformCutTabulationError( + f"{len(outside)} (stratum, cluster) pairs of the rows " + f"are not in the design frame (first: {list(outside)[:3]})" + ) + sizes = pd.Series(1, index=clusters_all).groupby(level=0).sum() + else: + sizes = ( + normalized[["stratum", "cluster"]] + .drop_duplicates() + .groupby("stratum")["cluster"] + .count() + ) + design_summary.update( + n_strata=int(len(sizes)), + n_clusters=int(sizes.sum()), + singleton_strata=[int(s) for s in sizes[sizes < 2].index], + ) + elif design is not None: + _design_clusters(design) cells = [ - _cell_entry(normalized, name, mask, config) + _cell_entry(normalized, name, mask, config, clusters_all) for name, mask in masks.items() ] per_seed, floors = _floors(normalized, masks, config) @@ -594,6 +688,7 @@ def tabulate_uniform_cut( "statistic_definitions": dict(STATISTIC_DEFINITIONS), "upstream_spec": dict(upstream_spec or {}), "pending_decisions": [dict(item) for item in pending_decisions], + "design": design_summary, "input_summary": { "n_observations": int(len(normalized)), "n_persons": int(normalized["person_id"].nunique()), diff --git a/src/populace_dynamics/uniform_cut_track_u/invented.py b/src/populace_dynamics/uniform_cut_track_u/invented.py index 00f00a28..43b38167 100644 --- a/src/populace_dynamics/uniform_cut_track_u/invented.py +++ b/src/populace_dynamics/uniform_cut_track_u/invented.py @@ -26,16 +26,31 @@ couples in the target cohort with an earning adult child as an OFUM (U4 drops the child's income for them); * couples still working at 67 with business, farm and rental losses - (negative items the codebooks allow) and large wealth; singles with - negative WEALTH1; cohabiting couples (relationship 22, not married; - the partner's birth year inferred from invented earnings); separated - members whose spouse lives elsewhere; + (negative items the codebooks allow) and large wealth; retired couples + with farm income and head IRA income, and retired singles with annuity + income (the head's annuity and IRA income is ``HEAD ANNUITIES`` before + 2013 and ``HEAD ANNUITIES`` plus ``HEAD IRAS`` in 2013, as in the + codebooks); singles with negative WEALTH1; cohabiting couples + (relationship 22, not married; the partner's birth year inferred from + invented earnings); separated members whose spouse lives elsewhere; +* couples with a legal wife (code 20) in which the head's marriage + history cannot date the marriage (``unknown``) or the wife has no + marriage-history record, and female + heads with a legal husband (code 90) who has no marriage-history + record (the relationship-code resolution of unresolved marital states; + the husband's income is put in the family file's "wife" items, an + invented assumption not checked against the codebooks); * members who enter an institution (sequence 51) in the first wave whose income year is at or after their 66th birthday year and stay, attached to the family they left, where the spouse becomes head (row U-inst); members who move out (71) or die (81) by that wave's interview; and members whose weight is zero from that wave. +Individual-file ages are ages at the interview: a person whose invented +birthday falls before the interview is one year older than the income +year minus the birth year, as in the PSID, so the birth-year law's seed +coordinate and the wave age can differ from the income-year age. + Every family unit's income adds up exactly under the codebook identities (:func:`populace_dynamics.data.family_income.reconcile_family_income`), and WEALTH1 equals its assets less its debts in every wave. Waves 2005 @@ -109,6 +124,8 @@ "negative_wealth_single": 8, "cohabiting_couple": 8, "separated": 4, + "unresolved_couple": 4, + "legal_husband_couple": 3, "institution_spouse": 6, "mover_out": 3, "decedent": 3, @@ -158,6 +175,9 @@ _N_STRATA = 8 _SEQ_INSTITUTION, _SEQ_MOVED, _SEQ_DIED = 51, 71, 81 _HEAD, _WIFE, _PARTNER, _PARENT, _GRANDCHILD = 10, 20, 22, 50, 60 +_LEGAL_HUSBAND = 90 +#: Relationship codes whose person fills the family file's "wife" items. +_SPOUSE_CODES = (_WIFE, _PARTNER, _LEGAL_HUSBAND) #: Individual-income items of an invented person, in 2004 dollars. _ITEMS = ( "labor", @@ -328,6 +348,8 @@ def person( "spouse_slot": spouse_slot, "reported_birth_year": reported, "ss_start_age": int(self.rng.integers(62, 67)), + # 1 when the invented birthday falls before the interview. + "birthday_before_interview": int(self.rng.random() < 0.6), "income": {item: float(income.get(item, 0.0)) for item in _ITEMS}, "noise": { wave: float(np.exp(self.rng.normal(0.0, 0.04))) @@ -398,6 +420,7 @@ def retired_couple(self) -> None: dividends=self.uniform(0, 5_000) if self.chance(0.3) else 0.0, rent=self.uniform(-3_000, 8_000) if self.chance(0.1) else 0.0, ira=self.uniform(1_000, 8_000) if self.chance(0.2) else 0.0, + farm=self.uniform(-2_000, 9_000) if self.chance(0.1) else 0.0, ) self.person( family, @@ -470,6 +493,7 @@ def retired_single(self) -> None: ss=self.uniform(7_000, 17_000), pension=self.uniform(2_000, 12_000) if self.chance(0.3) else 0.0, interest=self.uniform(0, 1_200) if self.chance(0.5) else 0.0, + annuity=self.uniform(500, 6_000) if self.chance(0.2) else 0.0, ) self.steady_roster(family, [(1, 1, _HEAD)]) self.wealth( @@ -766,6 +790,77 @@ def separated(self) -> None: vehicles=self.uniform(0, 8_000), ) + def unresolved_couple(self) -> None: + """A head and legal wife (code 20), one of whom has an unresolved + marital state: the head's history cannot date the marriage, or the + wife has no marriage-history record.""" + + family = self.family("unresolved_couple") + head_birth = self.target_year() + # Either the head's history cannot date the marriage, or the wife + # has no marriage-history record. + head_unknown = self.chance(0.5) + self.person( + family, + 1, + sex="male", + birth_year=head_birth, + marital="unknown" if head_unknown else "married", + spouse_slot=None if head_unknown else 2, + ss=self.uniform(9_000, 16_000), + interest=self.uniform(0, 1_000) if self.chance(0.5) else 0.0, + ) + self.person( + family, + 2, + sex="female", + birth_year=int(head_birth + self.rng.integers(-2, 4)), + marital="married" if head_unknown else "none", + spouse_slot=1 if head_unknown else None, + ss=self.uniform(4_000, 9_000), + ) + self.steady_roster(family, [(1, 1, _HEAD), (2, 2, _WIFE)]) + self.wealth( + family, + checking_saving=self.uniform(1_000, 15_000), + vehicles=self.uniform(1_000, 12_000), + ira_annuity=( + self.uniform(5_000, 60_000) if self.chance(0.5) else 0.0 + ), + home_equity=self.uniform(0, 90_000), + ) + + def legal_husband_couple(self) -> None: + """A female head whose legal husband (code 90) has no + marriage-history record.""" + + family = self.family("legal_husband_couple") + husband_birth = self.target_year() + self.person( + family, + 1, + sex="female", + birth_year=int(husband_birth + self.rng.integers(-1, 4)), + marital="married", + spouse_slot=2, + ss=self.uniform(5_000, 11_000), + ) + self.person( + family, + 2, + sex="male", + birth_year=husband_birth, + marital="none", + ss=self.uniform(9_000, 17_000), + ) + self.steady_roster(family, [(1, 1, _HEAD), (2, 2, _LEGAL_HUSBAND)]) + self.wealth( + family, + checking_saving=self.uniform(1_000, 12_000), + vehicles=self.uniform(1_000, 10_000), + home_equity=self.uniform(0, 80_000), + ) + def institution_spouse(self) -> None: family = self.family("institution_spouse") birth = self.target_year() @@ -1034,7 +1129,7 @@ def _anchor_frames( "interview": _interview(family, wave), "sequence": sequence, "relationship": relationship, - "age": (wave - 1) - int(person["birth_year"]), + "age": _wave_age(person, wave), "reported_birth_year": ( pd.NA if reported is None else int(reported) ), @@ -1052,6 +1147,16 @@ def _anchor_frames( return out +def _wave_age(person: Mapping[str, Any], wave: int) -> int: + """The invented age at the ``wave`` interview (spring of ``wave``).""" + + return ( + (wave - 1) + - int(person["birth_year"]) + + int(person["birthday_before_interview"]) + ) + + def _in_family( family: Mapping[str, Any], wave: int ) -> list[tuple[dict[str, Any], int]]: @@ -1070,7 +1175,7 @@ def _income_row(family: Mapping[str, Any], wave: int) -> dict[str, Any]: row: dict[str, Any] = {concept: 0 for concept in table} members = _in_family(family, wave) head = next(person for person, rel in members if rel == _HEAD) - wives = [person for person, rel in members if rel in (_WIFE, _PARTNER)] + wives = [person for person, rel in members if rel in _SPOUSE_CODES] wife = wives[0] if wives else None ofums = [ person @@ -1093,15 +1198,14 @@ def _income_row(family: Mapping[str, Any], wave: int) -> dict[str, Any]: row["head_farm"] += a["farm"] if role == "head": row["head_retirement_pensions"] += a["pension"] - row["head_annuities"] += a["annuity"] row["head_va_pension"] += a["va"] + row["head_other_retirement"] += a["other_retirement"] if wave == 2013: - row["head_other_retirement"] += a["other_retirement"] + row["head_annuities"] += a["annuity"] row["head_iras"] += a["ira"] else: - row["head_other_retirement"] += ( - a["other_retirement"] + a["ira"] - ) + # "Head's Income from Annuities and IRAs" (2005-2011). + row["head_annuities"] += a["annuity"] + a["ira"] elif wave == 2013: row["wife_retirement_pensions"] += a["pension"] + a["va"] row["wife_annuities"] += a["annuity"] @@ -1151,13 +1255,13 @@ def _income_row(family: Mapping[str, Any], wave: int) -> dict[str, Any]: children = sum( 1 for person in ofums if year - int(person["birth_year"]) < 18 ) - head_age = year - int(head["birth_year"]) + head_age = _wave_age(head, wave) row["interview"] = _interview(family, wave) row["fu_size"] = size row["n_children"] = children row["head_age"] = head_age row["head_sex"] = head["sex"] - row["wife_age"] = None if wife is None else year - int(wife["birth_year"]) + row["wife_age"] = None if wife is None else _wave_age(wife, wave) row["census_needs_standard"] = invented_needs_standard( year, size, children, head_age ) diff --git a/src/populace_dynamics/uniform_cut_track_u/rows.py b/src/populace_dynamics/uniform_cut_track_u/rows.py index 3adaa975..4d26a06a 100644 --- a/src/populace_dynamics/uniform_cut_track_u/rows.py +++ b/src/populace_dynamics/uniform_cut_track_u/rows.py @@ -3,15 +3,22 @@ Each row of the exercise-2 specification (``docs/design/boomers2004_uniform_cut_comparison.md``, sections 11 and 15) differs from the primary U0 in one field, except U6, which is defined -on U1 (plan ``critical-path-uniform-cut-20260923.md`` section 7). A row -here is a set of overrides of :class:`populace_dynamics.cohorts.age67. +on U1 (plan ``critical-path-uniform-cut-20260923.md`` section 7), and +U0-F, U0 restricted to the birth years 1941, 1943 and 1945. A row here +is a set of overrides of :class:`populace_dynamics.cohorts.age67. Age67Spec` and :class:`populace_dynamics.estimates.adjusted_poverty. AdjustedPovertySpec`; the defaults of those two classes are U0. +:data:`HEADLINE_RULE` is the specification's fallback rule (pending Max, +with plan section 10 decision 3): U0 is the headline when the 2005 and +2007 wealth supplements are staged, adjudicated and read before the #42 +registration, U0-F otherwise, by staging status only. + :func:`check_rows_against_block` holds :data:`REGISTERED_ROWS` to the specification's machine-readable block, so a row a run computes is the row the specification registers. U7 (employer DC balances) is not -built: it needs a label investigation of the PSID P-section items. +built: it needs a label investigation of the PSID P-section items, and +it must be built or removed before registration. """ from __future__ import annotations @@ -27,6 +34,8 @@ from populace_dynamics.estimates import adjusted_poverty as ap __all__ = [ + "FALLBACK_ROW", + "HEADLINE_RULE", "PRIMARY_ROW", "REGISTERED_ROWS", "SPECIFICATION_PATH", @@ -41,6 +50,15 @@ _ROOT / "docs" / "design" / "boomers2004_uniform_cut_comparison.md" ) PRIMARY_ROW = "U0" +#: The registered fallback row (U0 on 1941, 1943 and 1945). +FALLBACK_ROW = "U0-F" +#: The specification's headline rule (section 11; pending Max). +HEADLINE_RULE = "u0_if_2005_2007_wealth_staged_before_registration_else_u0f" +#: Block ``population`` values and the builder row they select. +_POPULATION_ROWS = { + "all_ten_birth_years": "U1", + "birth_years_1941_1943_1945": FALLBACK_ROW, +} _AGE67_FIELDS = frozenset(f.name for f in fields(age67.Age67Spec)) _INCOME_FIELDS = frozenset(f.name for f in fields(ap.AdjustedPovertySpec)) #: Block keys that describe a row without being a parameter. @@ -146,6 +164,18 @@ def as_dict(self) -> dict[str, Any]: "year; the scorecard's 'from 2004' has no readable source)" ), ), + TrackURow( + "U0-F", + "population", + "U0 restricted to the birth years 1941, 1943 and 1945 (the " + "blocked 1937 and 1939 left out and counted); the headline when " + "the 2005 and 2007 wealth supplements are not staged before " + "the #42 registration", + age67={"row": FALLBACK_ROW}, + awaiting=( + "Max (the fallback rule, with plan section 10 decision 3)" + ), + ), TrackURow( "U7", "financial assets", @@ -185,8 +215,8 @@ def as_dict(self) -> dict[str, Any]: "institution_income_rule": "family_of_record", }, awaiting=( - "the referee (the institution income rule is a builder " - "default)" + "the specification freeze (the institution income rule is " + "a builder default of u1-draft-3, not yet refereed)" ), ), ) @@ -209,8 +239,10 @@ def row_from_block(row_id: str, entry: Mapping[str, Any]) -> dict[str, Any]: """A block row's overrides: ``age67``, ``income``, ``built``, ``awaiting``. ``population: all_ten_birth_years`` and ``on: U1`` select the U1 - population; keys naming an :class:`~populace_dynamics.cohorts.age67. - Age67Spec` or :class:`~populace_dynamics.estimates.adjusted_poverty. + population; ``population: birth_years_1941_1943_1945`` selects U0-F + (with ``on: U0``, which names its base and changes nothing); keys + naming an :class:`~populace_dynamics.cohorts.age67.Age67Spec` or + :class:`~populace_dynamics.estimates.adjusted_poverty. AdjustedPovertySpec` field are overrides; ``status: not_built`` marks a row not built. Any other key is refused. """ @@ -218,11 +250,15 @@ def row_from_block(row_id: str, entry: Mapping[str, Any]) -> dict[str, Any]: age67_overrides: dict[str, Any] = {} income: dict[str, Any] = {} for key, value in entry.items(): - if key in ("population", "on"): - expected = "all_ten_birth_years" if key == "population" else "U1" - if value != expected: + if key == "population": + if value not in _POPULATION_ROWS: + raise ValueError(f"{row_id}: {key}={value!r} is not known") + age67_overrides["row"] = _POPULATION_ROWS[value] + elif key == "on": + if value not in ("U0", "U1"): raise ValueError(f"{row_id}: {key}={value!r} is not known") - age67_overrides["row"] = "U1" + if value == "U1": + age67_overrides.setdefault("row", "U1") elif key in _AGE67_FIELDS: age67_overrides[key] = value elif key in _INCOME_FIELDS: diff --git a/src/populace_dynamics/uniform_cut_track_u/runner.py b/src/populace_dynamics/uniform_cut_track_u/runner.py index 8f20266e..96505d43 100644 --- a/src/populace_dynamics/uniform_cut_track_u/runner.py +++ b/src/populace_dynamics/uniform_cut_track_u/runner.py @@ -8,13 +8,22 @@ status (:func:`populace_dynamics.estimates.adjusted_poverty. adjusted_incomes`) and tabulate the frozen statistic (:func:`populace_dynamics.estimates.uniform_cut_tabulation. -tabulate_uniform_cut`). For the primary row it also computes the two F17 -diagnostics: the component summaries +tabulate_uniform_cut`) with the design-based standard error on the full +sample design (:func:`design_frame`). For the headline row it also +computes the two F17 diagnostics: the component summaries (:mod:`populace_dynamics.uniform_cut_track_u.diagnostics`) and the official-concept poverty rate (:func:`official_concept_rates`: money income against the same thresholds, reported asset income kept, no annuity, no cut). +The headline row follows the specification's fallback rule +(:data:`populace_dynamics.uniform_cut_track_u.rows.HEADLINE_RULE`, pending +Max): U0 when every wave has WEALTH1, otherwise U0-F; a row whose +observation waves include a wave without WEALTH1 is reported as blocked, +with its structural counts, and not computed (unless ``allow_blocked``, +which leaves the blocked observations out and is refused for a +registered run). The rule depends on staging status only. + Provenance guards, before anything is computed: * ``data_provenance="invented"``: the inputs must be the invented @@ -27,7 +36,8 @@ returned and sealed by :func:`~populace_dynamics.cohorts.age67. load_age67_inputs`); the thresholds must be the committed Census capture, the SSI parameters and life tables the committed ones; no - wealth wave may be refused and no observation left out. + observation may be left out (a wave without WEALTH1 blocks the rows + that need it, and U0-F becomes the headline). The income concept and the tabulation apply their own guards as well. """ @@ -47,6 +57,8 @@ from populace_dynamics.estimates import uniform_cut_tabulation as ut from populace_dynamics.uniform_cut_track_u import diagnostics, invented from populace_dynamics.uniform_cut_track_u.rows import ( + FALLBACK_ROW, + HEADLINE_RULE, PRIMARY_ROW, REGISTERED_ROWS, TrackURow, @@ -59,7 +71,10 @@ "RUN_SCHEMA_VERSION", "TrackUParameters", "TrackURunError", + "blocked_waves", "committed_parameters", + "design_frame", + "headline_row", "official_concept_rates", "run_track_u", ] @@ -72,7 +87,9 @@ r"#issuecomment-[0-9]+" ) OFFICIAL_CONCEPT_CELLS: tuple[str, ...] = ut.DEFAULT_CELLS -#: The named deltas every output carries (specification section 12). +#: The named deltas every output carries (specification section 12, whose +#: bullets hold the same text; ``tests/test_boomers2004_uniform_cut_spec.py`` +#: checks that they agree). NAMED_DELTAS: tuple[str, ...] = ( "PSID versus SIPP wealth measurement", "realized 2004-2012 history versus DYNASIM's 1992-based projection, " @@ -83,19 +100,52 @@ "without the member's own income, which the PSID does not collect); " "attrition", "OFUM-owned assets inside family wealth", + "the family's wealth stands in for an OFUM cohort member's own wealth " + "(the Report's unit is the individual plus spouse, p. 24)", "imputed rent excluded", - "self-reported Social Security, possibly net of Medicare premiums", - "pension income that may include account withdrawals", + "self-reported Social Security, possibly net of Medicare Part B " + "premiums, so the 13 percent cut applies to a smaller base than the " + "gross benefit (baseline income is lower too; the net direction on " + "the change is not established)", + "retirement-account income beside annuitized balances: the head's " + "income from annuities and IRAs is removed (F4a), but the wife's " + "(before 2013) and the OFUMs' items combine pensions with annuity " + "income and stay, so any IRA or annuity income in them is counted " + "twice; an annuity already in payment may have no balance in WEALTH1, " + "so removing its income understates income", + "employer DC (401(k)) balances outside IRAs are not in WEALTH1 though " + "the Report counts them (p. 24): the primary annuity is understated " + "for their holders (row U7)", + "PSID other assets (W34) include cash value of life insurance, " + "collections and rights in a trust or estate, which the Report's list " + "(p. 22) does not name", + "the annuity is priced on population period life tables by age and " + "sex (NCHS 2000; U9 SSA 2004), while DYNASIM's mortality follows the " + "2002 Trustees projections (p. 20, fn. 4) and the Report ties the " + "annuity to family life expectancy (p. 24): with falling mortality, " + "period tables overstate the annuity", "exact-age sampling of alternate birth years (mean birth year 1941 " "against 1940.5)", - "farm income's asset portion is not separable and stays in income; " - "business income is split 50/50 between labor and asset parts by PSID " - "convention for working owners", + "U1 averages ages 66 and 68, and claiming between those ages is not " + "linear in age", + "the universe is alive at the interview after the income year", + "farm income's asset portion is imputed by the PSID business " + "convention (farm_asset_share); business income is split 50/50 " + "between labor and asset parts by PSID convention for working owners", "SSI units approximated from head, wife and OFUM totals (deeming by " "full attribution; OFUMs as one unit)", + "SSI deeming by full attribution can only overstate the offset (so " + "understate the change): under 20 CFR 416.1163 nothing is deemed when " + "the ineligible spouse's income is at most the couple-minus-individual " + "FBR", + "the SSI test uses annual amounts against 12 times the January FBR " + "(SSI accounting is monthly), and U3 omits rent and royalties from " + "countable income although 20 CFR 416.1121 counts them", "reported SSI may include state supplementary payments, so capping " "the offset at the federal benefit rate less reported SSI can " "understate the offset", + "U0-F, if it is the headline: mean birth year 1943 against 1940.5, " + "and every observation year at or after the 2008-09 asset shock", ) @@ -192,6 +242,50 @@ def _check_parameters(params: TrackUParameters, data_provenance: str) -> None: ) +def headline_row(inputs: age67.Age67Inputs) -> str: + """The headline row under the fallback rule (staging status only). + + U0 when every wave's WEALTH1 is available (the 2005 and 2007 wealth + supplements staged, adjudicated and read); U0-F otherwise. + """ + + return FALLBACK_ROW if inputs.wealth_refusals else PRIMARY_ROW + + +def blocked_waves(row: TrackURow, inputs: age67.Age67Inputs) -> list[int]: + """The row's observation waves whose WEALTH1 the inputs refuse.""" + + waves = {w for _, w, _, _ in age67.observation_plan(row.age67_spec())} + return sorted(waves & {int(w) for w in inputs.wealth_refusals}) + + +def design_frame( + inputs: age67.Age67Inputs, spec: age67.Age67Spec +) -> pd.DataFrame: + """The sample design the full-design standard error sums over. + + The distinct (stratum, cluster) pairs (ER31996, ER31997) of every + person with a positive cross-section weight in any of the row's + observation waves (specification section 10). + """ + + waves = sorted({w for _, w, _, _ in age67.observation_plan(spec)}) + positive: set[int] = set() + for wave in waves: + anchor = inputs.anchors[wave] + positive |= set( + anchor.loc[anchor["weight"] > 0, "person_id"].astype(int) + ) + design = inputs.design[inputs.design["person_id"].isin(positive)] + return ( + design[["stratum", "cluster"]] + .astype("int64") + .drop_duplicates() + .sort_values(["stratum", "cluster"]) + .reset_index(drop=True) + ) + + def _check_inputs( inputs: age67.Age67Inputs, data_provenance: str, @@ -222,11 +316,6 @@ def _check_inputs( "age67.load_age67_inputs; these inputs record " f"{(inputs.provenance or {}).get('kind')!r}" ) - if inputs.wealth_refusals: - raise TrackURunError( - "a registered run needs WEALTH1 for every wave; refused: " - f"{sorted(inputs.wealth_refusals)}" - ) if allow_blocked: raise TrackURunError( "a registered run may not leave blocked observations out" @@ -324,6 +413,12 @@ def _income_counts(adjusted: pd.DataFrame) -> dict[str, Any]: "n_asset_income_removed_nonzero": int( (adjusted["asset_income_removed"] != 0).sum() ), + "n_retirement_account_income_removed_nonzero": int( + (adjusted["retirement_account_income_removed"] != 0).sum() + ), + "n_farm_asset_income_removed_nonzero": int( + (adjusted["farm_asset_income_removed"] != 0).sum() + ), "n_ssi_offset_positive": int((adjusted["ssi_offset"] > 0).sum()), "n_ssi_new_positive": int((adjusted["ssi_new"] > 0).sum()), "n_poor_baseline": int(adjusted["poor_baseline"].sum()), @@ -398,6 +493,7 @@ def _compute_row( registration_pointer=registration_pointer, upstream_spec=adjusted.attrs["spec"], pending_decisions=pending, + design=design_frame(inputs, age67_spec), ) obs = cohort.observations result = { @@ -414,6 +510,11 @@ def _compute_row( "n_in_institution": int(members["in_institution"].sum()), "left_out": dict(members.attrs.get("left_out", {})), "observations_by_birth_year": _counts(members["birth_year"]), + "marital_resolution": _counts(members["marital_resolution"]), + "fu_head_age_source": _counts(members["fu_head_age_source"]), + "fu_head_spouse_age_source": _counts( + members["fu_head_spouse_age_source"] + ), "dispositions": ( _counts(cohort.dispositions["disposition"]) if not cohort.dispositions.empty @@ -426,6 +527,41 @@ def _compute_row( return result, members, adjusted, cohort +def _blocked_result( + row: TrackURow, inputs: age67.Age67Inputs, waves: list[int] +) -> dict[str, Any]: + """A row that needs a wave without WEALTH1: counts only, no income.""" + + cohort = age67.build_age67_cohort(inputs, row.age67_spec()) + obs = cohort.observations + blocked = ( + obs["wave"].isin(waves) if not obs.empty else pd.Series(dtype=bool) + ) + return { + "row": row.as_dict(), + "status": "blocked", + "reason": ( + f"WEALTH1 is refused for waves {waves} (the 2005 and 2007 " + "wealth supplements are not staged); reported with its " + "counts under the fallback rule" + ), + "blocked_waves": waves, + "population": { + "n_observations_built": int(len(obs)), + "n_observations_blocked": int(blocked.sum()), + "observations_by_birth_year": ( + _counts(obs["birth_year"]) if not obs.empty else {} + ), + "blocked_by_birth_year": ( + _counts(obs.loc[blocked, "birth_year"]) + if not obs.empty + else {} + ), + }, + "tabulation": None, + } + + def run_track_u( inputs: age67.Age67Inputs, params: TrackUParameters, @@ -439,14 +575,22 @@ def run_track_u( ) -> dict[str, Any]: """Every registered row, the F17 diagnostics and the provenance. - Rows not built (U7) are reported with their reason. An undefined - cell stays undefined with its reason (the tabulation's rule); nothing - is imputed or dropped silently. + Rows not built (U7) are reported with their reason. The headline row + is :func:`headline_row`'s (the fallback rule); a row whose waves lack + WEALTH1 is reported as blocked with its counts unless + ``allow_blocked`` (then its blocked observations are left out and + counted; refused for a registered run). An undefined cell stays + undefined with its reason (the tabulation's rule); nothing is imputed + or dropped silently. """ rows = REGISTERED_ROWS if rows is None else rows - if PRIMARY_ROW not in rows: - raise TrackURunError("the primary row U0 must run") + headline = headline_row(inputs) + if headline not in rows: + raise TrackURunError( + f"the headline row {headline} (fallback rule, staging status) " + "must run" + ) checks = _check_inputs( inputs, data_provenance, registration_pointer, allow_blocked ) @@ -463,6 +607,11 @@ def run_track_u( "tabulation": None, } continue + waves = blocked_waves(row, inputs) + if waves and not allow_blocked: + say(f"Track U row {row_id}: blocked (waves {waves})") + results[row_id] = _blocked_result(row, inputs, waves) + continue say(f"Track U row {row_id}") result, members, adjusted, cohort = _compute_row( row, @@ -474,9 +623,10 @@ def run_track_u( tabulation_config=tabulation_config, ) results[row_id] = result - if row_id == PRIMARY_ROW: + if row_id == headline: primary = (members, adjusted, cohort) - assert primary is not None + if primary is None: + raise TrackURunError(f"the headline row {headline} was not computed") members, adjusted, cohort = primary if data_provenance == ap.INVENTED: checks["invented_cohort"] = invented.check_invented_cohort(cohort) @@ -489,6 +639,13 @@ def run_track_u( "data_provenance": data_provenance, "registration_pointer": registration_pointer, "labels": labels, + "headline": { + "row": headline, + "rule": HEADLINE_RULE, + "wealth_refused_waves": sorted( + int(w) for w in inputs.wealth_refusals + ), + }, "cohort_provenance": { key: value for key, value in cohort.provenance.items() @@ -501,7 +658,7 @@ def run_track_u( "input_checks": checks, "rows": results, "f17_diagnostics": { - "row": PRIMARY_ROW, + "row": headline, "official_concept_poverty_rate": official_concept_rates( members, adjusted ), diff --git a/tests/README-tiers.md b/tests/README-tiers.md index 5ef7342c..eb5feb7f 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -38,9 +38,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 3,027 | -| `artifact` | 2,841 | +| `unit` | 3,044 | +| `artifact` | 2,844 | | `integration_psid` | 863 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 163 | -| **Total** | **7,414** | +| **Total** | **7,434** | diff --git a/tests/cohorts/test_age67.py b/tests/cohorts/test_age67.py index f4f045c4..e3949134 100644 --- a/tests/cohorts/test_age67.py +++ b/tests/cohorts/test_age67.py @@ -30,7 +30,7 @@ def _mh_row(pid, birth, **marriage): base = { "person_id": pid, - "sex": _SEX[pid], + "sex": _SEX.get(pid, "na"), "birth_year": birth, "birth_month": pd.NA, "marriage_order": pd.NA, @@ -173,6 +173,8 @@ def _family(wave: int) -> pd.DataFrame: *family_income.SOCIAL_SECURITY_CONCEPTS, *family_income.SSI_CONCEPTS, *family_income.ASSET_INCOME_CONCEPTS, + *family_income.HW_EARNED_CONCEPTS, + "head_annuities", ): frame[concept] = 0 return frame @@ -228,6 +230,8 @@ def test_observation_plans(): (1943, 2011, 2010, 1.0), (1945, 2013, 2012, 1.0), ) + fallback = age67.observation_plan(age67.Age67Spec(row="U0-F")) + assert fallback == u0[2:] u1 = age67.observation_plan(age67.Age67Spec(row="U1")) assert len(u1) == 5 + 8 + 1 assert (1936, 2005, 2004, 1.0) in u1 @@ -251,10 +255,19 @@ def test_u0_observations_and_dispositions(): head = obs.loc["1001:2009"] assert head["member_role"] == "head" assert head["married"] and head["member_married_coresident"] + assert head["marital_resolution"] == "marriage_history" assert head["spouse_person_id"] == 1002 assert head["spouse_age"] == 65 and head["spouse_sex"] == "female" + assert head["spouse_age_source"] == "derived_birth_year" assert head["fu_legal_wife_present"] assert head["wife_sex"] == "female" + # the annuitants of fu_head_rule: the head himself and his legal wife + assert head["fu_head_person_id"] == 1001 + assert head["fu_head_age"] == 67 and head["fu_head_sex"] == "male" + assert head["fu_head_spouse_present"] + assert head["fu_head_spouse_person_id"] == 1002 + assert head["fu_head_spouse_age"] == 65 + assert head["fu_head_spouse_sex"] == "female" assert head["member_age"] == 67 assert head["family_unit_id"] == 2009 * 100_000 + 11 assert head["wealth_status"] == "family_file" @@ -266,7 +279,13 @@ def test_u0_observations_and_dispositions(): assert ofum["birth_source"] == "derived_projection_age" assert ofum["member_role"] == "ofum" assert ofum["marital_status"] == "no_marriage_history" + # an OFUM (code 30) is not resolved by the relationship code + assert ofum["marital_resolution"] == "unresolved_non_married" assert not ofum["married"] + # its family's annuitants are the head 1001 and legal wife 1002 + assert ofum["fu_head_person_id"] == 1001 + assert ofum["fu_head_age"] == 69 + assert ofum["fu_head_spouse_age"] == 67 assert obs.loc["3001:2005", "wealth_status"] == ( "blocked_wealth_supplement_not_staged" ) @@ -433,41 +452,266 @@ def test_structural_summary_counts_only(): assert summary["n_observations_blocked"] == 1 assert summary["n_observations_computable_now"] == 3 assert summary["family_units_with_two_or_more_members"] == 1 - assert summary["observations_by_birth_year_and_wave"]["1943@2011"][ - "member_role" - ] == {"ofum": 1, "wife": 1} + cell = summary["observations_by_birth_year_and_wave"]["1943@2011"] + assert cell["member_role"] == {"ofum": 1, "wife": 1} + assert cell["marital_resolution"] == { + "marriage_history": 1, + "unresolved_non_married": 1, + } + assert cell["fu_head_age_source"] == {"derived_birth_year": 2} + receipt = summary["component_receipt_counts"]["2011"] + assert receipt["family_farm_income_nonzero"] == 0 + assert receipt["family_head_retirement_account_income_nonzero"] == 0 text = repr(summary) for forbidden in ("poor", "threshold", "annuity"): assert forbidden not in text def test_unresolved_marital_status_is_an_explicit_pending_default(): - """Members whose marriage history cannot date a state are non-married. + """Unresolved marital states are classified by relationship code. Plan F12 names MH85_23 but not what to do when it cannot resolve the - state ("unknown") or the person has no record - ("no_marriage_history"); the builder's reading (non-married, so the - non_married cell and a single-life annuity) is a named parameter with - a pending decision, not a silent fallthrough. + state ("unknown") or the person has no record ("no_marriage_ + history"); the referee's answer (Q7) is the relationship code, with + non-married as the named alternative. Both are parameters with a + pending decision, not a silent fallthrough. """ spec = age67.Age67Spec() - assert spec.unresolved_marital_status == "non_married" + assert spec.unresolved_marital_status == "relationship_code" with pytest.raises(ValueError, match="unresolved_marital_status"): age67.Age67Spec(unresolved_marital_status="married") decisions = {item.field: item for item in age67.pending_decisions()} item = decisions["unresolved_marital_status"] - assert item.default == "non_married" - assert "F12" in item.default_basis + assert item.default == "relationship_code" + assert item.alternatives == ("non_married",) + assert "F12" in item.default_basis and "Q7" in item.default_basis cohort = age67.build_age67_cohort(_inputs()) ofum = cohort.observations.set_index("observation_id").loc["6001:2011"] assert ofum["marital_status"] == "no_marriage_history" assert not ofum["married"] assert cohort.provenance["spec"]["unresolved_marital_status"] == ( - "non_married" + "relationship_code" ) +def _with_unresolved_couples() -> age67.Age67Inputs: + """Invented, all in 2013 (income year 2012): + + * family 91: head 9101 (male, born 1945, marriage history "unknown": + one undated marriage) with legal wife 9102 (female, no history, + individual-file age 70, zero weight: a nonsample spouse); + * family 92: female head 9201 (born 1945 by marriage history, age 67) + with legal husband 9202 (code 90, male, no marriage history, + individual-file age 67 in 2013, the earliest wave he is in a + family, so the law's seed gives 2012 - 67 = 1945). + """ + + anchors = _anchors() + rows = { + 9101: (9101, 91, 1, 10, 67, 60.0), + 9102: (9102, 91, 2, 20, 70, 0.0), + 9201: (9201, 92, 1, 10, 67, 50.0), + 9202: (9202, 92, 2, 90, 67, 50.0), + } + for wave, frame in anchors.items(): + extra = pd.DataFrame( + [ + row if wave == 2013 else (pid, 0, 0, 0, 0, 0.0) + for pid, row in rows.items() + ], + columns=[ + "person_id", + "interview", + "sequence", + "relationship", + "age", + "weight", + ], + ) + extra["reported_birth_year"] = pd.array( + [pd.NA] * len(extra), dtype="Int64" + ) + anchors[wave] = pd.concat([frame, extra], ignore_index=True) + history = _marriage_history() + unknown = _mh_row(9101, 1945, how_ended="other") + unknown["last_known_status"] = "married" + unknown["sex"] = "male" + head = _mh_row( + 9201, + 1945, + spouse_person_id=9202, + start_year=1970, + how_ended="intact", + last_known_status="married", + ) + head["sex"] = "female" + extra_history = pd.DataFrame([unknown, head]).astype(history.dtypes) + history = pd.concat([history, extra_history], ignore_index=True) + base = _inputs() + incomes = {wave: _family(wave) for wave in age67.WAVES} + wealth = dict(base.family_wealth) + template = _family(2011).iloc[[0]] + for interview in (91, 92): + incomes[2013] = pd.concat( + [ + incomes[2013], + template.assign( + wave=2013, income_year=2012, interview=interview + ), + ], + ignore_index=True, + ) + wealth[2013] = pd.concat( + [ + wealth[2013], + pd.DataFrame( + { + "wave": [2013], + "interview": [interview], + "wealth1": [500], + "wealth1_acc": [0], + } + ), + ], + ignore_index=True, + ) + sexes = {9101: "male", 9102: "female", 9201: "female", 9202: "male"} + return _inputs( + anchors=anchors, + marriage_history=history, + family_income=incomes, + family_wealth=wealth, + design=pd.concat( + [ + base.design, + pd.DataFrame( + { + "person_id": list(sexes), + "stratum": [1] * 4, + "cluster": [1, 1, 2, 2], + } + ), + ], + ignore_index=True, + ), + death_records=pd.concat( + [ + base.death_records, + pd.DataFrame( + {"person_id": list(sexes), "sex": list(sexes.values())} + ), + ], + ignore_index=True, + ), + ) + + +def test_relationship_code_resolves_unresolved_marital_states(): + """Referee Q7: the relationship code decides an unresolved state. + + 9101 (head, history "unknown") with a legal wife in the family is + married and co-resident with her; 9202 (legal husband, code 90, no + history) is married to the female head. Under "non_married" both are + non-married with no spouse. + """ + + inputs = _with_unresolved_couples() + obs = age67.build_age67_cohort(inputs).observations.set_index( + "observation_id" + ) + head = obs.loc["9101:2013"] + assert head["marital_status"] == "unknown" + assert head["marital_resolution"] == ( + "relationship_code_head_with_legal_spouse" + ) + assert head["married"] and head["member_married_coresident"] + assert head["spouse_person_id"] == 9102 + husband = obs.loc["9202:2013"] + assert husband["relationship"] == 90 + assert husband["marital_status"] == "no_marriage_history" + assert husband["marital_resolution"] == "relationship_code_legal_husband" + assert husband["married"] and husband["member_married_coresident"] + assert husband["spouse_person_id"] == 9201 + assert husband["member_role"] == "ofum" + # the female head and her legal husband are the family's annuitants + assert husband["fu_head_person_id"] == 9201 + assert husband["fu_head_sex"] == "female" + assert husband["fu_head_spouse_present"] + assert husband["fu_head_spouse_person_id"] == 9202 + assert husband["fu_head_spouse_sex"] == "male" + assert obs.loc["9201:2013", "marital_resolution"] == "marriage_history" + plain = age67.build_age67_cohort( + inputs, age67.Age67Spec(unresolved_marital_status="non_married") + ).observations.set_index("observation_id") + for oid in ("9101:2013", "9202:2013"): + assert plain.loc[oid, "marital_resolution"] == ( + "unresolved_non_married" + ) + assert not plain.loc[oid, "married"] + assert pd.isna(plain.loc[oid, "spouse_person_id"]) + + +def test_annuitant_ages_use_derived_birth_years(): + """Referee Q8: income-year ages from derived birth years. + + Invented: 1001's legal wife 1002 is born 1943 by marriage history; at + the 2009 interview her individual-file age is set to 66 (her birthday + fell before the interview). Income year 2008: the derived age is 65, + the wave age 66. 9102, a zero-weight nonsample wife outside the + universe with no history, gets her birth year from the law's seed + (2012 - 70 = 1942), so her income-year age equals the wave age, 70. + """ + + inputs = _with_unresolved_couples() + anchors = dict(inputs.anchors) + frame = anchors[2009].copy() + frame.loc[frame["person_id"].eq(1002), "age"] = 66 + anchors[2009] = frame + import dataclasses + + inputs = dataclasses.replace(inputs, anchors=anchors) + derived = age67.build_age67_cohort(inputs).observations.set_index( + "observation_id" + ) + assert derived.loc["1001:2009", "spouse_age"] == 65 + assert derived.loc["1001:2009", "spouse_age_source"] == ( + "derived_birth_year" + ) + assert derived.loc["1001:2009", "fu_head_spouse_age"] == 65 + head = derived.loc["9101:2013"] + assert head["spouse_age"] == 70 + assert head["spouse_age_source"] == "derived_birth_year" + assert head["fu_head_spouse_age"] == 70 + wave = age67.build_age67_cohort( + inputs, age67.Age67Spec(annuitant_age_source="wave_age") + ).observations.set_index("observation_id") + assert wave.loc["1001:2009", "spouse_age"] == 66 + assert wave.loc["1001:2009", "spouse_age_source"] == "wave_age" + assert wave.loc["1001:2009", "fu_head_spouse_age"] == 66 + with pytest.raises(ValueError, match="annuitant_age_source"): + age67.Age67Spec(annuitant_age_source="reported_birth_year") + decisions = {item.field: item for item in age67.pending_decisions()} + assert decisions["annuitant_age_source"].default == "derived_birth_year" + assert "Q8" in decisions["annuitant_age_source"].default_basis + + +def test_u0f_keeps_only_the_fallback_birth_years(): + inputs = _inputs() + fallback = age67.build_age67_cohort(inputs, age67.Age67Spec(row="U0-F")) + obs = fallback.observations + assert set(obs["birth_year"]) <= set(age67.FALLBACK_BIRTH_YEARS) + assert "3001:2005" not in set(obs["observation_id"]) + # nothing is blocked, so the income rows need no allow_blocked + rows = age67.income_rows(fallback, inputs) + assert rows.attrs["left_out"]["wealth_supplement_not_staged"] == 0 + assert set(rows["observation_id"]) == { + "1001:2009", + "1002:2011", + "6001:2011", + } + + def test_spec_validation_and_pending_decisions(): with pytest.raises(ValueError): age67.Age67Spec(row="U9") diff --git a/tests/estimates/test_adjusted_poverty.py b/tests/estimates/test_adjusted_poverty.py index b529fd6f..88f6e8c9 100644 --- a/tests/estimates/test_adjusted_poverty.py +++ b/tests/estimates/test_adjusted_poverty.py @@ -97,11 +97,11 @@ def _row(observation_id: str, **overrides) -> dict: member_married_coresident=False, spouse_age=None, spouse_sex=None, - head_age=2, - head_sex="male", - wife_age=None, - wife_sex=None, - fu_legal_wife_present=False, + fu_head_age=2, + fu_head_sex="male", + fu_head_spouse_present=False, + fu_head_spouse_age=None, + fu_head_spouse_sex=None, wife_present=False, fu_size=1, n_children=0, @@ -314,46 +314,224 @@ def test_negative_wealth_buys_no_annuity(): def test_joint_annuity_for_a_married_coresident_member(): - out = _run( - _row( - "a", - member_married_coresident=True, - spouse_age=2, - spouse_sex="female", - wealth1=1024, - fu_size=2, - wife_present=True, - ) + row = _row( + "a", + member_married_coresident=True, + spouse_age=2, + spouse_sex="female", + fu_head_spouse_present=True, + fu_head_spouse_age=2, + fu_head_spouse_sex="female", + wealth1=1024, + fu_size=2, + wife_present=True, ) - # 0.8 * 1024 / 1.024 = 800 - assert out.loc["a", "annuity"] == pytest.approx(800.0) - assert out.loc["a", "annuity_basis"] == "joint:male2+female2" - assert out.loc["a", "threshold"] == 1300.0 - + # the member is the head: both rules price the same couple + for lives in ("fu_head_rule", "member_rule"): + out = _run(row, annuity_lives=lives) + # 0.8 * 1024 / 1.024 = 800 + assert out.loc["a", "annuity"] == pytest.approx(800.0) + assert out.loc["a", "annuity_basis"] == "joint:male2+female2" + assert out.loc["a", "threshold"] == 1300.0 + + +def test_fu_head_rule_prices_on_the_head_and_legal_spouse(): + """Section 13: an OFUM woman aged 3 in a family headed by a man aged 2 + and his legal wife aged 2 (INVENTED; WEALTH1 1,024). + + fu_head_rule (default): joint 1.024, annuity 0.8 * 1024 / 1.024 = 800. + member_rule: single life on the female OFUM aged 3, t_p_x = 1, .75, 0 + -> .75 * .8 = .6, annuity 0.8 * 1024 / .6 = 1,365.33. + """ -def test_fu_head_rule_prices_on_the_head_and_legal_wife(): + assert ap.AdjustedPovertySpec().annuity_lives == "fu_head_rule" rows = [ _row( "ofum", member_role="ofum", member_age=3, member_sex="female", - head_age=2, - head_sex="male", - wife_age=2, - wife_sex="female", - fu_legal_wife_present=True, + fu_head_age=2, + fu_head_sex="male", + fu_head_spouse_present=True, + fu_head_spouse_age=2, + fu_head_spouse_sex="female", wife_present=True, fu_size=3, wealth1=1024, ) ] - member = _run(*rows) - head = _run(*rows, annuity_lives="fu_head_rule") - # member rule: single life on the female OFUM aged 3 -> - # t_p_x = 1, .75, 0 -> .75*.8 = .6 - assert member.loc["ofum", "annuity_factor"] == pytest.approx(0.6) + head = _run(*rows) + member = _run(*rows, annuity_lives="member_rule") assert head.loc["ofum", "annuity_factor"] == pytest.approx(1.024) + assert head.loc["ofum", "annuity"] == pytest.approx(800.0) + assert member.loc["ofum", "annuity_factor"] == pytest.approx(0.6) + assert member.loc["ofum", "annuity"] == pytest.approx(0.8 * 1024 / 0.6) + + +def test_fu_head_rule_with_a_legal_husband(): + """A female head aged 2 with a legal husband (code 90) aged 2: the + builder records him as the head's legal spouse, so the joint price is + .5 * (1.152 + .896) = 1.024 (INVENTED).""" + + out = _run( + _row( + "h", + member_role="ofum", + member_sex="male", + fu_head_sex="female", + fu_head_spouse_present=True, + fu_head_spouse_age=2, + fu_head_spouse_sex="male", + wealth1=1024, + fu_size=2, + ) + ) + assert out.loc["h", "annuity_factor"] == pytest.approx(1.024) + assert out.loc["h", "annuity_basis"] == "joint:female2+male2" + + +def test_fu_head_rule_needs_the_annuitants_ages(): + with pytest.raises(ap.AdjustedPovertyError, match="annuitant age"): + _run(_row("a", fu_head_age=None)) + with pytest.raises(ap.AdjustedPovertyError, match="annuitant age"): + _run( + _row( + "a", + fu_head_spouse_present=True, + fu_head_spouse_sex="female", + fu_head_spouse_age=None, + ) + ) + + +def test_head_annuity_and_ira_income_is_removed_under_replace(): + """F4a (INVENTED): money income 1,200 of which the head's annuity and + IRA income is 200. + + remove_head: B = 1,200 - 200 = 1,000; keep: B = 1,200; U5 (keep + reported asset income) keeps it too. In the 2013 file (income year + 2012) HEAD ANNUITIES 150 and HEAD IRAS 50 are both removed. + """ + + row = _row( + "a", + head_annuities=200, + hw_transfer=200, + head_ss=1000, + total_family_income=1200, + ) + assert ap.AdjustedPovertySpec().retirement_account_income_rule == ( + "remove_head" + ) + removed = _run(row) + assert removed.loc["a", "retirement_account_income_removed"] == 200 + assert removed.loc["a", "asset_income_removed"] == 200 + assert removed.loc["a", "baseline_income"] == pytest.approx(1000.0) + kept = _run(row, retirement_account_income_rule="keep") + assert kept.loc["a", "retirement_account_income_removed"] == 0 + assert kept.loc["a", "baseline_income"] == pytest.approx(1200.0) + u5 = _run(row, asset_income_rule="keep") + assert u5.loc["a", "asset_income_removed"] == 0 + assert u5.loc["a", "baseline_income"] == pytest.approx(1200.0) + with pytest.raises(ap.AdjustedPovertyError): + ap.AdjustedPovertySpec(retirement_account_income_rule="remove_all") + + +def test_head_iras_enter_only_where_the_family_file_has_them(): + thresholds_2012 = ap.PovertyThresholds( + weighted_average={2012: _WA_2010}, + matrix={}, + provenance={"kind": "invented"}, + ) + ssi_2012 = ap.SsiParameters( + fbr_individual_monthly={2012: 50.0}, + fbr_couple_monthly={2012: 75.0}, + general_income_exclusion_monthly=20.0, + earned_income_exclusion_monthly=65.0, + earned_income_share_excluded=0.5, + resource_limit_individual=2000.0, + resource_limit_couple=3000.0, + provenance={"kind": "invented"}, + ) + + def run(*rows): + return ap.adjusted_incomes( + _frame(*rows), + ap.AdjustedPovertySpec(real_interest_rate=RATE), + data_provenance=ap.INVENTED, + life_table=MOCK_LIFE_TABLE, + thresholds=thresholds_2012, + ssi=ssi_2012, + ).set_index("observation_id") + + row_2012 = _row( + "b", + income_year=2012, + head_annuities=150, + head_iras=50, + hw_transfer=200, + total_family_income=1200, + ) + out = run(row_2012) + assert out.loc["b", "retirement_account_income_removed"] == 200 + assert out.loc["b", "baseline_income"] == pytest.approx(1000.0) + # a 2012 row without HEAD IRAS is refused (the 2013 file has it) + missing = dict(row_2012) + missing.pop("head_iras") + with pytest.raises(ap.AdjustedPovertyError, match="head_iras"): + run(missing) + # before 2013 IRA income is inside HEAD ANNUITIES: a nonzero + # head_iras there is refused, a missing one is zero + with pytest.raises(ap.AdjustedPovertyError, match="head_iras"): + _run(_row("c", head_iras=10)) + out = _run( + _row("c", head_annuities=40, hw_transfer=40, total_family_income=40), + _row("d", head_iras=None, total_family_income=10), + ) + assert out.loc["c", "retirement_account_income_removed"] == 40 + assert out.loc["d", "retirement_account_income_removed"] == 0 + + +def test_farm_asset_share_imputes_farm_asset_income(): + """F4b (INVENTED): positive farm income 400 and a farm loss of 300. + + share 0.5 (default): remove 200 of the income (B = 1,400 - 200) and + the whole loss (B = 700 + 300 = 1,000); share 0: nothing removed. + """ + + rows = [ + _row( + "gain", + head_farm=400, + hw_taxable=400, + head_ss=1000, + total_family_income=1400, + ), + _row( + "loss", + head_farm=-300, + hw_taxable=-300, + head_ss=1000, + total_family_income=700, + ), + ] + assert ap.AdjustedPovertySpec().farm_asset_share == 0.5 + out = _run(*rows) + assert out["farm_asset_income_removed"].to_dict() == { + "gain": 200.0, + "loss": -300.0, + } + assert out.loc["gain", "baseline_income"] == pytest.approx(1200.0) + assert out.loc["loss", "baseline_income"] == pytest.approx(1000.0) + none = _run(*rows, farm_asset_share=0.0) + assert (none["farm_asset_income_removed"] == 0).all() + assert none.loc["gain", "baseline_income"] == pytest.approx(1400.0) + assert none.loc["loss", "baseline_income"] == pytest.approx(700.0) + kept = _run(*rows, asset_income_rule="keep") + assert (kept["farm_asset_income_removed"] == 0).all() + with pytest.raises(ap.AdjustedPovertyError, match="farm_asset_share"): + ap.AdjustedPovertySpec(farm_asset_share=1.5) def test_ssi_offset_for_existing_recipients(): @@ -596,6 +774,16 @@ def test_spec_validation_and_pending_decisions(): assert spec.real_interest_rate == 0.03 assert spec.ssi_rule == "offset_existing_recipients" decisions = {item.field: item for item in ap.pending_decisions()} + # the referee's 2 percent sensitivity (R6) and the new F4a/F4b fields + assert decisions["real_interest_rate"].alternatives == (0.02,) + assert "administrative costs" in ( + decisions["real_interest_rate"].default_basis + ) + assert decisions["retirement_account_income_rule"].alternatives == ( + "keep", + ) + assert decisions["farm_asset_share"].alternatives == (0.0,) + assert decisions["annuity_lives"].alternatives == ("member_rule",) assert decisions["ssi_rule"].awaiting.startswith("Max") assert "d189" in decisions["ssi_rule"].awaiting for name, item in decisions.items(): diff --git a/tests/estimates/test_uniform_cut_tabulation.py b/tests/estimates/test_uniform_cut_tabulation.py index e8349cfe..634c8574 100644 --- a/tests/estimates/test_uniform_cut_tabulation.py +++ b/tests/estimates/test_uniform_cut_tabulation.py @@ -40,12 +40,27 @@ def _rows() -> pd.DataFrame: ) +def _design(rows: pd.DataFrame, *extra: tuple[int, int]) -> pd.DataFrame: + """INVENTED design frame: the rows' (stratum, cluster) pairs plus + ``extra`` pairs that hold no row.""" + + pairs = rows[["stratum", "cluster"]].drop_duplicates() + if extra: + pairs = pd.concat( + [pairs, pd.DataFrame(extra, columns=["stratum", "cluster"])], + ignore_index=True, + ) + return pairs.reset_index(drop=True) + + def _cells(result: dict) -> dict: return {entry["cell"]: entry for entry in result["cells"]} def test_rates_and_changes_by_hand(): - result = ut.tabulate_uniform_cut(_rows(), data_provenance="invented") + result = ut.tabulate_uniform_cut( + _rows(), data_provenance="invented", design=_design(_rows()) + ) cells = _cells(result) expected = { # all: W = 10; poor B: o3 (2) -> 20; poor R: o2+o3+o4 (7) -> 70 @@ -80,7 +95,9 @@ def test_rates_and_changes_by_hand(): def test_design_standard_error_by_hand(): - result = ut.tabulate_uniform_cut(_rows(), data_provenance="invented") + result = ut.tabulate_uniform_cut( + _rows(), data_provenance="invented", design=_design(_rows()) + ) se = _cells(result)["all"]["design_se"]["delta"] # d = R - B: 0, 1, 0, 1, 0; r = .5; z = w (d - r) / 10: # -.05, .05, -.1, .2, -.1. Stratum 1 clusters: 0, -.1 (mean -.05, @@ -91,6 +108,42 @@ def test_design_standard_error_by_hand(): assert se["n_singleton_strata"] == 0 +def test_full_design_adds_clusters_without_observations(): + """Section 13: the full sample design (referee Q5, INVENTED). + + The design adds a third cluster to stratum 2 and a third stratum of + two clusters, none holding an observation. Stratum 2's cluster totals + become .2, -.1 and 0 (mean 1/30; squared deviations .027778, .017778, + .001111, sum .046667; times 3/2 = .07); stratum 1 still gives .01 and + stratum 3 gives 0, so SE = 100 * sqrt(.08). Relative to the rows + only (the u1-draft-3 estimator) it is 100 * sqrt(.1). + """ + + rows = _rows() + design = _design(rows, (2, 3), (3, 1), (3, 2)) + result = ut.tabulate_uniform_cut( + rows, data_provenance="invented", design=design + ) + se = _cells(result)["all"]["design_se"]["delta"] + assert se["se"] == pytest.approx(100 * math.sqrt(0.08)) + assert se["n_strata"] == 3 + assert result["design"] == { + "domain": "full_sample_design", + "n_strata": 3, + "n_clusters": 7, + "singleton_strata": [], + } + rows_only = ut.tabulate_uniform_cut( + rows, + data_provenance="invented", + config=ut.TabulationConfig(design_se_domain="tabulated_rows"), + ) + assert _cells(rows_only)["all"]["design_se"]["delta"]["se"] == ( + pytest.approx(100 * math.sqrt(0.1)) + ) + assert rows_only["design"]["domain"] == "tabulated_rows" + + def test_singleton_strata_are_left_out_and_counted(): rows = _rows() extra = rows.iloc[[0]].copy() @@ -99,10 +152,53 @@ def test_singleton_strata_are_left_out_and_counted(): extra["family_unit_id"] = 60 extra["stratum"] = 3 rows = pd.concat([rows, extra], ignore_index=True) - result = ut.tabulate_uniform_cut(rows, data_provenance="invented") - se = _cells(result)["all"]["design_se"]["delta"] + # relative to the rows, stratum 3 holds one cluster and drops out + rows_only = ut.tabulate_uniform_cut( + rows, + data_provenance="invented", + config=ut.TabulationConfig(design_se_domain="tabulated_rows"), + ) + se = _cells(rows_only)["all"]["design_se"]["delta"] assert se["n_singleton_strata"] == 1 assert se["n_strata"] == 2 + assert rows_only["design"]["singleton_strata"] == [3] + # on a full design where stratum 3 has two clusters, it enters + full = ut.tabulate_uniform_cut( + rows, data_provenance="invented", design=_design(rows, (3, 2)) + ) + se = _cells(full)["all"]["design_se"]["delta"] + assert se["n_singleton_strata"] == 0 + assert se["n_strata"] == 3 + # a singleton in the full design itself is still listed + lone = ut.tabulate_uniform_cut( + rows, data_provenance="invented", design=_design(rows) + ) + assert lone["design"]["singleton_strata"] == [3] + + +def test_full_design_needs_a_design_frame_covering_the_rows(): + rows = _rows() + with pytest.raises(ut.UniformCutTabulationError, match="design frame"): + ut.tabulate_uniform_cut(rows, data_provenance="invented") + partial = _design(rows).iloc[:3] + with pytest.raises(ut.UniformCutTabulationError, match="not in the"): + ut.tabulate_uniform_cut( + rows, data_provenance="invented", design=partial + ) + with pytest.raises(ut.UniformCutTabulationError, match="lacks"): + ut.tabulate_uniform_cut( + rows, + data_provenance="invented", + design=pd.DataFrame({"stratum": [1]}), + ) + # without design standard errors no frame is needed + ut.tabulate_uniform_cut( + rows, + data_provenance="invented", + config=ut.TabulationConfig(design_standard_errors=False), + ) + with pytest.raises(ut.UniformCutTabulationError, match="domain"): + ut.TabulationConfig(design_se_domain="whole_psid") def _reference_delta(rows: pd.DataFrame, mask: np.ndarray) -> float | None: @@ -117,7 +213,9 @@ def _reference_delta(rows: pd.DataFrame, mask: np.ndarray) -> float | None: def test_half_split_floor_uses_family_units(): rows = _rows() - result = ut.tabulate_uniform_cut(rows, data_provenance="invented") + result = ut.tabulate_uniform_cut( + rows, data_provenance="invented", design=_design(rows) + ) units = pd.DataFrame({"family_unit_id": rows["family_unit_id"].tolist()}) gaps = [] for seed in ut.DEFAULT_FLOOR_SEEDS: @@ -187,7 +285,9 @@ def test_half_split_floor_is_person_disjoint_under_u1(): """ rows = _u1_rows() - result = ut.tabulate_uniform_cut(rows, data_provenance="invented") + result = ut.tabulate_uniform_cut( + rows, data_provenance="invented", design=_design(rows) + ) units = pd.DataFrame({"split_unit": ut.floor_split_units(rows)}) gaps = [] for seed in ut.DEFAULT_FLOOR_SEEDS: @@ -209,7 +309,9 @@ def test_half_split_floor_is_person_disjoint_under_u1(): def test_floor_is_undefined_with_one_family_unit(): rows = _rows() rows["family_unit_id"] = 1 - result = ut.tabulate_uniform_cut(rows, data_provenance="invented") + result = ut.tabulate_uniform_cut( + rows, data_provenance="invented", design=_design(rows) + ) floor = _cells(result)["all"]["floor"]["delta"] assert floor["defined"] is False assert floor["mean"] is None @@ -221,7 +323,9 @@ def test_undefined_cells_are_reported_not_raised(): rows = _rows() rows["sex"] = "male" rows.loc[rows["married"], "weight"] = 0.0 - result = ut.tabulate_uniform_cut(rows, data_provenance="invented") + result = ut.tabulate_uniform_cut( + rows, data_provenance="invented", design=_design(rows) + ) undefined = { item["cell"]: item["reason"] for item in result["undefined_cells"] } @@ -231,7 +335,9 @@ def test_undefined_cells_are_reported_not_raised(): def test_invented_results_are_labelled(): - result = ut.tabulate_uniform_cut(_rows(), data_provenance="invented") + result = ut.tabulate_uniform_cut( + _rows(), data_provenance="invented", design=_design(_rows()) + ) assert result["labels"][0] == ut.INVENTED_DATA_LABEL assert set(OUTPUT_LABELS) <= set(result["labels"]) assert result["config"]["draws"] == 1 @@ -255,7 +361,9 @@ def test_provenance_guards(): rows = _rows() rows.attrs["provenance_kind"] = "psid_files" with pytest.raises(ut.UniformCutTabulationError, match="PSID"): - ut.tabulate_uniform_cut(rows, data_provenance="invented") + ut.tabulate_uniform_cut( + rows, data_provenance="invented", design=_design(rows) + ) @pytest.mark.parametrize("kind", [None, "caller_frames", "invented"]) @@ -292,7 +400,9 @@ def test_rows_are_validated(column, value): rows[column] = rows[column].astype(object) rows.loc[0, column] = value with pytest.raises(ut.UniformCutTabulationError): - ut.tabulate_uniform_cut(rows, data_provenance="invented") + ut.tabulate_uniform_cut( + rows, data_provenance="invented", design=_design(rows) + ) def test_config_validation(): diff --git a/tests/test_boomers2004_uniform_cut_spec.py b/tests/test_boomers2004_uniform_cut_spec.py index ae6adefb..1c001f14 100644 --- a/tests/test_boomers2004_uniform_cut_spec.py +++ b/tests/test_boomers2004_uniform_cut_spec.py @@ -3,14 +3,16 @@ ``docs/design/boomers2004_uniform_cut_comparison.md`` is read by downstream lanes through its machine-readable JSON block (§15). These tests hold that block to the code's defaults and constants (the income -concept, the cohort builder, the reader tables and the tabulation), so a -draft change and a code change cannot drift apart silently. They use -only the document and the code: no PSID value, no model output and no -comparator value. +concept, the cohort builder, the reader tables, the tabulation, the +registered rows and the runner's named deltas), so a draft change and a +code change cannot drift apart silently. They use only the document and +the code: no PSID value, no model output and no comparator value. """ from __future__ import annotations +import hashlib +import importlib.util import json import re from pathlib import Path @@ -21,12 +23,13 @@ from populace_dynamics.data import family_income from populace_dynamics.estimates import adjusted_poverty as ap from populace_dynamics.estimates import uniform_cut_tabulation as ut +from populace_dynamics.uniform_cut_track_u import rows as track_u_rows +from populace_dynamics.uniform_cut_track_u import runner -SPEC_PATH = ( - Path(__file__).resolve().parents[1] - / "docs" - / "design" - / "boomers2004_uniform_cut_comparison.md" +ROOT = Path(__file__).resolve().parents[1] +SPEC_PATH = ROOT / "docs" / "design" / "boomers2004_uniform_cut_comparison.md" +EVIDENCE = ( + Path.home() / "microcosm-launch-evidence" / "dynasim-parity-20260909" ) @@ -42,23 +45,33 @@ def block(text: str) -> dict: return json.loads(blocks[0]) +def _section(text: str, number: str, following: str) -> str: + return text.split(f"## {number}. ")[1].split(f"## {following}. ")[0] + + def test_block_identity_and_status(block): assert block["specification"] == "boomers2004_uniform_cut_exercise2" + assert block["version"] == "u1-draft-4" assert block["status"] == "draft_for_referee" assert block["claim_class"]["awaiting"] == "d189" assert block["acceptance_rule"] is None assert block["labels"] == list(ap.OUTPUT_LABELS) + assert "row_u7_not_built" in block["blocked_by"] def test_status_line_does_not_claim_ratification(text): status = text.split("- **Specification:**")[0] assert "draft for the referee" in status assert "Nothing here is ratified" in status + assert "`u1-draft-4`" in text.split("- **Plan item:**")[0] def test_population_matches_the_builder(block): population = block["population"] assert population["primary_birth_years"] == list(age67.PRIMARY_BIRTH_YEARS) + assert population["fallback_birth_years"] == list( + age67.FALLBACK_BIRTH_YEARS + ) assert population["u1_birth_years"] == [ age67.ALL_BIRTH_YEARS[0], age67.ALL_BIRTH_YEARS[-1], @@ -70,6 +83,11 @@ def test_population_matches_the_builder(block): assert population["unresolved_marital_status"] == ( spec.unresolved_marital_status ) + assert population["unresolved_marital_status"] in ( + age67.UNRESOLVED_MARITAL_RULES + ) + assert population["annuitant_age_source"] == spec.annuitant_age_source + assert population["annuitant_age_source"] in age67.ANNUITANT_AGE_SOURCES assert population["institution_income_rule"] == ( spec.institution_income_rule ) @@ -97,6 +115,20 @@ def test_population_matches_the_builder(block): assert population["design"] == {"stratum": "ER31996", "cluster": "ER31997"} +def test_headline_rule_matches_the_code(block): + headline = block["population"]["headline"] + assert headline["rule"] == track_u_rows.HEADLINE_RULE + assert headline["fallback_row"] == track_u_rows.FALLBACK_ROW + assert headline["awaiting"] + assert block["population"]["primary_row"] == track_u_rows.PRIMARY_ROW + fallback = age67.observation_plan(age67.Age67Spec(row="U0-F")) + assert sorted({b for b, _, _, _ in fallback}) == list( + age67.FALLBACK_BIRTH_YEARS + ) + # the fallback's waves all carry WEALTH1 in the family file + assert {w for _, w, _, _ in fallback} <= set(family_income.WEALTH_WAVES) + + def test_income_concept_matches_the_spec_defaults(block): spec = ap.AdjustedPovertySpec() concept = block["income_concept"] @@ -105,6 +137,19 @@ def test_income_concept_matches_the_spec_defaults(block): assert concept["asset_income_items"] == list( family_income.ASSET_INCOME_CONCEPTS ) + assert concept["retirement_account_income_rule"] == ( + spec.retirement_account_income_rule + ) + assert concept["retirement_account_income_items"] == list( + ap.RETIREMENT_ACCOUNT_INCOME_CONCEPTS + ) + for item in ap.RETIREMENT_ACCOUNT_INCOME_CONCEPTS: + assert any( + item in family_income.income_variables(wave) + for wave in family_income.INCOME_WAVES + ) + assert concept["farm_asset_share"] == spec.farm_asset_share + assert concept["farm_loss"] == ap.FARM_LOSS_RULE assert concept["annuitized_share"] == spec.annuitized_share annuity = concept["annuity"] assert annuity == { @@ -117,16 +162,62 @@ def test_income_concept_matches_the_spec_defaults(block): "lives": spec.annuity_lives, "negative_wealth": spec.negative_wealth_rule, } + decisions = {item.field: item for item in ap.pending_decisions()} + assert concept["sensitivities_unscored"] == { + "real_interest_rate": list( + decisions["real_interest_rate"].alternatives + ) + } assert block["threshold"]["rule"] == spec.threshold_rule assert block["threshold"]["capture_status"] == "not_captured" assert ap.THRESHOLDS_SHA256 is None - assert block["cut"]["rate"] == spec.cut_rate == block["target"]["cut_rate"] - assert block["cut"]["start_year"] == spec.cut_start_year is None ssi = block["ssi"] assert ssi["rule"] == spec.ssi_rule assert ssi["deeming"] == spec.ssi_deeming assert ssi["ofum_unit"] == spec.ofum_ssi_unit assert ssi["parameters"]["sha256"] == ap.SSI_PARAMETERS_SHA256 + assert ssi["parameters"]["file"] == str( + ap.SSI_PARAMETERS_PATH.relative_to(ROOT) + ) + + +def test_cut_and_threshold_years_match_the_code(block): + spec = ap.AdjustedPovertySpec() + cut = block["cut"] + assert cut["rate"] == spec.cut_rate == block["target"]["cut_rate"] + assert cut["start_year"] == spec.cut_start_year is None + assert cut["start_year_rule"] == ap.CUT_START_YEAR_RULE + # not code parameters: the cut base and behaviour are fixed in + # adjusted_incomes (specification section 16 lists them as pending) + assert cut["base"] == "all_social_security_of_the_unit" + assert cut["behavior"] == "none" + income_years = [wave - 1 for wave in family_income.INCOME_WAVES] + ssi_years = sorted(ap.load_ssi_parameters().fbr_individual_monthly) + assert block["threshold"]["years"] == [ + min(income_years), + max(income_years), + ] + assert block["threshold"]["years"] == [ssi_years[0], ssi_years[-1]] + + +def test_statistic_and_comparison_match_the_tabulation(block): + statistic = block["statistic"] + assert [statistic["headline"], *statistic["secondary"]] == list( + ut.STATISTICS + ) + assert statistic["unit"] == "percentage_points" + comparison = block["comparison"] + assert comparison["gap"] == "model_minus_report" + assert comparison["acceptance"]["rule"] is None + # the pending acceptance rule is an "awaiting" key the registered run's + # ratification scan finds + spec = importlib.util.spec_from_file_location( + "_registered", ROOT / "scripts" / "run_track_u_registered.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + assert "comparison.acceptance.awaiting" in module._awaiting(block) + assert "population.headline.awaiting" in module._awaiting(block) def test_rows_name_real_alternatives(block): @@ -139,13 +230,14 @@ def test_rows_name_real_alternatives(block): "U4", "U5", "U6", + "U0-F", "U7", "U8", "U9", "U10", "U-inst", } - for name, row in rows.items(): + for row in rows.values(): if row.get("status") == "not_built": continue overrides = { @@ -154,8 +246,9 @@ def test_rows_name_real_alternatives(block): if key in ap.AdjustedPovertySpec().as_dict() } ap.AdjustedPovertySpec(**overrides) - if "population" in row: - assert row["population"] == "all_ten_birth_years", name + assert rows["U1"]["population"] == "all_ten_birth_years" + assert rows["U0-F"]["population"] == "birth_years_1941_1943_1945" + assert rows["U0-F"]["on"] == "U0" assert rows["U-inst"]["presence"] == "in_family_or_institution" age67.Age67Spec( presence=rows["U-inst"]["presence"], @@ -165,8 +258,6 @@ def test_rows_name_real_alternatives(block): ap.AdjustedPovertySpec(cut_start_year=rows["U6"]["cut_start_year"]) # every row but U7 is built, and the code's rows equal the block's assert [name for name, row in rows.items() if "status" in row] == ["U7"] - from populace_dynamics.uniform_cut_track_u import rows as track_u_rows - assert track_u_rows.check_rows_against_block(block)["rows_equal_the_block"] @@ -182,19 +273,79 @@ def test_cells_and_uncertainty_match_the_tabulation(block): assert uncertainty["floor"]["min_usable_seeds"] == ut.MIN_FLOOR_SEEDS assert uncertainty["floor"]["split_unit"] == ut.FLOOR_SPLIT_UNIT assert config.as_dict()["floor_split_unit"] == ut.FLOOR_SPLIT_UNIT + design = uncertainty["design_se"] + assert design["domain"] == config.design_se_domain + assert design["domain"] in ut.DESIGN_SE_DOMAINS + assert config.design_standard_errors + + +def test_named_deltas_equal_the_runner(text): + section = _section(text, "12", "13") + bullets = [] + for line in section.splitlines(): + if line.startswith("- "): + bullets.append(line[2:]) + elif bullets and line.startswith(" ") and line.strip(): + bullets[-1] += " " + line.strip() + cleaned = [bullet.rstrip(";.") for bullet in bullets] + assert cleaned == list(runner.NAMED_DELTAS) def test_pending_decisions_are_listed_in_the_text(text): - section = text.split("## 16. Pending decisions")[1].split("## 17.")[0] + section = _section(text, "16", "17") assert "d189" in section assert "decision 8" in section and "cut_start_year" in section assert "institution_income_rule = family_of_record" in section assert "d194" in section + assert "fallback rule" in section ssi = next( item for item in ap.pending_decisions() if item.field == "ssi_rule" ) assert "d189" in ssi.awaiting assert "offset for existing recipients (default)" in section + flat = " ".join(section.split()) + for phrase in ( + "retirement-account income (remove the head's)", + "farm asset share (0.5", + "annuity lives (FU head rule)", + "annuitant ages (derived birth year)", + "unresolved marital status (relationship code)", + "design SE (full-design domain)", + "real rate (3 percent; 2 percent sensitivity)", + ): + assert phrase in flat, phrase + # every pending field of the code is listed by the code + assert {item.field for item in age67.pending_decisions()} == set( + age67.Age67Spec().as_dict() + ) + fields = {item.field for item in ap.pending_decisions()} + for name in ( + "retirement_account_income_rule", + "farm_asset_share", + "annuity_lives", + "real_interest_rate", + ): + assert name in fields + + +def test_referee_pass_is_recorded(block, text): + (entry,) = block["referee_passes"] + assert entry["object_version"] == "u1-draft-2" + assert entry["object_commit"] == "8d7e7431" + assert entry["required_changes"] == 17 + assert entry["applied_in"] == block["version"] + assert re.fullmatch(r"[0-9a-f]{64}", entry["sha256"]) + section = _section(text, "17", "18") + assert entry["sha256"] in section + for change in [f"R{n}" for n in range(1, 18)] + [ + f"O{n}" for n in range(1, 9) + ]: + assert f"| {change} " in section, change + assert "**Declined:**" in section + report = EVIDENCE / Path(entry["report"]).name + if not report.is_file(): + pytest.skip("the referee report is outside this checkout") + assert hashlib.sha256(report.read_bytes()).hexdigest() == entry["sha256"] def test_invented_cases_match_the_code(): @@ -211,3 +362,26 @@ def test_invented_cases_match_the_code(): assert ap.annuity_factor_joint( table, "male", 2, "female", 2, rate=0.25 ) == pytest.approx(1.024) + + +def test_annuity_price_table_matches_the_committed_tables(text): + """Section 5's prices, recomputed from the committed life tables.""" + + nchs = ap.load_nchs_2000_life_table() + ssa = ap.load_ssa_period_2004_life_table() + section = _section(text, "5", "6") + for age, rate in ((66, 0.03), (67, 0.03), (68, 0.03), (67, 0.02)): + values = [ + ap.annuity_factor_single(table, sex, age, rate=rate) + for table in (nchs, ssa) + for sex in ("male", "female") + ] + line = f"| {age} | {int(rate * 100)}% | " + " | ".join( + f"{value:.4f}" for value in values + ) + assert line in section, line + for table, rate in ((nchs, 0.03), (ssa, 0.03), (nchs, 0.02), (ssa, 0.02)): + joint = ap.annuity_factor_joint( + table, "male", 67, "female", 67, rate=rate + ) + assert f"{joint:.4f}" in section diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 18d659c1..a7376009 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,8 +1,8 @@ { "schema_version": 1, "counts": { - "unit": 3027, - "artifact": 2841, + "unit": 3044, + "artifact": 2844, "integration_psid": 863, "reproduction_legacy": 520, "oracle_policyengine": 163 diff --git a/tests/track_u/test_dry_run_script.py b/tests/track_u/test_dry_run_script.py index 78a3eacc..19a5352e 100644 --- a/tests/track_u/test_dry_run_script.py +++ b/tests/track_u/test_dry_run_script.py @@ -72,6 +72,15 @@ def test_the_dry_run_checks_hold(output): blocked["left_out_with_allow_blocked"]["wealth_supplement_not_staged"] > 0 ) + fallback = blocked["fallback_rule"] + assert fallback["headline"]["row"] == "U0-F" + assert fallback["row_status"]["U0-F"] == "computed" + assert fallback["row_status"]["U0"] == "blocked" + assert set(fallback["blocked_counts"]) == { + row + for row, status in fallback["row_status"].items() + if status == "blocked" + } assert checks["registered_guard_refuses_invented_inputs"]["refused"] assert checks["census_thresholds_not_captured"]["error"] == ( "ThresholdsNotCapturedError" @@ -86,3 +95,6 @@ def test_every_built_row_is_tabulated(output): statuses = {row: entry["status"] for row, entry in result["rows"].items()} assert statuses.pop("U7") == "not_built" assert set(statuses.values()) == {"computed"} + assert result["headline"]["row"] == "U0" + text = (output / "RESULTS.md").read_text() + assert "Headline row U0" in text and "full sample design" in text diff --git a/tests/track_u/test_invented_generator.py b/tests/track_u/test_invented_generator.py index 1e3a8762..3a03d5e8 100644 --- a/tests/track_u/test_invented_generator.py +++ b/tests/track_u/test_invented_generator.py @@ -188,6 +188,34 @@ def test_every_track_u_path_is_exercised(staged): ["head_business_asset", "head_rent", "head_farm"] ] assert (losses < 0).any().any() + # specification u1-draft-4: every relationship-code resolution path, + # a legal husband (code 90), the head's annuity and IRA income (and + # HEAD IRAS in 2013), positive farm income, and interview ages that + # exceed the income-year age + assert { + "marriage_history", + "relationship_code_head_with_legal_spouse", + "relationship_code_legal_wife", + "relationship_code_legal_husband", + "unresolved_non_married", + } <= set(u1.observations["marital_resolution"]) + assert (u1.observations["relationship"] == 90).any() + u1_rows = age67.income_rows(u1, staged) + assert (u1_rows["head_annuities"] > 0).any() + assert (u1_rows["head_farm"] > 0).any() + assert (staged.family_income[2013]["head_iras"] > 0).any() + anchor = staged.anchors[2009] + present = anchor[anchor["sequence"].between(1, 20)] + births = { + row.person_id: row.birth_year + for row in staged.marriage_history.itertuples() + } + older = [ + age - (2008 - births[pid]) + for pid, age in zip(present["person_id"], present["age"], strict=True) + if pid in births + ] + assert set(older) == {0, 1} def test_invented_thresholds_by_hand(): diff --git a/tests/track_u/test_invented_pipeline.py b/tests/track_u/test_invented_pipeline.py index 3f81f34e..86837aaa 100644 --- a/tests/track_u/test_invented_pipeline.py +++ b/tests/track_u/test_invented_pipeline.py @@ -107,6 +107,7 @@ def test_invented_chain_runs_end_to_end(): data_provenance="invented", upstream_spec=adjusted.attrs["spec"], pending_decisions=[d.as_dict() for d in ap.pending_decisions()], + design=inputs.design, ) assert result["labels"][0] == ut.INVENTED_DATA_LABEL cells = {entry["cell"]: entry for entry in result["cells"]} diff --git a/tests/track_u/test_registered_script.py b/tests/track_u/test_registered_script.py index 41ae99ea..907ae8f8 100644 --- a/tests/track_u/test_registered_script.py +++ b/tests/track_u/test_registered_script.py @@ -195,6 +195,43 @@ def test_the_artifact_is_created_exclusively(tmp_path): def test_the_registration_pointer_and_commit_are_required(): with pytest.raises(SystemExit): _script().main([]) + # the headline row is required too, and only U0 or U0-F + with pytest.raises(SystemExit): + _script().main( + ["--registration-pointer", POINTER, "--registered-commit", COMMIT] + ) + with pytest.raises(SystemExit): + _script().main( + [ + "--registration-pointer", + POINTER, + "--registered-commit", + COMMIT, + "--headline-row", + "U1", + ] + ) + + +class _Staging: + """INVENTED stand-in for loaded inputs: only the refused waves.""" + + def __init__(self, refused): + self.wealth_refusals = {wave: "not staged" for wave in refused} + + +def test_the_headline_row_must_match_the_staging(): + """The fallback rule (specification section 11): the registration + names the headline row, and the run refuses a staging that gives + another.""" + + script = _script() + assert script.check_headline("U0", _Staging(())) == "U0" + assert script.check_headline("U0-F", _Staging((2005, 2007))) == "U0-F" + with pytest.raises(ValueError, match="staging changed"): + script.check_headline("U0", _Staging((2005, 2007))) + with pytest.raises(ValueError, match="staging changed"): + script.check_headline("U0-F", _Staging(())) def test_uncaptured_thresholds_stop_the_run_before_any_psid_read( @@ -215,6 +252,8 @@ def no_psid(**_): POINTER, "--registered-commit", COMMIT, + "--headline-row", + "U0-F", "--output", str(output), ] diff --git a/tests/track_u/test_rows.py b/tests/track_u/test_rows.py index ea7c7a5a..0c9107ea 100644 --- a/tests/track_u/test_rows.py +++ b/tests/track_u/test_rows.py @@ -23,7 +23,7 @@ def block() -> dict: def test_the_code_rows_equal_the_committed_block(block): check = rows.check_rows_against_block(block) assert check["rows_equal_the_block"] - assert check["specification_version"] == block["version"] == "u1-draft-3" + assert check["specification_version"] == block["version"] == "u1-draft-4" assert check["rows_checked"] == sorted(rows.REGISTERED_ROWS) @@ -58,7 +58,13 @@ def test_each_row_changes_the_field_the_plan_names(): inst = registered["U-inst"].age67_spec() assert inst.presence == "in_family_or_institution" assert inst.institution_income_rule == "family_of_record" - assert "referee" in registered["U-inst"].awaiting + assert "not yet refereed" in registered["U-inst"].awaiting + # U0-F is U0 on the fallback birth years; its headline status awaits + # Max's ruling on the fallback rule + fallback = registered[rows.FALLBACK_ROW] + assert fallback.age67_spec() == age67.Age67Spec(row="U0-F") + assert fallback.income_spec() == ap.AdjustedPovertySpec() + assert "fallback rule" in fallback.awaiting def test_row_from_block_reads_populations_and_overrides(): @@ -79,10 +85,26 @@ def test_row_from_block_reads_populations_and_overrides(): assert not rows.row_from_block( "U7", {"financial_assets": "y", "status": "not_built"} )["built"] + assert rows.row_from_block( + "U0-F", + { + "population": "birth_years_1941_1943_1945", + "on": "U0", + "awaiting": "y", + }, + ) == { + "age67": {"row": "U0-F"}, + "income": {}, + "built": True, + "awaiting": "y", + } + assert rows.row_from_block("U0", {"on": "U0"})["age67"] == {} with pytest.raises(ValueError, match="unknown block key"): rows.row_from_block("U9", {"mortality": "nchs_2000"}) with pytest.raises(ValueError, match="not known"): - rows.row_from_block("U1", {"on": "U0"}) + rows.row_from_block("U1", {"on": "U2"}) + with pytest.raises(ValueError, match="not known"): + rows.row_from_block("U1", {"population": "birth_years_1937"}) with pytest.raises(ValueError, match="unknown status"): rows.row_from_block("U-inst", {"status": "counted_only"}) @@ -93,6 +115,7 @@ def test_row_from_block_reads_populations_and_overrides(): ("U2", {"ssi_rule": "full_static_recomputation"}), ("U6", {"cut_start_year": 2005}), ("U-inst", {"institution_income_rule": "excluded"}), + ("U0-F", {"population": "all_ten_birth_years"}), ("U7", {"status": None}), ], ) diff --git a/tests/track_u/test_runner.py b/tests/track_u/test_runner.py index 27c602c8..bf8c00f6 100644 --- a/tests/track_u/test_runner.py +++ b/tests/track_u/test_runner.py @@ -68,6 +68,13 @@ def test_every_row_runs_and_u7_is_reported_not_built(result): assert cell["delta"] >= 0 assert result["labels"][0] == ut.INVENTED_DATA_LABEL assert result["labels"][1:] == list(ap.OUTPUT_LABELS) + assert result["headline"]["row"] == rows.PRIMARY_ROW + for entry in result["rows"].values(): + if entry["status"] == "computed": + assert entry["tabulation"]["design"]["domain"] == ( + "full_sample_design" + ) + assert entry["tabulation"]["design"]["singleton_strata"] == [] assert result["cohort_provenance"]["kind"] == age67.INVENTED assert result["psid_files_sha256"] == {} assert result["input_checks"]["invented_inputs"]["regenerated"] @@ -108,6 +115,22 @@ def test_rows_change_only_what_they_register(result): # U9 prices on the SSA 2004 table assert rows_["U9"]["life_table"] == "ssa_period_2004" assert u0["life_table"] == "nchs_2000" + # U5 keeps the F4a and F4b items too; U0 removes them + counts = u0["income_concept_counts"] + assert counts["n_retirement_account_income_removed_nonzero"] > 0 + assert counts["n_farm_asset_income_removed_nonzero"] > 0 + for key in ( + "n_retirement_account_income_removed_nonzero", + "n_farm_asset_income_removed_nonzero", + ): + assert rows_["U5"]["income_concept_counts"][key] == 0 + # U0-F is U0 on 1941, 1943 and 1945 + fallback = rows_["U0-F"]["population"]["observations_by_birth_year"] + assert fallback == { + year: n + for year, n in u0["population"]["observations_by_birth_year"].items() + if year in ("1941", "1943", "1945") + } def test_f17_diagnostics_are_reported_for_u0(result): @@ -220,11 +243,14 @@ def test_a_registered_run_refuses_invented_or_blocked_inputs(params, staged): registration_pointer=POINTER, allow_blocked=True, ) + # refused waves no longer stop a registered run at the input check + # (the fallback rule applies); these inputs still fail the parameter + # check, since no Census capture is pinned blocked = dataclasses.replace( invented.invented_age67_inputs(), provenance={**staged.provenance, "kind": age67.PSID_FILES}, ) - with pytest.raises(runner.TrackURunError, match="WEALTH1"): + with pytest.raises(runner.TrackURunError, match="Census threshold"): runner.run_track_u( blocked, params, @@ -233,6 +259,54 @@ def test_a_registered_run_refuses_invented_or_blocked_inputs(params, staged): ) +def test_the_fallback_rule_as_staged_today(params): + """Specification section 11: without the 2005/2007 wealth, U0-F is the + headline and every row that needs those waves is reported blocked with + its counts (INVENTED data).""" + + blocked = invented.invented_age67_inputs() + assert runner.headline_row(blocked) == rows.FALLBACK_ROW + result = runner.run_track_u(blocked, params, data_provenance=ap.INVENTED) + assert result["headline"]["row"] == "U0-F" + assert result["headline"]["rule"] == rows.HEADLINE_RULE + assert result["headline"]["wealth_refused_waves"] == [2005, 2007] + statuses = {row: entry["status"] for row, entry in result["rows"].items()} + assert statuses.pop("U0-F") == "computed" + assert statuses.pop("U7") == "not_built" + assert set(statuses.values()) == {"blocked"} + u0 = result["rows"]["U0"] + assert u0["tabulation"] is None + assert u0["blocked_waves"] == [2005, 2007] + assert set(u0["population"]["blocked_by_birth_year"]) == {"1937", "1939"} + u6 = result["rows"]["U6"] + assert u6["blocked_waves"] == [2005, 2007] + assert result["f17_diagnostics"]["row"] == "U0-F" + tabulated = result["rows"]["U0-F"]["population"] + assert set(tabulated["observations_by_birth_year"]) == { + "1941", + "1943", + "1945", + } + assert tabulated["left_out"] == {"wealth_supplement_not_staged": 0} + + +def test_blocked_waves_and_design_frame(): + staged = invented.invented_age67_inputs(supplement_waves_staged=True) + blocked = invented.invented_age67_inputs() + registered = rows.REGISTERED_ROWS + assert runner.headline_row(staged) == rows.PRIMARY_ROW + assert runner.blocked_waves(registered["U0"], staged) == [] + assert runner.blocked_waves(registered["U0"], blocked) == [2005, 2007] + assert runner.blocked_waves(registered["U0-F"], blocked) == [] + design = runner.design_frame(staged, registered["U0"].age67_spec()) + assert list(design.columns) == ["stratum", "cluster"] + assert not design.duplicated().any() + # the invented design: eight strata of two clusters + assert design.groupby("stratum")["cluster"].nunique().to_dict() == ( + dict.fromkeys(range(1, 9), 2) + ) + + def test_a_registered_run_needs_the_committed_pinned_parameters(params): with pytest.raises(runner.TrackURunError, match="Census threshold"): runner._check_parameters(params, ap.REGISTERED_REAL) From 5e32237e18e490c29eae4f3028c358bc3511460e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 24 Sep 2026 06:54:47 -0400 Subject: [PATCH 3/9] List the Track U tabulation's pending decisions and record the u1-draft-4 structural counts - uniform_cut_tabulation.pending_decisions(): the design SE domain (full_sample_design, alternative tabulated_rows; referee Q5/R10) and the cells (plan decision 2(b)); the runner and the dry run carry them with the income-concept and builder decisions. - Specification u1-draft-4, sections 3, 14, 16 and 19: the structural counts recollected under the new rules on the staged PSID (counts only; evidence EVID/track-u-structure-u1d4-20260924/): 1 of the 35 unresolved U0 marital states resolves by relationship code (U1: 3 of 98); every annuitant age came from a derived birth year; among the 320 computable U0 observations, 5 have family farm income and 18 head annuity or IRA income (U1: 9 and 66 of 857). The dry-run evidence for this draft is EVID/track-u-dry-run-u1d4-20260924/. - Tier counts recollected (7,435). No income concept, threshold or poverty statistic was computed on PSID data. Co-Authored-By: Claude Opus 5.5 --- .../boomers2004_uniform_cut_comparison.md | 32 +++++++++++---- scripts/track_u_dry_run.py | 1 + .../estimates/uniform_cut_tabulation.py | 39 +++++++++++++++++++ .../uniform_cut_track_u/runner.py | 2 + tests/README-tiers.md | 4 +- .../estimates/test_uniform_cut_tabulation.py | 10 +++++ tests/tier_counts.json | 2 +- tests/track_u/test_dry_run_script.py | 1 + tests/track_u/test_runner.py | 6 +++ 9 files changed, 86 insertions(+), 11 deletions(-) diff --git a/docs/design/boomers2004_uniform_cut_comparison.md b/docs/design/boomers2004_uniform_cut_comparison.md index 6e5220bc..ffe2bcb4 100644 --- a/docs/design/boomers2004_uniform_cut_comparison.md +++ b/docs/design/boomers2004_uniform_cut_comparison.md @@ -227,10 +227,20 @@ moved out or died); the evidence file lists every disposition. Of the 483 U0 observations, 35 are OFUMs (95 of 1,349 under U1), 457 have marriage-history birth years, and 35 have an unresolved marital state (26 `no_marriage_history`, 9 `unknown`), which `u1-draft-1` counted as -non-married. The counts under this draft's rules (resolution paths, -annuitant age sources, farm and head retirement-account income) are -recollected by the same script; §14 records them. No income, threshold -or poverty status was computed. +non-married. + +Recollected under this draft's rules (the same script, evidence +`EVID/track-u-structure-u1d4-20260924/`, counts only): the observation +counts are unchanged, and U0-F has 320 observations. Of the 35 +unresolved U0 states, 1 resolves by relationship code (a head with a +co-resident legal spouse) and 34 stay non-married (U1: 3 of 98 resolve). +Every annuitant age (the head, the head's legal spouse and the member's +co-resident spouse) came from a derived birth year; no wave-age fallback +was needed. Among the 320 computable U0 observations (U0-F's), 5 belong +to families with nonzero farm income (none negative) and 18 to families +whose head has annuity or IRA income; among U1's 857 computable +observations, 9 and 66 (one farm loss). No income, threshold or poverty +status was computed. ## 4. Income concept @@ -760,7 +770,9 @@ the reachability guard: - `scripts/track_u_dry_run.py` (U9): every row on the invented cohort with invented thresholds and the committed life tables and SSI capture, and the fallback rule on the invented cohort as staged today; - evidence headed "INVENTED DATA - NOT A COMPARISON". + evidence headed "INVENTED DATA - NOT A COMPARISON", + `EVID/track-u-dry-run-20260924/` (`u1-draft-3`) and + `EVID/track-u-dry-run-u1d4-20260924/` (this draft). - `scripts/run_track_u_registered.py` (U10 entry point): refuses unless the pointer is an issue #42 comment, `HEAD` is the registered commit on a clean tree, §15 is ratified with nothing awaiting and nothing @@ -988,8 +1000,9 @@ holds it to the code's defaults. None of these is ratified. Each is a code parameter whose default is the plan's proposal, the referee's recommendation or a builder choice where -both are silent; `adjusted_poverty.pending_decisions()` and -`age67.pending_decisions()` return them with their basis. +both are silent; `adjusted_poverty.pending_decisions()`, +`age67.pending_decisions()` and `uniform_cut_tabulation.pending_decisions()` +(the design SE domain and the cells) return them with their basis. **Awaiting Max (d189, open):** @@ -1210,7 +1223,10 @@ Heeringa 2008); any SCF wealth aggregate (none is committed or saved). the invented generator adds couples resolved by relationship code, legal husbands, farm, annuity and IRA income, and interview ages; §15 gains `referee_passes`, `comparison`, the headline rule and the new - fields; §17 records the pass and the open questions. + fields; §17 records the pass and the open questions; the structural + counts are recollected under the new rules + (`EVID/track-u-structure-u1d4-20260924/`) and the dry run re-run + (`EVID/track-u-dry-run-u1d4-20260924/`). - `u1-draft-3` (2026-09-24, continuation on `dynamics-ex2-track-u-2-20260924`): row U-inst gets an income rule (`family_of_record`) and row U6 is built (`cut_start_year`, the start diff --git a/scripts/track_u_dry_run.py b/scripts/track_u_dry_run.py index c74fbc91..ead5fc4b 100644 --- a/scripts/track_u_dry_run.py +++ b/scripts/track_u_dry_run.py @@ -377,6 +377,7 @@ def results_markdown(result: dict[str, Any]) -> str: for item in ( result["pending_decisions"]["age67"] + result["pending_decisions"]["income_concept"] + + result["pending_decisions"]["tabulation"] ): lines.append( f"- `{item['field']}` = `{item['default']}` (awaiting " diff --git a/src/populace_dynamics/estimates/uniform_cut_tabulation.py b/src/populace_dynamics/estimates/uniform_cut_tabulation.py index 72770626..07ec87a5 100644 --- a/src/populace_dynamics/estimates/uniform_cut_tabulation.py +++ b/src/populace_dynamics/estimates/uniform_cut_tabulation.py @@ -82,6 +82,7 @@ INVENTED, OUTPUT_LABELS, REGISTERED_REAL, + PendingDecision, ) from populace_dynamics.harness.panel import split_panel_by_person @@ -103,6 +104,7 @@ "TabulationConfig", "UniformCutTabulationError", "floor_split_units", + "pending_decisions", "tabulate_uniform_cut", "tabulation_rows", ] @@ -515,6 +517,43 @@ def _floors( return per_seed, floors +def pending_decisions() -> tuple[PendingDecision, ...]: + """The open choices of :class:`TabulationConfig`, with their defaults. + + None is ratified; each awaits the specification freeze (or, for the + cells, plan section 10 decision 2(b)). + """ + + config = TabulationConfig() + freeze = "U1 specification freeze (Max's ratification by merge)" + return ( + PendingDecision( + "design_se_domain", + config.design_se_domain, + ("tabulated_rows",), + "referee (boomers2004-referee-20260924.md) Q5/R10: domain " + "estimation on the full sample design, every (stratum, " + "cluster) pair of the observation waves' positive-weight " + "persons with z = 0 outside the cell; relative to the " + "tabulated rows only, every stratum with one cluster present " + "drops out and the variance is understated", + freeze, + ), + PendingDecision( + "cells", + list(config.cells), + (), + "plan section 7: all (headline), men, women, married, " + "non_married (the Report's list of appendix tables names " + "Appendix Table 17 'by Gender and Marital Status') and the " + "four optional sex-by-marital cells; which splits Tables 19 " + "and 21 carry is unknown", + "Max (plan section 10 decision 2(b)) and the specification " + "freeze", + ), + ) + + def tabulation_rows( members: pd.DataFrame, adjusted: pd.DataFrame ) -> pd.DataFrame: diff --git a/src/populace_dynamics/uniform_cut_track_u/runner.py b/src/populace_dynamics/uniform_cut_track_u/runner.py index 96505d43..9d567899 100644 --- a/src/populace_dynamics/uniform_cut_track_u/runner.py +++ b/src/populace_dynamics/uniform_cut_track_u/runner.py @@ -436,6 +436,7 @@ def _income_counts(adjusted: pd.DataFrame) -> dict[str, Any]: def _pending(row: TrackURow) -> list[dict[str, Any]]: items = [item.as_dict() for item in age67.pending_decisions()] items += [item.as_dict() for item in ap.pending_decisions()] + items += [item.as_dict() for item in ut.pending_decisions()] if row.awaiting: items.append( { @@ -670,6 +671,7 @@ def run_track_u( "income_concept": [ item.as_dict() for item in ap.pending_decisions() ], + "tabulation": [item.as_dict() for item in ut.pending_decisions()], "rows": { row_id: row.awaiting for row_id, row in rows.items() diff --git a/tests/README-tiers.md b/tests/README-tiers.md index eb5feb7f..9eb402d4 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -38,9 +38,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 3,044 | +| `unit` | 3,045 | | `artifact` | 2,844 | | `integration_psid` | 863 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 163 | -| **Total** | **7,434** | +| **Total** | **7,435** | diff --git a/tests/estimates/test_uniform_cut_tabulation.py b/tests/estimates/test_uniform_cut_tabulation.py index 634c8574..a177f259 100644 --- a/tests/estimates/test_uniform_cut_tabulation.py +++ b/tests/estimates/test_uniform_cut_tabulation.py @@ -405,6 +405,16 @@ def test_rows_are_validated(column, value): ) +def test_pending_decisions_name_the_config_defaults(): + decisions = {item.field: item for item in ut.pending_decisions()} + config = ut.TabulationConfig() + assert decisions["design_se_domain"].default == config.design_se_domain + assert decisions["design_se_domain"].alternatives == ("tabulated_rows",) + assert "Q5" in decisions["design_se_domain"].default_basis + assert decisions["cells"].default == list(config.cells) + assert "2(b)" in decisions["cells"].awaiting + + def test_config_validation(): with pytest.raises(ut.UniformCutTabulationError): ut.TabulationConfig(cells=("men",)) diff --git a/tests/tier_counts.json b/tests/tier_counts.json index a7376009..19622fff 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 3044, + "unit": 3045, "artifact": 2844, "integration_psid": 863, "reproduction_legacy": 520, diff --git a/tests/track_u/test_dry_run_script.py b/tests/track_u/test_dry_run_script.py index 19a5352e..c99eee67 100644 --- a/tests/track_u/test_dry_run_script.py +++ b/tests/track_u/test_dry_run_script.py @@ -98,3 +98,4 @@ def test_every_built_row_is_tabulated(output): assert result["headline"]["row"] == "U0" text = (output / "RESULTS.md").read_text() assert "Headline row U0" in text and "full sample design" in text + assert "`design_se_domain` = `full_sample_design`" in text diff --git a/tests/track_u/test_runner.py b/tests/track_u/test_runner.py index bf8c00f6..44ba8021 100644 --- a/tests/track_u/test_runner.py +++ b/tests/track_u/test_runner.py @@ -69,6 +69,12 @@ def test_every_row_runs_and_u7_is_reported_not_built(result): assert result["labels"][0] == ut.INVENTED_DATA_LABEL assert result["labels"][1:] == list(ap.OUTPUT_LABELS) assert result["headline"]["row"] == rows.PRIMARY_ROW + assert { + item["field"] for item in result["pending_decisions"]["tabulation"] + } == { + "design_se_domain", + "cells", + } for entry in result["rows"].values(): if entry["status"] == "computed": assert entry["tabulation"]["design"]["domain"] == ( From c51e7c759c9af453e95e59786f94239f6b0d24b9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 24 Sep 2026 07:59:57 -0400 Subject: [PATCH 4/9] Refuse unpinned Census thresholds in the Track U runner; drop U3 "upper bound" labels Independent review of 310c53a5..5e32237e (Track U continuation and the first referee pass, u1-draft-4). - uniform_cut_track_u.runner._check_parameters: a registered_real run compared the thresholds' recorded SHA-256 with adjusted_poverty.THRESHOLDS_SHA256, which is None until the Census capture is committed, so a table labelled "census_capture" whose provenance omitted its SHA-256 (or recorded None) passed the check (None == None). The check now refuses every table while the pin is unset. Regression tests: the unpinned cases are refused, and with an invented pin a table recording it passes while others are refused. - adjusted_poverty.SSI_RULES and uniform_cut_track_u.rows (row U3): the descriptions still called U3 "an upper bound"; the specification (sections 8 and 11, referee suggestion O4) calls it the largest SSI response of the three registered rules and not a bound on DYNASIM's simulation. The run artifact now carries the specification's wording. - Tier counts recollected (7,436). No income concept, threshold or poverty statistic was computed on PSID data. Co-Authored-By: Claude Opus 5.5 --- .../estimates/adjusted_poverty.py | 4 +- .../uniform_cut_track_u/rows.py | 4 +- .../uniform_cut_track_u/runner.py | 13 ++++-- tests/README-tiers.md | 4 +- tests/tier_counts.json | 2 +- tests/track_u/test_runner.py | 45 +++++++++++++++++++ 6 files changed, 64 insertions(+), 8 deletions(-) diff --git a/src/populace_dynamics/estimates/adjusted_poverty.py b/src/populace_dynamics/estimates/adjusted_poverty.py index 8a292c13..588a5439 100644 --- a/src/populace_dynamics/estimates/adjusted_poverty.py +++ b/src/populace_dynamics/estimates/adjusted_poverty.py @@ -287,7 +287,9 @@ "none": "no SSI response (row U2)", "full_static_recomputation": ( "the offset plus take-up by every newly income- and " - "resource-eligible head/wife unit (row U3, an upper bound)" + "resource-eligible head/wife unit (row U3: the largest SSI " + "response of the three registered rules, not a bound on " + "DYNASIM's simulation)" ), } SSI_DEEMING: dict[str, str] = { diff --git a/src/populace_dynamics/uniform_cut_track_u/rows.py b/src/populace_dynamics/uniform_cut_track_u/rows.py index 4d26a06a..7023a965 100644 --- a/src/populace_dynamics/uniform_cut_track_u/rows.py +++ b/src/populace_dynamics/uniform_cut_track_u/rows.py @@ -136,7 +136,9 @@ def as_dict(self) -> dict[str, Any]: TrackURow( "U3", "SSI", - "full static SSI recomputation (upper bound)", + "full static SSI recomputation (the largest SSI response of " + "the three registered rules, not a bound on DYNASIM's " + "simulation)", income={"ssi_rule": "full_static_recomputation"}, ), TrackURow( diff --git a/src/populace_dynamics/uniform_cut_track_u/runner.py b/src/populace_dynamics/uniform_cut_track_u/runner.py index 9d567899..2d49588d 100644 --- a/src/populace_dynamics/uniform_cut_track_u/runner.py +++ b/src/populace_dynamics/uniform_cut_track_u/runner.py @@ -214,12 +214,19 @@ def _check_parameters(params: TrackUParameters, data_provenance: str) -> None: thresholds = ( {} if params.thresholds is None else dict(params.thresholds.provenance) ) - if thresholds.get("kind") != "census_capture" or ( - thresholds.get("sha256") != ap.THRESHOLDS_SHA256 + # Until the capture is committed the pin is None, and a table whose + # provenance simply omits its SHA-256 would otherwise compare equal to + # it: no threshold table passes while nothing is pinned. + if ( + ap.THRESHOLDS_SHA256 is None + or thresholds.get("kind") != "census_capture" + or thresholds.get("sha256") != ap.THRESHOLDS_SHA256 ): raise TrackURunError( "a registered run needs the committed, pinned Census threshold " - f"capture; the thresholds record {thresholds!r}" + "capture (adjusted_poverty.THRESHOLDS_SHA256 is " + f"{ap.THRESHOLDS_SHA256!r}); the thresholds record " + f"{thresholds!r}" ) if params.ssi.provenance.get("kind") != "policyengine_us_capture" or ( params.ssi.provenance.get("sha256") != ap.SSI_PARAMETERS_SHA256 diff --git a/tests/README-tiers.md b/tests/README-tiers.md index 9eb402d4..0200b372 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -39,8 +39,8 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| | `unit` | 3,045 | -| `artifact` | 2,844 | +| `artifact` | 2,845 | | `integration_psid` | 863 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 163 | -| **Total** | **7,435** | +| **Total** | **7,436** | diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 19622fff..5d02453e 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -2,7 +2,7 @@ "schema_version": 1, "counts": { "unit": 3045, - "artifact": 2844, + "artifact": 2845, "integration_psid": 863, "reproduction_legacy": 520, "oracle_policyengine": 163 diff --git a/tests/track_u/test_runner.py b/tests/track_u/test_runner.py index 44ba8021..9f07ee88 100644 --- a/tests/track_u/test_runner.py +++ b/tests/track_u/test_runner.py @@ -328,6 +328,51 @@ def test_a_registered_run_needs_the_committed_pinned_parameters(params): assert ap.THRESHOLDS_SHA256 is None with pytest.raises(runner.TrackURunError, match="pinned"): runner._check_parameters(fake_capture, ap.REGISTERED_REAL) + # regression (independent review, 2026-09-24): a table labelled as the + # capture whose provenance omits its SHA-256, or records None, matched + # the unset pin (None == None) and passed + for provenance in ( + {"kind": "census_capture"}, + {"kind": "census_capture", "sha256": None}, + ): + unpinned = dataclasses.replace( + params, + thresholds=dataclasses.replace( + params.thresholds, provenance=provenance + ), + ) + with pytest.raises(runner.TrackURunError, match="pinned"): + runner._check_parameters(unpinned, ap.REGISTERED_REAL) + + +def test_a_pinned_capture_passes_the_parameter_check(params, monkeypatch): + """Once a capture is pinned, a table recording that SHA-256 passes and + any other is refused (the pin here is INVENTED; nothing is captured).""" + + pin = "a" * 64 + monkeypatch.setattr(ap, "THRESHOLDS_SHA256", pin) + + def with_provenance(provenance): + return dataclasses.replace( + params, + thresholds=dataclasses.replace( + params.thresholds, provenance=provenance + ), + ) + + runner._check_parameters( + with_provenance({"kind": "census_capture", "sha256": pin}), + ap.REGISTERED_REAL, + ) + for provenance in ( + {"kind": "census_capture"}, + {"kind": "census_capture", "sha256": "b" * 64}, + {"kind": "invented", "sha256": pin}, + ): + with pytest.raises(runner.TrackURunError, match="pinned"): + runner._check_parameters( + with_provenance(provenance), ap.REGISTERED_REAL + ) invented_table = ap.LifeTable( name=ap.INVENTED_LIFE_TABLE, qx={sex: (0.1,) * 110 + (1.0,) for sex in ("male", "female")}, From 37a94ea263c092d8fd9a802ad7604047a22798b0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 24 Sep 2026 14:19:47 -0400 Subject: [PATCH 5/9] Correct four Track U documentation defects found in independent review Independent adversarial review of 310c53a5..c51e7c75 (Track U continuation and the first referee pass, u1-draft-4). No rule, default or row changes; the behaviour tests pin what the corrected text says. - adjusted_poverty module text: row U3 was still called "an upper bound" across a line break after c51e7c75 dropped the label from SSI_RULES and the rows; it now uses the specification's wording (referee O4). Regression test collapses whitespace before searching the module text, SSI_RULES, the row descriptions and the specification. - Specification section 9 said every member of a family unit gets the same B, R and T under fu_head_rule; under row U6 the cut follows each member's age-67 year (cut_applies), so two members of one family unit can get different R. The section now names the U4 and U6 exceptions. Tests: an invented family with 1936 and 1938 members under cut_start_year=2004, and a text check of section 9. - age67._annuitant_birth_years and specification section 5 said the R8 extension seeds a person from the earliest wave in which they are in a family; the code seeds from the earliest wave in which they hold a head, spouse or partner code (or, for a marriage-history spouse, are in a family). The text now states the implemented rule and records a review count on the staged PSID: every in-family person with those codes has a positive weight in 2005-2013, so the extension adds no one to U0 or U1. Test: an invented zero-weight wife who is an OFUM earlier keeps the seed of her first spouse-coded wave. - Specification section 3 listed code 90 beside the role codes; the builder's role for a code-90 legal husband is OFUM (U4 keeps him on the family-unit basis), which the section and question 11 now say, with a review count of 2 U0 and 4 U1 such observations. - family_income module text: an over-length line reflowed. - Tier counts recollected (7,440). Review counts on the staged PSID were structural only (relationship codes, universe membership, design strata); no income concept, threshold or poverty statistic was computed on PSID data. Co-Authored-By: Claude Opus 5.5 --- .../boomers2004_uniform_cut_comparison.md | 57 +++++++++++++++---- src/populace_dynamics/cohorts/age67.py | 17 ++++-- src/populace_dynamics/data/family_income.py | 3 +- .../estimates/adjusted_poverty.py | 9 +-- tests/README-tiers.md | 4 +- tests/cohorts/test_age67.py | 41 +++++++++++++ tests/estimates/test_adjusted_poverty.py | 45 +++++++++++++++ tests/test_boomers2004_uniform_cut_spec.py | 39 +++++++++++++ tests/tier_counts.json | 2 +- 9 files changed, 193 insertions(+), 24 deletions(-) diff --git a/docs/design/boomers2004_uniform_cut_comparison.md b/docs/design/boomers2004_uniform_cut_comparison.md index ffe2bcb4..1f286599 100644 --- a/docs/design/boomers2004_uniform_cut_comparison.md +++ b/docs/design/boomers2004_uniform_cut_comparison.md @@ -192,10 +192,17 @@ splits on family units linked through shared persons (§10). **Attached per observation** (`cohorts/age67.build_age67_cohort`): sex (ER32000 through the death-record reader); relationship to head (10 -head, 20 legal wife, 22 cohabiting "wife", 90 legal husband of head, -otherwise OFUM; the codes are verified in `IND2023ER_formats.sas` for -each wave); marital status at the end of the income year from the -marriage history (F12; separated counts as married); whether a legal +head, 20 legal wife, 22 cohabiting "wife", 90 legal husband of head; +the codes are verified in `IND2023ER_formats.sas` for each wave) and the +member's role: head (10), wife (20 or 22), otherwise OFUM. A code-90 +legal husband's role is therefore OFUM (row U4 gives him the family-unit +basis, and the F17 summaries do not identify his own Social Security), +although the annuity (§5) and the marital resolution below treat him as +the head's legal spouse; 2 of the 483 U0 observations and 4 of the 1,349 +U1 observations are code-90 husbands (a structural count made in the +independent review of this draft; question 11); marital status at the +end of the income year from the marriage history (F12; separated counts +as married); whether a legal spouse lives in the same family unit, and the spouse's age and sex; the family head and the head's co-resident legal spouse (code 20 or 90), with their ages and sexes (the annuitants of §5); and the sampling-error @@ -353,9 +360,17 @@ and WEALTH1 alone the alternative. minus derived birth year, from `estimates.career.derive_birth_years` extended beyond the universe to in-family heads, legal wives, cohabitors (22, 88), legal husbands and marriage-history spouses of - cohort members (typically zero-weight nonsample spouses; their seed - coordinate is the earliest of the five waves in which they are in a - family). Where the law leaves a person unresolved (including + cohort members who have no positive cross-section weight (such as + zero-weight nonsample spouses). The extension's seed coordinate is the + earliest of the five waves in which the person is in a family with one + of those codes, or, for a marriage-history spouse, in a family at all: + the analogue of the universe's earliest presence wave. On the staged + files every in-family person with one of those codes has a positive + cross-section weight in each of the five waves, so all of them are in + the universe and the extension adds no one to U0 or U1 (a structural + count made in the independent review of this draft); it matters only + for inputs where such a person has zero weight. Where the law leaves a + person unresolved (including conflicting marriage-history birth years, which the extension leaves out rather than failing the build), the individual-file age at the wave is used and counted (`wave_age`). The law gives the @@ -514,7 +529,8 @@ Deeming from an ineligible spouse: 416.1163. For cell c, with observation weights w, baseline adjusted income B, reform income R and threshold T (family-level under `fu_head_rule`: -every member of a family unit gets the same B, R and T): +every member of a family unit gets the same B, R and T, except in rows +U4 and U6 below): ```text P_B[c] = 100 × Σ_{i∈c} w_i·1{B_i < T_i} / Σ_{i∈c} w_i @@ -525,7 +541,12 @@ P_R[c] = 100 × Σ_{i∈c} w_i·1{R_i < T_i} / Σ_{i∈c} w_i Δ is the headline; P_B and P_R are the secondary rows (the Table 19 and Table 21 analogues). Under row U4, head and wife members use the head-and-wife basis and OFUM members the family basis, so B and T differ -by role there. +by role there. Under row U6 the cut follows each member's own age-67 +year (§7), so two cohort members of one family unit can get different +R. For example, under U1 a 1936 member observed at 68 and a 1938 member +of the same family unit observed at 66 both have income year 2004 (wave +2005), and with `cut_start_year = 2004` only the 1938 member's +observation is cut. **Cells.** `all` (headline); `men`, `women`, `married`, `non_married` (the Report's list of appendix tables includes Appendix Table 17, @@ -1130,7 +1151,12 @@ birth years (R8). 11. Is extending the head's legal spouse to code 90 (legal husband), in the annuity and in the relationship-code resolution, right, given that the family file's "wife" items' treatment of a code-90 husband - was not checked? + was not checked? The builder gives him role OFUM, so row U4 keeps him + on the family-unit basis; 2 of 483 U0 and 4 of 1,349 U1 observations + are code-90 husbands (§3). The 2011 family codebook groups "husband of + Head" with Wife/"Wife" in its composition and birth variables, but + describes the income items as the Wife's/"Wife's" only (independent + review; not settled). 12. Under the fallback rule, should the one-field alternatives follow the headline to U0-F's population (so that U2–U5 and U8–U10 are computed when the supplements are not staged), or stay on U0 and be reported @@ -1207,6 +1233,17 @@ Heeringa 2008); any SCF wealth aggregate (none is committed or saved). ## 19. Changelog +- `u1-draft-4`, independent review corrections (2026-09-24; no rule, + default or row changed): §3 states that a code-90 legal husband's role + is OFUM (U4 keeps him on the family-unit basis) and counts those + observations (2 in U0, 4 in U1), also added to question 11; §5 states + the seed wave the birth-year extension actually uses (the earliest wave + in which the person holds a head, spouse or partner code) and that the + extension adds no one on the staged files; §9 notes that under U6 two + members of one family unit can get different R; the income concept's + module text now describes U3 as the largest SSI response of the three + rules, as §8 does (O4). Counts only; no income, threshold or poverty + status was computed on PSID data. - `u1-draft-4` (2026-09-24, first referee pass applied, §17): §1 states the Report's unit and financial-asset definition from pp. 19–24 and re-sources the cells to Appendix Table 17's title; F4a removes the diff --git a/src/populace_dynamics/cohorts/age67.py b/src/populace_dynamics/cohorts/age67.py index c1be8da9..23ebaad5 100644 --- a/src/populace_dynamics/cohorts/age67.py +++ b/src/populace_dynamics/cohorts/age67.py @@ -837,12 +837,17 @@ def _annuitant_birth_years( birth-year law is extended to every in-family person (sequence 1-20) of the five waves coded head, legal wife, cohabiting "wife", first-year cohabitor or legal husband, and to every marriage-history spouse of a - target member, who is not in the universe (typically a zero-weight - nonsample spouse). The seed coordinate is the earliest of the five - waves in which the person is in a family, whatever the weight. The - law runs without a required population, so a person with conflicting - marriage-history birth years is left out (unresolved) instead of - failing the build; the caller then uses the wave age and counts it. + target member, who is not in the universe (a person with no positive + cross-section weight, such as a zero-weight nonsample spouse). The + seed coordinate is the earliest of the five waves in which the person + is in a family with one of those codes (a marriage-history spouse: in + a family at all), whatever the weight: the analogue of the universe's + earliest presence wave. On the staged PSID every in-family person + with one of those codes has a positive weight in each wave, so this + extension adds no one there. The law runs without a required + population, so a person with conflicting marriage-history birth years + is left out (unresolved) instead of failing the build; the caller + then uses the wave age and counts it. """ history = inputs.marriage_history diff --git a/src/populace_dynamics/data/family_income.py b/src/populace_dynamics/data/family_income.py index ebc496a7..bcd1b413 100644 --- a/src/populace_dynamics/data/family_income.py +++ b/src/populace_dynamics/data/family_income.py @@ -43,7 +43,8 @@ splits the head's ``HEAD ANNUITIES`` into annuities and ``HEAD IRAS-2012``; earlier files carry one ``WIFE RETIREMENT/ANNUITIES`` item ("Income from Pensions and Annuities"), and their ``HEAD ANNUITIES`` is -"Head's Income from Annuities and IRAs" (2005-2011 codebooks). Head and wife here are the PSID's; "wife" +"Head's Income from Annuities and IRAs" (2005-2011 codebooks). Head and +wife here are the PSID's; "wife" includes a cohabiting female partner ("Wife"). The accuracy flag of each Social Security, SSI and asset item is read raw (:data:`ACCURACY_CONCEPTS`; 0 is "Actual value"). diff --git a/src/populace_dynamics/estimates/adjusted_poverty.py b/src/populace_dynamics/estimates/adjusted_poverty.py index 588a5439..f399842f 100644 --- a/src/populace_dynamics/estimates/adjusted_poverty.py +++ b/src/populace_dynamics/estimates/adjusted_poverty.py @@ -82,10 +82,11 @@ with baseline SSI, SSI rises by the fall in countable Social Security income (after the $20 monthly general income exclusion), capped so SSI does not exceed the federal benefit rate (FBR); nobody newly enrols. - ``none`` (row U2). ``full_static_recomputation`` (row U3, an upper - bound): the offset, plus every head/wife unit that the cut makes newly - income-eligible and whose resource proxy passes the resource limit - takes up SSI. + ``none`` (row U2). ``full_static_recomputation`` (row U3, the largest + SSI response of the three registered rules, not a bound on DYNASIM's + simulation): the offset, plus every head/wife unit that the cut makes + newly income-eligible and whose resource proxy passes the resource + limit takes up SSI. * **Poverty**: poor when income is below the threshold (Census convention); baseline and reform status per row. diff --git a/tests/README-tiers.md b/tests/README-tiers.md index 0200b372..bcba6195 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -38,9 +38,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 3,045 | +| `unit` | 3,049 | | `artifact` | 2,845 | | `integration_psid` | 863 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 163 | -| **Total** | **7,436** | +| **Total** | **7,440** | diff --git a/tests/cohorts/test_age67.py b/tests/cohorts/test_age67.py index e3949134..5b0f62f2 100644 --- a/tests/cohorts/test_age67.py +++ b/tests/cohorts/test_age67.py @@ -719,3 +719,44 @@ def test_spec_validation_and_pending_decisions(): age67.Age67Spec(u1_single_observation_weight=0) fields = {item.field for item in age67.pending_decisions()} assert fields == set(age67.Age67Spec().as_dict()) + + +def test_extension_seed_is_the_first_wave_with_an_annuitant_code(): + """The birth-year law's extension seeds a person from the earliest wave + in which they are in a family with a head, spouse or partner code (a + marriage-history spouse: in a family at all), the analogue of the + universe's earliest presence wave. + + Regression (independent review, 2026-09-24): the docstring and + specification section 5 said "the earliest wave in which they are in + a family", which the code does not do. INVENTED: 9102 (zero weight, + no marriage history, the legal wife of 9101 in 2013 aged 70) is also + put in family 11 in 2009 as an OFUM (code 30) aged 62. That wave + would seed 2008 - 62 = 1946; her first spouse-coded wave, 2013, seeds + 2012 - 70 = 1942, so her income-year age in 2012 stays 70. + """ + + import dataclasses + + inputs = _with_unresolved_couples() + anchors = dict(inputs.anchors) + frame = anchors[2009].copy() + mask = frame["person_id"].eq(9102) + frame.loc[mask, ["interview", "sequence", "relationship", "age"]] = [ + 11, + 3, + 30, + 62, + ] + frame.loc[mask, "weight"] = 0.0 + anchors[2009] = frame + inputs = dataclasses.replace(inputs, anchors=anchors) + obs = age67.build_age67_cohort(inputs).observations.set_index( + "observation_id" + ) + head = obs.loc["9101:2013"] + assert head["fu_head_spouse_person_id"] == 9102 + assert head["fu_head_spouse_age"] == 70 + assert head["fu_head_spouse_age_source"] == "derived_birth_year" + # the OFUM record in 2009 changes nothing about family 11's annuitants + assert obs.loc["1001:2009", "fu_head_spouse_person_id"] == 1002 diff --git a/tests/estimates/test_adjusted_poverty.py b/tests/estimates/test_adjusted_poverty.py index 88f6e8c9..2a555b2a 100644 --- a/tests/estimates/test_adjusted_poverty.py +++ b/tests/estimates/test_adjusted_poverty.py @@ -803,3 +803,48 @@ def test_rows_are_validated(): _run(_row("a"), _row("a")) with pytest.raises(ap.AdjustedPovertyError, match="member_role"): _run(_row("a", member_role="lodger")) + + +def test_u6_cut_follows_each_members_age67_year_within_a_family(): + """Specification section 9: B, R and T are family-level under + ``fu_head_rule`` except in U4 and U6. Regression (independent review, + 2026-09-24): the U6 exception was not stated. + + INVENTED: one family unit "f" with family Social Security 1,000 and + money income 1,000 holds two cohort members, born 1936 (the head) and + 1938 (the wife). Primary: both are cut, 0.13 * 1,000 = 130, R = 870. + U6 (start 2004): 1936 + 67 = 2003 < 2004, so the head's observation is + uncut (R = 1,000); 1938 + 67 = 2005 >= 2004, so the wife's is cut + (R = 870). B and T stay equal across the two members. + """ + + rows = [ + _row( + "head", + family_unit_id="f", + birth_year=1936, + head_ss=1000, + total_family_income=1000, + fu_size=2, + wife_present=True, + ), + _row( + "wife", + family_unit_id="f", + member_role="wife", + member_sex="female", + birth_year=1938, + head_ss=1000, + total_family_income=1000, + fu_size=2, + wife_present=True, + ), + ] + primary = _run(*rows) + assert primary["reform_income"].tolist() == pytest.approx([870.0, 870.0]) + u6 = _run(*rows, cut_start_year=2004) + assert u6["cut_applies"].to_dict() == {"head": False, "wife": True} + assert u6.loc["head", "reform_income"] == pytest.approx(1000.0) + assert u6.loc["wife", "reform_income"] == pytest.approx(870.0) + for column in ("baseline_income", "threshold", "annuity"): + assert u6.loc["head", column] == u6.loc["wife", column], column diff --git a/tests/test_boomers2004_uniform_cut_spec.py b/tests/test_boomers2004_uniform_cut_spec.py index 1c001f14..69f92d20 100644 --- a/tests/test_boomers2004_uniform_cut_spec.py +++ b/tests/test_boomers2004_uniform_cut_spec.py @@ -385,3 +385,42 @@ def test_annuity_price_table_matches_the_committed_tables(text): table, "male", 67, "female", 67, rate=rate ) assert f"{joint:.4f}" in section + + +def test_u3_is_never_called_a_bound(text): + """Referee O4: U3 is the largest SSI response of the three rules, not a + bound on DYNASIM's simulation. + + Regression (independent review, 2026-09-24): after the labels were + dropped from ``SSI_RULES`` and the rows, the income concept's module + text still called U3 "an upper bound" across a line break, which a + line-by-line search misses; the check collapses whitespace first. + """ + + sources = { + "adjusted_poverty module text": ap.__doc__, + "adjusted_poverty.SSI_RULES": " ".join(ap.SSI_RULES.values()), + "registered rows": " ".join( + row.description for row in track_u_rows.REGISTERED_ROWS.values() + ), + "specification": text, + } + for name, source in sources.items(): + flat = " ".join(source.split()).lower() + assert "upper bound" not in flat, name + assert "the largest SSI response" in " ".join(ap.__doc__.split()) + + +def test_section_9_names_where_members_of_a_family_differ(text): + """Section 9's family-level claim holds under ``fu_head_rule`` except in + U4 (B and T by role) and U6 (the cut follows each member's age-67 + year, :func:`adjusted_poverty.cut_applies`); regression (independent + review, 2026-09-24): the U6 exception was missing.""" + + section = " ".join(_section(text, "9", "10").split()) + assert "except in rows U4 and U6" in section + assert "Under row U4" in section + assert "Under row U6 the cut follows each member's own age-67" in section + assert ap.CUT_START_YEAR_RULE == ( + "cut_when_birth_year_plus_67_at_or_after_start" + ) diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 5d02453e..f0bae8dd 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 3045, + "unit": 3049, "artifact": 2845, "integration_psid": 863, "reproduction_legacy": 520, From b862d2d70bedff935ad824e8feb6e1affea6b6ba Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 24 Sep 2026 16:24:05 -0400 Subject: [PATCH 6/9] Apply the second Track U referee pass to the exercise-2 specification (u1-draft-5) Second referee report boomers2004-referee-2-20260924.md (SHA-256 2113ba9b...d5af2) on u1-draft-4 (37a94ea2): ratifiable after S1-S9. Each change was checked against the cleared exercise-2 definitions extract (a3978b68...0384), the u1d4 structural counts, the PSID 1968 family codebook, the family_income transfer labels or the Census poverty-measures page before it was applied. - S1: section 1 states the target as the cleared extract defines it (the 1936-45 column, 36 rows, whole-number cells, the 2004 start and no behavioural response). - S2: cells are the Report's Total, Gender, Marital Status and Gender and Marital Status rows. age67 emits marital_status_4 (married, widowed, divorced, never_married, unclassified); the tabulation's cells, REPORT_ROWS and unclassified_marital_cells ("excluded_counted") follow, non_married is gone, and the official-concept diagnostic uses the same cells. - S3: the other 21 rows are named omissions (NOT_COMPUTED_REPORT_ROWS, listed in the run artifact); corrected to say only the own lifetime-earnings measure is stated to include uncovered earnings. - S4: the comparator interval is +/-0.5 per level and 1 for the difference. - S5: cut_start_year defaults to 2004; row U6 is withdrawn. - S6: named deltas drop "imputed rent excluded" and add the income-source and unresolved-marital deltas. - S7: row U-inst is withdrawn; institution_income_rule defaults to "excluded" (the referee's Q9 answer). - S8: U2-F ... U10-F register each one-field alternative on U0-F's population, awaiting the fallback rule. - S9: the extract, the pass and the S-item dispositions are recorded (section 17.3; referee_passes gains the second entry). Optional O2-O6 and O9 applied; O1 left to the freeze. The runner records the spouse's age source. No income, threshold or poverty status was computed on PSID data. Co-Authored-By: Claude Opus 5.5 --- .../boomers2004_uniform_cut_comparison.md | 630 +++++++++++++----- scripts/run_track_u_registered.py | 5 +- scripts/track_u_component_diagnostics.py | 7 +- src/populace_dynamics/cohorts/age67.py | 209 ++++-- .../estimates/adjusted_poverty.py | 55 +- .../estimates/uniform_cut_tabulation.py | 322 +++++++-- .../uniform_cut_track_u/__init__.py | 3 +- .../uniform_cut_track_u/diagnostics.py | 6 +- .../uniform_cut_track_u/invented.py | 4 +- .../uniform_cut_track_u/rows.py | 240 +++---- .../uniform_cut_track_u/runner.py | 60 +- tests/cohorts/test_age67.py | 123 +++- tests/estimates/test_adjusted_poverty.py | 76 ++- .../estimates/test_uniform_cut_tabulation.py | 133 +++- tests/test_boomers2004_uniform_cut_spec.py | 165 +++-- tests/track_u/test_invented_generator.py | 12 +- tests/track_u/test_registered_script.py | 2 +- tests/track_u/test_rows.py | 79 ++- tests/track_u/test_runner.py | 94 ++- 19 files changed, 1641 insertions(+), 584 deletions(-) diff --git a/docs/design/boomers2004_uniform_cut_comparison.md b/docs/design/boomers2004_uniform_cut_comparison.md index 1f286599..586312de 100644 --- a/docs/design/boomers2004_uniform_cut_comparison.md +++ b/docs/design/boomers2004_uniform_cut_comparison.md @@ -1,15 +1,17 @@ # Boomers 2004 uniform cut comparison: specification for DynaSim scorecard exercise 2 -- **Status:** draft for the referee. Nothing here is ratified. One - referee pass is recorded (§17); this draft applies it and awaits a - second pass on what the first did not see. Max has not ruled on - exercise 2 (cos decision d189, open, default "yes to Track U with the - proposed SSI rule; downloads needed from Max") or on the fallback rule - (§11). Every choice that awaits him or the specification freeze is an - explicit parameter in the code with the recommended default, and §16 - lists them. +- **Status:** draft for the referee. Nothing here is ratified. Two + referee passes are recorded (§17). The first was applied in + `u1-draft-4`; this draft applies the second, whose verdict was that the + draft is ratifiable after nine required changes (S1–S9, §17.3). No + independent check of this draft's application of them has been made + yet. Max has not ruled on exercise 2 (cos decision d189, open, default + "yes to Track U with the proposed SSI rule; downloads needed from + Max") or on the fallback rule (§11). Every choice that awaits him or + the specification freeze is an explicit parameter in the code with the + recommended default, and §16 lists them. - **Specification:** `boomers2004_uniform_cut_exercise2`, version - `u1-draft-4`, drafted 2026-09-24. §19 is the changelog. + `u1-draft-5`, drafted 2026-09-24. §19 is the changelog. - **Plan item:** U1 of the Track U plan, `EVID/critical-path-uniform-cut-20260923.md` (§7 fields F1–F17, §8 work items), where `EVID` = @@ -32,7 +34,7 @@ the range `EVID/RESTRICTED-FILES.md` clears; the page mapping (printed *n* = PDF *n* + 1) was checked from the footer digits of PDF pages 20 and 25 alone. Statements checked there now cite the Report page. No - lane opened printed page 25 or later for this draft, any results table + lane opened printed page 25 or later for `u1-draft-4`, any results table of the Report, the exercise-2 or COLA comparator directories, any comparator seal or reconciliation file, the Track A comparison memo, the issue #42 result comments, or page 3 or later of the Urban 2010 @@ -40,11 +42,23 @@ disclosed or used: a qualitative fragment of printed page 41 (late boomers; no number; no exercise-2 cell), shown when the plan mapped page numbers (plan §14), and the plan's reading of printed page 25 - (reporting splits, 2003 dollars). `RESTRICTED-FILES.md` names printed - pp. 19–24 as the readable methods range, so page-25 readings are cited - here only where another readable source confirms them (the cells now - cite the list of appendix tables). §18 lists what this draft read and - what it did not verify. + (reporting splits, 2003 dollars). `RESTRICTED-FILES.md` now clears + printed p. 25 layout lines 1–6 (the end of the Sample Criteria + subsection), and the cleared extract quotes them; both page-25 facts + come from those lines, and this draft cites them there. For + `u1-draft-5` the builder also read the cleared exercise-2 definitions + extract (SHA-256 + `a3978b683b4275424b6d12e9fe45f021fae277ccf9731a7883952564b6ed0384`), + which `RESTRICTED-FILES.md` lists as cleared for builders; statements + taken from it cite the Report page it quotes and say "cleared extract", + and the builder opened none of those pages (nor any page of the Report + PDF for this draft). When the second referee wrote, no values scan of + the extract was recorded (`RESTRICTED-FILES.md`, 2026-09-24 14:35). + As re-read during this pass, `RESTRICTED-FILES.md` lists the extract + as having passed the validation lane's values scan with 0 genuine leaks + at this hash, citing a changelog entry for 2026-09-24 16:10 that the + file did not yet hold. §18 lists what this draft read and what it did + not verify. ## 1. Target @@ -55,8 +69,12 @@ Institute #2004-05, May 2004): - **Tables:** 19, "Adjusted Poverty Rates at Age 67", and 21, "Adjusted Poverty Rates at Age 67, Assuming 13 Percent Reduction in Social Security Benefits" (list of tables, printed p. 5). The comparator is - the change from Table 19 to Table 21. Which rows those tables carry is - unknown to this builder (plan §10, decision 2(b)). + the change from Table 19 to Table 21 in the column headed "1936-45" + under "Birth Cohort". Both tables carry the same 36 data rows (a Total + row and 35 subgroup rows under eight section headers) and print every + cell as a whole number (cleared exercise-2 definitions extract, + `EVID/exercise2-definitions-cleared-20260924.md`, SHA-256 + `a3978b68…0384`, "cleared extract" below). §9 maps the rows to cells. - **Cohort:** persons born 1936–1945 ("near-retirees"), each analysed when they reach 67 (p. 24). - **Model:** DYNASIM, from a sample of about 100,000 individuals from the @@ -67,10 +85,18 @@ Institute #2004-05, May 2004): each individual's income and wealth are the household's, including the spouse's if married (p. 24). The income resources projected at 67 include earnings, SSI and co-resident income, meaning income of - non-spouse co-resident family members (p. 19). Whether co-residents - count in family size is not stated in pp. 19–24. The Report's income - differs from Census money income by imputing income from financial - assets as the real (price-indexed) annuity that 80 percent of those + non-spouse co-resident family members (p. 19). Results are reported + separately for married and non-married individuals and for men and + women, in 2003 dollars (p. 25, lines 1–6, cleared extract). Whether + co-residents count in family size is not stated anywhere in the + Report, nor whether their income enters the adjusted-poverty income: it + is part of total income (p. 31) and excluded from replacement-rate + income (p. 36; Table 13, note a), both as quoted in the cleared + extract. Imputed rent is excluded from the income used for poverty + rates (fn. 17, p. 31, cleared extract), as it is here. The Report's + income differs from Census money income by imputing income from + financial assets as the real (price-indexed) annuity that 80 percent of + those assets would buy (p. 24), recalculated each year, with a 50 percent survivor annuity for married couples (fn. 10). The Census measure, by contrast, counts interest and dividends, and counts retirement-account @@ -87,20 +113,30 @@ Institute #2004-05, May 2004): 65-and-over poverty threshold" (p. 24). - **Cut:** a 13 percent reduction in Social Security benefits: OCACT "estimates that benefits would need to be reduced immediately by 13 - percent" (p. 22, citing the 2003 Trustees Report). Its start year, - coverage, SSI treatment and any behavioural response are not in - pp. 19–24 (plan §10, decisions 2 and 8). + percent" (p. 22, citing the 2003 Trustees Report). The scenario assumes + "that current and future benefits would be cut immediately (beginning + in 2004) by 13 percent" and "no behavioral response to cuts in Social + Security benefits" (p. 37 and its fn. 19, as quoted in the cleared + extract; this builder did not open that page). The Report gives no + month within 2004 and does not say which benefit types the cut covers + or whether SSI is recomputed (cleared extract, "Things this extract + does not settle"). The comparator values are sealed on the comparator side. This builder -has not seen them and does not know which cells they cover. +has not seen them; it knows the tables' labels, notes and printed +precision only from the cleared extract. ## 2. Sources | Source | Use | Status | |---|---|---| | Plan, `critical-path-uniform-cut-20260923.md` | Every field proposal (§7) | Read in full | -| The Report: table of contents, lists of tables and appendix tables (PDF pp. 5–7); Section III, printed pp. 19–24 (PDF pp. 20–25) | §1, §4, §5 citations (u1-draft-4) | Read; nothing later | +| The Report: table of contents, lists of tables and appendix tables (PDF pp. 5–7); Section III, printed pp. 19–24 (PDF pp. 20–25) | §1, §4, §5 citations (u1-draft-4) | Read for `u1-draft-4`, nothing later; no page of the PDF was opened for `u1-draft-5` | +| Cleared exercise-2 definitions extract (`EVID/exercise2-definitions-cleared-20260924.md`, SHA-256 `a3978b68…0384`) | Row and column labels and printed precision of Tables 19 and 21; the cut's start year and behaviour; the income-measure statements (§1, §7, §9, §10a, §12) | Read in full; the only source for Report pages beyond PDF pp. 5–7 and 20–25 and printed p. 25 lines 1–6 | | Referee report `EVID/boomers2004-referee-20260924.md` | §17 | Read in full; every required change checked against its source | +| Second referee report `EVID/boomers2004-referee-2-20260924.md` (SHA-256 `2113ba9b…d5af2`) | §17.3 | Read in full; S1–S9 checked against their sources before they were applied | +| PSID 1968 family-file codebook (`psid-data/family/1968/fam1968_codebook.pdf`) | The PSID's first income year, 1967 (V74, "1967 Income from Labor - HEAD"), for the lifetime-earnings omission (§9) | Read for that entry (its whole-sample frequency counts were seen and not used) | +| Census Bureau, "Poverty measures" guidance page (census.gov, fetched 2026-09-24) | "Poverty status cannot be determined for people in: Institutional group quarters (such as prisons or nursing homes)" (§11, U-inst withdrawn) | Read for that sentence | | PSID family files 2005–2013 (`psid-data/family//FAMER.*`) | Family income components, Social Security, SSI, asset income, the head's annuity and IRA income, farm income, FU composition, PSID's needs standard; WEALTH1 in 2009–2013 | Staged; labels verified by `data/family_income.py`; codebook entries for `HEAD ANNUITIES`, `HEAD IRAS`, `WIFE RETIREMENT/ANNUITIES`, `OTHER FU MEMBR RETIREMENT/ANNUITIES`, ER52214, ER52216, ER52368 and W21, W33 read | | PSID wealth supplements 2005 and 2007 | WEALTH1 for the 1937 and 1939 birth years (U0) and six U1 observation cells | **Not staged.** The reader refuses with the missing files named (§3) | | PSID cross-year individual file (`ind2023er`) | Anchors per wave, cross-section weights, relationship to head, age, sampling-error stratum and cluster | Staged; labels verified by `cohorts/age67.py`, including relationship code 90 ("Legal husband of Head") | @@ -148,11 +184,11 @@ U0-F is the headline, by staging status only. **Universe per wave (F2).** Sequence 1–20 (in a responding family) and a positive core/immigrant cross-section weight. The cross-section weight is also positive for movers-out (71–80) and decedents (81–89); no row -admits them and the dispositions count them. Registered row U-inst adds -sequence 51–59 (institutions). +admits them and the dispositions count them. Institutionalized persons +(sequence 51–59) are outside the universe too and counted in the +dispositions; row U-inst, which admitted them, is withdrawn (§11). -**Institution income rule (U-inst; builder default of `u1-draft-3`, -pending the freeze, not yet refereed).** The plan gives +**Institution income rule (code option, not registered).** The plan gives institutionalized persons no income rule, and the PSID collects no income for them: the individual-file Social Security items are "Inap.: … in an institution" (codebook, ER34137–ER34143 for 2011). The PSID does @@ -161,21 +197,25 @@ institution it "attaches an institutional status data record to the family they left" (2011 User Guide, §2.4), and the family file's record-count variable counts "any institutionalized individuals associated with the family" among the records "having the same -family-level data". The default `institution_income_rule = -family_of_record` therefore gives an institutionalized observation the -family unit whose interview number its record carries: that family's +family-level data". The option `institution_income_rule = +family_of_record` (the `u1-draft-3` builder rule) gives an +institutionalized observation the family unit whose interview number its +record carries: that family's income concept, wealth, size, children and threshold, and (under `fu_head_rule`, §5) that family's annuity. Its role is OFUM: the relationship code of a person in an institution is to the previous wave's head (codebook note on ER34103), so it is neither the current head nor the wife, and it has no co-resident spouse. The member's own income is missing from the family's, and `# IN FU` ("the actual number -of persons currently in the FU") does not count them (§14). Both are -named deltas (§12). An institutionalized person whose interview number -has no family-file record that wave is a disposition -(`institution_family_of_record_missing`). The alternative `excluded` -keeps institutionalized persons out of the universe (disposition -`institution_excluded_by_rule`), which makes U-inst's population U0's. +of persons currently in the FU") does not count them (§14); both would +be named deltas of the option. An institutionalized person whose +interview number has no family-file record that wave is a disposition +(`institution_family_of_record_missing`). The code's default is now +`excluded` (the second referee's Q9 answer, §17.3): institutionalized +persons stay out of the universe (disposition +`institution_excluded_by_rule`), so the presence option +`in_family_or_institution` then has U0's population. No registered row +reads the rule. **Birth year.** `estimates.career.derive_birth_years` (the first-estimates §3.1 law, as Track A uses), total over the union of the five waves' @@ -200,9 +240,10 @@ basis, and the F17 summaries do not identify his own Social Security), although the annuity (§5) and the marital resolution below treat him as the head's legal spouse; 2 of the 483 U0 observations and 4 of the 1,349 U1 observations are code-90 husbands (a structural count made in the -independent review of this draft; question 11); marital status at the +independent review of `u1-draft-4`; question 11); marital status at the end of the income year from the marriage history (F12; separated counts -as married); whether a legal +as married) and the four-way status of §9 (`marital_status_4`: married, +widowed, divorced, never married or unclassified); whether a legal spouse lives in the same family unit, and the spouse's age and sex; the family head and the head's co-resident legal spouse (code 20 or 90), with their ages and sexes (the annuitants of §5); and the sampling-error @@ -212,9 +253,12 @@ date the state (`unknown`), or who has no marriage-history record `unresolved_marital_status="relationship_code"` (pending the freeze; referee Q7): married and co-resident when the member is head (10) with a co-resident legal spouse (20 or 90), or is the head's legal wife (20) or -legal husband (90); otherwise non-married. The alternative `non_married` -classifies them all as non-married (the `u1-draft-3` rule). Each -resolution path is counted (`marital_resolution`). The marital pairs +legal husband (90); otherwise the state stays unresolved: not married, +with four-way status `unclassified`, in no marital cell (§9). The +alternative `non_married` leaves them all unresolved (the `u1-draft-3` +rule, under which they counted as non-married). Each resolution path is +counted (`marital_resolution`; its label `unresolved_non_married` means +not married and unclassified). The marital pairs indicator of the individual file is not used. An institutionalized member is never resolved by code: its relationship code is to the previous wave's head. @@ -236,11 +280,11 @@ marriage-history birth years, and 35 have an unresolved marital state (26 `no_marriage_history`, 9 `unknown`), which `u1-draft-1` counted as non-married. -Recollected under this draft's rules (the same script, evidence +Recollected under `u1-draft-4`'s rules (the same script, evidence `EVID/track-u-structure-u1d4-20260924/`, counts only): the observation counts are unchanged, and U0-F has 320 observations. Of the 35 unresolved U0 states, 1 resolves by relationship code (a head with a -co-resident legal spouse) and 34 stay non-married (U1: 3 of 98 resolve). +co-resident legal spouse) and 34 stay unresolved (U1: 3 of 98 resolve). Every annuitant age (the head, the head's legal spouse and the member's co-resident spouse) came from a derived birth year; no wave-age fallback was needed. Among the 320 computable U0 observations (U0-F's), 5 belong @@ -249,6 +293,14 @@ whose head has annuity or IRA income; among U1's 857 computable observations, 9 and 66 (one farm loss). No income, threshold or poverty status was computed. +Four-way marital status (§9), summed from the same evidence's +marital-status and resolution counts: U0 has 331 married (330 by the +marriage history, 1 by relationship code), 44 widowed, 60 divorced, 14 +never married and 34 unclassified; U0-F has 227, 17, 43, 9 and 24; U1 +has 917, 124, 164, 49 and 95. No U0 or U0-F target person was in an +institution at the observation wave (no `not_present:institution` +disposition); U1 has 3 (1942: 2, 1944: 1). + ## 4. Income concept **Money income (F3).** Primary: the family unit's `TOTAL FAMILY INCOME` @@ -259,7 +311,22 @@ staged files this identity holds exactly for every family in all five waves (`reconcile_family_income`, counts only). Row U4: head and wife income only (head and wife taxable and transfer income and Social Security), size = the member plus a co-resident wife or partner; an OFUM -member keeps the family-unit basis. +member keeps the family-unit basis. In code the co-resident wife or +partner is the family file's `AGE OF WIFE` not 0 (`wife_present`), so +for a female head whose legal spouse is a code-90 legal husband U4's +size is 2 only where the family file records an age in the "wife" field +for him (question 11). + +**Tax treatment and co-resident basis.** The primary is pre-tax (`TOTAL +FAMILY INCOME`). The Report states tax treatment only for +replacement-rate income ("pre-tax and post-transfer", p. 36), not for +the adjusted-poverty income (cleared extract). The family-unit basis +follows the Report's co-resident income (p. 19) and its Appendix Table 1 +row "Living Arrangements of the Aged", under which "resources of +co-residing family members are imputed using donor families sampled from +current co-residing aged individuals in SIPP" (PDF p. 75, cleared +extract); whether that income enters the poverty measure is not stated +(§1). **Asset income (F4).** Primary: remove the reported asset income of the unit and add the annuity. Removed items (family file, every wave): head @@ -427,8 +494,11 @@ the 65-and-over rule). lists one spreadsheet per year, `thresh04.xlsx` … `thresh12.xlsx`, under `https://www2.census.gov/programs-surveys/cps/tables/time-series/historical-poverty-thresholds/`. The files have not been downloaded (downloads need Max's approval in -this environment, cos decision d194). Once the nine files are staged in -a directory, `scripts/capture_track_u_parameters.py --census-dir DIR` +this environment, cos decision d194). d194's text names ten files, +`thresh03.xlsx`–`thresh12.xlsx`; the income years need only the nine +from `thresh04`, and the capture reads each year's file by name +(`thresh04.xlsx` … `thresh12.xlsx`), so a staged `thresh03.xlsx` is not +read. Once the nine files are staged in a directory, `scripts/capture_track_u_parameters.py --census-dir DIR` parses them by their printed labels, checks that the 65-and-over rows sit below the under-65 rows and that thresholds rise with size, records each file's URL, bytes and SHA-256, and writes @@ -444,22 +514,23 @@ be assigned. **Primary (F11):** reform income subtracts 13 percent of all Social Security income of the unit (head, wife and OFUM amounts under the family -basis) in the income year. No behavioural response. Every U0 income year -is 2004 or later, so the unread start year matters only for U1's 1936 -row: row U6 (on U1) leaves 1936 uncut if the cut starts in 2004. The -Report's methods say "reduced immediately" (p. 22); the scorecard's -"from 2004" has no source this builder could read (plan §10, decision -8). - -**Row U6 (built; the start year awaits decision 8).** -`AdjustedPovertySpec.cut_start_year` (default `None`: every observation -is cut) is the row's parameter. With a start year, an observation is cut -only when its member turns 67 (birth year + 67) in or after it: the -observation stands for the member's status at 67, so the rule keys on -the age-67 year, not on the income year (U1 observes 1936 at 68, in -2004). With 2004, only the 1936 birth year (67 in 2003) is uncut; its -reform income equals its baseline, and with no fall in countable income -the SSI rules give it no offset and no new take-up. +basis) in the income year, for observations whose member turns 67 in +2004 or later (`cut_start_year = 2004`, pending the freeze). No +behavioural response (fn. 19, cleared extract). The Report cuts "current +and future benefits … immediately (beginning in 2004)" (p. 37; also +p. 44) and analyses each person "when they reach age 67" (p. 24), so its +1936 birth year, 67 in 2003, is uncut at its age-67 year. The rule keys +on the age-67 year (birth year + 67), not the income year, because each +observation stands for the member's status at 67. Every U0 and U0-F +observation turns 67 in 2004 or later and is cut; under U1 the 1936 +birth year, observed at 68 in income year 2004, is uncut: its reform +income equals its baseline, and with no fall in countable income the SSI +rules give it no offset and no new take-up. The Report gives no month +within 2004 and does not say how DYNASIM dates a person's age-67 year +within a calendar year. The alternative `cut_start_year = None` (every +observation cut, the `u1-draft-4` primary) has no source and is not +registered. Row U6 of `u1-draft-3` and `-4` is withdrawn: with 2004 as +the primary it would equal U1. ## 8. SSI response @@ -476,6 +547,13 @@ the federal benefit rate, but the PSID's reported SSI may include a state supplementary payment; where it does, the cap binds early and the offset is understated (named delta, §12). +The Report's Appendix Table 1 row "SSI Benefits" describes DYNASIM's SSI +module: "Eligibility is deterministic; 2 equations predict take-up of the +aged; key predictors include demographics, state supplement, resources" +(PDF p. 75, cleared extract). That is context for F13 and U3 only: the +Report does not say whether the 13 percent scenario re-runs SSI (cleared +extract, "Things this extract does not settle"). + - **SSI units (builder approximation):** head and wife form one unit, a couple unit when both receive SSI (couple FBR) and an individual unit otherwise; its Social Security includes the spouse's (deeming @@ -529,8 +607,8 @@ Deeming from an ineligible spouse: 416.1163. For cell c, with observation weights w, baseline adjusted income B, reform income R and threshold T (family-level under `fu_head_rule`: -every member of a family unit gets the same B, R and T, except in rows -U4 and U6 below): +every member of a family unit gets the same B, R and T, except in row +U4 and under U1 below): ```text P_B[c] = 100 × Σ_{i∈c} w_i·1{B_i < T_i} / Σ_{i∈c} w_i @@ -541,21 +619,55 @@ P_R[c] = 100 × Σ_{i∈c} w_i·1{R_i < T_i} / Σ_{i∈c} w_i Δ is the headline; P_B and P_R are the secondary rows (the Table 19 and Table 21 analogues). Under row U4, head and wife members use the head-and-wife basis and OFUM members the family basis, so B and T differ -by role there. Under row U6 the cut follows each member's own age-67 -year (§7), so two cohort members of one family unit can get different -R. For example, under U1 a 1936 member observed at 68 and a 1938 member -of the same family unit observed at 66 both have income year 2004 (wave -2005), and with `cut_start_year = 2004` only the 1938 member's +by role there. Under U1 the cut follows each member's own age-67 year +(§7), so two cohort members of one family unit can get different R. For +example, under U1 a 1936 member observed at 68 and a 1938 member of the +same family unit observed at 66 both have income year 2004 (wave 2005), +and with the primary's `cut_start_year = 2004` only the 1938 member's observation is cut. -**Cells.** `all` (headline); `men`, `women`, `married`, `non_married` -(the Report's list of appendix tables includes Appendix Table 17, -"Adjusted Poverty Rates at Age 67, by Gender and Marital Status"; which -of these splits Tables 19 and 21 carry is unknown, plan §10 decision -2(b)); the four sex-by-marital cells (optional); one diagnostic cell per -birth year (not scored). Only cells present in the sealed comparator are -scored; the others are reported as "not in comparator". The cell list -freezes at registration. +**Cells.** Tables 19 and 21 carry the same 36 rows (cleared extract). +Track U v1 computes the fifteen below in the 1936-45 column; the other +21 are named omissions (below). Marital status is the member's state at +the end of the income year (F12; separated counts as married). + +| Cell | Report row (label as printed) | Role | +|---|---|---| +| `all` | Total | headline | +| `women`, `men` | Gender: Female; Male | scored | +| `married`, `widowed`, `divorced`, `never_married` | Marital Status: Married; Widowed; Divorced; Never married | scored | +| `women_married`, `women_widowed`, `women_divorced`, `women_never_married`, `men_married`, `men_widowed`, `men_divorced`, `men_never_married` | Gender and Marital Status: Female: Married … Male: Never married | secondary (compared and reported with n) | +| `birth_year_` | none | diagnostic, not compared | + +A cohabiting member counts under his or her legal status. A member whose +marital state neither the marriage history nor `unresolved_marital_status` +(§3) resolves has no four-way status: under +`unclassified_marital_cells = "excluded_counted"` (pending the freeze) +the member enters `all`, `women` and `men` and no marital cell, and each +marital cell reports how many were left out. A member resolved as +married by relationship code is `married`. Under the unregistered +alternative `separated_is_married = false`, a separated member is also +unclassified. The cell list is frozen at registration. In code: +`age67.marital_status_4` and the tabulation's `REPORT_ROWS`, +`DEFAULT_CELLS` (headline and scored) and `OPTIONAL_CELLS` (secondary). + +**Report rows not computed in v1 (named omissions).** Race/Ethnicity +(four rows), Education (three), Labor Force Experience (four), and +Lifetime Earnings (Own) and (Shared) (five quintiles each): 21 rows. +Track U has no reader for them, and the Report leaves parts of them +undefined: whether "High school graduate" includes some college, the +labor-force-experience measure, and the population over which the +earnings quintiles are cut (cleared extract). Both lifetime-earnings +measures average wage-indexed earnings at ages 22–62, and the own +measure includes uncovered earnings and earnings above the taxable +maximum (p. 26, cleared extract). The PSID does not observe earnings at +every one of those ages for these cohorts: its first income year is +1967 (the 1968 family file, codebook V74, "1967 Income from Labor - +HEAD"), when the 1936–1944 birth years were already past 22, and from +the 1999 wave it is biennial, so odd income years from 1997 on are not +observed. The run artifact lists each omitted row with this reason as +"not computed" (`uniform_cut_tabulation.NOT_COMPUTED_REPORT_ROWS`, the +runner's `report_rows_not_computed`); none is imputed. **Undefined cells.** An empty cell or one with zero total weight has no statistic; it is reported with its reason and never imputed. @@ -565,7 +677,7 @@ headline row (§11): - *Official-concept poverty rate* for the same sample: money income (TOTAL FAMILY INCOME, reported asset income kept, no annuity, no cut) - against the row's threshold, weighted per cell + against the row's threshold, weighted per headline and scored cell (`uniform_cut_track_u.runner.official_concept_rates`). It involves the threshold, so it runs only inside the registered run (and on invented data in the dry run), never on real data before registration. @@ -619,15 +731,17 @@ headline row (§11): ## 10a. Comparison and acceptance -- **Comparator:** for each scored cell, the Report's change from Table 19 - to Table 21 (and, as secondary rows, the two levels), sealed on the - comparator side and opened only after the run artifact is committed. +- **Comparator:** for each headline, scored and secondary cell (§9), the + Report's change from Table 19 to Table 21 in the "1936-45" column (and, + as secondary rows, the two levels), sealed on the comparator side and + opened only after the run artifact is committed. - **Gap:** our Δ minus the Report's Δ, in percentage points, per cell and registered row; likewise for P_B and P_R. -- **Comparator interval:** the Report prints rates; if each is printed to - d decimals, each level carries ±0.5·10^−d and the printed difference - ±10^−d. The interval appears only in the comparison memo, never in the - run artifact. +- **Comparator interval:** the Report prints every cell as a whole number + (cleared extract), so each printed level carries ±0.5 percentage points + of rounding, and the printed difference (Table 21 minus Table 19) lies + within 1 point of the difference of the unrounded levels. The interval + appears only in the comparison memo, never in the run artifact. - **Acceptance:** none by default (plan §10 decision 6, pending Max): the memo reports our value, floor, design SE, the comparator with its interval, and the gap, and declares no pass or fail. A rule, if any, @@ -636,24 +750,30 @@ headline row (§11): ## 11. Registered rows -Each alternative differs from U0 in one field, except U6 (defined on U1) -and U0-F. U0 is the headline row unless the fallback rule below makes -U0-F the headline; no row may be promoted after results exist. Every -registered row is computed in the same one-shot run and published with -the others. A row that is not built when the registration is posted is +Each alternative differs from U0 (or, for the -F rows, from U0-F) in one +field; U1 and U0-F change the population itself. U0 is the headline row +unless the fallback rule below makes U0-F the headline; no row may be +promoted after results exist. Every registered row is computed in the +same one-shot run and published with the others. A row that is not built when the registration is posted is removed from this table and listed as a named omission. **Fallback rule (pending Max, with plan §10 decision 3).** If the 2005 and 2007 wealth supplements are staged, adjudicated and read before the #42 registration is posted, U0 is the headline and U0-F an alternative; -otherwise U0-F is the headline, and U0 and U1 are reported as blocked -with their counts. The rule depends on staging status only. In code +otherwise U0-F is the headline, and the rows defined on U0 or U1 are +reported as blocked with their counts. The rule depends on staging status only. In code (`uniform_cut_track_u.runner.headline_row`), every row whose observation waves include a wave without WEALTH1 is reported as blocked with its -counts, which under the fallback is every row except U0-F and U7: the -one-field alternatives are defined on U0 and need 1937 and 1939 too. -Whether they should instead follow the headline to U0-F's population is -not decided (§16, §17). The registration comment states the headline +counts, which under the fallback is every row defined on U0 or U1 (U0, +U1, U2–U5 and U8–U10): they need 1937 and 1939 too. So that the +registration carries its alternatives in either staging state, each +one-field alternative defined on U0 is also registered on U0-F's +population: U2-F, U3-F, U4-F, U5-F, U8-F, U9-F and U10-F (the field and +value of U2 … U10, population `birth_years_1941_1943_1945`). Under the +fallback the U0 versions are blocked and the -F versions computed; with +the supplements staged both are computed. This is the recommended +default of the pending fallback decision (§16); Max may instead keep the +alternatives on U0 only. The registration comment states the headline row, and the registered run refuses when the staged PSID gives a different one (`scripts/run_track_u_registered.py --headline-row`). @@ -665,13 +785,25 @@ different one (`scripts/run_track_u_registered.py --headline-row`). | U3 | SSI | Full static recomputation (the largest SSI response of the three) | Yes | | U4 | Income unit | Head and wife only | Yes | | U5 | Asset income | Keep reported asset income and add the annuity | Yes | -| U6 | Cut (on U1) | 1936 uncut if the cut starts in 2004 (`cut_start_year = 2004`) | Yes; the start year awaits decision 8 | | U0-F | Population | Birth years 1941, 1943, 1945 only (the blocked 1937 and 1939 left out and counted) | Yes; the fallback rule awaits Max | | U7 | Financial assets | Plus employer DC balances | **No** (label investigation); must be built before registration or removed | | U8 | Threshold | PSID `CENSUS NEEDS STANDARD` | Yes | | U9 | Mortality | SSA period life table 2004 | Yes | | U10 | Threshold | Census size-by-children matrix | Yes (needs the capture) | -| U-inst | Universe | Add institutions (sequence 51–59), family-of-record income rule (§3) | Yes; the rule awaits the freeze and a referee | +| U2-F … U10-F | as U2 … U10 | on U0-F's population | Yes; with U0-F, they await the fallback rule | + +Institutionalized persons (sequence 51–59) are outside the universe and +counted in the dispositions. Row U-inst of `u1-draft-3` and `-4` is +withdrawn: on the staged PSID none of U0's or U0-F's target persons was +in an institution at the observation wave (u1d4 structural counts, +dispositions; U1 has 3), so U-inst would add no institutionalized +observation, and the Census cannot determine poverty status for people +in institutional group quarters. The `family_of_record` rule stays in the +code as an unregistered option. + +Row U6 of `u1-draft-3` and `-4` (the cut's start year on U1) is +withdrawn: the primary now starts the cut in 2004 (§7), so U6 would +equal U1. ## 12. Named deltas @@ -683,10 +815,11 @@ found while building). Each bullet is the text of - PSID versus SIPP wealth measurement; - realized 2004-2012 history versus DYNASIM's 1992-based projection, including the 2008-09 asset shock for the 1941-45 cohorts at 67; - realized COLAs versus 2002 Trustees assumptions; -- immigrant under-coverage; institutionalized persons (row U-inst: the family-of-record rule assigns the family's income and threshold without the member's own income, which the PSID does not collect); attrition; +- immigrant under-coverage; institutionalized persons are outside the universe (none of U0's target persons was in an institution at the observation wave on the staged files); attrition; - OFUM-owned assets inside family wealth; - the family's wealth stands in for an OFUM cohort member's own wealth (the Report's unit is the individual plus spouse, p. 24); -- imputed rent excluded; +- the Report's total income lists income from financial assets, imputed rent, Social Security, DB pensions, retirement-account income, earnings, SSI and non-spouse co-resident income (p. 31, cleared extract), and its poverty income is described only as differing from Census money income by the annuity (p. 24); the primary keeps every other PSID money-income source (veterans' pensions, unemployment and workers' compensation, TANF and other welfare, child support, alimony, help from relatives and others, miscellaneous transfers); +- members whose marital state is unresolved (34 of U0's 483 observations, 24 of U0-F's 320, structural counts) are left out of the marital cells (§9); - self-reported Social Security, possibly net of Medicare Part B premiums, so the 13 percent cut applies to a smaller base than the gross benefit (baseline income is lower too; the net direction on the change is not established); - retirement-account income beside annuitized balances: the head's income from annuities and IRAs is removed (F4a), but the wife's (before 2013) and the OFUMs' items combine pensions with annuity income and stay, so any IRA or annuity income in them is counted twice; an annuity already in payment may have no balance in WEALTH1, so removing its income understates income; - employer DC (401(k)) balances outside IRAs are not in WEALTH1 though the Report counts them (p. 24): the primary annuity is understated for their holders (row U7); @@ -700,7 +833,7 @@ found while building). Each bullet is the text of - SSI deeming by full attribution can only overstate the offset (so understate the change): under 20 CFR 416.1163 nothing is deemed when the ineligible spouse's income is at most the couple-minus-individual FBR; - the SSI test uses annual amounts against 12 times the January FBR (SSI accounting is monthly), and U3 omits rent and royalties from countable income although 20 CFR 416.1121 counts them; - reported SSI may include state supplementary payments, so capping the offset at the federal benefit rate less reported SSI can understate the offset; -- U0-F, if it is the headline: mean birth year 1943 against 1940.5, and every observation year at or after the 2008-09 asset shock. +- rows on U0-F's population (U0-F and U2-F ... U10-F): mean birth year 1943 against 1940.5, and every observation year at or after the 2008-09 asset shock. ## 13. Invented worked cases @@ -727,16 +860,17 @@ Invented life table (ages 0–4): male qx 0, 0, 0.2, 0.5, 1; female qx 0, | SSI offset, capped (SSI 550) | min(130, 600 − 550) | +50 | | SSI, SS under the exclusion (SS 200) | 0 − 0 | 0 | | U3 new take-up (SS 900, no other income) | before 660 ≥ 600; after 543 < 600 | +57 | -| U6, birth year 1936 (T = 1,000, SS 1,050) | primary: R = 1,050 − 136.5 = 913.5 (poor); `cut_start_year = 2004`: 1936 + 67 = 2003 < 2004, no cut, R = 1,050 | uncut | -| U6, SSI recipient born 1936 (SS 1,000, SSI 100) | primary offset 130; uncut: no fall | 0 | -| U-inst, member in an institution attached to family 11 | family 11's income 1,000 and WEALTH1 500; role OFUM; no co-resident spouse | assigned | +| Cut start, birth year 1936 (T = 1,000, SS 1,050) | `cut_start_year = None`: R = 1,050 − 136.5 = 913.5 (poor); primary (2004): 1936 + 67 = 2003 < 2004, no cut, R = 1,050 | uncut | +| Cut start, SSI recipient born 1936 (SS 1,000, SSI 100) | `cut_start_year = None`: offset 130; primary (2004): uncut, no fall | 0 | +| Institution option (`family_of_record`, not registered), member in an institution attached to family 11 | family 11's income 1,000 and WEALTH1 500; role OFUM; no co-resident spouse | assigned | +| Four-way status (§9): unresolved, not resolved by code; separated under `separated_is_married = false` | no marital row | unclassified | | F4a, head annuity and IRA income 200 (money income 1,200) | `remove_head`: B = 1,200 − 200 = 1,000; `keep`: B = 1,200 | 1,000 / 1,200 | | F4a, 2013 file: `HEAD ANNUITIES` 150 and `HEAD IRAS` 50 | both removed | 200 removed | | F4b, farm income 400 | share 0.5: remove 200; share 0: remove 0 | 200 / 0 | | F4b, farm loss −300 | share 0.5: remove −300 (income rises by 300); share 0: 0 | −300 / 0 | | Lives, OFUM woman aged 3 in a family headed by a man aged 2 and his legal wife aged 2 (WEALTH1 1,024) | `fu_head_rule`: joint 1.024, annuity 800; `member_rule`: single female 3, 0.75·0.8 = 0.6, annuity 1,365.3 | 800 / 1,365.3 | | Lives, female head aged 2 with a legal husband (code 90) aged 2 | joint on the head and her legal husband | 1.024 | -| Unresolved marital state, head with a legal wife in the family | `relationship_code`: married, co-resident with the wife; `non_married`: non-married, no spouse | married / non-married | +| Unresolved marital state, head with a legal wife in the family | `relationship_code`: married, co-resident with the wife; `non_married`: not married (four-way status unclassified), no spouse | married / unclassified | | Spouse age, birth year 1943 by marriage history, individual-file age 66 at the 2009 interview | income year 2008: `derived_birth_year` 65; `wave_age` 66 | 65 / 66 | Invented tabulation (five observations, weights 1, 1, 2, 4, 2): P_B = 20, @@ -746,7 +880,12 @@ the rows occupy (two strata of two clusters each; stratum contributions stratum 2 and a third stratum of two clusters, none holding an observation: stratum 2's cluster totals become 0.2, −0.1 and 0 (mean 1/30, contribution 1.5 × 0.04667 = 0.07), stratum 3 contributes 0, and -the SE is 100·√0.08. +the SE is 100·√0.08. With four-way statuses married, married, widowed, +divorced and unclassified (the last a woman): `married` has P_B = 0 and +P_R = 50, `widowed` 100 and 100, `divorced` 0 and 100; `never_married` +is empty and undefined; the unclassified woman is in `all` and `women` +and reported as left out of each marital cell (1, and 0 for the men's +cells). F17 weighted quantile (invented values 10, 20, 30, 40 with weights 1, 1, 1, 1): the median is the smallest value whose cumulative weight share @@ -756,9 +895,10 @@ reaches 0.5, 20; p90 is 40. On master since #456 (merge `8d7e7431`; built on branch `dynamics-ex2-track-u-20260924`), continued on -`dynamics-ex2-track-u-2-20260924` (`u1-draft-3` and this draft; not -merged); all opt-in; registered in `POST_REVIEW_SOURCE_EXCLUSIONS` and -the reachability guard: +`dynamics-ex2-track-u-2-20260924` (`u1-draft-3` to this draft; not +merged; `origin/master` at `088772f1` merged in for this draft); all +opt-in; registered in `POST_REVIEW_SOURCE_EXCLUSIONS` and the +reachability guard: - `src/populace_dynamics/data/family_income.py`: label-verified family income for waves 2005–2013 (81 items a wave, 85 in 2013, including the @@ -767,20 +907,23 @@ the reachability guard: 2005 and 2007 wealth supplements; reconciliation counts. - `src/populace_dynamics/cohorts/age67.py`: the age-67 observations for U0, U1 and U0-F, dispositions, the relationship-code resolution of - unresolved marital states, the head and head's legal spouse with their + unresolved marital states, the four-way marital status of §9 + (`marital_status_4`), the head and head's legal spouse with their income-year ages (the birth-year law extended to in-family heads and spouses), provenance seal, structural summary. - `src/populace_dynamics/estimates/adjusted_poverty.py`: annuity prices, threshold lookup, SSI rules, baseline and reform income (with F4a and F4b), poverty status; refuses PSID-built rows without a registration. -- `src/populace_dynamics/estimates/uniform_cut_tabulation.py`: cells, +- `src/populace_dynamics/estimates/uniform_cut_tabulation.py`: the + Report's rows as cells (`REPORT_ROWS`, `NOT_COMPUTED_REPORT_ROWS`), Δ/P_B/P_R, half-split floors, design SE on the full sample design; refuses real data without a registration pointer. - `scripts/capture_track_u_parameters.py` (SSI capture written; Census parser ready), `scripts/track_u_structure.py` (counts only). - `src/populace_dynamics/uniform_cut_track_u/` (`u1-draft-3`, extended - here): `rows.py` (the registered rows, held equal to §15's rows, and - the headline rule); `invented.py` (U5: a seeded INVENTED age-67 + here): `rows.py` (the registered rows, held equal to §15's rows, + including the -F alternatives on U0-F's population, and the headline + rule); `invented.py` (U5: a seeded INVENTED age-67 population in the reader shapes, the INVENTED threshold table and the re-generation checks; this draft adds couples resolved by relationship code, legal husbands, farm, annuity and IRA income, and interview ages @@ -793,7 +936,7 @@ the reachability guard: capture, and the fallback rule on the invented cohort as staged today; evidence headed "INVENTED DATA - NOT A COMPARISON", `EVID/track-u-dry-run-20260924/` (`u1-draft-3`) and - `EVID/track-u-dry-run-u1d4-20260924/` (this draft). + `EVID/track-u-dry-run-u1d4-20260924/` (`u1-draft-4`). - `scripts/run_track_u_registered.py` (U10 entry point): refuses unless the pointer is an issue #42 comment, `HEAD` is the registered commit on a clean tree, §15 is ratified with nothing awaiting and nothing @@ -807,21 +950,20 @@ the reachability guard: family's `# IN FU` equals its number of in-family records (sequence 1–20) in all five waves, and none of the 389–426 families per wave with an attached institution record counts it in `# IN FU`: the - reading the institution rule relies on. + reading the unregistered `family_of_record` institution rule relies + on. Blocked: 1. **2005 and 2007 wealth supplements** (Max, simba login): U0 loses 1937 and 1939 until staged, adjudicated and read; under the fallback rule - U0-F is then the headline and the rows defined on U0 or U1 are - reported as blocked. + U0-F is then the headline, the rows defined on U0 or U1 are + reported as blocked, and the -F alternatives are computed. 2. **Census thresholds** (download approval, cos decision d194): no primary threshold until captured; the dry run uses an INVENTED table. 3. **Registration** on issue #42 after ratification; no real-data poverty statistic may be computed before it. -4. **Comparator coverage** of the frozen cells (comparator side, values - redacted). -5. **Row U7** is not built (label investigation of the P-section +4. **Row U7** is not built (label investigation of the P-section employer DC items); it must be built or removed before registration. ## 15. Machine-readable parameter block @@ -832,7 +974,7 @@ holds it to the code's defaults. ```json { "specification": "boomers2004_uniform_cut_exercise2", - "version": "u1-draft-4", + "version": "u1-draft-5", "status": "draft_for_referee", "claim_class": { "proposed": "track_u_psid_realized_measurement_not_a_projection", @@ -846,7 +988,15 @@ holds it to the code's defaults. "cut_rate": 0.13, "unit": "individual_with_household_income_and_wealth_including_spouse", "financial_assets": "non_pension_wealth_plus_ira_keogh_401k_balances", - "comparator_values": "sealed_comparator_side_not_opened_by_builder" + "comparator_values": "sealed_comparator_side_not_opened_by_builder", + "column": "1936-45", + "report_rows": 36, + "printed_precision": "whole_numbers", + "definitions_extract": { + "file": "EVID/exercise2-definitions-cleared-20260924.md", + "sha256": "a3978b683b4275424b6d12e9fe45f021fae277ccf9731a7883952564b6ed0384", + "cleared_for_builders": "2026-09-24" + } }, "population": { "primary_row": "U0", @@ -873,7 +1023,7 @@ holds it to the code's defaults. "separated_is_married": true, "unresolved_marital_status": "relationship_code", "annuitant_age_source": "derived_birth_year", - "institution_income_rule": "family_of_record", + "institution_income_rule": "excluded", "design": {"stratum": "ER31996", "cluster": "ER31997"} }, "income_concept": { @@ -914,7 +1064,7 @@ holds it to the code's defaults. "rate": 0.13, "base": "all_social_security_of_the_unit", "behavior": "none", - "start_year": null, + "start_year": 2004, "start_year_rule": "cut_when_birth_year_plus_67_at_or_after_start" }, "ssi": { @@ -934,11 +1084,13 @@ holds it to the code's defaults. "unit": "percentage_points" }, "cells": { - "scored_candidates": ["all", "men", "women", "married", "non_married"], - "optional": ["men_married", "men_non_married", "women_married", "women_non_married"], + "column": "1936-45", + "scored": ["all", "women", "men", "married", "widowed", "divorced", "never_married"], + "secondary": ["women_married", "women_widowed", "women_divorced", "women_never_married", "men_married", "men_widowed", "men_divorced", "men_never_married"], "diagnostic": "birth_year", - "source": "appendix_table_17_title_by_gender_and_marital_status", - "scored_only_if_in_comparator": true + "report_labels": "tables_19_21_stub_labels_cleared_extract", + "unclassified_marital_cells": "excluded_counted", + "not_computed": ["race_ethnicity", "education", "labor_force_experience", "lifetime_earnings_own", "lifetime_earnings_shared"] }, "uncertainty": { "draws": 1, @@ -958,7 +1110,7 @@ holds it to the code's defaults. }, "comparison": { "gap": "model_minus_report", - "comparator_interval": "display_rounding", + "comparator_interval": "whole_number_rounding_level_0_5_difference_1", "acceptance": {"rule": null, "awaiting": "Max (plan section 10 decision 6)"} }, "rows": { @@ -968,13 +1120,18 @@ holds it to the code's defaults. "U3": {"ssi_rule": "full_static_recomputation"}, "U4": {"income_unit": "head_wife"}, "U5": {"asset_income_rule": "keep"}, - "U6": {"on": "U1", "cut_start_year": 2004, "awaiting": "plan section 10 decision 8"}, "U0-F": {"population": "birth_years_1941_1943_1945", "on": "U0", "awaiting": "Max (fallback rule, with plan section 10 decision 3)"}, "U7": {"financial_assets": "wealth1_plus_employer_dc", "status": "not_built"}, "U8": {"threshold_rule": "psid_census_needs_standard"}, "U9": {"mortality_basis": "ssa_period_2004"}, "U10": {"threshold_rule": "census_matrix_65plus"}, - "U-inst": {"presence": "in_family_or_institution", "institution_income_rule": "family_of_record", "awaiting": "specification freeze (rule not yet refereed)"} + "U2-F": {"population": "birth_years_1941_1943_1945", "ssi_rule": "none", "awaiting": "Max (fallback rule, with plan section 10 decision 3)"}, + "U3-F": {"population": "birth_years_1941_1943_1945", "ssi_rule": "full_static_recomputation", "awaiting": "Max (fallback rule, with plan section 10 decision 3)"}, + "U4-F": {"population": "birth_years_1941_1943_1945", "income_unit": "head_wife", "awaiting": "Max (fallback rule, with plan section 10 decision 3)"}, + "U5-F": {"population": "birth_years_1941_1943_1945", "asset_income_rule": "keep", "awaiting": "Max (fallback rule, with plan section 10 decision 3)"}, + "U8-F": {"population": "birth_years_1941_1943_1945", "threshold_rule": "psid_census_needs_standard", "awaiting": "Max (fallback rule, with plan section 10 decision 3)"}, + "U9-F": {"population": "birth_years_1941_1943_1945", "mortality_basis": "ssa_period_2004", "awaiting": "Max (fallback rule, with plan section 10 decision 3)"}, + "U10-F": {"population": "birth_years_1941_1943_1945", "threshold_rule": "census_matrix_65plus", "awaiting": "Max (fallback rule, with plan section 10 decision 3)"} }, "diagnostics_f17": { "row": "headline", @@ -1005,6 +1162,16 @@ holds it to the code's defaults. "verdict": "revise_before_ratification", "required_changes": 17, "applied_in": "u1-draft-4" + }, + { + "report": "EVID/boomers2004-referee-2-20260924.md", + "sha256": "2113ba9bd25764252e19fa5de28d5c12f7745154a31be0220953d831518d5af2", + "object_version": "u1-draft-4", + "object_commit": "37a94ea2", + "object_blob_sha256": "1b461e0ad29a65b03706d445ab220264e4dba3ef99b3cf970cb40179b9e9c8a7", + "verdict": "ratifiable_after_required_changes", + "required_changes": 9, + "applied_in": "u1-draft-5" } ], "blocked_by": [ @@ -1023,7 +1190,8 @@ None of these is ratified. Each is a code parameter whose default is the plan's proposal, the referee's recommendation or a builder choice where both are silent; `adjusted_poverty.pending_decisions()`, `age67.pending_decisions()` and `uniform_cut_tabulation.pending_decisions()` -(the design SE domain and the cells) return them with their basis. +(the design SE domain, the cells and the unclassified-marital rule) +return them with their basis. **Awaiting Max (d189, open):** @@ -1038,21 +1206,26 @@ d189's text describes Track U as a "Python income concept, not Axiom", so a yes on d189 may be read as covering plan decision 4; confirm when ruling. -**Awaiting Max, not yet on a decision card (plan §10):** decision 2 (a -values-redacted definitions extract by the comparator side), decision 4 +**Awaiting Max, not yet on a decision card (plan §10):** decision 2 +(the definitions extract: it exists and was cleared for builders on +2026-09-24; no card or ruling by Max on it was found), decision 4 (Python SSI arithmetic labelled "not Axiom"), decision 6 (acceptance rule; default none, report gaps, §10a), decision 7 (ratify this specification by merge; post the #42 registration), decision 8 (the scorecard's "from -2004": row U6 is built with `cut_start_year = 2004`, and the primary's -`cut_start_year = None` cuts every observation), approval to download the -Census threshold files (cos decision d194), and the fallback rule of §11 -(with decision 3), including whether the one-field alternatives follow -the headline to U0-F's population when the supplements are not staged -(default: they stay on U0 and are reported as blocked). +2004"): the cleared extract supplies the source, the Report's "beginning +in 2004" (pp. 37 and 44); the primary now uses `cut_start_year = 2004`, +which moves only U1's 1936 birth year; Max's confirmation of the +scorecard wording remains open, and the fallback rule of §11 (with +decision 3), including the recommended default that each one-field +alternative is also registered on U0-F's population (U2-F … U10-F), so +the registration carries its alternatives whether or not the supplements +are staged; Max may instead keep the alternatives on U0 only (blocked +under the fallback). Approval to download the Census threshold files is +on its own card (cos decision d194, open). **Awaiting the specification freeze (defaults shown):** cut rate (0.13; the code lists it), cut base (all Social Security of the unit; not a code -parameter, decision 2(a)), cut start year (none), income unit (family +parameter, decision 2(a)), cut start year (2004), income unit (family unit), asset-income rule (replace), retirement-account income (remove the head's), farm asset share (0.5 of positive income, whole loss), annuitized share (0.8), real rate (3 percent; 2 percent sensitivity), @@ -1066,10 +1239,12 @@ SSI unit (one individual unit), row (U0, with the U0-F fallback rule), presence (in family), separated is married (true), unresolved marital status (relationship code), U1 1936 weight (1: each birth year then carries one cross-section's weight), seed wave (earliest presence wave), -design SE (full-design domain), cells (after decision 2(b)), and the -institution income rule (`institution_income_rule = family_of_record`; -alternative `excluded`; only row U-inst reads it, and no referee has -reviewed it yet). +design SE (full-design domain), cells (§9, including +`unclassified_marital_cells`, default `excluded_counted`), a small-cell +flag for the comparison memo (none by default; the second referee +suggests flagging cells under an unweighted *n* of 30, O1), and the +institution income rule (used by no registered row; default `excluded`, +the second referee's Q9 answer; alternative `family_of_record`). A registered run refuses a §15 block that still names anything as `awaiting`, so ratification must resolve each of these (and the code's @@ -1138,37 +1313,93 @@ direction stated (§8, §12); Q4 weight 1 (§3); Q5 the full design (R10); Q6 register U0-F (R11); Q7 the relationship code (R9); Q8 derived birth years (R8). -### 17.2 Questions for the next pass - -9. Is `family_of_record` the right income rule for institutionalized - members under U-inst (their family's income, annuity and threshold, - without their own income, which the PSID does not collect), or should - they be `excluded` from the universe, or get a rule not built here - (for example a one-person unit)? -10. Row U6 keys the cut on the year the member turns 67, not on the - income year observed (U1 observes 1936 at 68, in 2004). Is that the - reading the plan's "the 1936 cohort unaffected" intends? -11. Is extending the head's legal spouse to code 90 (legal husband), in - the annuity and in the relationship-code resolution, right, given - that the family file's "wife" items' treatment of a code-90 husband - was not checked? The builder gives him role OFUM, so row U4 keeps him - on the family-unit basis; 2 of 483 U0 and 4 of 1,349 U1 observations - are code-90 husbands (§3). The 2011 family codebook groups "husband of - Head" with Wife/"Wife" in its composition and birth variables, but - describes the income items as the Wife's/"Wife's" only (independent - review; not settled). -12. Under the fallback rule, should the one-field alternatives follow the - headline to U0-F's population (so that U2–U5 and U8–U10 are computed - when the supplements are not staged), or stay on U0 and be reported - as blocked (the default)? -13. The birth-year law extension leaves a person with conflicting - marriage-history birth years unresolved and falls back to the wave - age. Is that acceptable, or should the build fail closed as the law - does for its required population? -14. F4a removes the head's annuity income, including annuities already - in payment that may carry no WEALTH1 balance. Should the rule remove - only `HEAD IRAS` in 2013 and keep `HEAD ANNUITIES` (not possible - before 2013, where the item combines both)? +### 17.2 Questions 9–14 and the second referee's answers + +The second referee (§17.3) answered the questions `u1-draft-4` put; the +answers replace the questions. + +9. **Institution income rule.** Answer: use `excluded`, and do not + register U-inst: no U0 or U0-F target person was in an institution at + the observation wave on the staged PSID, so the row adds no + institutionalized observation, and `family_of_record` would measure a + family's poverty and assign it to someone the family's size and income + leave out. Adopted: U-inst is withdrawn (S7) and `excluded` is the + code default of the unregistered option. +10. **The cut keyed on the age-67 year.** Answer: yes; the Report + analyses each person "when they reach age 67" (p. 24) and cuts + benefits "beginning in 2004" (pp. 37, 44, cleared extract). Adopted + in the primary (S5); U6 is withdrawn. +11. **Code 90 as the head's legal spouse.** Answer: right for the annuity + and the marital resolution (fn. 10 gives married couples a 50 percent + survivor annuity); U4 and the SSI unit stay open, with small exposure + (2 of 483 U0 observations are code-90 husbands). Adopted; §4 states + U4's size rule for a female head with a legal husband, and §3 records + the structural counts. +12. **Alternatives under the fallback.** Answer: they should follow the + headline to U0-F. Adopted as the recommended default of the pending + fallback decision (S8): U2-F … U10-F are registered. +13. **Conflicting marriage-history birth years falling back to the wave + age.** Answer: acceptable; on the staged files no wave-age fallback + occurs. Adopted; the registered run now records every annuitant age + source, including the member's co-resident spouse (O6). +14. **Removing only `HEAD IRAS` in 2013.** Answer: no; the pooled U0 and + U0-F would then give 1945 a different income definition from 1941 and + 1943. Kept: `remove_head` in every wave, with its named delta; §3 + records the 2013 counts (O8). + +Question 11's family-file part (how the file records a code-90 +husband's income) is not settled by reading; §3's structural counts +show what the staged files carry in the "wife" fields. + +### 17.3 Second pass (applied in `u1-draft-5`) + +- **Report:** `EVID/boomers2004-referee-2-20260924.md`, SHA-256 + `2113ba9bd25764252e19fa5de28d5c12f7745154a31be0220953d831518d5af2` + (rehashed for this draft; second referee, Claude Code subagent, Opus + 5.5, 2026-09-24). +- **Object:** `u1-draft-4` at `37a94ea2`, blob SHA-256 + `1b461e0ad29a65b03706d445ab220264e4dba3ef99b3cf970cb40179b9e9c8a7` + (rehashed for this draft with `git show`). +- **Verdict:** the draft is ratifiable after nine required changes + (S1–S9), not before; ten optional suggestions (O1–O10); answers to + questions 9–14 (§17.2). +- **Cleared extract:** `EVID/exercise2-definitions-cleared-20260924.md`, + SHA-256 + `a3978b683b4275424b6d12e9fe45f021fae277ccf9731a7883952564b6ed0384` + (rehashed for this draft). + +Each required change was checked against its cited source before it was +applied: the cleared extract; the Report's pages this builder read for +`u1-draft-4` (none opened for this draft); the structural-count evidence +`EVID/track-u-structure-u1d4-20260924/` (the marital and disposition +counts re-summed); the PSID 1968 family codebook; the transfer-item +labels in `data/family_income.py`; and the Census "Poverty measures" +page. + +| Change | Disposition | +|---|---| +| S1 (§1 from the extract) | Applied as written. Verified in the extract: Tables 19 and 21 have 36 data rows under eight section headers, the column group "Birth Cohort" with "1936-45", whole-number cells; the p. 37 scenario sentence and fn. 19; total income (p. 31), replacement-rate income (p. 36, Table 13 note a) and fn. 17. O9 added: the reporting splits and 2003 dollars are cited to printed p. 25 lines 1–6 through the extract | +| S2 (cells, four-way marital status) | Applied. Verified: the extract's row labels; the referee's counts re-summed from the u1d4 evidence (U0: married 330 + 1 by relationship code, widowed 44, divorced 60, never married 14, unclassified 34; U0-F: 226 + 1, 17, 43, 9, 24). Code: `age67.marital_status_4`, the tabulation's `REPORT_ROWS`, `DEFAULT_CELLS` and `OPTIONAL_CELLS`, `unclassified_marital_cells`, the runner's official-concept cells; the invented cohort covers all five statuses (test). The JSON is S2's | +| S3 (named omissions) | Applied, with one correction: the extract attributes the inclusion of uncovered earnings and earnings above the taxable maximum to the own measure only (p. 26), where S3 said both measures. The PSID's first income year, 1967, which S3 flagged as general knowledge, was confirmed in the 1968 family codebook (V74); the biennial gap from the 1999 wave is added. Code: `NOT_COMPUTED_REPORT_ROWS`, listed in the run artifact | +| S4 (comparator interval) | Applied as written. The difference bound follows from \|(a′ − b′) − (a − b)\| ≤ \|a′ − a\| + \|b′ − b\| ≤ 1 | +| S5 (start year 2004) | Applied. Verified in the extract (PDF pp. 38 and 45, printed 37 and 44; the exposure arithmetic puts 1936 at 67 in 2003) and p. 24. `cut_start_year` defaults to 2004 with alternative `None`; U6 is withdrawn; §7, §9, §11, §13, §16 and the tests follow | +| S6 (named deltas) | Applied as written. Verified: the p. 31 list and p. 24's contrast in the extract; the transfer items are `data/family_income.py` labels (for 2004: ER27954 TANF, ER27958 other welfare, ER27960 VA pension, ER27968 unemployment compensation, ER27970 workers compensation, ER27972 child support, ER27974 alimony, ER27976 and ER27978 help from relatives and others, ER27980 miscellaneous transfers). `runner.NAMED_DELTAS` changed with it | +| S7 (withdraw U-inst) | Applied. Verified: the u1d4 dispositions list no `not_present:institution` for any U0 or U0-F birth year and 3 for U1 (1942: 2; 1944: 1); the Census sentence on institutional group quarters (census.gov, fetched 2026-09-24). Beyond S7, the code default of `institution_income_rule` is now `excluded` (the Q9 answer), and §3 records a check of the referee's unverified caveat (the seed wave) | +| S8 (alternatives on U0-F) | Applied. `rows.FALLBACK_ALTERNATIVES` builds U2-F … U10-F from the U0 rows; each awaits the fallback rule. Under the fallback the runner computes them and blocks the U0 versions (test). Wording: §11 says U1 and U0-F "change the population itself" where S8 had "except U1 and U0-F" | +| S9 (record the extract and this pass) | Applied. The builder-boundary sentence on the values scan also records what `RESTRICTED-FILES.md` said when re-read during this pass | + +| Suggestion | Disposition | +|---|---| +| O1 (small cells) | Not adopted: a small-cell flag is a reporting rule for the memo, which the freeze should set with decision 6; listed in §16 (none by default). The artifact carries `n_observations` per cell | +| O2 (tax treatment) | Applied (§4) | +| O3 (co-resident basis) | Applied (§4) | +| O4 (SSI context) | Applied (§8) | +| O5 (Census file count) | Applied (§6): d194 names ten files; the capture reads nine by name | +| O6 (wave-age fallbacks) | Applied: the runner's population summary records `spouse_age_source` beside the head's and the head's spouse's sources | +| O7 (code-90 heads, U4 size) | Applied: §4 states the size rule; §3 records the counts | +| O8 (2013 annuity and IRA counts) | Applied: §3 records the counts | +| O9 (printed p. 25 lines 1–6) | Applied (§1, builder boundary) | +| O10 (for the orchestrator) | The master merge is done (`088772f1`, §14); the values-scan record is noted in the builder boundary | ## 18. What this draft read and did not verify @@ -1205,11 +1436,22 @@ eCFR current text of 20 CFR 416.1112, 416.1121, 416.1124, 416.1163 and 416.1205; and the nine Federal Register notices of §8, fetched from govinfo.gov into the session scratch directory to be read (not committed). +`u1-draft-5` also read: the second referee report in full; the cleared +exercise-2 definitions extract in full (both rehashed); +`EVID/RESTRICTED-FILES.md` (twice; it changed during this pass); the +u1d4 structural-count evidence (README and JSON, counts only); the +Track U code and tests it changes; the 1968 family codebook entry V74; +the transfer-item labels in `data/family_income.py`; the cos records of +d189 and d194 (their text only); and the Census "Poverty measures" +guidance page (fetched 2026-09-24). It opened no page of the Report PDF. + **Ran on staged PSID:** label verification, the component-identity reconciliation counts and the structural counts of §3, the F17 component summaries of §9 for row U0 (`u1-draft-3`), and (`u1-draft-4`) a design count (strata and clusters among positive-weight persons per wave) and a -count of in-family code-90 records per wave. No income concept, +count of in-family code-90 records per wave; (`u1-draft-5`) the +structural counts of §3 recollected under this draft's code, with the +institution-option and code-90 counts. No income concept, annuity, threshold assignment, poverty status or poverty rate was computed on PSID data. @@ -1224,8 +1466,12 @@ whether PSID Social Security amounts are net of Medicare premiums; whether SSI and Social Security are kept apart by respondents; where a wife's IRA income is recorded before 2013; how the family file records a code-90 legal husband's income; the P-section DC items for U7; the -coverage of the sealed comparator; anything in the Report on printed -page 25 or later; whether an institutionalized member's income enters +sealed comparator's values (its labels are in the cleared extract); +anything in the Report beyond printed p. 25 lines 1–6, except as quoted +in the cleared extract, whose quotes from restricted pages the +clearance review did not recheck; the Report's definition of "Married" +and when it measures marital status (the extract says neither is +defined); whether an institutionalized member's income enters any family-file item (no documentation found that it does); DYNASIM's actual annuity rate, mortality and survivor conventions beyond pp. 19–24; the domain-estimation citation the referee gives (West, Berglund and @@ -1233,6 +1479,26 @@ Heeringa 2008); any SCF wealth aggregate (none is committed or saved). ## 19. Changelog +- `u1-draft-5` (2026-09-24, second referee pass applied, §17.3): + `origin/master` (`088772f1`) merged in; §1 states the target as the + cleared exercise-2 definitions extract defines it (the "1936-45" + column, 36 rows, whole-number cells, the 2004 start and no behavioural + response); §9 maps fifteen cells to the Report's Total, Gender, Marital + Status and Gender and Marital Status rows with a four-way marital + status (`marital_status_4`; unresolved members are unclassified, left + out of the marital cells and counted) and names the other 21 rows as + omissions; §10a fixes the comparator interval at ±0.5 per level and 1 + for the difference; §7 makes `cut_start_year = 2004` the primary and + withdraws U6; §11 withdraws U-inst and registers U2-F … U10-F on + U0-F's population as the fallback decision's recommended default; §12 + drops "imputed rent excluded" and adds the income-source and + unresolved-marital deltas; `institution_income_rule` defaults to + `excluded`; the runner records the spouse's age source and the Report + rows not computed; §4, §6 and §8 take the optional O2–O5; §15 gains the + target's column, rows, precision and extract, the new cells, the -F + rows and the second `referee_passes` entry; §17 records the pass and + the answers to questions 9–14. No income, threshold or poverty status + was computed on PSID data. - `u1-draft-4`, independent review corrections (2026-09-24; no rule, default or row changed): §3 states that a code-90 legal husband's role is OFUM (U4 keeps him on the family-unit basis) and counts those diff --git a/scripts/run_track_u_registered.py b/scripts/run_track_u_registered.py index 94c47fc4..b38955a1 100644 --- a/scripts/run_track_u_registered.py +++ b/scripts/run_track_u_registered.py @@ -33,13 +33,14 @@ wealth supplements are staged, adjudicated and read, U0-F otherwise); a mismatch means the staging changed after registration and the run refuses (a new registration is needed). Rows that need a wave without - WEALTH1 are reported as blocked with their counts. + WEALTH1 are reported as blocked with their counts; the -F alternatives + on U0-F's population are computed in either staging state. The artifact publishes regardless of outcome. It never reads the sealed comparator; the seal is opened only after this artifact is committed. As of this script's writing none of these preconditions holds: the -specification is a draft (``u1-draft-4``), Max has not ruled on exercise +specification is a draft (``u1-draft-5``), Max has not ruled on exercise 2 (cos decision d189) or on the fallback rule, the Census thresholds are not captured (d194) and the 2005/2007 wealth supplements are not staged. diff --git a/scripts/track_u_component_diagnostics.py b/scripts/track_u_component_diagnostics.py index da414aa4..4b7e73d3 100644 --- a/scripts/track_u_component_diagnostics.py +++ b/scripts/track_u_component_diagnostics.py @@ -7,8 +7,9 @@ Security summaries set against the committed SSA Annual Statistical Supplement 2025 Table 5.A4 cells, the SSI amounts against the committed federal benefit rates, and counts of ``# IN FU`` against the individual -records (the U-inst rule's reading). WEALTH1 has no committed or saved -published comparator, so it is reported without one. +records (the reading of the unregistered institution rule). WEALTH1 has +no committed or saved published comparator, so it is reported without +one. It computes **no** income concept, annuity, threshold, poverty status or poverty rate, and no official-concept poverty rate (plan @@ -200,7 +201,7 @@ def summary_markdown(result: dict[str, Any]) -> str: _sentence(diag["published_sources"]["wealth1"]["not_compared"]) + " Waves 2005 and 2007 have no WEALTH1 (supplements not staged).", "", - "## `# IN FU` against the individual records (U-inst rule)", + "## `# IN FU` against the individual records (institution rule)", "", "| Wave | Families | # IN FU = in-family records | Institution " "records | Families with them | … # IN FU = in-family only | … " diff --git a/src/populace_dynamics/cohorts/age67.py b/src/populace_dynamics/cohorts/age67.py index 23ebaad5..fbdb2d66 100644 --- a/src/populace_dynamics/cohorts/age67.py +++ b/src/populace_dynamics/cohorts/age67.py @@ -28,15 +28,22 @@ #42 registration (a rule on staging status, never on results). Universe per wave (F2): sequence 1-20 (in a responding family) with a -positive cross-section weight; the registered option U-inst adds -sequence 51-59 (institution). The PSID collects no income for an +positive cross-section weight. Institutionalized persons (sequence +51-59) are outside it and counted in the dispositions. Row U-inst, which +added them, was withdrawn in the specification's u1-draft-5 (second +referee, S7: none of U0's target persons was in an institution at the +observation wave on the staged PSID). The unregistered code option +``presence="in_family_or_institution"`` still admits them, and +``institution_income_rule`` (pending the freeze; no registered row reads +it) then decides their income. The PSID collects no income for an institutionalized person (the individual-file Social Security items are -"Inap.: ... in an institution", codebook ER34137-ER34143 for 2011) and -the plan gives them no income rule, so ``institution_income_rule`` (a -builder default of u1-draft-3, pending the freeze and not yet refereed) -supplies one: +"Inap.: ... in an institution", codebook ER34137-ER34143 for 2011): -* ``family_of_record`` (default): the PSID attaches an institutionalized +* ``excluded`` (default, the second referee's Q9 answer): + institutionalized persons stay out of the poverty universe (a + disposition, ``institution_excluded_by_rule``), so the option's + population is U0's and only its counts differ. +* ``family_of_record``: the PSID attaches an institutionalized sample member's record to the family they left (2011 User Guide, section 2.4) and counts it among the individual records "having the same family-level data" as that family (family-file codebook text for @@ -50,9 +57,6 @@ currently in the FU") is read as not counting them: a named delta. An institutionalized person whose interview number has no family-file record in that wave is a disposition, not an observation. -* ``excluded``: institutionalized persons stay out of the poverty - universe (a disposition, as under the primary's universe), so U-inst - equals U0 in population and only its counts differ. The cross-section weight is also positive for movers-out (71-80) and decedents (81-89), whom neither option admits; the dispositions count @@ -87,9 +91,20 @@ under ``relationship_code`` (default, the referee's Q7 answer), married and co-resident with that spouse when the member is the head with a co-resident legal spouse (code 20 or 90) or is the head's legal wife (20) -or legal husband (90); otherwise non-married. ``non_married`` classifies -them all as non-married. ``marital_resolution`` records the path taken -for every observation. +or legal husband (90); otherwise not married (``married`` false). +``non_married`` leaves them all not married. ``marital_resolution`` +records the path taken for every observation. + +Four-way marital status (``marital_status_4``, specification section 9 +after the second referee's S2): the Report's Tables 19 and 21 carry the +marital rows Married, Widowed, Divorced and Never married (cleared +exercise-2 definitions extract). A member is ``married`` when +``married`` is true (by the marriage history or by the relationship +code), otherwise the marriage history's ``widowed``, ``divorced`` or +``never_married``; every other state is ``unclassified`` (an unresolved +state the relationship code does not resolve, and ``separated``, which +arises only under ``separated_is_married=False``). A cohabiting member +counts under his or her legal status. Annuitant ages (``annuitant_age_source``, pending the freeze): under ``derived_birth_year`` (default, the referee's Q8 answer) every @@ -151,6 +166,8 @@ "PSID_FILES", "ROWS", "TARGET_AGE", + "MARITAL_STATUS_4", + "UNCLASSIFIED_MARITAL_STATUS", "UNRESOLVED_MARITAL_RULES", "WAVES", "Age67Cohort", @@ -159,6 +176,7 @@ "build_age67_cohort", "income_rows", "load_age67_inputs", + "marital_status_4", "observation_plan", "pending_decisions", "read_design_variables", @@ -178,30 +196,48 @@ PSID_FILES = "psid_files" INVENTED = "invented" CALLER_FRAMES = "caller_frames" -#: The income rules for an institutionalized observation (row U-inst). +#: The income rules for an institutionalized observation under the +#: unregistered option ``presence="in_family_or_institution"`` (row U-inst +#: was withdrawn in u1-draft-5). INSTITUTION_INCOME_RULES: dict[str, str] = { + "excluded": ( + "institutionalized persons stay out of the poverty universe (a " + "disposition); the option's population is then U0's (the second " + "referee's Q9 answer)" + ), "family_of_record": ( "the family unit whose interview number the institutionalized " "member's record carries (the family they left): its income, " "wealth, size and children, with the member's own age and sex, " - "role ofum and no co-resident spouse (builder default, pending " - "the freeze; not yet refereed)" - ), - "excluded": ( - "institutionalized persons stay out of the poverty universe (a " - "disposition); U-inst then equals U0 in population" + "role ofum and no co-resident spouse (the u1-draft-3 builder " + "rule; unregistered)" ), } +#: The Report's marital rows (Tables 19 and 21, "Marital Status": Married, +#: Widowed, Divorced, Never married; cleared exercise-2 definitions +#: extract), as the values of ``marital_status_4``. +MARITAL_STATUS_4: tuple[str, ...] = ( + "married", + "widowed", + "divorced", + "never_married", +) +#: ``marital_status_4`` of a member none of the Report's marital rows +#: holds (specification section 9). +UNCLASSIFIED_MARITAL_STATUS = "unclassified" #: How a member whose marriage history cannot resolve the state is #: classified. UNRESOLVED_MARITAL_RULES: dict[str, str] = { "relationship_code": ( "married and co-resident when the member is the head with a " "co-resident legal spouse (code 20 or 90) or is the head's legal " - "wife (20) or legal husband (90); otherwise non-married (the " - "referee's Q7 answer)" + "wife (20) or legal husband (90); otherwise not married, with " + "marital_status_4 'unclassified' (the referee's Q7 answer)" + ), + "non_married": ( + "every unresolved member is not married (marital_status_4 " + "'unclassified')" ), - "non_married": "every unresolved member is non-married", } #: Where an annuitant's age comes from. ANNUITANT_AGE_SOURCES: dict[str, str] = { @@ -313,7 +349,7 @@ class Age67Spec: seed_wave_rule: str = "earliest_presence_wave" unresolved_marital_status: str = "relationship_code" annuitant_age_source: str = "derived_birth_year" - institution_income_rule: str = "family_of_record" + institution_income_rule: str = "excluded" def __post_init__(self) -> None: if self.row not in ROWS: @@ -367,8 +403,13 @@ def pending_decisions() -> tuple[psid2010.PendingDecision, ...]: "presence", spec.presence, ("in_family_or_institution",), - "plan F2 (U-inst adds institutions; no income rule exists " - "for them)", + "plan F2; row U-inst (institutions admitted) was withdrawn in " + "u1-draft-5 (second referee, boomers2004-referee-2-20260924.md, " + "S7 and Q9): on the staged PSID none of U0's or U0-F's target " + "persons was in an institution at the observation wave, and the " + "Census cannot determine poverty status for people in " + "institutional group quarters; the alternative is an " + "unregistered code option", freeze, ), psid2010.PendingDecision( @@ -409,8 +450,9 @@ def pending_decisions() -> tuple[psid2010.PendingDecision, ...]: "history'); the relationship code decides (married and " "co-resident as head with a co-resident legal spouse, 20 or " "90, or as the head's legal wife, 20, or legal husband, 90), " - "otherwise non-married; the draft-2 rule classified them all " - "as non-married", + "otherwise not married (marital_status_4 'unclassified', left " + "out of the marital cells, specification section 9); the " + "draft-2 rule left them all not married", freeze, ), psid2010.PendingDecision( @@ -429,17 +471,15 @@ def pending_decisions() -> tuple[psid2010.PendingDecision, ...]: psid2010.PendingDecision( "institution_income_rule", spec.institution_income_rule, - ("excluded",), - "builder default (u1-draft-3), not yet refereed (the first " - "referee pass reviewed u1-draft-2, which had no rule): plan F2 " - "adds institutions under U-inst but gives them no income rule " - "and the PSID collects none for them; the PSID attaches an " - "institutionalized member's record to the family they left " - "(2011 User Guide section 2.4), so the observation takes that " - "family's income concept and threshold (the member's own " - "income is missing and the family's size does not count them: " - "a named delta). Applies only under presence " - "in_family_or_institution", + ("family_of_record",), + "second referee's Q9 answer (boomers2004-referee-2-20260924.md): " + "institutionalized persons stay out of the universe; " + "family_of_record (the u1-draft-3 builder rule: the family the " + "member left, 2011 User Guide section 2.4, without the member's " + "own income, which the PSID does not collect) would measure a " + "family's poverty and assign it to someone its size and income " + "leave out. Used by no registered row (U-inst withdrawn, S7); " + "applies only under presence in_family_or_institution", freeze, ), ) @@ -974,6 +1014,24 @@ def _resolve_by_relationship( return "unresolved_non_married", pd.NA +def marital_status_4(marital_status: str, married: bool) -> str: + """The member's row among the Report's four marital rows, if any. + + ``married`` when ``married`` is true (by the marriage history, or by + the relationship code under ``unresolved_marital_status``); the + marriage history's ``widowed``, ``divorced`` or ``never_married`` + otherwise; :data:`UNCLASSIFIED_MARITAL_STATUS` for every other state + (an unresolved ``unknown`` or ``no_marriage_history`` state, and + ``separated``, which arises only under ``separated_is_married=False``). + """ + + if married: + return "married" + if marital_status in MARITAL_STATUS_4[1:]: + return str(marital_status) + return UNCLASSIFIED_MARITAL_STATUS + + def _annuitant_age( pid: int | None, income_year: int, @@ -1017,13 +1075,14 @@ def build_age67_cohort( ``sequence``, ``relationship``, ``member_role``, ``stratum``, ``cluster``, ``marital_status``, ``marital_resolution`` (``marriage_history`` or the path an unresolved state took), - ``married``, ``spouse_person_id``, ``member_married_coresident``, + ``married``, ``marital_status_4`` (:func:`marital_status_4`), + ``spouse_person_id``, ``member_married_coresident``, ``spouse_age``, ``spouse_age_source``, ``spouse_sex``, ``fu_legal_wife_present``, ``wife_sex``, the head and the head's legal spouse (``fu_head_person_id``, ``fu_head_age``, ``fu_head_age_source``, ``fu_head_sex``, ``fu_head_spouse_present``, - ``fu_head_spouse_person_id``, ``fu_head_spouse_age``, - ``fu_head_spouse_age_source``, ``fu_head_spouse_sex``; ages are + ``fu_head_spouse_person_id``, ``fu_head_spouse_relationship`` (20 or + 90), ``fu_head_spouse_age``, ``fu_head_spouse_age_source``, ``fu_head_spouse_sex``; ages are income-year ages under ``annuitant_age_source``), ``wealth_status``, ``in_institution`` and ``income_status`` (``family_file`` for a member of the family unit, ``family_of_record`` for an institutionalized @@ -1139,7 +1198,9 @@ def build_age67_cohort( continue disposition_rows.append({**base, "disposition": "observation"}) family_rows = by_family.get(interview) - head, head_spouse, _ = _head_and_legal_spouse(family_rows) + head, head_spouse, head_spouse_code = _head_and_legal_spouse( + family_rows + ) if pid in episodes: state = psid2010.marital_state_at( episodes[pid], @@ -1248,6 +1309,9 @@ def build_age67_cohort( "marital_status": state["status"], "marital_resolution": resolution, "married": bool(married), + "marital_status_4": marital_status_4( + state["status"], bool(married) + ), "spouse_person_id": spouse, "member_married_coresident": bool(coresident), "spouse_age": spouse_age, @@ -1265,6 +1329,9 @@ def build_age67_cohort( "fu_head_spouse_person_id": ( pd.NA if head_spouse is None else head_spouse ), + "fu_head_spouse_relationship": ( + pd.NA if head_spouse_code is None else head_spouse_code + ), "fu_head_spouse_age": head_spouse_age, "fu_head_spouse_age_source": head_spouse_age_source, "fu_head_spouse_sex": ( @@ -1296,6 +1363,7 @@ def build_age67_cohort( "fu_head_person_id", "fu_head_age", "fu_head_spouse_person_id", + "fu_head_spouse_relationship", "fu_head_spouse_age", ): observations[column] = observations[column].astype("Int64") @@ -1426,6 +1494,44 @@ def _counts(series: pd.Series) -> dict[str, int]: } +def _legal_husband_counts( + rows: pd.DataFrame, fam: pd.DataFrame +) -> dict[str, int | None]: + """Counts for observations whose head has a legal husband (code 90). + + ``wife_age_present`` counts those whose family records an ``AGE OF + WIFE`` (not 0, "no Wife/'Wife' in FU"): the U4 unit size is the member + plus a co-resident wife (``wife_present``), so it shows whether the + family file carries the legal husband in its "wife" fields. + """ + + husband = ( + rows["fu_head_spouse_relationship"] + .eq(_LEGAL_HUSBAND) + .fillna(False) + .to_numpy(dtype=bool) + ) + fam_h = fam.loc[husband] + + def count(column: str, test: Any) -> int | None: + return int(test(fam_h[column]).sum()) if column in fam_h else None + + return { + "n_observations": int(husband.sum()), + "n_member_is_head": int( + (husband & rows["member_role"].eq("head").to_numpy()).sum() + ), + "n_member_is_the_husband": int( + ( + husband & rows["relationship"].eq(_LEGAL_HUSBAND).to_numpy() + ).sum() + ), + "wife_age_present": count("wife_present", lambda s: s.astype(bool)), + "wife_labor_income_nonzero": count("wife_labor", lambda s: s != 0), + "wife_social_security_nonzero": count("wife_ss", lambda s: s != 0), + } + + def structural_summary( cohort: Age67Cohort, inputs: Age67Inputs ) -> dict[str, Any]: @@ -1482,6 +1588,12 @@ def structural_summary( ), "marital_status": _counts(rows["marital_status"]), "marital_resolution": _counts(rows["marital_resolution"]), + # Specification section 9 (u1-draft-5): the Report's four + # marital rows, and by sex. + "marital_status_4": _counts(rows["marital_status_4"]), + "sex_by_marital_status_4": _counts( + rows["sex"].astype(str) + ":" + rows["marital_status_4"] + ), "spouse_age_source": _counts(rows["spouse_age_source"]), "fu_head_age_source": _counts(rows["fu_head_age_source"]), "fu_head_spouse_present": _counts(rows["fu_head_spouse_present"]), @@ -1555,6 +1667,17 @@ def structural_summary( != 0 ).sum() ), + "family_head_annuities_nonzero": int( + (fam["head_annuities"] != 0).sum() + ), + "family_head_iras_nonzero": ( + int((fam["head_iras"] != 0).sum()) + if "head_iras" in fam + else None + ), + # Question 11 (u1-draft-5): the family file's "wife" fields + # where the head's legal spouse is a legal husband (code 90). + "head_legal_husband": _legal_husband_counts(rows, fam), "family_farm_income_nonzero": int((fam["head_farm"] != 0).sum()), "family_farm_income_negative": int((fam["head_farm"] < 0).sum()), "social_security_accuracy_nonzero": ( diff --git a/src/populace_dynamics/estimates/adjusted_poverty.py b/src/populace_dynamics/estimates/adjusted_poverty.py index f399842f..1ec86282 100644 --- a/src/populace_dynamics/estimates/adjusted_poverty.py +++ b/src/populace_dynamics/estimates/adjusted_poverty.py @@ -69,14 +69,18 @@ Census thresholds are **not captured** in this repository yet (see :func:`load_poverty_thresholds`). * **Cut** (F11). Reform income subtracts ``cut_rate`` (0.13) times all - Social Security of the unit; there is no behavioural response. - ``cut_start_year`` (default ``None``: every observation is cut, the F11 - primary) is row U6's parameter: with a start year, an observation is - cut only when the year its member turns 67 (birth year + 67) is at or - after it, so with 2004 the 1936 birth year (67 in 2003, observed at 68 - under U1) is uncut. The plan defines U6 on U1 with the scorecard's - "from 2004", which has no source the builder could read (plan section - 10, decision 8); the start year is pending that ruling. + Social Security of the unit; there is no behavioural response (Report + p. 37, fn. 19, as quoted in the cleared exercise-2 definitions + extract). ``cut_start_year`` (default 2004, the Report's "cut + immediately (beginning in 2004)", p. 37, cleared extract): an + observation is cut only when the year its member turns 67 (birth year + + 67) is at or after it, so every U0 and U0-F observation is cut and, + under U1, the 1936 birth year (67 in 2003, observed at 68) is not. + ``None`` (every observation cut, the u1-draft-4 primary) is an + unregistered alternative; row U6, which carried 2004 on U1, was + withdrawn in u1-draft-5 (second referee S5). Max's confirmation of + the scorecard's "from 2004" wording is pending (plan section 10, + decision 8). * **SSI response** (F13, pending Max, decision record d189). ``offset_existing_recipients`` (proposed primary): for each SSI unit with baseline SSI, SSI rises by the fall in countable Social Security @@ -250,7 +254,7 @@ ) #: How a farm loss is treated under ``farm_asset_share`` (F4b). FARM_LOSS_RULE = "removed_whole_when_share_positive" -#: The rule ``cut_start_year`` applies (row U6). +#: The rule ``cut_start_year`` applies (F11 primary). CUT_START_YEAR_RULE = "cut_when_birth_year_plus_67_at_or_after_start" #: Income years whose family file carries ``head_iras``. _HEAD_IRA_INCOME_YEARS: frozenset[int] = frozenset( @@ -371,7 +375,7 @@ class AdjustedPovertySpec: """Every income-concept choice; defaults are the plan's primary.""" cut_rate: float = 0.13 - cut_start_year: int | None = None + cut_start_year: int | None = 2004 annuitized_share: float = 0.8 real_interest_rate: float = 0.03 annuity_timing: str = "immediate" @@ -490,23 +494,30 @@ def pending_decisions() -> tuple[PendingDecision, ...]: spec.cut_rate, (), "Report p. 22 (OCACT estimates 'that benefits would need to be " - "reduced immediately by 13 percent'); the scenario's start " - "year and coverage are not in the methods (plan section 10 " - "decisions 2 and 8)", + "reduced immediately by 13 percent') and p. 37 ('cut " + "immediately (beginning in 2004) by 13 percent', cleared " + "exercise-2 definitions extract); the Report does not say " + "which benefit types the cut covers or whether SSI is " + "recomputed", _FREEZE, ), PendingDecision( "cut_start_year", spec.cut_start_year, - (2004,), - "plan F11: the primary cuts every observation (None); row U6 " - "(on U1) leaves an observation uncut when its member turned 67 " - "before the start year, 2004 being the scorecard's 'from 2004', " - "which has no source the builder could read; under U0 every " - "age-67 year is 2004 or later, so only U1's 1936 birth year " - "moves", - "Max (plan section 10 decision 8: the scenario's start year; " - "not yet on a decision card)", + (None,), + "second referee S5 (boomers2004-referee-2-20260924.md): the " + "Report cuts 'current and future benefits ... immediately " + "(beginning in 2004)' (p. 37; also p. 44) and analyses each " + "person 'when they reach age 67' (p. 24), both as quoted in the " + "cleared exercise-2 definitions extract; the cut applies when " + "the member's age-67 year (birth year + 67) is 2004 or later, " + "so every U0 and U0-F observation is cut and only U1's 1936 " + "birth year (67 in 2003) is not; None (every observation cut, " + "the u1-draft-4 primary) has no source and is not registered; " + "the Report gives no month within 2004", + "Max (plan section 10 decision 8: confirming the scorecard's " + "'from 2004' wording; not yet on a decision card) and the " + "specification freeze", ), PendingDecision( "income_unit", diff --git a/src/populace_dynamics/estimates/uniform_cut_tabulation.py b/src/populace_dynamics/estimates/uniform_cut_tabulation.py index 07ec87a5..ffbd8531 100644 --- a/src/populace_dynamics/estimates/uniform_cut_tabulation.py +++ b/src/populace_dynamics/estimates/uniform_cut_tabulation.py @@ -17,13 +17,24 @@ delta = P_R - P_B (percentage points) ``delta`` is the headline; ``P_B`` and ``P_R`` are the secondary rows -(the Table 19 and Table 21 analogues). Cells: ``all`` (headline), -``men``, ``women``, ``married`` and ``non_married`` (the reporting -splits the plan reads in the Report's methods, plan section 7), the four -sex-by-marital cells (optional) and one diagnostic cell per birth year -(not scored). An empty cell, or one with -zero total weight, is undefined and reported with its reason, never -imputed. +(the Table 19 and Table 21 analogues). Cells (specification section 9, +u1-draft-5, after the second referee's S2): the Report's Tables 19 and 21 +carry 36 rows (cleared exercise-2 definitions extract), of which Track U +v1 computes fifteen in the 1936-45 column: ``all`` (Total, headline); +the scored ``women``, ``men``, ``married``, ``widowed``, ``divorced`` and +``never_married`` (Gender; Marital Status); and the eight secondary +sex-by-marital cells (Gender and Marital Status), compared and reported +with their counts. :data:`REPORT_ROWS` names each cell's Report row; +:data:`NOT_COMPUTED_REPORT_ROWS` lists the other 21 rows (race and +ethnicity, education, labor-force experience, lifetime earnings) as named +omissions. One diagnostic cell per birth year is not compared. The +marital cells read ``marital_status_4`` +(:func:`populace_dynamics.cohorts.age67.marital_status_4`); a member with +no four-way status (``unclassified``) enters ``all``, ``women`` and +``men`` and no marital cell, and each marital cell reports how many were +left out (``unclassified_marital_cells="excluded_counted"``, pending the +freeze). An empty cell, or one with zero total weight, is undefined and +reported with its reason, never imputed. Uncertainty (plan field F15; the run is deterministic, K = 1): @@ -88,6 +99,7 @@ __all__ = [ "CELL_DEFINITIONS", + "COMPARATOR_COLUMN", "DEFAULT_CELLS", "DESIGN_SE_DOMAINS", "DEFAULT_FLOOR_SEEDS", @@ -95,14 +107,20 @@ "FLOOR_FRACTION", "FLOOR_SPLIT_UNIT", "INVENTED_DATA_LABEL", + "MARITAL_STATUSES", "MIN_FLOOR_SEEDS", + "NOT_COMPUTED_REPORT_ROWS", "OPTIONAL_CELLS", + "REPORT_ROWS", "REQUIRED_COLUMNS", "SCHEMA_VERSION", "STATISTICS", "STATISTIC_ID", "TabulationConfig", + "UNCLASSIFIED", + "UNCLASSIFIED_MARITAL_CELL_RULES", "UniformCutTabulationError", + "cell_mask", "floor_split_units", "pending_decisions", "tabulate_uniform_cut", @@ -121,7 +139,7 @@ "family_unit_id", "weight", "sex", - "married", + "marital_status_4", "birth_year", "stratum", "cluster", @@ -136,30 +154,136 @@ "reform_rate": "100 * sum w 1{R < T} / sum w (the Table 21 analogue)", } +#: The Report's column the comparator reads (Tables 19 and 21, "Birth +#: Cohort"; cleared exercise-2 definitions extract). +COMPARATOR_COLUMN = "1936-45" +#: The values of ``marital_status_4`` that the marital cells read, in the +#: order of the Report's "Marital Status" rows, and the value of a member +#: none of them holds (equal to +#: :data:`populace_dynamics.cohorts.age67.MARITAL_STATUS_4` and +#: ``UNCLASSIFIED_MARITAL_STATUS``, which a test holds). +MARITAL_STATUSES: tuple[str, ...] = ( + "married", + "widowed", + "divorced", + "never_married", +) +UNCLASSIFIED = "unclassified" +_SEXES = (("women", "female"), ("men", "male")) +#: The Report's row labels, as printed (cleared extract). +_MARITAL_LABELS = { + "married": "Married", + "widowed": "Widowed", + "divorced": "Divorced", + "never_married": "Never married", +} +#: Each computed cell's row in Tables 19 and 21 ("section: label" as +#: printed; the Total row has no section header). +REPORT_ROWS: dict[str, str] = { + "all": "Total", + "women": "Gender: Female", + "men": "Gender: Male", + **{ + status: f"Marital Status: {label}" + for status, label in _MARITAL_LABELS.items() + }, + **{ + f"{cell}_{status}": ( + f"Gender and Marital Status: " + f"{'Female' if sex == 'female' else 'Male'}: {label}" + ) + for cell, sex in _SEXES + for status, label in _MARITAL_LABELS.items() + }, +} +#: Headline and scored cells (the Total, Gender and Marital Status rows). +DEFAULT_CELLS: tuple[str, ...] = ("all", "women", "men", *MARITAL_STATUSES) +#: Secondary cells, compared and reported with their counts (the Gender +#: and Marital Status rows). +OPTIONAL_CELLS: tuple[str, ...] = tuple( + f"{cell}_{status}" for cell, _ in _SEXES for status in MARITAL_STATUSES +) CELL_DEFINITIONS: dict[str, str] = { "all": "every observation (headline)", - "men": "sex == male", "women": "sex == female", - "married": "legally married at the end of the income year (F12)", - "non_married": "not legally married (cohabitors included, F12)", - "men_married": "optional: male and married", - "men_non_married": "optional: male and not married", - "women_married": "optional: female and married", - "women_non_married": "optional: female and not married", + "men": "sex == male", + "married": ( + "married at the end of the income year (F12; separated counts as " + "married; by the marriage history or the relationship code)" + ), + "widowed": "widowed at the end of the income year (marriage history)", + "divorced": "divorced at the end of the income year (marriage history)", + "never_married": "never married (marriage history)", + **{ + f"{cell}_{status}": f"secondary: {sex} and {status}" + for cell, sex in _SEXES + for status in MARITAL_STATUSES + }, +} +#: The Report rows of Tables 19 and 21 that Track U v1 does not compute +#: (specification section 9, named omissions; cleared extract). +NOT_COMPUTED_REPORT_ROWS: dict[str, dict[str, Any]] = { + "race_ethnicity": { + "section": "Race/Ethnicity", + "rows": ( + "White, non-hispanic", + "Black, non-hispanic", + "Hispanic", + "Other", + ), + "reason": "Track U v1 has no reader for race and ethnicity", + }, + "education": { + "section": "Education", + "rows": ( + "High school dropout", + "High school graduate", + "College graduate", + ), + "reason": ( + "Track U v1 has no reader for education, and the Report does " + "not say whether 'High school graduate' includes some college" + ), + }, + "labor_force_experience": { + "section": "Labor Force Experience", + "rows": ( + "Less than 20 years", + "20 to 29 years", + "30 to 34 years", + "35 or more years", + ), + "reason": ( + "Track U v1 has no reader for it, and the Report does not " + "define the measure" + ), + }, + **{ + f"lifetime_earnings_{kind}": { + "section": f"Lifetime Earnings ({kind.title()})", + "rows": tuple( + f"{n} Quintile" for n in ("1st", "2nd", "3rd", "4th", "5th") + ), + "reason": ( + "Track U v1 has no reader for it; the measure averages " + "wage-indexed earnings at ages 22-62, which the PSID does " + "not observe at every one of those ages for these cohorts " + "(its first income year is 1967, and odd income years from " + "1997 on are not observed), and the Report does not say " + "over which population the quintiles are cut" + ), + } + for kind in ("own", "shared") + }, +} +#: How members with no four-way marital status enter the marital cells. +UNCLASSIFIED_MARITAL_CELL_RULES: dict[str, str] = { + "excluded_counted": ( + "a member whose marital_status_4 is 'unclassified' enters all, " + "women and men and no marital cell; each marital cell reports how " + "many were left out (second referee S2)" + ), } -DEFAULT_CELLS: tuple[str, ...] = ( - "all", - "men", - "women", - "married", - "non_married", -) -OPTIONAL_CELLS: tuple[str, ...] = ( - "men_married", - "men_non_married", - "women_married", - "women_non_married", -) #: Diagnostic cells, one per birth year present (not scored). DIAGNOSTIC_BIRTH_YEAR_CELLS = "birth_year_" @@ -192,6 +316,7 @@ class TabulationConfig: """Cells and uncertainty settings; defaults are the plan's proposal.""" cells: tuple[str, ...] = DEFAULT_CELLS + OPTIONAL_CELLS + unclassified_marital_cells: str = "excluded_counted" birth_year_diagnostics: bool = True floor_seeds: tuple[int, ...] = DEFAULT_FLOOR_SEEDS design_standard_errors: bool = True @@ -215,10 +340,19 @@ def __post_init__(self) -> None: raise UniformCutTabulationError( f"design_se_domain must be one of {sorted(DESIGN_SE_DOMAINS)}" ) + if self.unclassified_marital_cells not in ( + UNCLASSIFIED_MARITAL_CELL_RULES + ): + raise UniformCutTabulationError( + "unclassified_marital_cells must be one of " + f"{sorted(UNCLASSIFIED_MARITAL_CELL_RULES)}" + ) def as_dict(self) -> dict[str, Any]: return { "cells": list(self.cells), + "comparator_column": COMPARATOR_COLUMN, + "unclassified_marital_cells": self.unclassified_marital_cells, "birth_year_diagnostics": self.birth_year_diagnostics, "floor_seeds": list(self.floor_seeds), "floor_fraction": FLOOR_FRACTION, @@ -258,7 +392,14 @@ def _normalize(rows: pd.DataFrame) -> pd.DataFrame: sexes = set(out["sex"].astype(str)) if not sexes <= {"male", "female"}: raise UniformCutTabulationError(f"sex outside male/female: {sexes}") - for column in ("married", "poor_baseline", "poor_reform"): + statuses = out["marital_status_4"] + allowed = {*MARITAL_STATUSES, UNCLASSIFIED} + if statuses.isna().any() or not set(statuses.astype(str)) <= allowed: + raise UniformCutTabulationError( + f"marital_status_4 outside {sorted(allowed)}" + ) + out["marital_status_4"] = statuses.astype(str) + for column in ("poor_baseline", "poor_reform"): values = out[column] if ( values.isna().any() @@ -277,21 +418,55 @@ def _normalize(rows: pd.DataFrame) -> pd.DataFrame: return out +def _cell_parts(name: str) -> tuple[str | None, str | None]: + """``(sex, marital status)`` a cell restricts to (``None``: any).""" + + if name == "all": + return None, None + for cell, sex in _SEXES: + if name == cell: + return sex, None + if name.startswith(f"{cell}_") and name[len(cell) + 1 :] in ( + MARITAL_STATUSES + ): + return sex, name[len(cell) + 1 :] + if name in MARITAL_STATUSES: + return None, name + raise UniformCutTabulationError(f"unknown cell {name!r}") + + +def cell_mask(rows: pd.DataFrame, name: str) -> np.ndarray: + """The observations of cell ``name`` (columns ``sex`` and + ``marital_status_4``). + + A marital cell holds the members whose ``marital_status_4`` is its + status; an ``unclassified`` member is in no marital cell + (``unclassified_marital_cells="excluded_counted"``). + """ + + sex, status = _cell_parts(name) + mask = np.ones(len(rows), dtype=bool) + if sex is not None: + mask &= rows["sex"].astype(str).eq(sex).to_numpy() + if status is not None: + mask &= rows["marital_status_4"].astype(str).eq(status).to_numpy() + return mask + + +def _unclassified_left_out(rows: pd.DataFrame, name: str) -> int | None: + """How many members of a marital cell's sex domain are unclassified.""" + + sex, status = _cell_parts(name) + if status is None: + return None + domain = rows["marital_status_4"].eq(UNCLASSIFIED).to_numpy() + if sex is not None: + domain = domain & rows["sex"].eq(sex).to_numpy() + return int(domain.sum()) + + def _cell_masks(rows: pd.DataFrame, config: TabulationConfig) -> dict: - male = rows["sex"].eq("male").to_numpy() - married = rows["married"].to_numpy() - masks = { - "all": np.ones(len(rows), dtype=bool), - "men": male, - "women": ~male, - "married": married, - "non_married": ~married, - "men_married": male & married, - "men_non_married": male & ~married, - "women_married": ~male & married, - "women_non_married": ~male & ~married, - } - out = {name: masks[name] for name in config.cells} + out = {name: cell_mask(rows, name) for name in config.cells} if config.birth_year_diagnostics: for year in sorted(set(rows["birth_year"].tolist())): out[f"birth_year_{year}"] = rows["birth_year"].eq(year).to_numpy() @@ -388,11 +563,26 @@ def _cell_entry( entry: dict[str, Any] = { "cell": name, "definition": CELL_DEFINITIONS.get( - name, "diagnostic: birth year (not scored)" + name, "diagnostic: birth year (not compared)" ), - "scored_candidate": name in DEFAULT_CELLS, - "optional": name in OPTIONAL_CELLS, + "report_row": REPORT_ROWS.get(name), + "role": ( + "headline" + if name == "all" + else ( + "scored" + if name in DEFAULT_CELLS + else ("secondary" if name in OPTIONAL_CELLS else "diagnostic") + ) + ), + "scored": name in DEFAULT_CELLS, + "secondary": name in OPTIONAL_CELLS, "diagnostic": name.startswith("birth_year_"), + "n_marital_unclassified_left_out": ( + None + if name.startswith("birth_year_") + else _unclassified_left_out(rows, name) + ), "n_observations": int(mask.sum()), "n_persons": int(rows.loc[mask, "person_id"].nunique()), "n_family_units": int(rows.loc[mask, "family_unit_id"].nunique()), @@ -520,8 +710,7 @@ def _floors( def pending_decisions() -> tuple[PendingDecision, ...]: """The open choices of :class:`TabulationConfig`, with their defaults. - None is ratified; each awaits the specification freeze (or, for the - cells, plan section 10 decision 2(b)). + None is ratified; each awaits the specification freeze. """ config = TabulationConfig() @@ -543,13 +732,26 @@ def pending_decisions() -> tuple[PendingDecision, ...]: "cells", list(config.cells), (), - "plan section 7: all (headline), men, women, married, " - "non_married (the Report's list of appendix tables names " - "Appendix Table 17 'by Gender and Marital Status') and the " - "four optional sex-by-marital cells; which splits Tables 19 " - "and 21 carry is unknown", - "Max (plan section 10 decision 2(b)) and the specification " - "freeze", + "second referee S2 (boomers2004-referee-2-20260924.md) and the " + "cleared exercise-2 definitions extract: Tables 19 and 21 carry " + "36 rows; Track U v1 computes the Total, Gender and Marital " + "Status rows (all headline; women, men, married, widowed, " + "divorced, never_married scored) and the eight Gender and " + "Marital Status rows (secondary) in the 1936-45 column; the " + "other 21 rows are named omissions (NOT_COMPUTED_REPORT_ROWS)", + freeze, + ), + PendingDecision( + "unclassified_marital_cells", + config.unclassified_marital_cells, + (), + "second referee S2: a member with no four-way marital status " + "(an unresolved state the relationship code does not resolve; " + "34 of U0's 483 observations and 24 of U0-F's 320 on the staged " + "PSID, structural counts) enters all, women and men and no " + "marital cell, and each marital cell reports how many were " + "left out", + freeze, ), ) @@ -733,6 +935,16 @@ def tabulate_uniform_cut( "n_persons": int(normalized["person_id"].nunique()), "n_family_units": int(normalized["family_unit_id"].nunique()), "n_zero_weight": int((normalized["weight"] == 0).sum()), + "marital_status_4": { + str(key): int(value) + for key, value in normalized["marital_status_4"] + .value_counts() + .sort_index() + .items() + }, + "n_marital_unclassified": int( + normalized["marital_status_4"].eq(UNCLASSIFIED).sum() + ), }, "cells": cells, "undefined_cells": undefined, diff --git a/src/populace_dynamics/uniform_cut_track_u/__init__.py b/src/populace_dynamics/uniform_cut_track_u/__init__.py index 8541901a..4cc11d14 100644 --- a/src/populace_dynamics/uniform_cut_track_u/__init__.py +++ b/src/populace_dynamics/uniform_cut_track_u/__init__.py @@ -8,7 +8,8 @@ :mod:`populace_dynamics.estimates.uniform_cut_tabulation`; this package composes it: -* :mod:`.rows` -- the registered rows (U0-U10, U-inst) as builder and +* :mod:`.rows` -- the registered rows (U0-U5, U0-F, U7-U10 and the -F + alternatives U2-F ... U10-F on U0-F's population) as builder and income-concept parameters, checked against the specification block; * :mod:`.invented` -- the INVENTED age-67 cohort generator and the INVENTED threshold table (dry runs and tests only); diff --git a/src/populace_dynamics/uniform_cut_track_u/diagnostics.py b/src/populace_dynamics/uniform_cut_track_u/diagnostics.py index fda4294c..65e6b263 100644 --- a/src/populace_dynamics/uniform_cut_track_u/diagnostics.py +++ b/src/populace_dynamics/uniform_cut_track_u/diagnostics.py @@ -572,8 +572,10 @@ def institution_record_counts(inputs: age67.Age67Inputs) -> dict[str, Any]: (51-59) attached to a responding family, and among the families with at least one such record, how many have ``# IN FU`` equal to the in-family count alone and how many equal to the in-family count plus - the institution records. This checks the U-inst rule's reading that - ``# IN FU`` does not count institutionalized members. + the institution records. This checks the reading of the unregistered + ``family_of_record`` institution rule (row U-inst, withdrawn in + u1-draft-5) that ``# IN FU`` does not count institutionalized + members. """ out: dict[str, Any] = {} diff --git a/src/populace_dynamics/uniform_cut_track_u/invented.py b/src/populace_dynamics/uniform_cut_track_u/invented.py index 43b38167..de8f8ea2 100644 --- a/src/populace_dynamics/uniform_cut_track_u/invented.py +++ b/src/populace_dynamics/uniform_cut_track_u/invented.py @@ -42,7 +42,9 @@ invented assumption not checked against the codebooks); * members who enter an institution (sequence 51) in the first wave whose income year is at or after their 66th birthday year and stay, attached - to the family they left, where the spouse becomes head (row U-inst); + to the family they left, where the spouse becomes head (the + unregistered institution option; row U-inst was withdrawn in + u1-draft-5); members who move out (71) or die (81) by that wave's interview; and members whose weight is zero from that wave. diff --git a/src/populace_dynamics/uniform_cut_track_u/rows.py b/src/populace_dynamics/uniform_cut_track_u/rows.py index 7023a965..30b73670 100644 --- a/src/populace_dynamics/uniform_cut_track_u/rows.py +++ b/src/populace_dynamics/uniform_cut_track_u/rows.py @@ -2,13 +2,23 @@ Each row of the exercise-2 specification (``docs/design/boomers2004_uniform_cut_comparison.md``, sections 11 and -15) differs from the primary U0 in one field, except U6, which is defined -on U1 (plan ``critical-path-uniform-cut-20260923.md`` section 7), and -U0-F, U0 restricted to the birth years 1941, 1943 and 1945. A row here -is a set of overrides of :class:`populace_dynamics.cohorts.age67. -Age67Spec` and :class:`populace_dynamics.estimates.adjusted_poverty. +15) differs from the primary U0 in one field; U1 (all ten birth years) +and U0-F (U0 restricted to the birth years 1941, 1943 and 1945) change +the population itself. Each one-field alternative defined on U0 (U2, +U3, U4, U5, U8, U9, U10) is also registered on U0-F's population as its +``-F`` row (U2-F ... U10-F; :data:`FALLBACK_ALTERNATIVES`), so that the +registration carries its alternatives whether or not the 2005 and 2007 +wealth supplements are staged (second referee, S8). A row here is a set +of overrides of :class:`populace_dynamics.cohorts.age67.Age67Spec` and +:class:`populace_dynamics.estimates.adjusted_poverty. AdjustedPovertySpec`; the defaults of those two classes are U0. +Rows U6 (the cut's start year, on U1) and U-inst (institutions admitted) +of u1-draft-3 and -4 are withdrawn in u1-draft-5 (second referee, S5 and +S7): the primary now starts the cut in 2004 (``cut_start_year``), so U6 +would equal U1, and no U0 target person was in an institution at the +observation wave on the staged PSID. + :data:`HEADLINE_RULE` is the specification's fallback rule (pending Max, with plan section 10 decision 3): U0 is the headline when the 2005 and 2007 wealth supplements are staged, adjudicated and read before the #42 @@ -34,6 +44,7 @@ from populace_dynamics.estimates import adjusted_poverty as ap __all__ = [ + "FALLBACK_ALTERNATIVES", "FALLBACK_ROW", "HEADLINE_RULE", "PRIMARY_ROW", @@ -109,117 +120,120 @@ def as_dict(self) -> dict[str, Any]: } +_FALLBACK_AWAITING = "Max (the fallback rule, with plan section 10 decision 3)" +#: The one-field alternatives defined on U0 that are also registered on +#: U0-F's population, and the name of each ``-F`` row (second referee S8). +FALLBACK_ALTERNATIVES: dict[str, str] = { + row_id: f"{row_id}-F" + for row_id in ("U2", "U3", "U4", "U5", "U8", "U9", "U10") +} + +_BASE_ROWS: tuple[TrackURow, ...] = ( + TrackURow( + "U0", + "-", + "primary: exact age (odd birth years 1937-1945 at 67), family " + "unit, reported asset income replaced by the annuity, Census " + "weighted-average 65+ thresholds, SSI offset for existing " + "recipients", + ), + TrackURow( + "U1", + "population", + "all ten birth years: even birth years at 66 and 68 with half " + "weight each, 1936 at 68 only", + age67={"row": "U1"}, + ), + TrackURow( + "U2", + "SSI", + "no SSI response", + income={"ssi_rule": "none"}, + ), + TrackURow( + "U3", + "SSI", + "full static SSI recomputation (the largest SSI response of " + "the three registered rules, not a bound on DYNASIM's " + "simulation)", + income={"ssi_rule": "full_static_recomputation"}, + ), + TrackURow( + "U4", + "income unit", + "head and wife income only", + income={"income_unit": "head_wife"}, + ), + TrackURow( + "U5", + "asset income", + "keep reported asset income and add the annuity", + income={"asset_income_rule": "keep"}, + ), + TrackURow( + "U0-F", + "population", + "U0 restricted to the birth years 1941, 1943 and 1945 (the " + "blocked 1937 and 1939 left out and counted); the headline when " + "the 2005 and 2007 wealth supplements are not staged before " + "the #42 registration", + age67={"row": FALLBACK_ROW}, + awaiting=_FALLBACK_AWAITING, + ), + TrackURow( + "U7", + "financial assets", + "WEALTH1 plus employer DC balances", + built=False, + not_built_reason=( + "needs a label investigation of the PSID P-section pension " + "account items (plan section 4); not built" + ), + ), + TrackURow( + "U8", + "threshold", + "PSID CENSUS NEEDS STANDARD", + income={"threshold_rule": "psid_census_needs_standard"}, + ), + TrackURow( + "U9", + "mortality", + "SSA period life table for 2004", + income={"mortality_basis": "ssa_period_2004"}, + ), + TrackURow( + "U10", + "threshold", + "Census size-by-related-children matrix (65+ rows for sizes 1 " + "and 2)", + income={"threshold_rule": "census_matrix_65plus"}, + ), +) + + +def _on_fallback(row: TrackURow) -> TrackURow: + """``row``'s field and value on U0-F's population (its -F row).""" + + return TrackURow( + FALLBACK_ALTERNATIVES[row.row_id], + row.field_changed, + f"{row.description}, on U0-F's population (birth years 1941, " + "1943 and 1945)", + age67={**dict(row.age67), "row": FALLBACK_ROW}, + income=dict(row.income), + awaiting=_FALLBACK_AWAITING, + ) + + REGISTERED_ROWS: dict[str, TrackURow] = { row.row_id: row for row in ( - TrackURow( - "U0", - "-", - "primary: exact age (odd birth years 1937-1945 at 67), family " - "unit, reported asset income replaced by the annuity, Census " - "weighted-average 65+ thresholds, SSI offset for existing " - "recipients", - ), - TrackURow( - "U1", - "population", - "all ten birth years: even birth years at 66 and 68 with half " - "weight each, 1936 at 68 only", - age67={"row": "U1"}, - ), - TrackURow( - "U2", - "SSI", - "no SSI response", - income={"ssi_rule": "none"}, - ), - TrackURow( - "U3", - "SSI", - "full static SSI recomputation (the largest SSI response of " - "the three registered rules, not a bound on DYNASIM's " - "simulation)", - income={"ssi_rule": "full_static_recomputation"}, - ), - TrackURow( - "U4", - "income unit", - "head and wife income only", - income={"income_unit": "head_wife"}, - ), - TrackURow( - "U5", - "asset income", - "keep reported asset income and add the annuity", - income={"asset_income_rule": "keep"}, - ), - TrackURow( - "U6", - "cut (on U1)", - "U1 with the 1936 birth year uncut if the cut starts in 2004 " - "(the cut reaches an observation only when its member turns 67 " - "in or after 2004)", - age67={"row": "U1"}, - income={"cut_start_year": 2004}, - awaiting=( - "Max, plan section 10 decision 8 (the scenario's start " - "year; the scorecard's 'from 2004' has no readable source)" - ), - ), - TrackURow( - "U0-F", - "population", - "U0 restricted to the birth years 1941, 1943 and 1945 (the " - "blocked 1937 and 1939 left out and counted); the headline when " - "the 2005 and 2007 wealth supplements are not staged before " - "the #42 registration", - age67={"row": FALLBACK_ROW}, - awaiting=( - "Max (the fallback rule, with plan section 10 decision 3)" - ), - ), - TrackURow( - "U7", - "financial assets", - "WEALTH1 plus employer DC balances", - built=False, - not_built_reason=( - "needs a label investigation of the PSID P-section pension " - "account items (plan section 4); not built" - ), - ), - TrackURow( - "U8", - "threshold", - "PSID CENSUS NEEDS STANDARD", - income={"threshold_rule": "psid_census_needs_standard"}, - ), - TrackURow( - "U9", - "mortality", - "SSA period life table for 2004", - income={"mortality_basis": "ssa_period_2004"}, - ), - TrackURow( - "U10", - "threshold", - "Census size-by-related-children matrix (65+ rows for sizes 1 " - "and 2)", - income={"threshold_rule": "census_matrix_65plus"}, - ), - TrackURow( - "U-inst", - "universe", - "add persons in institutions (sequence 51-59) with the " - "family-of-record income rule", - age67={ - "presence": "in_family_or_institution", - "institution_income_rule": "family_of_record", - }, - awaiting=( - "the specification freeze (the institution income rule is " - "a builder default of u1-draft-3, not yet refereed)" - ), + *_BASE_ROWS, + *( + _on_fallback(row) + for row in _BASE_ROWS + if row.row_id in FALLBACK_ALTERNATIVES ), ) } diff --git a/src/populace_dynamics/uniform_cut_track_u/runner.py b/src/populace_dynamics/uniform_cut_track_u/runner.py index 2d49588d..89283e76 100644 --- a/src/populace_dynamics/uniform_cut_track_u/runner.py +++ b/src/populace_dynamics/uniform_cut_track_u/runner.py @@ -95,14 +95,24 @@ "realized 2004-2012 history versus DYNASIM's 1992-based projection, " "including the 2008-09 asset shock for the 1941-45 cohorts at 67", "realized COLAs versus 2002 Trustees assumptions", - "immigrant under-coverage; institutionalized persons (row U-inst: " - "the family-of-record rule assigns the family's income and threshold " - "without the member's own income, which the PSID does not collect); " - "attrition", + "immigrant under-coverage; institutionalized persons are outside the " + "universe (none of U0's target persons was in an institution at the " + "observation wave on the staged files); attrition", "OFUM-owned assets inside family wealth", "the family's wealth stands in for an OFUM cohort member's own wealth " "(the Report's unit is the individual plus spouse, p. 24)", - "imputed rent excluded", + "the Report's total income lists income from financial assets, " + "imputed rent, Social Security, DB pensions, retirement-account " + "income, earnings, SSI and non-spouse co-resident income (p. 31, " + "cleared extract), and its poverty income is described only as " + "differing from Census money income by the annuity (p. 24); the " + "primary keeps every other PSID money-income source (veterans' " + "pensions, unemployment and workers' compensation, TANF and other " + "welfare, child support, alimony, help from relatives and others, " + "miscellaneous transfers)", + "members whose marital state is unresolved (34 of U0's 483 " + "observations, 24 of U0-F's 320, structural counts) are left out of " + "the marital cells (§9)", "self-reported Social Security, possibly net of Medicare Part B " "premiums, so the 13 percent cut applies to a smaller base than the " "gross benefit (baseline income is lower too; the net direction on " @@ -144,8 +154,9 @@ "reported SSI may include state supplementary payments, so capping " "the offset at the federal benefit rate less reported SSI can " "understate the offset", - "U0-F, if it is the headline: mean birth year 1943 against 1940.5, " - "and every observation year at or after the 2008-09 asset shock", + "rows on U0-F's population (U0-F and U2-F ... U10-F): mean birth " + "year 1943 against 1940.5, and every observation year at or after the " + "2008-09 asset shock", ) @@ -339,18 +350,6 @@ def _counts(series: pd.Series) -> dict[str, int]: } -def _cell_masks(members: pd.DataFrame) -> dict[str, np.ndarray]: - male = members["sex"].astype(str).eq("male").to_numpy() - married = members["married"].astype(bool).to_numpy() - return { - "all": np.ones(len(members), dtype=bool), - "men": male, - "women": ~male, - "married": married, - "non_married": ~married, - } - - def official_concept_rates( members: pd.DataFrame, adjusted: pd.DataFrame ) -> dict[str, Any]: @@ -361,11 +360,15 @@ def official_concept_rates( annuity, no cut) is below the row's threshold (the same threshold as the adjusted concept, so the 65-and-over rule, not the Census householder-age convention). Weighted rate per cell of - :data:`OFFICIAL_CONCEPT_CELLS`, in percent; an empty or zero-weight - cell is undefined. Not scored. + :data:`OFFICIAL_CONCEPT_CELLS` (the tabulation's headline and scored + cells, :func:`populace_dynamics.estimates.uniform_cut_tabulation. + cell_mask`), in percent; an empty or zero-weight cell is undefined. + Not scored. """ - joined = members[["observation_id", "weight", "sex", "married"]].merge( + joined = members[ + ["observation_id", "weight", "sex", "marital_status_4"] + ].merge( adjusted[["observation_id", "money_income", "threshold"]], on="observation_id", how="inner", @@ -376,7 +379,8 @@ def official_concept_rates( poor = (joined["money_income"] < joined["threshold"]).to_numpy() weight = joined["weight"].to_numpy(dtype=np.float64) cells = {} - for name, mask in _cell_masks(joined).items(): + for name in OFFICIAL_CONCEPT_CELLS: + mask = ut.cell_mask(joined, name) total = float(weight[mask].sum()) if not mask.any() or total <= 0: cells[name] = { @@ -519,6 +523,11 @@ def _compute_row( "left_out": dict(members.attrs.get("left_out", {})), "observations_by_birth_year": _counts(members["birth_year"]), "marital_resolution": _counts(members["marital_resolution"]), + "marital_status_4": _counts(members["marital_status_4"]), + # Referee 2, O6: every annuitant age source, including the + # member's co-resident spouse, so a wave-age fallback is + # recorded. + "spouse_age_source": _counts(members["spouse_age_source"]), "fu_head_age_source": _counts(members["fu_head_age_source"]), "fu_head_spouse_age_source": _counts( members["fu_head_spouse_age_source"] @@ -672,6 +681,11 @@ def run_track_u( ), "components": diagnostics.component_diagnostics(cohort, inputs), }, + "comparator_column": ut.COMPARATOR_COLUMN, + "report_rows_not_computed": { + key: {**entry, "rows": list(entry["rows"])} + for key, entry in ut.NOT_COMPUTED_REPORT_ROWS.items() + }, "named_deltas": list(NAMED_DELTAS), "pending_decisions": { "age67": [item.as_dict() for item in age67.pending_decisions()], diff --git a/tests/cohorts/test_age67.py b/tests/cohorts/test_age67.py index 5b0f62f2..d6078a13 100644 --- a/tests/cohorts/test_age67.py +++ b/tests/cohorts/test_age67.py @@ -352,17 +352,27 @@ def _with_institution_member() -> age67.Age67Inputs: ) -def test_institution_rule_family_of_record_is_the_default(): - """U-inst: the institutionalized member takes its family of record. +def test_institution_rule_family_of_record_option(): + """The unregistered family-of-record option (row U-inst, withdrawn in + u1-draft-5): the institutionalized member takes its family of record. Invented 9001 is in an institution in 2009 attached to family 11 (person 1001's family): the observation is an OFUM of that family, with no co-resident spouse, and merges family 11's income and wealth. + The default rule is ``excluded`` (the second referee's Q9 answer). """ - assert age67.Age67Spec().institution_income_rule == "family_of_record" + assert age67.Age67Spec().institution_income_rule == "excluded" + decisions = {item.field: item for item in age67.pending_decisions()} + item = decisions["institution_income_rule"] + assert item.default == "excluded" + assert item.alternatives == ("family_of_record",) + assert "Q9" in item.default_basis and "S7" in item.default_basis inputs = _with_institution_member() - spec = age67.Age67Spec(presence="in_family_or_institution") + spec = age67.Age67Spec( + presence="in_family_or_institution", + institution_income_rule="family_of_record", + ) cohort = age67.build_age67_cohort(inputs, spec) obs = cohort.observations.set_index("observation_id") member = obs.loc["9001:2009"] @@ -389,21 +399,26 @@ def test_institution_rule_family_of_record_is_the_default(): def test_institution_rule_excluded_and_missing_family_of_record(): inputs = _with_institution_member() + # the default rule keeps the member out even when the presence option + # admits institutions: the option's population is then U0's excluded = age67.build_age67_cohort( - inputs, - age67.Age67Spec( - presence="in_family_or_institution", - institution_income_rule="excluded", - ), + inputs, age67.Age67Spec(presence="in_family_or_institution") ) assert "9001:2009" not in set(excluded.observations["observation_id"]) assert _dispositions(excluded)[(9001, 2009)] == ( "institution_excluded_by_rule" ) + assert set(excluded.observations["observation_id"]) == set( + age67.build_age67_cohort(inputs).observations["observation_id"] + ) # 7001 is in an institution in 2007 attached to interview 7, which has # no 2007 family-file record in the invented frames cohort = age67.build_age67_cohort( - inputs, age67.Age67Spec(presence="in_family_or_institution") + inputs, + age67.Age67Spec( + presence="in_family_or_institution", + institution_income_rule="family_of_record", + ), ) assert "7001:2007" not in set(cohort.observations["observation_id"]) assert _dispositions(cohort)[(7001, 2007)] == ( @@ -459,7 +474,13 @@ def test_structural_summary_counts_only(): "unresolved_non_married": 1, } assert cell["fu_head_age_source"] == {"derived_birth_year": 2} + assert sum(cell["marital_status_4"].values()) == 2 + assert cell["marital_status_4"].get("unclassified") == 1 + assert sum(cell["sex_by_marital_status_4"].values()) == 2 receipt = summary["component_receipt_counts"]["2011"] + assert receipt["family_head_annuities_nonzero"] == 0 + assert receipt["family_head_iras_nonzero"] is None + assert receipt["head_legal_husband"]["n_observations"] == 0 assert receipt["family_farm_income_nonzero"] == 0 assert receipt["family_head_retirement_account_income_nonzero"] == 0 text = repr(summary) @@ -639,8 +660,13 @@ def test_relationship_code_resolves_unresolved_marital_states(): assert husband["fu_head_sex"] == "female" assert husband["fu_head_spouse_present"] assert husband["fu_head_spouse_person_id"] == 9202 + assert husband["fu_head_spouse_relationship"] == 90 assert husband["fu_head_spouse_sex"] == "male" assert obs.loc["9201:2013", "marital_resolution"] == "marriage_history" + # both resolve into the Report's "Married" row (specification section 9) + for oid in ("9101:2013", "9202:2013", "9201:2013"): + assert obs.loc[oid, "marital_status_4"] == "married" + assert obs.loc["1001:2009", "fu_head_spouse_relationship"] == 20 plain = age67.build_age67_cohort( inputs, age67.Age67Spec(unresolved_marital_status="non_married") ).observations.set_index("observation_id") @@ -650,6 +676,83 @@ def test_relationship_code_resolves_unresolved_marital_states(): ) assert not plain.loc[oid, "married"] assert pd.isna(plain.loc[oid, "spouse_person_id"]) + # unresolved: in no marital cell of the Report + assert plain.loc[oid, "marital_status_4"] == "unclassified" + summary = age67.structural_summary( + age67.build_age67_cohort(inputs), inputs + ) + husband_counts = summary["component_receipt_counts"]["2013"][ + "head_legal_husband" + ] + # family 92: the female head 9201 and her legal husband 9202 (code 90) + assert husband_counts["n_observations"] == 2 + assert husband_counts["n_member_is_head"] == 1 + assert husband_counts["n_member_is_the_husband"] == 1 + # the invented family frames carry no AGE OF WIFE or wife labor item + assert husband_counts["wife_age_present"] is None + assert husband_counts["wife_social_security_nonzero"] == 0 + + +@pytest.mark.parametrize( + ("status", "married", "expected"), + [ + ("married", True, "married"), + ("widowed", False, "widowed"), + ("divorced", False, "divorced"), + ("never_married", False, "never_married"), + # resolved as married by the relationship code + ("unknown", True, "married"), + ("no_marriage_history", True, "married"), + # unresolved, and separated under separated_is_married=False + ("unknown", False, "unclassified"), + ("no_marriage_history", False, "unclassified"), + ("separated", False, "unclassified"), + ], +) +def test_marital_status_4_maps_to_the_reports_marital_rows( + status, married, expected +): + """Specification section 9 (second referee S2): Tables 19 and 21 carry + Married, Widowed, Divorced and Never married (cleared extract); every + other state is unclassified and enters no marital cell.""" + + assert age67.marital_status_4(status, married) == expected + assert age67.MARITAL_STATUS_4 == ( + "married", + "widowed", + "divorced", + "never_married", + ) + assert age67.UNCLASSIFIED_MARITAL_STATUS == "unclassified" + + +def test_a_separated_member_is_married_or_unclassified(): + """Specification section 9: separated counts as married (F12); under + the unregistered separated_is_married=False the member is unclassified. + + Invented: 1001's marriage (1965) ends in a separation dated 2000; + he is observed in 2009 (income year 2008). + """ + + import dataclasses + + inputs = _inputs() + history = inputs.marriage_history.copy() + mask = history["person_id"].eq(1001) + history.loc[mask, "how_ended"] = "separated" + history.loc[mask, "last_known_status"] = "separated" + history.loc[mask, "end_year"] = 2000 + history.loc[mask, "separation_year"] = 2000 + inputs = dataclasses.replace(inputs, marriage_history=history) + for flag, status, four in ( + (True, "married", "married"), + (False, "separated", "unclassified"), + ): + obs = age67.build_age67_cohort( + inputs, age67.Age67Spec(separated_is_married=flag) + ).observations.set_index("observation_id") + assert obs.loc["1001:2009", "marital_status"] == status + assert obs.loc["1001:2009", "marital_status_4"] == four def test_annuitant_ages_use_derived_birth_years(): diff --git a/tests/estimates/test_adjusted_poverty.py b/tests/estimates/test_adjusted_poverty.py index 2a555b2a..642ce2b9 100644 --- a/tests/estimates/test_adjusted_poverty.py +++ b/tests/estimates/test_adjusted_poverty.py @@ -90,6 +90,9 @@ def _row(observation_id: str, **overrides) -> dict: row.update( observation_id=observation_id, family_unit_id=observation_id, + # an invented member who turns 67 in income year 2010, so the + # primary's 2004 cut start reaches it + birth_year=1943, income_year=2010, member_role="head", member_age=2, @@ -591,16 +594,18 @@ def test_ssi_offset_for_existing_recipients(): def test_cut_start_year_leaves_earlier_age67_years_uncut(): - """Row U6: with a 2004 start, the 1936 birth year (67 in 2003) is uncut. + """Primary (2004, second referee S5): the 1936 birth year (67 in 2003) + is uncut; the unregistered ``cut_start_year=None`` cuts everyone. INVENTED rows, threshold T = 1,000 (one person, 65+ row): - * "1936": B = 1,050 (not poor). Primary: cut 0.13 * 1,050 = 136.5, - R = 913.5 (poor). U6: 1936 + 67 = 2003 < 2004, no cut, R = 1,050. + * "1936": B = 1,050 (not poor). ``None``: cut 0.13 * 1,050 = 136.5, + R = 913.5 (poor). Primary: 1936 + 67 = 2003 < 2004, no cut, + R = 1,050. * "1937": 1937 + 67 = 2004 >= 2004, cut under both: R = 913.5. * "ssi": birth year 1936, SSI recipient with SS 1,000, SSI 100: - primary offset (1,000 - 240) - (870 - 240) = 130 (cap 500); U6 has - no cut, so no fall in countable income and no offset. + ``None`` offset (1,000 - 240) - (870 - 240) = 130 (cap 500); the + primary has no cut, so no fall in countable income and no offset. """ rows = [ @@ -615,29 +620,30 @@ def test_cut_start_year_leaves_earlier_age67_years_uncut(): total_family_income=1100, ), ] + assert ap.AdjustedPovertySpec().cut_start_year == 2004 + everyone = _run(*rows, cut_start_year=None) + assert everyone["cut_applies"].all() + assert everyone.loc["1936", "cut"] == pytest.approx(136.5) + assert everyone.loc["1936", "reform_income"] == pytest.approx(913.5) + assert everyone.loc["1936", "poor_reform"] + assert everyone.loc["ssi", "ssi_offset"] == pytest.approx(130.0) primary = _run(*rows) - assert primary["cut_applies"].all() - assert primary.loc["1936", "cut"] == pytest.approx(136.5) - assert primary.loc["1936", "reform_income"] == pytest.approx(913.5) - assert primary.loc["1936", "poor_reform"] - assert primary.loc["ssi", "ssi_offset"] == pytest.approx(130.0) - u6 = _run(*rows, cut_start_year=2004) - assert u6["cut_applies"].to_dict() == { + assert primary["cut_applies"].to_dict() == { "1936": False, "1937": True, "ssi": False, } - assert u6.loc["1936", "cut"] == 0.0 - assert u6.loc["1936", "reform_income"] == pytest.approx(1050.0) - assert not u6.loc["1936", "poor_reform"] - assert u6.loc["1937", "reform_income"] == pytest.approx(913.5) - assert u6.loc["1937", "poor_reform"] - assert u6.loc["ssi", "ssi_offset"] == 0.0 - assert u6.loc["ssi", "reform_income"] == pytest.approx( - u6.loc["ssi", "baseline_income"] + assert primary.loc["1936", "cut"] == 0.0 + assert primary.loc["1936", "reform_income"] == pytest.approx(1050.0) + assert not primary.loc["1936", "poor_reform"] + assert primary.loc["1937", "reform_income"] == pytest.approx(913.5) + assert primary.loc["1937", "poor_reform"] + assert primary.loc["ssi", "ssi_offset"] == 0.0 + assert primary.loc["ssi", "reform_income"] == pytest.approx( + primary.loc["ssi", "baseline_income"] ) # full static recomputation: an uncut unit cannot become newly eligible - u3 = _run(*rows, cut_start_year=2004, ssi_rule="full_static_recomputation") + u3 = _run(*rows, ssi_rule="full_static_recomputation") assert (u3.loc[["1936", "ssi"], "ssi_new"] == 0).all() @@ -805,17 +811,19 @@ def test_rows_are_validated(): _run(_row("a", member_role="lodger")) -def test_u6_cut_follows_each_members_age67_year_within_a_family(): +def test_u1_cut_follows_each_members_age67_year_within_a_family(): """Specification section 9: B, R and T are family-level under - ``fu_head_rule`` except in U4 and U6. Regression (independent review, - 2026-09-24): the U6 exception was not stated. + ``fu_head_rule`` except in U4 and under U1, where the primary's 2004 + cut start follows each member's age-67 year. Regression (independent + review, 2026-09-24): the exception was not stated. INVENTED: one family unit "f" with family Social Security 1,000 and money income 1,000 holds two cohort members, born 1936 (the head) and - 1938 (the wife). Primary: both are cut, 0.13 * 1,000 = 130, R = 870. - U6 (start 2004): 1936 + 67 = 2003 < 2004, so the head's observation is + 1938 (the wife), as U1 observes them in income year 2004. Primary + (start 2004): 1936 + 67 = 2003 < 2004, so the head's observation is uncut (R = 1,000); 1938 + 67 = 2005 >= 2004, so the wife's is cut - (R = 870). B and T stay equal across the two members. + (0.13 * 1,000 = 130, R = 870). ``cut_start_year=None``: both are cut, + R = 870. B and T stay equal across the two members. """ rows = [ @@ -840,11 +848,13 @@ def test_u6_cut_follows_each_members_age67_year_within_a_family(): wife_present=True, ), ] + everyone = _run(*rows, cut_start_year=None) + assert everyone["reform_income"].tolist() == pytest.approx([870.0, 870.0]) primary = _run(*rows) - assert primary["reform_income"].tolist() == pytest.approx([870.0, 870.0]) - u6 = _run(*rows, cut_start_year=2004) - assert u6["cut_applies"].to_dict() == {"head": False, "wife": True} - assert u6.loc["head", "reform_income"] == pytest.approx(1000.0) - assert u6.loc["wife", "reform_income"] == pytest.approx(870.0) + assert primary["cut_applies"].to_dict() == {"head": False, "wife": True} + assert primary.loc["head", "reform_income"] == pytest.approx(1000.0) + assert primary.loc["wife", "reform_income"] == pytest.approx(870.0) for column in ("baseline_income", "threshold", "annuity"): - assert u6.loc["head", column] == u6.loc["wife", column], column + assert ( + primary.loc["head", column] == primary.loc["wife", column] + ), column diff --git a/tests/estimates/test_uniform_cut_tabulation.py b/tests/estimates/test_uniform_cut_tabulation.py index a177f259..8e803d44 100644 --- a/tests/estimates/test_uniform_cut_tabulation.py +++ b/tests/estimates/test_uniform_cut_tabulation.py @@ -30,7 +30,13 @@ def _rows() -> pd.DataFrame: "family_unit_id": [10, 10, 20, 30, 40], "weight": [1.0, 1.0, 2.0, 4.0, 2.0], "sex": ["male", "female", "female", "male", "female"], - "married": [True, True, False, False, False], + "marital_status_4": [ + "married", + "married", + "widowed", + "divorced", + "unclassified", + ], "birth_year": [1941, 1943, 1941, 1945, 1945], "stratum": [1, 1, 1, 2, 2], "cluster": [1, 1, 2, 1, 2], @@ -69,31 +75,127 @@ def test_rates_and_changes_by_hand(): "men": (0.0, 80.0, 80.0), # women o2, o3, o5: W = 5; B: 2 -> 40; R: 1 + 2 -> 60 "women": (40.0, 60.0, 20.0), + # married o1, o2: W = 2; R: o2 (1) -> 50 "married": (0.0, 50.0, 50.0), - # non-married o3, o4, o5: W = 8; B 2 -> 25; R 6 -> 75 - "non_married": (25.0, 75.0, 50.0), + # widowed o3: W = 2, poor in both + "widowed": (100.0, 100.0, 0.0), + # divorced o4: W = 4, poor after the cut only + "divorced": (0.0, 100.0, 100.0), "men_married": (0.0, 0.0, 0.0), - "men_non_married": (0.0, 100.0, 100.0), + "men_divorced": (0.0, 100.0, 100.0), "women_married": (0.0, 100.0, 100.0), - "women_non_married": (50.0, 50.0, 0.0), + "women_widowed": (100.0, 100.0, 0.0), "birth_year_1941": (200 / 3, 200 / 3, 0.0), "birth_year_1943": (0.0, 100.0, 100.0), "birth_year_1945": (0.0, 400 / 6, 400 / 6), } - assert set(cells) == set(expected) + # o5 (female, unclassified) is in all and women but no marital cell; + # these cells hold nobody + empty = { + "never_married", + "men_widowed", + "men_never_married", + "women_divorced", + "women_never_married", + } + assert set(cells) == set(expected) | empty for name, (base, reform, delta) in expected.items(): assert cells[name]["baseline_rate"] == pytest.approx(base), name assert cells[name]["reform_rate"] == pytest.approx(reform), name assert cells[name]["delta"] == pytest.approx(delta), name + for name in empty: + assert not cells[name]["defined"], name + assert cells[name]["undefined_reason"] == "empty cell" assert cells["all"]["n_persons"] == 5 assert cells["all"]["n_family_units"] == 4 assert cells["all"]["weight_total"] == 10.0 assert cells["birth_year_1941"]["diagnostic"] - assert cells["men"]["scored_candidate"] - assert cells["men_married"]["optional"] + assert cells["birth_year_1941"]["role"] == "diagnostic" + assert cells["all"]["role"] == "headline" + assert cells["men"]["scored"] and cells["men"]["role"] == "scored" + assert cells["men_married"]["secondary"] + assert cells["men_married"]["role"] == "secondary" + assert cells["all"]["report_row"] == "Total" + assert cells["women_never_married"]["report_row"] == ( + "Gender and Marital Status: Female: Never married" + ) + assert cells["birth_year_1941"]["report_row"] is None + # unclassified members left out of each marital cell (o5, a woman) + assert cells["all"]["n_marital_unclassified_left_out"] is None + assert cells["women"]["n_marital_unclassified_left_out"] is None + assert cells["married"]["n_marital_unclassified_left_out"] == 1 + assert cells["women_widowed"]["n_marital_unclassified_left_out"] == 1 + assert cells["men_divorced"]["n_marital_unclassified_left_out"] == 0 + assert result["input_summary"]["n_marital_unclassified"] == 1 + assert result["input_summary"]["marital_status_4"] == { + "divorced": 1, + "married": 2, + "unclassified": 1, + "widowed": 1, + } + assert result["config"]["comparator_column"] == "1936-45" + assert result["config"]["unclassified_marital_cells"] == ( + "excluded_counted" + ) json.dumps(result) +def test_cells_are_the_reports_rows(): + """Specification section 9 (second referee S2, S3): Tables 19 and 21 + carry 36 rows (cleared extract); fifteen are computed and the other 21 + are named omissions.""" + + assert ut.DEFAULT_CELLS == ( + "all", + "women", + "men", + "married", + "widowed", + "divorced", + "never_married", + ) + assert ut.OPTIONAL_CELLS == ( + "women_married", + "women_widowed", + "women_divorced", + "women_never_married", + "men_married", + "men_widowed", + "men_divorced", + "men_never_married", + ) + computed = ut.DEFAULT_CELLS + ut.OPTIONAL_CELLS + assert set(ut.REPORT_ROWS) == set(computed) == set(ut.CELL_DEFINITIONS) + assert len(set(ut.REPORT_ROWS.values())) == 15 + assert ut.REPORT_ROWS["never_married"] == "Marital Status: Never married" + assert ut.REPORT_ROWS["men_widowed"] == ( + "Gender and Marital Status: Male: Widowed" + ) + omitted = ut.NOT_COMPUTED_REPORT_ROWS + assert [len(entry["rows"]) for entry in omitted.values()] == [ + 4, + 3, + 4, + 5, + 5, + ] + assert len(computed) + sum(len(e["rows"]) for e in omitted.values()) == 36 + assert [entry["section"] for entry in omitted.values()] == [ + "Race/Ethnicity", + "Education", + "Labor Force Experience", + "Lifetime Earnings (Own)", + "Lifetime Earnings (Shared)", + ] + assert all(entry["reason"] for entry in omitted.values()) + assert "not computed" not in " ".join(ut.CELL_DEFINITIONS.values()) + # the tabulation's marital values are the cohort builder's + from populace_dynamics.cohorts import age67 + + assert ut.MARITAL_STATUSES == age67.MARITAL_STATUS_4 + assert ut.UNCLASSIFIED == age67.UNCLASSIFIED_MARITAL_STATUS + + def test_design_standard_error_by_hand(): result = ut.tabulate_uniform_cut( _rows(), data_provenance="invented", design=_design(_rows()) @@ -322,7 +424,7 @@ def test_floor_is_undefined_with_one_family_unit(): def test_undefined_cells_are_reported_not_raised(): rows = _rows() rows["sex"] = "male" - rows.loc[rows["married"], "weight"] = 0.0 + rows.loc[rows["marital_status_4"].eq("married"), "weight"] = 0.0 result = ut.tabulate_uniform_cut( rows, data_provenance="invented", design=_design(rows) ) @@ -391,6 +493,8 @@ def test_registered_real_needs_rows_built_from_psid_files(kind): [ ("weight", -1.0), ("sex", "unknown"), + ("marital_status_4", "separated"), + ("marital_status_4", None), ("poor_reform", None), ("stratum", None), ], @@ -412,7 +516,12 @@ def test_pending_decisions_name_the_config_defaults(): assert decisions["design_se_domain"].alternatives == ("tabulated_rows",) assert "Q5" in decisions["design_se_domain"].default_basis assert decisions["cells"].default == list(config.cells) - assert "2(b)" in decisions["cells"].awaiting + assert "S2" in decisions["cells"].default_basis + assert "freeze" in decisions["cells"].awaiting + item = decisions["unclassified_marital_cells"] + assert item.default == config.unclassified_marital_cells + assert item.default == "excluded_counted" + assert set(ut.UNCLASSIFIED_MARITAL_CELL_RULES) == {"excluded_counted"} def test_config_validation(): @@ -422,3 +531,7 @@ def test_config_validation(): ut.TabulationConfig(cells=("all", "retirees")) with pytest.raises(ut.UniformCutTabulationError): ut.TabulationConfig(floor_seeds=(0, 0)) + with pytest.raises(ut.UniformCutTabulationError): + ut.TabulationConfig(cells=("all", "non_married")) + with pytest.raises(ut.UniformCutTabulationError): + ut.TabulationConfig(unclassified_marital_cells="non_married") diff --git a/tests/test_boomers2004_uniform_cut_spec.py b/tests/test_boomers2004_uniform_cut_spec.py index 69f92d20..7d8a75a5 100644 --- a/tests/test_boomers2004_uniform_cut_spec.py +++ b/tests/test_boomers2004_uniform_cut_spec.py @@ -51,7 +51,7 @@ def _section(text: str, number: str, following: str) -> str: def test_block_identity_and_status(block): assert block["specification"] == "boomers2004_uniform_cut_exercise2" - assert block["version"] == "u1-draft-4" + assert block["version"] == "u1-draft-5" assert block["status"] == "draft_for_referee" assert block["claim_class"]["awaiting"] == "d189" assert block["acceptance_rule"] is None @@ -63,7 +63,8 @@ def test_status_line_does_not_claim_ratification(text): status = text.split("- **Specification:**")[0] assert "draft for the referee" in status assert "Nothing here is ratified" in status - assert "`u1-draft-4`" in text.split("- **Plan item:**")[0] + assert "`u1-draft-5`" in text.split("- **Plan item:**")[0] + assert "Two" in status and "referee passes are recorded" in status def test_population_matches_the_builder(block): @@ -181,11 +182,43 @@ def test_income_concept_matches_the_spec_defaults(block): ) +def test_target_records_the_cleared_extract(block): + """Second referee S1 and S9: the comparator column, the Report's 36 + rows, whole-number cells and the cleared extract's path and hash.""" + + target = block["target"] + assert target["column"] == ut.COMPARATOR_COLUMN == "1936-45" + assert target["report_rows"] == 36 + assert target["report_rows"] == len(ut.DEFAULT_CELLS) + len( + ut.OPTIONAL_CELLS + ) + sum(len(e["rows"]) for e in ut.NOT_COMPUTED_REPORT_ROWS.values()) + assert target["printed_precision"] == "whole_numbers" + extract = target["definitions_extract"] + assert extract["file"] == ( + "EVID/exercise2-definitions-cleared-20260924.md" + ) + assert extract["sha256"] == ( + "a3978b683b4275424b6d12e9fe45f021fae277ccf9731a7883952564b6ed0384" + ) + local = EVIDENCE / Path(extract["file"]).name + if not local.is_file(): + pytest.skip("the cleared extract is outside this checkout") + assert hashlib.sha256(local.read_bytes()).hexdigest() == ( + extract["sha256"] + ) + + def test_cut_and_threshold_years_match_the_code(block): spec = ap.AdjustedPovertySpec() cut = block["cut"] assert cut["rate"] == spec.cut_rate == block["target"]["cut_rate"] - assert cut["start_year"] == spec.cut_start_year is None + # second referee S5: the Report's "beginning in 2004" + assert cut["start_year"] == spec.cut_start_year == 2004 + decision = {item.field: item for item in ap.pending_decisions()}[ + "cut_start_year" + ] + assert decision.alternatives == (None,) + assert "S5" in decision.default_basis assert cut["start_year_rule"] == ap.CUT_START_YEAR_RULE # not code parameters: the cut base and behaviour are fixed in # adjusted_incomes (specification section 16 lists them as pending) @@ -222,21 +255,34 @@ def test_statistic_and_comparison_match_the_tabulation(block): def test_rows_name_real_alternatives(block): rows = block["rows"] - assert set(rows) == { - "U0", - "U1", - "U2", - "U3", - "U4", - "U5", - "U6", - "U0-F", - "U7", - "U8", - "U9", - "U10", - "U-inst", + fallback = {f"{row}-F" for row in ("U2", "U3", "U4", "U5")} | { + f"{row}-F" for row in ("U8", "U9", "U10") } + # second referee S5 and S7 withdrew U6 and U-inst; S8 added the -F + # alternatives on U0-F's population + assert ( + set(rows) + == { + "U0", + "U1", + "U2", + "U3", + "U4", + "U5", + "U0-F", + "U7", + "U8", + "U9", + "U10", + } + | fallback + ) + assert set(track_u_rows.FALLBACK_ALTERNATIVES.values()) == fallback + for base, alternative in track_u_rows.FALLBACK_ALTERNATIVES.items(): + entry = dict(rows[alternative]) + assert entry.pop("population") == "birth_years_1941_1943_1945" + assert entry.pop("awaiting") == rows["U0-F"]["awaiting"] + assert entry == rows[base], alternative for row in rows.values(): if row.get("status") == "not_built": continue @@ -249,13 +295,6 @@ def test_rows_name_real_alternatives(block): assert rows["U1"]["population"] == "all_ten_birth_years" assert rows["U0-F"]["population"] == "birth_years_1941_1943_1945" assert rows["U0-F"]["on"] == "U0" - assert rows["U-inst"]["presence"] == "in_family_or_institution" - age67.Age67Spec( - presence=rows["U-inst"]["presence"], - institution_income_rule=rows["U-inst"]["institution_income_rule"], - ) - assert rows["U6"]["on"] == "U1" - ap.AdjustedPovertySpec(cut_start_year=rows["U6"]["cut_start_year"]) # every row but U7 is built, and the code's rows equal the block's assert [name for name, row in rows.items() if "status" in row] == ["U7"] assert track_u_rows.check_rows_against_block(block)["rows_equal_the_block"] @@ -263,8 +302,16 @@ def test_rows_name_real_alternatives(block): def test_cells_and_uncertainty_match_the_tabulation(block): cells = block["cells"] - assert cells["scored_candidates"] == list(ut.DEFAULT_CELLS) - assert cells["optional"] == list(ut.OPTIONAL_CELLS) + assert cells["column"] == ut.COMPARATOR_COLUMN + assert cells["scored"] == list(ut.DEFAULT_CELLS) + assert cells["secondary"] == list(ut.OPTIONAL_CELLS) + assert cells["unclassified_marital_cells"] == ( + ut.TabulationConfig().unclassified_marital_cells + ) + assert cells["not_computed"] == list(ut.NOT_COMPUTED_REPORT_ROWS) + assert block["comparison"]["comparator_interval"] == ( + "whole_number_rounding_level_0_5_difference_1" + ) uncertainty = block["uncertainty"] config = ut.TabulationConfig() assert uncertainty["draws"] == config.as_dict()["draws"] == 1 @@ -294,8 +341,10 @@ def test_named_deltas_equal_the_runner(text): def test_pending_decisions_are_listed_in_the_text(text): section = _section(text, "16", "17") assert "d189" in section - assert "decision 8" in section and "cut_start_year" in section - assert "institution_income_rule = family_of_record" in section + assert "decision 8" in section and "cut_start_year = 2004" in section + assert "the institution income rule (used by no registered row" in ( + " ".join(section.split()) + ) assert "d194" in section assert "fallback rule" in section ssi = next( @@ -312,6 +361,9 @@ def test_pending_decisions_are_listed_in_the_text(text): "unresolved marital status (relationship code)", "design SE (full-design domain)", "real rate (3 percent; 2 percent sensitivity)", + "cut start year (2004)", + "U2-F … U10-F", + "no card or ruling by Max on it was found", ): assert phrase in flat, phrase # every pending field of the code is listed by the code @@ -328,24 +380,42 @@ def test_pending_decisions_are_listed_in_the_text(text): assert name in fields -def test_referee_pass_is_recorded(block, text): - (entry,) = block["referee_passes"] - assert entry["object_version"] == "u1-draft-2" - assert entry["object_commit"] == "8d7e7431" - assert entry["required_changes"] == 17 - assert entry["applied_in"] == block["version"] - assert re.fullmatch(r"[0-9a-f]{64}", entry["sha256"]) +def test_referee_passes_are_recorded(block, text): + first, second = block["referee_passes"] + assert first["object_version"] == "u1-draft-2" + assert first["object_commit"] == "8d7e7431" + assert first["required_changes"] == 17 + assert first["applied_in"] == "u1-draft-4" + assert second["object_version"] == "u1-draft-4" + assert second["object_commit"] == "37a94ea2" + assert second["required_changes"] == 9 + assert second["applied_in"] == block["version"] section = _section(text, "17", "18") - assert entry["sha256"] in section + for entry in (first, second): + assert re.fullmatch(r"[0-9a-f]{64}", entry["sha256"]) + assert entry["sha256"] in section + assert second["object_blob_sha256"] in section + first_pass = section.split("### 17.2")[0] for change in [f"R{n}" for n in range(1, 18)] + [ f"O{n}" for n in range(1, 9) ]: - assert f"| {change} " in section, change - assert "**Declined:**" in section - report = EVIDENCE / Path(entry["report"]).name - if not report.is_file(): - pytest.skip("the referee report is outside this checkout") - assert hashlib.sha256(report.read_bytes()).hexdigest() == entry["sha256"] + assert f"| {change} " in first_pass, change + assert "**Declined:**" in first_pass + second_pass = section.split("### 17.3")[1] + for change in [f"S{n}" for n in range(1, 10)] + [ + f"O{n}" for n in range(1, 11) + ]: + assert f"| {change} " in second_pass, change + answers = section.split("### 17.2")[1].split("### 17.3")[0] + for number in range(9, 15): + assert f"{number}. **" in answers, number + for entry in (first, second): + report = EVIDENCE / Path(entry["report"]).name + if not report.is_file(): + pytest.skip("the referee reports are outside this checkout") + assert hashlib.sha256(report.read_bytes()).hexdigest() == ( + entry["sha256"] + ) def test_invented_cases_match_the_code(): @@ -413,14 +483,15 @@ def test_u3_is_never_called_a_bound(text): def test_section_9_names_where_members_of_a_family_differ(text): """Section 9's family-level claim holds under ``fu_head_rule`` except in - U4 (B and T by role) and U6 (the cut follows each member's age-67 - year, :func:`adjusted_poverty.cut_applies`); regression (independent - review, 2026-09-24): the U6 exception was missing.""" + U4 (B and T by role) and under U1, where the primary's 2004 start + follows each member's age-67 year (:func:`adjusted_poverty. + cut_applies`); regression (independent review, 2026-09-24): the + exception was missing (it was row U6's until u1-draft-5).""" section = " ".join(_section(text, "9", "10").split()) - assert "except in rows U4 and U6" in section + assert "except in row U4 and under U1" in section assert "Under row U4" in section - assert "Under row U6 the cut follows each member's own age-67" in section + assert "Under U1 the cut follows each member's own age-67" in section assert ap.CUT_START_YEAR_RULE == ( "cut_when_birth_year_plus_67_at_or_after_start" ) diff --git a/tests/track_u/test_invented_generator.py b/tests/track_u/test_invented_generator.py index 3a03d5e8..6c7733f1 100644 --- a/tests/track_u/test_invented_generator.py +++ b/tests/track_u/test_invented_generator.py @@ -174,8 +174,18 @@ def test_every_track_u_path_is_exercised(staged): assert (rows["wealth1"] < 0).any() assert (rows[list(family_income.SSI_CONCEPTS)].sum(axis=1) > 0).any() assert (rows["n_children"] > 0).any() + # every four-way marital status of specification section 9 + assert set(obs["marital_status_4"]) == { + *age67.MARITAL_STATUS_4, + age67.UNCLASSIFIED_MARITAL_STATUS, + } + # the unregistered institution option (U-inst withdrawn, u1-draft-5) u_inst = age67.build_age67_cohort( - staged, age67.Age67Spec(presence="in_family_or_institution") + staged, + age67.Age67Spec( + presence="in_family_or_institution", + institution_income_rule="family_of_record", + ), ) institution = u_inst.observations[u_inst.observations["in_institution"]] assert len(institution) > 0 diff --git a/tests/track_u/test_registered_script.py b/tests/track_u/test_registered_script.py index 907ae8f8..7e081315 100644 --- a/tests/track_u/test_registered_script.py +++ b/tests/track_u/test_registered_script.py @@ -111,7 +111,7 @@ def test_the_registered_state_passes(tmp_path): { "specification": { **RATIFIED, - "rows": {"U6": {"on": "U1", "awaiting": "decision 8"}}, + "rows": {"U2-F": {"awaiting": "the fallback rule"}}, } }, "awaited", diff --git a/tests/track_u/test_rows.py b/tests/track_u/test_rows.py index 0c9107ea..690cfc03 100644 --- a/tests/track_u/test_rows.py +++ b/tests/track_u/test_rows.py @@ -23,7 +23,7 @@ def block() -> dict: def test_the_code_rows_equal_the_committed_block(block): check = rows.check_rows_against_block(block) assert check["rows_equal_the_block"] - assert check["specification_version"] == block["version"] == "u1-draft-4" + assert check["specification_version"] == block["version"] == "u1-draft-5" assert check["rows_checked"] == sorted(rows.REGISTERED_ROWS) @@ -38,12 +38,12 @@ def test_each_row_changes_the_field_the_plan_names(): ) assert registered["U4"].income_spec().income_unit == "head_wife" assert registered["U5"].income_spec().asset_income_rule == "keep" - # U6 is defined on U1 and differs in the cut's start year only - assert registered["U6"].age67_spec() == registered["U1"].age67_spec() - assert registered["U6"].income_spec().cut_start_year == 2004 - assert registered["U6"].awaiting and "decision 8" in ( - registered["U6"].awaiting - ) + # u1-draft-5 (second referee S5, S7): U6 and U-inst are withdrawn; the + # primary cuts from 2004, so U1 carries the 1936 birth year uncut + assert "U6" not in registered and "U-inst" not in registered + assert registered["U0"].income_spec().cut_start_year == 2004 + assert registered["U1"].income_spec().cut_start_year == 2004 + assert registered["U0"].age67_spec().presence == "in_family" assert not registered["U7"].built assert "label investigation" in registered["U7"].not_built_reason assert registered["U8"].income_spec().threshold_rule == ( @@ -55,10 +55,6 @@ def test_each_row_changes_the_field_the_plan_names(): assert registered["U10"].income_spec().threshold_rule == ( "census_matrix_65plus" ) - inst = registered["U-inst"].age67_spec() - assert inst.presence == "in_family_or_institution" - assert inst.institution_income_rule == "family_of_record" - assert "not yet refereed" in registered["U-inst"].awaiting # U0-F is U0 on the fallback birth years; its headline status awaits # Max's ruling on the fallback rule fallback = registered[rows.FALLBACK_ROW] @@ -67,6 +63,37 @@ def test_each_row_changes_the_field_the_plan_names(): assert "fallback rule" in fallback.awaiting +def test_the_one_field_alternatives_are_registered_on_u0f_too(): + """Second referee S8: U2-F ... U10-F carry the field and value of U2 + ... U10 on U0-F's population, each awaiting the fallback rule.""" + + registered = rows.REGISTERED_ROWS + assert rows.FALLBACK_ALTERNATIVES == { + "U2": "U2-F", + "U3": "U3-F", + "U4": "U4-F", + "U5": "U5-F", + "U8": "U8-F", + "U9": "U9-F", + "U10": "U10-F", + } + for base, alternative in rows.FALLBACK_ALTERNATIVES.items(): + row = registered[alternative] + assert row.age67_spec() == age67.Age67Spec(row=rows.FALLBACK_ROW) + assert row.income_spec() == registered[base].income_spec() + assert row.field_changed == registered[base].field_changed + assert row.awaiting == registered[rows.FALLBACK_ROW].awaiting + assert row.built + # the -F rows need no wave without WEALTH1 + assert { + w + for alternative in rows.FALLBACK_ALTERNATIVES.values() + for _, w, _, _ in age67.observation_plan( + registered[alternative].age67_spec() + ) + } == {2009, 2011, 2013} + + def test_row_from_block_reads_populations_and_overrides(): assert rows.row_from_block( "U1", {"population": "all_ten_birth_years"} @@ -77,11 +104,24 @@ def test_row_from_block_reads_populations_and_overrides(): "awaiting": None, } parsed = rows.row_from_block( - "U6", {"on": "U1", "cut_start_year": 2004, "awaiting": "x"} + "X", {"on": "U1", "cut_start_year": 2004, "awaiting": "x"} ) assert parsed["age67"] == {"row": "U1"} assert parsed["income"] == {"cut_start_year": 2004} assert parsed["awaiting"] == "x" + assert rows.row_from_block( + "U3-F", + { + "population": "birth_years_1941_1943_1945", + "ssi_rule": "full_static_recomputation", + "awaiting": "y", + }, + ) == { + "age67": {"row": "U0-F"}, + "income": {"ssi_rule": "full_static_recomputation"}, + "built": True, + "awaiting": "y", + } assert not rows.row_from_block( "U7", {"financial_assets": "y", "status": "not_built"} )["built"] @@ -106,15 +146,16 @@ def test_row_from_block_reads_populations_and_overrides(): with pytest.raises(ValueError, match="not known"): rows.row_from_block("U1", {"population": "birth_years_1937"}) with pytest.raises(ValueError, match="unknown status"): - rows.row_from_block("U-inst", {"status": "counted_only"}) + rows.row_from_block("U8", {"status": "counted_only"}) @pytest.mark.parametrize( ("row_id", "edit"), [ ("U2", {"ssi_rule": "full_static_recomputation"}), - ("U6", {"cut_start_year": 2005}), - ("U-inst", {"institution_income_rule": "excluded"}), + ("U1", {"cut_start_year": 2005}), + ("U4-F", {"income_unit": "family_unit"}), + ("U9-F", {"population": "all_ten_birth_years"}), ("U0-F", {"population": "all_ten_birth_years"}), ("U7", {"status": None}), ], @@ -131,9 +172,13 @@ def test_a_block_that_differs_from_the_code_is_refused(block, row_id, edit): def test_a_resolved_awaiting_note_must_be_resolved_in_both_places(block): changed = copy.deepcopy(block) - del changed["rows"]["U6"]["awaiting"] - with pytest.raises(ValueError, match="row U6 differs"): + del changed["rows"]["U10-F"]["awaiting"] + with pytest.raises(ValueError, match="row U10-F differs"): rows.check_rows_against_block(changed) + withdrawn = copy.deepcopy(block) + withdrawn["rows"]["U6"] = {"on": "U1", "cut_start_year": 2004} + with pytest.raises(ValueError, match="differ from the block"): + rows.check_rows_against_block(withdrawn) missing = copy.deepcopy(block) del missing["rows"]["U10"] with pytest.raises(ValueError, match="differ from the block"): diff --git a/tests/track_u/test_runner.py b/tests/track_u/test_runner.py index 9f07ee88..362618d5 100644 --- a/tests/track_u/test_runner.py +++ b/tests/track_u/test_runner.py @@ -74,7 +74,20 @@ def test_every_row_runs_and_u7_is_reported_not_built(result): } == { "design_se_domain", "cells", + "unclassified_marital_cells", } + # second referee S3: the Report rows not computed are listed + assert result["comparator_column"] == "1936-45" + assert list(result["report_rows_not_computed"]) == list( + ut.NOT_COMPUTED_REPORT_ROWS + ) + assert ( + sum( + len(entry["rows"]) + for entry in result["report_rows_not_computed"].values() + ) + == 21 + ) for entry in result["rows"].values(): if entry["status"] == "computed": assert entry["tabulation"]["design"]["domain"] == ( @@ -89,7 +102,7 @@ def test_every_row_runs_and_u7_is_reported_not_built(result): def test_rows_change_only_what_they_register(result): rows_ = result["rows"] - u0, u1, u6 = rows_["U0"], rows_["U1"], rows_["U6"] + u0, u1 = rows_["U0"], rows_["U1"] # U2: no SSI response at all assert rows_["U2"]["income_concept_counts"]["n_ssi_offset_positive"] == 0 assert rows_["U2"]["income_concept_counts"]["n_ssi_new_positive"] == 0 @@ -100,24 +113,43 @@ def test_rows_change_only_what_they_register(result): # U5 removes no reported asset income counts = rows_["U5"]["income_concept_counts"] assert counts["n_asset_income_removed_nonzero"] == 0 - # U6 is U1 with the 1936 birth year uncut + # the primary's 2004 start (second referee S5): U1's 1936 birth year + # (67 in 2003) is uncut, every other observation is cut n_1936 = u1["population"]["observations_by_birth_year"]["1936"] - assert u6["population"] == u1["population"] - assert u6["income_concept_counts"]["n_cut_applies"] == ( - u1["income_concept_counts"]["n_cut_applies"] - n_1936 + assert u1["income_concept_counts"]["n_cut_applies"] == ( + u1["population"]["n_observations_tabulated"] - n_1936 ) - assert _all(u6)["baseline_rate"] == pytest.approx( - _all(u1)["baseline_rate"] + assert u0["income_concept_counts"]["n_cut_applies"] == ( + u0["population"]["n_observations_tabulated"] ) - assert _all(u6)["reform_rate"] <= _all(u1)["reform_rate"] - cells = {cell["cell"]: cell for cell in u6["tabulation"]["cells"]} + cells = {cell["cell"]: cell for cell in u1["tabulation"]["cells"]} assert cells["birth_year_1936"]["delta"] == 0.0 - # U-inst adds the institutionalized members to U0's population - inst = rows_["U-inst"]["population"] - assert inst["n_in_institution"] > 0 - assert inst["n_observations_tabulated"] == ( - u0["population"]["n_observations_tabulated"] + inst["n_in_institution"] + # no registered row admits institutions (U-inst withdrawn, S7) + for entry in rows_.values(): + if entry["status"] == "computed": + assert entry["population"]["n_in_institution"] == 0 + assert entry["age67_spec"]["presence"] == "in_family" + # the -F rows are their U0 rows on U0-F's population (S8) + for base, alternative in rows.FALLBACK_ALTERNATIVES.items(): + assert rows_[alternative]["income_spec"] == rows_[base]["income_spec"] + assert rows_[alternative]["population"] == ( + rows_["U0-F"]["population"] + ) + # the four-way marital status reaches the rows and the tabulation + status = u0["population"]["marital_status_4"] + assert set(status) == { + "married", + "widowed", + "divorced", + "never_married", + "unclassified", + } + cells = {cell["cell"]: cell for cell in u0["tabulation"]["cells"]} + assert cells["married"]["n_marital_unclassified_left_out"] == ( + status["unclassified"] ) + assert cells["married"]["report_row"] == "Marital Status: Married" + assert u0["population"]["spouse_age_source"] # U9 prices on the SSA 2004 table assert rows_["U9"]["life_table"] == "ssa_period_2004" assert u0["life_table"] == "nchs_2000" @@ -163,7 +195,12 @@ def test_official_concept_rates_by_hand(): "observation_id": ["a", "b", "c", "d"], "weight": [1.0, 1.0, 2.0, 4.0], "sex": ["male", "female", "female", "male"], - "married": [True, False, True, False], + "marital_status_4": [ + "married", + "unclassified", + "married", + "widowed", + ], } ) adjusted = pd.DataFrame( @@ -179,7 +216,12 @@ def test_official_concept_rates_by_hand(): assert cells["men"]["rate"] == pytest.approx(100.0) assert cells["women"]["rate"] == pytest.approx(0.0) assert cells["married"]["rate"] == pytest.approx(100 / 3) - assert cells["non_married"]["rate"] == pytest.approx(80.0) + # widowed: d only (weight 4, poor) + assert cells["widowed"]["rate"] == pytest.approx(100.0) + # b (unclassified) is in all and women but no marital cell + assert not cells["divorced"]["defined"] + assert cells["never_married"]["undefined_reason"] == "empty cell" + assert set(cells) == set(ut.DEFAULT_CELLS) with pytest.raises(runner.TrackURunError, match="do not match"): runner.official_concept_rates(members, adjusted.iloc[:3]) @@ -279,13 +321,29 @@ def test_the_fallback_rule_as_staged_today(params): statuses = {row: entry["status"] for row, entry in result["rows"].items()} assert statuses.pop("U0-F") == "computed" assert statuses.pop("U7") == "not_built" + # second referee S8: the -F alternatives are computed under the + # fallback; the U0 and U1 versions are blocked + for alternative in rows.FALLBACK_ALTERNATIVES.values(): + assert statuses.pop(alternative) == "computed", alternative + assert set(statuses) == { + "U0", + "U1", + "U2", + "U3", + "U4", + "U5", + "U8", + "U9", + "U10", + } assert set(statuses.values()) == {"blocked"} u0 = result["rows"]["U0"] assert u0["tabulation"] is None assert u0["blocked_waves"] == [2005, 2007] assert set(u0["population"]["blocked_by_birth_year"]) == {"1937", "1939"} - u6 = result["rows"]["U6"] - assert u6["blocked_waves"] == [2005, 2007] + assert result["rows"]["U1"]["blocked_waves"] == [2005, 2007] + assert result["rows"]["U2"]["blocked_waves"] == [2005, 2007] + assert result["rows"]["U2-F"]["tabulation"]["cells"] assert result["f17_diagnostics"]["row"] == "U0-F" tabulated = result["rows"]["U0-F"]["population"] assert set(tabulated["observations_by_birth_year"]) == { From 4666857605fec69632ee364a63b66ce78cdbdf8b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 24 Sep 2026 16:31:39 -0400 Subject: [PATCH 7/9] Record u1-draft-5 structural counts; correct the U-inst premise Recollected the structural counts under the u1-draft-5 code (b862d2d7; evidence track-u-structure-u1d5-20260924, counts only) and built each row with the withdrawn U-inst settings. Admitting institutions adds one U0 observation (a 1937-born person in no wave's in-family universe, in an institution in 2005) and none to U0-F, so the second referee's S7 premise that U-inst adds nothing holds for U0-F only. The withdrawal stands on the Census and Q9 reasons; section 3, section 11, the named delta (runner.NAMED_DELTAS and section 12), section 17 and the age67 and rows texts now say so. Section 3 records the four-way marital counts by sex, the annuitant age sources (no wave-age fallback), the code-90 husband counts (the family file records no AGE OF WIFE for them; question 11, O7) and the 2013 HEAD ANNUITIES and HEAD IRAS counts (question 14, O8); section 4 states U4's size rule for a female head with a legal husband. No income, threshold or poverty status was computed on PSID data. Co-Authored-By: Claude Opus 5.5 --- .../boomers2004_uniform_cut_comparison.md | 71 +++++++++++++++---- src/populace_dynamics/cohorts/age67.py | 18 +++-- .../uniform_cut_track_u/rows.py | 5 +- .../uniform_cut_track_u/runner.py | 4 +- 4 files changed, 74 insertions(+), 24 deletions(-) diff --git a/docs/design/boomers2004_uniform_cut_comparison.md b/docs/design/boomers2004_uniform_cut_comparison.md index 586312de..2eab6ebe 100644 --- a/docs/design/boomers2004_uniform_cut_comparison.md +++ b/docs/design/boomers2004_uniform_cut_comparison.md @@ -297,9 +297,40 @@ Four-way marital status (§9), summed from the same evidence's marital-status and resolution counts: U0 has 331 married (330 by the marriage history, 1 by relationship code), 44 widowed, 60 divorced, 14 never married and 34 unclassified; U0-F has 227, 17, 43, 9 and 24; U1 -has 917, 124, 164, 49 and 95. No U0 or U0-F target person was in an -institution at the observation wave (no `not_present:institution` -disposition); U1 has 3 (1942: 2, 1944: 1). +has 917, 124, 164, 49 and 95. No U0 or U0-F target person of the +primary universe was in an institution at the observation wave (no +`not_present:institution` disposition); U1 has 3 (1942: 2, 1944: 1). + +Recollected under `u1-draft-5`'s code (the same script at `b862d2d7`, +evidence `EVID/track-u-structure-u1d5-20260924/`, counts only): the +observation counts and every disposition equal the u1d4 evidence, and +the four-way counts above are confirmed. By sex, U0-F's secondary cells +hold: women married 97, widowed 14, divorced 30, never married 8, +unclassified 16; men married 130, widowed 3, divorced 13, never married +1, unclassified 8 (unweighted observations; the smallest secondary +cells hold one to three people, second referee O1). Every annuitant age +(the member's co-resident spouse, the head and the head's legal spouse) +came from a derived birth year in U0, U0-F and U1; none fell back to the +wave age (question 13). Question 11: in the computable observations +every code-90 legal husband is himself the cohort member (2 in U0, all in +U0-F's waves; 4 in U1), no cohort-member head has one, and in each of +those families the family file records no `AGE OF WIFE` and no +nonzero wife labor income or wife Social Security; so the family file +does not carry these legal husbands in its "wife" fields, and U4 gives +the husband himself the family-unit basis as an OFUM (§4). Question 14: +in 2013 (U0's 1945 birth year, 111 observations) 2 heads' families +report nonzero `HEAD ANNUITIES` and 5 nonzero `HEAD IRAS`, 7 in all (none +both); the pooled 2009 and 2011 item (`HEAD ANNUITIES`, annuities and +IRAs) is nonzero for 5 and 6. The withdrawn row U-inst's settings +(`presence = in_family_or_institution`, `family_of_record`), built for +each row in the same evidence (`institution-option-check.json`): U0 has +484 observations, one more than the primary: a 1937-born person in an +institution in the 2005 wave who is in no wave's in-family universe, so +not in the primary universe at all (admitting institutions grows the +universe the birth-year law runs over from 30,674 to 30,753 persons); +U0-F is unchanged at 320; +U1 has 1,353 (the 3 above plus that person). Under `excluded` every row +equals the primary. No income, threshold or poverty status was computed. ## 4. Income concept @@ -315,7 +346,9 @@ member keeps the family-unit basis. In code the co-resident wife or partner is the family file's `AGE OF WIFE` not 0 (`wife_present`), so for a female head whose legal spouse is a code-90 legal husband U4's size is 2 only where the family file records an age in the "wife" field -for him (question 11). +for him (question 11). On the staged files no computable cohort member +is such a head, and none of the families of the code-90 husbands who are +cohort members records an `AGE OF WIFE` (§3). **Tax treatment and co-resident basis.** The primary is pre-tax (`TOTAL FAMILY INCOME`). The Report states tax treatment only for @@ -794,12 +827,18 @@ different one (`scripts/run_track_u_registered.py --headline-row`). Institutionalized persons (sequence 51–59) are outside the universe and counted in the dispositions. Row U-inst of `u1-draft-3` and `-4` is -withdrawn: on the staged PSID none of U0's or U0-F's target persons was -in an institution at the observation wave (u1d4 structural counts, -dispositions; U1 has 3), so U-inst would add no institutionalized -observation, and the Census cannot determine poverty status for people -in institutional group quarters. The `family_of_record` rule stays in the -code as an unregistered option. +withdrawn: the Census cannot determine poverty status for people in +institutional group quarters, and `family_of_record` would measure a +family's poverty and assign it to someone the family's size and income +leave out (the second referee's Q9 answer). On the staged PSID the row +is nearly vacuous: none of the primary universe's U0 or U0-F target +persons was in an institution at the observation wave (u1d4 structural +counts, dispositions; U1 has 3), but admitting institutions also admits +persons who appear only through institution records, and that adds one +U0 observation (born 1937, in the 2005 wave, which the fallback blocks) +and none to U0-F (§3; the second referee's reason, that U-inst adds no +observation, holds for U0-F only). The `family_of_record` rule stays in +the code as an unregistered option. Row U6 of `u1-draft-3` and `-4` (the cut's start year on U1) is withdrawn: the primary now starts the cut in 2004 (§7), so U6 would @@ -815,7 +854,7 @@ found while building). Each bullet is the text of - PSID versus SIPP wealth measurement; - realized 2004-2012 history versus DYNASIM's 1992-based projection, including the 2008-09 asset shock for the 1941-45 cohorts at 67; - realized COLAs versus 2002 Trustees assumptions; -- immigrant under-coverage; institutionalized persons are outside the universe (none of U0's target persons was in an institution at the observation wave on the staged files); attrition; +- immigrant under-coverage; institutionalized persons are outside the universe (on the staged files admitting them would add 1 observation to U0's 483, born 1937, and none to U0-F's 320); attrition; - OFUM-owned assets inside family wealth; - the family's wealth stands in for an OFUM cohort member's own wealth (the Report's unit is the individual plus spouse, p. 24); - the Report's total income lists income from financial assets, imputed rent, Social Security, DB pensions, retirement-account income, earnings, SSI and non-spouse co-resident income (p. 31, cleared extract), and its poverty income is described only as differing from Census money income by the annuity (p. 24); the primary keeps every other PSID money-income source (veterans' pensions, unemployment and workers' compensation, TANF and other welfare, child support, alimony, help from relatives and others, miscellaneous transfers); @@ -1324,7 +1363,13 @@ answers replace the questions. institutionalized observation, and `family_of_record` would measure a family's poverty and assign it to someone the family's size and income leave out. Adopted: U-inst is withdrawn (S7) and `excluded` is the - code default of the unregistered option. + code default of the unregistered option. The count is corrected: the + referee's unverified caveat was about seed waves, but admitting + institutions also enlarges the universe the birth-year law runs over, + and that brings in one 1937-born U0 target person who is in no wave's + in-family universe, so U-inst would add 1 observation to U0's 483 + (none to U0-F's 320; U1 gains 4, not 3). The conceptual reasons + stand, so the withdrawal stands (§3, §11). 10. **The cut keyed on the age-67 year.** Answer: yes; the Report analyses each person "when they reach age 67" (p. 24) and cuts benefits "beginning in 2004" (pp. 37, 44, cleared extract). Adopted @@ -1384,7 +1429,7 @@ page. | S4 (comparator interval) | Applied as written. The difference bound follows from \|(a′ − b′) − (a − b)\| ≤ \|a′ − a\| + \|b′ − b\| ≤ 1 | | S5 (start year 2004) | Applied. Verified in the extract (PDF pp. 38 and 45, printed 37 and 44; the exposure arithmetic puts 1936 at 67 in 2003) and p. 24. `cut_start_year` defaults to 2004 with alternative `None`; U6 is withdrawn; §7, §9, §11, §13, §16 and the tests follow | | S6 (named deltas) | Applied as written. Verified: the p. 31 list and p. 24's contrast in the extract; the transfer items are `data/family_income.py` labels (for 2004: ER27954 TANF, ER27958 other welfare, ER27960 VA pension, ER27968 unemployment compensation, ER27970 workers compensation, ER27972 child support, ER27974 alimony, ER27976 and ER27978 help from relatives and others, ER27980 miscellaneous transfers). `runner.NAMED_DELTAS` changed with it | -| S7 (withdraw U-inst) | Applied. Verified: the u1d4 dispositions list no `not_present:institution` for any U0 or U0-F birth year and 3 for U1 (1942: 2; 1944: 1); the Census sentence on institutional group quarters (census.gov, fetched 2026-09-24). Beyond S7, the code default of `institution_income_rule` is now `excluded` (the Q9 answer), and §3 records a check of the referee's unverified caveat (the seed wave) | +| S7 (withdraw U-inst) | Applied, with its count corrected. Verified: the u1d4 dispositions list no `not_present:institution` for any U0 or U0-F birth year and 3 for U1 (1942: 2; 1944: 1); the Census sentence on institutional group quarters (census.gov, fetched 2026-09-24). Not verified as written: "U-inst would add no institutionalized observation". Building the rows with institutions admitted (§3) adds one U0 observation, a 1937-born person in no wave's in-family universe, because admitting institutions enlarges the birth-year law's universe; U0-F is unchanged. The §3, §11 and §12 texts say so instead of S7's "none of U0's target persons". The withdrawal stands on the Census and Q9 reasons. Beyond S7, the code default of `institution_income_rule` is now `excluded` (the Q9 answer) | | S8 (alternatives on U0-F) | Applied. `rows.FALLBACK_ALTERNATIVES` builds U2-F … U10-F from the U0 rows; each awaits the fallback rule. Under the fallback the runner computes them and blocks the U0 versions (test). Wording: §11 says U1 and U0-F "change the population itself" where S8 had "except U1 and U0-F" | | S9 (record the extract and this pass) | Applied. The builder-boundary sentence on the values scan also records what `RESTRICTED-FILES.md` said when re-read during this pass | diff --git a/src/populace_dynamics/cohorts/age67.py b/src/populace_dynamics/cohorts/age67.py index fbdb2d66..063f0b0f 100644 --- a/src/populace_dynamics/cohorts/age67.py +++ b/src/populace_dynamics/cohorts/age67.py @@ -31,8 +31,10 @@ positive cross-section weight. Institutionalized persons (sequence 51-59) are outside it and counted in the dispositions. Row U-inst, which added them, was withdrawn in the specification's u1-draft-5 (second -referee, S7: none of U0's target persons was in an institution at the -observation wave on the staged PSID). The unregistered code option +referee, S7 and Q9: the Census cannot determine poverty status for people +in institutional group quarters; on the staged PSID admitting them adds +one U0 observation, born 1937, and none to U0-F). The unregistered code +option ``presence="in_family_or_institution"`` still admits them, and ``institution_income_rule`` (pending the freeze; no registered row reads it) then decides their income. The PSID collects no income for an @@ -405,11 +407,13 @@ def pending_decisions() -> tuple[psid2010.PendingDecision, ...]: ("in_family_or_institution",), "plan F2; row U-inst (institutions admitted) was withdrawn in " "u1-draft-5 (second referee, boomers2004-referee-2-20260924.md, " - "S7 and Q9): on the staged PSID none of U0's or U0-F's target " - "persons was in an institution at the observation wave, and the " - "Census cannot determine poverty status for people in " - "institutional group quarters; the alternative is an " - "unregistered code option", + "S7 and Q9): the Census cannot determine poverty status for " + "people in institutional group quarters, and on the staged PSID " + "admitting institutions adds one U0 observation (a 1937-born " + "person in no wave's in-family universe, in an institution in " + "the 2005 wave) and none to U0-F (structural counts, " + "u1-draft-5); " + "the alternative is an unregistered code option", freeze, ), psid2010.PendingDecision( diff --git a/src/populace_dynamics/uniform_cut_track_u/rows.py b/src/populace_dynamics/uniform_cut_track_u/rows.py index 30b73670..c3b66611 100644 --- a/src/populace_dynamics/uniform_cut_track_u/rows.py +++ b/src/populace_dynamics/uniform_cut_track_u/rows.py @@ -16,8 +16,9 @@ Rows U6 (the cut's start year, on U1) and U-inst (institutions admitted) of u1-draft-3 and -4 are withdrawn in u1-draft-5 (second referee, S5 and S7): the primary now starts the cut in 2004 (``cut_start_year``), so U6 -would equal U1, and no U0 target person was in an institution at the -observation wave on the staged PSID. +would equal U1; the Census cannot determine poverty status for people in +institutional group quarters, and on the staged PSID U-inst would add one +U0 observation (born 1937) and none to U0-F. :data:`HEADLINE_RULE` is the specification's fallback rule (pending Max, with plan section 10 decision 3): U0 is the headline when the 2005 and diff --git a/src/populace_dynamics/uniform_cut_track_u/runner.py b/src/populace_dynamics/uniform_cut_track_u/runner.py index 89283e76..f20b8e08 100644 --- a/src/populace_dynamics/uniform_cut_track_u/runner.py +++ b/src/populace_dynamics/uniform_cut_track_u/runner.py @@ -96,8 +96,8 @@ "including the 2008-09 asset shock for the 1941-45 cohorts at 67", "realized COLAs versus 2002 Trustees assumptions", "immigrant under-coverage; institutionalized persons are outside the " - "universe (none of U0's target persons was in an institution at the " - "observation wave on the staged files); attrition", + "universe (on the staged files admitting them would add 1 observation " + "to U0's 483, born 1937, and none to U0-F's 320); attrition", "OFUM-owned assets inside family wealth", "the family's wealth stands in for an OFUM cohort member's own wealth " "(the Report's unit is the individual plus spouse, p. 24)", From 1fa6e6066bfe738960f394b7bbe6dd063f87f7ed Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 24 Sep 2026 18:56:39 -0400 Subject: [PATCH 8/9] Record the u1-draft-5 invented dry run and recollect the tier manifest The builder's last step, left uncommitted when its session hit a usage limit: spec section 14 lists EVID/track-u-dry-run-u1d5-20260924/ (run at 46668576), and the tier manifest counts the tests 46668576 added (unit 3,125; total 7,519). The tier-policy test passes on a full collection; black and ruff are clean. Co-Authored-By: Claude Opus 5.5 --- docs/design/boomers2004_uniform_cut_comparison.md | 7 +++++-- tests/README-tiers.md | 4 ++-- tests/tier_counts.json | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/design/boomers2004_uniform_cut_comparison.md b/docs/design/boomers2004_uniform_cut_comparison.md index 2eab6ebe..ad15eb71 100644 --- a/docs/design/boomers2004_uniform_cut_comparison.md +++ b/docs/design/boomers2004_uniform_cut_comparison.md @@ -974,8 +974,11 @@ reachability guard: with invented thresholds and the committed life tables and SSI capture, and the fallback rule on the invented cohort as staged today; evidence headed "INVENTED DATA - NOT A COMPARISON", - `EVID/track-u-dry-run-20260924/` (`u1-draft-3`) and - `EVID/track-u-dry-run-u1d4-20260924/` (`u1-draft-4`). + `EVID/track-u-dry-run-20260924/` (`u1-draft-3`), + `EVID/track-u-dry-run-u1d4-20260924/` (`u1-draft-4`) and + `EVID/track-u-dry-run-u1d5-20260924/` (this draft, at `46668576`: the + fallback staging makes U0-F the headline, blocks the rows defined on U0 + or U1 and computes the -F rows). - `scripts/run_track_u_registered.py` (U10 entry point): refuses unless the pointer is an issue #42 comment, `HEAD` is the registered commit on a clean tree, §15 is ratified with nothing awaiting and nothing diff --git a/tests/README-tiers.md b/tests/README-tiers.md index 516ba956..9864fde0 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -38,9 +38,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 3,109 | +| `unit` | 3,125 | | `artifact` | 2,848 | | `integration_psid` | 863 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 163 | -| **Total** | **7,503** | +| **Total** | **7,519** | diff --git a/tests/tier_counts.json b/tests/tier_counts.json index f588290d..3ac9b370 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 3109, + "unit": 3125, "artifact": 2848, "integration_psid": 863, "reproduction_legacy": 520, From bc82604e1c11e1482548ab413eaa214ffb9f45ae Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 24 Sep 2026 19:13:23 -0400 Subject: [PATCH 9/9] Restore the decision-8 awaiting in the Track U block; correct three u1-draft-5 texts Independent review of the second referee pass (37a94ea2..1fa6e606). - Section 15: withdrawing row U6 removed the block's only "awaiting" for plan decision 8, which adjusted_poverty.pending_decisions (the cut_start_year decision) and section 16 still list as awaiting Max's confirmation of the scorecard's "from 2004" wording. The cut block now carries that "awaiting", so the registered run's ratification scan refuses the block until Max rules. A new test holds every code decision awaiting Max (ssi_rule, cut_start_year) to an "awaiting" key the scan finds; it fails on 1fa6e606. - Builder boundary: it said RESTRICTED-FILES.md did not yet hold the 2026-09-24 16:10 changelog entry for the cleared extract's values scan; the file now holds it (values scan clean, 0 genuine leaks, 0 unclassified). A test holds the sentence to the file; it fails on 1fa6e606. - Section 16 lists the decisions awaiting Max without a card one per line, so decision 8 and the fallback rule no longer read as one item; section 11 no longer says every registered row is computed (under the fallback rule the rows defined on U0 or U1 are reported as blocked); section 14 records that the invented dry run re-run at 1fa6e606 equals the u1d5 evidence except for the commit and output path. - Tier manifest recollected (unit 3,127; total 7,521). No rule, default or row changed. No income, threshold, annuity or poverty status was computed on PSID data. Co-Authored-By: Claude Opus 5.5 --- .../boomers2004_uniform_cut_comparison.md | 98 ++++++++++++------- tests/README-tiers.md | 4 +- tests/test_boomers2004_uniform_cut_spec.py | 71 +++++++++++++- tests/tier_counts.json | 2 +- 4 files changed, 137 insertions(+), 38 deletions(-) diff --git a/docs/design/boomers2004_uniform_cut_comparison.md b/docs/design/boomers2004_uniform_cut_comparison.md index ad15eb71..96a1de41 100644 --- a/docs/design/boomers2004_uniform_cut_comparison.md +++ b/docs/design/boomers2004_uniform_cut_comparison.md @@ -3,13 +3,15 @@ - **Status:** draft for the referee. Nothing here is ratified. Two referee passes are recorded (§17). The first was applied in `u1-draft-4`; this draft applies the second, whose verdict was that the - draft is ratifiable after nine required changes (S1–S9, §17.3). No - independent check of this draft's application of them has been made - yet. Max has not ruled on exercise 2 (cos decision d189, open, default - "yes to Track U with the proposed SSI rule; downloads needed from - Max") or on the fallback rule (§11). Every choice that awaits him or - the specification freeze is an explicit parameter in the code with the - recommended default, and §16 lists them. + draft is ratifiable after nine required changes (S1–S9, §17.3). An + independent review (2026-09-24) checked this draft's application of + them against their sources, the code and the tests; its corrections + are listed in §19. Max has not ruled on exercise 2 (cos decision + d189, open, default "yes to Track U with the proposed SSI rule; + downloads needed from Max") or on the fallback rule (§11). Every + choice that awaits him or the specification freeze is an explicit + parameter in the code with the recommended default, and §16 lists + them. - **Specification:** `boomers2004_uniform_cut_exercise2`, version `u1-draft-5`, drafted 2026-09-24. §19 is the changelog. - **Plan item:** U1 of the Track U plan, @@ -54,11 +56,14 @@ and the builder opened none of those pages (nor any page of the Report PDF for this draft). When the second referee wrote, no values scan of the extract was recorded (`RESTRICTED-FILES.md`, 2026-09-24 14:35). - As re-read during this pass, `RESTRICTED-FILES.md` lists the extract - as having passed the validation lane's values scan with 0 genuine leaks - at this hash, citing a changelog entry for 2026-09-24 16:10 that the - file did not yet hold. §18 lists what this draft read and what it did - not verify. + `RESTRICTED-FILES.md` now records one (its changelog entry for + 2026-09-24 16:10, read in the independent review of this draft): the + validation lane's values scan of the extract at this hash is clean, + with 0 genuine leaks and 0 unclassified tokens among the 1,103 numeric + tokens it scanned. (The builder of this draft reported that, when it + re-read the file, the verdict was already listed under "Cleared for + builders" but the changelog entry it cites was not yet there.) §18 + lists what this draft read and what it did not verify. ## 1. Target @@ -786,9 +791,11 @@ headline row (§11): Each alternative differs from U0 (or, for the -F rows, from U0-F) in one field; U1 and U0-F change the population itself. U0 is the headline row unless the fallback rule below makes U0-F the headline; no row may be -promoted after results exist. Every registered row is computed in the -same one-shot run and published with the others. A row that is not built when the registration is posted is -removed from this table and listed as a named omission. +promoted after results exist. Every registered row is run in the same +one-shot run (computed, or reported as blocked with its counts under the +fallback rule below) and published with the others. A row that is not +built when the registration is posted is removed from this table and +listed as a named omission. **Fallback rule (pending Max, with plan §10 decision 3).** If the 2005 and 2007 wealth supplements are staged, adjudicated and read before the @@ -978,7 +985,9 @@ reachability guard: `EVID/track-u-dry-run-u1d4-20260924/` (`u1-draft-4`) and `EVID/track-u-dry-run-u1d5-20260924/` (this draft, at `46668576`: the fallback staging makes U0-F the headline, blocks the rows defined on U0 - or U1 and computes the -F rows). + or U1 and computes the -F rows; the independent review re-ran it at + `1fa6e606`, and its output equals this evidence except for the + recorded commit and output path). - `scripts/run_track_u_registered.py` (U10 entry point): refuses unless the pointer is an issue #42 comment, `HEAD` is the registered commit on a clean tree, §15 is ratified with nothing awaiting and nothing @@ -1107,7 +1116,8 @@ holds it to the code's defaults. "base": "all_social_security_of_the_unit", "behavior": "none", "start_year": 2004, - "start_year_rule": "cut_when_birth_year_plus_67_at_or_after_start" + "start_year_rule": "cut_when_birth_year_plus_67_at_or_after_start", + "awaiting": "Max (plan section 10 decision 8: confirming the scorecard's 'from 2004' wording)" }, "ssi": { "rule": "offset_existing_recipients", @@ -1248,22 +1258,27 @@ d189's text describes Track U as a "Python income concept, not Axiom", so a yes on d189 may be read as covering plan decision 4; confirm when ruling. -**Awaiting Max, not yet on a decision card (plan §10):** decision 2 -(the definitions extract: it exists and was cleared for builders on -2026-09-24; no card or ruling by Max on it was found), decision 4 -(Python SSI arithmetic labelled "not Axiom"), decision 6 (acceptance rule; -default none, report gaps, §10a), decision 7 (ratify this specification -by merge; post the #42 registration), decision 8 (the scorecard's "from -2004"): the cleared extract supplies the source, the Report's "beginning -in 2004" (pp. 37 and 44); the primary now uses `cut_start_year = 2004`, -which moves only U1's 1936 birth year; Max's confirmation of the -scorecard wording remains open, and the fallback rule of §11 (with -decision 3), including the recommended default that each one-field -alternative is also registered on U0-F's population (U2-F … U10-F), so -the registration carries its alternatives whether or not the supplements -are staged; Max may instead keep the alternatives on U0 only (blocked -under the fallback). Approval to download the Census threshold files is -on its own card (cos decision d194, open). +**Awaiting Max, not yet on a decision card (plan §10):** + +- decision 2 (the definitions extract: it exists and was cleared for + builders on 2026-09-24; no card or ruling by Max on it was found); +- decision 4 (Python SSI arithmetic labelled "not Axiom"); +- decision 6 (acceptance rule; default none, report gaps, §10a); +- decision 7 (ratify this specification by merge; post the #42 + registration); +- decision 8 (the scorecard's "from 2004"): the cleared extract supplies + the source, the Report's "beginning in 2004" (pp. 37 and 44); the + primary now uses `cut_start_year = 2004`, which moves only U1's 1936 + birth year; Max's confirmation of the scorecard wording remains open + (§15 `cut.awaiting`); +- the fallback rule of §11 (with decision 3), including the recommended + default that each one-field alternative is also registered on U0-F's + population (U2-F … U10-F), so the registration carries its + alternatives whether or not the supplements are staged; Max may + instead keep the alternatives on U0 only (blocked under the fallback). + +Approval to download the Census threshold files is on its own card (cos +decision d194, open). **Awaiting the specification freeze (defaults shown):** cut rate (0.13; the code lists it), cut base (all Social Security of the unit; not a code @@ -1527,6 +1542,23 @@ Heeringa 2008); any SCF wealth aggregate (none is committed or saved). ## 19. Changelog +- `u1-draft-5`, independent review corrections (2026-09-24; no rule, + default or row changed): §15's `cut` block gains an `awaiting` for + plan decision 8 (Max's confirmation of the scorecard's "from 2004" + wording), which the withdrawal of row U6 had left with no + machine-readable `awaiting` although the code's + `cut_start_year` decision and §16 still list it as awaiting Max; a + test now holds every code decision awaiting Max to an `awaiting` + key the registered run's ratification scan finds. The builder + boundary records the extract's values-scan verdict as + `RESTRICTED-FILES.md` now holds it (changelog entry 2026-09-24 + 16:10), in place of a sentence saying that entry was missing. §16's + list of decisions awaiting Max without a card is set out one item + per line, so decision 8 and the fallback rule no longer read as one + item; §11 no longer says every registered row is computed (under the + fallback rule the rows defined on U0 or U1 are reported as blocked). + Counts only; no income, threshold or poverty status was + computed on PSID data. - `u1-draft-5` (2026-09-24, second referee pass applied, §17.3): `origin/master` (`088772f1`) merged in; §1 states the target as the cleared exercise-2 definitions extract defines it (the "1936-45" diff --git a/tests/README-tiers.md b/tests/README-tiers.md index 9864fde0..5522c224 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -38,9 +38,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 3,125 | +| `unit` | 3,127 | | `artifact` | 2,848 | | `integration_psid` | 863 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 163 | -| **Total** | **7,519** | +| **Total** | **7,521** | diff --git a/tests/test_boomers2004_uniform_cut_spec.py b/tests/test_boomers2004_uniform_cut_spec.py index 7d8a75a5..ef44dcc9 100644 --- a/tests/test_boomers2004_uniform_cut_spec.py +++ b/tests/test_boomers2004_uniform_cut_spec.py @@ -67,6 +67,32 @@ def test_status_line_does_not_claim_ratification(text): assert "Two" in status and "referee passes are recorded" in status +def test_builder_boundary_records_the_extract_values_scan(text): + """The builder boundary states the cleared extract's values-scan + verdict as ``RESTRICTED-FILES.md`` records it. + + Regression (independent review of u1-draft-5, 2026-09-24): the + boundary said the governance file did not yet hold the 16:10 changelog + entry recording the scan; it does, so the sentence was stale. + """ + + boundary = " ".join( + text.split("- **Builder boundary:**")[1] + .split("## 1. Target")[0] + .split() + ) + assert "did not yet hold" not in boundary + assert "values scan of the extract at this hash is clean" in boundary + assert "0 genuine leaks" in boundary + assert "2026-09-24 16:10" in boundary + restricted = EVIDENCE / "RESTRICTED-FILES.md" + if not restricted.is_file(): + pytest.skip("RESTRICTED-FILES.md is outside this checkout") + governance = " ".join(restricted.read_text(encoding="utf-8").split()) + assert "2026-09-24 16:10: exercise-2 values scan" in governance + assert "Exercise 2 values scan: values scan clean." in governance + + def test_population_matches_the_builder(block): population = block["population"] assert population["primary_birth_years"] == list(age67.PRIMARY_BIRTH_YEARS) @@ -244,13 +270,54 @@ def test_statistic_and_comparison_match_the_tabulation(block): assert comparison["acceptance"]["rule"] is None # the pending acceptance rule is an "awaiting" key the registered run's # ratification scan finds + module = _registered_script() + assert "comparison.acceptance.awaiting" in module._awaiting(block) + assert "population.headline.awaiting" in module._awaiting(block) + + +def _registered_script(): spec = importlib.util.spec_from_file_location( "_registered", ROOT / "scripts" / "run_track_u_registered.py" ) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) - assert "comparison.acceptance.awaiting" in module._awaiting(block) - assert "population.headline.awaiting" in module._awaiting(block) + return module + + +def test_every_choice_awaiting_max_is_awaited_in_the_block(block): + """Every code parameter whose pending decision awaits Max (other than + the freeze, Max's ratification by merge) carries an ``awaiting`` key in + the section 15 block, so the registered run's ratification scan refuses + the block until Max rules. + + Regression (independent review of u1-draft-5, 2026-09-24): withdrawing + row U6 removed the block's only ``awaiting`` for plan decision 8 (the + cut's start year, now the primary's ``cut_start_year = 2004``), which + ``adjusted_poverty.pending_decisions`` and section 16 still list as + awaiting Max's confirmation of the scorecard's "from 2004" wording. + """ + + where = {"ssi_rule": "ssi.awaiting", "cut_start_year": "cut.awaiting"} + awaiting_max = { + item.field + for decisions in ( + age67.pending_decisions(), + ap.pending_decisions(), + ut.pending_decisions(), + ) + for item in decisions + if item.awaiting.startswith("Max") + } + assert awaiting_max == set(where) + found = _registered_script()._awaiting(block) + for field, path in where.items(): + assert path in found, field + decision = {item.field: item for item in ap.pending_decisions()}[ + "cut_start_year" + ] + assert "decision 8" in decision.awaiting + assert "decision 8" in block["cut"]["awaiting"] + assert "d189" in block["ssi"]["awaiting"] def test_rows_name_real_alternatives(block): diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 3ac9b370..64b5de8c 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 3125, + "unit": 3127, "artifact": 2848, "integration_psid": 863, "reproduction_legacy": 520,