From d53f3fd45be946e193ff56aad4413417fedd22d9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 24 Sep 2026 02:28:35 -0400 Subject: [PATCH 1/4] Oracle: statutory benefit computation years (42 USC 415(b)(2)) ss.benefits.aime divides every worker's highest 35 indexed years by 420. 42 USC 415(b)(2) sets the number of benefit computation years to the elapsed years (calendar years after the later of 1950 and the year of attaining 21, before the year of attaining 62 or of earlier death) less 5, never fewer than 2. The two agree from the 1929 birth cohort on; Track C step 1 traced all 251 of its Axiom-vs-oracle AIME disagreements (people born before 1929) to that count. ss/benefits.py is inside the first-estimates birth-evidence source seal (reviewed commit 1c7f3540) and is called by the sealed ledger and the gate-2c couple-earnings proxy, so it is not edited. Instead: - New opt-in module ss/statutory_aime.py (quotes 415(b)(1), (b)(2)(A), (b)(2)(B)(i) and (iii), (b)(3)(A)(ii)(I), and 104(j)(2) of the 1972 amendments): elapsed years, benefit computation years (old-age, death before 62, and a disability onset with the one-fifth dropout capped at 5, 2-year minimum), the 415(b)(3) indexing year, the statutory AIME, and oracle_aime() with a named ComputationYears convention. LEGACY_FIXED_35 is the unchanged benefits.aime. It refuses workers attaining 62 before 1975 (104(j)(2)), a death after a disability entitlement, and history years before 1951. Child-care dropout years are not encoded. - scenario_benefits.eligibility_pia_for_clock takes computation_years, default STATUTORY. - Track A passes TRACK_A_COMPUTATION_YEARS = LEGACY_FIXED_35 for its oracle retirement level (Registration 13's arithmetic); approximate_pia stays on benefits.aime. - The new module is registered in POST_REVIEW_SOURCE_EXCLUSIONS, the pinned tuple and the reachability guard. - Tests: hand-computed counts and AIMEs (born 1913, 1917, 1920, 1928, 1929, 1935; attaining 21 after 1950; the 2-year minimum; death and disability cases), equality with benefits.aime for every birth year 1929-1995 on invented careers, a Track A legacy test, the sealed-ledger test under both conventions, and a source scan pinning every legacy user. Tier manifest: unit 2922, total 7249. Co-Authored-By: Claude Opus 5.5 --- scripts/first_estimates_birth_evidence.py | 4 + .../cola_track_a/benefits.py | 25 + src/populace_dynamics/scenario_benefits.py | 36 +- src/populace_dynamics/ss/statutory_aime.py | 410 +++++++++++++++++ tests/README-tiers.md | 4 +- tests/cola_track_a/test_assembly.py | 69 ++- .../estimates/test_birth_evidence_artifact.py | 8 + tests/ss/test_statutory_aime.py | 432 ++++++++++++++++++ tests/test_scenario_benefits.py | 35 +- tests/tier_counts.json | 2 +- 10 files changed, 1009 insertions(+), 16 deletions(-) create mode 100644 src/populace_dynamics/ss/statutory_aime.py create mode 100644 tests/ss/test_statutory_aime.py diff --git a/scripts/first_estimates_birth_evidence.py b/scripts/first_estimates_birth_evidence.py index 6b0121a7..5d65b0e5 100644 --- a/scripts/first_estimates_birth_evidence.py +++ b/scripts/first_estimates_birth_evidence.py @@ -222,6 +222,10 @@ Path("src/populace_dynamics/cola_track_a/opening.py"), Path("src/populace_dynamics/cola_track_a/runner.py"), Path("src/populace_dynamics/cola_track_a/statutory.py"), + # The opt-in statutory AIME (42 USC 415(b)(2) computation years) sits + # beside the sealed ss.benefits, which it calls but does not edit; the + # historical ledger never imports it. + Path("src/populace_dynamics/ss/statutory_aime.py"), ) POST_REVIEW_SHARED_SOURCE_BLOBS = { Path( diff --git a/src/populace_dynamics/cola_track_a/benefits.py b/src/populace_dynamics/cola_track_a/benefits.py index e76be70c..304e68dc 100644 --- a/src/populace_dynamics/cola_track_a/benefits.py +++ b/src/populace_dynamics/cola_track_a/benefits.py @@ -64,6 +64,18 @@ Levels the oracle does not compute (decision 2(b) and the A6 pre-eligibility-death hook) follow :class:`LevelPolicy`; :func:`approximate_pia` is the disclosed approximation. + +Benefit computation years (:data:`TRACK_A_COMPUTATION_YEARS`). Every +oracle AIME in this module divides by 35 years +(:attr:`~populace_dynamics.ss.statutory_aime.ComputationYears. +LEGACY_FIXED_35`, the unchanged ``ss.benefits.aime``), the arithmetic of +Track A's Registration 13 (``runs/replication_urban2010_cola_v1.json``, +registered commit ``002264d9``). 42 USC 415(b)(2) gives fewer years to +workers born before 1929 (elapsed years less 5); the Track C step 1 +comparison (2026-09-23) traced all 251 of its AIME disagreements, among +7486 people, to that count. The assembly keeps the count Registration 13 +used; moving a future Track A run to the statutory count is a +registration-level choice that has not been made. """ from __future__ import annotations @@ -90,8 +102,10 @@ from populace_dynamics.engine.loop import ProjectionResult from populace_dynamics.ss import benefits from populace_dynamics.ss.params import SSAParameters +from populace_dynamics.ss.statutory_aime import ComputationYears __all__ = [ + "TRACK_A_COMPUTATION_YEARS", "BenefitContext", "StateLookups", "PiaRecord", @@ -102,6 +116,11 @@ #: The oracle's PIA formula covers eligibility from 1979 (plan section 10). FIRST_ORACLE_ELIGIBILITY_YEAR = 1979 +#: The AIME computation-year convention of Track A's oracle retirement +#: level: the legacy fixed 35 that Registration 13 was computed with (module +#: docstring), not the statutory 415(b)(2) count. :func:`approximate_pia` +#: is always legacy. +TRACK_A_COMPUTATION_YEARS = ComputationYears.LEGACY_FIXED_35 _RETIREMENT_AGE = 62 _MONTHS = 12 _OPENING_AUX = ("survivor", "spouse", "other", "unclassified") @@ -163,6 +182,11 @@ def approximate_pia( It is not the statutory DI or pre-eligibility-death computation: the divisor is always 35 years (no elapsed or dropout years), so it understates short careers. It sets a weight, never a reform ratio. + It always calls the unchanged ``benefits.aime`` + (``ComputationYears.LEGACY_FIXED_35``), the approximation Track A + registered. The statutory counts for a disability onset or a death + before 62 exist as ``ss.statutory_aime.aime(..., disability_year=...)`` + or ``death_year=...``; using them here would change that approximation. """ kept = { @@ -257,6 +281,7 @@ def _level( history=self._history(person_id), birth_year=birth, params=self.ctx.params, + computation_years=TRACK_A_COMPUTATION_YEARS, ) else: self.cache[key] = approximate_pia( diff --git a/src/populace_dynamics/scenario_benefits.py b/src/populace_dynamics/scenario_benefits.py index 1d1d1386..fb8086e4 100644 --- a/src/populace_dynamics/scenario_benefits.py +++ b/src/populace_dynamics/scenario_benefits.py @@ -60,7 +60,9 @@ benefit level of a worker who died before eligibility is likewise a raising hook (:func:`survivor_of_preeligibility_death_benefit_level`). Callers may supply any PIA to the path functions; this module does not - say where a DI or survivor PIA comes from. + say where a DI or survivor PIA comes from. (The statutory computation + years for those cases exist in :mod:`populace_dynamics.ss.statutory_aime`; + neither hook uses them without a ruling.) * No family maximum, earnings test, recomputation, payment-month timing, whole-dollar payment rounding, or pre-1983 June-effective COLA timing. The annual convention is the sealed ledger's: the amount for payment @@ -90,6 +92,14 @@ plan names no specific approximation, so there is no proposed primary to default to; a caller may still pass any PIA to the path functions. +The age-62 AIME's benefit computation years are an explicit parameter of +:func:`eligibility_pia_for_clock` (``computation_years``). The default is +the statute, :attr:`~populace_dynamics.ss.statutory_aime.ComputationYears. +STATUTORY` (42 USC 415(b)(2): elapsed years less 5, fewer than 35 for +workers born before 1929). ``LEGACY_FIXED_35`` is the sealed ledger's +``benefits.aime`` (always 35), which Track A's Registration 13 used and +which the Track A assembly still passes explicitly. + One convention inside the ``ENTITLEMENT`` alternative is fixed here and not named in the plan, so the specification freeze (plan item A1) must confirm it: for a spouse or widow(er), "entitlement" is the insured @@ -112,8 +122,9 @@ from populace_dynamics.estimates.ledgers import REPORT_YEARS, floor_to_dime from populace_dynamics.estimates.parameters import COLASeries -from populace_dynamics.ss import benefits +from populace_dynamics.ss import benefits, statutory_aime from populace_dynamics.ss.params import SSAParameters +from populace_dynamics.ss.statutory_aime import ComputationYears __all__ = [ "DI_BENEFIT_LEVEL_RULING", @@ -777,14 +788,21 @@ def eligibility_pia_for_clock( history: Mapping[int, float], birth_year: int, params: SSAParameters, + computation_years: ComputationYears = ComputationYears.STATUTORY, ) -> float: """The PIA at eligibility for the worker on ``clock``. - Only the old-age case is computed, with the existing oracle exactly as - the sealed ledger calls it (``benefits.aime`` then ``benefits.pia`` in - the year the worker attains 62). ``history`` is used as supplied; the - caller applies any cutoff law. The other bases route to hooks that - raise ``NotImplementedError``. + Only the old-age case is computed: the oracle AIME, then + ``benefits.pia`` in the year the worker attains 62. The AIME counts + its benefit computation years by ``computation_years`` + (:func:`populace_dynamics.ss.statutory_aime.oracle_aime`): + ``STATUTORY`` (the default) follows 42 USC 415(b)(2) and equals the + sealed ledger's ``benefits.aime`` for workers born 1929 or later; it + refuses workers attaining 62 before 1975 and history years before + 1951. ``LEGACY_FIXED_35`` is ``benefits.aime`` exactly as the sealed + ledger calls it, for every birth year. ``history`` is used as + supplied; the caller applies any cutoff law. The other bases route to + hooks that raise ``NotImplementedError``. """ birth = _year(birth_year, "birth_year") @@ -798,7 +816,9 @@ def eligibility_pia_for_clock( raise ValueError( "An age-62 clock must start in the year the worker attains 62." ) - aime_value = benefits.aime(dict(history), birth, params) + aime_value = statutory_aime.oracle_aime( + history, birth, params, computation_years=computation_years + ) return benefits.pia(aime_value, clock.eligibility_year, params) diff --git a/src/populace_dynamics/ss/statutory_aime.py b/src/populace_dynamics/ss/statutory_aime.py new file mode 100644 index 00000000..656651a2 --- /dev/null +++ b/src/populace_dynamics/ss/statutory_aime.py @@ -0,0 +1,410 @@ +"""Statutory benefit computation years and AIME (42 USC 415(b)). + +Python oracle (transitional, not Axiom). :func:`populace_dynamics.ss. +benefits.aime` averages every worker's highest 35 indexed years over 420 +months. 42 USC 415(b)(2) instead makes the number of benefit computation +years depend on the worker's elapsed years. For anyone born 1929 or later +who is alive at 62, the two agree: 40 elapsed years less 5 is 35. For +people born before 1929 the statute gives fewer than 35 years, and the +Track C step 1 comparison (the Axiom 415(b) candidate against this oracle +on the Track A cohort, 2026-09-23) traced all 251 of its disagreements to +that count. This module computes the statutory count and the AIME that +uses it. ``ss/__init__.py`` asks that the oracle's rule coverage not be +extended; this module corrects the old-age count the oracle already +computes, and adds the death and disability counts only as functions of +dates the caller supplies (no production path passes them yet). + +Statute text. Quoted from 42 USC 415(b) as the Axiom corpus holds it +(uscode.house.gov release point 119-100, expression date 2026-06-26, text +SHA-256 +``5b41d1cdacd39f4c49da7cf41dbbf22e1ea29b125be7105ca2daa12292e06eae``; +a copy is the encoder workspace ``source.txt`` under +``microcosm-launch-evidence/dynasim-parity-20260909/parallel-oasdi-20260920/ +encoder-recovery/``). The builder-visible excerpt file of the COLA work +(``tr2008-inputs-20260922/usc-42-415-excerpts.txt``) holds 415(a)(3)(B) +and 415(i) only, not 415(b). + +415(b)(1):: + + An individual’s average indexed monthly earnings shall be equal to the + quotient obtained by dividing— (A) the total (after adjustment under + paragraph (3)) of his wages paid in and self-employment income credited + to his benefit computation years (determined under paragraph (2)), by + (B) the number of months in those years. + +415(b)(2)(A), first sentence:: + + The number of an individual’s benefit computation years equals the + number of elapsed years reduced— (i) in the case of an individual who + is entitled to old-age insurance benefits (except as provided in the + second sentence of this subparagraph), or who has died, by 5 years, and + (ii) in the case of an individual who is entitled to disability + insurance benefits, by the number of years equal to one-fifth of such + individual’s elapsed years (disregarding any resulting fractional part + of a year), but not by more than 5 years. + +415(b)(2)(A), last sentence:: + + The number of an individual’s benefit computation years as determined + under this subparagraph shall in no case be less than 2. + +415(b)(2)(B)(i):: + + the term “benefit computation years” means those computation base + years, equal in number to the number determined under subparagraph (A), + for which the total of such individual’s wages and self-employment + income, after adjustment under paragraph (3), is the largest; + +415(b)(2)(B)(iii):: + + the term “number of elapsed years” means (except as otherwise provided + by section 104(j)(2) of the Social Security Amendments of 1972) the + number of calendar years after 1950 (or, if later, the year in which + the individual attained age 21) and before the year in which the + individual died, or, if it occurred earlier (but after 1960), the year + in which he attained age 62; except that such term excludes any + calendar year any part of which is included in a period of disability. + +415(b)(3)(A)(ii)(I), the indexing year:: + + the national average wage index (as defined in section 409(k)(1) of + this title ) for the second calendar year preceding the earliest of the + year of the individual’s death, eligibility for an old-age insurance + benefit, or eligibility for a disability insurance benefit + +415(a)(3)(B) (in the excerpt file) deems eligibility for old-age benefits +to begin with the month of attaining 62 and for disability benefits with +the month the period of disability began. + +Section 104(j)(2) of the Social Security Amendments of 1972 (Pub. L. +92-603, 86 Stat. 1341), from the statutory note "Effective Date of 1972 +Amendment" under 42 USC 414 (law.cornell.edu/uscode/text/42/414, read +2026-09-24, page SHA-256 +``731e1dbc8648c689d2d97546980739a1a53477da0669d566d3d32ec9f3966a2d``; +bracketed text is the note's):: + + In the case of a man who attains age 62 prior to 1975, the number of + his elapsed years for purposes of section 215(b)(3) of the Social + Security Act [42 U.S.C. 415(b)(3)] shall be equal to (A) the number + determined under such section as in effect on September 1, 1972, or + (B) if less, the number determined as though he attained age 65 in + 1975, [...] + +The oracle has no sex input and does not encode the 1972 law, so it +refuses anyone who attains 62 before 1975. That also makes the "(but +after 1960)" condition always hold. Quotations keep the sources' +characters; only line breaks were added. + +What is encoded: + +* the elapsed years of 415(b)(2)(B)(iii), for a worker alive at 62, a + worker who died (``death_year``), or a worker with one period of + disability that began in ``disability_year`` and runs through the end + of the elapsed-year window; +* the 415(b)(2)(A) reduction: 5 years for old-age entitlement or death, + one-fifth of the elapsed years (at most 5) for disability entitlement, + and the 2-year minimum; +* the 415(b)(3)(A) indexing year (the second year before the earliest of + death, attaining 62 and the start of the period of disability), with + later years entering unindexed through the existing + :func:`~populace_dynamics.ss.benefits.indexed_history`. + +What is not encoded (each is a documented limitation, not an +approximation this module applies silently): + +* the child-care dropout years of 415(b)(2)(A)'s third sentence (a + disabled worker living with a child under 3, up to a combined reduction + of 3): for such a worker the returned count can be too high and the AIME + too low; +* the second sentence of 415(b)(2)(A) (clause (ii) continuing into later + old-age eligibility unless 12 months pass without entitlement): a caller + whose old-age benefit follows a disability entitlement passes the + ``disability_year`` itself; +* periods of disability that ended, more than one period, and the + 12-month rule of 415(a)(3)(B); a death after a disability entitlement is + refused; +* 104(j)(2) of the 1972 amendments and the 1960 condition (see above); +* the choice of computation base years (415(b)(2)(B)(ii)): like + ``benefits.aime``, the AIME ranks every year the caller supplies, and + the caller applies the entitlement or death cutoff. Years before 1951 + are refused because they cannot be computation base years. + +The legacy convention. ``ss/benefits.py`` is not edited. It sits inside +the source identity the first-estimates birth-evidence reducer seals +(``scripts/first_estimates_birth_evidence.py``: every file under +``src/populace_dynamics`` outside ``POST_REVIEW_SOURCE_EXCLUSIONS`` must +equal reviewed commit ``1c7f3540``), and two sealed callers use it: the +first-estimates ledger (``estimates/ledgers.py``, ``runs/first_estimates_ +v1.json``) and the gate-2c couple-earnings AIME proxy +(``data/couple_earnings.py``). ``benefits.aime`` therefore stays what it +was, and this module names it :attr:`ComputationYears.LEGACY_FIXED_35`. +Callers pick a convention explicitly through :func:`oracle_aime`; the +legacy one is for code paths whose committed artifacts were computed with +it (``tests/ss/test_statutory_aime.py`` pins where it is used). +""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from enum import Enum + +from populace_dynamics.ss import benefits +from populace_dynamics.ss.params import SSAParameters + +__all__ = [ + "ComputationYears", + "LEGACY_FIXED_COMPUTATION_YEARS", + "elapsed_years", + "benefit_computation_years", + "indexing_year", + "aime_with_computation_years", + "aime", + "oracle_aime", +] + +#: The fixed divisor years of ``ss.benefits.aime`` (the legacy convention). +LEGACY_FIXED_COMPUTATION_YEARS = 35 +#: 415(b)(2)(B)(iii): elapsed years are the calendar years after 1950 ... +ELAPSED_YEARS_AFTER = 1950 +#: ... or, if later, after the year of attaining 21 ... +ELAPSED_START_AGE = 21 +#: ... and before the year of attaining 62 (or of death, if earlier). +ELIGIBILITY_AGE = 62 +#: 415(b)(2)(A)(i): the reduction for old-age entitlement or death. +OLD_AGE_OR_DEATH_REDUCTION = 5 +#: 415(b)(2)(A)(ii): one-fifth of the elapsed years, at most 5. +DISABILITY_REDUCTION_DIVISOR = 5 +DISABILITY_REDUCTION_MAXIMUM = 5 +#: 415(b)(2)(A): "shall in no case be less than 2". +MINIMUM_COMPUTATION_YEARS = 2 +#: 104(j)(2) of the 1972 amendments governs men attaining 62 before 1975. +FIRST_AGE_62_YEAR_ENCODED = 1975 +#: 415(b)(3)(A)(ii)(I): the second calendar year preceding. +INDEXING_LAG_YEARS = 2 +#: ``benefits.indexed_history`` indexes to its birth-year argument + 60. +_BENEFITS_INDEXING_AGE = 60 +_MONTHS = 12 + + +class ComputationYears(str, Enum): + """How an oracle AIME counts its benefit computation years. + + ``STATUTORY``: 42 USC 415(b)(2), through :func:`aime`. + ``LEGACY_FIXED_35``: always 35, the unchanged + :func:`populace_dynamics.ss.benefits.aime`; kept only for code paths + whose committed artifacts were computed with it. + """ + + STATUTORY = "statutory_415_b_2" + LEGACY_FIXED_35 = "legacy_fixed_35" + + +def _year(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{label} must be an integer year, not {value!r}.") + return int(value) + + +def _optional_year(value: object, label: str) -> int | None: + return None if value is None else _year(value, label) + + +def _checked_dates( + birth_year: object, death_year: object, disability_year: object +) -> tuple[int, int | None, int | None]: + birth = _year(birth_year, "birth_year") + death = _optional_year(death_year, "death_year") + onset = _optional_year(disability_year, "disability_year") + if birth + ELIGIBILITY_AGE < FIRST_AGE_62_YEAR_ENCODED: + raise ValueError( + f"Born {birth}: attains 62 before {FIRST_AGE_62_YEAR_ENCODED}. " + "Section 104(j)(2) of the Social Security Amendments of 1972 " + "sets a man's elapsed years then, and the oracle has no sex " + "input and does not encode the 1972 law." + ) + if death is not None and death < birth: + raise ValueError(f"death_year {death} precedes birth_year {birth}.") + if onset is not None and onset < birth: + raise ValueError( + f"disability_year {onset} precedes birth_year {birth}." + ) + if death is not None and onset is not None: + raise ValueError( + "A death after a disability entitlement is not encoded: " + "415(b)(2)(A) reduces for death by 5 years (clause (i)) and " + "continues clause (ii) only into later disability or old-age " + "eligibility; the oracle does not settle that case." + ) + return birth, death, onset + + +def elapsed_years( + birth_year: int, + *, + death_year: int | None = None, + disability_year: int | None = None, +) -> int: + """Number of elapsed years, 42 USC 415(b)(2)(B)(iii). + + The calendar years after the later of 1950 and the year of attaining + 21, and before the earlier of the year of death (``death_year``, if + given) and the year of attaining 62. ``disability_year`` is the + calendar year in which the worker's period of disability began; that + year and every later year are excluded (any part of each is in the + period, which is taken to run through the end of the window). + Never negative. + """ + + birth, death, onset = _checked_dates( + birth_year, death_year, disability_year + ) + after = max(ELAPSED_YEARS_AFTER, birth + ELAPSED_START_AGE) + before = birth + ELIGIBILITY_AGE + if death is not None: + before = min(before, death) + if onset is not None: + before = min(before, onset) + return max(0, before - after - 1) + + +def benefit_computation_years( + birth_year: int, + *, + death_year: int | None = None, + disability_year: int | None = None, +) -> int: + """Number of benefit computation years, 42 USC 415(b)(2)(A). + + With neither date, the worker is taken to be entitled to old-age + benefits and alive at 62; with ``death_year``, to have died; with + ``disability_year``, to be entitled to disability benefits (clause + (ii), which also carries into a later old-age benefit). Elapsed years + less 5 (clause (i)) or less one-fifth of them, at most 5 (clause + (ii)), and never fewer than 2. The child-care dropout years of the + third sentence are not encoded. + """ + + elapsed = elapsed_years( + birth_year, death_year=death_year, disability_year=disability_year + ) + if disability_year is None: + reduction = OLD_AGE_OR_DEATH_REDUCTION + else: + reduction = min( + DISABILITY_REDUCTION_MAXIMUM, + elapsed // DISABILITY_REDUCTION_DIVISOR, + ) + return max(MINIMUM_COMPUTATION_YEARS, elapsed - reduction) + + +def indexing_year( + birth_year: int, + *, + death_year: int | None = None, + disability_year: int | None = None, +) -> int: + """The wage-indexing year, 42 USC 415(b)(3)(A)(ii)(I). + + The second calendar year before the earliest of the year of death, + the year of attaining 62 (old-age eligibility, 415(a)(3)(B)(i)) and + the year the period of disability began (disability eligibility, + 415(a)(3)(B)(ii)). With neither date it is the year of attaining 60, + the year :func:`populace_dynamics.ss.benefits.aime` indexes to. + """ + + birth, death, onset = _checked_dates( + birth_year, death_year, disability_year + ) + earliest = birth + ELIGIBILITY_AGE + for year in (death, onset): + if year is not None: + earliest = min(earliest, year) + return earliest - INDEXING_LAG_YEARS + + +def aime_with_computation_years( + history: Mapping[int, float], + computation_years: int, + params: SSAParameters, + *, + indexing_year: int, +) -> int: + """The AIME arithmetic of ``benefits.aime`` with any year count. + + Earnings limited to the contribution and benefit base + (``benefits.creditable_history``), indexed to ``indexing_year`` with + later years unindexed (``benefits.indexed_history``), the highest + ``computation_years`` values (years absent from ``history`` count as + zero), summed in the same order and floored over + ``12 * computation_years`` months. With 35 years and the year of + attaining 60 it is ``benefits.aime`` operation for operation. + """ + + count = _year(computation_years, "computation_years") + if count < 1: + raise ValueError(f"computation_years must be positive, not {count}.") + index = _year(indexing_year, "indexing_year") + early = sorted(year for year in history if int(year) <= 1950) + if early: + raise ValueError( + f"Years {early} precede 1951 and cannot be computation base " + "years (42 USC 415(b)(2)(B)(ii))." + ) + creditable = benefits.creditable_history(dict(history), params) + indexed = benefits.indexed_history( + creditable, index - _BENEFITS_INDEXING_AGE, params + ) + top = sorted(indexed.values(), reverse=True)[:count] + top += [0.0] * (count - len(top)) + return math.floor(sum(top) / (count * _MONTHS)) + + +def aime( + history: Mapping[int, float], + birth_year: int, + params: SSAParameters, + *, + death_year: int | None = None, + disability_year: int | None = None, +) -> int: + """Average indexed monthly earnings under 42 USC 415(b). + + The statutory number of benefit computation years + (:func:`benefit_computation_years`) and indexing year + (:func:`indexing_year`), then :func:`aime_with_computation_years`. + For a worker born 1929 or later with neither date it equals + ``benefits.aime(history, birth_year, params)`` exactly. + ``history`` must hold only the worker's computation base years; the + caller applies the entitlement or death cutoff. + """ + + return aime_with_computation_years( + history, + benefit_computation_years( + birth_year, death_year=death_year, disability_year=disability_year + ), + params, + indexing_year=indexing_year( + birth_year, death_year=death_year, disability_year=disability_year + ), + ) + + +def oracle_aime( + history: Mapping[int, float], + birth_year: int, + params: SSAParameters, + *, + computation_years: ComputationYears, +) -> int: + """The old-age (age-62) oracle AIME under a named convention. + + ``STATUTORY`` is :func:`aime` for a worker alive at 62; + ``LEGACY_FIXED_35`` is the unchanged ``benefits.aime``. + """ + + convention = ComputationYears(computation_years) + if convention is ComputationYears.LEGACY_FIXED_35: + return benefits.aime(dict(history), birth_year, params) + return aime(history, birth_year, params) diff --git a/tests/README-tiers.md b/tests/README-tiers.md index f1adbf1c..ee8e990d 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -38,9 +38,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 2,881 | +| `unit` | 2,922 | | `artifact` | 2,796 | | `integration_psid` | 849 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 162 | -| **Total** | **7,208** | +| **Total** | **7,249** | diff --git a/tests/cola_track_a/test_assembly.py b/tests/cola_track_a/test_assembly.py index 5aaf6db7..7ef58a45 100644 --- a/tests/cola_track_a/test_assembly.py +++ b/tests/cola_track_a/test_assembly.py @@ -75,8 +75,9 @@ INVENTED_DATA_LABEL, ) from populace_dynamics.estimates.parameters import COLASeries -from populace_dynamics.ss import benefits +from populace_dynamics.ss import benefits, statutory_aime from populace_dynamics.ss.params import SSAParameters +from populace_dynamics.ss.statutory_aime import ComputationYears SHAPE = (2, MAX_AGE + 1) @@ -1217,6 +1218,72 @@ def pia(pid, birth): ) +def test_track_a_retirement_level_keeps_the_legacy_35_years(): + # INVENTED: 1 (born 1925, never married) claimed at 62 in 1987 and is + # alive in 2030; covered earnings 1968-1986 only. Track A's oracle + # retirement level divides by 35 years (Registration 13's arithmetic), + # not by the 31 statutory computation years for the 1925 cohort. + assert ( + track_benefits.TRACK_A_COMPUTATION_YEARS + is ComputationYears.LEGACY_FIXED_35 + ) + persons = [_static(1, 1925)] + persons[0]["marital_status_opening"] = "never_married" + careers = {1: {year: 30_000.0 for year in range(1968, 1987)}} + final = [ + _state( + 1, + 1925, + 2030, + marital_status="never_married", + claimed=True, + claim_year=1987, + ) + ] + initial = [_state(1, 1925, 2010, marital_status="never_married")] + cohort, projection = _handmade_cohort(persons, careers, final, initial) + params = invented_params() + context = track_benefits.BenefitContext( + cohort=cohort, params=params, baseline=invented_cola(), config=CONFIG + ) + rows, _ = track_benefits.reference_benefit_rows( + projection, draw=0, row=REGISTERED_ROWS["R0"], context=context + ) + (row,) = rows + clock = sb.WorkerClock.at_age_62(1925, entitlement_year=1987) + factor = claiming.benefit_factor(62 * 12, 1925, params) + + def worker_path(computation_years): + pia = sb.eligibility_pia_for_clock( + sb.WorkerClock.at_age_62(1925), + history=careers[1], + birth_year=1925, + params=params, + computation_years=computation_years, + ) + return pia, sb.worker_scenario_paths( + eligibility_pia=pia, + claim_age_factor=factor, + clock=clock, + baseline=invented_cola(), + ) + + legacy_pia, legacy = worker_path(ComputationYears.LEGACY_FIXED_35) + statutory_pia, _ = worker_path(ComputationYears.STATUTORY) + assert legacy_pia == benefits.pia( + benefits.aime(careers[1], 1925, params), 1987, params + ) + assert statutory_aime.benefit_computation_years(1925) == 31 + assert statutory_pia > legacy_pia + component = row["benefit_components"]["retired_worker"] + assert component["base"] == pytest.approx( + 12 * legacy.baseline_monthly_by_payment_year[2030] + ) + assert component["reform"] == pytest.approx( + 12 * legacy.reform_monthly_by_payment_year[2030] + ) + + def test_widow_benefit_equals_the_a6_composition(): # INVENTED: worker 2 (born 1955) claimed 2019 and died in 2021; # widow 1 (born 1958) has no own claim. diff --git a/tests/estimates/test_birth_evidence_artifact.py b/tests/estimates/test_birth_evidence_artifact.py index ba93cba6..bdd756d9 100644 --- a/tests/estimates/test_birth_evidence_artifact.py +++ b/tests/estimates/test_birth_evidence_artifact.py @@ -126,6 +126,7 @@ def test_post_review_sources_are_outside_historical_reducer_identity(): Path("src/populace_dynamics/cola_track_a/opening.py"), Path("src/populace_dynamics/cola_track_a/runner.py"), Path("src/populace_dynamics/cola_track_a/statutory.py"), + Path("src/populace_dynamics/ss/statutory_aime.py"), ) assert reducer.POST_REVIEW_SHARED_SOURCE_BLOBS == { Path( @@ -391,6 +392,13 @@ def test_post_review_exclusions_are_unreachable_from_birth_evidence(): "the opt-in Track A assembly became reachable from the " f"birth-evidence reducer: {sorted(track_a_modules & reachable)}" ) + statutory_aime = "populace_dynamics.ss.statutory_aime" + assert statutory_aime in module_paths + assert "populace_dynamics.ss.benefits" in reachable + assert statutory_aime not in reachable, ( + "the opt-in statutory AIME became reachable from the " + "birth-evidence reducer" + ) def test_reducer_accepts_explicit_unresolved_upstream_boundary(): diff --git a/tests/ss/test_statutory_aime.py b/tests/ss/test_statutory_aime.py new file mode 100644 index 00000000..7e75d507 --- /dev/null +++ b/tests/ss/test_statutory_aime.py @@ -0,0 +1,432 @@ +"""Statutory benefit computation years and AIME (42 USC 415(b)(2)). + +Every parameter bundle and earnings history here is INVENTED; the +expected values are computed by hand in the comments from the statute's +counting rules, not read back from the code under test. +""" + +from __future__ import annotations + +import ast +import math +import random +from pathlib import Path + +import pytest + +from populace_dynamics.ss import benefits +from populace_dynamics.ss import statutory_aime as sa +from populace_dynamics.ss.params import SSAParameters +from populace_dynamics.ss.statutory_aime import ComputationYears + +ROOT = Path(__file__).resolve().parents[2] + + +def _flat_params(nawi_overrides: dict[int, float] | None = None): + """INVENTED: a flat wage index (indexing is neutral) and no binding + contribution and benefit base, so AIMEs are hand-computable.""" + nawi = {year: 10_000.0 for year in range(1951, 2071)} + nawi.update(nawi_overrides or {}) + return SSAParameters( + nawi=nawi, + wage_base={1937: 1.0e12}, + pia_factors=(0.9, 0.32, 0.15), + fra_months_by_birth_year=[(1900, 66 * 12)], + early_monthly_rates=(5 / 900, 5 / 1200), + early_first_bracket_months=36, + pe_us_revision="INVENTED", + delayed_credit_by_birth_year=[(1900, 0.08)], + ) + + +def _growth_params() -> SSAParameters: + """INVENTED: 4 percent wage growth and a binding, rising base.""" + return SSAParameters( + nawi={ + year: 2_800.0 * 1.04 ** (year - 1951) for year in range(1951, 2071) + }, + wage_base={1937: 3_000.0, 1975: 14_100.0, 1990: 51_300.0}, + pia_factors=(0.9, 0.32, 0.15), + fra_months_by_birth_year=[(1900, 792), (1955, 804)], + early_monthly_rates=(5 / 900, 5 / 1200), + early_first_bracket_months=36, + pe_us_revision="INVENTED", + delayed_credit_by_birth_year=[(1900, 0.08)], + ) + + +# --------------------------------------------------------------------------- +# Elapsed years and benefit computation years (hand counts) +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + ("birth", "elapsed", "computation"), + [ + # Attains 21 in 1941 and 62 in 1982: 1951-1981 elapsed. + (1920, 31, 26), + # Attains 21 in 1949 and 62 in 1990: 1951-1989 elapsed. + (1928, 39, 34), + # Attains 21 in 1950 and 62 in 1991: 1951-1990 elapsed. + (1929, 40, 35), + # Attains 21 in 1956 and 62 in 1997: 1957-1996 elapsed. + (1935, 40, 35), + # Attains 21 in 1934 and 62 in 1975: 1951-1974 elapsed. + (1913, 24, 19), + # Attains 21 in 1938 and 62 in 1979: 1951-1978 elapsed. + (1917, 28, 23), + ], +) +def test_old_age_counts_by_birth_year(birth, elapsed, computation): + assert sa.elapsed_years(birth) == elapsed + assert sa.benefit_computation_years(birth) == computation + + +@pytest.mark.parametrize("birth", [1929, 1940, 1960, 1975, 1990, 2005]) +def test_attaining_21_after_1950_always_gives_40_elapsed_and_35_years(birth): + # Years after the year of attaining 21 and before the year of attaining + # 62: (birth + 22) through (birth + 61), 40 years; less 5 is 35. + assert sa.elapsed_years(birth) == 40 + assert sa.benefit_computation_years(birth) == 35 + assert sa.benefit_computation_years(birth) == ( + sa.LEGACY_FIXED_COMPUTATION_YEARS + ) + + +def test_death_before_62_counts_to_the_year_before_death(): + # Born 1960 (attains 21 in 1981), died 2000: 1982-1999 elapsed = 18; + # clause (i) "or who has died" reduces by 5 -> 13. + assert sa.elapsed_years(1960, death_year=2000) == 18 + assert sa.benefit_computation_years(1960, death_year=2000) == 13 + # A death at or after 62 leaves the old-age count unchanged. + for death in (2022, 2023, 2040): + assert sa.benefit_computation_years(1960, death_year=death) == 35 + assert sa.benefit_computation_years(1920, death_year=1990) == 26 + + +@pytest.mark.parametrize( + ("death", "elapsed"), + [ + (1985, 3), # 1982-1984: 3 - 5 < 2 -> 2 + (1984, 2), # 1982-1983 + (1982, 0), # the year after attaining 21: no elapsed year + (1981, 0), # the year of attaining 21: never negative + (1970, 0), + ], +) +def test_two_year_minimum(death, elapsed): + assert sa.elapsed_years(1960, death_year=death) == elapsed + assert sa.benefit_computation_years(1960, death_year=death) == 2 + + +@pytest.mark.parametrize( + ("onset", "elapsed", "dropout", "computation"), + [ + # Born 1960 (attains 21 in 1981). The onset year and later are + # excluded ("any part of which is included in a period of + # disability"). Clause (ii): one-fifth of the elapsed years, + # fraction disregarded, at most 5. + (1983, 1, 0, 2), # 1982: 1 - 0 = 1 -> minimum 2 + (1987, 5, 1, 4), # 1982-1986 + (1990, 8, 1, 7), # 1982-1989 + (2005, 23, 4, 19), # 1982-2004 + (2015, 33, 5, 28), # 1982-2014: 6 would exceed the cap of 5 + (2021, 39, 5, 34), # 1982-2020 + ], +) +def test_disability_dropout_years(onset, elapsed, dropout, computation): + assert sa.elapsed_years(1960, disability_year=onset) == elapsed + assert elapsed - dropout == computation or computation == 2 + assert sa.benefit_computation_years(1960, disability_year=onset) == ( + computation + ) + + +def test_disability_after_62_keeps_the_age_62_window(): + # Born 1950 (attains 21 in 1971, 62 in 2012); a period of disability + # beginning in 2014 excludes nothing before 2012: 1972-2011 = 40, + # clause (ii) reduces by min(5, 8) = 5. + assert sa.elapsed_years(1950, disability_year=2014) == 40 + assert sa.benefit_computation_years(1950, disability_year=2014) == 35 + + +def test_indexing_year_is_the_second_year_before_the_earliest_event(): + assert sa.indexing_year(1960) == 2020 # attains 62 in 2022 + assert sa.indexing_year(1960, death_year=2000) == 1998 + assert sa.indexing_year(1960, death_year=2030) == 2020 + assert sa.indexing_year(1960, disability_year=2005) == 2003 + assert sa.indexing_year(1920) == 1980 + + +# --------------------------------------------------------------------------- +# AIME (hand-computed on INVENTED flat parameters) +# --------------------------------------------------------------------------- +def test_aime_born_1920_divides_by_26_years(): + params = _flat_params() + # 12,000 in each of 1968-1981 (14 years): total 168,000. + history = {year: 12_000.0 for year in range(1968, 1982)} + # Statutory: 168,000 / (26 * 12 = 312) = 538.46 -> 538. + assert sa.aime(history, 1920, params) == 538 + # Legacy: 168,000 / 420 = 400. + assert benefits.aime(history, 1920, params) == 400 + + +def test_aime_born_1928_divides_by_34_years(): + params = _flat_params() + # 12,000 in each of 1968-1989 (22 years): total 264,000. + history = {year: 12_000.0 for year in range(1968, 1990)} + # Statutory: 264,000 / 408 = 647.06 -> 647; legacy: / 420 = 628.57. + assert sa.aime(history, 1928, params) == 647 + assert benefits.aime(history, 1928, params) == 628 + + +def test_aime_born_1929_and_1935_is_unchanged(): + params = _flat_params() + # 1929: 12,000 in 1968-1990 (23 years) = 276,000 / 420 = 657.14. + history_1929 = {year: 12_000.0 for year in range(1968, 1991)} + assert sa.aime(history_1929, 1929, params) == 657 + assert benefits.aime(history_1929, 1929, params) == 657 + # 1935: 20,000 in 1968-1996 (29 years) = 580,000 / 420 = 1380.95. + history_1935 = {year: 20_000.0 for year in range(1968, 1997)} + assert sa.aime(history_1935, 1935, params) == 1380 + assert benefits.aime(history_1935, 1935, params) == 1380 + + +def test_aime_keeps_the_highest_years_when_there_are_more_than_the_count(): + params = _flat_params() + # Born 1920: 26 computation years. 30 years 1952-1981 of 1,000 each, + # plus 4 of them raised by 26,000: the top 26 are the 4 raised years + # and 22 plain ones: (4 * 27,000 + 22 * 1,000) / 312 = 416.67 -> 416. + history = {year: 1_000.0 for year in range(1952, 1982)} + for year in (1960, 1965, 1970, 1975): + history[year] = 27_000.0 + assert sa.aime(history, 1920, params) == 416 + + +def test_aime_for_a_death_before_62_uses_the_minimum_and_death_indexing(): + params = _flat_params() + # Born 1960, died 1984: 2 computation years (the minimum). The top + # two of 30,000 / 18,000 / 6,000 are 48,000; / 24 = 2,000. + history = {1982: 30_000.0, 1983: 18_000.0, 1984: 6_000.0} + assert sa.aime(history, 1960, params, death_year=1984) == 2_000 + # The legacy divisor would give 54,000 / 420 = 128.57 -> 128. + assert benefits.aime(history, 1960, params) == 128 + + +def test_aime_indexes_to_the_second_year_before_death(): + # INVENTED wage index: 1,000 before 1998, 2,000 in 1998-2019, 4,000 + # from 2020. Born 1960, died 2000 (13 computation years, index 1998): + # 1990's 10,000 -> 10,000 * 2,000 / 1,000 = 20,000; 1999 is after the + # indexing year and enters unindexed at 5,000. 25,000 / 156 = 160.26. + index = {year: 1_000.0 for year in range(1951, 1998)} + index.update({year: 2_000.0 for year in range(1998, 2020)}) + index.update({year: 4_000.0 for year in range(2020, 2071)}) + params = _flat_params(index) + history = {1990: 10_000.0, 1999: 5_000.0} + assert sa.aime(history, 1960, params, death_year=2000) == 160 + # Alive at 62 (index 2020): 1990 -> 40,000 and 1999 -> 10,000; + # 50,000 / 420 = 119.05 -> 119, the legacy result for this cohort too. + assert sa.aime(history, 1960, params) == 119 + assert benefits.aime(history, 1960, params) == 119 + + +def test_aime_for_a_disability_onset_uses_dropout_years_and_onset_indexing(): + params = _flat_params() + # Born 1960, period of disability from 2005: 19 computation years, + # index year 2003. 24,000 in each of 1990-2004 (15 years) = 360,000; + # / (19 * 12 = 228) = 1578.95 -> 1578. + history = {year: 24_000.0 for year in range(1990, 2005)} + assert sa.aime(history, 1960, params, disability_year=2005) == 1578 + + +# --------------------------------------------------------------------------- +# Unchanged results from the 1929 birth cohort on (INVENTED random careers) +# --------------------------------------------------------------------------- +def _random_history(rng: random.Random, birth: int) -> dict[int, float]: + first = max(1968, birth + 18) + last = min(2060, birth + 70) + history = {} + for year in range(first, last + 1): + if rng.random() < 0.8: + history[year] = rng.choice( + (0.0, rng.uniform(0.0, 20_000.0), rng.uniform(0.0, 2.0e5)) + ) + return history + + +def test_statutory_equals_legacy_for_every_birth_year_from_1929(): + params = _growth_params() + rng = random.Random(20260924) + for birth in range(1929, 1996): + for _ in range(15): + history = _random_history(rng, birth) + statutory = sa.aime(history, birth, params) + assert statutory == benefits.aime(history, birth, params) + assert statutory == sa.oracle_aime( + history, + birth, + params, + computation_years=ComputationYears.STATUTORY, + ) + + +def _reference_aime(history, birth, params, count, index_year): + """Independent restatement of 415(b)(1)-(3) for the property test.""" + indexed = [] + for year, earnings in history.items(): + creditable = min(float(earnings), params.wage_base_for(year)) + if year < index_year: + creditable = ( + creditable * params.nawi[index_year] / params.nawi[year] + ) + indexed.append(creditable) + top = sorted(indexed, reverse=True)[:count] + top += [0.0] * (count - len(top)) + return math.floor(sum(top) / (12 * count)) + + +def test_before_1929_equals_the_oracle_arithmetic_with_the_statutory_count(): + # The Track C attribution check's form: the oracle's own creditable and + # indexed history, the highest (elapsed - 5) years, floored. + params = _growth_params() + rng = random.Random(1917) + for birth in range(1913, 1929): + # Elapsed: 1951 through the year before attaining 62; less 5. + count = (birth + 62 - 1950 - 1) - 5 + assert sa.benefit_computation_years(birth) == count < 35 + for _ in range(10): + history = _random_history(rng, birth) + expected = _reference_aime( + history, birth, params, count, birth + 60 + ) + assert sa.aime(history, birth, params) == expected + assert ( + sa.aime_with_computation_years( + history, count, params, indexing_year=birth + 60 + ) + == expected + ) + assert sa.aime_with_computation_years( + history, 35, params, indexing_year=birth + 60 + ) == benefits.aime(history, birth, params) + + +def test_oracle_aime_legacy_is_the_unchanged_benefits_aime(): + params = _growth_params() + rng = random.Random(35) + # Every birth year, including those the statutory encoding refuses. + for birth in (1905, 1912, 1920, 1928, 1929, 1960): + for _ in range(5): + history = _random_history(rng, birth) + assert sa.oracle_aime( + history, + birth, + params, + computation_years=ComputationYears.LEGACY_FIXED_35, + ) == benefits.aime(history, birth, params) + assert ComputationYears("legacy_fixed_35") is ( + ComputationYears.LEGACY_FIXED_35 + ) + assert ComputationYears("statutory_415_b_2") is ComputationYears.STATUTORY + + +# --------------------------------------------------------------------------- +# Refusals +# --------------------------------------------------------------------------- +def test_refuses_workers_attaining_62_before_1975(): + # 104(j)(2) of the 1972 amendments sets a man's elapsed years then. + with pytest.raises(ValueError, match="104"): + sa.benefit_computation_years(1912) + with pytest.raises(ValueError, match="104"): + sa.aime({1968: 1_000.0}, 1900, _flat_params()) + assert sa.benefit_computation_years(1913) == 19 + + +def test_refuses_undecided_or_invalid_dates(): + with pytest.raises(ValueError, match="death after a disability"): + sa.benefit_computation_years( + 1960, death_year=2010, disability_year=2005 + ) + with pytest.raises(ValueError, match="precedes birth_year"): + sa.elapsed_years(1960, death_year=1959) + with pytest.raises(ValueError, match="precedes birth_year"): + sa.elapsed_years(1960, disability_year=1950) + with pytest.raises(TypeError): + sa.elapsed_years(1960.0) + with pytest.raises(TypeError): + sa.elapsed_years(True) + + +def test_refuses_years_that_cannot_be_computation_base_years(): + params = _flat_params() + with pytest.raises(ValueError, match="1951"): + sa.aime({1950: 1_000.0, 1968: 1_000.0}, 1929, params) + with pytest.raises(ValueError, match="positive"): + sa.aime_with_computation_years( + {1968: 1.0}, 0, params, indexing_year=1990 + ) + + +# --------------------------------------------------------------------------- +# Where the legacy convention is used (source scan, no imports) +# --------------------------------------------------------------------------- +#: Every module that calls ``benefits.aime`` (the legacy fixed-35 +#: convention) directly, with the committed evidence it serves. +LEGACY_BENEFITS_AIME_CALLERS = { + # Sealed first-estimates ledger (birth-evidence source identity). + "src/populace_dynamics/estimates/ledgers.py", + # Sealed gate-2c couple-earnings AIME proxy. + "src/populace_dynamics/data/couple_earnings.py", + # Track A's disclosed approximation (Registration 13). + "src/populace_dynamics/cola_track_a/benefits.py", + # The LEGACY_FIXED_35 branch of oracle_aime itself. + "src/populace_dynamics/ss/statutory_aime.py", + # The cross-engine PIA artifact (born 1958/1964: 35 either way); its + # notes record a comparison against the fixed 35. + "scripts/build_cross_engine_pia_artifact.py", + # The R7 earnings-sharing replication, whose PIA supply admits every + # birth year with an age-60 year in the wage-index series. + "scripts/replication_r7_sharing.py", +} +#: The sealed package initializer re-exports ``benefits.aime`` as +#: ``ss.aime``; nothing may import it under that name or call it. +LEGACY_AIME_REEXPORTS = {"src/populace_dynamics/ss/__init__.py"} +#: Every module that names ``ComputationYears.LEGACY_FIXED_35``. +LEGACY_CONVENTION_USERS = { + "src/populace_dynamics/cola_track_a/benefits.py", + "src/populace_dynamics/ss/statutory_aime.py", +} + + +def _python_sources(): + for base in ("src", "scripts"): + for path in sorted((ROOT / base).rglob("*.py")): + yield path.relative_to(ROOT).as_posix(), ast.parse( + path.read_text(encoding="utf-8") + ) + + +def test_the_legacy_convention_is_used_only_where_pinned(): + callers, users, aliases = set(), set(), set() + for relative, tree in _python_sources(): + for node in ast.walk(tree): + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "aime" + and isinstance(node.func.value, ast.Name) + and node.func.value.id in ("benefits", "ss") + ): + callers.add(relative) + if ( + isinstance(node, ast.Attribute) + and node.attr == "LEGACY_FIXED_35" + ): + users.add(relative) + if isinstance(node, ast.ImportFrom) and node.module in ( + "populace_dynamics.ss", + "populace_dynamics.ss.benefits", + ): + if any(alias.name == "aime" for alias in node.names): + aliases.add(relative) + assert callers == LEGACY_BENEFITS_AIME_CALLERS + assert users == LEGACY_CONVENTION_USERS + assert aliases == LEGACY_AIME_REEXPORTS diff --git a/tests/test_scenario_benefits.py b/tests/test_scenario_benefits.py index 4acb021d..93189644 100644 --- a/tests/test_scenario_benefits.py +++ b/tests/test_scenario_benefits.py @@ -22,8 +22,9 @@ PayrollRateLegs, ReportParameters, ) -from populace_dynamics.ss import benefits +from populace_dynamics.ss import benefits, statutory_aime from populace_dynamics.ss.params import SSAParameters +from populace_dynamics.ss.statutory_aime import ComputationYears # Invented realized history: a deterministic closed-form path over the # committed loader's 1979-2022 span (0.012 to 0.090), with a few zero years. @@ -734,6 +735,10 @@ def test_di_benefit_level_hook_awaits_a_ruling(): def test_age_62_pia_matches_the_sealed_ledger_computation(): + # INVENTED claimants. The legacy convention is the sealed ledger's + # computation for every birth year; the statutory default agrees with + # it from the 1929 birth cohort on and divides the 1925 claimant's + # total by 31 computation years (1951-1986 elapsed, less 5), not 35. parameters = _report_parameters(_realized()) ledger = ledgers.build_benefit_ledger( _invented_claimants(), parameters, draw_index=0 @@ -746,15 +751,37 @@ def test_age_62_pia_matches_the_sealed_ledger_computation(): for year, amount in claimant.earnings_by_year.items() if year <= claimant.claim_year } - value = sb.eligibility_pia_for_clock( - sb.WorkerClock.at_age_62(claimant.birth_year), + clock = sb.WorkerClock.at_age_62(claimant.birth_year) + legacy = sb.eligibility_pia_for_clock( + clock, history=history, birth_year=claimant.birth_year, params=parameters.ssa, + computation_years=ComputationYears.LEGACY_FIXED_35, ) - assert struct.pack("= 1929: + assert struct.pack(" legacy with pytest.raises(ValueError, match="attains 62"): sb.eligibility_pia_for_clock( sb.WorkerClock( diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 5719ce57..fc6cd628 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 2881, + "unit": 2922, "artifact": 2796, "integration_psid": 849, "reproduction_legacy": 520, From 5c83bbbda77caa140eac02dbdfc947ea4d61561d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 24 Sep 2026 04:59:51 -0400 Subject: [PATCH 2/4] Pin Track C's PIA check to Track A's computation years After the merge of master (#455), scripts/track_c_aime_agreement.py checked that Track A's PIA rests on its oracle AIME by calling scenario_benefits.eligibility_pia_for_clock without computation_years. This branch made that function default to the statutory 415(b)(2) count, while the script's oracle side is the fixed-35 benefits.aime and Track A passes LEGACY_FIXED_35. A real run would therefore stop with a PIA mismatch at the first person born before 1929 (the oracle-fix rerun's NOTE records the same stop: person 6002, born 1921). The merged tree also failed tests/ss/test_statutory_aime.py's source scan, because the script called benefits.aime directly without being pinned. - The check moves into check_track_a_pia() and passes computation_years=TRACK_A_COMPUTATION_YEARS, as the Track A assembly does. - oracle_aime() calls ss.statutory_aime.oracle_aime under ORACLE_CONVENTION = LEGACY_FIXED_35. That call is benefits.aime, so the numbers the 20260923 run recorded are unchanged. main() refuses to run if Track A's convention stops being LEGACY_FIXED_35, because the attribution and the report text assume 35 years. - The report prose and the module docstring name the convention, and ss/statutory_aime.py joins the recorded source list. - The source scan pins the script as a LEGACY_FIXED_35 user. - ss/statutory_aime.py: the 104(j)(2) provenance line said all bracketed text was the note's, but "[...]" is an omission made here. It now says so, and that no copy of the page is saved with the evidence. The list of what is not encoded adds the birth-year convention: the year of attaining 21 or 62 is taken to be birth year plus age. - Tests (INVENTED data): check_track_a_pia passes for people born 1925 and 1960 and refuses a PIA that does not rest on oracle_aime. The default statutory call gives the 1925 person another PIA. The convention guard refuses STATUTORY. Removing the computation_years pin makes the new check test fail. - Tier manifest recollected: unit 3036, total 7382. Co-Authored-By: Claude Opus 5.5 --- scripts/track_c_aime_agreement.py | 93 +++++++++++++++++----- src/populace_dynamics/ss/statutory_aime.py | 7 +- tests/README-tiers.md | 4 +- tests/ss/test_statutory_aime.py | 4 + tests/test_track_c_aime_agreement.py | 53 ++++++++++++ tests/tier_counts.json | 2 +- 6 files changed, 139 insertions(+), 24 deletions(-) diff --git a/scripts/track_c_aime_agreement.py b/scripts/track_c_aime_agreement.py index 6fcbf28b..207565bd 100644 --- a/scripts/track_c_aime_agreement.py +++ b/scripts/track_c_aime_agreement.py @@ -5,8 +5,12 @@ * the oracle's AIME, computed exactly as Track A computes it (``ss.benefits.aime`` over the person's Track A career, indexed to the - year of attaining 60, the call ``scenario_benefits. - eligibility_pia_for_clock`` makes for the age-62 clock), with Track A's + year of attaining 60: the call ``scenario_benefits. + eligibility_pia_for_clock`` makes for the age-62 clock under Track A's + computation-year convention, ``cola_track_a.benefits. + TRACK_A_COMPUTATION_YEARS`` = ``ComputationYears.LEGACY_FIXED_35``; the + function's default, the statutory 42 USC 415(b)(2) count, differs for + people born before 1929), with Track A's statutory parameters (the committed capture before 1975, the TR2008 intermediate AWI from 1975: ``cola_track_a.runner.tr2008_ssa_parameters``); * the AIME the actual Axiom rules engine returns for the same person, @@ -92,9 +96,13 @@ from populace_dynamics import scenario_benefits as sb # noqa: E402 from populace_dynamics.cola_track_a.benefits import ( # noqa: E402 FIRST_ORACLE_ELIGIBILITY_YEAR, + TRACK_A_COMPUTATION_YEARS, ) -from populace_dynamics.ss import benefits # noqa: E402 +from populace_dynamics.ss import benefits, statutory_aime # noqa: E402 from populace_dynamics.ss.params import SSAParameters # noqa: E402 +from populace_dynamics.ss.statutory_aime import ( # noqa: E402 + ComputationYears, +) SCHEMA_VERSION = "populace_dynamics.track_c.aime_agreement.v1" HEADER = ( @@ -118,6 +126,10 @@ PANEL_FIRST_YEAR = 1968 #: The oracle's fixed benefit computation years (ss.benefits). ORACLE_COMPUTATION_YEARS = 35 +#: The computation-year convention of the oracle side: Track A's. The +#: attribution and the report text assume it is the legacy fixed 35 +#: (:data:`ORACLE_COMPUTATION_YEARS`); :func:`main` refuses to run otherwise. +ORACLE_CONVENTION = ComputationYears.LEGACY_FIXED_35 _RETIREMENT_AGE = 62 _INDEXING_AGE = 60 _MONTHS = 12 @@ -322,9 +334,60 @@ def candidate_computation_years(birth_year: int) -> int: def oracle_aime( career: Mapping[int, float], birth_year: int, params: SSAParameters ) -> int: - """Track A's oracle AIME (``eligibility_pia_for_clock``'s call).""" + """Track A's oracle AIME (``eligibility_pia_for_clock``'s call). + + ``ss.statutory_aime.oracle_aime`` under :data:`ORACLE_CONVENTION`, + which is ``ss.benefits.aime`` (always 35 years). + """ + + return statutory_aime.oracle_aime( + dict(career), + int(birth_year), + params, + computation_years=ORACLE_CONVENTION, + ) + + +def require_track_a_convention( + track_a_convention: ComputationYears = TRACK_A_COMPUTATION_YEARS, +) -> None: + """Refuse to run unless Track A still uses :data:`ORACLE_CONVENTION`.""" + + if ComputationYears(track_a_convention) is not ORACLE_CONVENTION: + raise SystemExit( + "Track A's oracle computation years are " + f"{ComputationYears(track_a_convention).value}; this script " + f"compares against {ORACLE_CONVENTION.value} " + f"({ORACLE_COMPUTATION_YEARS} years) and must be updated first" + ) - return benefits.aime(dict(career), int(birth_year), params) + +def check_track_a_pia( + inputs: Iterable[PersonInput], params: SSAParameters +) -> None: + """Fail unless Track A's retirement PIA rests on :func:`oracle_aime`. + + Calls ``eligibility_pia_for_clock`` as Track A's assembly does, with + ``computation_years=TRACK_A_COMPUTATION_YEARS``. The function's + default is the statutory count, which gives a different PIA to people + born before 1929. + """ + + for person in inputs: + birth = person.selection.birth_year + track_a_pia = sb.eligibility_pia_for_clock( + sb.WorkerClock.at_age_62(birth), + history=person.career, + birth_year=birth, + params=params, + computation_years=TRACK_A_COMPUTATION_YEARS, + ) + if track_a_pia != benefits.pia( + oracle_aime(person.career, birth, params), + birth + _RETIREMENT_AGE, + params, + ): + raise ValueError(f"{person.selection.person_id}: PIA mismatch") def oracle_arithmetic_with_count( @@ -1003,7 +1066,8 @@ def render_results(result: Mapping[str, Any]) -> str: add( "- Oracle side: `ss.benefits.aime(career, birth_year, params)`, " "the call `scenario_benefits.eligibility_pia_for_clock` makes for " - "Track A's age-62 clock, with Track A's parameters " + "Track A's age-62 clock under Track A's computation-year " + "convention (`LEGACY_FIXED_35`), with Track A's parameters " f"(`{result['parameters']['track_a_revision']}`). It limits each " "year to the contribution and benefit base, indexes to the year of " "attaining 60, takes the highest 35 years (absent years count as " @@ -1382,6 +1446,7 @@ def _sha256_file(path: Path) -> str: "scripts/track_c_aime_agreement.py", "src/populace_dynamics/axiom_benefit_bridge.py", "src/populace_dynamics/ss/benefits.py", + "src/populace_dynamics/ss/statutory_aime.py", "src/populace_dynamics/ss/params.py", "src/populace_dynamics/scenario_benefits.py", "src/populace_dynamics/cola_track_a/benefits.py", @@ -1601,6 +1666,7 @@ def main(argv: list[str] | None = None) -> int: output = args.output_dir.expanduser() if (output / "result.json").exists(): raise SystemExit(f"{output / 'result.json'} exists; not overwriting") + require_track_a_convention() binding = bridge.reviewed_case_a_binding() binding.verify() params, base, parameter_info = _parameters() @@ -1655,20 +1721,7 @@ def main(argv: list[str] | None = None) -> int: for s in selected ] # Track A's PIA rests on exactly this AIME (eligibility_pia_for_clock). - for person in inputs: - birth = person.selection.birth_year - track_a_pia = sb.eligibility_pia_for_clock( - sb.WorkerClock.at_age_62(birth), - history=person.career, - birth_year=birth, - params=params, - ) - if track_a_pia != benefits.pia( - oracle_aime(person.career, birth, params), - birth + _RETIREMENT_AGE, - params, - ): - raise ValueError(f"{person.selection.person_id}: PIA mismatch") + check_track_a_pia(inputs, params) wage_index = track_a_wage_index(params) base_series = track_a_contribution_base( params, bridge.FIRST_COMPUTATION_BASE_YEAR, cutoff diff --git a/src/populace_dynamics/ss/statutory_aime.py b/src/populace_dynamics/ss/statutory_aime.py index 656651a2..351562f6 100644 --- a/src/populace_dynamics/ss/statutory_aime.py +++ b/src/populace_dynamics/ss/statutory_aime.py @@ -81,7 +81,9 @@ Amendment" under 42 USC 414 (law.cornell.edu/uscode/text/42/414, read 2026-09-24, page SHA-256 ``731e1dbc8648c689d2d97546980739a1a53477da0669d566d3d32ec9f3966a2d``; -bracketed text is the note's):: +the bracketed citation is the note's and ``[...]`` marks the rest of the +paragraph, omitted here; unlike the 415(b) text, no copy of this page is +saved with the evidence):: In the case of a man who attains age 62 prior to 1975, the number of his elapsed years for purposes of section 215(b)(3) of the Social @@ -124,6 +126,9 @@ 12-month rule of 415(a)(3)(B); a death after a disability entitlement is refused; * 104(j)(2) of the 1972 amendments and the 1960 condition (see above); +* dates within a year: the year in which the worker attains 21 or 62 is + taken to be the birth year plus that age (the oracle has birth years + only, as in ``ss.benefits``); * the choice of computation base years (415(b)(2)(B)(ii)): like ``benefits.aime``, the AIME ranks every year the caller supplies, and the caller applies the entitlement or death cutoff. Years before 1951 diff --git a/tests/README-tiers.md b/tests/README-tiers.md index d45ec7d5..8d492cb5 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -38,9 +38,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 3,034 | +| `unit` | 3,036 | | `artifact` | 2,801 | | `integration_psid` | 862 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 163 | -| **Total** | **7,380** | +| **Total** | **7,382** | diff --git a/tests/ss/test_statutory_aime.py b/tests/ss/test_statutory_aime.py index 7e75d507..d33dd950 100644 --- a/tests/ss/test_statutory_aime.py +++ b/tests/ss/test_statutory_aime.py @@ -393,6 +393,10 @@ def test_refuses_years_that_cannot_be_computation_base_years(): LEGACY_CONVENTION_USERS = { "src/populace_dynamics/cola_track_a/benefits.py", "src/populace_dynamics/ss/statutory_aime.py", + # Track C step 1 compares the Axiom engine with Track A's oracle AIME + # (track-c-aime-agreement-20260923) and refuses to run if Track A's + # convention changes. + "scripts/track_c_aime_agreement.py", } diff --git a/tests/test_track_c_aime_agreement.py b/tests/test_track_c_aime_agreement.py index 3495abeb..ee34f69b 100644 --- a/tests/test_track_c_aime_agreement.py +++ b/tests/test_track_c_aime_agreement.py @@ -19,8 +19,11 @@ import pytest import populace_dynamics.axiom_benefit_bridge as bridge +from populace_dynamics import scenario_benefits as sb +from populace_dynamics.cola_track_a.benefits import TRACK_A_COMPUTATION_YEARS from populace_dynamics.ss import benefits from populace_dynamics.ss.params import SSAParameters +from populace_dynamics.ss.statutory_aime import ComputationYears ROOT = Path(__file__).resolve().parents[1] @@ -218,6 +221,56 @@ def test_oracle_arithmetic_at_35_years_is_the_oracle(): assert fewer > tc.oracle_aime(career, 1930, INVENTED_PARAMS) +def test_the_oracle_side_is_track_a_s_legacy_convention(): + # The attribution counts disagreements "instead of the oracle's fixed + # 35", so the oracle side must be Track A's legacy convention. + assert tc.ORACLE_CONVENTION is ComputationYears.LEGACY_FIXED_35 + assert TRACK_A_COMPUTATION_YEARS is tc.ORACLE_CONVENTION + assert tc.ORACLE_COMPUTATION_YEARS == 35 + tc.require_track_a_convention() + tc.require_track_a_convention(ComputationYears.LEGACY_FIXED_35) + with pytest.raises(SystemExit, match="statutory_415_b_2"): + tc.require_track_a_convention(ComputationYears.STATUTORY) + # INVENTED career of a person born 1925 (31 statutory computation + # years): the oracle side is still the fixed-35 benefits.aime. + career = _career(1925) + assert tc.oracle_aime(career, 1925, INVENTED_PARAMS) == benefits.aime( + career, 1925, INVENTED_PARAMS + ) + + +def test_track_a_pia_check_passes_track_a_s_computation_years(monkeypatch): + # INVENTED people born 1925 and 1960. The check calls + # eligibility_pia_for_clock as Track A's assembly does; its default + # (the statutory count) gives the 1925 person another PIA, so a call + # without computation_years would stop the real run with a PIA + # mismatch for everyone born before 1929. + people = [_person(1, 1925), _person(2, 1960)] + tc.check_track_a_pia(people, INVENTED_PARAMS) + early = people[0] + statutory = sb.eligibility_pia_for_clock( + sb.WorkerClock.at_age_62(1925), + history=early.career, + birth_year=1925, + params=INVENTED_PARAMS, + ) + oracle = benefits.pia( + tc.oracle_aime(early.career, 1925, INVENTED_PARAMS), + 1987, + INVENTED_PARAMS, + ) + assert statutory != oracle + # A PIA that does not rest on the oracle AIME is refused. + original = tc.oracle_aime + monkeypatch.setattr( + tc, + "oracle_aime", + lambda career, birth, params: original(career, birth, params) + 100, + ) + with pytest.raises(ValueError, match="1: PIA mismatch"): + tc.check_track_a_pia(people, INVENTED_PARAMS) + + def test_transport_zero_fills_outside_the_panel_and_labels_rows(): career = _career(1930) career[1975] = 90000.0 # INVENTED amount above the invented base diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 74f2cd45..154a64d4 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 3034, + "unit": 3036, "artifact": 2801, "integration_psid": 862, "reproduction_legacy": 520, From 71c9474353388f930dd136d77f3fbbb492266621 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 24 Sep 2026 05:58:07 -0400 Subject: [PATCH 3/4] Track C AIME agreement: bind to the Track A artifact by recorded key; make the oracle's computation years a choice On master, scripts/track_c_aime_agreement.py stopped before any engine call. Its check that the cohort is the one in the registered Track A artifact (runs/replication_urban2010_cola_v1.json, committed in 7b580dc9) compared the whole cohort diagnostics dict. bfe9fa3e (merged in #454) later added opening_stock_entitlement_clamped_by_clock_rule to the diagnostics, so the comparison failed on that key alone. Artifact binding - track_a_artifact_binding compares every diagnostic the artifact records and refuses a missing or changed one. The source provenance and the parameter revision are still compared as whole values. - Diagnostics the artifact does not record are allowed and returned as additive keys. An additive _by_ key must be a mapping of integers summing to the recorded . - result.json records the compared keys, the additive keys and their reconciliation; RESULTS.md lists them. require_track_a_artifact raises SystemExit naming each failed check, missing key and changed key. - A 20-person smoke run on the real cohort (--limit, not kept) passed every check. It found one additive key, whose clock-rule counts (1, 2, 46) sum to the recorded 49. Oracle choice - --oracle-computation-years statutory|legacy_fixed_35, default statutory. statutory is ss.statutory_aime's 42 USC 415(b)(2) count. legacy_fixed_35 is ss.benefits.aime: Track A's convention and the oracle of the 2026-09-23 report. - The choice is named in the header, the labels, a new "oracle" record, and the not-compared list. RESULTS.md names it in the title, the result paragraph, the oracle-side description and the attribution note. - Each record now has oracle_aime (the chosen oracle's AIME), track_a_oracle_aime (always Track A's fixed-35 AIME) and oracle_computation_years. Summaries count the people whose two AIMEs differ. - The count attribution applies only when the candidate's count differs from the oracle's. Under the statutory oracle the two counts are equal (tested for births 1913-2000), so it never applies there. - check_oracle_pia checks eligibility_pia_for_clock against the recorded AIME under both the chosen convention and Track A's. Its record is kept in result.json. - The script still refuses to run if Track A's convention stops being LEGACY_FIXED_35. - Schema v2. --render-only refuses a v1 result, whose prose belongs to the script at the commit that result records. Tests (INVENTED data) - The additive-key regression, and refusal of missing, changed and non-summing keys, with the SystemExit message. - The oracle choice and its parser default; classification under both oracles; the PIA check under both conventions and without the computation_years pin; the rendered oracle names; and the actual-engine test under both oracles. - New artifact-tier module tests/test_track_c_track_a_artifact_binding.py reads the committed artifact's recorded keys. They are a subset of what the current code produces for INVENTED people, and the recorded 2011 cohort binds when an INVENTED split of its clamped total is added. - These regression tests fail against the previous script. Tier manifest recollected: unit 3047, artifact 2803, total 7395. Co-Authored-By: Claude Opus 5.5 --- scripts/track_c_aime_agreement.py | 710 ++++++++++++++---- tests/README-tiers.md | 6 +- tests/ss/test_statutory_aime.py | 8 +- tests/test_track_c_aime_agreement.py | 535 ++++++++++--- .../test_track_c_track_a_artifact_binding.py | 122 +++ tests/tier_counts.json | 4 +- 6 files changed, 1131 insertions(+), 254 deletions(-) create mode 100644 tests/test_track_c_track_a_artifact_binding.py diff --git a/scripts/track_c_aime_agreement.py b/scripts/track_c_aime_agreement.py index 207565bd..0b61a5f9 100644 --- a/scripts/track_c_aime_agreement.py +++ b/scripts/track_c_aime_agreement.py @@ -3,16 +3,27 @@ For every person of the real Track A 2011-wave PSID cohort whose AIME the transitional Python oracle computes in Track A, this script records -* the oracle's AIME, computed exactly as Track A computes it - (``ss.benefits.aime`` over the person's Track A career, indexed to the - year of attaining 60: the call ``scenario_benefits. - eligibility_pia_for_clock`` makes for the age-62 clock under Track A's - computation-year convention, ``cola_track_a.benefits. - TRACK_A_COMPUTATION_YEARS`` = ``ComputationYears.LEGACY_FIXED_35``; the - function's default, the statutory 42 USC 415(b)(2) count, differs for - people born before 1929), with Track A's - statutory parameters (the committed capture before 1975, the TR2008 - intermediate AWI from 1975: ``cola_track_a.runner.tr2008_ssa_parameters``); +* the oracle's AIME over the person's Track A career, indexed to the year + of attaining 60, under the computation-year convention chosen with + ``--oracle-computation-years`` (every output names it): + + - ``statutory`` (the default): ``ss.statutory_aime.oracle_aime`` with + ``ComputationYears.STATUTORY``, the 42 USC 415(b)(2) count (elapsed + years less 5, at least 2), which is also the default of + ``scenario_benefits.eligibility_pia_for_clock``; + - ``legacy_fixed_35``: ``ComputationYears.LEGACY_FIXED_35``, the + unchanged ``ss.benefits.aime`` (always 35 years). This is the call + ``eligibility_pia_for_clock`` makes for the age-62 clock under Track + A's own convention (``cola_track_a.benefits. + TRACK_A_COMPUTATION_YEARS``); it was the oracle of the 2026-09-23 + report (``track-c-aime-agreement-20260923``). + + The two can differ only for people born before 1929, who have fewer + than 35 statutory computation years. Track A's own AIME (the legacy + convention) is recorded beside the chosen oracle's for every person. + Both use Track A's statutory parameters (the committed capture before + 1975, the TR2008 intermediate AWI from 1975: + ``cola_track_a.runner.tr2008_ssa_parameters``); * the AIME the actual Axiom rules engine returns for the same person, through :mod:`populace_dynamics.axiom_benefit_bridge` (the pinned engine binary and compiled 415(b) candidate that reproduced SSA's published @@ -57,12 +68,24 @@ The attribution check (verification only, not a third model) reruns the oracle's own creditable and indexed history with the candidate's number -of benefit computation years instead of the oracle's fixed 35. +of benefit computation years instead of the oracle's. Under the legacy +oracle (35 years) it can explain disagreements for people born before +1929; the statutory oracle already uses the candidate's count, so under +it no disagreement is attributed to the count. + +Binding to the registered Track A run. The cohort and parameters must be +those of ``runs/replication_urban2010_cola_v1.json``: every cohort +diagnostic the artifact records must be present with an equal value, and +its source provenance and parameter revision must be equal. Diagnostics +the current code adds that the artifact does not record are allowed, +recorded in the output, and, where one is a breakdown of a recorded total +(``_by_``), must sum to it. Usage:: python scripts/track_c_aime_agreement.py \\ - --output-dir [--workers 4] [--allow-dirty] + --output-dir [--workers 4] [--allow-dirty] \\ + [--oracle-computation-years statutory|legacy_fixed_35] python scripts/track_c_aime_agreement.py --render-only Writes ``result.json`` and ``RESULTS.md`` (and ``executions/``) there. @@ -104,32 +127,36 @@ ComputationYears, ) -SCHEMA_VERSION = "populace_dynamics.track_c.aime_agreement.v1" -HEADER = ( - "Track C step 1: per-person AIME agreement between the actual Axiom " - "rules engine and the transitional Python oracle on the real Track A " - "PSID cohort. The Axiom RuleSpec candidates are UNACCEPTED. This is " - "not the COLA statistic: no benefit path, reform or age-profile value " - "is computed." -) -LABELS = ( - "PSID-seeded closed cohort", - "Axiom RuleSpec candidates unaccepted (diagnostic only)", - "Python oracle: transitional, not Axiom", - "not the COLA statistic", -) +#: v2 (2026-09-24): the oracle's computation years are a recorded choice +#: (``oracle``), ``oracle_aime`` is the chosen oracle's AIME and +#: ``track_a_oracle_aime`` always Track A's, and the Track A artifact +#: binding records the diagnostics compared and the additive ones. +SCHEMA_VERSION = "populace_dynamics.track_c.aime_agreement.v2" DEFAULT_OUTPUT_DIR = Path( "~/microcosm-launch-evidence/dynasim-parity-20260909/" "track-c-aime-agreement-20260923" ) #: The first year of the PSID family earnings panel (career law). PANEL_FIRST_YEAR = 1968 -#: The oracle's fixed benefit computation years (ss.benefits). -ORACLE_COMPUTATION_YEARS = 35 -#: The computation-year convention of the oracle side: Track A's. The -#: attribution and the report text assume it is the legacy fixed 35 -#: (:data:`ORACLE_COMPUTATION_YEARS`); :func:`main` refuses to run otherwise. -ORACLE_CONVENTION = ComputationYears.LEGACY_FIXED_35 +#: Track A's fixed benefit computation years (``ss.benefits.aime``). +LEGACY_COMPUTATION_YEARS = statutory_aime.LEGACY_FIXED_COMPUTATION_YEARS +#: ``--oracle-computation-years`` choices. +ORACLE_CHOICES: Mapping[str, ComputationYears] = { + "statutory": ComputationYears.STATUTORY, + "legacy_fixed_35": ComputationYears.LEGACY_FIXED_35, +} +#: The default oracle: the statutory 42 USC 415(b)(2) count. +DEFAULT_ORACLE = "statutory" +#: What each oracle is, in the words every output uses. +ORACLE_NAMES: Mapping[ComputationYears, str] = { + ComputationYears.STATUTORY: ( + "statutory computation years (42 USC 415(b)(2), " + "ss.statutory_aime), not Track A's fixed 35" + ), + ComputationYears.LEGACY_FIXED_35: ( + "Track A's legacy fixed 35 computation years (ss.benefits.aime)" + ), +} _RETIREMENT_AGE = 62 _INDEXING_AGE = 60 _MONTHS = 12 @@ -331,63 +358,179 @@ def candidate_computation_years(birth_year: int) -> int: return max(2, elapsed - 5) +def oracle_convention(choice: str | ComputationYears) -> ComputationYears: + """The convention of an ``--oracle-computation-years`` choice.""" + + if isinstance(choice, ComputationYears): + return choice + try: + return ORACLE_CHOICES[str(choice)] + except KeyError: + raise ValueError( + f"unknown oracle computation years {choice!r}; choose one of " + f"{', '.join(ORACLE_CHOICES)}" + ) from None + + +def oracle_choice(convention: ComputationYears) -> str: + """The ``--oracle-computation-years`` value of a convention.""" + + for choice, value in ORACLE_CHOICES.items(): + if value is ComputationYears(convention): + return choice + raise ValueError(f"no oracle choice for {convention!r}") + + +def header_for(convention: ComputationYears) -> str: + """The result header, naming the oracle compared against.""" + + return ( + "Track C step 1: per-person AIME agreement between the actual Axiom " + "rules engine and the transitional Python oracle on the real Track " + "A PSID cohort. Oracle: " + + ORACLE_NAMES[ComputationYears(convention)] + + ". The Axiom RuleSpec candidates are UNACCEPTED. This is not the " + "COLA statistic: no benefit path, reform or age-profile value is " + "computed." + ) + + +def labels_for(convention: ComputationYears) -> tuple[str, ...]: + """The result labels, the oracle's among them.""" + + return ( + "PSID-seeded closed cohort", + "Axiom RuleSpec candidates unaccepted (diagnostic only)", + "Python oracle: transitional, not Axiom", + "oracle: " + ORACLE_NAMES[ComputationYears(convention)], + "not the COLA statistic", + ) + + +def oracle_document(convention: ComputationYears) -> dict[str, Any]: + """What ``result.json`` records about the oracle compared against.""" + + convention = ComputationYears(convention) + return { + "choice": oracle_choice(convention), + "computation_years": convention.value, + "name": ORACLE_NAMES[convention], + "call": ( + "ss.statutory_aime.oracle_aime(career, birth_year, params, " + f"computation_years=ComputationYears.{convention.name})" + ), + "track_a_computation_years": TRACK_A_COMPUTATION_YEARS.value, + "is_track_a_convention": convention is TRACK_A_COMPUTATION_YEARS, + "choices": { + choice: value.value for choice, value in ORACLE_CHOICES.items() + }, + "default_choice": DEFAULT_ORACLE, + } + + def oracle_aime( - career: Mapping[int, float], birth_year: int, params: SSAParameters + career: Mapping[int, float], + birth_year: int, + params: SSAParameters, + *, + convention: ComputationYears, ) -> int: - """Track A's oracle AIME (``eligibility_pia_for_clock``'s call). + """The oracle's age-62 AIME under a named convention. - ``ss.statutory_aime.oracle_aime`` under :data:`ORACLE_CONVENTION`, - which is ``ss.benefits.aime`` (always 35 years). + ``ss.statutory_aime.oracle_aime``: ``STATUTORY`` is the 42 USC + 415(b)(2) count, ``LEGACY_FIXED_35`` is ``ss.benefits.aime`` (always + 35 years), Track A's. """ return statutory_aime.oracle_aime( dict(career), int(birth_year), params, - computation_years=ORACLE_CONVENTION, + computation_years=ComputationYears(convention), ) +def oracle_computation_year_count( + birth_year: int, convention: ComputationYears +) -> int: + """How many computation years the oracle divides by for a person.""" + + if ComputationYears(convention) is ComputationYears.LEGACY_FIXED_35: + return LEGACY_COMPUTATION_YEARS + return statutory_aime.benefit_computation_years(int(birth_year)) + + def require_track_a_convention( track_a_convention: ComputationYears = TRACK_A_COMPUTATION_YEARS, ) -> None: - """Refuse to run unless Track A still uses :data:`ORACLE_CONVENTION`.""" + """Refuse to run unless Track A still uses the legacy fixed 35. + + The report text names Track A's AIME as the fixed-35 one under either + oracle choice, and the ``legacy_fixed_35`` choice is described as + Track A's own. + """ - if ComputationYears(track_a_convention) is not ORACLE_CONVENTION: + convention = ComputationYears(track_a_convention) + if convention is not ComputationYears.LEGACY_FIXED_35: raise SystemExit( - "Track A's oracle computation years are " - f"{ComputationYears(track_a_convention).value}; this script " - f"compares against {ORACLE_CONVENTION.value} " - f"({ORACLE_COMPUTATION_YEARS} years) and must be updated first" + f"Track A's oracle computation years are {convention.value}; " + "this script describes Track A's AIME as " + f"{ComputationYears.LEGACY_FIXED_35.value} " + f"({LEGACY_COMPUTATION_YEARS} years) and must be updated first" ) -def check_track_a_pia( - inputs: Iterable[PersonInput], params: SSAParameters -) -> None: - """Fail unless Track A's retirement PIA rests on :func:`oracle_aime`. - - Calls ``eligibility_pia_for_clock`` as Track A's assembly does, with - ``computation_years=TRACK_A_COMPUTATION_YEARS``. The function's - default is the statutory count, which gives a different PIA to people - born before 1929. +def check_oracle_pia( + inputs: Iterable[PersonInput], + params: SSAParameters, + convention: ComputationYears, +) -> dict[str, Any]: + """Fail unless the oracle PIAs rest on the AIMEs this script records. + + For every person and for both the chosen convention and Track A's + (``TRACK_A_COMPUTATION_YEARS``, as Track A's assembly passes it), + ``eligibility_pia_for_clock`` with that ``computation_years`` must + equal ``benefits.pia`` of :func:`oracle_aime` under it. Under the + legacy choice the two conventions are one. The function's default is + the statutory count, so a call that dropped ``computation_years`` + would fail the legacy check for people born before 1929. """ + conventions = tuple( + dict.fromkeys( + (ComputationYears(convention), TRACK_A_COMPUTATION_YEARS) + ) + ) + persons = 0 for person in inputs: + persons += 1 birth = person.selection.birth_year - track_a_pia = sb.eligibility_pia_for_clock( - sb.WorkerClock.at_age_62(birth), - history=person.career, - birth_year=birth, - params=params, - computation_years=TRACK_A_COMPUTATION_YEARS, - ) - if track_a_pia != benefits.pia( - oracle_aime(person.career, birth, params), - birth + _RETIREMENT_AGE, - params, - ): - raise ValueError(f"{person.selection.person_id}: PIA mismatch") + for checked in conventions: + pia = sb.eligibility_pia_for_clock( + sb.WorkerClock.at_age_62(birth), + history=person.career, + birth_year=birth, + params=params, + computation_years=checked, + ) + if pia != benefits.pia( + oracle_aime(person.career, birth, params, convention=checked), + birth + _RETIREMENT_AGE, + params, + ): + raise ValueError( + f"{person.selection.person_id}: PIA mismatch " + f"({checked.value})" + ) + return { + "persons": persons, + "conventions_checked": [checked.value for checked in conventions], + "check": ( + "scenario_benefits.eligibility_pia_for_clock(age-62 clock, " + "computation_years=c) == ss.benefits.pia(oracle AIME under c, " + "year of attaining 62) for every person and convention c" + ), + } def oracle_arithmetic_with_count( @@ -400,7 +543,8 @@ def oracle_arithmetic_with_count( Verification only: ``ss.benefits`` creditable and indexed histories, the highest ``computation_years`` values (zero-padded), divided by - ``12 * computation_years`` and floored. With 35 it is the oracle. + ``12 * computation_years`` and floored. With 35 it is the legacy + oracle, with the statutory count the statutory one. """ creditable = benefits.creditable_history(dict(career), params) @@ -580,6 +724,7 @@ class PassContext: binding: bridge.AxiomEngineBinding | None cutoff_year: int anchor_wave: int + oracle_convention: ComputationYears runner: bridge.Runner | None = None pia_check_params: SSAParameters | None = None @@ -618,7 +763,11 @@ def compare_person( entitlement = _entitlement_for_pass(context, person) if entitlement is None: raise ValueError(f"{selection.person_id}: no entitlement year") - track_a_aime = oracle_aime(person.career, birth, context.params) + convention = ComputationYears(context.oracle_convention) + full_oracle = oracle_aime( + person.career, birth, context.params, convention=convention + ) + oracle_count = oracle_computation_year_count(birth, convention) record: dict[str, Any] = { "person_id": selection.person_id, "birth_year": birth, @@ -627,19 +776,28 @@ def compare_person( "eligibility_year": birth + _RETIREMENT_AGE, "indexing_year": birth + _INDEXING_AGE, "entitlement_year_sent": entitlement, - "track_a_oracle_aime": track_a_aime, + "oracle_computation_years": oracle_count, + "oracle_aime": full_oracle, + "track_a_oracle_aime": oracle_aime( + person.career, + birth, + context.params, + convention=TRACK_A_COMPUTATION_YEARS, + ), } compared_career = { year: value for year, value in person.career.items() if int(year) < entitlement } - comparison_oracle = track_a_aime + comparison_oracle = full_oracle if len(compared_career) != len(person.career): - comparison_oracle = oracle_aime(compared_career, birth, context.params) + comparison_oracle = oracle_aime( + compared_career, birth, context.params, convention=convention + ) record["oracle_aime_on_sent_years"] = comparison_oracle record["truncation_effect_on_oracle"] = comparison_oracle - ( - track_a_aime + full_oracle ) count = candidate_computation_years(birth) record["candidate_computation_years"] = count @@ -711,9 +869,7 @@ def compare_person( ) if difference == 0: attribution = ATTR_EXACT - elif count != ORACLE_COMPUTATION_YEARS and result.aime == Decimal( - at_count - ): + elif count != oracle_count and result.aime == Decimal(at_count): attribution = ATTR_COUNT else: attribution = ATTR_UNEXPLAINED @@ -835,6 +991,9 @@ def summarize_records(records: Sequence[Mapping[str, Any]]) -> dict: ) return { "persons": len(records), + "oracle_aime_differs_from_track_a": sum( + r["oracle_aime"] != r["track_a_oracle_aime"] for r in records + ), "status": dict(sorted(status.items())), "executed": len(executed), "exact_matches": exact, @@ -977,8 +1136,19 @@ def _row(*cells: Any) -> str: def render_results(result: Mapping[str, Any]) -> str: - """RESULTS.md from ``result.json`` (deterministic; no new numbers).""" + """RESULTS.md from ``result.json`` (deterministic; no new numbers). + + Refuses a result without an ``oracle`` record (schema v1, written + before the oracle choice): its report text belongs to the script at + the commit its provenance records. + """ + if "oracle" not in result: + raise ValueError( + "result.json records no oracle choice (schema " + f"{result.get('schema_version', 'unknown')}); render it with the " + "script at the commit its provenance records" + ) lines: list[str] = [] add = lines.append passes = result["passes"] @@ -986,7 +1156,14 @@ def render_results(result: Mapping[str, Any]) -> str: overall = primary["overall"] selection = result["selection"] cohort = result["cohort"] - title = "# Track C step 1: AIME agreement, Axiom engine vs Python oracle" + oracle = result["oracle"] + convention = ComputationYears(oracle["computation_years"]) + statutory = convention is ComputationYears.STATUTORY + title = ( + "# Track C step 1: AIME agreement, Axiom engine vs Python oracle " + f"({'statutory' if statutory else 'legacy fixed-35'} computation " + "years)" + ) add(title) add("") if result.get("partial"): @@ -995,6 +1172,11 @@ def render_results(result: Mapping[str, Any]) -> str: "was compared. Do not cite these numbers." ) add("") + add( + f"**Oracle: {oracle['name']}** " + f"(`--oracle-computation-years {oracle['choice']}`)." + ) + add("") add( "**The Axiom RuleSpec candidates are unaccepted.** Agreement is " "diagnostic only. The Python oracle is transitional (not Axiom). " @@ -1006,10 +1188,26 @@ def render_results(result: Mapping[str, Any]) -> str: add("## Result") add("") st = overall["status"] - count35 = primary["by_candidate_computation_years"]["35_computation_years"] + by_count = primary["by_candidate_computation_years"] + count35 = by_count["35_computation_years"] + fewer = by_count["fewer_than_35_computation_years"] attribution = overall["attribution"] + if statutory: + attribution_text = ( + "oracle's own arithmetic with the candidate's computation-year " + "count (elapsed years minus 5, fewer than 35 for people born " + "before 1929; the statutory oracle uses that count too, so no " + "disagreement can be attributed to it) and " + ) + else: + attribution_text = ( + "oracle's own arithmetic with the candidate's computation-year " + "count (elapsed years minus 5, fewer than 35 for people born " + "before 1929; the legacy oracle always uses 35) and " + ) add( - f"Primary pass: {overall['executed']} of {overall['persons']} " + f"Primary pass, oracle with {oracle['name']}: " + f"{overall['executed']} of {overall['persons']} " f"selected persons were executed on the engine " f"({st.get('refused', 0)} refused, {st.get('failed', 0)} failed). " f"The engine's AIME equals the oracle's for " @@ -1017,15 +1215,28 @@ def render_results(result: Mapping[str, Any]) -> str: f"({_pct(overall['exact_match_share_of_executed'])}); among the " f"{count35['executed']} executed people with 35 candidate " f"computation years (born 1929 or later) it equals the oracle's " - f"for {count35['exact_matches']}. Of the " + f"for {count35['exact_matches']}, and among the " + f"{fewer['executed']} with fewer (born before 1929) for " + f"{fewer['exact_matches']}. Of the " f"{overall['nonzero_differences']['count']} disagreements, " f"{attribution.get(ATTR_COUNT, 0)} are reproduced exactly by the " - "oracle's own arithmetic with the candidate's computation-year " - "count (elapsed years minus 5, fewer than 35 for people born " - "before 1929; the oracle always uses 35) and " - f"{attribution.get(ATTR_UNEXPLAINED, 0)} are unexplained." + + attribution_text + + f"{attribution.get(ATTR_UNEXPLAINED, 0)} are unexplained." ) add("") + if "oracle_aime_differs_from_track_a" in overall: + add( + "This oracle's AIME differs from Track A's (the fixed-35 " + "`LEGACY_FIXED_35` AIME, recorded per person as " + f"`track_a_oracle_aime`) for " + f"{overall['oracle_aime_differs_from_track_a']} of the " + f"{overall['persons']} selected people: " + f"{fewer.get('oracle_aime_differs_from_track_a', 0)} of the " + f"{fewer['persons']} with fewer than 35 candidate computation " + f"years and {count35.get('oracle_aime_differs_from_track_a', 0)}" + f" of the {count35['persons']} with 35." + ) + add("") add( "The inputs are the cohort's PSID careers under declarations both " "sides share (below); agreement shows the engine and the oracle " @@ -1063,16 +1274,37 @@ def render_results(result: Mapping[str, Any]) -> str: ) + "." ) - add( - "- Oracle side: `ss.benefits.aime(career, birth_year, params)`, " - "the call `scenario_benefits.eligibility_pia_for_clock` makes for " - "Track A's age-62 clock under Track A's computation-year " - "convention (`LEGACY_FIXED_35`), with Track A's parameters " - f"(`{result['parameters']['track_a_revision']}`). It limits each " - "year to the contribution and benefit base, indexes to the year of " - "attaining 60, takes the highest 35 years (absent years count as " - "zero) and floors the total over 420 months." - ) + if statutory: + add( + "- Oracle side: `ss.statutory_aime.oracle_aime(career, " + "birth_year, params, computation_years=ComputationYears." + "STATUTORY)`, the default of " + "`scenario_benefits.eligibility_pia_for_clock` for the age-62 " + "clock (Track A itself passes the legacy fixed 35), with Track " + f"A's parameters (`{result['parameters']['track_a_revision']}`)." + " It limits each year to the contribution and benefit base, " + "indexes to the year of attaining 60, takes the highest n years " + "(absent years count as zero), n being the elapsed years minus " + "5 (42 USC 415(b)(2); 35 for people born 1929 or later), and " + "floors the total over 12n months. Track A's fixed-35 AIME is " + "recorded beside it for every person but is not what the engine " + "is compared with." + ) + else: + add( + "- Oracle side: `ss.benefits.aime(career, birth_year, params)` " + "(`ss.statutory_aime.oracle_aime` with `computation_years=" + "ComputationYears.LEGACY_FIXED_35`), the call " + "`scenario_benefits.eligibility_pia_for_clock` makes for Track " + "A's age-62 clock under Track A's computation-year convention, " + "with Track A's parameters " + f"(`{result['parameters']['track_a_revision']}`). It limits " + "each year to the contribution and benefit base, indexes to the " + "year of attaining 60, takes the highest 35 years (absent years " + "count as zero) and floors the total over 420 months. For " + "people born before 1929, 42 USC 415(b)(2) gives fewer than 35 " + "computation years (the `statutory` choice)." + ) add( "- Axiom side: `axiom_benefit_bridge` requests on the pinned " "engine (hashes below). The 415(b) candidate counts benefit " @@ -1167,8 +1399,14 @@ def summary_row(label: str, s: Mapping[str, Any]) -> str: add( f"`{ATTR_COUNT}` means the engine's AIME equals the oracle's own " "arithmetic (its creditable and indexed history) rerun with the " - "candidate's number of computation years instead of 35. " - f"`{ATTR_UNEXPLAINED}` lists every other disagreement; their " + "candidate's number of computation years instead of the oracle's " + + ( + "(the statutory oracle already uses the candidate's count, so " + "no record can have this attribution). " + if statutory + else "(35, the legacy oracle's). " + ) + + f"`{ATTR_UNEXPLAINED}` lists every other disagreement; their " "requests and engine outputs are in `executions/` (up to the cap " "recorded under `executions` in `result.json`; each written " "record names its files)." @@ -1303,9 +1541,9 @@ def summary_row(label: str, s: Mapping[str, Any]) -> str: add(f"Attribution: {json.dumps(d['attribution'])}.") add("") add( - "Effect of the truncation on the oracle itself (oracle AIME on " - "the years before the recorded entitlement minus Track A's " - "full-career oracle AIME): " + "Effect of the truncation on the oracle itself (the oracle's " + "AIME on the years before the recorded entitlement minus its " + "full-career AIME, both with the oracle's computation years): " f"{conv['persons_with_career_years_dropped']} persons have " f"career years at or after entitlement, " f"{conv['persons_with_nonzero_years_dropped']} of them nonzero; " @@ -1327,6 +1565,21 @@ def summary_row(label: str, s: Mapping[str, Any]) -> str: "(69,846.57), which gives " f"{pia['published_bend_points_2026']}." ) + consistency = result.get("oracle_pia_consistency") + if consistency is not None: + add("") + add( + "Before any engine call the script checked, for all " + f"{consistency['persons']} selected people, that " + "`scenario_benefits.eligibility_pia_for_clock` rests on the " + "AIMEs recorded here under " + + " and ".join( + f"`{value}`" for value in consistency["conventions_checked"] + ) + + " (its PIA equals `ss.benefits.pia` of that AIME at the year " + "of attaining 62). The legacy convention is the one Track A's " + "assembly passes." + ) add("") add( f"Formula check only, not a Track A quantity: for the " @@ -1384,13 +1637,7 @@ def summary_row(label: str, s: Mapping[str, Any]) -> str: ) artifact = cohort.get("track_a_artifact") if artifact is not None: - add( - f"- The cohort's diagnostics and source provenance and the " - f"parameter revision equal those recorded in `{artifact['path']}`" - f" (SHA-256 `{artifact['sha256']}`; checks " - f"{json.dumps(artifact['checks'])}); only those fields of the " - "artifact were used." - ) + add(_artifact_binding_text(artifact)) add( f"- Wage index series digest `{result['parameters']['wage_index']['source_sha256']}`" f" ({result['parameters']['wage_index']['years']}); contribution " @@ -1402,7 +1649,8 @@ def summary_row(label: str, s: Mapping[str, Any]) -> str: add("") add( "- `result.json`: every selected person in every pass (status, " - "both AIMEs, the difference, attribution, row counts by kind, " + "the oracle's, Track A's and the engine's AIMEs, the difference, " + "attribution, row counts by kind, " "request and stdout digests, refusal reasons), the summaries, the " "declarations and the provenance." ) @@ -1513,6 +1761,38 @@ def _json_normal(value: Any) -> Any: return json.loads(json.dumps(value, sort_keys=True, default=str)) +#: An additive diagnostic ``_by_`` breaks a recorded total +#: down (the naming ``cola_track_a/opening.py`` uses). +_BREAKDOWN_SEPARATOR = "_by_" + + +def _is_count(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def _breakdown_check( + key: str, value: Any, recorded: Mapping[str, Any] +) -> dict[str, Any] | None: + """How an additive ``_by_`` key reconciles, if it is one.""" + + if _BREAKDOWN_SEPARATOR not in key: + return None + total_key = key.split(_BREAKDOWN_SEPARATOR, 1)[0] + if total_key not in recorded: + return None + total = recorded[total_key] + parts = list(value.values()) if isinstance(value, Mapping) else None + countable = ( + parts is not None and all(map(_is_count, parts)) and _is_count(total) + ) + return { + "total_key": total_key, + "recorded_total": total, + "sum": sum(parts) if countable else None, + "sums_to_recorded_total": bool(countable and sum(parts) == total), + } + + def track_a_artifact_binding( artifact: Mapping[str, Any], *, @@ -1520,27 +1800,128 @@ def track_a_artifact_binding( diagnostics: Mapping[str, Any], source_provenance: Mapping[str, Any], parameters_revision: str, -) -> dict[str, bool]: - """Checks that the cohort and parameters are the committed Track A run's. +) -> dict[str, Any]: + """Compare the cohort and parameters with the committed Track A run. Reads only the artifact's parameter revision and its cohort - diagnostics; nothing about benefits or age groups. + diagnostics and source provenance; nothing about benefits or age + groups. Every diagnostic the artifact records must be present in + ``diagnostics`` with an equal value (compared as JSON); a missing or + changed one fails ``cohort_diagnostics_equal_on_recorded_keys``. + Diagnostics the artifact does not record (keys the code added after + it was written) are allowed and returned under + ``additive_diagnostic_keys``; one named ``_by_`` for a + recorded integer ```` must be a mapping of integers summing to + it (``additive_breakdowns_sum_to_recorded_totals``). The source + provenance and the parameter revision must be equal as a whole. + + Returns ``checks`` (all must be true) and what was compared. """ - recorded = dict(artifact["cohorts"][str(anchor_wave)]) + recorded = _json_normal(dict(artifact["cohorts"][str(anchor_wave)])) recorded_source = recorded.pop("source_provenance") - return { + fresh = _json_normal(dict(diagnostics)) + missing = sorted(set(recorded) - set(fresh)) + changed = sorted( + key for key in recorded if key in fresh and fresh[key] != recorded[key] + ) + additive = {key: fresh[key] for key in sorted(set(fresh) - set(recorded))} + breakdowns = { + key: _breakdown_check(key, value, recorded) + for key, value in additive.items() + } + checks = { "ssa_parameters_revision_equal": ( artifact["ssa_parameters_revision"] == parameters_revision ), - "cohort_diagnostics_equal": ( - _json_normal(recorded) == _json_normal(dict(diagnostics)) + "cohort_diagnostics_equal_on_recorded_keys": not missing + and not changed, + "additive_breakdowns_sum_to_recorded_totals": all( + check["sums_to_recorded_total"] + for check in breakdowns.values() + if check is not None ), "cohort_source_provenance_equal": ( - _json_normal(recorded_source) - == _json_normal(dict(source_provenance)) + recorded_source == _json_normal(dict(source_provenance)) ), } + return { + "checks": checks, + "recorded_diagnostic_keys": sorted(recorded), + "recorded_keys_missing": missing, + "recorded_keys_changed": { + key: {"recorded": recorded[key], "fresh": fresh[key]} + for key in changed + }, + "additive_diagnostic_keys": additive, + "additive_breakdowns": breakdowns, + } + + +def require_track_a_artifact( + artifact_bytes: bytes, + *, + path: Path | str = TRACK_A_ARTIFACT, + anchor_wave: int, + diagnostics: Mapping[str, Any], + source_provenance: Mapping[str, Any], + parameters_revision: str, +) -> dict[str, Any]: + """The recorded binding, or ``SystemExit`` naming every failed check.""" + + binding = track_a_artifact_binding( + json.loads(artifact_bytes), + anchor_wave=anchor_wave, + diagnostics=diagnostics, + source_provenance=source_provenance, + parameters_revision=parameters_revision, + ) + if not all(binding["checks"].values()): + failed = sorted(k for k, ok in binding["checks"].items() if not ok) + raise SystemExit( + f"the cohort or parameters differ from {path}: failed {failed}; " + f"recorded keys missing {binding['recorded_keys_missing']}; " + f"recorded keys changed {binding['recorded_keys_changed']}; " + f"additive breakdowns {binding['additive_breakdowns']}" + ) + return { + "path": str(path), + "sha256": hashlib.sha256(artifact_bytes).hexdigest(), + **binding, + } + + +def _artifact_binding_text(artifact: Mapping[str, Any]) -> str: + """The RESULTS.md line for the Track A artifact binding.""" + + recorded = artifact.get("recorded_diagnostic_keys", ()) + text = ( + f"- Every cohort diagnostic recorded in `{artifact['path']}` " + f"(SHA-256 `{artifact['sha256']}`; {len(recorded)} keys), its " + "source provenance and its parameter revision equal the fresh " + f"cohort's (checks {json.dumps(artifact['checks'])}); only those " + "fields of the artifact were used." + ) + additive = artifact.get("additive_diagnostic_keys") or {} + if not additive: + return text + " The fresh cohort records no other diagnostic." + parts = [] + for key, value in additive.items(): + part = f"`{key}` = `{json.dumps(value, sort_keys=True)}`" + check = (artifact.get("additive_breakdowns") or {}).get(key) + if check is not None: + part += ( + f" (sum {check['sum']}; recorded `{check['total_key']}` " + f"{check['recorded_total']})" + ) + parts.append(part) + return ( + text + " The fresh cohort also records diagnostics the artifact " + "does not; they have no recorded value to compare with, and a " + "breakdown of a recorded total must sum to it: " + + "; ".join(parts) + + "." + ) def _parameters(): @@ -1587,6 +1968,37 @@ def _parameters(): ) +def not_compared(convention: ComputationYears) -> list[str]: + """What the run does not compare, for the chosen oracle.""" + + items = [ + "PIA for anyone but the 1964 birth cohort, and Track A's own PIA " + "for anyone: the 415(a) candidate covers only 2026 first " + "eligibility, with 2026 bend points derived from SSA's " + "published 2024 AWI rather than Track A's TR2008 AWI", + "Track A's DI levels and pre-eligibility-death levels (the " + "disclosed oracle approximation, approximate_pia): the bridge " + "supports ordinary old-age entitlement without disability only", + "COLA increases, early or delayed claiming adjustments, spouse " + "and survivor benefits, and every Track A benefit path", + "members born before 1917 or after 1968 (see selection " + "exclusions) and the 2009-wave R6 population", + "the five-group COLA age profile: not computed; of the Track " + "A artifact the script uses only the parameter revision and the " + "2011-wave cohort diagnostics and source provenance", + ] + if ComputationYears(convention) is not TRACK_A_COMPUTATION_YEARS: + items.append( + "Track A's own AIME against the engine: the oracle compared " + f"is {ComputationYears(convention).value}, while Track A " + "(cola_track_a.benefits.TRACK_A_COMPUTATION_YEARS) uses " + f"{TRACK_A_COMPUTATION_YEARS.value}; Track A's AIME is " + "recorded per person as track_a_oracle_aime, and " + "--oracle-computation-years legacy_fixed_35 compares it" + ) + return items + + def _write_execution(directory: Path, stem: str, result) -> None: directory.mkdir(parents=True, exist_ok=True) (directory / f"{stem}.request.json").write_bytes(result.prepared.wire) @@ -1635,7 +2047,9 @@ def _pia_summary( } -def main(argv: list[str] | None = None) -> int: +def build_parser() -> argparse.ArgumentParser: + """The command line.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument( "--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR.expanduser() @@ -1647,7 +2061,22 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--render-only", type=Path, default=None) parser.add_argument("--examples-per-class", type=int, default=3) parser.add_argument("--max-unexplained-executions", type=int, default=500) - args = parser.parse_args(argv) + parser.add_argument( + "--oracle-computation-years", + choices=tuple(ORACLE_CHOICES), + default=DEFAULT_ORACLE, + help=( + "the oracle's benefit computation years: 'statutory' (42 USC " + "415(b)(2), ss.statutory_aime; the default) or " + "'legacy_fixed_35' (Track A's ss.benefits.aime, the oracle of " + "the 2026-09-23 report)" + ), + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) if args.render_only is not None: result = json.loads((args.render_only / "result.json").read_text()) @@ -1667,6 +2096,8 @@ def main(argv: list[str] | None = None) -> int: if (output / "result.json").exists(): raise SystemExit(f"{output / 'result.json'} exists; not overwriting") require_track_a_convention() + convention = oracle_convention(args.oracle_computation_years) + print(f"oracle: {ORACLE_NAMES[convention]}", file=sys.stderr) binding = bridge.reviewed_case_a_binding() binding.verify() params, base, parameter_info = _parameters() @@ -1674,24 +2105,14 @@ def main(argv: list[str] | None = None) -> int: cohort, provenance, cohort_info = _load_cohort( args.anchor_wave, reference_year ) - artifact_bytes = (ROOT / TRACK_A_ARTIFACT).read_bytes() - artifact_checks = track_a_artifact_binding( - json.loads(artifact_bytes), + cohort_info["track_a_artifact"] = require_track_a_artifact( + (ROOT / TRACK_A_ARTIFACT).read_bytes(), + path=TRACK_A_ARTIFACT, anchor_wave=args.anchor_wave, diagnostics=cohort.diagnostics, source_provenance=cohort.source_provenance, parameters_revision=params.pe_us_revision, ) - if not all(artifact_checks.values()): - raise SystemExit( - f"the cohort or parameters differ from {TRACK_A_ARTIFACT}: " - f"{artifact_checks}" - ) - cohort_info["track_a_artifact"] = { - "path": str(TRACK_A_ARTIFACT), - "sha256": hashlib.sha256(artifact_bytes).hexdigest(), - "checks": artifact_checks, - } cutoff = cohort.start_year persons = cohort.persons selections = select_persons( @@ -1720,8 +2141,9 @@ def main(argv: list[str] | None = None) -> int: ) for s in selected ] - # Track A's PIA rests on exactly this AIME (eligibility_pia_for_clock). - check_track_a_pia(inputs, params) + # The oracle PIAs rest on exactly these AIMEs (eligibility_pia_for_clock + # under the chosen convention and under Track A's). + pia_consistency = check_oracle_pia(inputs, params, convention) wage_index = track_a_wage_index(params) base_series = track_a_contribution_base( params, bridge.FIRST_COMPUTATION_BASE_YEAR, cutoff @@ -1769,6 +2191,7 @@ def on_result(record, result): binding=binding, cutoff_year=cutoff, anchor_wave=args.anchor_wave, + oracle_convention=convention, pia_check_params=base, ) for name in (PASS_PRIMARY, PASS_DIAGNOSTIC, PASS_STRICT) @@ -1798,8 +2221,9 @@ def on_result(record, result): exclusions = Counter(s.exclusion for s in selections if not s.selected) result = { "schema_version": SCHEMA_VERSION, - "header": HEADER, - "labels": list(LABELS), + "header": header_for(convention), + "labels": list(labels_for(convention)), + "oracle": oracle_document(convention), "partial": args.limit is not None, "cohort": cohort_info, "selection": { @@ -1857,25 +2281,11 @@ def on_result(record, result): for (name, attribution), count in sorted(examples.items()) }, }, + "oracle_pia_consistency": pia_consistency, "pia_check": _pia_summary( passes[PASS_PRIMARY]["records"], params, base ), - "not_compared": [ - "PIA for anyone but the 1964 birth cohort, and Track A's own PIA " - "for anyone: the 415(a) candidate covers only 2026 first " - "eligibility, with 2026 bend points derived from SSA's " - "published 2024 AWI rather than Track A's TR2008 AWI", - "Track A's DI levels and pre-eligibility-death levels (the " - "disclosed oracle approximation, approximate_pia): the bridge " - "supports ordinary old-age entitlement without disability only", - "COLA increases, early or delayed claiming adjustments, spouse " - "and survivor benefits, and every Track A benefit path", - "members born before 1917 or after 1968 (see selection " - "exclusions) and the 2009-wave R6 population", - "the five-group COLA age profile: not computed; of the Track " - "A artifact the script uses only the parameter revision and the " - "2011-wave cohort diagnostics and source provenance", - ], + "not_compared": not_compared(convention), "provenance": { "engine_binding": binding.document(), "engine_versions": sorted(engine_versions), diff --git a/tests/README-tiers.md b/tests/README-tiers.md index 8d492cb5..efa9ba8c 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -38,9 +38,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 3,036 | -| `artifact` | 2,801 | +| `unit` | 3,047 | +| `artifact` | 2,803 | | `integration_psid` | 862 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 163 | -| **Total** | **7,382** | +| **Total** | **7,395** | diff --git a/tests/ss/test_statutory_aime.py b/tests/ss/test_statutory_aime.py index d33dd950..e18bc504 100644 --- a/tests/ss/test_statutory_aime.py +++ b/tests/ss/test_statutory_aime.py @@ -393,9 +393,11 @@ def test_refuses_years_that_cannot_be_computation_base_years(): LEGACY_CONVENTION_USERS = { "src/populace_dynamics/cola_track_a/benefits.py", "src/populace_dynamics/ss/statutory_aime.py", - # Track C step 1 compares the Axiom engine with Track A's oracle AIME - # (track-c-aime-agreement-20260923) and refuses to run if Track A's - # convention changes. + # Track C step 1 compares the Axiom engine with the oracle AIME under + # --oracle-computation-years (statutory by default; legacy_fixed_35 is + # the oracle of track-c-aime-agreement-20260923), records Track A's + # legacy AIME beside it, and refuses to run if Track A's convention + # changes. "scripts/track_c_aime_agreement.py", } diff --git a/tests/test_track_c_aime_agreement.py b/tests/test_track_c_aime_agreement.py index ee34f69b..a8114cec 100644 --- a/tests/test_track_c_aime_agreement.py +++ b/tests/test_track_c_aime_agreement.py @@ -40,6 +40,8 @@ def _load_script(): tc = _load_script() +STATUTORY = ComputationYears.STATUTORY +LEGACY = ComputationYears.LEGACY_FIXED_35 # INVENTED parameters: a smooth wage index with cent values and a flat # contribution base. Not SSA's series. @@ -152,7 +154,14 @@ def fake_binding(tmp_path): ) -def _context(name=tc.PASS_PRIMARY, *, binding=None, runner=None, **kw): +def _context( + name=tc.PASS_PRIMARY, + *, + binding=None, + runner=None, + oracle=STATUTORY, + **kw, +): params = kw.pop("params", INVENTED_PARAMS) return tc.PassContext( name=name, @@ -162,6 +171,7 @@ def _context(name=tc.PASS_PRIMARY, *, binding=None, runner=None, **kw): binding=binding, cutoff_year=CUTOFF, anchor_wave=2011, + oracle_convention=oracle, runner=runner, **kw, ) @@ -208,67 +218,167 @@ def test_conventions_entitlement_year_and_candidate_count(): assert tc.candidate_computation_years(1960) == 35 -def test_oracle_arithmetic_at_35_years_is_the_oracle(): +def test_the_candidate_count_is_the_statutory_oracle_s_count(): + # The report says no disagreement can be attributed to the count under + # the statutory oracle: the two counts agree for every birth year the + # statutory oracle accepts (attaining 62 in 1975 or later). + for birth in range(1913, 2001): + assert tc.candidate_computation_years( + birth + ) == tc.oracle_computation_year_count(birth, STATUTORY), birth + assert tc.oracle_computation_year_count(1913, LEGACY) == 35 + + +def test_oracle_arithmetic_at_the_oracle_s_count_is_the_oracle(): career = _career(1930) career[1990] = 45191.8 # INVENTED non-dyadic amount - assert tc.oracle_arithmetic_with_count( - career, 1930, INVENTED_PARAMS, 35 - ) == tc.oracle_aime(career, 1930, INVENTED_PARAMS) - assert tc.oracle_aime(career, 1930, INVENTED_PARAMS) == benefits.aime( - career, 1930, INVENTED_PARAMS + legacy = tc.oracle_aime(career, 1930, INVENTED_PARAMS, convention=LEGACY) + assert ( + tc.oracle_arithmetic_with_count(career, 1930, INVENTED_PARAMS, 35) + == legacy + ) + assert legacy == benefits.aime(career, 1930, INVENTED_PARAMS) + # Born 1930: 35 statutory years, so the two oracles agree. + assert tc.oracle_computation_year_count(1930, STATUTORY) == 35 + assert ( + tc.oracle_aime(career, 1930, INVENTED_PARAMS, convention=STATUTORY) + == legacy ) fewer = tc.oracle_arithmetic_with_count(career, 1930, INVENTED_PARAMS, 20) - assert fewer > tc.oracle_aime(career, 1930, INVENTED_PARAMS) - - -def test_the_oracle_side_is_track_a_s_legacy_convention(): - # The attribution counts disagreements "instead of the oracle's fixed - # 35", so the oracle side must be Track A's legacy convention. - assert tc.ORACLE_CONVENTION is ComputationYears.LEGACY_FIXED_35 - assert TRACK_A_COMPUTATION_YEARS is tc.ORACLE_CONVENTION - assert tc.ORACLE_COMPUTATION_YEARS == 35 + assert fewer > legacy + # INVENTED career of a person born 1920: 26 statutory years. The + # statutory oracle is the arithmetic at that count; the legacy one at 35. + early = _career(1920) + assert tc.oracle_computation_year_count(1920, STATUTORY) == 26 + assert tc.oracle_computation_year_count(1920, LEGACY) == 35 + statutory = tc.oracle_aime( + early, 1920, INVENTED_PARAMS, convention=STATUTORY + ) + assert statutory == tc.oracle_arithmetic_with_count( + early, 1920, INVENTED_PARAMS, 26 + ) + assert tc.oracle_aime( + early, 1920, INVENTED_PARAMS, convention=LEGACY + ) == benefits.aime(early, 1920, INVENTED_PARAMS) + assert statutory > benefits.aime(early, 1920, INVENTED_PARAMS) + + +def test_the_oracle_choice_defaults_to_statutory_and_is_named(): + parser = tc.build_parser() + assert parser.parse_args([]).oracle_computation_years == "statutory" + assert tc.DEFAULT_ORACLE == "statutory" + legacy_args = ["--oracle-computation-years", "legacy_fixed_35"] + assert ( + parser.parse_args(legacy_args).oracle_computation_years + == "legacy_fixed_35" + ) + with pytest.raises(SystemExit): + parser.parse_args(["--oracle-computation-years", "fixed_35"]) + assert tc.oracle_convention("statutory") is STATUTORY + assert tc.oracle_convention("legacy_fixed_35") is LEGACY + assert tc.oracle_convention(LEGACY) is LEGACY + with pytest.raises(ValueError, match="unknown oracle computation years"): + tc.oracle_convention("statutory_415_b_2") + assert tc.oracle_choice(STATUTORY) == "statutory" + assert tc.oracle_choice(LEGACY) == "legacy_fixed_35" + for convention, phrase in ( + (STATUTORY, "statutory computation years (42 USC 415(b)(2)"), + (LEGACY, "Track A's legacy fixed 35 computation years"), + ): + assert phrase in tc.header_for(convention) + assert "oracle: " + tc.ORACLE_NAMES[convention] in tc.labels_for( + convention + ) + document = tc.oracle_document(convention) + assert document["computation_years"] == convention.value + assert document["choice"] == tc.oracle_choice(convention) + assert document["is_track_a_convention"] is (convention is LEGACY) + assert document["track_a_computation_years"] == LEGACY.value + assert f"ComputationYears.{convention.name}" in document["call"] + # Only the non-Track-A oracle adds a not-compared item about Track A. + assert len(tc.not_compared(STATUTORY)) == len(tc.not_compared(LEGACY)) + 1 + assert "track_a_oracle_aime" in tc.not_compared(STATUTORY)[-1] + + +def test_track_a_s_convention_is_still_the_legacy_fixed_35(): + # The report names Track A's AIME as the fixed-35 one under either + # choice, so a change of Track A's convention stops the script. + assert TRACK_A_COMPUTATION_YEARS is LEGACY + assert tc.LEGACY_COMPUTATION_YEARS == 35 tc.require_track_a_convention() - tc.require_track_a_convention(ComputationYears.LEGACY_FIXED_35) + tc.require_track_a_convention(LEGACY) with pytest.raises(SystemExit, match="statutory_415_b_2"): - tc.require_track_a_convention(ComputationYears.STATUTORY) - # INVENTED career of a person born 1925 (31 statutory computation - # years): the oracle side is still the fixed-35 benefits.aime. - career = _career(1925) - assert tc.oracle_aime(career, 1925, INVENTED_PARAMS) == benefits.aime( - career, 1925, INVENTED_PARAMS - ) + tc.require_track_a_convention(STATUTORY) -def test_track_a_pia_check_passes_track_a_s_computation_years(monkeypatch): +@pytest.mark.parametrize( + ("oracle", "checked"), + [ + (STATUTORY, [STATUTORY.value, LEGACY.value]), + (LEGACY, [LEGACY.value]), + ], +) +def test_oracle_pia_check_follows_the_chosen_oracle( + monkeypatch, oracle, checked +): # INVENTED people born 1925 and 1960. The check calls - # eligibility_pia_for_clock as Track A's assembly does; its default - # (the statutory count) gives the 1925 person another PIA, so a call - # without computation_years would stop the real run with a PIA - # mismatch for everyone born before 1929. + # eligibility_pia_for_clock with the chosen convention and with Track + # A's, as Track A's assembly does. people = [_person(1, 1925), _person(2, 1960)] - tc.check_track_a_pia(people, INVENTED_PARAMS) + document = tc.check_oracle_pia(people, INVENTED_PARAMS, oracle) + assert document["persons"] == 2 + assert document["conventions_checked"] == checked + # The two conventions give the 1925 person different PIAs, so each + # check is a real constraint. early = people[0] - statutory = sb.eligibility_pia_for_clock( - sb.WorkerClock.at_age_62(1925), - history=early.career, - birth_year=1925, - params=INVENTED_PARAMS, - ) - oracle = benefits.pia( - tc.oracle_aime(early.career, 1925, INVENTED_PARAMS), + pias = { + convention: sb.eligibility_pia_for_clock( + sb.WorkerClock.at_age_62(1925), + history=early.career, + birth_year=1925, + params=INVENTED_PARAMS, + computation_years=convention, + ) + for convention in (STATUTORY, LEGACY) + } + assert pias[STATUTORY] != pias[LEGACY] + assert pias[LEGACY] == benefits.pia( + tc.oracle_aime(early.career, 1925, INVENTED_PARAMS, convention=LEGACY), 1987, INVENTED_PARAMS, ) - assert statutory != oracle - # A PIA that does not rest on the oracle AIME is refused. + # A PIA that does not rest on the recorded oracle AIME is refused. original = tc.oracle_aime monkeypatch.setattr( tc, "oracle_aime", - lambda career, birth, params: original(career, birth, params) + 100, + lambda career, birth, params, *, convention: original( + career, birth, params, convention=convention + ) + + 100, ) with pytest.raises(ValueError, match="1: PIA mismatch"): - tc.check_track_a_pia(people, INVENTED_PARAMS) + tc.check_oracle_pia(people, INVENTED_PARAMS, oracle) + + +def test_oracle_pia_check_needs_the_computation_years_pin(monkeypatch): + # A PIA function that ignores computation_years (always its statutory + # default) must fail the Track A (legacy) check for someone born 1925. + original = sb.eligibility_pia_for_clock + + def unpinned(clock, *, history, birth_year, params, computation_years): + return original( + clock, history=history, birth_year=birth_year, params=params + ) + + monkeypatch.setattr(tc.sb, "eligibility_pia_for_clock", unpinned) + people = [_person(1, 1925)] + with pytest.raises(ValueError, match="1: PIA mismatch \\(legacy"): + tc.check_oracle_pia(people, INVENTED_PARAMS, STATUTORY) + with pytest.raises(ValueError, match="1: PIA mismatch \\(legacy"): + tc.check_oracle_pia(people, INVENTED_PARAMS, LEGACY) + # Born 1960: 35 years either way, so the unpinned call passes. + tc.check_oracle_pia([_person(2, 1960)], INVENTED_PARAMS, LEGACY) def test_transport_zero_fills_outside_the_panel_and_labels_rows(): @@ -353,12 +463,15 @@ def test_strict_transport_is_refused_with_classified_reasons(): assert "axiom_aime" not in accepted -def test_executed_records_classify_agreement(fake_binding): +@pytest.mark.parametrize("oracle", [STATUTORY, LEGACY]) +def test_executed_records_classify_agreement(fake_binding, oracle): person = _person(1, 1935) - oracle = tc.oracle_aime(person.career, 1935, INVENTED_PARAMS) - engine = FakeEngine(aime=str(oracle)) + aime = tc.oracle_aime( + person.career, 1935, INVENTED_PARAMS, convention=oracle + ) + engine = FakeEngine(aime=str(aime)) record, result = tc.compare_person( - person, _context(binding=fake_binding, runner=engine) + person, _context(binding=fake_binding, runner=engine, oracle=oracle) ) assert record["status"] == "executed" assert record["exact_match"] is True @@ -366,31 +479,73 @@ def test_executed_records_classify_agreement(fake_binding): assert record["attribution"] == tc.ATTR_EXACT assert record["entitlement_year_sent"] == 2011 assert record["request_sha256"] == result.prepared.request_sha256 + # Born 1935: 35 years under either oracle, and Track A's AIME is the + # oracle's. + assert record["oracle_computation_years"] == 35 + assert record["oracle_aime"] == record["track_a_oracle_aime"] == aime request = engine.requests[0] assert request["outputs"] == [bridge.AIME_OUTPUT] # Window 1957-2010: eleven zero years before 1968, then the career. assert len(request["batches"]) == 2011 - 1957 + record, _ = tc.compare_person( + person, + _context( + binding=fake_binding, + runner=FakeEngine(str(aime + 1)), + oracle=oracle, + ), + ) + assert record["attribution"] == tc.ATTR_UNEXPLAINED + assert record["difference"] == "1" + + +def test_the_legacy_oracle_attributes_the_count_the_statutory_one_uses( + fake_binding, +): + # INVENTED career of a person born 1920 (26 candidate computation + # years); the engine answers with the oracle arithmetic at 26 years. early = _person(2, 1920) count = tc.candidate_computation_years(1920) at_count = tc.oracle_arithmetic_with_count( early.career, 1920, INVENTED_PARAMS, count ) - record, _ = tc.compare_person( + legacy, _ = tc.compare_person( early, - _context(binding=fake_binding, runner=FakeEngine(str(at_count))), + _context( + binding=fake_binding, + runner=FakeEngine(str(at_count)), + oracle=LEGACY, + ), ) - assert record["attribution"] == tc.ATTR_COUNT - assert Decimal(record["difference"]) == at_count - ( - record["track_a_oracle_aime"] + assert legacy["oracle_computation_years"] == 35 + assert legacy["oracle_aime"] == legacy["track_a_oracle_aime"] + assert legacy["attribution"] == tc.ATTR_COUNT + assert Decimal(legacy["difference"]) == at_count - legacy["oracle_aime"] + assert Decimal(legacy["difference"]) > 0 + statutory, _ = tc.compare_person( + early, + _context( + binding=fake_binding, + runner=FakeEngine(str(at_count)), + oracle=STATUTORY, + ), ) - - record, _ = tc.compare_person( - person, - _context(binding=fake_binding, runner=FakeEngine(str(oracle + 1))), + assert statutory["oracle_computation_years"] == count == 26 + assert statutory["oracle_aime"] == at_count + assert statutory["track_a_oracle_aime"] == legacy["oracle_aime"] + assert statutory["attribution"] == tc.ATTR_EXACT + # Under the statutory oracle the count explains nothing: a different + # engine answer is unexplained even if it were the legacy arithmetic. + other, _ = tc.compare_person( + early, + _context( + binding=fake_binding, + runner=FakeEngine(str(legacy["oracle_aime"])), + oracle=STATUTORY, + ), ) - assert record["attribution"] == tc.ATTR_UNEXPLAINED - assert record["difference"] == "1" + assert other["attribution"] == tc.ATTR_UNEXPLAINED def test_refusals_and_engine_failures_are_recorded(fake_binding): @@ -431,7 +586,9 @@ def test_diagnostic_pass_compares_the_same_truncated_years(fake_binding): opening_entitlement_year=2006, ) truncated = {y: v for y, v in career.items() if y < 2006} - oracle_truncated = tc.oracle_aime(truncated, 1944, INVENTED_PARAMS) + oracle_truncated = tc.oracle_aime( + truncated, 1944, INVENTED_PARAMS, convention=STATUTORY + ) record, _ = tc.compare_person( person, _context( @@ -443,7 +600,7 @@ def test_diagnostic_pass_compares_the_same_truncated_years(fake_binding): assert record["entitlement_year_sent"] == 2006 assert record["oracle_aime_on_sent_years"] == oracle_truncated assert record["truncation_effect_on_oracle"] == oracle_truncated - ( - tc.oracle_aime(career, 1944, INVENTED_PARAMS) + tc.oracle_aime(career, 1944, INVENTED_PARAMS, convention=STATUTORY) ) assert record["truncation_effect_on_oracle"] < 0 assert record["career_years_not_sent"] == "2006-2010" @@ -452,7 +609,9 @@ def test_diagnostic_pass_compares_the_same_truncated_years(fake_binding): def test_pia_formula_check_records_the_1964_cohort(fake_binding): person = _person(1, 1964) - oracle = tc.oracle_aime(person.career, 1964, INVENTED_PARAMS) + oracle = tc.oracle_aime( + person.career, 1964, INVENTED_PARAMS, convention=STATUTORY + ) pia = benefits.pia(float(oracle), 2026, INVENTED_PARAMS) engine = FakeEngine(aime=str(oracle), pia=tc._float_text(pia)) record, result = tc.compare_person( @@ -476,7 +635,10 @@ def test_run_pass_and_summaries(fake_binding): people = [_person(i, 1930 + i) for i in range(1, 7)] answers = { p.selection.person_id: tc.oracle_aime( - p.career, p.selection.birth_year, INVENTED_PARAMS + p.career, + p.selection.birth_year, + INVENTED_PARAMS, + convention=STATUTORY, ) for p in people } @@ -496,6 +658,8 @@ def by_id(argv, wire, timeout): overall = summary["overall"] assert overall["executed"] == 6 assert overall["exact_matches"] == 5 + # Born 1931-1936: 35 years, so the statutory oracle is Track A's. + assert overall["oracle_aime_differs_from_track_a"] == 0 assert overall["exact_match_share_of_executed"] == pytest.approx(5 / 6) assert overall["difference_counts"] == {"0": 5, "2": 1} assert overall["nonzero_differences"]["count"] == 1 @@ -533,40 +697,144 @@ def test_difference_bins_are_half_open_and_exhaustive(): assert sum(bins.values()) == sum(counts.values()) -def test_track_a_artifact_binding_compares_cohort_and_parameters(): - # INVENTED artifact fragment and cohort diagnostics. - diagnostics = {"members": 3, "a3_spec": {"m4_waves": (2011,)}} - source = {"kind": "psid_files", "content_sha256": "INVENTED"} - artifact = { +# INVENTED artifact fragment and cohort diagnostics, in the shape of +# runs/replication_urban2010_cola_v1.json's cohorts and of +# cola_track_a.opening's diagnostics. +_SOURCE = {"kind": "psid_files", "content_sha256": "INVENTED"} +_RECORDED = { + "members": 3, + "a3_spec": {"m4_waves": [2011]}, + "opening_stock_entitlement_clamped": 5, +} +_ADDITIVE_KEY = "opening_stock_entitlement_clamped_by_clock_rule" +#: INVENTED breakdown that sums to the recorded total (5). +_ADDITIVE_VALUE = {"linked_worker_birth_plus_62": 2, "own_birth_plus_62": 3} + + +def _artifact(recorded=None): + return { "ssa_parameters_revision": "INVENTED+tr2008", "cohorts": { "2011": { - "members": 3, - "a3_spec": {"m4_waves": [2011]}, - "source_provenance": dict(source), + **(_RECORDED if recorded is None else recorded), + "source_provenance": dict(_SOURCE), } }, } - checks = tc.track_a_artifact_binding( - artifact, + + +def _binding(diagnostics, *, artifact=None, revision="INVENTED+tr2008"): + return tc.track_a_artifact_binding( + _artifact() if artifact is None else artifact, anchor_wave=2011, diagnostics=diagnostics, - source_provenance=source, - parameters_revision="INVENTED+tr2008", + source_provenance=_SOURCE, + parameters_revision=revision, ) - assert all(checks.values()) - changed = tc.track_a_artifact_binding( - artifact, - anchor_wave=2011, - diagnostics={**diagnostics, "members": 4}, - source_provenance=source, - parameters_revision="INVENTED+other", + + +def test_track_a_artifact_binding_compares_cohort_and_parameters(): + diagnostics = {**_RECORDED, "a3_spec": {"m4_waves": (2011,)}} + binding = _binding(diagnostics) + assert all(binding["checks"].values()) + assert binding["recorded_diagnostic_keys"] == sorted(_RECORDED) + assert binding["additive_diagnostic_keys"] == {} + changed = _binding( + {**diagnostics, "members": 4}, revision="INVENTED+other" ) - assert changed == { + assert changed["checks"] == { "ssa_parameters_revision_equal": False, - "cohort_diagnostics_equal": False, + "cohort_diagnostics_equal_on_recorded_keys": False, + "additive_breakdowns_sum_to_recorded_totals": True, "cohort_source_provenance_equal": True, } + assert changed["recorded_keys_changed"] == { + "members": {"recorded": 3, "fresh": 4} + } + missing = _binding( + {k: v for k, v in diagnostics.items() if k != "members"} + ) + assert not missing["checks"]["cohort_diagnostics_equal_on_recorded_keys"] + assert missing["recorded_keys_missing"] == ["members"] + + +def test_an_additive_diagnostic_key_is_recorded_not_refused(): + # Regression: #454 added opening_stock_entitlement_clamped_by_clock_rule + # to the cohort diagnostics after the Track A artifact was written, and + # the whole-dict comparison stopped every real run before any engine + # call (cohort_diagnostics_equal: False). + fresh = {**_RECORDED, _ADDITIVE_KEY: dict(_ADDITIVE_VALUE)} + recorded = dict(_artifact()["cohorts"]["2011"]) + recorded.pop("source_provenance") + assert tc._json_normal(recorded) != tc._json_normal(fresh) + binding = _binding(fresh) + assert all(binding["checks"].values()) + assert binding["additive_diagnostic_keys"] == { + _ADDITIVE_KEY: _ADDITIVE_VALUE + } + assert binding["additive_breakdowns"] == { + _ADDITIVE_KEY: { + "total_key": "opening_stock_entitlement_clamped", + "recorded_total": 5, + "sum": 5, + "sums_to_recorded_total": True, + } + } + # A breakdown that does not sum to the recorded total is refused. + off = _binding({**fresh, _ADDITIVE_KEY: {"own_birth_plus_62": 4}}) + assert off["checks"] == { + "ssa_parameters_revision_equal": True, + "cohort_diagnostics_equal_on_recorded_keys": True, + "additive_breakdowns_sum_to_recorded_totals": False, + "cohort_source_provenance_equal": True, + } + not_counts = _binding({**fresh, _ADDITIVE_KEY: ["own_birth_plus_62"]}) + assert not all(not_counts["checks"].values()) + # An additive key that breaks down no recorded total is recorded only. + other = _binding({**fresh, "invented_new_counter": 7}) + assert all(other["checks"].values()) + assert other["additive_diagnostic_keys"]["invented_new_counter"] == 7 + assert other["additive_breakdowns"]["invented_new_counter"] is None + # Changing a recorded key is still refused alongside an additive one. + changed = _binding({**fresh, "opening_stock_entitlement_clamped": 6}) + assert not changed["checks"]["cohort_diagnostics_equal_on_recorded_keys"] + + +def test_require_track_a_artifact_refuses_and_records(tmp_path): + fresh = {**_RECORDED, _ADDITIVE_KEY: dict(_ADDITIVE_VALUE)} + artifact_bytes = json.dumps(_artifact()).encode() + kwargs = dict( + path="INVENTED/artifact.json", + anchor_wave=2011, + source_provenance=_SOURCE, + parameters_revision="INVENTED+tr2008", + ) + document = tc.require_track_a_artifact( + artifact_bytes, diagnostics=fresh, **kwargs + ) + assert document["path"] == "INVENTED/artifact.json" + assert document["sha256"] == hashlib.sha256(artifact_bytes).hexdigest() + assert document["additive_diagnostic_keys"] == { + _ADDITIVE_KEY: _ADDITIVE_VALUE + } + text = tc._artifact_binding_text(document) + assert "3 keys" in text + assert f"`{_ADDITIVE_KEY}`" in text + assert "(sum 5; recorded `opening_stock_entitlement_clamped` 5)" in text + plain = tc._artifact_binding_text( + tc.require_track_a_artifact( + artifact_bytes, diagnostics=dict(_RECORDED), **kwargs + ) + ) + assert "records no other diagnostic" in plain + with pytest.raises(SystemExit) as refused: + tc.require_track_a_artifact( + artifact_bytes, diagnostics={**fresh, "members": 4}, **kwargs + ) + message = str(refused.value) + assert "INVENTED/artifact.json" in message + assert "cohort_diagnostics_equal_on_recorded_keys" in message + assert "'members': {'recorded': 3, 'fresh': 4}" in message def test_decimal_and_float_text_are_plain(): @@ -581,14 +849,20 @@ def test_decimal_and_float_text_are_plain(): assert series.source_sha256 is not None -def _invented_result(fake_binding): +def _invented_result(fake_binding, oracle=STATUTORY): people = [_person(1, 1935), _person(2, 1920, "retired_worker")] records = tc.run_pass( people, - _context(binding=fake_binding, runner=FakeEngine("100")), + _context( + binding=fake_binding, runner=FakeEngine("100"), oracle=oracle + ), ) - strict = tc.run_pass(people, _context(tc.PASS_STRICT)) + strict = tc.run_pass(people, _context(tc.PASS_STRICT, oracle=oracle)) return { + "schema_version": tc.SCHEMA_VERSION, + "header": tc.header_for(oracle), + "labels": list(tc.labels_for(oracle)), + "oracle": tc.oracle_document(oracle), "partial": True, "cohort": { "anchor_wave": 2011, @@ -597,6 +871,11 @@ def _invented_result(fake_binding): "psid_files_bundle_sha256": "INVENTED", "a3_content_sha256": "INVENTED", "track_a_seal": "INVENTED", + "track_a_artifact": { + "path": "INVENTED/artifact.json", + "sha256": "INVENTED", + **_binding({**_RECORDED, _ADDITIVE_KEY: _ADDITIVE_VALUE}), + }, }, "selection": { "selected": 2, @@ -615,6 +894,12 @@ def _invented_result(fake_binding): tc.PASS_PRIMARY: {"summary": tc.summarize_pass(records)}, tc.PASS_STRICT: {"summary": tc.summarize_pass(strict)}, }, + "oracle_pia_consistency": { + "persons": 2, + "conventions_checked": list( + dict.fromkeys((oracle.value, LEGACY.value)) + ), + }, "pia_check": { "persons": 0, "agree": 0, @@ -642,10 +927,11 @@ def _invented_result(fake_binding): } +@pytest.mark.parametrize("oracle", [STATUTORY, LEGACY]) def test_render_labels_candidates_and_computes_no_age_profile( - fake_binding, tmp_path + fake_binding, tmp_path, oracle ): - result = _invented_result(fake_binding) + result = _invented_result(fake_binding, oracle) text = tc.render_results(result) assert "PARTIAL SMOKE RUN" in text assert "candidates are unaccepted" in text @@ -659,11 +945,55 @@ def test_render_labels_candidates_and_computes_no_age_profile( # Person 2 (born 1920) has 26 candidate computation years. assert "| 26 | 1920-1920 | 1 |" in text assert "derives them from SSA's published 2024 AWI" in text + # The artifact binding names the additive diagnostic. + assert f"`{_ADDITIVE_KEY}`" in text (tmp_path / "result.json").write_text(json.dumps(result)) assert tc.main(["--render-only", str(tmp_path)]) == 0 assert (tmp_path / "RESULTS.md").read_text() == text +def test_render_names_the_oracle_compared_against(fake_binding): + statutory = tc.render_results(_invented_result(fake_binding, STATUTORY)) + legacy = tc.render_results(_invented_result(fake_binding, LEGACY)) + title = "# Track C step 1: AIME agreement, Axiom engine vs Python oracle" + assert statutory.splitlines()[0] == title + ( + " (statutory computation years)" + ) + assert legacy.splitlines()[0] == title + ( + " (legacy fixed-35 computation years)" + ) + for text, convention in ((statutory, STATUTORY), (legacy, LEGACY)): + name = tc.ORACLE_NAMES[convention] + assert f"**Oracle: {name}**" in text + assert f"Primary pass, oracle with {name}:" in text + assert ( + f"`--oracle-computation-years {tc.oracle_choice(convention)}`" + in text + ) + assert "computation_years=ComputationYears.STATUTORY" in statutory + assert "the statutory oracle uses that count too" in statutory + assert "no record can have this attribution" in statutory + assert "`ss.benefits.aime(career, birth_year, params)`" in legacy + assert "the legacy oracle always uses 35" in legacy + assert "(35, the legacy oracle's)" in legacy + # Person 2 (born 1920): the statutory oracle's AIME is not Track A's. + assert "differs from Track A's" in statutory + assert "for 1 of the 2 selected people: 1 of the 1 with fewer" in ( + " ".join(statutory.split()) + ) + assert "for 0 of the 2 selected people" in " ".join(legacy.split()) + assert "`statutory_415_b_2` and `legacy_fixed_35`" in statutory + assert "`legacy_fixed_35` (its PIA" in legacy + + +def test_render_refuses_a_result_without_an_oracle_record(fake_binding): + result = _invented_result(fake_binding) + result.pop("oracle") + result["schema_version"] = "populace_dynamics.track_c.aime_agreement.v1" + with pytest.raises(ValueError, match="aime_agreement.v1"): + tc.render_results(result) + + def test_actual_engine_agrees_with_the_oracle_on_invented_careers(): binding = bridge.reviewed_case_a_binding() missing = [str(path) for path in binding.missing_files()] @@ -679,12 +1009,25 @@ def test_actual_engine_agrees_with_the_oracle_on_invented_careers(): career, _provenance(career), ) - record, result = tc.compare_person(modern, _context(binding=binding)) - assert record["status"] == "executed" - assert result.engine_version == "0.2.2" - assert record["exact_match"] is True - assert record["attribution"] == tc.ATTR_EXACT - early, _ = tc.compare_person(_person(2, 1920), _context(binding=binding)) - assert early["candidate_computation_years"] == 26 - assert early["attribution"] == tc.ATTR_COUNT - assert Decimal(early["axiom_aime"]) > early["track_a_oracle_aime"] + for oracle in (STATUTORY, LEGACY): + record, result = tc.compare_person( + modern, _context(binding=binding, oracle=oracle) + ) + assert record["status"] == "executed" + assert result.engine_version == "0.2.2" + assert record["exact_match"] is True + assert record["attribution"] == tc.ATTR_EXACT + early = _person(2, 1920) + legacy, _ = tc.compare_person( + early, _context(binding=binding, oracle=LEGACY) + ) + assert legacy["candidate_computation_years"] == 26 + assert legacy["attribution"] == tc.ATTR_COUNT + assert Decimal(legacy["axiom_aime"]) > legacy["track_a_oracle_aime"] + statutory, _ = tc.compare_person( + early, _context(binding=binding, oracle=STATUTORY) + ) + assert statutory["oracle_computation_years"] == 26 + assert statutory["attribution"] == tc.ATTR_EXACT + assert Decimal(statutory["axiom_aime"]) == statutory["oracle_aime"] + assert statutory["request_sha256"] == legacy["request_sha256"] diff --git a/tests/test_track_c_track_a_artifact_binding.py b/tests/test_track_c_track_a_artifact_binding.py new file mode 100644 index 00000000..0fcc0ed7 --- /dev/null +++ b/tests/test_track_c_track_a_artifact_binding.py @@ -0,0 +1,122 @@ +"""Track C step 1's binding to the committed Track A artifact, by key. + +Reads the cohort diagnostics recorded in the committed +``runs/replication_urban2010_cola_v1.json`` (Registration 13's artifact) +and the diagnostics the current ``cola_track_a.opening`` code produces for +INVENTED people (``cola_track_a.invented``; no PSID data). The per-rule +breakdowns below are INVENTED splits of the recorded totals. + +Regression: #454 added ``opening_stock_entitlement_clamped_by_clock_rule`` +to the cohort diagnostics after the artifact was committed, and +``scripts/track_c_aime_agreement.py`` compared the whole diagnostics dict, +so every real run stopped before any engine call. +""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from functools import lru_cache +from pathlib import Path + +from populace_dynamics.cohorts import psid2010 +from populace_dynamics.cola_track_a import ( + TrackAConfig, + invented, + prepare_track_a_cohort, +) + +ROOT = Path(__file__).resolve().parents[1] +ARTIFACT = ROOT / "runs" / "replication_urban2010_cola_v1.json" +ADDITIVE_KEY = "opening_stock_entitlement_clamped_by_clock_rule" +TOTAL_KEY = "opening_stock_entitlement_clamped" + + +def _load_script(): + path = ROOT / "scripts" / "track_c_aime_agreement.py" + spec = importlib.util.spec_from_file_location( + "track_c_aime_artifact_binding", path + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + # Dataclasses resolve their module through sys.modules. + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +tc = _load_script() + + +@lru_cache(maxsize=1) +def _code_diagnostics() -> dict: + """The diagnostics the current code produces (INVENTED people).""" + cohort = prepare_track_a_cohort( + psid2010.build_psid2010_cohort( + invented.invented_psid2010_inputs(seed=7) + ), + data_provenance="invented", + config=TrackAConfig(), + ) + return dict(cohort.diagnostics) + + +def _recorded(wave: str = "2011") -> tuple[dict, dict, dict]: + artifact = json.loads(ARTIFACT.read_text()) + recorded = dict(artifact["cohorts"][wave]) + source = recorded.pop("source_provenance") + return artifact, recorded, source + + +def test_the_artifact_records_a_subset_of_the_code_s_diagnostics(): + fresh = set(_code_diagnostics()) + for wave in ("2011", "2009"): + _, recorded, _ = _recorded(wave) + # Every recorded key must still be produced, or Track C refuses. + assert set(recorded) <= fresh, set(recorded) - fresh + # The key #454 added is produced but not recorded. + assert ADDITIVE_KEY in fresh - set(recorded) + assert TOTAL_KEY in recorded + + +def test_the_recorded_2011_cohort_binds_with_the_added_breakdown(): + artifact, recorded, source = _recorded() + total = recorded[TOTAL_KEY] + assert isinstance(total, int) and total > 0 + # INVENTED split of the recorded total over two clock rules. + split = {"linked_worker_birth_plus_62": 1, "own_birth_plus_62": total - 1} + fresh = {**recorded, ADDITIVE_KEY: split} + # The whole-dict comparison the script used to make fails. + assert tc._json_normal(recorded) != tc._json_normal(fresh) + kwargs = dict( + anchor_wave=2011, + source_provenance=source, + parameters_revision=artifact["ssa_parameters_revision"], + ) + binding = tc.track_a_artifact_binding( + artifact, diagnostics=fresh, **kwargs + ) + assert all(binding["checks"].values()), binding["checks"] + assert binding["recorded_diagnostic_keys"] == sorted(recorded) + assert binding["additive_diagnostic_keys"] == {ADDITIVE_KEY: split} + assert binding["additive_breakdowns"][ADDITIVE_KEY] == { + "total_key": TOTAL_KEY, + "recorded_total": total, + "sum": total, + "sums_to_recorded_total": True, + } + # An INVENTED split one short of the recorded total is refused. + short = tc.track_a_artifact_binding( + artifact, + diagnostics={**recorded, ADDITIVE_KEY: {"own_birth_plus_62": 1}}, + **kwargs, + ) + assert not short["checks"]["additive_breakdowns_sum_to_recorded_totals"] + # Dropping a recorded key is refused. + dropped = {k: v for k, v in fresh.items() if k != TOTAL_KEY} + missing = tc.track_a_artifact_binding( + artifact, diagnostics=dropped, **kwargs + ) + assert missing["recorded_keys_missing"] == [TOTAL_KEY] + assert not missing["checks"]["cohort_diagnostics_equal_on_recorded_keys"] diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 154a64d4..e30ad2ad 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,8 +1,8 @@ { "schema_version": 1, "counts": { - "unit": 3036, - "artifact": 2801, + "unit": 3047, + "artifact": 2803, "integration_psid": 862, "reproduction_legacy": 520, "oracle_policyengine": 163 From afb462b1355a47f63850b1f5f60f202a6ebb4397 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 24 Sep 2026 06:34:03 -0400 Subject: [PATCH 4/4] Track C: bind only reviewed additive diagnostics; require --output-dir Review of 71c94743. Its artifact binding fixed the stop at master (the whole-dict comparison refused the key bfe9fa3e added) but accepted any diagnostic the artifact lacks: a key not named _by_ was recorded and allowed, and one so named was allowed whenever its counts summed to the recorded total, even with a negative count or a count above what the artifact records for that category. The old whole-dict comparison refused every new diagnostic, so the binding was weaker than the check it replaced. - KNOWN_ADDITIVE_DIAGNOSTICS lists the reviewed additive diagnostics with the commit that added each and the recorded diagnostics it breaks down. It holds one entry: opening_stock_entitlement_clamped_by_clock_rule (bfe9fa3e), a breakdown of opening_stock_entitlement_clamped by opening_stock_clock_rules. - The binding refuses any other diagnostic the artifact lacks (additive_diagnostic_keys_known) and requires a listed one to sum to its recorded total and to count only recorded categories, each at most its recorded count (additive_breakdowns_within_recorded_categories). Recorded keys, source provenance and parameter revision are compared as before. result.json names unknown keys; RESULTS.md names the commit and the category check. - On the real 2011 cohort the binding passes: the one additive key (1 + 2 + 46 = 49) fits the recorded clock-rule counts (248, 58, 1340). With that key removed, the fresh cohort's Track A seal is 142f72d0..., the seal of the 2026-09-23 run that passed the old whole-dict check. - --output-dir no longer defaults to the 2026-09-23 evidence directory; a run without it is refused before any git, data or engine step. - Schema v3. --render-only renders only v3 results; v1 and v2 prose belongs to the script at the commit each records. Tests (INVENTED data, plus the committed artifact's recorded keys): an unreviewed key, a _by_-named unreviewed key, a count above the recorded category, an unrecorded category and a negative count are each refused; the listed table equals what the current code adds to both recorded waves; the run refuses a missing --output-dir; v2 results are not rendered. These tests fail against 71c94743. The artifact-tier "one short" split is now one short of the recorded total. Tier manifest recollected: unit 3053, artifact 2804, total 7402. Co-Authored-By: Claude Opus 5.5 --- scripts/track_c_aime_agreement.py | 145 +++++++++++------ tests/README-tiers.md | 6 +- tests/test_track_c_aime_agreement.py | 154 +++++++++++++++--- .../test_track_c_track_a_artifact_binding.py | 56 ++++++- tests/tier_counts.json | 4 +- 5 files changed, 285 insertions(+), 80 deletions(-) diff --git a/scripts/track_c_aime_agreement.py b/scripts/track_c_aime_agreement.py index 0b61a5f9..6205ab6c 100644 --- a/scripts/track_c_aime_agreement.py +++ b/scripts/track_c_aime_agreement.py @@ -76,10 +76,12 @@ Binding to the registered Track A run. The cohort and parameters must be those of ``runs/replication_urban2010_cola_v1.json``: every cohort diagnostic the artifact records must be present with an equal value, and -its source provenance and parameter revision must be equal. Diagnostics -the current code adds that the artifact does not record are allowed, -recorded in the output, and, where one is a breakdown of a recorded total -(``_by_``), must sum to it. +its source provenance and parameter revision must be equal. A +diagnostic the artifact does not record must be a reviewed one listed in +``KNOWN_ADDITIVE_DIAGNOSTICS`` (with the commit that added it); it is +recorded in the output and must reconcile with the recorded diagnostics it +breaks down (sum to the recorded total, fit within the recorded +per-category counts). Any other new diagnostic refuses the run. Usage:: @@ -127,15 +129,14 @@ ComputationYears, ) -#: v2 (2026-09-24): the oracle's computation years are a recorded choice -#: (``oracle``), ``oracle_aime`` is the chosen oracle's AIME and -#: ``track_a_oracle_aime`` always Track A's, and the Track A artifact +#: v2 (2026-09-24, commit 71c94743): the oracle's computation years are a +#: recorded choice (``oracle``), ``oracle_aime`` is the chosen oracle's AIME +#: and ``track_a_oracle_aime`` always Track A's, and the Track A artifact #: binding records the diagnostics compared and the additive ones. -SCHEMA_VERSION = "populace_dynamics.track_c.aime_agreement.v2" -DEFAULT_OUTPUT_DIR = Path( - "~/microcosm-launch-evidence/dynasim-parity-20260909/" - "track-c-aime-agreement-20260923" -) +#: v3 (2026-09-24): the binding admits only the additive diagnostics in +#: ``KNOWN_ADDITIVE_DIAGNOSTICS`` and records ``added_by``, the category +#: check and ``unknown_additive_diagnostic_keys``. +SCHEMA_VERSION = "populace_dynamics.track_c.aime_agreement.v3" #: The first year of the PSID family earnings panel (career law). PANEL_FIRST_YEAR = 1968 #: Track A's fixed benefit computation years (``ss.benefits.aime``). @@ -1138,16 +1139,18 @@ def _row(*cells: Any) -> str: def render_results(result: Mapping[str, Any]) -> str: """RESULTS.md from ``result.json`` (deterministic; no new numbers). - Refuses a result without an ``oracle`` record (schema v1, written - before the oracle choice): its report text belongs to the script at + Refuses a result of any other schema (v1 has no oracle choice, v2 an + earlier artifact binding): its report text belongs to the script at the commit its provenance records. """ - if "oracle" not in result: + if result.get("schema_version") != SCHEMA_VERSION or "oracle" not in ( + result + ): raise ValueError( - "result.json records no oracle choice (schema " - f"{result.get('schema_version', 'unknown')}); render it with the " - "script at the commit its provenance records" + "result.json has schema " + f"{result.get('schema_version', 'unknown')}, not {SCHEMA_VERSION};" + " render it with the script at the commit its provenance records" ) lines: list[str] = [] add = lines.append @@ -1761,9 +1764,21 @@ def _json_normal(value: Any) -> Any: return json.loads(json.dumps(value, sort_keys=True, default=str)) -#: An additive diagnostic ``_by_`` breaks a recorded total -#: down (the naming ``cola_track_a/opening.py`` uses). -_BREAKDOWN_SEPARATOR = "_by_" +#: Cohort diagnostics the current code produces that the Track A artifact +#: does not record. Each was added after the artifact was written (the +#: commit is named), and each is a count breakdown of recorded +#: diagnostics: its counts must sum to the recorded ``total`` and name only +#: categories the recorded ``categories`` mapping counts, none above its +#: recorded count. Any other diagnostic the artifact lacks refuses the +#: run, as the whole-dict comparison did: a new key is reviewed and listed +#: here before Track C binds to a cohort that carries it. +KNOWN_ADDITIVE_DIAGNOSTICS: Mapping[str, Mapping[str, str]] = { + "opening_stock_entitlement_clamped_by_clock_rule": { + "added_by": "bfe9fa3e", + "total": "opening_stock_entitlement_clamped", + "categories": "opening_stock_clock_rules", + }, +} def _is_count(value: Any) -> bool: @@ -1773,23 +1788,40 @@ def _is_count(value: Any) -> bool: def _breakdown_check( key: str, value: Any, recorded: Mapping[str, Any] ) -> dict[str, Any] | None: - """How an additive ``_by_`` key reconciles, if it is one.""" + """How a known additive diagnostic reconciles; ``None`` if unknown.""" - if _BREAKDOWN_SEPARATOR not in key: - return None - total_key = key.split(_BREAKDOWN_SEPARATOR, 1)[0] - if total_key not in recorded: + known = KNOWN_ADDITIVE_DIAGNOSTICS.get(key) + if known is None: return None - total = recorded[total_key] - parts = list(value.values()) if isinstance(value, Mapping) else None + total_key, categories_key = known["total"], known["categories"] + total = recorded.get(total_key) + categories = recorded.get(categories_key) + parts = dict(value) if isinstance(value, Mapping) else None countable = ( - parts is not None and all(map(_is_count, parts)) and _is_count(total) + parts is not None + and all(_is_count(count) and count >= 0 for count in parts.values()) + and _is_count(total) + ) + within = ( + countable + and isinstance(categories, Mapping) + and all( + name in categories + and _is_count(categories[name]) + and count <= categories[name] + for name, count in parts.items() + ) ) return { + "added_by": known["added_by"], "total_key": total_key, "recorded_total": total, - "sum": sum(parts) if countable else None, - "sums_to_recorded_total": bool(countable and sum(parts) == total), + "sum": sum(parts.values()) if countable else None, + "sums_to_recorded_total": bool( + countable and sum(parts.values()) == total + ), + "categories_key": categories_key, + "within_recorded_categories": bool(within), } @@ -1808,11 +1840,12 @@ def track_a_artifact_binding( groups. Every diagnostic the artifact records must be present in ``diagnostics`` with an equal value (compared as JSON); a missing or changed one fails ``cohort_diagnostics_equal_on_recorded_keys``. - Diagnostics the artifact does not record (keys the code added after - it was written) are allowed and returned under - ``additive_diagnostic_keys``; one named ``_by_`` for a - recorded integer ```` must be a mapping of integers summing to - it (``additive_breakdowns_sum_to_recorded_totals``). The source + A diagnostic the artifact does not record must be one of + :data:`KNOWN_ADDITIVE_DIAGNOSTICS` (``additive_diagnostic_keys_known``) + and reconcile with the recorded diagnostics it breaks down: its counts + sum to the recorded total (``additive_breakdowns_sum_to_recorded_ + totals``) and fit within the recorded per-category counts + (``additive_breakdowns_within_recorded_categories``). The source provenance and the parameter revision must be equal as a whole. Returns ``checks`` (all must be true) and what was compared. @@ -1830,16 +1863,19 @@ def track_a_artifact_binding( key: _breakdown_check(key, value, recorded) for key, value in additive.items() } + known = [check for check in breakdowns.values() if check is not None] checks = { "ssa_parameters_revision_equal": ( artifact["ssa_parameters_revision"] == parameters_revision ), "cohort_diagnostics_equal_on_recorded_keys": not missing and not changed, + "additive_diagnostic_keys_known": len(known) == len(breakdowns), "additive_breakdowns_sum_to_recorded_totals": all( - check["sums_to_recorded_total"] - for check in breakdowns.values() - if check is not None + check["sums_to_recorded_total"] for check in known + ), + "additive_breakdowns_within_recorded_categories": all( + check["within_recorded_categories"] for check in known ), "cohort_source_provenance_equal": ( recorded_source == _json_normal(dict(source_provenance)) @@ -1854,6 +1890,9 @@ def track_a_artifact_binding( for key in changed }, "additive_diagnostic_keys": additive, + "unknown_additive_diagnostic_keys": sorted( + key for key, check in breakdowns.items() if check is None + ), "additive_breakdowns": breakdowns, } @@ -1882,6 +1921,9 @@ def require_track_a_artifact( f"the cohort or parameters differ from {path}: failed {failed}; " f"recorded keys missing {binding['recorded_keys_missing']}; " f"recorded keys changed {binding['recorded_keys_changed']}; " + "diagnostics neither recorded nor in " + "KNOWN_ADDITIVE_DIAGNOSTICS " + f"{binding['unknown_additive_diagnostic_keys']}; " f"additive breakdowns {binding['additive_breakdowns']}" ) return { @@ -1911,16 +1953,19 @@ def _artifact_binding_text(artifact: Mapping[str, Any]) -> str: check = (artifact.get("additive_breakdowns") or {}).get(key) if check is not None: part += ( - f" (sum {check['sum']}; recorded `{check['total_key']}` " - f"{check['recorded_total']})" + f" (added in `{check['added_by']}`; sum {check['sum']}, " + f"recorded `{check['total_key']}` {check['recorded_total']};" + f" within recorded `{check['categories_key']}`: " + f"{'yes' if check['within_recorded_categories'] else 'no'})" ) parts.append(part) return ( text + " The fresh cohort also records diagnostics the artifact " - "does not; they have no recorded value to compare with, and a " - "breakdown of a recorded total must sum to it: " - + "; ".join(parts) - + "." + "does not. Each is a reviewed breakdown listed in " + "`KNOWN_ADDITIVE_DIAGNOSTICS` with the commit that added it; its " + "counts must sum to the recorded total and name only recorded " + "categories, none above its recorded count (an unlisted diagnostic " + "refuses the run): " + "; ".join(parts) + "." ) @@ -2052,7 +2097,10 @@ def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument( - "--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR.expanduser() + "--output-dir", + type=Path, + default=None, + help="evidence directory for a run (required unless --render-only)", ) parser.add_argument("--anchor-wave", type=int, default=2011) parser.add_argument("--workers", type=int, default=4) @@ -2084,6 +2132,11 @@ def main(argv: list[str] | None = None) -> int: print(args.render_only / "RESULTS.md") return 0 + if args.output_dir is None: + raise SystemExit( + "--output-dir is required for a run: name a new evidence " + "directory (the script never overwrites a result.json)" + ) started = datetime.datetime.now(datetime.timezone.utc).isoformat() head = _git("rev-parse", "HEAD") dirty = _git("status", "--porcelain") diff --git a/tests/README-tiers.md b/tests/README-tiers.md index efa9ba8c..3f372c6f 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -38,9 +38,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 3,047 | -| `artifact` | 2,803 | +| `unit` | 3,053 | +| `artifact` | 2,804 | | `integration_psid` | 862 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 163 | -| **Total** | **7,395** | +| **Total** | **7,402** | diff --git a/tests/test_track_c_aime_agreement.py b/tests/test_track_c_aime_agreement.py index a8114cec..fcd206bf 100644 --- a/tests/test_track_c_aime_agreement.py +++ b/tests/test_track_c_aime_agreement.py @@ -704,11 +704,25 @@ def test_difference_bins_are_half_open_and_exhaustive(): _RECORDED = { "members": 3, "a3_spec": {"m4_waves": [2011]}, + "opening_stock_clock_rules": { + "a3_receipt_start": 7, + "linked_worker_birth_plus_62": 4, + "own_birth_plus_62": 10, + }, "opening_stock_entitlement_clamped": 5, } _ADDITIVE_KEY = "opening_stock_entitlement_clamped_by_clock_rule" -#: INVENTED breakdown that sums to the recorded total (5). +#: INVENTED breakdown that sums to the recorded total (5) and fits within +#: the recorded clock-rule counts (4 and 10). _ADDITIVE_VALUE = {"linked_worker_birth_plus_62": 2, "own_birth_plus_62": 3} +_ALL_CHECKS_TRUE = { + "ssa_parameters_revision_equal": True, + "cohort_diagnostics_equal_on_recorded_keys": True, + "additive_diagnostic_keys_known": True, + "additive_breakdowns_sum_to_recorded_totals": True, + "additive_breakdowns_within_recorded_categories": True, + "cohort_source_provenance_equal": True, +} def _artifact(recorded=None): @@ -733,20 +747,24 @@ def _binding(diagnostics, *, artifact=None, revision="INVENTED+tr2008"): ) +def _failed(binding): + return sorted(k for k, ok in binding["checks"].items() if not ok) + + def test_track_a_artifact_binding_compares_cohort_and_parameters(): diagnostics = {**_RECORDED, "a3_spec": {"m4_waves": (2011,)}} binding = _binding(diagnostics) - assert all(binding["checks"].values()) + assert binding["checks"] == _ALL_CHECKS_TRUE assert binding["recorded_diagnostic_keys"] == sorted(_RECORDED) assert binding["additive_diagnostic_keys"] == {} + assert binding["unknown_additive_diagnostic_keys"] == [] changed = _binding( {**diagnostics, "members": 4}, revision="INVENTED+other" ) assert changed["checks"] == { + **_ALL_CHECKS_TRUE, "ssa_parameters_revision_equal": False, "cohort_diagnostics_equal_on_recorded_keys": False, - "additive_breakdowns_sum_to_recorded_totals": True, - "cohort_source_provenance_equal": True, } assert changed["recorded_keys_changed"] == { "members": {"recorded": 3, "fresh": 4} @@ -754,50 +772,94 @@ def test_track_a_artifact_binding_compares_cohort_and_parameters(): missing = _binding( {k: v for k, v in diagnostics.items() if k != "members"} ) - assert not missing["checks"]["cohort_diagnostics_equal_on_recorded_keys"] + assert _failed(missing) == ["cohort_diagnostics_equal_on_recorded_keys"] assert missing["recorded_keys_missing"] == ["members"] -def test_an_additive_diagnostic_key_is_recorded_not_refused(): - # Regression: #454 added opening_stock_entitlement_clamped_by_clock_rule - # to the cohort diagnostics after the Track A artifact was written, and - # the whole-dict comparison stopped every real run before any engine - # call (cohort_diagnostics_equal: False). +def test_the_reviewed_additive_diagnostic_is_recorded_not_refused(): + # Regression: #454 (bfe9fa3e) added + # opening_stock_entitlement_clamped_by_clock_rule to the cohort + # diagnostics after the Track A artifact was written, and the + # whole-dict comparison stopped every real run before any engine call + # (cohort_diagnostics_equal: False). + assert tc.KNOWN_ADDITIVE_DIAGNOSTICS == { + _ADDITIVE_KEY: { + "added_by": "bfe9fa3e", + "total": "opening_stock_entitlement_clamped", + "categories": "opening_stock_clock_rules", + } + } fresh = {**_RECORDED, _ADDITIVE_KEY: dict(_ADDITIVE_VALUE)} recorded = dict(_artifact()["cohorts"]["2011"]) recorded.pop("source_provenance") assert tc._json_normal(recorded) != tc._json_normal(fresh) binding = _binding(fresh) - assert all(binding["checks"].values()) + assert binding["checks"] == _ALL_CHECKS_TRUE assert binding["additive_diagnostic_keys"] == { _ADDITIVE_KEY: _ADDITIVE_VALUE } assert binding["additive_breakdowns"] == { _ADDITIVE_KEY: { + "added_by": "bfe9fa3e", "total_key": "opening_stock_entitlement_clamped", "recorded_total": 5, "sum": 5, "sums_to_recorded_total": True, + "categories_key": "opening_stock_clock_rules", + "within_recorded_categories": True, } } # A breakdown that does not sum to the recorded total is refused. off = _binding({**fresh, _ADDITIVE_KEY: {"own_birth_plus_62": 4}}) - assert off["checks"] == { - "ssa_parameters_revision_equal": True, - "cohort_diagnostics_equal_on_recorded_keys": True, - "additive_breakdowns_sum_to_recorded_totals": False, - "cohort_source_provenance_equal": True, - } + assert _failed(off) == ["additive_breakdowns_sum_to_recorded_totals"] not_counts = _binding({**fresh, _ADDITIVE_KEY: ["own_birth_plus_62"]}) - assert not all(not_counts["checks"].values()) - # An additive key that breaks down no recorded total is recorded only. - other = _binding({**fresh, "invented_new_counter": 7}) - assert all(other["checks"].values()) - assert other["additive_diagnostic_keys"]["invented_new_counter"] == 7 - assert other["additive_breakdowns"]["invented_new_counter"] is None + assert "additive_breakdowns_sum_to_recorded_totals" in _failed(not_counts) # Changing a recorded key is still refused alongside an additive one. changed = _binding({**fresh, "opening_stock_entitlement_clamped": 6}) - assert not changed["checks"]["cohort_diagnostics_equal_on_recorded_keys"] + assert "cohort_diagnostics_equal_on_recorded_keys" in _failed(changed) + + +@pytest.mark.parametrize( + "breakdown", + [ + # Sums to 5 but counts 5 under a rule the artifact counts 4 times. + {"linked_worker_birth_plus_62": 5}, + # Sums to 5 but names a rule the artifact never counts. + {"invented_rule": 2, "own_birth_plus_62": 3}, + # Sums to 5 only through a negative count. + {"own_birth_plus_62": 6, "linked_worker_birth_plus_62": -1}, + ], +) +def test_an_additive_breakdown_must_fit_the_recorded_categories(breakdown): + # Regression: the 71c94743 check accepted each of these (it tested the + # sum only). + binding = _binding({**_RECORDED, _ADDITIVE_KEY: breakdown}) + assert "additive_breakdowns_within_recorded_categories" in ( + _failed(binding) + ) + assert not all(binding["checks"].values()) + + +@pytest.mark.parametrize( + "extra", + [ + # Not a breakdown of anything recorded. + {"invented_new_counter": 7}, + # Named like a breakdown of a recorded total and summing to it, but + # never reviewed. + {"opening_stock_entitlement_clamped_by_invented_rule": {"a": 5}}, + ], +) +def test_an_unreviewed_diagnostic_the_artifact_lacks_is_refused(extra): + # Regression: the whole-dict comparison refused any new diagnostic; the + # 71c94743 check accepted both of these (the first as "recorded only", + # the second through its _by_ name). Only the reviewed + # KNOWN_ADDITIVE_DIAGNOSTICS may be absent from the artifact. + fresh = {**_RECORDED, _ADDITIVE_KEY: dict(_ADDITIVE_VALUE), **extra} + binding = _binding(fresh) + assert _failed(binding) == ["additive_diagnostic_keys_known"] + assert binding["unknown_additive_diagnostic_keys"] == sorted(extra) + assert binding["additive_breakdowns"][next(iter(extra))] is None def test_require_track_a_artifact_refuses_and_records(tmp_path): @@ -818,9 +880,14 @@ def test_require_track_a_artifact_refuses_and_records(tmp_path): _ADDITIVE_KEY: _ADDITIVE_VALUE } text = tc._artifact_binding_text(document) - assert "3 keys" in text + assert "4 keys" in text assert f"`{_ADDITIVE_KEY}`" in text - assert "(sum 5; recorded `opening_stock_entitlement_clamped` 5)" in text + assert ( + "(added in `bfe9fa3e`; sum 5, recorded " + "`opening_stock_entitlement_clamped` 5; within recorded " + "`opening_stock_clock_rules`: yes)" + ) in text + assert "an unlisted diagnostic refuses the run" in text plain = tc._artifact_binding_text( tc.require_track_a_artifact( artifact_bytes, diagnostics=dict(_RECORDED), **kwargs @@ -835,6 +902,29 @@ def test_require_track_a_artifact_refuses_and_records(tmp_path): assert "INVENTED/artifact.json" in message assert "cohort_diagnostics_equal_on_recorded_keys" in message assert "'members': {'recorded': 3, 'fresh': 4}" in message + with pytest.raises(SystemExit) as unknown: + tc.require_track_a_artifact( + artifact_bytes, + diagnostics={**fresh, "invented_new_counter": 7}, + **kwargs, + ) + message = str(unknown.value) + assert "additive_diagnostic_keys_known" in message + assert "KNOWN_ADDITIVE_DIAGNOSTICS ['invented_new_counter']" in message + + +def test_a_run_requires_an_output_dir(monkeypatch): + # Regression: the default --output-dir named the 2026-09-23 evidence + # directory, so a run without the flag was aimed at an earlier run's + # directory. The check precedes every git, engine and data step. + assert tc.build_parser().parse_args([]).output_dir is None + + def no_git(*args): + raise AssertionError("reached git before refusing") + + monkeypatch.setattr(tc, "_git", no_git) + with pytest.raises(SystemExit, match="--output-dir is required"): + tc.main([]) def test_decimal_and_float_text_are_plain(): @@ -986,12 +1076,22 @@ def test_render_names_the_oracle_compared_against(fake_binding): assert "`legacy_fixed_35` (its PIA" in legacy -def test_render_refuses_a_result_without_an_oracle_record(fake_binding): +def test_render_refuses_a_result_of_another_schema(fake_binding): result = _invented_result(fake_binding) result.pop("oracle") result["schema_version"] = "populace_dynamics.track_c.aime_agreement.v1" with pytest.raises(ValueError, match="aime_agreement.v1"): tc.render_results(result) + # A 71c94743 (v2) result has an oracle record but the earlier artifact + # binding; its prose belongs to the script at that commit. + v2 = _invented_result(fake_binding) + v2["schema_version"] = "populace_dynamics.track_c.aime_agreement.v2" + with pytest.raises(ValueError, match="aime_agreement.v2"): + tc.render_results(v2) + no_oracle = _invented_result(fake_binding) + no_oracle.pop("oracle") + with pytest.raises(ValueError, match="render it with the script"): + tc.render_results(no_oracle) def test_actual_engine_agrees_with_the_oracle_on_invented_careers(): diff --git a/tests/test_track_c_track_a_artifact_binding.py b/tests/test_track_c_track_a_artifact_binding.py index 0fcc0ed7..72c315eb 100644 --- a/tests/test_track_c_track_a_artifact_binding.py +++ b/tests/test_track_c_track_a_artifact_binding.py @@ -31,6 +31,7 @@ ARTIFACT = ROOT / "runs" / "replication_urban2010_cola_v1.json" ADDITIVE_KEY = "opening_stock_entitlement_clamped_by_clock_rule" TOTAL_KEY = "opening_stock_entitlement_clamped" +CATEGORIES_KEY = "opening_stock_clock_rules" def _load_script(): @@ -80,6 +81,30 @@ def test_the_artifact_records_a_subset_of_the_code_s_diagnostics(): assert TOTAL_KEY in recorded +def test_the_reviewed_additive_diagnostics_are_exactly_what_the_code_adds(): + # A diagnostic the code starts producing after this test was written + # fails here, and Track C refuses to bind until it is reviewed and + # listed in KNOWN_ADDITIVE_DIAGNOSTICS. + produced = _code_diagnostics() + for wave in ("2011", "2009"): + _, recorded, _ = _recorded(wave) + assert set(produced) - set(recorded) == set( + tc.KNOWN_ADDITIVE_DIAGNOSTICS + ) + for known in tc.KNOWN_ADDITIVE_DIAGNOSTICS.values(): + assert known["total"] in recorded + assert known["categories"] in recorded + # What each listing claims holds for the code's own (INVENTED) + # diagnostics: the breakdown sums to its total and fits its categories. + for key, known in tc.KNOWN_ADDITIVE_DIAGNOSTICS.items(): + breakdown = produced[key] + categories = produced[known["categories"]] + assert sum(breakdown.values()) == produced[known["total"]] + assert all( + 0 <= count <= categories[name] for name, count in breakdown.items() + ) + + def test_the_recorded_2011_cohort_binds_with_the_added_breakdown(): artifact, recorded, source = _recorded() total = recorded[TOTAL_KEY] @@ -101,18 +126,45 @@ def test_the_recorded_2011_cohort_binds_with_the_added_breakdown(): assert binding["recorded_diagnostic_keys"] == sorted(recorded) assert binding["additive_diagnostic_keys"] == {ADDITIVE_KEY: split} assert binding["additive_breakdowns"][ADDITIVE_KEY] == { + "added_by": "bfe9fa3e", "total_key": TOTAL_KEY, "recorded_total": total, "sum": total, "sums_to_recorded_total": True, + "categories_key": CATEGORIES_KEY, + "within_recorded_categories": True, } # An INVENTED split one short of the recorded total is refused. + one_short = { + "linked_worker_birth_plus_62": 1, + "own_birth_plus_62": total - 2, + } short = tc.track_a_artifact_binding( + artifact, diagnostics={**recorded, ADDITIVE_KEY: one_short}, **kwargs + ) + assert not short["checks"]["additive_breakdowns_sum_to_recorded_totals"] + # An INVENTED split that sums to the total but puts it all under the + # least-counted recorded clock rule, which the artifact counts fewer + # times than that, is refused. + rules = recorded[CATEGORIES_KEY] + fewest = min(rules, key=rules.get) + assert rules[fewest] < total + crowded = tc.track_a_artifact_binding( artifact, - diagnostics={**recorded, ADDITIVE_KEY: {"own_birth_plus_62": 1}}, + diagnostics={**recorded, ADDITIVE_KEY: {fewest: total}}, **kwargs, ) - assert not short["checks"]["additive_breakdowns_sum_to_recorded_totals"] + assert crowded["checks"]["additive_breakdowns_sum_to_recorded_totals"] + assert not crowded["checks"][ + "additive_breakdowns_within_recorded_categories" + ] + # A diagnostic the code does not list is refused. + unknown = tc.track_a_artifact_binding( + artifact, + diagnostics={**fresh, "invented_new_counter": 7}, + **kwargs, + ) + assert not unknown["checks"]["additive_diagnostic_keys_known"] # Dropping a recorded key is refused. dropped = {k: v for k, v in fresh.items() if k != TOTAL_KEY} missing = tc.track_a_artifact_binding( diff --git a/tests/tier_counts.json b/tests/tier_counts.json index e30ad2ad..d6398ecd 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,8 +1,8 @@ { "schema_version": 1, "counts": { - "unit": 3047, - "artifact": 2803, + "unit": 3053, + "artifact": 2804, "integration_psid": 862, "reproduction_legacy": 520, "oracle_policyengine": 163