Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file modified app/public/paper/policybench.pdf
Binary file not shown.
6 changes: 3 additions & 3 deletions app/public/paper/web/index.html

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions app/src/paperSnapshot.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"pdfVersion": "1607c71abaf3",
"pdfVersion": "f15264138ec2",
"responseWindow": "June 12 and October 8, 2026",
"snapshotDate": "2026-10-10",
"webVersion": "31cd4141373a"
"webVersion": "33be2303d996"
}
2 changes: 1 addition & 1 deletion paper/index.qmd
Original file line number Diff line number Diff line change
Expand Up @@ -892,7 +892,7 @@ reference_audit = pd.DataFrame(
["US SNAP, SSI and CHIP (September 2026 wave)", "Whether listed mortgage interest is on the occupied home, unlisted hours worked, survivor-benefit taxability, and whether an adult tax dependent is the filers' child (three Medicaid outputs) excluded as unlisted inputs. SNAP October-December references regenerated with the FY2026 figures, the last published before the freeze. SNAP references regenerated for every month of 2026 with the engine's SNAP rounding defects fixed as policyengine-us fixed them after the freeze (cents in the allotment, an unrounded minimum benefit, floored net income, unrounded income standards). One Michigan SNAP reference regenerated with the fix for the SNAP child support option, whose engine values carried the opposite meaning (Michigan deducts child support from net income, and the engine excluded it from gross income; fixed upstream in policyengine-us #9586 on 2026-09-24). The heat-and-eat utility allowance granted without an elderly or disabled member, which P.L. 119-21 ended, excluded as an engine defect. The CHIP flag dissolved: employer coverage bars CHIP."],
["US projected 2026 parameters (September 2026 wave)", "No judge flag: a recomputation with every projected 2026 value held found references resting on price-index projections. Wisconsin, Minnesota, Michigan and Missouri amounts and Maryland's withholding allowance had been published before the freeze; California's indexed amounts, the IRS sales tax tables, Idaho's threshold and Maryland's return deduction had not. Scored references regenerated with the published or last-published amounts; Wisconsin's reach only the corrected values of Wisconsin outputs excluded for engine defects."],
[f"US references on policyengine-us {r.september_upgrade.engine_version} (September 2026 upgrade)", f"Once PolicyBench had ported the conventions, an investigator traced every output that still moved between policyengine-us {r.september_upgrade.previous_engine_version} and {r.september_upgrade.engine_version} to its upstream change and settled it against primary sources and their publication dates. An independent reviewer tried to refute each settlement. The upgrade changes {r.september_upgrade.scored_change_count} scored references, for law published before the freeze that {r.september_upgrade.previous_engine_version} did not encode or misapplied: New Jersey's 2026-2028 child tax credit schedule (P.L.2026, c.26), Arizona's 200 percent SNAP categorical eligibility limit from March 2026, child support received counted as income for school meals (7 CFR 245.6(a)(5)(ii)), and New York's Empire State child credit phase-out rounding. The investigation found, and PolicyBench excluded, {r.september_upgrade.new_exclusion_count} federal income tax outputs that depend on an unlisted input: {r.september_upgrade.engine_version} counts a listed state and local tax refund as income, which is right only if the refunded tax reduced federal tax in the prior year (26 U.S.C. 111(a)). PolicyBench re-reviewed the {r.september_upgrade.rechecked_count} excluded outputs that move; all stay excluded. The re-review of a California household's excluded SNAP output flagged its head's Medicaid eligibility, which turns on the same unlisted input, SSI's disability criterion, through the Working Disabled Program; PolicyBench excluded it."],
*([[f"US references on policyengine-us {r.policyengine_us_version} (October 2026 upgrade)", f"The move from policyengine-us {r.previous_policyengine_us_version} fixes the engine defects behind {r.engine_upgrade_restored_count} outputs ({r.engine_upgrade_restored_count - r.engine_upgrade_restored_ruled_count} that earlier releases excluded return to scoring at the new version's values; the {r.engine_upgrade_restored_ruled_count} ruled on {r.ruling_date} while still scored stay scored) and changes {r.engine_upgrade_scored_change_count} scored {'reference' if r.engine_upgrade_scored_change_count == 1 else 'references'} beyond the dollar tolerance; {r.engine_upgrade_new_exclusion_count} {'output leaves' if r.engine_upgrade_new_exclusion_count == 1 else 'outputs leave'} scoring. PolicyBench re-reviewed the {r.engine_upgrade_rechecked_count} excluded outputs whose values move; all stay excluded. {r.engine_upgrade_restored_sentence} {r.engine_upgrade_scored_change_sentence} {r.engine_upgrade_new_exclusion_sentence}"]] if r.engine_upgrade_count > 1 else []),
*([[f"US references on policyengine-us {r.policyengine_us_version} (October 2026 upgrade)", f"The move from policyengine-us {r.previous_policyengine_us_version} {r.engine_upgrade_restored_split_clause} and changes {r.engine_upgrade_scored_change_count} scored {'reference' if r.engine_upgrade_scored_change_count == 1 else 'references'} beyond the dollar tolerance; {r.engine_upgrade_new_exclusion_count} {'output leaves' if r.engine_upgrade_new_exclusion_count == 1 else 'outputs leave'} scoring. PolicyBench re-reviewed the {r.engine_upgrade_rechecked_count} excluded outputs whose values move; all stay excluded. {r.engine_upgrade_restored_sentence} {r.engine_upgrade_scored_change_sentence} {r.engine_upgrade_new_exclusion_sentence}"]] if r.engine_upgrade_count > 1 else []),
["US federal and Virginia income tax (October 2026 review)", f"No judge flag: a review of release dashboard-data-20260930 swept every reference on policyengine-us {r.review_exclusion_engine_version} under readings of two inputs no prompt states. The state income tax part of the federal state and local tax deduction is policyengine-us's formula estimate of state income tax withheld on federal adjusted gross income; read as the household paying its 2026 liability during the year, or paying none, {r.review_sweep_newly_excluded_count_word('salt_withholding')} scored federal income tax outputs move, and PolicyBench excluded them; the readings also move {r.review_sweep_already_excluded_count_word('salt_withholding')} federal outputs already excluded. policyengine-us treats every Medicare-eligible person as enrolled and counts a modeled Part B premium as a medical expense; without it, {r.review_sweep_moved_household_count_word('medicare_part_b')} household's federal and Virginia income tax move, and both are excluded."],
],
columns=["Discrepancy class", "Review outcome"],
Expand Down
4 changes: 2 additions & 2 deletions paper/snapshot/20260501/manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -169,12 +169,12 @@
"rendered_paper_artifacts": {
"pdf": {
"path": "app/public/paper/policybench.pdf",
"sha256": "1607c71abaf35fae6f6d51576450c91747f8fb158485fc6ea64431e4c4c683c3"
"sha256": "f15264138ec20def17884219075533bd2604a7ba879f3bd11c8973b800f97c23"
},
"web": {
"files": {
"figures/positive_zero_scatter.png": "f13ca00d6a6a2232ee658733507d6e0d39dfc8f639b74ec4b3f016c85acc7fb6",
"index.html": "31cd4141373acbcd2d474841f9e4dbf488df8dd33e62b70699a5ffe47bd42551",
"index.html": "33be2303d996e4cbd9271c528a3836c76560d6613db42f2193ed98a125717f9e",
"pe-tokens.css": "8f24d8da26f583c8ffddffcdcd172b6d52cbecfec20eda55bd39d7aa829f41d8",
"policybench-theme.css": "0e12c5fd615558259e5bce0167a38424e54f9ceb280666c4afd660d759cd1cb9",
"site_libs/clipboard/clipboard.min.js": "e17a1d816e13c0826e0ed7febfabc3277f45571234bde0bf9120829a7169edc9",
Expand Down
97 changes: 79 additions & 18 deletions policybench/paper_results.py
Original file line number Diff line number Diff line change
Expand Up @@ -288,8 +288,10 @@ def partition_engine_upgrade_changes(
"""Split an engine upgrade's changed outputs into four disjoint groups.

``excluded`` is the exclusion record once the upgrade was decided, and
``restored`` the excluded outputs the upgrade returned to scoring at the
new engine's values. A change to an excluded output is a new exclusion; a
``restored`` the outputs whose engine defects the upgrade fixes, scored at
the new engine's values (those a published board excluded return to
scoring; any a ruling decided while still scored stay scored). A change to
an excluded output is a new exclusion; a
change to a restored output is a restoration. Any other change is scored,
and moves beyond the exact-match tolerance ($1 for an amount, any change
for a 0/1 flag) or within it. Every change lands in exactly one group, so
Expand Down Expand Up @@ -389,6 +391,49 @@ def restored_split_sentence(restored: int, ruled: int, ruling_date: str) -> str:
)


def restored_split_clause(restored: int, ruled: int, ruling_date: str) -> str:
"""The upgrades table's form of ``restored_split_sentence``: a verb phrase
finishing "The move from policyengine-us X ...", with numerals as the rest
of that cell writes counts. It names the ``restored`` outputs whose engine
defects the upgrade fixes and, in parentheses, splits them as the sentence
does: ``restored - ruled`` that earlier releases excluded return to
scoring, and ``ruled`` that rulings on ``ruling_date`` decided while they
were still scored stay scored. A lone output in either group is "one", not
"1"."""
if not 0 <= ruled <= restored:
raise ValueError(f"{ruled} ruled of {restored} fixed outputs")
if not restored:
return "fixes none of the engine defects behind excluded outputs"
returned = restored - ruled
defects = "defect" if restored == 1 else "defects"
noun = "output" if restored == 1 else "outputs"
lead = f"fixes the engine {defects} behind {restored} {noun}"
if not ruled:
excluded = (
"an earlier release excluded it, and it returns"
if restored == 1
else "earlier releases excluded them, and they return"
)
return f"{lead} ({excluded} to scoring at the new version's values)"
if not returned:
decided = (
"it while it was still scored, and it stays"
if ruled == 1
else "them while they were still scored, and they stay"
)
return (
f"{lead} (the {ruling_date} rulings decided {decided} scored at the new "
"version's values)"
)
back = "one that earlier releases excluded returns"
if returned > 1:
back = f"{returned} that earlier releases excluded return"
kept = f"the one ruled on {ruling_date} while still scored stays"
if ruled > 1:
kept = f"the {ruled} ruled on {ruling_date} while still scored stay"
return f"{lead} ({back} to scoring at the new version's values; {kept} scored)"


def _key(entry: dict) -> tuple[str, str]:
return entry["scenario_id"], entry.get("variable", "snap")

Expand Down Expand Up @@ -430,8 +475,9 @@ class EngineUpgrade:
holds those excluded once it was decided: records decided by the end of
its date and not removed by then. A record decided after the upgrade never
counts here, and a record a later revision removed still does.
``restored`` holds the outputs the upgrade returned to scoring (its
``regenerated_exclusions``).
``restored`` holds the outputs whose engine defects the upgrade fixes (its
``regenerated_exclusions``): those a published board excluded return to
scoring, and any a ruling decided while still scored stay scored.
"""

def __init__(
Expand Down Expand Up @@ -1851,7 +1897,10 @@ def engine_upgrade_new_exclusion_count(self) -> int:

@property
def engine_upgrade_restored_count(self) -> int:
"""Excluded outputs the last upgrade returned to scoring."""
"""Outputs whose engine defects the last upgrade fixes (its
``regenerated_exclusions``): those a published board excluded return
to scoring, and those the 2026-10-06 rulings decided while still
scored (``engine_upgrade_restored_ruled_count``) stay scored."""
last = self.last_engine_upgrade
return 0 if last is None else last.restored_count

Expand All @@ -1861,7 +1910,7 @@ def engine_upgrade_restored_count_word(self) -> str:

@property
def engine_upgrade_restored_ruled_count(self) -> int:
"""Restored outputs the 2026-10-06 rulings had decided to exclude. No
"""Fixed outputs the 2026-10-06 rulings had decided to exclude. No
published board excluded them: they were still scored when the rulings
came, so they stay scored rather than return to scoring."""
last = self.last_engine_upgrade
Expand All @@ -1883,6 +1932,16 @@ def engine_upgrade_restored_split_sentence(self) -> str:
RULING_DATE,
)

@property
def engine_upgrade_restored_split_clause(self) -> str:
"""The same split as a clause for the upgrades table, after "The move
from policyengine-us X"."""
return restored_split_clause(
self.engine_upgrade_restored_count,
self.engine_upgrade_restored_ruled_count,
RULING_DATE,
)

@cached_property
def engine_upgrade_timing(self) -> dict | None:
"""The last upgrade's sweep timing record, when that upgrade is the
Expand Down Expand Up @@ -1949,11 +2008,11 @@ def engine_upgrade_timing_sentence(self) -> str:

@property
def engine_upgrade_restored_fixes(self) -> list[tuple[str, int, list[str]]]:
"""The defects behind the outputs the last upgrade returned to scoring:
each one's name, how many outputs it restores and the upstream pull
requests its regenerated_exclusions entries name (not the related
ones), in the sidecar's order. A record with several root causes names
each."""
"""The defects the last upgrade fixes behind its fixed outputs (those
that return to scoring and those still scored alike): each one's name,
how many outputs it fixes and the upstream pull requests its
regenerated_exclusions entries name (not the related ones), in the
sidecar's order. A record with several root causes names each."""
last = self.last_engine_upgrade
fixes: dict[str, tuple[int, list[str]]] = {}
for entry in [] if last is None else last.revision["regenerated_exclusions"]:
Expand All @@ -1971,7 +2030,7 @@ def engine_upgrade_restored_fixes(self) -> list[tuple[str, int, list[str]]]:

@property
def engine_upgrade_restored_fix_phrase(self) -> str:
"""The fixes behind the restored outputs as one series, each with the
"""The fixes behind the fixed outputs as one series, each with the
upstream pull requests it names and, past one, how many outputs."""
parts = []
for label, count, prs in self.engine_upgrade_restored_fixes:
Expand All @@ -1982,15 +2041,17 @@ def engine_upgrade_restored_fix_phrase(self) -> str:

@property
def engine_upgrade_restored_sentence(self) -> str:
"""One sentence naming the fixes behind the restored outputs."""
"""One sentence naming the fixes behind all the fixed outputs. They
are "fixed", not "restored": the ones the 2026-10-06 rulings decided
while still scored never left scoring."""
phrase = self.engine_upgrade_restored_fix_phrase
if not phrase:
return ""
return f"The restored outputs take the upstream fixes for {phrase}."
return f"The fixed outputs take the upstream fixes for {phrase}."

@property
def engine_upgrade_restored_targets(self) -> dict[str, list[dict]]:
"""The last upgrade's restored outputs by how their corrected value is
"""The last upgrade's fixed outputs by how their corrected value is
known (the sidecar's ``target.kind``): ``record`` for the value the
exclusion record carries, ``fix_modules`` for the value the audit's
fix modules give on a later engine that still has the defect. Refuses,
Expand All @@ -2007,14 +2068,14 @@ def engine_upgrade_restored_targets(self) -> dict[str, list[dict]]:
raise ValueError(f"{key} has an unknown target kind {target['kind']!r}")
tolerance = entry["tolerance"]
if not 0 <= tolerance <= 1:
raise ValueError(f"{key} is restored on a tolerance of {tolerance}")
raise ValueError(f"{key} is fixed on a tolerance of {tolerance}")
variable = entry["variable"]
if abs(
entry["regenerated"] - target["value"]
) > tolerance or moves_beyond_tolerance(
variable, target["value"], entry["regenerated"]
):
raise ValueError(f"{key} is restored off its target")
raise ValueError(f"{key} is fixed off its target")
if target["kind"] == "record":
if target["value"] != entry["record"]["alternative_value"]:
raise ValueError(f"{key}'s target is not its record's value")
Expand All @@ -2027,7 +2088,7 @@ def engine_upgrade_restored_targets(self) -> dict[str, list[dict]]:

@property
def engine_upgrade_restored_target_sentence(self) -> str:
"""How the restored outputs' corrected values are known: by the
"""How the fixed outputs' corrected values are known: by the
exclusion record, or by the audit's fix modules on a later engine."""
targets = self.engine_upgrade_restored_targets
by_record, by_modules = targets["record"], targets["fix_modules"]
Expand Down
27 changes: 23 additions & 4 deletions tests/test_disclosures.py
Original file line number Diff line number Diff line change
Expand Up @@ -688,6 +688,25 @@ def test_rendered_pdf_takes_its_engine_times_from_the_timing_record():
assert sentence in text, sentence


def test_rendered_html_calls_the_upgrade_fixes_fixed_not_restored():
"""Round-2 final review nits: the upgrades table renders its split from
restored_split_clause, and the sentence naming the upstream fixes, in the
table and in the October paragraph, calls all the fixed outputs "fixed",
since the ones ruled on 2026-10-06 while still scored never left scoring."""
text = _html_text()
assert r.engine_upgrade_restored_split_clause in text
sentence = r.engine_upgrade_restored_sentence
assert sentence.startswith("The fixed outputs take the upstream fixes for ")
assert text.count(sentence) == 2
assert "restored outputs take" not in text


def test_rendered_pdf_calls_the_upgrade_fixes_fixed_not_restored():
text = _straight_quotes(re.sub(r"\s+", " ", _pdf_running_text()))
assert "restored outputs take" not in text
assert "fixed outputs take the upstream fixes for" in text


def _engine_groups(exclusions) -> list[tuple[str, int]]:
"""Excluded outputs by the engine version of their value, oldest first."""
by_engine = Counter(
Expand Down Expand Up @@ -961,11 +980,11 @@ def test_card_counts_follow_the_frozen_records():

r = PaperResults()
card = re.sub(r"\s+", " ", BENCHMARK_CARD.read_text())
# The October move: what it restores, from where, and why.
restored, ruled = r.engine_upgrade_restored_count, r.ruled_regenerated_count
# The October move: what it fixes, from where, and why.
fixed, ruled = r.engine_upgrade_restored_count, r.ruled_regenerated_count
assert (
f"The new version fixes the defects behind {count_word(restored)} outputs. "
f"Release 20261006 excluded {count_word(restored - ruled)} of them, which "
f"The new version fixes the defects behind {count_word(fixed)} outputs. "
f"Release 20261006 excluded {count_word(fixed - ruled)} of them, which "
f"return to scoring; the other {count_word(ruled)} are the 2026-10-06 "
"defects below, which stay scored."
) in card
Expand Down
Loading
Loading