Catch reference keys policyengine-core drops and citation URLs filed in documentation

changelog.d/reference-key-hygiene.fixed.md

@@ -0,0 +1 @@
+Move citations that policyengine-core was dropping, or that sat in variable `documentation`, into `reference`: a Pell Grant value under a misspelled `refrence:` key, eight `metadata.references` aliases, and the URLs of 39 variables, including the doubled URL in `az_taxable_income`. Add code-health checks for misplaced citation/unit keys and known citation-key misspellings in both `.yaml` and `.yml` files, and stop the reference URL checks accepting a second URL fused after a fragment `=`.

policyengine_us/parameters/gov/ed/pell_grant/head/marginal_rate.yaml

@@ -90,8 +90,9 @@ brackets:
             - title: THE EFC FORMULA, 2023–2024
               href: https://fsapartners.ed.gov/sites/default/files/2022-08/2324EFCFormulaGuide.pdf#page=29
       2024-08-01:
-          value: 20_600
-          refrence:
+        value: 20_600
+        metadata:
+          reference:
             - title: FAFSA® Pell Eligibility and SAI Guide | Step 2 | Table A5
               href: https://fsapartners.ed.gov/sites/default/files/2024-01/20242025FAFSAPellEligibilityandSAIGuide.pdf#page=18
     rate:

policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/catch_up/age_threshold.yaml

@@ -5,6 +5,6 @@ metadata:
   unit: year
   period: year
   label: Pension contribution catch-up age threshold
-  references:
+  reference:
   - title: 26 CFR § 1.414(v)-1 - Catch-up contributions. (g)(3)(ii)
     href: https://www.law.cornell.edu/cfr/text/26/1.414(v)-1#g_3_ii

policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/catch_up/limit/ira.yaml

@@ -12,7 +12,7 @@ metadata:
     rounding:
       type: downwards
       interval: 100
-  references:
+  reference:
   - title: 26 U.S. Code § 219 - Retirement savings (b)(5)(B)
     href: https://www.law.cornell.edu/uscode/text/26/219
   - title: Cost-of-Living Adjustments for Retirement Items

policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/limit/401k.yaml

@@ -17,7 +17,7 @@ metadata:
     rounding:
       type: downwards
       interval: 500
-  references:
+  reference:
   - title: 26 CFR § 1.402(g)-1 - Limitation on exclusion for elective deferrals. (d))(i)
     href: https://law.cornell.edu/cfr/text/26/1.402(g)-1
   # Inflation adjusted amounts

policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/limit/annual_additions.yaml

@@ -18,7 +18,7 @@ metadata:
     rounding:
       type: downwards
       interval: 1000
-  references:
+  reference:
   - title: 26 U.S. Code § 415 - Limitations on benefits and contributions under qualified plans
     href: https://www.law.cornell.edu/uscode/text/26/415
   - title: Cost-of-Living Adjustments for Retirement Items

policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/limit/ira.yaml

@@ -13,7 +13,7 @@ metadata:
     rounding:
       type: downwards
       interval: 500
-  references:
+  reference:
   - title: 26 U.S. Code § 219 - Retirement savings (b)(5)(A)
     href: https://www.law.cornell.edu/uscode/text/26/219
   # Inflation adjusted amounts

policyengine_us/parameters/gov/states/ca/tax/income/amt/amti/sources.yaml

@@ -14,7 +14,7 @@ metadata:
    unit: list  
    period: year 
    label: California altenative minimum taxable income adjustments 
-   references: 
+   reference:
     - title: 2021 Schedule P (540) Alternative Minimum Tax and Credit Limitations - Residents
       href: https://www.ftb.ca.gov/forms/2021/2021-540-p.pdf#page=1
     - title: 2022 Schedule P (540) Alternative Minimum Tax and Credit Limitations - Residents

policyengine_us/parameters/gov/states/de/tax/income/subtractions/subtractions.yaml

@@ -22,7 +22,7 @@ metadata:
   unit: list
   period: year
   label: Delaware adjusted gross income subtractions
-  references:
+  reference:
   - title: Delaware Code, § 1106 Modifcations (b)
     href: https://delcode.delaware.gov/title30/c011/sc02/index.html
   - title: Delaware HB 285 (151st General Assembly), amending 30 Del. C. § 1106(b)(10)

policyengine_us/parameters/gov/states/id/tax/income/subtractions/subtractions.yaml

@@ -29,7 +29,7 @@ metadata:
   unit: list
   period: year
   label: Idaho adjusted gross income subtractions
-  references: 
+  reference:
   - title: 2022 Idaho CodeSection 63-3022D - deduction of expenses for household and dependent care services
     href: https://law.justia.com/codes/idaho/2022/title-63/chapter-30/section-63-3022d/
   - title: Individual Income Tax Forms and Instructions 2021

policyengine_us/reforms/state_dependent_exemptions/repeal_state_dependent_exemptions.py

@@ -8,7 +8,7 @@ class hi_regular_exemptions(Variable):
         entity = TaxUnit
         label = "Hawaii regular exemptions"
         unit = USD
-        documentation = "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=20"
+        reference = "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=20"
         definition_period = YEAR
         defined_for = StateCode.HI
 

policyengine_us/tests/code_health/test_no_concatenated_reference_urls.py

@@ -19,10 +19,12 @@
   leading or trailing spaces, or a note such as "(Line 3)" (put notes in a
   Python comment).
 
-A scheme may follow whitespace, an opening bracket or quote, or "=" (a URL
-passed as a query value, as in azleg.gov's viewdocument/?docName=https://...).
-It may follow "/" only when that slash closes an archive.org wrapper such as
-https://web.archive.org/web/<timestamp>/https://...
+A scheme may follow whitespace or an opening bracket or quote. It may follow
+"=" only when that "=" ends a query parameter (a URL passed as a query value,
+as in azleg.gov's viewdocument/?docName=https://... or ?a=1&url=https://...);
+an "=" inside a fragment, as in x.pdf#page=https://..., marks two URLs glued
+together. It may follow "/" only when that slash closes an archive.org wrapper
+such as https://web.archive.org/web/<timestamp>/https://...
 """
 
 import ast
@@ -35,12 +37,15 @@
 PACKAGE = Path(__file__).resolve().parents[2]
 ATTRIBUTES = {"reference", "documentation"}
 SCHEME = re.compile(r"https?://", re.IGNORECASE)
-ALLOWED_BEFORE_SCHEME = set("([{<\"'`=")
+ALLOWED_BEFORE_SCHEME = set("([{<\"'`")
 BARE_URL = re.compile(r"https?://\S+", re.IGNORECASE)
 ARCHIVE_WRAPPER = re.compile(
     r"(?:^|[\s(\[{<\"'`=])(?:https?://)?(?:web\.)?archive\.org/web/[^\s/]+/$",
     re.IGNORECASE,
 )
+# "?name=" or "?a=1&name=" with no "#" since the "?": the end of a query
+# parameter whose value is a URL.
+QUERY_VALUE = re.compile(r"\?(?:[^#\s]*&)?[^?&#=\s]*=$")
 
 
 def glued_url_offsets(text: str) -> list[int]:
@@ -53,7 +58,7 @@ def glued_url_offsets(text: str) -> list[int]:
         before = text[start - 1]
         if before.isspace() or before in ALLOWED_BEFORE_SCHEME:
             continue
-        if before == "/" and ARCHIVE_WRAPPER.search(text, 0, start):
+        if is_wrapped_url(text, start):
             continue
         offsets.append(start)
     return offsets
@@ -86,9 +91,9 @@ def is_wrapped_url(text: str, start: int) -> bool:
     """True if the URL scheme at `start` sits inside another URL: as a query
     value (`...?docName=https://...`) or after an archive.org wrapper."""
     before = text[start - 1]
-    return before == "=" or (
-        before == "/" and bool(ARCHIVE_WRAPPER.search(text, 0, start))
-    )
+    if before == "=":
+        return bool(QUERY_VALUE.search(text, 0, start))
+    return before == "/" and bool(ARCHIVE_WRAPPER.search(text, 0, start))
 
 
 def is_bare_url_or_has_none(text: str) -> bool:
@@ -135,6 +140,9 @@ def find_glued_urls(source: str, filename: str = "<string>") -> list[str]:
         "https://example.gov/a)https://example.gov/b",
         "HTTPS://EXAMPLE.GOV/A.PDFHTTPS://EXAMPLE.GOV/B.PDF",
         "https://example.gov/web/2024/https://example.gov/b",
+        "https://example.gov/a.pdf#page=https://example.gov/b.pdf",
+        "https://example.gov/a.pdf#page=2&view=https://example.gov/b.pdf",
+        "https://example.gov/a?x=1#page=https://example.gov/b.pdf",
     ],
 )
 def test_glued_urls_are_flagged(text):
@@ -155,6 +163,7 @@ def test_glued_urls_are_flagged(text):
         "[IT-201-I](https://www.tax.ny.gov/it201i.pdf)",
         '"https://example.gov/a"',
         "https://www.cbo.gov/system/files/2026-02/51138-2026-02-Revenue.xlsx",
+        "https://example.gov/view?a=1&url=https://example.gov/b.pdf",
     ],
 )
 def test_separated_and_wrapped_urls_are_not_flagged(text):
@@ -176,6 +185,7 @@ def test_separated_and_wrapped_urls_are_not_flagged(text):
         "https://example.gov/a.pdf(https://example.gov/b.pdf)",
         "https://example.gov/a.pdf[https://example.gov/b.pdf]",
         "https://example.gov/web/2024/https://example.gov/b",
+        "https://example.gov/a.pdf#page=https://example.gov/b.pdf",
     ],
 )
 def test_reference_entries_that_are_not_one_bare_url_are_flagged(text):
@@ -189,6 +199,7 @@ def test_reference_entries_that_are_not_one_bare_url_are_flagged(text):
         "https://www.azleg.gov/viewdocument/?docName=https://www.azleg.gov/ars/43/01001.htm",
         "https://web.archive.org/web/20250720165524/https://www.mass.gov/doc/heap",
         "https://web.archive.org/web/2021id_/https://example.gov/a?url=https://b.gov",
+        "https://example.gov/view?a=1&url=https://example.gov/b.pdf",
         "26 U.S.C. 32(b)",
         "IRS Publication 596, page 3",
         "",