diff --git a/changelog.d/reference-key-hygiene.fixed.md b/changelog.d/reference-key-hygiene.fixed.md new file mode 100644 index 00000000000..66948cc8672 --- /dev/null +++ b/changelog.d/reference-key-hygiene.fixed.md @@ -0,0 +1 @@ +Move citations that policyengine-core was dropping, or that sat in variable `documentation`, into `reference`: a Pell Grant value under a misspelled `refrence:` key, eight `metadata.references` aliases, and the URLs of 39 variables, including the doubled URL in `az_taxable_income`. Add code-health checks for misplaced citation/unit keys and known citation-key misspellings in both `.yaml` and `.yml` files, and stop the reference URL checks accepting a second URL fused after a fragment `=`. diff --git a/policyengine_us/parameters/gov/ed/pell_grant/head/marginal_rate.yaml b/policyengine_us/parameters/gov/ed/pell_grant/head/marginal_rate.yaml index 59698a22ee5..cacf62736e6 100644 --- a/policyengine_us/parameters/gov/ed/pell_grant/head/marginal_rate.yaml +++ b/policyengine_us/parameters/gov/ed/pell_grant/head/marginal_rate.yaml @@ -90,8 +90,9 @@ brackets: - title: THE EFC FORMULA, 2023–2024 href: https://fsapartners.ed.gov/sites/default/files/2022-08/2324EFCFormulaGuide.pdf#page=29 2024-08-01: - value: 20_600 - refrence: + value: 20_600 + metadata: + reference: - title: FAFSA® Pell Eligibility and SAI Guide | Step 2 | Table A5 href: https://fsapartners.ed.gov/sites/default/files/2024-01/20242025FAFSAPellEligibilityandSAIGuide.pdf#page=18 rate: diff --git a/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/catch_up/age_threshold.yaml b/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/catch_up/age_threshold.yaml index d87b93ac41d..e9fce231f72 100644 --- a/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/catch_up/age_threshold.yaml +++ b/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/catch_up/age_threshold.yaml @@ -5,6 +5,6 @@ metadata: unit: year period: year label: Pension contribution catch-up age threshold - references: + reference: - title: 26 CFR § 1.414(v)-1 - Catch-up contributions. (g)(3)(ii) href: https://www.law.cornell.edu/cfr/text/26/1.414(v)-1#g_3_ii diff --git a/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/catch_up/limit/ira.yaml b/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/catch_up/limit/ira.yaml index 034263f142f..593a7c54a59 100644 --- a/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/catch_up/limit/ira.yaml +++ b/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/catch_up/limit/ira.yaml @@ -12,7 +12,7 @@ metadata: rounding: type: downwards interval: 100 - references: + reference: - title: 26 U.S. Code § 219 - Retirement savings (b)(5)(B) href: https://www.law.cornell.edu/uscode/text/26/219 - title: Cost-of-Living Adjustments for Retirement Items diff --git a/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/limit/401k.yaml b/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/limit/401k.yaml index 000c686dab2..77a423bfe68 100644 --- a/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/limit/401k.yaml +++ b/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/limit/401k.yaml @@ -17,7 +17,7 @@ metadata: rounding: type: downwards interval: 500 - references: + reference: - title: 26 CFR § 1.402(g)-1 - Limitation on exclusion for elective deferrals. (d))(i) href: https://law.cornell.edu/cfr/text/26/1.402(g)-1 # Inflation adjusted amounts diff --git a/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/limit/annual_additions.yaml b/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/limit/annual_additions.yaml index d2a9512c9d1..c192036f571 100644 --- a/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/limit/annual_additions.yaml +++ b/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/limit/annual_additions.yaml @@ -18,7 +18,7 @@ metadata: rounding: type: downwards interval: 1000 - references: + reference: - title: 26 U.S. Code § 415 - Limitations on benefits and contributions under qualified plans href: https://www.law.cornell.edu/uscode/text/26/415 - title: Cost-of-Living Adjustments for Retirement Items diff --git a/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/limit/ira.yaml b/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/limit/ira.yaml index 04b5bb06907..b9117984543 100644 --- a/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/limit/ira.yaml +++ b/policyengine_us/parameters/gov/irs/gross_income/retirement_contributions/limit/ira.yaml @@ -13,7 +13,7 @@ metadata: rounding: type: downwards interval: 500 - references: + reference: - title: 26 U.S. Code § 219 - Retirement savings (b)(5)(A) href: https://www.law.cornell.edu/uscode/text/26/219 # Inflation adjusted amounts diff --git a/policyengine_us/parameters/gov/states/ca/tax/income/amt/amti/sources.yaml b/policyengine_us/parameters/gov/states/ca/tax/income/amt/amti/sources.yaml index ac2b7c877b1..620c19bce9b 100644 --- a/policyengine_us/parameters/gov/states/ca/tax/income/amt/amti/sources.yaml +++ b/policyengine_us/parameters/gov/states/ca/tax/income/amt/amti/sources.yaml @@ -14,7 +14,7 @@ metadata: unit: list period: year label: California altenative minimum taxable income adjustments - references: + reference: - title: 2021 Schedule P (540) Alternative Minimum Tax and Credit Limitations - Residents href: https://www.ftb.ca.gov/forms/2021/2021-540-p.pdf#page=1 - title: 2022 Schedule P (540) Alternative Minimum Tax and Credit Limitations - Residents diff --git a/policyengine_us/parameters/gov/states/de/tax/income/subtractions/subtractions.yaml b/policyengine_us/parameters/gov/states/de/tax/income/subtractions/subtractions.yaml index 179f77e6ab0..da1d01a94a0 100644 --- a/policyengine_us/parameters/gov/states/de/tax/income/subtractions/subtractions.yaml +++ b/policyengine_us/parameters/gov/states/de/tax/income/subtractions/subtractions.yaml @@ -22,7 +22,7 @@ metadata: unit: list period: year label: Delaware adjusted gross income subtractions - references: + reference: - title: Delaware Code, § 1106 Modifcations (b) href: https://delcode.delaware.gov/title30/c011/sc02/index.html - title: Delaware HB 285 (151st General Assembly), amending 30 Del. C. § 1106(b)(10) diff --git a/policyengine_us/parameters/gov/states/id/tax/income/subtractions/subtractions.yaml b/policyengine_us/parameters/gov/states/id/tax/income/subtractions/subtractions.yaml index 0c9d9e9b325..ee01efbc66d 100644 --- a/policyengine_us/parameters/gov/states/id/tax/income/subtractions/subtractions.yaml +++ b/policyengine_us/parameters/gov/states/id/tax/income/subtractions/subtractions.yaml @@ -29,7 +29,7 @@ metadata: unit: list period: year label: Idaho adjusted gross income subtractions - references: + reference: - title: 2022 Idaho CodeSection 63-3022D - deduction of expenses for household and dependent care services href: https://law.justia.com/codes/idaho/2022/title-63/chapter-30/section-63-3022d/ - title: Individual Income Tax Forms and Instructions 2021 diff --git a/policyengine_us/reforms/state_dependent_exemptions/repeal_state_dependent_exemptions.py b/policyengine_us/reforms/state_dependent_exemptions/repeal_state_dependent_exemptions.py index 67d9c0d6e8e..07bb78dd917 100644 --- a/policyengine_us/reforms/state_dependent_exemptions/repeal_state_dependent_exemptions.py +++ b/policyengine_us/reforms/state_dependent_exemptions/repeal_state_dependent_exemptions.py @@ -8,7 +8,7 @@ class hi_regular_exemptions(Variable): entity = TaxUnit label = "Hawaii regular exemptions" unit = USD - documentation = "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=20" + reference = "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=20" definition_period = YEAR defined_for = StateCode.HI diff --git a/policyengine_us/tests/code_health/test_no_concatenated_reference_urls.py b/policyengine_us/tests/code_health/test_no_concatenated_reference_urls.py index 33f50c9a9de..2c1fdc92f9c 100644 --- a/policyengine_us/tests/code_health/test_no_concatenated_reference_urls.py +++ b/policyengine_us/tests/code_health/test_no_concatenated_reference_urls.py @@ -19,10 +19,12 @@ leading or trailing spaces, or a note such as "(Line 3)" (put notes in a Python comment). -A scheme may follow whitespace, an opening bracket or quote, or "=" (a URL -passed as a query value, as in azleg.gov's viewdocument/?docName=https://...). -It may follow "/" only when that slash closes an archive.org wrapper such as -https://web.archive.org/web//https://... +A scheme may follow whitespace or an opening bracket or quote. It may follow +"=" only when that "=" ends a query parameter (a URL passed as a query value, +as in azleg.gov's viewdocument/?docName=https://... or ?a=1&url=https://...); +an "=" inside a fragment, as in x.pdf#page=https://..., marks two URLs glued +together. It may follow "/" only when that slash closes an archive.org wrapper +such as https://web.archive.org/web//https://... """ import ast @@ -35,12 +37,15 @@ PACKAGE = Path(__file__).resolve().parents[2] ATTRIBUTES = {"reference", "documentation"} SCHEME = re.compile(r"https?://", re.IGNORECASE) -ALLOWED_BEFORE_SCHEME = set("([{<\"'`=") +ALLOWED_BEFORE_SCHEME = set("([{<\"'`") BARE_URL = re.compile(r"https?://\S+", re.IGNORECASE) ARCHIVE_WRAPPER = re.compile( r"(?:^|[\s(\[{<\"'`=])(?:https?://)?(?:web\.)?archive\.org/web/[^\s/]+/$", re.IGNORECASE, ) +# "?name=" or "?a=1&name=" with no "#" since the "?": the end of a query +# parameter whose value is a URL. +QUERY_VALUE = re.compile(r"\?(?:[^#\s]*&)?[^?&#=\s]*=$") def glued_url_offsets(text: str) -> list[int]: @@ -53,7 +58,7 @@ def glued_url_offsets(text: str) -> list[int]: before = text[start - 1] if before.isspace() or before in ALLOWED_BEFORE_SCHEME: continue - if before == "/" and ARCHIVE_WRAPPER.search(text, 0, start): + if is_wrapped_url(text, start): continue offsets.append(start) return offsets @@ -86,9 +91,9 @@ def is_wrapped_url(text: str, start: int) -> bool: """True if the URL scheme at `start` sits inside another URL: as a query value (`...?docName=https://...`) or after an archive.org wrapper.""" before = text[start - 1] - return before == "=" or ( - before == "/" and bool(ARCHIVE_WRAPPER.search(text, 0, start)) - ) + if before == "=": + return bool(QUERY_VALUE.search(text, 0, start)) + return before == "/" and bool(ARCHIVE_WRAPPER.search(text, 0, start)) def is_bare_url_or_has_none(text: str) -> bool: @@ -135,6 +140,9 @@ def find_glued_urls(source: str, filename: str = "") -> list[str]: "https://example.gov/a)https://example.gov/b", "HTTPS://EXAMPLE.GOV/A.PDFHTTPS://EXAMPLE.GOV/B.PDF", "https://example.gov/web/2024/https://example.gov/b", + "https://example.gov/a.pdf#page=https://example.gov/b.pdf", + "https://example.gov/a.pdf#page=2&view=https://example.gov/b.pdf", + "https://example.gov/a?x=1#page=https://example.gov/b.pdf", ], ) def test_glued_urls_are_flagged(text): @@ -155,6 +163,7 @@ def test_glued_urls_are_flagged(text): "[IT-201-I](https://www.tax.ny.gov/it201i.pdf)", '"https://example.gov/a"', "https://www.cbo.gov/system/files/2026-02/51138-2026-02-Revenue.xlsx", + "https://example.gov/view?a=1&url=https://example.gov/b.pdf", ], ) def test_separated_and_wrapped_urls_are_not_flagged(text): @@ -176,6 +185,7 @@ def test_separated_and_wrapped_urls_are_not_flagged(text): "https://example.gov/a.pdf(https://example.gov/b.pdf)", "https://example.gov/a.pdf[https://example.gov/b.pdf]", "https://example.gov/web/2024/https://example.gov/b", + "https://example.gov/a.pdf#page=https://example.gov/b.pdf", ], ) def test_reference_entries_that_are_not_one_bare_url_are_flagged(text): @@ -189,6 +199,7 @@ def test_reference_entries_that_are_not_one_bare_url_are_flagged(text): "https://www.azleg.gov/viewdocument/?docName=https://www.azleg.gov/ars/43/01001.htm", "https://web.archive.org/web/20250720165524/https://www.mass.gov/doc/heap", "https://web.archive.org/web/2021id_/https://example.gov/a?url=https://b.gov", + "https://example.gov/view?a=1&url=https://example.gov/b.pdf", "26 U.S.C. 32(b)", "IRS Publication 596, page 3", "", diff --git a/policyengine_us/tests/code_health/test_parameter_yaml_keys.py b/policyengine_us/tests/code_health/test_parameter_yaml_keys.py new file mode 100644 index 00000000000..44bd79bebe4 --- /dev/null +++ b/policyengine_us/tests/code_health/test_parameter_yaml_keys.py @@ -0,0 +1,327 @@ +"""Check canonical citation metadata and sibling keys core can ignore. + +Core copies metadata on parameters, dated values, scales and nodes without +moving sibling ``reference`` or ``unit`` keys into it. Where an allowed-key +list is supplied, its unknown-key warning exempts those legacy siblings. +YAML-defined nodes have no allowed-key list: a ``unit`` key can become a child +or raise a parsing error. Nodes skip YAML children and files named +``reference``, but still load directories with that name. The 14 citations +beside dated values in gov/irs/income/exemption/amount.yaml were lost until +#9609 moved them under ``metadata``. + +Citation aliases such as ``metadata.references`` remain under their supplied +names instead of populating canonical ``metadata.reference``. Core does not +warn about unknown metadata keys, so the raw scan checks the two known citation +misspellings while leaving other metadata opaque. + +Core emits ``ParameterKeyWarning`` for unknown siblings only where it validates +an allowed-key list. ``policyengine_us.model_api`` ignores warnings, so the last +test reloads the parameter tree with that warning recorded. These tests do not +change core's legacy loader compatibility. +""" + +from pathlib import Path +import warnings + +import policyengine_core.warnings as core_warnings +import pytest +import yaml +from policyengine_core.parameters import Parameter, ParameterNode + +REPO = Path(__file__).resolve().parents[2] +PARAMETER_DIRS = (REPO / "parameters", REPO / "params_on_demand") +# Repository placement convention; core's legacy exemptions do not migrate keys. +METADATA_ONLY_KEYS = {"reference", "unit"} +MISSPELLED_METADATA_REFERENCE_KEYS = {"references", "refrence"} +Loader = getattr(yaml, "CSafeLoader", yaml.SafeLoader) + + +def keys_outside_metadata(text, keys=METADATA_ONLY_KEYS): + """Return (line, key path) for misplaced keys and known citation aliases. + + Metadata is opaque except for direct citation misspellings and YAML merges + supplying metadata entries. Canonical citation payloads and other metadata + values are not traversed. + """ + found = [] + + def walk(node, path, in_metadata=False): + if isinstance(node, yaml.MappingNode): + for key_node, value_node in node.value: + key = key_node.value + key_path = [*path, str(key)] + if in_metadata: + if key in MISSPELLED_METADATA_REFERENCE_KEYS: + found.append((key_node.start_mark.line + 1, ".".join(key_path))) + if key_node.tag == "tag:yaml.org,2002:merge": + walk(value_node, key_path, in_metadata=True) + continue + if key == "metadata": + if isinstance(value_node, yaml.MappingNode): + walk(value_node, key_path, in_metadata=True) + continue + if key in keys: + found.append((key_node.start_mark.line + 1, ".".join(key_path))) + walk(value_node, key_path) + elif isinstance(node, yaml.SequenceNode): + for index, item in enumerate(node.value): + walk(item, [*path, str(index)], in_metadata=in_metadata) + + walk(yaml.compose(text, Loader=Loader), []) + return found + + +@pytest.mark.parametrize( + "text, expected", + [ + ( + "description: Top level, beside values.\n" + "values:\n" + " 2020-01-01: 1\n" + "reference:\n" + " - title: Rev. Proc. 2019-44\n" + " href: https://www.irs.gov/pub/irs-drop/rp-19-44.pdf\n", + [(4, "reference")], + ), + ( + "values:\n" + " 2026-01-01:\n" + " value: 1\n" + " metadata:\n" + " reference: https://a.gov/2026.pdf\n" + " 2027-01-01:\n" + " value: 2\n" + " reference: https://a.gov/2027.pdf\n", + [(8, "values.2027-01-01.reference")], + ), + ( + "reference: https://a.gov/node.pdf\nchild:\n values:\n 2020-01-01: 1\n", + [(1, "reference")], + ), + ( + "brackets:\n" + " - threshold:\n" + " 2020-01-01: 0\n" + " rate:\n" + " 2020-01-01: 0.1\n" + " reference: https://a.gov/scale.pdf\n", + [(6, "brackets.0.reference")], + ), + ( + "brackets:\n" + " - threshold:\n" + " values:\n" + " 2020-01-01: 0\n" + " reference: https://a.gov/threshold.pdf\n", + [(5, "brackets.0.threshold.reference")], + ), + ( + "unit: currency-USD\nvalues:\n 2020-01-01: 1\n", + [(1, "unit")], + ), + ], +) +def test_keys_outside_metadata_are_found(text, expected): + assert keys_outside_metadata(text) == expected + + +@pytest.mark.parametrize("misspelling", sorted(MISSPELLED_METADATA_REFERENCE_KEYS)) +@pytest.mark.parametrize( + "text, line, key_path", + [ + ( + "values:\n" + " 2020-01-01: 1\n" + "metadata:\n" + " {misspelling}: https://a.gov/parameter.pdf\n", + 4, + "metadata.{misspelling}", + ), + ( + "values:\n" + " 2020-01-01:\n" + " value: 1\n" + " metadata:\n" + " {misspelling}: https://a.gov/value.pdf\n", + 5, + "values.2020-01-01.metadata.{misspelling}", + ), + ( + "brackets:\n" + " - threshold:\n" + " 2020-01-01: 0\n" + " metadata:\n" + " {misspelling}: https://a.gov/bracket.pdf\n", + 5, + "brackets.0.metadata.{misspelling}", + ), + ( + "metadata:\n <<:\n {misspelling}: https://a.gov/merged.pdf\n", + 3, + "metadata.<<.{misspelling}", + ), + ( + "metadata:\n" + " defaults: &citations\n" + " {misspelling}: https://a.gov/alias.pdf\n" + " <<: *citations\n", + 3, + "metadata.<<.{misspelling}", + ), + ( + "metadata:\n" + " defaults: &citations\n" + " {misspelling}: https://a.gov/alias-list.pdf\n" + " other: &other\n" + " label: Example\n" + " <<: [*other, *citations]\n", + 3, + "metadata.<<.1.{misspelling}", + ), + ], +) +def test_metadata_citation_misspellings_are_found(misspelling, text, line, key_path): + assert keys_outside_metadata(text.format(misspelling=misspelling)) == [ + (line, key_path.format(misspelling=misspelling)) + ] + + +@pytest.mark.parametrize( + "text", + [ + "description: Reference under the parameter's metadata.\n" + "values:\n" + " 2020-01-01: 1\n" + "metadata:\n" + " unit: currency-USD\n" + " reference:\n" + " - title: Rev. Proc. 2019-44\n" + " href: https://www.irs.gov/pub/irs-drop/rp-19-44.pdf\n", + "values:\n" + " 2027-01-01:\n" + " value: 2\n" + " metadata:\n" + " reference: https://a.gov/2027.pdf\n", + "metadata:\n" + " reference: https://a.gov/node.pdf\n" + "reference_amount:\n" + " values:\n" + " 2020-01-01: 1\n", + "brackets:\n" + " - threshold:\n" + " 2020-01-01: 0\n" + " metadata:\n" + " reference: https://a.gov/bracket.pdf\n", + "2020-01-01: 1\n", + "metadata:\n" + " custom:\n" + " references: Arbitrary nested metadata\n" + " refrence: Also opaque\n" + " metadata:\n" + " references: Still nested inside an arbitrary field\n", + "metadata:\n" + ' "<<":\n' + " references: A quoted literal key is not a YAML merge\n" + " refrence: Also opaque\n", + "metadata:\n" + " reference:\n" + " - title: Citation\n" + " href: https://a.gov/citation.pdf\n" + " references: Citation payloads are opaque\n", + "metadata:\n" + " defaults: &defaults\n" + " unit: currency-USD\n" + " reference: https://a.gov/merged.pdf\n" + " <<: *defaults\n", + "metadata:\n" + " defaults: &defaults\n" + " unit: currency-USD\n" + " citations: &citations\n" + " reference: https://a.gov/merged-list.pdf\n" + " <<: [*defaults, *citations]\n", + ], +) +def test_keys_inside_metadata_are_accepted(text): + assert keys_outside_metadata(text) == [] + + +@pytest.mark.parametrize("misspelling", sorted(MISSPELLED_METADATA_REFERENCE_KEYS)) +def test_correcting_metadata_citation_alias_preserves_value(misspelling): + text = ( + "values:\n" + " '2020-01-01':\n" + " value: 1\n" + "metadata:\n" + f" {misspelling}:\n" + " - title: Example law\n" + " href: https://a.gov/law\n" + ) + corrected_text = text.replace(f" {misspelling}:", " reference:") + original = Parameter("example", data=yaml.load(text, Loader=Loader)) + corrected = Parameter("example", data=yaml.load(corrected_text, Loader=Loader)) + citations = [{"title": "Example law", "href": "https://a.gov/law"}] + + assert original.values_list[0].value == corrected.values_list[0].value == 1 + assert "reference" not in original.metadata + assert original.metadata[misspelling] == citations + assert corrected.metadata["reference"] == citations + assert misspelling not in corrected.metadata + assert keys_outside_metadata(text) == [(5, f"metadata.{misspelling}")] + assert keys_outside_metadata(corrected_text) == [] + + +def parameter_files(directories=PARAMETER_DIRS): + for directory in directories: + for suffix in (".yaml", ".yml"): + yield from sorted( + path for path in directory.rglob(f"*{suffix}") if path.is_file() + ) + + +def test_parameter_files_include_both_yaml_extensions(tmp_path): + nested = tmp_path / "nested" + nested.mkdir() + expected = {tmp_path / "parameter.yaml", nested / "parameter.yml"} + for path in expected: + path.write_text("values:\n 2020-01-01: 1\n") + (tmp_path / "README.md").write_text("Not a parameter file.\n") + + assert set(parameter_files((tmp_path,))) == expected + + +def test_parameter_files_are_found(): + files = list(parameter_files()) + assert len(files) > 1_000 + assert REPO / "parameters/gov/irs/income/exemption/amount.yaml" in files + + +def test_no_reference_or_unit_outside_metadata(): + errors = [ + f"{path.relative_to(REPO.parent)}:{line}: {key_path}" + for path in parameter_files() + for line, key_path in keys_outside_metadata(path.read_text()) + ] + assert not errors, ( + "Parameter citation/unit keys must use canonical metadata placement. " + "Move sibling reference/unit keys under metadata, and rename direct " + "metadata.references or metadata.refrence keys to metadata.reference:\n" + + "\n".join(errors) + ) + + +def test_parameter_files_have_no_keys_core_ignores(): + parameter_key_warning = getattr(core_warnings, "ParameterKeyWarning", None) + if parameter_key_warning is None: + pytest.skip("policyengine-core < 3.32 does not flag unknown keys") + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", parameter_key_warning) + for directory in PARAMETER_DIRS: + ParameterNode("", directory_path=str(directory)) + messages = sorted( + { + str(warning.message) + for warning in caught + if issubclass(warning.category, parameter_key_warning) + } + ) + assert not messages, "\n".join(messages) diff --git a/policyengine_us/tests/code_health/test_reference_urls.py b/policyengine_us/tests/code_health/test_reference_urls.py index 79bd611fd08..70cd5c89eb8 100644 --- a/policyengine_us/tests/code_health/test_reference_urls.py +++ b/policyengine_us/tests/code_health/test_reference_urls.py @@ -3,6 +3,10 @@ Python joins adjacent string literals, so a ``reference`` tuple written without commas between its URLs collapses into one string. policyengine-core wraps that string as a one-element list, and it renders as a single dead link. + +A variable's ``documentation`` is prose. Any URL it cites must also be listed +in ``reference``, and documentation that is nothing but URLs is a citation +filed in the wrong attribute. """ import ast @@ -14,9 +18,24 @@ URL = re.compile(r"https?://") # A URL may embed another URL as a Wayback Machine snapshot target -# (web.archive.org/web//https://...) or as a query value -# (viewdocument/?docName=https://...). -EMBEDDED_URL_PREFIX = re.compile(r"(?:web\.archive\.org/web/\d+[a-z_]*/|=)$") +# (web.archive.org/web//https://...) or as the value of a query +# parameter (viewdocument/?docName=https://... or ?a=1&url=https://...). The +# parameter must sit in the query, before any "#": an "=" inside a fragment, +# as in x.pdf#page=https://..., marks two URLs fused together. +EMBEDDED_URL_PREFIX = re.compile( + r"(?:web\.archive\.org/web/\d+[a-z_]*/|\?(?:[^#\s]*&)?[^?&#=\s]*=)$" +) +# Punctuation that ends a sentence rather than a URL cited in prose. +TRAILING_PUNCTUATION = ".,;:!?'\"" + + +def top_level_url_starts(text): + """Offsets of the URLs in ``text`` that are not embedded in another URL.""" + return [ + match.start() + for match in URL.finditer(text) + if not EMBEDDED_URL_PREFIX.search(text[: match.start()]) + ] def reference_url_problem(href): @@ -28,11 +47,9 @@ def reference_url_problem(href): """ if not isinstance(href, str): return None - starts = [match.start() for match in URL.finditer(href)] - if not starts: + if not URL.search(href): return None - top_level = [i for i in starts if not EMBEDDED_URL_PREFIX.search(href[:i])] - if len(top_level) > 1: + if len(top_level_url_starts(href)) > 1: return "contains more than one URL (missing comma between literals?)" if not re.fullmatch(r"https?://\S+", href) or href.endswith((",", ";")): return "is not a single bare URL" @@ -49,6 +66,45 @@ def _entries(reference): ] +def _cited_urls(token): + """The URLs in a whitespace-free ``token`` of prose, each cut at the next + URL fused onto it and stripped of the punctuation or closing bracket that + ends the sentence.""" + starts = top_level_url_starts(token) + ends = [*starts[1:], len(token)] + return [_strip_prose(token[start:end]) for start, end in zip(starts, ends)] + + +def _strip_prose(url): + url = url.rstrip(TRAILING_PUNCTUATION) + while url.endswith(")") and url.count(")") > url.count("("): + url = url[:-1].rstrip(TRAILING_PUNCTUATION) + return url + + +def documentation_url_problems(documentation, reference): + """Describe what is wrong with the URLs in a variable's documentation.""" + if not isinstance(documentation, str) or not URL.search(documentation): + return [] + problems = [] + tokens = documentation.split() + if all(URL.match(token) for token in tokens): + problems.append("is only URLs: move them into reference") + cited = [] + for token in tokens: + urls = _cited_urls(token) + if len(urls) > 1: + problems.append(f"{token!r} contains more than one URL") + cited.extend(urls) + listed = set(_entries(reference)) + problems.extend( + f"cites {url!r}, which reference does not list" + for url in dict.fromkeys(cited) + if url not in listed + ) + return problems + + @pytest.mark.parametrize( "href", [ @@ -57,6 +113,8 @@ def _entries(reference): "placeholder", "http://web.archive.org/web/20091202182220/http://www.mass.gov/g.pdf", "https://www.azleg.gov/viewdocument/?docName=https://www.azleg.gov/ars/43/01072.htm", + "https://a.gov/view?a=1&url=https://b.gov/2.pdf", + "https://web.archive.org/web/2021id_/https://a.gov/view?url=https://b.gov", ], ) def test_reference_url_problem_accepts(href): @@ -75,12 +133,86 @@ def test_reference_url_problem_accepts(href): "https://a.gov/1.pdf, ", "https://a.gov/1.pdf, # Line 7", "https://a.gov/1.pdf (Line 3)", + # A fragment truncated before two literals were joined: "=" ends a + # fragment here, not a query parameter. + "https://a.gov/1.pdf#page=https://b.gov/2.pdf", + "https://a.gov/1.pdf#page=2&view=https://b.gov/2.pdf", + "https://a.gov/view?a=1#page=https://b.gov/2.pdf", ], ) def test_reference_url_problem_rejects(href): assert reference_url_problem(href) is not None +@pytest.mark.parametrize( + "documentation, reference", + [ + (None, None), + ("Prose without a link.", None), + ( + "See the list at https://a.gov/1202#e_3_A.", + "https://a.gov/1202#e_3_A", + ), + ( + "Explained in the manual (https://a.gov/manual.pdf), section 2.", + [dict(title="Manual", href="https://a.gov/manual.pdf")], + ), + ( + "Wikipedia: https://en.wikipedia.org/wiki/Tax_(disambiguation).", + ("https://en.wikipedia.org/wiki/Tax_(disambiguation)",), + ), + ], +) +def test_documentation_url_problems_accepts(documentation, reference): + assert documentation_url_problems(documentation, reference) == [] + + +@pytest.mark.parametrize( + "documentation, reference, expected", + [ + # az_taxable_income before this check: one URL written twice and no + # reference. The URL is reported once. + ( + "https://a.gov/140.pdf#page=8\nhttps://a.gov/140.pdf#page=8", + None, + [ + "is only URLs: move them into reference", + "cites 'https://a.gov/140.pdf#page=8', which reference does not list", + ], + ), + # Listing the URL in reference too does not make the documentation + # prose. + ( + "https://a.gov/140.pdf#page=8\nhttps://a.gov/140.pdf#page=8", + "https://a.gov/140.pdf#page=8", + ["is only URLs: move them into reference"], + ), + ( + "https://a.gov/statute.htm", + "A.R.S. 43-1022", + [ + "is only URLs: move them into reference", + "cites 'https://a.gov/statute.htm', which reference does not list", + ], + ), + ( + "See https://a.gov/1.pdf.", + None, + ["cites 'https://a.gov/1.pdf', which reference does not list"], + ), + ( + "See https://a.gov/1.pdf#page=2https://b.gov/2.pdf", + ("https://a.gov/1.pdf#page=2", "https://b.gov/2.pdf"), + [ + "'https://a.gov/1.pdf#page=2https://b.gov/2.pdf' contains more than one URL" + ], + ), + ], +) +def test_documentation_url_problems_rejects(documentation, reference, expected): + assert documentation_url_problems(documentation, reference) == expected + + def test_variable_references_are_single_urls(): from policyengine_us.system import system @@ -128,9 +260,42 @@ def _evaluate_reference(node, constants): return ast.literal_eval(node) +def test_variable_documentation_urls_are_references(): + from policyengine_us.system import system + + errors = [ + f"{name}: documentation {problem}" + for name, variable in sorted(system.variables.items()) + for problem in documentation_url_problems( + variable.documentation, variable.reference + ) + ] + assert not errors, "\n".join(errors) + + +def _class_attributes(node, constants, names, location, errors): + """Evaluate the class attributes in ``names`` that ``node`` assigns.""" + attributes = {} + for statement in node.body: + if not isinstance(statement, ast.Assign): + continue + for target in statement.targets: + if not (isinstance(target, ast.Name) and target.id in names): + continue + try: + attributes[target.id] = _evaluate_reference(statement.value, constants) + except (KeyError, ValueError, TypeError, SyntaxError): + errors.append( + f"{location}: {target.id} is not built from literals, so " + "this test cannot check it" + ) + return attributes + + def test_reform_variable_references_are_single_urls(): # Reform variables are defined inside reform factories, so they are not in - # the baseline system; check their reference assignments in source instead. + # the baseline system; check their reference and documentation + # assignments in source instead. errors = [] for path in sorted((REPO / "reforms").rglob("*.py")): tree = ast.parse(path.read_text()) @@ -138,27 +303,17 @@ def test_reform_variable_references_are_single_urls(): for node in ast.walk(tree): if not isinstance(node, ast.ClassDef): continue - for statement in node.body: - if not ( - isinstance(statement, ast.Assign) - and any( - isinstance(target, ast.Name) and target.id == "reference" - for target in statement.targets - ) - ): - continue - location = f"{path.relative_to(REPO)}::{node.name}" - try: - reference = _evaluate_reference(statement.value, constants) - except (KeyError, ValueError, TypeError, SyntaxError): - errors.append( - f"{location}: reference is not built from literals, so " - "this test cannot check it" - ) - continue - for href in _entries(reference): - if problem := reference_url_problem(href): - errors.append(f"{location}: {href!r} {problem}") + location = f"{path.relative_to(REPO)}::{node.name}" + attributes = _class_attributes( + node, constants, {"reference", "documentation"}, location, errors + ) + for href in _entries(attributes.get("reference")): + if problem := reference_url_problem(href): + errors.append(f"{location}: {href!r} {problem}") + for problem in documentation_url_problems( + attributes.get("documentation"), attributes.get("reference") + ): + errors.append(f"{location}: documentation {problem}") assert not errors, "\n".join(errors) diff --git a/policyengine_us/variables/gov/aca/eligibility/aca_magi_fraction.py b/policyengine_us/variables/gov/aca/eligibility/aca_magi_fraction.py index 863fa84d810..9464de7bdd2 100644 --- a/policyengine_us/variables/gov/aca/eligibility/aca_magi_fraction.py +++ b/policyengine_us/variables/gov/aca/eligibility/aca_magi_fraction.py @@ -6,13 +6,19 @@ class aca_magi_fraction(Variable): entity = TaxUnit label = "ACA-related modified AGI as fraction of prior-year FPL" documentation = ( - "ACA-related MAGI as fraction of federal poverty line.\n" - "Documentation on use of prior-year FPL in the following reference:\n" - " title: 2022 IRS Form 8962 (ACA PTC) instructions, Line 4\n" - " href: https://www.irs.gov/pub/irs-pdf/i8962.pdf#page=7\n" - "Documentation on truncation of fraction in the following reference:\n" - " title: 2022 IRS Form 8962 instructions, Line 5 Worksheet 2\n" - " href: https://www.irs.gov/pub/irs-pdf/i8962.pdf#page=8" + "ACA-related MAGI as fraction of federal poverty line. The first " + "reference documents the use of the prior-year FPL; the second " + "documents the truncation of the fraction." + ) + reference = ( + dict( + title="2022 IRS Form 8962 (ACA PTC) instructions, Line 4", + href="https://www.irs.gov/pub/irs-pdf/i8962.pdf#page=7", + ), + dict( + title="2022 IRS Form 8962 instructions, Line 5 Worksheet 2", + href="https://www.irs.gov/pub/irs-pdf/i8962.pdf#page=8", + ), ) definition_period = YEAR diff --git a/policyengine_us/variables/gov/hhs/medicare/eligibility/is_medicare_eligible.py b/policyengine_us/variables/gov/hhs/medicare/eligibility/is_medicare_eligible.py index fd5a6ce2fbf..0772dde903c 100644 --- a/policyengine_us/variables/gov/hhs/medicare/eligibility/is_medicare_eligible.py +++ b/policyengine_us/variables/gov/hhs/medicare/eligibility/is_medicare_eligible.py @@ -16,6 +16,10 @@ class is_medicare_eligible(Variable): "(42 U.S.C. 426-1) are not, as the model has no inputs for those " "conditions." ) + reference = dict( + title="CMS, Original Medicare (Part A and B) Eligibility and Enrollment", + href="https://www.cms.gov/medicare/enrollment-renewal/health-plans/original-part-a-b", + ) def formula(person, period, parameters): p = parameters(period).gov.hhs.medicare.eligibility diff --git a/policyengine_us/variables/gov/states/al/tax/income/deductions/al_standard_deduction.py b/policyengine_us/variables/gov/states/al/tax/income/deductions/al_standard_deduction.py index 5797597958a..1af7cbe52bb 100644 --- a/policyengine_us/variables/gov/states/al/tax/income/deductions/al_standard_deduction.py +++ b/policyengine_us/variables/gov/states/al/tax/income/deductions/al_standard_deduction.py @@ -7,7 +7,7 @@ class al_standard_deduction(Variable): label = "Alabama standard deduction" unit = USD # The Code of Alabama 1975 Section 40-18-15 (b)(4). - documentation = "https://alison.legislature.state.al.us/code-of-alabama" + reference = "https://alison.legislature.state.al.us/code-of-alabama" definition_period = YEAR defined_for = StateCode.AL diff --git a/policyengine_us/variables/gov/states/al/tax/income/exemptions/al_dependent_exemption.py b/policyengine_us/variables/gov/states/al/tax/income/exemptions/al_dependent_exemption.py index de4ef25e57d..165931c3946 100644 --- a/policyengine_us/variables/gov/states/al/tax/income/exemptions/al_dependent_exemption.py +++ b/policyengine_us/variables/gov/states/al/tax/income/exemptions/al_dependent_exemption.py @@ -7,7 +7,7 @@ class al_dependent_exemption(Variable): label = "Alabama dependent exemption" unit = USD # The Code of Alabama 1975 Section 40-18-19 (a)(9). - documentation = "https://alison.legislature.state.al.us/code-of-alabama" + reference = "https://alison.legislature.state.al.us/code-of-alabama" definition_period = YEAR defined_for = StateCode.AL diff --git a/policyengine_us/variables/gov/states/al/tax/income/exemptions/al_personal_exemption.py b/policyengine_us/variables/gov/states/al/tax/income/exemptions/al_personal_exemption.py index 218565ec5b2..39122e79af8 100644 --- a/policyengine_us/variables/gov/states/al/tax/income/exemptions/al_personal_exemption.py +++ b/policyengine_us/variables/gov/states/al/tax/income/exemptions/al_personal_exemption.py @@ -7,7 +7,7 @@ class al_personal_exemption(Variable): label = "Alabama personal exemption" unit = USD # The Code of Alabama 1975 Section 40-18-19 (a)(8). - documentation = "https://alison.legislature.state.al.us/code-of-alabama" + reference = "https://alison.legislature.state.al.us/code-of-alabama" definition_period = YEAR defined_for = StateCode.AL diff --git a/policyengine_us/variables/gov/states/ar/tax/income/credits/ar_cdcc.py b/policyengine_us/variables/gov/states/ar/tax/income/credits/ar_cdcc.py index afae402d1fb..c82c5cd6e92 100644 --- a/policyengine_us/variables/gov/states/ar/tax/income/credits/ar_cdcc.py +++ b/policyengine_us/variables/gov/states/ar/tax/income/credits/ar_cdcc.py @@ -9,9 +9,7 @@ class ar_cdcc(Variable): entity = TaxUnit label = "Arkansas Child and Dependent Care Credit" unit = USD - documentation = ( - "https://codes.findlaw.com/ar/title-26-taxation/ar-code-sect-26-51-502/" - ) + reference = "https://codes.findlaw.com/ar/title-26-taxation/ar-code-sect-26-51-502/" definition_period = YEAR defined_for = StateCode.AR diff --git a/policyengine_us/variables/gov/states/az/tax/income/az_agi.py b/policyengine_us/variables/gov/states/az/tax/income/az_agi.py index d2eaf43c057..b19209cbfe4 100644 --- a/policyengine_us/variables/gov/states/az/tax/income/az_agi.py +++ b/policyengine_us/variables/gov/states/az/tax/income/az_agi.py @@ -7,7 +7,7 @@ class az_agi(Variable): label = "Arizona adjusted gross income" unit = USD definition_period = YEAR - documentation = "https://azdor.gov/sites/default/files/2023-03/FORMS_INDIVIDUAL_2020_140NRBOOKLET.pdf#page=18" # Line 52 + reference = "https://azdor.gov/sites/default/files/2023-03/FORMS_INDIVIDUAL_2020_140NRBOOKLET.pdf#page=18" # Line 52 defined_for = StateCode.AZ def formula(tax_unit, period, parameters): diff --git a/policyengine_us/variables/gov/states/az/tax/income/az_filing_status.py b/policyengine_us/variables/gov/states/az/tax/income/az_filing_status.py index 15939508d84..adf26010700 100644 --- a/policyengine_us/variables/gov/states/az/tax/income/az_filing_status.py +++ b/policyengine_us/variables/gov/states/az/tax/income/az_filing_status.py @@ -14,9 +14,9 @@ class az_filing_status(Variable): possible_values = ArizonaFilingStatus default_value = ArizonaFilingStatus.SINGLE definition_period = YEAR - documentation = ( - "https://www.azleg.gov/viewdocument/?docName=https://www.azleg.gov/ars/43/01001.htm\n" # Section 43-1001(5) - "https://azdor.gov/forms/individual/form-140a-arizona-resident-personal-income-tax-booklet" # Page 5 (page 9 of pdf), Box 5. + reference = ( + "https://www.azleg.gov/viewdocument/?docName=https://www.azleg.gov/ars/43/01001.htm", # Section 43-1001(5) + "https://azdor.gov/forms/individual/form-140a-arizona-resident-personal-income-tax-booklet", # Page 5 (page 9 of pdf), Box 5. ) label = "Arizona filing status" diff --git a/policyengine_us/variables/gov/states/az/tax/income/az_taxable_income.py b/policyengine_us/variables/gov/states/az/tax/income/az_taxable_income.py index 0d694ce24a2..b6636d99bf7 100644 --- a/policyengine_us/variables/gov/states/az/tax/income/az_taxable_income.py +++ b/policyengine_us/variables/gov/states/az/tax/income/az_taxable_income.py @@ -6,7 +6,7 @@ class az_taxable_income(Variable): entity = TaxUnit label = "Arizona taxable income" unit = USD - documentation = "https://azdor.gov/sites/default/files/2023-03/FORMS_INDIVIDUAL_2022_140Ai.pdf#page=8\nhttps://azdor.gov/sites/default/files/2023-03/FORMS_INDIVIDUAL_2022_140Ai.pdf#page=8" + reference = "https://azdor.gov/sites/default/files/2023-03/FORMS_INDIVIDUAL_2022_140Ai.pdf#page=8" definition_period = YEAR defined_for = StateCode.AZ diff --git a/policyengine_us/variables/gov/states/az/tax/income/credits/dependent_credit/az_dependent_tax_credit.py b/policyengine_us/variables/gov/states/az/tax/income/credits/dependent_credit/az_dependent_tax_credit.py index e7ce13ffe14..1aed9ddc978 100644 --- a/policyengine_us/variables/gov/states/az/tax/income/credits/dependent_credit/az_dependent_tax_credit.py +++ b/policyengine_us/variables/gov/states/az/tax/income/credits/dependent_credit/az_dependent_tax_credit.py @@ -9,7 +9,7 @@ class az_dependent_tax_credit(Variable): entity = TaxUnit label = "Arizona dependent tax credit" unit = USD - documentation = "https://www.azleg.gov/viewdocument/?docName=https://www.azleg.gov/ars/43/01073-01.htm" + reference = "https://www.azleg.gov/viewdocument/?docName=https://www.azleg.gov/ars/43/01073-01.htm" definition_period = YEAR defined_for = StateCode.AZ diff --git a/policyengine_us/variables/gov/states/az/tax/income/credits/dependent_credit/az_dependent_tax_credit_potential.py b/policyengine_us/variables/gov/states/az/tax/income/credits/dependent_credit/az_dependent_tax_credit_potential.py index 4893697c0c1..af778feb0af 100644 --- a/policyengine_us/variables/gov/states/az/tax/income/credits/dependent_credit/az_dependent_tax_credit_potential.py +++ b/policyengine_us/variables/gov/states/az/tax/income/credits/dependent_credit/az_dependent_tax_credit_potential.py @@ -6,7 +6,7 @@ class az_dependent_tax_credit_potential(Variable): entity = TaxUnit label = "Arizona dependent tax credit" unit = USD - documentation = "https://www.azleg.gov/viewdocument/?docName=https://www.azleg.gov/ars/43/01073-01.htm" + reference = "https://www.azleg.gov/viewdocument/?docName=https://www.azleg.gov/ars/43/01073-01.htm" definition_period = YEAR defined_for = StateCode.AZ diff --git a/policyengine_us/variables/gov/states/az/tax/income/deductions/standard/az_base_standard_deduction.py b/policyengine_us/variables/gov/states/az/tax/income/deductions/standard/az_base_standard_deduction.py index 0a741401e38..9458056bba5 100644 --- a/policyengine_us/variables/gov/states/az/tax/income/deductions/standard/az_base_standard_deduction.py +++ b/policyengine_us/variables/gov/states/az/tax/income/deductions/standard/az_base_standard_deduction.py @@ -6,7 +6,7 @@ class az_base_standard_deduction(Variable): entity = TaxUnit label = "Arizona base standard deduction" unit = USD - documentation = "https://www.azleg.gov/viewdocument/?docName=https://www.azleg.gov/ars/43/01041.htm" + reference = "https://www.azleg.gov/viewdocument/?docName=https://www.azleg.gov/ars/43/01041.htm" definition_period = YEAR defined_for = StateCode.AZ diff --git a/policyengine_us/variables/gov/states/az/tax/income/deductions/standard/az_increased_standard_deduction_for_charitable_contributions.py b/policyengine_us/variables/gov/states/az/tax/income/deductions/standard/az_increased_standard_deduction_for_charitable_contributions.py index 774c702b52c..2855d2b092e 100644 --- a/policyengine_us/variables/gov/states/az/tax/income/deductions/standard/az_increased_standard_deduction_for_charitable_contributions.py +++ b/policyengine_us/variables/gov/states/az/tax/income/deductions/standard/az_increased_standard_deduction_for_charitable_contributions.py @@ -6,7 +6,7 @@ class az_increased_standard_deduction_for_charitable_contributions(Variable): entity = TaxUnit label = "Arizona increased standard deduction for charitable contributions" unit = USD - documentation = "https://www.azleg.gov/viewdocument/?docName=https://www.azleg.gov/ars/43/01041.htm" + reference = "https://www.azleg.gov/viewdocument/?docName=https://www.azleg.gov/ars/43/01041.htm" definition_period = YEAR defined_for = StateCode.AZ diff --git a/policyengine_us/variables/gov/states/az/tax/income/deductions/standard/az_standard_deduction.py b/policyengine_us/variables/gov/states/az/tax/income/deductions/standard/az_standard_deduction.py index dbc75370a0e..05b30f43fb8 100644 --- a/policyengine_us/variables/gov/states/az/tax/income/deductions/standard/az_standard_deduction.py +++ b/policyengine_us/variables/gov/states/az/tax/income/deductions/standard/az_standard_deduction.py @@ -6,7 +6,7 @@ class az_standard_deduction(Variable): entity = TaxUnit label = "Arizona standard deduction" unit = USD - documentation = "https://www.azleg.gov/viewdocument/?docName=https://www.azleg.gov/ars/43/01041.htm" + reference = "https://www.azleg.gov/viewdocument/?docName=https://www.azleg.gov/ars/43/01041.htm" definition_period = YEAR defined_for = StateCode.AZ diff --git a/policyengine_us/variables/gov/states/az/tax/income/subtractions/az_529a_able_account_subtraction.py b/policyengine_us/variables/gov/states/az/tax/income/subtractions/az_529a_able_account_subtraction.py index d7801c97a13..36801bd5360 100644 --- a/policyengine_us/variables/gov/states/az/tax/income/subtractions/az_529a_able_account_subtraction.py +++ b/policyengine_us/variables/gov/states/az/tax/income/subtractions/az_529a_able_account_subtraction.py @@ -6,8 +6,10 @@ class az_529a_able_account_subtraction(Variable): entity = TaxUnit label = "Arizona 529A ABLE account subtraction" unit = USD - documentation = "https://www.azleg.gov/ars/43/01022.htm" - reference = "A.R.S. 43-1022 - Subtractions from Arizona Gross Income" + reference = dict( + title="A.R.S. 43-1022 - Subtractions from Arizona Gross Income", + href="https://www.azleg.gov/ars/43/01022.htm", + ) definition_period = YEAR defined_for = StateCode.AZ diff --git a/policyengine_us/variables/gov/states/az/tax/income/subtractions/az_adoption_expense_subtraction.py b/policyengine_us/variables/gov/states/az/tax/income/subtractions/az_adoption_expense_subtraction.py index 627d5c59899..f76463ddeac 100644 --- a/policyengine_us/variables/gov/states/az/tax/income/subtractions/az_adoption_expense_subtraction.py +++ b/policyengine_us/variables/gov/states/az/tax/income/subtractions/az_adoption_expense_subtraction.py @@ -6,8 +6,10 @@ class az_adoption_expense_subtraction(Variable): entity = TaxUnit label = "Arizona adoption expense subtraction" unit = USD - documentation = "https://www.azleg.gov/ars/43/01022.htm" - reference = "A.R.S. 43-1022 - Subtractions from Arizona Gross Income" + reference = dict( + title="A.R.S. 43-1022 - Subtractions from Arizona Gross Income", + href="https://www.azleg.gov/ars/43/01022.htm", + ) definition_period = YEAR defined_for = StateCode.AZ diff --git a/policyengine_us/variables/gov/states/az/tax/income/subtractions/college_savings/az_529_college_savings_plan_subtraction.py b/policyengine_us/variables/gov/states/az/tax/income/subtractions/college_savings/az_529_college_savings_plan_subtraction.py index 1172a362836..e1c42971da3 100644 --- a/policyengine_us/variables/gov/states/az/tax/income/subtractions/college_savings/az_529_college_savings_plan_subtraction.py +++ b/policyengine_us/variables/gov/states/az/tax/income/subtractions/college_savings/az_529_college_savings_plan_subtraction.py @@ -6,7 +6,7 @@ class az_529_college_savings_plan_subtraction(Variable): entity = TaxUnit label = "Arizona 529 college savings plan subtraction" unit = USD - documentation = "https://azdor.gov/sites/default/files/2023-03/FORMS_INDIVIDUAL_2022_140i.pdf#page=15" + reference = "https://azdor.gov/sites/default/files/2023-03/FORMS_INDIVIDUAL_2022_140i.pdf#page=15" definition_period = YEAR defined_for = StateCode.AZ diff --git a/policyengine_us/variables/gov/states/ca/tax/income/credits/ca_cdcc.py b/policyengine_us/variables/gov/states/ca/tax/income/credits/ca_cdcc.py index d5197011ad7..a845d62dd6a 100644 --- a/policyengine_us/variables/gov/states/ca/tax/income/credits/ca_cdcc.py +++ b/policyengine_us/variables/gov/states/ca/tax/income/credits/ca_cdcc.py @@ -6,7 +6,7 @@ class ca_cdcc(Variable): entity = TaxUnit label = "California Child and Dependent Care Expenses Credit" unit = USD - documentation = "https://www.ftb.ca.gov/forms/2020/2020-3506-instructions.html" + reference = "https://www.ftb.ca.gov/forms/2020/2020-3506-instructions.html" definition_period = YEAR defined_for = StateCode.CA diff --git a/policyengine_us/variables/gov/states/ca/tax/income/credits/ca_federal_cdcc.py b/policyengine_us/variables/gov/states/ca/tax/income/credits/ca_federal_cdcc.py index 418559d0fa8..bc339f5c3a2 100644 --- a/policyengine_us/variables/gov/states/ca/tax/income/credits/ca_federal_cdcc.py +++ b/policyengine_us/variables/gov/states/ca/tax/income/credits/ca_federal_cdcc.py @@ -6,7 +6,7 @@ class ca_federal_cdcc(Variable): entity = TaxUnit label = "Child and Dependent Care Expenses Credit replicated to include California limitations" unit = USD - documentation = "https://www.ftb.ca.gov/about-ftb/data-reports-plans/Summary-of-Federal-Income-Tax-Changes/index.html#PL-117-2-9631" + reference = "https://www.ftb.ca.gov/about-ftb/data-reports-plans/Summary-of-Federal-Income-Tax-Changes/index.html#PL-117-2-9631" definition_period = YEAR defined_for = StateCode.CA diff --git a/policyengine_us/variables/gov/states/ca/tax/income/deductions/standard/ca_standard_deduction.py b/policyengine_us/variables/gov/states/ca/tax/income/deductions/standard/ca_standard_deduction.py index 723a61b199e..27c3ceea1a1 100644 --- a/policyengine_us/variables/gov/states/ca/tax/income/deductions/standard/ca_standard_deduction.py +++ b/policyengine_us/variables/gov/states/ca/tax/income/deductions/standard/ca_standard_deduction.py @@ -6,7 +6,7 @@ class ca_standard_deduction(Variable): entity = TaxUnit label = "California standard deduction" unit = USD - documentation = "https://www.ftb.ca.gov/forms/2021/2021-540.pdf" + reference = "https://www.ftb.ca.gov/forms/2021/2021-540.pdf" definition_period = YEAR defined_for = StateCode.CA diff --git a/policyengine_us/variables/gov/states/co/tax/income/credits/cdcc/co_cdcc.py b/policyengine_us/variables/gov/states/co/tax/income/credits/cdcc/co_cdcc.py index 158651c4217..15e9eb63a2d 100644 --- a/policyengine_us/variables/gov/states/co/tax/income/credits/cdcc/co_cdcc.py +++ b/policyengine_us/variables/gov/states/co/tax/income/credits/cdcc/co_cdcc.py @@ -6,15 +6,13 @@ class co_cdcc(Variable): entity = TaxUnit label = "Colorado Child Care Expenses Credit" unit = USD - documentation = ( - "https://advance.lexis.com/documentpage/?pdmfid=1000516&crid=d14880b7-7410-4295-bcf1-2e099e57d8f3&pdistocdocslideraccess=true&config=014FJAAyNGJkY2Y4Zi1mNjgyLTRkN2YtYmE4OS03NTYzNzYzOTg0OGEKAFBvZENhdGFsb2d592qv2Kywlf8caKqYROP5&pddocfullpath=%2Fshared%2Fdocument%2Fstatutes-legislation%2Furn%3AcontentItem%3A65HV-06G3-CGX8-050B-00008-00&pdcomponentid=234177&pdtocnodeidentifier=ABPAACAACAABAACABA&ecomp=k2vckkk&prid=e2e32763-f8fa-4832-8191-f70124d877f6\n" - "https://tax.colorado.gov/sites/tax/files/documents/DR_104_Book_2022.pdf#page=46" - ) reference = ( # C.R.S. 39-22-119(1.7) (2026+ regime added by HB24-1134, SECTION 1) "https://leg.colorado.gov/sites/default/files/2024a_1134_signed.pdf#page=3", # Income Tax Topics: Child and Dependent Care Expenses Credit (Revised January 2026) "https://tax.colorado.gov/sites/tax/files/documents/ITT_Child_and_Dependent_Care_Expenses_Credit_Jan_2026.pdf", + "https://advance.lexis.com/documentpage/?pdmfid=1000516&crid=d14880b7-7410-4295-bcf1-2e099e57d8f3&pdistocdocslideraccess=true&config=014FJAAyNGJkY2Y4Zi1mNjgyLTRkN2YtYmE4OS03NTYzNzYzOTg0OGEKAFBvZENhdGFsb2d592qv2Kywlf8caKqYROP5&pddocfullpath=%2Fshared%2Fdocument%2Fstatutes-legislation%2Furn%3AcontentItem%3A65HV-06G3-CGX8-050B-00008-00&pdcomponentid=234177&pdtocnodeidentifier=ABPAACAACAABAACABA&ecomp=k2vckkk&prid=e2e32763-f8fa-4832-8191-f70124d877f6", + "https://tax.colorado.gov/sites/tax/files/documents/DR_104_Book_2022.pdf#page=46", ) definition_period = YEAR defined_for = StateCode.CO diff --git a/policyengine_us/variables/gov/states/hi/tax/income/deductions/hi_deductions.py b/policyengine_us/variables/gov/states/hi/tax/income/deductions/hi_deductions.py index 5e84412ac6c..a92860b971e 100644 --- a/policyengine_us/variables/gov/states/hi/tax/income/deductions/hi_deductions.py +++ b/policyengine_us/variables/gov/states/hi/tax/income/deductions/hi_deductions.py @@ -6,9 +6,9 @@ class hi_deductions(Variable): entity = TaxUnit label = "Hawaii deductions" unit = USD - documentation = ( - "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=15\n" # Itemized Deduction - "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=20" # Standard Deduction + reference = ( + "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=15", # Itemized Deduction + "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=20", # Standard Deduction ) definition_period = YEAR defined_for = StateCode.HI diff --git a/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_casualty_loss_deduction.py b/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_casualty_loss_deduction.py index 29ec2090d92..69fde86cf7f 100644 --- a/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_casualty_loss_deduction.py +++ b/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_casualty_loss_deduction.py @@ -6,9 +6,9 @@ class hi_casualty_loss_deduction(Variable): entity = TaxUnit label = "Hawaii casualty loss deduction" unit = USD - documentation = ( - "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=18\n" - "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=32" # total itemized deduction worksheet + reference = ( + "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=18", + "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=32", # total itemized deduction worksheet ) definition_period = YEAR defined_for = StateCode.HI diff --git a/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_interest_deduction.py b/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_interest_deduction.py index 4a58a37629d..5d90b997a73 100644 --- a/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_interest_deduction.py +++ b/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_interest_deduction.py @@ -6,9 +6,9 @@ class hi_interest_deduction(Variable): entity = TaxUnit label = "Hawaii interest deduction" unit = USD - documentation = ( - "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=17\n" - "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=32" # total itemized deduction worksheet + reference = ( + "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=17", + "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=32", # total itemized deduction worksheet ) definition_period = YEAR defined_for = StateCode.HI diff --git a/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_itemized_deductions.py b/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_itemized_deductions.py index 2358892da1a..0a31d2ef99a 100644 --- a/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_itemized_deductions.py +++ b/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_itemized_deductions.py @@ -6,9 +6,9 @@ class hi_itemized_deductions(Variable): entity = TaxUnit label = "Hawaii itemized deduction" unit = USD - documentation = ( - "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=15\n" - "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=32" # total itemized deduction worksheet + reference = ( + "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=15", + "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=32", # total itemized deduction worksheet ) definition_period = YEAR defined_for = StateCode.HI diff --git a/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_medical_expense_deduction.py b/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_medical_expense_deduction.py index fea377e4596..3f8256d91c1 100644 --- a/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_medical_expense_deduction.py +++ b/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_medical_expense_deduction.py @@ -6,9 +6,9 @@ class hi_medical_expense_deduction(Variable): entity = TaxUnit label = "Hawaii medical expense deduction" unit = USD - documentation = ( - "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=15\n" - "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=32" # total itemized deduction worksheet + reference = ( + "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=15", + "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=32", # total itemized deduction worksheet ) definition_period = YEAR defined_for = StateCode.HI diff --git a/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_reduced_itemized_deductions.py b/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_reduced_itemized_deductions.py index c68fabcfe61..b63c15e4543 100644 --- a/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_reduced_itemized_deductions.py +++ b/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_reduced_itemized_deductions.py @@ -6,9 +6,9 @@ class hi_reduced_itemized_deductions(Variable): entity = TaxUnit label = "Hawaii reduced itemized deduction" unit = USD - documentation = ( - "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=15\n" - "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=32" # total itemized deduction worksheet + reference = ( + "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=15", + "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=32", # total itemized deduction worksheet ) definition_period = YEAR defined_for = StateCode.HI diff --git a/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_total_itemized_deductions.py b/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_total_itemized_deductions.py index b4524d6ef82..e3b7826c340 100644 --- a/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_total_itemized_deductions.py +++ b/policyengine_us/variables/gov/states/hi/tax/income/deductions/itemized/hi_total_itemized_deductions.py @@ -6,9 +6,9 @@ class hi_total_itemized_deductions(Variable): entity = TaxUnit label = "Hawaii total itemized deduction" unit = USD - documentation = ( - "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=15\n" - "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=32" # total itemized deduction worksheet + reference = ( + "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=15", + "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=32", # total itemized deduction worksheet ) definition_period = YEAR defined_for = StateCode.HI diff --git a/policyengine_us/variables/gov/states/hi/tax/income/deductions/standard/hi_standard_deduction.py b/policyengine_us/variables/gov/states/hi/tax/income/deductions/standard/hi_standard_deduction.py index d85230bcacf..8fcc21f050a 100644 --- a/policyengine_us/variables/gov/states/hi/tax/income/deductions/standard/hi_standard_deduction.py +++ b/policyengine_us/variables/gov/states/hi/tax/income/deductions/standard/hi_standard_deduction.py @@ -6,7 +6,7 @@ class hi_standard_deduction(Variable): entity = TaxUnit label = "Hawaii standard deduction" unit = USD - documentation = "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf" + reference = "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf" definition_period = YEAR defined_for = StateCode.HI diff --git a/policyengine_us/variables/gov/states/hi/tax/income/exemptions/hi_disabled_exemptions.py b/policyengine_us/variables/gov/states/hi/tax/income/exemptions/hi_disabled_exemptions.py index 4a6057f816c..3931ac4904a 100644 --- a/policyengine_us/variables/gov/states/hi/tax/income/exemptions/hi_disabled_exemptions.py +++ b/policyengine_us/variables/gov/states/hi/tax/income/exemptions/hi_disabled_exemptions.py @@ -6,7 +6,7 @@ class hi_disabled_exemptions(Variable): entity = TaxUnit label = "Hawaii disabled exemptions" unit = USD - documentation = "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=20" + reference = "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=20" definition_period = YEAR defined_for = StateCode.HI diff --git a/policyengine_us/variables/gov/states/hi/tax/income/exemptions/hi_exemptions.py b/policyengine_us/variables/gov/states/hi/tax/income/exemptions/hi_exemptions.py index 6960fae829f..4617222e1fd 100644 --- a/policyengine_us/variables/gov/states/hi/tax/income/exemptions/hi_exemptions.py +++ b/policyengine_us/variables/gov/states/hi/tax/income/exemptions/hi_exemptions.py @@ -6,7 +6,7 @@ class hi_exemptions(Variable): entity = TaxUnit label = "Hawaii exemptions" unit = USD - documentation = "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=20" + reference = "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=20" definition_period = YEAR defined_for = StateCode.HI diff --git a/policyengine_us/variables/gov/states/hi/tax/income/exemptions/hi_regular_exemptions.py b/policyengine_us/variables/gov/states/hi/tax/income/exemptions/hi_regular_exemptions.py index 01bf8f2c3bf..199c0122c99 100644 --- a/policyengine_us/variables/gov/states/hi/tax/income/exemptions/hi_regular_exemptions.py +++ b/policyengine_us/variables/gov/states/hi/tax/income/exemptions/hi_regular_exemptions.py @@ -6,7 +6,7 @@ class hi_regular_exemptions(Variable): entity = TaxUnit label = "Hawaii regular exemptions" unit = USD - documentation = "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=20" + reference = "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=20" definition_period = YEAR defined_for = StateCode.HI diff --git a/policyengine_us/variables/gov/states/hi/tax/income/subtractions/hi_military_pay_exclusion.py b/policyengine_us/variables/gov/states/hi/tax/income/subtractions/hi_military_pay_exclusion.py index ca26c22f349..73ca5c0deab 100644 --- a/policyengine_us/variables/gov/states/hi/tax/income/subtractions/hi_military_pay_exclusion.py +++ b/policyengine_us/variables/gov/states/hi/tax/income/subtractions/hi_military_pay_exclusion.py @@ -6,7 +6,7 @@ class hi_military_pay_exclusion(Variable): entity = TaxUnit label = "Hawaii military reserve or national guard duty pay exclusion" unit = USD - documentation = "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=13" + reference = "https://files.hawaii.gov/tax/forms/2022/n11ins.pdf#page=13" definition_period = YEAR defined_for = StateCode.HI diff --git a/policyengine_us/variables/gov/states/ky/tax/income/credits/personal/ky_aged_personal_tax_credits.py b/policyengine_us/variables/gov/states/ky/tax/income/credits/personal/ky_aged_personal_tax_credits.py index 8a1a50ae7e5..e3298ba60b1 100644 --- a/policyengine_us/variables/gov/states/ky/tax/income/credits/personal/ky_aged_personal_tax_credits.py +++ b/policyengine_us/variables/gov/states/ky/tax/income/credits/personal/ky_aged_personal_tax_credits.py @@ -6,7 +6,7 @@ class ky_aged_personal_tax_credits(Variable): entity = Person unit = USD label = "Kentucky personal tax credits aged amount" - documentation = ( + reference = ( "https://apps.legislature.ky.gov/law/statutes/statute.aspx?id=53500#page=3" ) definition_period = YEAR diff --git a/policyengine_us/variables/gov/states/ky/tax/income/credits/personal/ky_blind_personal_tax_credits.py b/policyengine_us/variables/gov/states/ky/tax/income/credits/personal/ky_blind_personal_tax_credits.py index 940c0115996..766fff9788a 100644 --- a/policyengine_us/variables/gov/states/ky/tax/income/credits/personal/ky_blind_personal_tax_credits.py +++ b/policyengine_us/variables/gov/states/ky/tax/income/credits/personal/ky_blind_personal_tax_credits.py @@ -6,7 +6,7 @@ class ky_blind_personal_tax_credits(Variable): entity = Person unit = USD label = "Kentucky personal tax credits blind amount" - documentation = ( + reference = ( "https://apps.legislature.ky.gov/law/statutes/statute.aspx?id=53500#page=3" ) definition_period = YEAR diff --git a/policyengine_us/variables/gov/states/ri/tax/income/deductions/standard/ri_standard_deduction.py b/policyengine_us/variables/gov/states/ri/tax/income/deductions/standard/ri_standard_deduction.py index 0ca68dd612e..779a53639bd 100644 --- a/policyengine_us/variables/gov/states/ri/tax/income/deductions/standard/ri_standard_deduction.py +++ b/policyengine_us/variables/gov/states/ri/tax/income/deductions/standard/ri_standard_deduction.py @@ -6,7 +6,7 @@ class ri_standard_deduction(Variable): entity = TaxUnit label = "Rhode Island standard deduction" unit = USD - documentation = "https://tax.ri.gov/sites/g/files/xkgbur541/files/2022-12/ADV_2022_40_Inflation_Adjustments.pdf" + reference = "https://tax.ri.gov/sites/g/files/xkgbur541/files/2022-12/ADV_2022_40_Inflation_Adjustments.pdf" definition_period = YEAR defined_for = StateCode.RI diff --git a/policyengine_us/variables/household/income/person/self_employment/business_is_qualified.py b/policyengine_us/variables/household/income/person/self_employment/business_is_qualified.py index dfa1b4830cb..00fe8b7d79b 100644 --- a/policyengine_us/variables/household/income/person/self_employment/business_is_qualified.py +++ b/policyengine_us/variables/household/income/person/self_employment/business_is_qualified.py @@ -8,5 +8,8 @@ class business_is_qualified(Variable): unit = USD documentation = "Whether all income from self-employment, partnerships and S-corporations is from qualified businesses. A qualified trade or business is any trade or business other than a specified service trade or business, or employment. The list of specified service trades can be found at https://www.law.cornell.edu/uscode/text/26/1202#e_3_A." definition_period = YEAR - reference = "https://www.law.cornell.edu/uscode/text/26/199A#d_1" + reference = ( + "https://www.law.cornell.edu/uscode/text/26/199A#d_1", + "https://www.law.cornell.edu/uscode/text/26/1202#e_3_A", + ) default_value = True diff --git a/policyengine_us/variables/household/income/person/self_employment/business_is_sstb.py b/policyengine_us/variables/household/income/person/self_employment/business_is_sstb.py index 5e6faefe599..fa13336ad22 100644 --- a/policyengine_us/variables/household/income/person/self_employment/business_is_sstb.py +++ b/policyengine_us/variables/household/income/person/self_employment/business_is_sstb.py @@ -8,4 +8,7 @@ class business_is_sstb(Variable): unit = USD documentation = "Whether all income from self-employment, partnerships and S-corporations is from qualified businesses. A qualified trade or business is any specified service trade or business, or employment. The list of specified service trades can be found at https://www.law.cornell.edu/uscode/text/26/1202#e_3_A." definition_period = YEAR - reference = "https://www.law.cornell.edu/uscode/text/26/199A#d_1" + reference = ( + "https://www.law.cornell.edu/uscode/text/26/199A#d_1", + "https://www.law.cornell.edu/uscode/text/26/1202#e_3_A", + )