From fba45621b6c7836de72198b8ff5697f31f5d8339 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 7 Oct 2026 15:58:07 -0400 Subject: [PATCH 1/5] Cite NZ spec sources by symbol, not line number benefit_unit_rule.json cited concepts.py:880-919 and concepts.py lines 892-901 for the relationship pointers. #1120 shifted both ranges by one line on main, and #1134 (open) adds seven more lines above them: at its head, lines 892-901 are the partner pointer's text rather than the "always parent 1" sentence, and 880-919 stops before reference_person_id. The file also cited executor.py by line, and scenarios.json cited the harness as "ops main run.py:168", which moves with ops main. Cite the Relationships block, _pointer("parent_1_person_id"), the executor functions, and WEEKS_IN_MODEL_YEAR at ops ba6e7d6e instead. Add unpinned_python_line_citations: no NZ spec resource may cite a line of a Python source unless it pins that source at a commit (as as_rate_bridge.json pins the harness), with Hypothesis cases per refusal branch, plus checks that the cited pointers match the concepts source and that every dotted microcosm symbol the package names resolves. Re-pin the two resource hashes and the NZ spec fingerprint in the golden. Co-Authored-By: Claude Opus 5.5 --- changelog.d/nz-spec-symbol-citations.fixed.md | 1 + .../microcosm/build/nz/benefit_unit_rule.json | 6 +- .../src/microcosm/build/nz/scenarios.json | 2 +- .../shared/test_nz_spec_package.py | 405 +++++++++++++++++- .../tests/golden/nz_country_spec.json | 6 +- 5 files changed, 410 insertions(+), 10 deletions(-) create mode 100644 changelog.d/nz-spec-symbol-citations.fixed.md diff --git a/changelog.d/nz-spec-symbol-citations.fixed.md b/changelog.d/nz-spec-symbol-citations.fixed.md new file mode 100644 index 000000000..351221e44 --- /dev/null +++ b/changelog.d/nz-spec-symbol-citations.fixed.md @@ -0,0 +1 @@ +Cite source code by symbol, not line number, in the New Zealand spec package. `benefit_unit_rule.json` cited the relationship pointers as `concepts.py:880-919` and `concepts.py lines 892-901`, which #1120 shifted by one line, and the graph executor as `executor.py lines 636-643` and `lines 1550-1573`. It now names the `# --- Relationships` block from `_pointer("partner_person_id")` through `_pointer("reference_person_id")`, `_pointer("parent_1_person_id")`, and `microcosm.graph.executor._structural_columns` and `_validate_population_declaration`. `scenarios.json` cited the conformance harness as `ops main run.py:168`, which moves with ops main. It now names `WEEKS_IN_MODEL_YEAR` in `nz-lane/emtr_reproduction/run.py` at ops `ba6e7d6e`. A new engine-free test refuses any line citation of a Python source in the package unless the same resource pins that source at a commit, as `as_rate_bridge.json` pins the harness. It also checks that the benefit unit rule's pointer citations match the concepts source and that every dotted `microcosm.*` symbol the package names resolves. The two resource hashes and the NZ spec fingerprint in `tests/golden/nz_country_spec.json` are re-pinned. The spec-engine `package_fingerprint` moves, and no test or artifact pins it. The `spec_sha256` and `documentation_sha256` are unchanged. No behaviour or data changes. diff --git a/packages/microcosm-build/src/microcosm/build/nz/benefit_unit_rule.json b/packages/microcosm-build/src/microcosm/build/nz/benefit_unit_rule.json index 0fe9753e6..4808c6848 100644 --- a/packages/microcosm-build/src/microcosm/build/nz/benefit_unit_rule.json +++ b/packages/microcosm-build/src/microcosm/build/nz/benefit_unit_rule.json @@ -2,7 +2,7 @@ "version": 1, "country": "nz", "method_card_row": "MC7", - "description": "The New Zealand benefit unit (engine Family entity): an adult, their co-resident partner, and their dependent children; every other adult forms their own unit. The unit id family.family_id and the membership person.person_family_id are structural graph columns: the graph executor counts each entity's id column and each person-table membership column as structural (microcosm.graph.executor._structural_columns, executor.py lines 636-643) and rejects a non-structural node that declares one as an output (_validate_population_declaration, executor.py lines 1550-1573). family.family_household_id, the unit's link to its household, is not structural; it is an ordinary family column. The planned shared transport CREATE kernel (packages G4 and G5a) is to build the units from the donor concept pointers, refusing ambiguous or cross-household units. Family weights are never stored: each unit inherits its household's weight through the Frame.", + "description": "The New Zealand benefit unit (engine Family entity): an adult, their co-resident partner, and their dependent children; every other adult forms their own unit. The unit id family.family_id and the membership person.person_family_id are structural graph columns: the graph executor counts each entity's id column and each person-table membership column as structural (microcosm.graph.executor._structural_columns) and rejects a non-structural node that declares one as an output (microcosm.graph.executor._validate_population_declaration). family.family_household_id, the unit's link to its household, is not structural; it is an ordinary family column. The planned shared transport CREATE kernel (packages G4 and G5a) is to build the units from the donor concept pointers, refusing ambiguous or cross-household units. Family weights are never stored: each unit inherits its household's weight through the Frame.", "status": "awaiting D1 (MC7; the dependent-child rule is spec data under amendment MC11a, which is proposed and also awaits D1)", "entity": "family", "engine_entity": "Family", @@ -19,7 +19,7 @@ "parent_2_person_id" ], "household_reference_person": "reference_person_id", - "citation": "packages/microcosm-frame/src/microcosm/frame/concepts.py:880-919" + "citation": "packages/microcosm-frame/src/microcosm/frame/concepts.py, the \"# --- Relationships\" block: _pointer(\"partner_person_id\") through _pointer(\"reference_person_id\")" }, "composition": [ { @@ -32,7 +32,7 @@ }, { "member": "dependent_child", - "rule": "a dependent child joins the unit of its co-resident parent parent_1_person_id; a lone co-resident parent is always parent 1 (the pointer definition, concepts.py lines 892-901; the pointer checks flag parent_2_person_id without parent_1_person_id as parent_order, concepts.py _check_pointers), so parent_2_person_id never chooses the unit" + "rule": "a dependent child joins the unit of its co-resident parent parent_1_person_id; a lone co-resident parent is always parent 1 (the pointer definition, concepts.py _pointer(\"parent_1_person_id\"); the pointer checks flag parent_2_person_id without parent_1_person_id as parent_order, concepts.py _check_pointers), so parent_2_person_id never chooses the unit" }, { "member": "other_adult", diff --git a/packages/microcosm-build/src/microcosm/build/nz/scenarios.json b/packages/microcosm-build/src/microcosm/build/nz/scenarios.json index b947b1981..0f0e86db2 100644 --- a/packages/microcosm-build/src/microcosm/build/nz/scenarios.json +++ b/packages/microcosm-build/src/microcosm/build/nz/scenarios.json @@ -150,7 +150,7 @@ "annualisation": { "weeks_per_year": 52, "method_card_row": "MC12", - "note": "the conformance harness uses 365/7 weeks (ops main run.py:168); the model states and uses 52" + "note": "the conformance harness uses 365/7 weeks (WEEKS_IN_MODEL_YEAR in TheAxiomFoundation/ops nz-lane/emtr_reproduction/run.py at ba6e7d6e); the model states and uses 52" }, "groupings": { "method_card_row": "MC3", diff --git a/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py b/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py index fe7320b6c..640ba21b2 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py @@ -19,12 +19,17 @@ (``modal_area``); - every scenario names exactly one method-card row and a tier from {entitlement, calibration}, and changes exactly one knob from the centre - (``scenario_errors``). + (``scenario_errors``); +- no resource cites a line number of a Python source unless the same + resource pins that source at a commit (``unpinned_python_line_citations``): + this repository's sources are cited by symbol, because their line numbers + move under unrelated edits. Differential checks compare two records of one fact: the loader's target references with a direct parse, the rules bindings' module digests with the -committed Axiom input surface, and the export contract's formula-owned list -with the bound variables. +committed Axiom input surface, the export contract's formula-owned list +with the bound variables, and the benefit unit rule's symbol citations with +the sources they name. """ from __future__ import annotations @@ -32,10 +37,13 @@ import copy import dataclasses import hashlib +import importlib +import inspect import json import math import re from collections.abc import Callable, Iterator, Mapping, Sequence +from pathlib import PurePosixPath from typing import Any import pytest @@ -49,6 +57,7 @@ reference_fact_selectors, ) from microcosm.build.spec_engine import load_yaml12 +from microcosm.frame import concepts as frame_concepts from test_support.paths import paths_for _BUILD = paths_for("microcosm-build") @@ -97,6 +106,20 @@ def _nested_keys(value: object) -> Iterator[str]: yield from _nested_keys(child) +def _nested_strings(value: object) -> Iterator[str]: + """Every string in a payload, mapping keys included, at any depth.""" + + if isinstance(value, Mapping): + for key, child in value.items(): + yield str(key) + yield from _nested_strings(child) + elif isinstance(value, list): + for child in value: + yield from _nested_strings(child) + elif isinstance(value, str): + yield value + + def _package_payloads() -> dict[str, object]: payloads: dict[str, object] = {} for path in sorted(NZ_ROOT.rglob("*")): @@ -429,6 +452,102 @@ def scenario_errors(document: Mapping[str, Any]) -> list[str]: return errors +#: A Python source named in prose: ``concepts.py``, or a path ending in one. +_PY_SOURCE = r"(?P[A-Za-z0-9_./-]*[A-Za-z0-9_]\.py)(?![A-Za-z0-9_])" +#: A line locator into a Python source, in each spelling the package has used +#: or GitHub produces: ``x.py:12``, ``x.py:12-19``, ``x.py#L12``, ``x.py L12``, +#: ``x.py line 12``, ``x.py, lines 12-19``, ``x.py (lines 12-19)``, and the +#: reverse ``lines 12-19 of x.py``. +_PY_LINE_CITATIONS = ( + re.compile(_PY_SOURCE + r"(?:\s*:\s*|#L|\s+L|,?\s*\(?\s*[Ll]ines?\s+)\d"), + re.compile( + r"\b[Ll]ines?\s+\d+(?:\s*[-–]\s*\d+)?(?:\s*,\s*\d+)*\s+(?:of|in)\s+" + + _PY_SOURCE + ), +) +#: A commit pin. A branch name such as ``main`` pins nothing. +_COMMIT_PIN = re.compile(r"[0-9a-fA-F]{7,40}") + + +def python_line_citations(value: object) -> list[tuple[str, str]]: + """Each (file name, string) where a string cites a line of a Python source.""" + + return [ + (PurePosixPath(match["file"]).name, text) + for text in _nested_strings(value) + for pattern in _PY_LINE_CITATIONS + for match in pattern.finditer(text) + ] + + +def commit_pinned_python_sources(value: object) -> set[str]: + """File names of the Python sources a payload pins at a commit. + + A pin is a mapping, at any depth, whose ``path`` names a ``.py`` file and + whose ``commit`` is 7-40 hex digits. + """ + + pinned: set[str] = set() + if isinstance(value, Mapping): + path, commit = value.get("path"), value.get("commit") + if ( + isinstance(path, str) + and path.endswith(".py") + and isinstance(commit, str) + and _COMMIT_PIN.fullmatch(commit) + ): + pinned.add(PurePosixPath(path).name) + for child in value.values(): + pinned |= commit_pinned_python_sources(child) + elif isinstance(value, list): + for child in value: + pinned |= commit_pinned_python_sources(child) + return pinned + + +def unpinned_python_line_citations(document: object) -> list[tuple[str, str]]: + """Line citations of Python sources that the document does not pin. + + A line number names code only at a fixed commit. This repository's sources + shift under unrelated edits, so a resource cites them by symbol + (``concepts.py _pointer("parent_1_person_id")``). A resource cites another + repository's source by line only when it also pins that source at a + commit, as as_rate_bridge.json's ``source`` does. Pins match by file name. + """ + + pinned = commit_pinned_python_sources(document) + return [ + (name, text) + for name, text in python_line_citations(document) + if name not in pinned + ] + + +#: A dotted Microcosm symbol named in prose, such as +#: ``microcosm.graph.executor._structural_columns``. +_DOTTED_MICROCOSM_SYMBOL = re.compile( + r"(? object: + """Import the longest module prefix of a dotted name, then get the rest.""" + + parts = dotted.split(".") + for cut in range(len(parts), 0, -1): + name = ".".join(parts[:cut]) + try: + target: object = importlib.import_module(name) + except ModuleNotFoundError as error: + if error.name != name: + raise + continue + for attribute in parts[cut:]: + target = getattr(target, attribute) + return target + raise ModuleNotFoundError(dotted, name=dotted) + + # --------------------------------------------------------------------------- # Reference sets # --------------------------------------------------------------------------- @@ -1474,6 +1593,286 @@ def test_every_refusal_branch_names_the_broken_scenario( assert expected in scenario_errors(document) +# --------------------------------------------------------------------------- +# Source citations +# --------------------------------------------------------------------------- + + +_PY_STEM = st.from_regex(r"[a-z_][a-z0-9_]{0,15}", fullmatch=True) +_PY_DIRECTORY = st.sampled_from( + ( + "", + "packages/microcosm-frame/src/microcosm/frame/", + "microcosm/graph/", + "nz-lane/emtr_reproduction/", + ) +) +#: Line-locator spellings: {file} is the cited source, {a} and {b} lines. +_LINE_LOCATORS = ( + "{file}:{a}", + "{file}:{a}-{b}", + "{file}:{a},{b}", + "{file}#L{a}", + "{file}#L{a}-L{b}", + "{file} L{a}", + "{file} line {a}", + "{file} lines {a}-{b}", + "{file}, lines {a}-{b}", + "{file} (lines {a}-{b})", + "Lines {a}-{b} of {file}", + "line {a} in {file}", +) +#: Symbol-citation spellings, which carry no line number. +_SYMBOL_LOCATORS = ( + "{file} {symbol}", + '{file} _pointer("{symbol}")', + '{file}, the "# --- Relationships" block', + "{file}: {symbol}", + "{symbol} in {file} at main", +) +#: Prose around a citation. It has no digit, so it can neither extend a +#: planted line number nor plant a citation of its own. +_PROSE = st.text( + alphabet="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ,;()'\"-_", + max_size=40, +) +_BRANCH_NAMES = ("main", "HEAD", "launch/ga-path-execution-gate", "ba6e7d", "") + + +def _plant(text: str, depth: int, as_key: bool) -> object: + planted: object = {text: "cited"} if as_key else text + for level in range(depth): + planted = {"level": level, "note": [planted]} + return planted + + +class TestSourceCitations: + def test_no_resource_cites_a_python_line_it_does_not_pin(self) -> None: + offenders = { + name: hits + for name, payload in _package_payloads().items() + if (hits := unpinned_python_line_citations(payload)) + } + assert offenders == {} + + def test_the_harness_line_citations_are_scanned_and_pinned(self) -> None: + bridge = _load("as_rate_bridge.json") + assert {name for name, _ in python_line_citations(bridge)} == {"run.py"} + assert commit_pinned_python_sources(bridge) == {"run.py"} + assert re.fullmatch(r"[0-9a-f]{40}", bridge["source"]["commit"]) + + @pytest.mark.parametrize( + "replaced", + [ + "packages/microcosm-frame/src/microcosm/frame/concepts.py:880-919", + "a lone co-resident parent is always parent 1 (the pointer " + "definition, concepts.py lines 892-901; the pointer checks ...)", + "(microcosm.graph.executor._structural_columns, executor.py lines 636-643)", + "(_validate_population_declaration, executor.py lines 1550-1573)", + "the conformance harness uses 365/7 weeks (ops main run.py:168)", + ], + ) + def test_the_line_citations_the_package_carried_are_refused( + self, replaced: str + ) -> None: + assert unpinned_python_line_citations({"rule": replaced}) + + @settings(max_examples=300, deadline=None) + @given( + stem=_PY_STEM, + directory=_PY_DIRECTORY, + locator=st.sampled_from(_LINE_LOCATORS), + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + prefix=_PROSE, + suffix=_PROSE, + depth=st.integers(0, 3), + as_key=st.booleans(), + ) + def test_an_unpinned_line_citation_is_refused_at_any_depth( + self, + stem: str, + directory: str, + locator: str, + a: int, + b: int, + prefix: str, + suffix: str, + depth: int, + as_key: bool, + ) -> None: + citation = locator.format(file=f"{directory}{stem}.py", a=a, b=b) + text = f"{prefix} {citation} {suffix}" + hits = unpinned_python_line_citations(_plant(text, depth, as_key)) + assert (f"{stem}.py", text) in hits + + @settings(max_examples=200, deadline=None) + @given( + stem=_PY_STEM, + directory=_PY_DIRECTORY, + pin_directory=_PY_DIRECTORY, + locator=st.sampled_from(_LINE_LOCATORS), + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + depth=st.integers(0, 3), + as_key=st.booleans(), + commit=_HEX40_TEXT, + length=st.integers(7, 40), + ) + def test_a_line_citation_of_a_source_pinned_at_a_commit_passes( + self, + stem: str, + directory: str, + pin_directory: str, + locator: str, + a: int, + b: int, + depth: int, + as_key: bool, + commit: str, + length: int, + ) -> None: + citation = locator.format(file=f"{directory}{stem}.py", a=a, b=b) + document = { + "source": { + "repository": "TheAxiomFoundation/ops", + "path": f"{pin_directory}{stem}.py", + "commit": commit[:length], + }, + "notes": [_plant(citation, depth, as_key)], + } + assert python_line_citations(document) + assert unpinned_python_line_citations(document) == [] + + @settings(max_examples=200, deadline=None) + @given( + stem=_PY_STEM, + other=_PY_STEM, + locator=st.sampled_from(_LINE_LOCATORS), + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + commit=_HEX40_TEXT, + branch=st.sampled_from(_BRANCH_NAMES), + pin_other_file=st.booleans(), + ) + def test_a_pin_of_another_file_or_at_a_branch_does_not_cover_a_citation( + self, + stem: str, + other: str, + locator: str, + a: int, + b: int, + commit: str, + branch: str, + pin_other_file: bool, + ) -> None: + if pin_other_file: + assume(other != stem) + pin = {"path": f"{other}.py", "commit": commit} + else: + pin = {"path": f"{stem}.py", "commit": branch} + citation = locator.format(file=f"{stem}.py", a=a, b=b) + hits = unpinned_python_line_citations({"source": pin, "note": citation}) + assert (f"{stem}.py", citation) in hits + + @settings(max_examples=200, deadline=None) + @given( + stem=_PY_STEM, + directory=_PY_DIRECTORY, + locator=st.sampled_from(_SYMBOL_LOCATORS), + symbol=_PY_STEM, + prefix=_PROSE, + suffix=_PROSE, + ) + def test_a_symbol_citation_is_not_a_line_citation( + self, + stem: str, + directory: str, + locator: str, + symbol: str, + prefix: str, + suffix: str, + ) -> None: + citation = locator.format(file=f"{directory}{stem}.py", symbol=symbol) + assert python_line_citations({"rule": f"{prefix} {citation} {suffix}"}) == [] + + def test_benefit_unit_rule_cites_the_relationship_pointers_by_symbol( + self, + ) -> None: + rule = _load("benefit_unit_rule.json") + source = inspect.getsource(frame_concepts) + block = source.split("# --- Relationships", 1)[1].split("# --- ", 1)[0] + block_pointers = re.findall(r'_pointer\(\s*"(\w+)"', block) + assert block_pointers == [ + "partner_person_id", + "parent_1_person_id", + "parent_2_person_id", + "reference_person_id", + ] + pointers = rule["pointer_concepts"] + named = [ + pointers["partner"], + *pointers["parents"], + pointers["household_reference_person"], + ] + assert named == block_pointers + assert pointers["citation"] == ( + "packages/microcosm-frame/src/microcosm/frame/concepts.py, the " + f'"# --- Relationships" block: _pointer("{block_pointers[0]}") ' + f'through _pointer("{block_pointers[-1]}")' + ) + cited = { + name + for text in _nested_strings(rule) + for name in re.findall(r'_pointer\("(\w+)"\)', text) + } + assert cited == { + "partner_person_id", + "parent_1_person_id", + "reference_person_id", + } + (child,) = [ + row for row in rule["composition"] if row["member"] == "dependent_child" + ] + assert 'concepts.py _pointer("parent_1_person_id")' in child["rule"] + parent_1 = frame_concepts.concept("fact:person.parent_1_person_id") + assert "always parent 1" in parent_1.definition + assert "concepts.py _check_pointers" in child["rule"] + assert callable(frame_concepts._check_pointers) + + def test_every_dotted_microcosm_symbol_in_the_package_resolves(self) -> None: + named = { + match + for payload in _package_payloads().values() + for text in _nested_strings(payload) + for match in _DOTTED_MICROCOSM_SYMBOL.findall(text) + } + assert { + "microcosm.graph.executor._structural_columns", + "microcosm.graph.executor._validate_population_declaration", + } <= named + unresolved = [] + for dotted in sorted(named): + try: + resolve_dotted_symbol(dotted) + except (ImportError, AttributeError) as error: + unresolved.append(f"{dotted}: {error}") + assert unresolved == [] + + @pytest.mark.parametrize( + ("dotted", "error"), + [ + ("microcosm.graph.executor._no_such_symbol", AttributeError), + ("microcosm.no_such_shard.module.symbol", ModuleNotFoundError), + ], + ) + def test_an_unresolvable_dotted_symbol_raises( + self, dotted: str, error: type[Exception] + ) -> None: + with pytest.raises(error): + resolve_dotted_symbol(dotted) + + # --------------------------------------------------------------------------- # Remaining resources # --------------------------------------------------------------------------- diff --git a/packages/microcosm-build/tests/golden/nz_country_spec.json b/packages/microcosm-build/tests/golden/nz_country_spec.json index 2f0ff95ad..03f04d6c7 100644 --- a/packages/microcosm-build/tests/golden/nz_country_spec.json +++ b/packages/microcosm-build/tests/golden/nz_country_spec.json @@ -1,6 +1,6 @@ { "country": "nz", - "fingerprint": "df8850d5893e9311882573ff6b4baa41c9dda41f6838d9a140199ddf0cc7fd48", + "fingerprint": "b7a6622b1602bf3eebcfa8ceb7000c2460f0cbe1d39d0e60cccdaa8f5e5c1ca6", "gate_ids": [ "calibration_per_family_fit", "national_and_region_admin_aggregates", @@ -37,7 +37,7 @@ "as_area_crosswalk.json": "cfb76c99898204598b0609e5462b2fba9d081872a13ac5dd973cb406bb6f7469", "as_rate_bridge.json": "5eea897159508fb71820f22cd8ccf4993075a750462bd97312d9e89ab01cba15", "axiom_rules_bindings.json": "72905af77132ff6a3c01ada4b6210be4c369d9c6dc49dc8c1b673e4bac0b5fb1", - "benefit_unit_rule.json": "9e507e78f180566a1eece74da2a35d88478dbe13487c3a64ee2b76c9ecc7a653", + "benefit_unit_rule.json": "2dcc3a890a4b11c3905550e0933e30305e8f5f6144a1af433cec720f710dc49f", "country_package.json": "b83de9e12238b41481810ddb7fbdbaaa71d457cbfa9da83f3f53cbf479cee43b", "currency_bridge.json": "aa6a4b286a6a1a015088f32ac013148553422c3d45eeef7d7a15af08388857ca", "export_contract.json": "f0629a3990a47f3f94114c4d7b6680636ef7b1bd34bb8e950976d3065f029fda", @@ -46,7 +46,7 @@ "holdout_references.json": "52b5a2360502156da1392a68350e4e7f61b5fd420fa74428819b65886a9f8655", "precal_references.json": "43b13849e3d74ebca7d7a9d46817463e3ad1333c3aedd7a3eac39e870510fba0", "release_contract.json": "6cd5e3a4a2bf30e4025e04e964c393b24040937d13db7e6a07b3966a55f3b114", - "scenarios.json": "5dbf2b9603fa55fd9abd7319e188f882ffe471e99e7d691b81980522a9bef82f", + "scenarios.json": "1ca4227290e1a8755886c030e9e6b93ce8fa427238b53ad7671971341af0435b", "source_stages.json": "cc1eef0769c79917f33c5ba0f99edf861070143168db48e0145b71efeee751d1", "spec/bundle.yaml": "25ef74ecbc00c3f2ecca2d30ddd5e46a582657863c217da33ee9a396569d2cf2", "spec/catalogs.yaml": "3ab2025af39cba93ead707b9463e428430a45b30696a43d0906cbcd97ed36631", From b50e40aa718d7cfb615174dca82db962aeb4037a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 8 Oct 2026 09:13:58 -0400 Subject: [PATCH 2/5] Cite US and UK spec sources by symbol or at a commit, never by drifting line Port PR #1141's NZ citation guard to every country spec package as a shared validator (test_support/microcosm_build/spec_source_citations.py) and an engine-free test (test_spec_source_citations.py). The guard refuses a line citation of a file in this repository, pinned or not, and a line citation of any other source unless its string names the commit or the resource pins the file in a path/commit mapping. US: ecps_parity_known_gaps.json now cites asec_pool symbols, the PUF support tool's functions, the build script's --asec-h5 arguments and the Build J summary's keys; where #720's Census person-column restore had made a sentence false, it now names the columns the restore leaves out. The remaining us-data citations in source_stages.json and target_parity_manifest.json name archived commit 42ed5d45. UK: uk_data_target_parity.json (and its generator) names incumbent commit 8629dbbf; uk_population_targets.json names registry ref 12a1e028. Reading every cited range at its commit corrected six wrong pointers (SSI take-up, SCF net worth fit, hmrc_cgt.py 195, frs_only.py, _land.py 52-73, the salary-sacrifice drop). Re-pins the source_stages.json hash, the regenerated sources.yaml, the US spec and documentation hashes, three inventory EXPECTED_HASHES and us-f0-coverage.json. Co-Authored-By: Claude Opus 5.5 --- .../spec-source-citations-us-uk.fixed.md | 1 + docs/evidence/spec-engine/us-f0-coverage.json | 20 +- .../build/spec_engine/inventory_coverage.py | 6 +- .../build/uk/uk_data_target_parity.json | 26 +- .../build/uk/uk_population_targets.json | 94 +-- .../build/uk_runtime/data_target_parity.py | 26 +- .../build/us/ecps_parity_known_gaps.json | 16 +- .../src/microcosm/build/us/source_stages.json | 14 +- .../src/microcosm/build/us/spec/sources.yaml | 132 ++-- .../build/us/target_parity_manifest.json | 2 +- .../shared/test_spec_source_citations.py | 633 ++++++++++++++++++ .../us/test_us_bundle_core_contracts.py | 2 +- .../test_us_financial_assistance_exclusion.py | 13 +- .../us/test_us_other_health_insurance.py | 13 +- .../us/test_us_survivor_benefits_exclusion.py | 13 +- .../test_us_wic_nutritional_risk_exclusion.py | 13 +- .../us/test_us_multispine_pool_tool.py | 2 +- .../microcosm_build/spec_source_citations.py | 243 +++++++ .../microcosm_build/us_spec_bundle.py | 2 +- tools/build_us_target_parity_manifest.py | 3 +- tools/generate_us_bundle_from_constants.py | 2 +- 21 files changed, 1098 insertions(+), 178 deletions(-) create mode 100644 changelog.d/spec-source-citations-us-uk.fixed.md create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py create mode 100644 test_support/microcosm_build/spec_source_citations.py diff --git a/changelog.d/spec-source-citations-us-uk.fixed.md b/changelog.d/spec-source-citations-us-uk.fixed.md new file mode 100644 index 000000000..56117f286 --- /dev/null +++ b/changelog.d/spec-source-citations-us-uk.fixed.md @@ -0,0 +1 @@ +Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, and a line citation of any other source unless its string names the commit (`commit `, `at `, `@` or a `/blob//` link) or the resource pins that file in a `path`/`commit` mapping. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` name archived commit `42ed5d45`. The uk-data citations in `uk/uk_data_target_parity.json` name incumbent commit `8629dbbf`, and those in `uk/uk_population_targets.json` name the registry ref `12a1e028`. Reading each cited range at its commit also corrected six citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index f99b29cdf..34336af8b 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,7 +4,7 @@ "version": 1 }, "country": "us", - "documentation_sha256": "0931cbd3aae2abcd8c072b78eba40c1c006b1bf7fa23c6a6681f9a895e47070e", + "documentation_sha256": "b3d98ce90c62d273061fed19ee8db943ef7d64f53e697ddb9c9180a47f09daa3", "field_usage": { "authored_normative_field_count": 32404, "claim_count": 49, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20" + "sha256": "5d84124a6109a8c0d22445fb36c12e56e29817cc694f15f7984dd6bff90aa26e" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20" + "sha256": "5d84124a6109a8c0d22445fb36c12e56e29817cc694f15f7984dd6bff90aa26e" }, "status": "covered" }, @@ -1775,12 +1775,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + "sha256": "4e0d2a606457f5e9d92c2d1e88fb682e861535f8b59e276c989ae06b67cf4e0d", "stages": 37 }, "failures": [], "observed": { - "sha256": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + "sha256": "4e0d2a606457f5e9d92c2d1e88fb682e861535f8b59e276c989ae06b67cf4e0d", "stages": 38 }, "status": "covered" @@ -1887,7 +1887,7 @@ "alpha", "zeta" ], - "sha256": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007" + "sha256": "9a86a97784e022d5654f6d420e3fff58f3564bf11d2ffc66155e68ed37500d31" }, "failures": [], "observed": { @@ -1911,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007" + "sha256": "9a86a97784e022d5654f6d420e3fff58f3564bf11d2ffc66155e68ed37500d31" }, "status": "covered" }, @@ -1975,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "61a5484eab9fae1d6651b3ae15f1793d4934ea2f6ed2c08d05716b745c221014" + "sha256": "934b244e467d82cf620eaf56a5cf388d7af20896d03be2ad5869c7aa9083ccd4" }, "status": "covered" }, @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "d1df6b316e4937e809f5f020ea2d3a7bd38bccc1845f3ae6e104f7eed6a73323" + "spec_sha256": "5df3ce86a59c61b45ca76cb90cf87aa80818d89fd7f835440a7dafb3f66ff578" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "d1df6b316e4937e809f5f020ea2d3a7bd38bccc1845f3ae6e104f7eed6a73323" + "spec_sha256": "5df3ce86a59c61b45ca76cb90cf87aa80818d89fd7f835440a7dafb3f66ff578" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 3fddff951..000cd963d 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -350,18 +350,18 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007", + "full_checkpoint": "9a86a97784e022d5654f6d420e3fff58f3564bf11d2ffc66155e68ed37500d31", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", "graph_nodes": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e", "geography_assignment": "44e7f5f6cd7ceeef85d532c3d46306d03de4f4ee811376e6d29920742746b65c", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20", + "late_resource_semantics": "5d84124a6109a8c0d22445fb36c12e56e29817cc694f15f7984dd6bff90aa26e", "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", "seed_map": "0fa116d9f5dae4ba0378f190d6c822b92f3dcf1e3fe80c9880bcaf87431c283c", "seed_protocol": "b4afa376b62fbc073fd10d74e40921f5ab60c1efe366e2c8504872091ea69d08", - "source_manifest": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + "source_manifest": "4e0d2a606457f5e9d92c2d1e88fb682e861535f8b59e276c989ae06b67cf4e0d", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", } diff --git a/packages/microcosm-build/src/microcosm/build/uk/uk_data_target_parity.json b/packages/microcosm-build/src/microcosm/build/uk/uk_data_target_parity.json index cb4efa40e..e1a36b060 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/uk_data_target_parity.json +++ b/packages/microcosm-build/src/microcosm/build/uk/uk_data_target_parity.json @@ -190,7 +190,7 @@ "policyengine_uk_data.datasets.imputations.services.services", "policyengine_uk_data.datasets.imputations.consumption" ], - "evidence": "uk-data datasets/imputations/services/services.py:33 and datasets/imputations/consumption.py:790,830; Chronicle#200/#202.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 datasets/imputations/services/services.py:33 and datasets/imputations/consumption.py:790,830; Chronicle#200/#202.", "fence": { "origin": "uk-data post-imputation-rescaled bus subsidy and fares, including London share and quintile rates.", "purpose": "Preserve service-consumption totals and distribution.", @@ -206,7 +206,7 @@ "policyengine_uk_data.datasets.imputations.services.services", "policyengine_uk_data.tests.test_aggregates" ], - "evidence": "uk-data datasets/imputations/frs_only.py and uk-data tests/test_aggregates.py:5; uk-data#199 and uk-data#474.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 datasets/imputations/services/services.py:137-148 and tests/test_aggregates.py:5; uk-data#199 and uk-data#474.", "fence": { "origin": "uk-data imputed DfE education spending without an active calibration target.", "purpose": "Retain the imputed input while avoiding an unsupported aggregate claim.", @@ -221,7 +221,7 @@ "covers": [ "policyengine_uk_data.datasets.imputations.services.services" ], - "evidence": "uk-data datasets/imputations/services/services.py:18; Chronicle transport facts landed in #200/#202.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 datasets/imputations/services/services.py:18; Chronicle transport facts landed in #200/#202.", "fence": { "origin": "uk-data rescaled rail_usage to a national subsidy total.", "purpose": "Anchor the imputed service-consumption mass.", @@ -782,7 +782,7 @@ "policyengine_uk_data.targets.sources.la_council_tax", "policyengine_uk_data.datasets.local_areas.local_authorities.loss" ], - "evidence": "uk-data targets/sources/la_council_tax.py and datasets/local_areas/local_authorities/loss.py:251-300; Microcosm council_tax/band_a..i contract rows (mhclg, welshgov and scotgov by_area families) bind 2,511 authority cells on the taxbase basis with the 296 English band-H cells and Shetland's band-H cell signed deferred (microcosm#929).", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/la_council_tax.py and datasets/local_areas/local_authorities/loss.py:251-300; Microcosm council_tax/band_a..i contract rows (mhclg, welshgov and scotgov by_area families) bind 2,511 authority cells on the taxbase basis with the 296 English band-H cells and Shetland's band-H cell signed deferred (microcosm#929).", "status": "ported_local_declared" }, { @@ -862,7 +862,7 @@ "covers": [ "policyengine_uk_data.targets.sources.voa_council_tax" ], - "evidence": "uk-data targets/sources/voa_council_tax.py:235,275; Microcosm mhclg.council_tax_stock.* (England, composed from the MHCLG taxbase authority rows per region), welshgov.council_tax_stock.* and scotgov.council_tax_stock.* targets on the councils' taxbase basis (microcosm#929).", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/voa_council_tax.py:235,275; Microcosm mhclg.council_tax_stock.* (England, composed from the MHCLG taxbase authority rows per region), welshgov.council_tax_stock.* and scotgov.council_tax_stock.* targets on the councils' taxbase basis (microcosm#929).", "status": "ported_national" }, { @@ -880,7 +880,7 @@ "covers": [ "policyengine_uk_data.targets.sources.hmrc_cgt" ], - "evidence": "uk-data targets/sources/hmrc_cgt.py:195; Microcosm hmrc_cgt_conditioning_facts.json (vendored HMRC Table 2.1a 2024-25 rows read by the donor stage and the amounts redraw) and uk_population_targets.json CGT rows.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/hmrc_cgt.py:196; Microcosm hmrc_cgt_conditioning_facts.json (vendored HMRC Table 2.1a 2024-25 rows read by the donor stage and the amounts redraw) and uk_population_targets.json CGT rows.", "status": "ported_national" }, { @@ -898,7 +898,7 @@ "covers": [ "policyengine_uk_data.targets.sources.hmrc_spi" ], - "evidence": "uk-data targets/sources/hmrc_spi.py:230; Microcosm HMRC SPI amount/count-by-total-income-band targets and replay fences.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/hmrc_spi.py:230; Microcosm HMRC SPI amount/count-by-total-income-band targets and replay fences.", "status": "ported_national" }, { @@ -938,7 +938,7 @@ "covers": [ "policyengine_uk_data.targets.sources.obr" ], - "evidence": "uk-data targets/sources/obr.py:284; Microcosm target ids obr.council_tax and country legs obr.council_tax_{england,scotland,wales}.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/obr.py:284; Microcosm target ids obr.council_tax and country legs obr.council_tax_{england,scotland,wales}.", "status": "ported_national" }, { @@ -947,7 +947,7 @@ "covers": [ "policyengine_uk_data.targets.sources.obr" ], - "evidence": "uk-data targets/sources/obr.py:348; Microcosm target ids obr.ni, obr.ni_employee, obr.ni_employer, and obr.ni_self_employed.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/obr.py:348; Microcosm target ids obr.ni, obr.ni_employee, obr.ni_employer, and obr.ni_self_employed.", "status": "ported_national" }, { @@ -956,7 +956,7 @@ "covers": [ "policyengine_uk_data.targets.sources.obr" ], - "evidence": "uk-data targets/sources/obr.py:402-409 and uk-data#88.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/obr.py:402-409 and uk-data#88.", "fence": { "origin": "uk-data skipped the Class 3 OBR row pending uk-data#88.", "purpose": "Avoid pretending voluntary Class 3 contributions are represented by another NIC variable.", @@ -980,7 +980,7 @@ "covers": [ "policyengine_uk_data.targets.sources.obr" ], - "evidence": "uk-data targets/sources/obr.py:458 welfare rows; Microcosm uk_population_targets.json OBR benefit-spending targets.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/obr.py:458 welfare rows; Microcosm uk_population_targets.json OBR benefit-spending targets.", "status": "ported_national" }, { @@ -998,7 +998,7 @@ "covers": [ "policyengine_uk_data.targets.sources.ons_tenure" ], - "evidence": "uk-data targets/sources/ons_tenure.py:69; Microcosm local ons.tenure.* contract rows.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/ons_tenure.py:69; Microcosm local ons.tenure.* contract rows.", "fence": { "origin": "uk-data calibrated five England tenure controls.", "purpose": "Preserve an England-level representation check independently of the local Census tenure surface.", @@ -1068,7 +1068,7 @@ "covers": [ "policyengine_uk_data.targets.sources.obr" ], - "evidence": "uk-data targets/sources/obr.py:617-728; Microcosm uk_population_targets.json tv_licence, private-school, and salary-sacrifice target rows.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/obr.py:570-728; Microcosm uk_population_targets.json tv_licence, private-school, and salary-sacrifice target rows.", "status": "ported_national" }, { diff --git a/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json b/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json index 34c63bc39..47e45c028 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json +++ b/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json @@ -683,9 +683,9 @@ "obr/salary_sacrifice_users_total": "R10 (microcosm#1069 c9): the salary-sacrifice users total contradicts the staged support and the two headcounts either side of £2,000 already bind it; it leaves the fit." }, "unmapped_declarations": { - "hmrc.salary_sacrifice.it_relief_basic_rate": "New declaration backed by chronicle#139 salary-sacrifice facts; the incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394.", - "hmrc.salary_sacrifice.it_relief_higher_rate": "New declaration backed by chronicle#139 salary-sacrifice facts; the incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394.", - "hmrc.salary_sacrifice.it_relief_additional_rate": "New declaration backed by chronicle#139 salary-sacrifice facts; the incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394.", + "hmrc.salary_sacrifice.it_relief_basic_rate": "New declaration backed by chronicle#139 salary-sacrifice facts; the incumbent source CSV is still 410-Gone at 12a1e028 and its rows are dropped by the catch-all except Exception in uk-data targets/sources/hmrc_salary_sacrifice.py lines 34-109 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3, which only logs the download error and returns the salary_sacrifice_contributions target alone, so build_loss_matrix.py never receives them.", + "hmrc.salary_sacrifice.it_relief_higher_rate": "New declaration backed by chronicle#139 salary-sacrifice facts; the incumbent source CSV is still 410-Gone at 12a1e028 and its rows are dropped by the catch-all except Exception in uk-data targets/sources/hmrc_salary_sacrifice.py lines 34-109 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3, which only logs the download error and returns the salary_sacrifice_contributions target alone, so build_loss_matrix.py never receives them.", + "hmrc.salary_sacrifice.it_relief_additional_rate": "New declaration backed by chronicle#139 salary-sacrifice facts; the incumbent source CSV is still 410-Gone at 12a1e028 and its rows are dropped by the catch-all except Exception in uk-data targets/sources/hmrc_salary_sacrifice.py lines 34-109 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3, which only logs the download error and returns the salary_sacrifice_contributions target alone, so build_loss_matrix.py never receives them.", "ons.population.female_90_plus": "María ruling 2026-08-21: split the incumbent terminal sex bands into uniform five-year 85_89 bands plus 90_plus tails. The incumbent six-year 85_90 terminal rows leave ages 91+ unconstrained; Microcosm keeps the uniform five-year bands and constrains the tail. The 85_89-vs-85_90 value gap is the single-age-90 share and is a signed semantic difference.", "ons.population.male_90_plus": "María ruling 2026-08-21: split the incumbent terminal sex bands into uniform five-year 85_89 bands plus 90_plus tails. The incumbent six-year 85_90 terminal rows leave ages 91+ unconstrained; Microcosm keeps the uniform five-year bands and constrains the tail. The 85_89-vs-85_90 value gap is the single-age-90 share and is a signed semantic difference.", "hmrc.tfc.government_top_up": "Post-freeze fact family, no uk-data metric.", @@ -835,23 +835,23 @@ "suppressed_ancestors": { "hmrc/salary_sacrifice_it_relief_basic_rate": { "contract_target_id": "hmrc.salary_sacrifice.it_relief_basic_rate", - "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394." + "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are dropped by the catch-all except Exception in uk-data targets/sources/hmrc_salary_sacrifice.py lines 34-109 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3, which only logs the download error and returns the salary_sacrifice_contributions target alone, so build_loss_matrix.py never receives them." }, "hmrc/salary_sacrifice_it_relief_higher_rate": { "contract_target_id": "hmrc.salary_sacrifice.it_relief_higher_rate", - "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394." + "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are dropped by the catch-all except Exception in uk-data targets/sources/hmrc_salary_sacrifice.py lines 34-109 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3, which only logs the download error and returns the salary_sacrifice_contributions target alone, so build_loss_matrix.py never receives them." }, "hmrc/salary_sacrifice_it_relief_additional_rate": { "contract_target_id": "hmrc.salary_sacrifice.it_relief_additional_rate", - "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394." + "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are dropped by the catch-all except Exception in uk-data targets/sources/hmrc_salary_sacrifice.py lines 34-109 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3, which only logs the download error and returns the salary_sacrifice_contributions target alone, so build_loss_matrix.py never receives them." }, "hmrc/salary_sacrifice_employee_nics_relief": { "contract_target_id": "hmrc.salary_sacrifice.nics_relief_employee", - "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394." + "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are dropped by the catch-all except Exception in uk-data targets/sources/hmrc_salary_sacrifice.py lines 34-109 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3, which only logs the download error and returns the salary_sacrifice_contributions target alone, so build_loss_matrix.py never receives them." }, "hmrc/salary_sacrifice_employer_nics_relief": { "contract_target_id": "hmrc.salary_sacrifice.nics_relief_employer", - "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394." + "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are dropped by the catch-all except Exception in uk-data targets/sources/hmrc_salary_sacrifice.py lines 34-109 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3, which only logs the download error and returns the salary_sacrifice_contributions target alone, so build_loss_matrix.py never receives them." } }, "accounting_notes": { @@ -2200,7 +2200,7 @@ "policyengine": { "metric_name": "obr/council_tax", "value_variable": "council_tax_less_benefit", - "notes": "OBR Table 4.1 reports net council tax receipts (net of council tax reduction), so the binding is council_tax_less_benefit, not gross council_tax (compute/council_tax.py:21-43). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane)." + "notes": "OBR Table 4.1 reports net council tax receipts (net of council tax reduction), so the binding is council_tax_less_benefit, not gross council_tax (uk-data targets/compute/council_tax.py:21-43 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane)." }, "axiom": { "metric_name": "obr/council_tax", @@ -2243,7 +2243,7 @@ "equals": "ENGLAND" } ], - "notes": "OBR Table 4.1 reports net council tax receipts (net of council tax reduction), so the binding is council_tax_less_benefit, not gross council_tax (compute/council_tax.py:21-43). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane)." + "notes": "OBR Table 4.1 reports net council tax receipts (net of council tax reduction), so the binding is council_tax_less_benefit, not gross council_tax (uk-data targets/compute/council_tax.py:21-43 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane)." }, "axiom": { "metric_name": "obr/council_tax_england", @@ -2286,7 +2286,7 @@ "equals": "SCOTLAND" } ], - "notes": "OBR Table 4.1 reports net council tax receipts (net of council tax reduction), so the binding is council_tax_less_benefit, not gross council_tax (compute/council_tax.py:21-43). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane)." + "notes": "OBR Table 4.1 reports net council tax receipts (net of council tax reduction), so the binding is council_tax_less_benefit, not gross council_tax (uk-data targets/compute/council_tax.py:21-43 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane)." }, "axiom": { "metric_name": "obr/council_tax_scotland", @@ -2329,7 +2329,7 @@ "equals": "WALES" } ], - "notes": "OBR Table 4.1 reports net council tax receipts (net of council tax reduction), so the binding is council_tax_less_benefit, not gross council_tax (compute/council_tax.py:21-43). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane)." + "notes": "OBR Table 4.1 reports net council tax receipts (net of council tax reduction), so the binding is council_tax_less_benefit, not gross council_tax (uk-data targets/compute/council_tax.py:21-43 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane)." }, "axiom": { "metric_name": "obr/council_tax_wales", @@ -2426,7 +2426,7 @@ "map_to": "household" } ], - "notes": "EFO incapacity-benefits line; uk-data sums benunit-level esa_income and person-level esa_contrib to household (compute/income.py:98-102). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane). Scoped to Great Britain (microcosm#1095, uk-data#490): OBR table 4.9 reports this line inside 'DWP social security', whose figures cover Great Britain and residents overseas (DWP benefit expenditure and caseload tables, Spring 2026, Notes 3), and lists Northern Ireland social security on rows of its own, so the model column sums Great Britain households only. The Chronicle fact still carries the UK code K02000001." + "notes": "EFO incapacity-benefits line; uk-data sums benunit-level esa_income and person-level esa_contrib to household (uk-data targets/compute/income.py:98-102 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane). Scoped to Great Britain (microcosm#1095, uk-data#490): OBR table 4.9 reports this line inside 'DWP social security', whose figures cover Great Britain and residents overseas (DWP benefit expenditure and caseload tables, Spring 2026, Notes 3), and lists Northern Ireland social security on rows of its own, so the model column sums Great Britain households only. The Chronicle fact still carries the UK code K02000001." }, "axiom": { "metric_name": "obr/esa", @@ -2868,7 +2868,7 @@ "map_to": "benunit" } ], - "notes": "Split of UC spend by benefit-cap status (compute/benefits.py:119-127). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane). Scoped to Great Britain (microcosm#1095, uk-data#490): OBR table 4.9 reports this line inside 'DWP social security', whose figures cover Great Britain and residents overseas (DWP benefit expenditure and caseload tables, Spring 2026, Notes 3), and lists Northern Ireland social security on rows of its own, so the model column sums Great Britain households only. The Chronicle fact still carries the UK code K02000001." + "notes": "Split of UC spend by benefit-cap status (uk-data targets/compute/benefits.py:101-109 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane). Scoped to Great Britain (microcosm#1095, uk-data#490): OBR table 4.9 reports this line inside 'DWP social security', whose figures cover Great Britain and residents overseas (DWP benefit expenditure and caseload tables, Spring 2026, Notes 3), and lists Northern Ireland social security on rows of its own, so the model column sums Great Britain households only. The Chronicle fact still carries the UK code K02000001." }, "axiom": { "metric_name": "obr/universal_credit_in_cap", @@ -2948,7 +2948,7 @@ "map_to": "benunit" } ], - "notes": "Split of UC spend by benefit-cap status (compute/benefits.py:119-127). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane). Scoped to Great Britain (microcosm#1095, uk-data#490): OBR table 4.9 reports this line inside 'DWP social security', whose figures cover Great Britain and residents overseas (DWP benefit expenditure and caseload tables, Spring 2026, Notes 3), and lists Northern Ireland social security on rows of its own, so the model column sums Great Britain households only. The Chronicle fact still carries the UK code K02000001." + "notes": "Split of UC spend by benefit-cap status (uk-data targets/compute/benefits.py:101-109 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane). Scoped to Great Britain (microcosm#1095, uk-data#490): OBR table 4.9 reports this line inside 'DWP social security', whose figures cover Great Britain and residents overseas (DWP benefit expenditure and caseload tables, Spring 2026, Notes 3), and lists Northern Ireland social security on rows of its own, so the model column sums Great Britain households only. The Chronicle fact still carries the UK code K02000001." }, "axiom": { "metric_name": "obr/universal_credit_outside_cap", @@ -3091,7 +3091,7 @@ }, "from_entity": "person", "map_to": "household", - "notes": "Income-tax relief attributed to salary sacrifice, banded on counterfactual adjusted net income against the live parameter thresholds (compute/income.py:23-50)." + "notes": "Income-tax relief attributed to salary sacrifice, banded on counterfactual adjusted net income against the live parameter thresholds (uk-data targets/compute/income.py:23-50 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "hmrc/salary_sacrifice_it_relief_basic_rate", @@ -3133,7 +3133,7 @@ }, "from_entity": "person", "map_to": "household", - "notes": "Income-tax relief attributed to salary sacrifice, banded on counterfactual adjusted net income against the live parameter thresholds (compute/income.py:23-50)." + "notes": "Income-tax relief attributed to salary sacrifice, banded on counterfactual adjusted net income against the live parameter thresholds (uk-data targets/compute/income.py:23-50 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "hmrc/salary_sacrifice_it_relief_higher_rate", @@ -3174,7 +3174,7 @@ }, "from_entity": "person", "map_to": "household", - "notes": "Income-tax relief attributed to salary sacrifice, banded on counterfactual adjusted net income against the live parameter thresholds (compute/income.py:23-50)." + "notes": "Income-tax relief attributed to salary sacrifice, banded on counterfactual adjusted net income against the live parameter thresholds (uk-data targets/compute/income.py:23-50 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "hmrc/salary_sacrifice_it_relief_additional_rate", @@ -3210,7 +3210,7 @@ "output_variable": "ni_employee", "from_entity": "person", "map_to": "household", - "notes": "NICs relief attributed to salary sacrifice (compute/income.py:59-68). uk-data's SPP-Review-based obr/salary_sacrifice_employee_ni_relief row (source lost) retires onto this hmrc-backed fact - vintage difference for the microcosm#622 parity gate." + "notes": "NICs relief attributed to salary sacrifice (uk-data targets/compute/income.py:59-68 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). uk-data's SPP-Review-based obr/salary_sacrifice_employee_ni_relief row (source lost) retires onto this hmrc-backed fact - vintage difference for the microcosm#622 parity gate." }, "axiom": { "metric_name": "hmrc/salary_sacrifice_employee_nics_relief", @@ -3246,7 +3246,7 @@ "output_variable": "ni_employer", "from_entity": "person", "map_to": "household", - "notes": "NICs relief attributed to salary sacrifice (compute/income.py:59-68). uk-data's SPP-Review-based obr/salary_sacrifice_employer_ni_relief row (source lost) retires onto this hmrc-backed fact - vintage difference for the microcosm#622 parity gate." + "notes": "NICs relief attributed to salary sacrifice (uk-data targets/compute/income.py:59-68 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). uk-data's SPP-Review-based obr/salary_sacrifice_employer_ni_relief row (source lost) retires onto this hmrc-backed fact - vintage difference for the microcosm#622 parity gate." }, "axiom": { "metric_name": "hmrc/salary_sacrifice_employer_nics_relief", @@ -3293,7 +3293,7 @@ "from_entity": "person", "map_to": "household", "threshold_price_base_year": 2023, - "notes": "The GBP 2,000 cap is defined at FRS base-year (2023) prices; deflate salary-sacrifice amounts before banding (compute/income.py:71-95)." + "notes": "The GBP 2,000 cap is defined at FRS base-year (2023) prices; deflate salary-sacrifice amounts before banding (uk-data targets/compute/income.py:71-95 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "obr/salary_sacrifice_users_below_cap", @@ -3336,7 +3336,7 @@ "from_entity": "person", "map_to": "household", "threshold_price_base_year": 2023, - "notes": "The GBP 2,000 cap is defined at FRS base-year (2023) prices; deflate salary-sacrifice amounts before banding (compute/income.py:71-95)." + "notes": "The GBP 2,000 cap is defined at FRS base-year (2023) prices; deflate salary-sacrifice amounts before banding (uk-data targets/compute/income.py:71-95 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "obr/salary_sacrifice_users_above_cap", @@ -3373,7 +3373,7 @@ "from_entity": "person", "map_to": "household", "groupby_variable": "total_income", - "notes": "Banded by person-level total income; band edges come from the fact dimension (compute/income.py:8-20)." + "notes": "Banded by person-level total income; band edges come from the fact dimension (uk-data targets/compute/income.py:8-20 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "hmrc/employment_income_income_band", @@ -3461,7 +3461,7 @@ "from_entity": "person", "map_to": "household", "groupby_variable": "total_income", - "notes": "Banded by person-level total income; band edges come from the fact dimension (compute/income.py:8-20)." + "notes": "Banded by person-level total income; band edges come from the fact dimension (uk-data targets/compute/income.py:8-20 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "hmrc/self_employment_income_income_band", @@ -3549,7 +3549,7 @@ "from_entity": "person", "map_to": "household", "groupby_variable": "total_income", - "notes": "Banded by person-level total income; band edges come from the fact dimension (compute/income.py:8-20)." + "notes": "Banded by person-level total income; band edges come from the fact dimension (uk-data targets/compute/income.py:8-20 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "hmrc/dividend_income_income_band", @@ -3637,7 +3637,7 @@ "from_entity": "person", "map_to": "household", "groupby_variable": "total_income", - "notes": "Banded by person-level total income; band edges come from the fact dimension (compute/income.py:8-20). The x1.9 property-income undercount adjustment is a populace-side value declaration, not a fact property." + "notes": "Banded by person-level total income; band edges come from the fact dimension (uk-data targets/compute/income.py:8-20 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). The x1.9 property-income undercount adjustment is a populace-side value declaration, not a fact property." }, "axiom": { "metric_name": "hmrc/property_income_income_band", @@ -3725,7 +3725,7 @@ "from_entity": "person", "map_to": "household", "groupby_variable": "total_income", - "notes": "Banded by person-level total income; band edges come from the fact dimension (compute/income.py:8-20)." + "notes": "Banded by person-level total income; band edges come from the fact dimension (uk-data targets/compute/income.py:8-20 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "hmrc/private_pension_income_income_band", @@ -3813,7 +3813,7 @@ "from_entity": "person", "map_to": "household", "groupby_variable": "total_income", - "notes": "Banded by person-level total income; band edges come from the fact dimension (compute/income.py:8-20)." + "notes": "Banded by person-level total income; band edges come from the fact dimension (uk-data targets/compute/income.py:8-20 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "hmrc/state_pension_income_band", @@ -4044,7 +4044,7 @@ } ], "value_variable": "household_count", - "notes": "0/1 household indicator, not a benunit headcount (compute/benefits.py:14-21). The publisher's GBP 320.9m total-reduction companion is a computed annualization - excluded per the facts-only ADR. Scoped to Great Britain (microcosm#1095, uk-data#490): the DWP benefit cap statistics cover Great Britain (K03000001), so the model counts Great Britain households only." + "notes": "0/1 household indicator, not a benunit headcount (uk-data targets/compute/benefits.py:14-21 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). The publisher's GBP 320.9m total-reduction companion is a computed annualization - excluded per the facts-only ADR. Scoped to Great Britain (microcosm#1095, uk-data#490): the DWP benefit cap statistics cover Great Britain (K03000001), so the model counts Great Britain households only." }, "axiom": { "metric_name": "dwp/benefit_capped_households", @@ -5749,7 +5749,7 @@ "from_entity": "benunit", "map_to": "household", "band_period_factor": 12, - "notes": "Benunit counts by annual UC award band: publisher bands are monthly award amounts, multiplied by 12 to band annual model UC (compute/benefits.py:84-98; half-open [lower, upper)). Administrative family classification uses model standard allowance above the model-year maximum single rate for couples, separately from retained claimant roles; zero allowance is UNKNOWN. Child presence retains own-benefit-unit reported-under-20/qualifying-child proxy and does not establish DWP declared-child attachment or ineligible-partner exceptions. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Family%20Type.html The last bound source band ends at £2,500 monthly, inclusive; the separately published £2,500.01 or over category remains unbound. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Monthly%20Award%20Amount%20(bands).html Count only the Great Britain regions, matching K03000001; DWP removes Northern Ireland-office claims from these statistics. Source: https://www.gov.uk/government/publications/universal-credit-statistics-background-information-and-methodology/universal-credit-statistics-background-information-and-methodology#creating-a-great-britain-dataset", + "notes": "Benunit counts by annual UC award band: publisher bands are monthly award amounts, multiplied by 12 to band annual model UC (uk-data targets/compute/benefits.py:84-98 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3; half-open [lower, upper)). Administrative family classification uses model standard allowance above the model-year maximum single rate for couples, separately from retained claimant roles; zero allowance is UNKNOWN. Child presence retains own-benefit-unit reported-under-20/qualifying-child proxy and does not establish DWP declared-child attachment or ineligible-partner exceptions. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Family%20Type.html The last bound source band ends at £2,500 monthly, inclusive; the separately published £2,500.01 or over category remains unbound. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Monthly%20Award%20Amount%20(bands).html Count only the Great Britain regions, matching K03000001; DWP removes Northern Ireland-office claims from these statistics. Source: https://www.gov.uk/government/publications/universal-credit-statistics-background-information-and-methodology/universal-credit-statistics-background-information-and-methodology#creating-a-great-britain-dataset", "household_conditions": [ { "variable": "region", @@ -5841,7 +5841,7 @@ "from_entity": "benunit", "map_to": "household", "band_period_factor": 12, - "notes": "Benunit counts by annual UC award band: publisher bands are monthly award amounts, multiplied by 12 to band annual model UC (compute/benefits.py:84-98; half-open [lower, upper)). Administrative family classification uses model standard allowance above the model-year maximum single rate for couples, separately from retained claimant roles; zero allowance is UNKNOWN. Child presence retains own-benefit-unit reported-under-20/qualifying-child proxy and does not establish DWP declared-child attachment or ineligible-partner exceptions. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Family%20Type.html The last bound source band ends at £2,500 monthly, inclusive; the separately published £2,500.01 or over category remains unbound. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Monthly%20Award%20Amount%20(bands).html Count only the Great Britain regions, matching K03000001; DWP removes Northern Ireland-office claims from these statistics. Source: https://www.gov.uk/government/publications/universal-credit-statistics-background-information-and-methodology/universal-credit-statistics-background-information-and-methodology#creating-a-great-britain-dataset", + "notes": "Benunit counts by annual UC award band: publisher bands are monthly award amounts, multiplied by 12 to band annual model UC (uk-data targets/compute/benefits.py:84-98 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3; half-open [lower, upper)). Administrative family classification uses model standard allowance above the model-year maximum single rate for couples, separately from retained claimant roles; zero allowance is UNKNOWN. Child presence retains own-benefit-unit reported-under-20/qualifying-child proxy and does not establish DWP declared-child attachment or ineligible-partner exceptions. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Family%20Type.html The last bound source band ends at £2,500 monthly, inclusive; the separately published £2,500.01 or over category remains unbound. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Monthly%20Award%20Amount%20(bands).html Count only the Great Britain regions, matching K03000001; DWP removes Northern Ireland-office claims from these statistics. Source: https://www.gov.uk/government/publications/universal-credit-statistics-background-information-and-methodology/universal-credit-statistics-background-information-and-methodology#creating-a-great-britain-dataset", "household_conditions": [ { "variable": "region", @@ -5933,7 +5933,7 @@ "from_entity": "benunit", "map_to": "household", "band_period_factor": 12, - "notes": "Benunit counts by annual UC award band: publisher bands are monthly award amounts, multiplied by 12 to band annual model UC (compute/benefits.py:84-98; half-open [lower, upper)). Administrative family classification uses model standard allowance above the model-year maximum single rate for couples, separately from retained claimant roles; zero allowance is UNKNOWN. Child presence retains own-benefit-unit reported-under-20/qualifying-child proxy and does not establish DWP declared-child attachment or ineligible-partner exceptions. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Family%20Type.html The last bound source band ends at £2,500 monthly, inclusive; the separately published £2,500.01 or over category remains unbound. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Monthly%20Award%20Amount%20(bands).html Count only the Great Britain regions, matching K03000001; DWP removes Northern Ireland-office claims from these statistics. Source: https://www.gov.uk/government/publications/universal-credit-statistics-background-information-and-methodology/universal-credit-statistics-background-information-and-methodology#creating-a-great-britain-dataset", + "notes": "Benunit counts by annual UC award band: publisher bands are monthly award amounts, multiplied by 12 to band annual model UC (uk-data targets/compute/benefits.py:84-98 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3; half-open [lower, upper)). Administrative family classification uses model standard allowance above the model-year maximum single rate for couples, separately from retained claimant roles; zero allowance is UNKNOWN. Child presence retains own-benefit-unit reported-under-20/qualifying-child proxy and does not establish DWP declared-child attachment or ineligible-partner exceptions. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Family%20Type.html The last bound source band ends at £2,500 monthly, inclusive; the separately published £2,500.01 or over category remains unbound. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Monthly%20Award%20Amount%20(bands).html Count only the Great Britain regions, matching K03000001; DWP removes Northern Ireland-office claims from these statistics. Source: https://www.gov.uk/government/publications/universal-credit-statistics-background-information-and-methodology/universal-credit-statistics-background-information-and-methodology#creating-a-great-britain-dataset", "household_conditions": [ { "variable": "region", @@ -6025,7 +6025,7 @@ "from_entity": "benunit", "map_to": "household", "band_period_factor": 12, - "notes": "Benunit counts by annual UC award band: publisher bands are monthly award amounts, multiplied by 12 to band annual model UC (compute/benefits.py:84-98; half-open [lower, upper)). Administrative family classification uses model standard allowance above the model-year maximum single rate for couples, separately from retained claimant roles; zero allowance is UNKNOWN. Child presence retains own-benefit-unit reported-under-20/qualifying-child proxy and does not establish DWP declared-child attachment or ineligible-partner exceptions. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Family%20Type.html The last bound source band ends at £2,500 monthly, inclusive; the separately published £2,500.01 or over category remains unbound. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Monthly%20Award%20Amount%20(bands).html Count only the Great Britain regions, matching K03000001; DWP removes Northern Ireland-office claims from these statistics. Source: https://www.gov.uk/government/publications/universal-credit-statistics-background-information-and-methodology/universal-credit-statistics-background-information-and-methodology#creating-a-great-britain-dataset", + "notes": "Benunit counts by annual UC award band: publisher bands are monthly award amounts, multiplied by 12 to band annual model UC (uk-data targets/compute/benefits.py:84-98 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3; half-open [lower, upper)). Administrative family classification uses model standard allowance above the model-year maximum single rate for couples, separately from retained claimant roles; zero allowance is UNKNOWN. Child presence retains own-benefit-unit reported-under-20/qualifying-child proxy and does not establish DWP declared-child attachment or ineligible-partner exceptions. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Family%20Type.html The last bound source band ends at £2,500 monthly, inclusive; the separately published £2,500.01 or over category remains unbound. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Monthly%20Award%20Amount%20(bands).html Count only the Great Britain regions, matching K03000001; DWP removes Northern Ireland-office claims from these statistics. Source: https://www.gov.uk/government/publications/universal-credit-statistics-background-information-and-methodology/universal-credit-statistics-background-information-and-methodology#creating-a-great-britain-dataset", "household_conditions": [ { "variable": "region", @@ -10760,7 +10760,7 @@ } ], "value_variable": "household_count", - "notes": "Household indicator (compute/demographics.py:60-77)." + "notes": "Household indicator (uk-data targets/compute/demographics.py:60-77 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "ons/scotland_households_3plus_children", @@ -11180,7 +11180,7 @@ } ], "value_variable": "household_count", - "notes": "Bound on the frame column household.ons_household_type == 'lone_parent_non_dependent_children_households', written by the frs_relationships stage (microcosm#791) from the FRS household grid (relhrp, R01-R14); the value is Chronicle's ons.household_type dimension value id, so the contract row, the frame and the published fact share one string and no engine variable is involved. ONS dependency is under 16, or 16-18 in full-time education (the engine's is_child, age under 18, is not used). Exactly one family, a lone parent whose children in the household are all non-dependent. The summed-age head-age proxy inherited from uk-data compute/households.py:45-52 is retired; the relationship grid makes the cell exact." + "notes": "Bound on the frame column household.ons_household_type == 'lone_parent_non_dependent_children_households', written by the frs_relationships stage (microcosm#791) from the FRS household grid (relhrp, R01-R14); the value is Chronicle's ons.household_type dimension value id, so the contract row, the frame and the published fact share one string and no engine variable is involved. ONS dependency is under 16, or 16-18 in full-time education (the engine's is_child, age under 18, is not used). Exactly one family, a lone parent whose children in the household are all non-dependent. The summed-age head-age proxy inherited from uk-data targets/compute/households.py:45-52 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3 is retired; the relationship grid makes the cell exact." }, "axiom": { "metric_name": "ons/lone_parent_non_dependent_children_households", @@ -13191,7 +13191,7 @@ "value_variable": "savings_interest_income", "from_entity": "person", "map_to": "household", - "notes": "uk-data holds the last actual flat for later years - a populace-side declaration (sources/ons_savings.py:39-44)." + "notes": "uk-data holds the last actual flat for later years - a populace-side declaration (uk-data targets/sources/ons_savings.py:39-44 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "ons/savings_interest_income", @@ -13264,7 +13264,7 @@ "policyengine": { "metric_name": "ons/land_value", "value_variable": "land_value", - "notes": "National balance sheet land series for the total_economy sector; uk-data carries the last estimate flat to later years (populace-side declaration, sources/_land.py:52-73)." + "notes": "National balance sheet land series for the total_economy sector; uk-data carries the last estimate flat to later years (populace-side declaration, uk-data targets/sources/_land.py:48-52 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "ons/land_value", @@ -13294,7 +13294,7 @@ "policyengine": { "metric_name": "ons/household_land_value", "value_variable": "household_land_value", - "notes": "National balance sheet land series for the households_and_npish sector; uk-data carries the last estimate flat to later years (populace-side declaration, sources/_land.py:52-73)." + "notes": "National balance sheet land series for the households_and_npish sector; uk-data carries the last estimate flat to later years (populace-side declaration, uk-data targets/sources/_land.py:54-59 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "ons/household_land_value", @@ -13324,7 +13324,7 @@ "policyengine": { "metric_name": "ons/corporate_land_value", "value_variable": "corporate_land_value", - "notes": "National balance sheet land series for the non_financial_corporations sector; uk-data carries the last estimate flat to later years (populace-side declaration, sources/_land.py:52-73)." + "notes": "National balance sheet land series for the non_financial_corporations sector; uk-data carries the last estimate flat to later years (populace-side declaration, uk-data targets/sources/_land.py:60-65 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "ons/corporate_land_value", @@ -13357,7 +13357,7 @@ "value_variable": "student_loan_repayment", "from_entity": "person", "map_to": "household", - "notes": "One fact per devolved country (Scotland, Wales, Northern Ireland); the resolved geography binds the model country filter (compute/other.py:116-137)." + "notes": "One fact per devolved country (Scotland, Wales, Northern Ireland); the resolved geography binds the model country filter (uk-data targets/compute/other.py:116-137 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "slc/student_loan_repayment/devolved", @@ -13522,7 +13522,7 @@ "plan_2_full_time", "plan_2_part_time" ], - "notes": "Publisher splits full-time/part-time (and masters/doctoral) products the model does not distinguish; the listed source lines sum to this target (compute/other.py:116-137)." + "notes": "Publisher splits full-time/part-time (and masters/doctoral) products the model does not distinguish; the listed source lines sum to this target (uk-data targets/compute/other.py:116-137 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "slc/student_loan_repayment/england/plan_2", @@ -13583,7 +13583,7 @@ "plan_3_postgraduate_masters", "plan_3_postgraduate_doctoral" ], - "notes": "Publisher splits full-time/part-time (and masters/doctoral) products the model does not distinguish; the listed source lines sum to this target (compute/other.py:116-137)." + "notes": "Publisher splits full-time/part-time (and masters/doctoral) products the model does not distinguish; the listed source lines sum to this target (uk-data targets/compute/other.py:116-137 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "slc/student_loan_repayment/england/postgraduate", @@ -13644,7 +13644,7 @@ "plan_5_full_time", "plan_5_part_time" ], - "notes": "Publisher splits full-time/part-time (and masters/doctoral) products the model does not distinguish; the listed source lines sum to this target (compute/other.py:116-137)." + "notes": "Publisher splits full-time/part-time (and masters/doctoral) products the model does not distinguish; the listed source lines sum to this target (uk-data targets/compute/other.py:116-137 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "slc/student_loan_repayment/england/plan_5", @@ -13696,7 +13696,7 @@ "value_variable": "person_count", "from_entity": "person", "map_to": "household", - "notes": "England borrowers on this plan earning above the repayment threshold (proxied by positive repayments, matching what the FRS captures via PAYE; compute/other.py:77-96)." + "notes": "England borrowers on this plan earning above the repayment threshold (proxied by positive repayments, matching what the FRS captures via PAYE; uk-data targets/compute/other.py:77-96 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "slc/plan_2_borrowers_above_threshold", @@ -13744,7 +13744,7 @@ "value_variable": "person_count", "from_entity": "person", "map_to": "household", - "notes": "England borrowers on this plan liable to repay, including below-threshold holders (compute/other.py:99-113)." + "notes": "England borrowers on this plan liable to repay, including below-threshold holders (uk-data targets/compute/other.py:99-113 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "slc/plan_2_borrowers_liable", @@ -13797,7 +13797,7 @@ "value_variable": "person_count", "from_entity": "person", "map_to": "household", - "notes": "England borrowers on this plan earning above the repayment threshold (proxied by positive repayments, matching what the FRS captures via PAYE; compute/other.py:77-96)." + "notes": "England borrowers on this plan earning above the repayment threshold (proxied by positive repayments, matching what the FRS captures via PAYE; uk-data targets/compute/other.py:77-96 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "slc/plan_5_borrowers_above_threshold", @@ -13845,7 +13845,7 @@ "value_variable": "person_count", "from_entity": "person", "map_to": "household", - "notes": "England borrowers on this plan liable to repay, including below-threshold holders (compute/other.py:99-113)." + "notes": "England borrowers on this plan liable to repay, including below-threshold holders (uk-data targets/compute/other.py:99-113 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "slc/plan_5_borrowers_liable", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/data_target_parity.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/data_target_parity.py index 554400887..f3d73dc02 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/data_target_parity.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/data_target_parity.py @@ -184,14 +184,14 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "concern_id": "national_obr_national_insurance", "status": "ported_national", "classification": "red_line_national_family", - "evidence": "uk-data targets/sources/obr.py:348; Microcosm target ids obr.ni, obr.ni_employee, obr.ni_employer, and obr.ni_self_employed.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/obr.py:348; Microcosm target ids obr.ni, obr.ni_employee, obr.ni_employer, and obr.ni_self_employed.", }, { "concern_id": "national_obr_national_insurance_class_3", "status": "reviewed_exclusion", "classification": "not_modeled", "reason": "The incumbent deliberately skipped Class 3 and no Microcosm dataset path exists.", - "evidence": "uk-data targets/sources/obr.py:402-409 and uk-data#88.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/obr.py:402-409 and uk-data#88.", "fence": _fence( "uk-data skipped the Class 3 OBR row pending uk-data#88.", "Avoid pretending voluntary Class 3 contributions are represented by another NIC variable.", @@ -202,31 +202,31 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "concern_id": "national_obr_council_tax", "status": "ported_national", "classification": "red_line_national_family", - "evidence": "uk-data targets/sources/obr.py:284; Microcosm target ids obr.council_tax and country legs obr.council_tax_{england,scotland,wales}.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/obr.py:284; Microcosm target ids obr.council_tax and country legs obr.council_tax_{england,scotland,wales}.", }, { "concern_id": "national_obr_welfare", "status": "ported_national", "classification": "national_registry_family", - "evidence": "uk-data targets/sources/obr.py:458 welfare rows; Microcosm uk_population_targets.json OBR benefit-spending targets.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/obr.py:458 welfare rows; Microcosm uk_population_targets.json OBR benefit-spending targets.", }, { "concern_id": "national_tv_licence_and_policy_statics", "status": "ported_national", "classification": "national_registry_family", - "evidence": "uk-data targets/sources/obr.py:617-728; Microcosm uk_population_targets.json tv_licence, private-school, and salary-sacrifice target rows.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/obr.py:570-728; Microcosm uk_population_targets.json tv_licence, private-school, and salary-sacrifice target rows.", }, { "concern_id": "national_hmrc_spi_income_bands", "status": "ported_national", "classification": "national_registry_family", - "evidence": "uk-data targets/sources/hmrc_spi.py:230; Microcosm HMRC SPI amount/count-by-total-income-band targets and replay fences.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/hmrc_spi.py:230; Microcosm HMRC SPI amount/count-by-total-income-band targets and replay fences.", }, { "concern_id": "national_hmrc_cgt_size_bands", "status": "ported_national", "classification": "national_registry_family", - "evidence": "uk-data targets/sources/hmrc_cgt.py:195; Microcosm hmrc_cgt_conditioning_facts.json (vendored HMRC Table 2.1a 2024-25 rows read by the donor stage and the amounts redraw) and uk_population_targets.json CGT rows.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/hmrc_cgt.py:196; Microcosm hmrc_cgt_conditioning_facts.json (vendored HMRC Table 2.1a 2024-25 rows read by the donor stage and the amounts redraw) and uk_population_targets.json CGT rows.", }, { "concern_id": "national_hmrc_salary_sacrifice", @@ -265,7 +265,7 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "status": "routed", "classification": "national_grain_follow_up", "reason": "The incumbent England-national tenure controls are not part of the current national contract; local tenure is declared separately.", - "evidence": "uk-data targets/sources/ons_tenure.py:69; Microcosm local ons.tenure.* contract rows.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/ons_tenure.py:69; Microcosm local ons.tenure.* contract rows.", "fence": _fence( "uk-data calibrated five England tenure controls.", "Preserve an England-level representation check independently of the local Census tenure surface.", @@ -318,7 +318,7 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "concern_id": "national_council_tax_stock", "status": "ported_national", "classification": "red_line_national_family", - "evidence": "uk-data targets/sources/voa_council_tax.py:235,275; Microcosm mhclg.council_tax_stock.* (England, composed from the MHCLG taxbase authority rows per region), welshgov.council_tax_stock.* and scotgov.council_tax_stock.* targets on the councils' taxbase basis (microcosm#929).", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/voa_council_tax.py:235,275; Microcosm mhclg.council_tax_stock.* (England, composed from the MHCLG taxbase authority rows per region), welshgov.council_tax_stock.* and scotgov.council_tax_stock.* targets on the councils' taxbase basis (microcosm#929).", }, { "concern_id": "national_housing_rate_headcount_products", @@ -360,7 +360,7 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "concern_id": "local_council_tax_band_counts", "status": "ported_local_declared", "classification": "local_registry_family", - "evidence": "uk-data targets/sources/la_council_tax.py and datasets/local_areas/local_authorities/loss.py:251-300; Microcosm council_tax/band_a..i contract rows (mhclg, welshgov and scotgov by_area families) bind 2,511 authority cells on the taxbase basis with the 296 English band-H cells and Shetland's band-H cell signed deferred (microcosm#929).", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 targets/sources/la_council_tax.py and datasets/local_areas/local_authorities/loss.py:251-300; Microcosm council_tax/band_a..i contract rows (mhclg, welshgov and scotgov by_area families) bind 2,511 authority cells on the taxbase basis with the 296 English band-H cells and Shetland's band-H cell signed deferred (microcosm#929).", }, { "concern_id": "local_private_rent_pipr", @@ -434,7 +434,7 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "status": "routed", "classification": "imputation_anchor", "reason": "The £21.6bn rail anchor belongs to WS-E/F (#148), not the local target solve.", - "evidence": "uk-data datasets/imputations/services/services.py:18; Chronicle transport facts landed in #200/#202.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 datasets/imputations/services/services.py:18; Chronicle transport facts landed in #200/#202.", "fence": _fence( "uk-data rescaled rail_usage to a national subsidy total.", "Anchor the imputed service-consumption mass.", @@ -446,7 +446,7 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "status": "routed", "classification": "imputation_anchor", "reason": "BUS05 subsidy/fare totals are consumed by the #789 rows; the NTS0705 quintile gradient remains routed to WS-E/F (#148).", - "evidence": "uk-data datasets/imputations/services/services.py:33 and datasets/imputations/consumption.py:790,830; Chronicle#200/#202.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 datasets/imputations/services/services.py:33 and datasets/imputations/consumption.py:790,830; Chronicle#200/#202.", "fence": _fence( "uk-data post-imputation-rescaled bus subsidy and fares, including London share and quintile rates.", "Preserve service-consumption totals and distribution.", @@ -458,7 +458,7 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "status": "reviewed_exclusion", "classification": "untargeted_both_sides", "reason": "Education spending was imputed but its aggregate test was commented out; corrected post-ebf733c DfE entitlement data from uk-data#474 remains input-side.", - "evidence": "uk-data datasets/imputations/frs_only.py and uk-data tests/test_aggregates.py:5; uk-data#199 and uk-data#474.", + "evidence": "uk-data at 8629dbbfe82727278d47ee5ff09fd1da4acefa38 datasets/imputations/services/services.py:137-148 and tests/test_aggregates.py:5; uk-data#199 and uk-data#474.", "fence": _fence( "uk-data imputed DfE education spending without an active calibration target.", "Retain the imputed input while avoiding an unsupported aggregate claim.", diff --git a/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json b/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json index 81b23b3d4..31ae0ab0f 100644 --- a/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json +++ b/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json @@ -40,7 +40,7 @@ "missing_columns": ["NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP"] } ], - "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-69 passes the three prebuilt 2022-2024 artifacts via --asec-h5; experiments/build_j_recert/base_j.summary.json lines 55-75 records their SHA-256 digests; tools/build_us_puf_support_base.py lines 85-88, 647-687, and 748-755 consumes those mappings and does not refresh omitted raw Census columns." + "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh passes the three prebuilt 2022-2024 artifacts via its --asec-h5 2024=, 2023=, and 2022= arguments; experiments/build_j_recert/base_j.summary.json base_source.sources[].sha256 records their SHA-256 digests; tools/build_us_puf_support_base.py _parse_args (--asec-h5) and _pooled_asec_sources_from_args consume those mappings, and _asec_sources_from_args binds each year to its pinned Census person file, from which microcosm.build.us_runtime.asec_pool._prepare_year_input restores only microcosm.build.us_runtime.asec_census_person_columns.ASEC_CENSUS_PERSON_COLUMNS, none of which is NOW_OWNGRP, NOW_HIPAID, or NOW_GRPFTYP." } }, "financial_assistance": { @@ -118,7 +118,7 @@ } ], "aggregate_identity": "For every locked artifact, HFINVAL equals the household sum of FFINVAL; neither aggregate identifies a recipient person.", - "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-69 passes only the three prebuilt 2022-2024 artifacts via --asec-h5; experiments/build_j_recert/base_j.summary.json lines 55-75 records their SHA-256 digests; tools/build_us_puf_support_base.py lines 85-88 and 659-699 consumes only those mappings; packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py lines 61-75 reads the prebuilt person/household tables without refreshing omitted Census columns." + "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh passes only the three prebuilt 2022-2024 artifacts via its --asec-h5 2024=, 2023=, and 2022= arguments; experiments/build_j_recert/base_j.summary.json base_source.sources[].sha256 records their SHA-256 digests; tools/build_us_puf_support_base.py _parse_args (--asec-h5) and _pooled_asec_sources_from_args consume those mappings, and _asec_sources_from_args binds each year to its pinned Census person file; microcosm.build.us_runtime.asec_pool.load_asec_h5_tables reads the prebuilt person/household tables, and microcosm.build.us_runtime.asec_pool._prepare_year_input restores only microcosm.build.us_runtime.asec_census_person_columns.ASEC_CENSUS_PERSON_COLUMNS, which excludes FIN_VAL, FIN_YN, and I_FINVAL." } }, "is_unmarried_partner_of_household_head": { @@ -173,10 +173,10 @@ "semantic_non_substitutes": { "A_EXPRRP": "Only the 2024 locked input carries it, and code 13 is the combined CPSRelationshipCode.PARTNER_OR_ROOMMATE category rather than the retired PERRP unmarried-partner category.", "PECOHAB": "Only the 2024 locked input carries the cohabiting-partner line pointer; it cannot recover partner status for the equally weighted 2022 and 2023 source years.", - "derived_A_EXPRRP": "packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py lines 397-422 derives older-year relationship codes from line, spouse, sex, and parent pointers and never emits code 13.", + "derived_A_EXPRRP": "microcosm.build.us_runtime.asec_pool._with_relationship_recode derives relationship codes from line, spouse, sex, and parent pointers only for a person table without A_EXPRRP, such as the raw 2022 and 2023 H5 inputs, and that fallback never emits code 13; since microcosm #720 the pooling tools bind each year's pinned Census person file and microcosm.build.us_runtime.asec_pool._prepare_year_input restores Census A_EXPRRP, whose reviewed domain includes code 13, before the recode, so pooled 2022 and 2023 persons no longer take the fallback.", "rejection": "Using only the 2024 alternatives and filling the equally weighted 2022-2023 source years as false, or splitting the combined partner/roommate category, would synthesize person statuses absent from the locked inputs." }, - "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-69 passes only the three prebuilt 2022-2024 artifacts via --asec-h5; experiments/build_j_recert/base_j.summary.json lines 55-75 records their SHA-256 digests and equal source-year shares; packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py lines 61-75 reads those tables without refreshing omitted Census columns." + "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh passes only the three prebuilt 2022-2024 artifacts via its --asec-h5 2024=, 2023=, and 2022= arguments; experiments/build_j_recert/base_j.summary.json base_source.sources[].sha256 and base_source.metadata.sources[].share record their SHA-256 digests and equal source-year shares; microcosm.build.us_runtime.asec_pool.load_asec_h5_tables reads those tables, and since microcosm #720 microcosm.build.us_runtime.asec_pool._prepare_year_input restores the 2022 and 2023 inputs' missing A_EXPRRP from pinned Census person files but not PERRP, PECOHAB, or A_FAMREL." } }, "is_wic_at_nutritional_risk": { @@ -283,7 +283,7 @@ "retired_bernoulli_draw": "The early-1980s category priors are aggregate eligibility-estimation factors, not person-level assessment observations.", "rejection": "Assigning False to nonrecipients, allocating an SPM amount to a person, or replaying the retired Bernoulli draw would synthesize individual nutritional-risk assessments absent from the hermetic sources." }, - "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-69 passes only the three prebuilt 2022-2024 artifacts via --asec-h5; experiments/build_j_recert/base_j.summary.json lines 55-75 records their SHA-256 digests; packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py lines 61-75 reads only their person and household tables without refreshing or linking separate WIC certification records." + "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh passes only the three prebuilt 2022-2024 artifacts via its --asec-h5 2024=, 2023=, and 2022= arguments; experiments/build_j_recert/base_j.summary.json base_source.sources[].sha256 records their SHA-256 digests; microcosm.build.us_runtime.asec_pool.load_asec_h5_tables reads only their person and household tables without refreshing or linking separate WIC certification records." } }, "survivor_benefits": { @@ -354,7 +354,7 @@ ], "aggregate_identity": "For every locked artifact, HSURVAL equals the household sum of FSURVAL; neither aggregate identifies the recipient person.", "semantic_rejection": "The retired derivation requires a reported person amount. Its CPS-only QRF cannot train without that leaf, and assigning FSURVAL or HSURVAL to a head, first member, or arbitrary split would synthesize a survivor-benefit observation absent from every locked person table.", - "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-69 passes only the three prebuilt 2022-2024 artifacts via --asec-h5; experiments/build_j_recert/base_j.summary.json lines 55-75 records their SHA-256 digests; packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py lines 61-75 reads the prebuilt person tables without refreshing omitted Census columns." + "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh passes only the three prebuilt 2022-2024 artifacts via its --asec-h5 2024=, 2023=, and 2022= arguments; experiments/build_j_recert/base_j.summary.json base_source.sources[].sha256 records their SHA-256 digests; microcosm.build.us_runtime.asec_pool.load_asec_h5_tables reads the prebuilt person tables, and microcosm.build.us_runtime.asec_pool._prepare_year_input restores only microcosm.build.us_runtime.asec_census_person_columns.ASEC_CENSUS_PERSON_COLUMNS, which excludes SRVS_VAL." } }, "takes_up_dc_ptc": { @@ -453,7 +453,7 @@ "PUF_other_credits": "Archived puf.py maps federal P08000 to other_credits; the locked PUF lacks state geography and cannot identify a DC credit claimant.", "rejection": "A published aggregate number of claims supplies no tax-unit claimant identity. Replaying a seeded draw, sorting eligible DC units, or otherwise allocating the count would synthesize the boolean values the locked microdata do not report." }, - "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 70-81 and 103-108 passes only the three prebuilt ASEC artifacts and processed PUF via --asec-h5/--puf-h5; experiments/build_j_recert/base_j.summary.json lines 55-75 and 317-318 records their SHA-256 digests." + "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh passes only the three prebuilt ASEC artifacts and processed PUF via its --asec-h5 2024=, 2023=, and 2022= and --puf-h5 arguments; experiments/build_j_recert/base_j.summary.json base_source.sources[].sha256 and puf_sha256 record their SHA-256 digests." } }, "takes_up_early_head_start_if_eligible": { @@ -526,7 +526,7 @@ "funded_enrollment": "Capacity or federally supported slots, not observed occupancy or person identity.", "source": "https://headstart.gov/sites/default/files/pdf/service-snapshot-ehs-2023-2024.pdf" }, - "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-81 passes only the three prebuilt ASEC artifacts and processed PUF via --asec-h5/--puf-h5; experiments/build_j_recert/base_j.summary.json lines 55-75 and 317-318 records their SHA-256 digests. The separately pinned SIPP donor does not observe the required Early Head Start domain." + "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh passes only the three prebuilt ASEC artifacts and processed PUF via its --asec-h5 2024=, 2023=, and 2022= and --puf-h5 arguments; experiments/build_j_recert/base_j.summary.json base_source.sources[].sha256 and puf_sha256 record their SHA-256 digests. The separately pinned SIPP donor does not observe the required Early Head Start domain." } } } diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index 72b56c12a..fb4581e86 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -725,7 +725,7 @@ "nonnegative_outputs": [ "workers_compensation" ], - "notes": "Exact annual person-level WC_VAL carry from archived cps.py lines 1559-1571; code-1 DIS_VAL slots remain excluded from the separate disability_benefits leaf rather than added here. The CPS-only workers_compensation output is replaced on the PUF support half with the archived eight predictors and at most 5,000 ASEC training people (extended_cps.py lines 135-194, 234-248, 639-745, and 1014-1073). Microcosm pins 100 trees and typed person weights as reproducibility hardening." + "notes": "Exact annual person-level WC_VAL carry from archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 1559-1571; code-1 DIS_VAL slots remain excluded from the separate disability_benefits leaf rather than added here. The CPS-only workers_compensation output is replaced on the PUF support half with the archived eight predictors and at most 5,000 ASEC training people (datasets/cps/extended_cps.py lines 135-194, 234-248, 639-745, and 1014-1073). Microcosm pins 100 trees and typed person weights as reproducibility hardening." }, { "stage": "weeks_unemployed_input", @@ -807,7 +807,7 @@ "nonnegative_outputs": [ "weeks_unemployed" ], - "notes": "Restores the final retired eCPS person-year weeks_unemployed input. Archived cps.py lines 1438-1442 carries Census ASEC LKWEEKS directly, mapping only the historical -1 not-in-universe sentinel to zero; the SHA-pinned 2023 ASEC public-use archive supplies the omitted 2022-income-year source rows exactly. Its 146,133 person records match the locked 2022 H5 one-for-one on PH_SEQ, P_SEQ, A_LINENO, A_AGE, A_FNLWGT, UC_VAL, and WKSWORK. The source has 4,200 positive LKWEEKS rows, range 0--51, 3.1456094276% A_FNLWGT-weighted positive support, and 172,563,622.04 weighted person-weeks; Census allocation flags are retained because the retired direct carry did not filter them. Archived calibration/puf_impute.py lines 703-785 replaces only the PUF support half with a QRF using age, sex, three tax-unit roles, joint filing, and unemployment compensation, caps training at 5,000 ASEC people, clips predictions to 0--52, and forces predictions to zero without positive unemployment compensation. Microcosm pins 100 trees and typed person weights as reproducibility hardening, and rounds its interpolating QRF draws back to the source's integer-week domain; no 2022 source value is filled statistically." + "notes": "Restores the final retired eCPS person-year weeks_unemployed input. Archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 1438-1442 carries Census ASEC LKWEEKS directly, mapping only the historical -1 not-in-universe sentinel to zero; the SHA-pinned 2023 ASEC public-use archive supplies the omitted 2022-income-year source rows exactly. Its 146,133 person records match the locked 2022 H5 one-for-one on PH_SEQ, P_SEQ, A_LINENO, A_AGE, A_FNLWGT, UC_VAL, and WKSWORK. The source has 4,200 positive LKWEEKS rows, range 0--51, 3.1456094276% A_FNLWGT-weighted positive support, and 172,563,622.04 weighted person-weeks; Census allocation flags are retained because the retired direct carry did not filter them. Archived calibration/puf_impute.py lines 703-785 replaces only the PUF support half with a QRF using age, sex, three tax-unit roles, joint filing, and unemployment compensation, caps training at 5,000 ASEC people, clips predictions to 0--52, and forces predictions to zero without positive unemployment compensation. Microcosm pins 100 trees and typed person weights as reproducibility hardening, and rounds its interpolating QRF draws back to the source's integer-week domain; no 2022 source value is filled statistically." }, { "stage": "capital_gain_distributions", @@ -1190,7 +1190,7 @@ "bond_assets", "stock_assets" ], - "notes": "Microcosm #374 restores the retired 50/50 SIPP/SCF financial-asset blend: a seeded recipient-household draw selects one complete three-leaf vector, and the reference person carries it while all other people receive zero. The SIPP mapping matches archived commit 42ed5d45's final 13-predictor asset QRF, while the explicitly requested immutable pu2023.csv artifact replaces that commit's later pu2024 default. The full SCF joins all five implicates to the summary extract on (y1, yy1). Archived commit 42ed5d45 utils/asset_imputation.py lines 15-19 maps summary-extract networth directly to the scf_net_worth QRF anchor; calibration/source_impute.py lines 1146-1164 fits that anchor with these eight predictors and calibration/source_impute.py lines 1324-1338 forces its construction-only balance-sheet components to sum back to it before exporting net_worth. Microcosm therefore persists that signed household anchor directly, including indebted households, without manufacturing or exporting the internal scf_* components. Legacy auto_loan_balance preserves the retired pipeline's name for original financed amounts, not remaining principal. Auto outputs and net_worth stay Household-grain; only the three SSI liquid-asset leaves are head-carried to Person. The OBBBA qualifying-interest proxy uses Treasury/IRS's approximately six million qualifying loans issued annually and is bounded by total auto-loan interest. Negative amount/rate codes are floored at the source boundary for the nonnegative auto outputs; signed net_worth is never clipped." + "notes": "Microcosm #374 restores the retired 50/50 SIPP/SCF financial-asset blend: a seeded recipient-household draw selects one complete three-leaf vector, and the reference person carries it while all other people receive zero. The SIPP mapping matches archived commit 42ed5d45's final 13-predictor asset QRF, while the explicitly requested immutable pu2023.csv artifact replaces that commit's later pu2024 default. The full SCF joins all five implicates to the summary extract on (y1, yy1). Archived commit 42ed5d45 utils/asset_imputation.py lines 15-19 maps summary-extract networth directly to the scf_net_worth QRF anchor; calibration/source_impute.py lines 1146-1164 add that anchor to the QRF outputs, lines 1227-1240 fit it with these eight predictors (SCF_PREDICTORS, lines 168-177), and calibration/source_impute.py lines 1324-1338 forces its construction-only balance-sheet components to sum back to it before exporting net_worth. Microcosm therefore persists that signed household anchor directly, including indebted households, without manufacturing or exporting the internal scf_* components. Legacy auto_loan_balance preserves the retired pipeline's name for original financed amounts, not remaining principal. Auto outputs and net_worth stay Household-grain; only the three SSI liquid-asset leaves are head-carried to Person. The OBBBA qualifying-interest proxy uses Treasury/IRS's approximately six million qualifying loans issued annually and is bounded by total auto-loan interest. Negative amount/rate codes are floored at the source boundary for the nonnegative auto outputs; signed net_worth is never clipped." }, { "stage": "ssi_disability_criteria", @@ -1568,7 +1568,7 @@ "outputs": [ "takes_up_ssi_if_eligible" ], - "notes": "Restores the final retired eCPS SSI take-up export from source-backed evidence at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe. datasets/cps/cps.py line 584 maps the direct ASEC SSI_VAL reporter signal, lines 650-657 derive the take-up leaf, and lines 1497-1499 export it; datasets/cps/takeup.py lines 10-35 define the retired stable random assignment. The retired all-age 0.50 rate has no full-population administrative source and is not reused. utils/ssi_targets.py lines 41-74 preserve the archived age-target method and cite SSA SSI Monthly Statistics, December 2024, Table 1, row Total with—Federal payment, by the three published age bands; those recipient counts enter the build only as ledger-fed calibration registry targets (role ssa_ssi_age_band_recipients, microcosm#469 and microcosm#470), never as count-matching goals in this stage. Direct SSI_VAL reporters remain true. Every other source-person identity draws once against the documented band prior — the anchored-mass-corrected count-truthful threshold (band target minus basis reporter floor) over (basis candidate capacity minus basis reporter floor) on weighted uncapped_ssi > 0 candidates, zero once the reporter floor meets the target, falling back to the basis reporter share of capacity when capacity cannot subsample the target — seeded and stable per person_source_id, with the decision fanned identically to both support clones. The capacity basis is this frame's weights by default, or a prior attempt's delivered-weight us_ssi_take_up.json diagnostics (microcosm#507/#508); either way the draw happens exactly once, with no reconcile loop. Delivered recipients on release weights are gated per enforced band (18_64, 65_plus) within a documented relative tolerance; the under-18 band is fenced pending microcosm#453/#509 and its gap ships in the release scorecard." + "notes": "Restores the final retired eCPS SSI take-up export from source-backed evidence at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe. datasets/cps/cps.py lines 1497-1499 map the direct ASEC SSI_VAL reporter signal to the temporary ssi_reported anchor, line 584 loads the retired SSI rate, lines 650-657 derive the take-up leaf from both, and lines 759-766 drop the anchor and save the leaf; datasets/cps/takeup.py lines 10-35 define the retired stable random assignment. The retired all-age 0.50 rate has no full-population administrative source and is not reused. utils/ssi_targets.py lines 41-74 preserve the archived age-target method and cite SSA SSI Monthly Statistics, December 2024, Table 1, row Total with—Federal payment, by the three published age bands; those recipient counts enter the build only as ledger-fed calibration registry targets (role ssa_ssi_age_band_recipients, microcosm#469 and microcosm#470), never as count-matching goals in this stage. Direct SSI_VAL reporters remain true. Every other source-person identity draws once against the documented band prior — the anchored-mass-corrected count-truthful threshold (band target minus basis reporter floor) over (basis candidate capacity minus basis reporter floor) on weighted uncapped_ssi > 0 candidates, zero once the reporter floor meets the target, falling back to the basis reporter share of capacity when capacity cannot subsample the target — seeded and stable per person_source_id, with the decision fanned identically to both support clones. The capacity basis is this frame's weights by default, or a prior attempt's delivered-weight us_ssi_take_up.json diagnostics (microcosm#507/#508); either way the draw happens exactly once, with no reconcile loop. Delivered recipients on release weights are gated per enforced band (18_64, 65_plus) within a documented relative tolerance; the under-18 band is fenced pending microcosm#453/#509 and its gap ships in the release scorecard." }, { "stage": "sipp_tips", @@ -1861,7 +1861,7 @@ "nonnegative_outputs": [ "employment_income_last_year" ], - "notes": "Archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 1680-1783 returns without creating values when no prior-year artifact exists; otherwise it joins adjacent ASEC files on PERIDNUM, requires I_ERNVAL == I_SEVAL == 0 on the prior record, treats only -1/-9999 as unavailable, records availability before falling back to current WSAL_VAL/SEMP_VAL, and preserves signed self-employment losses. After support cloning, datasets/cps/extended_cps.py lines 140-194, 639-745, and 1014-1073 jointly QRF-imputes both earnings leaves onto the PUF half from the exact eight predictors with a 5,000-person cap. Microcosm adds typed design weights. The archived finalizer drops formula-owned employment_income_last_year; the signed self-employment leaf and availability flag persist." + "notes": "Archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 1680-1783 returns without creating values when no prior-year artifact exists; otherwise it joins adjacent ASEC files on PERIDNUM, requires I_ERNVAL == I_SEVAL == 0 on the prior record, treats only -1/-9999 as unavailable, records availability before falling back to current WSAL_VAL/SEMP_VAL, and preserves signed self-employment losses. After support cloning, datasets/cps/extended_cps.py lines 140-194, 234-248, 639-745, and 1014-1073 jointly QRF-imputes both earnings leaves onto the PUF half from the exact eight predictors with a 5,000-person cap. Microcosm adds typed design weights. The archived finalizer drops formula-owned employment_income_last_year; the signed self-employment leaf and availability flag persist." }, { "stage": "immigration_status", @@ -2329,7 +2329,7 @@ "outputs": [ "takes_up_wic_if_eligible" ], - "notes": "The retired eCPS derives PolicyEngine-US wic_category_str and assigns a stable seeded Bernoulli at category-specific rates (archived cps.py lines 684-691, wic_takeup.yaml lines 1-33, and randomness.py lines 5-28). The archived rates exactly match USDA FNS CY2022 average-month coverage: pregnant 45.6%, all postpartum 68.9%, breastfeeding 66.3%, infants 78.4%, and children 46.0%. Microcosm deliberately runs this stage after pregnancy; the retired file computed WIC categories before its later pregnancy seed, which accidentally made the pregnant rate unreachable. The hermetic sources carry no breastfeeding assessment, so breastfeeding remains unavailable and mothers of infants use FNS's all-postpartum rate rather than an invented breastfeeding status. Stable source-identity hashes preserve paired support-clone outcomes. The separate is_wic_at_nutritional_risk input remains a reviewed source-unavailability exclusion." + "notes": "The retired eCPS derives PolicyEngine-US wic_category_str and assigns a stable seeded Bernoulli at category-specific rates (archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/cps/cps.py lines 684-691, parameters/take_up/wic_takeup.yaml lines 1-33, and utils/randomness.py lines 5-28). The archived rates exactly match USDA FNS CY2022 average-month coverage: pregnant 45.6%, all postpartum 68.9%, breastfeeding 66.3%, infants 78.4%, and children 46.0%. Microcosm deliberately runs this stage after pregnancy; the retired file computed WIC categories before its later pregnancy seed, which accidentally made the pregnant rate unreachable. The hermetic sources carry no breastfeeding assessment, so breastfeeding remains unavailable and mothers of infants use FNS's all-postpartum rate rather than an invented breastfeeding status. Stable source-identity hashes preserve paired support-clone outcomes. The separate is_wic_at_nutritional_risk input remains a reviewed source-unavailability exclusion." }, { "stage": "snap_abawd_discretionary_exemption", @@ -2558,7 +2558,7 @@ "nonnegative_outputs": [ "pre_subsidy_rent" ], - "notes": "Exact CPS H_TENURE and SPM carries plus the archived 10-predictor ACS rent QRF. receives_housing_assistance receives the retired second-stage CPS-to-PUF QRF, and takes_up_housing_assistance_if_eligible is kept exactly equal to that measured/imputed receipt anchor. Archived cps.py lines 664-682 preserved reporters before adding eligible nonreporters from a 0.50 draw; parameters/take_up/housing_assistance.yaml lines 12-15 say 0.50 was an ad hoc eCPS benchmark adjustment pending fuller calibration. Archived db/etl_housing_assistance.py lines 35-169 use a live HUD workbook to produce household-grain state counts, not SPM-unit recipient identities; that workbook is not pinned in the hermetic build and allocating its counts to SPM units would synthesize recipients. Microcosm never turns a reporter off and does not add unobserved take-up. Rent and both tenure enums are cloned unchanged." + "notes": "Exact CPS H_TENURE and SPM carries plus the archived 10-predictor ACS rent QRF. receives_housing_assistance receives the retired second-stage CPS-to-PUF QRF, and takes_up_housing_assistance_if_eligible is kept exactly equal to that measured/imputed receipt anchor. Archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 664-682 preserved reporters before adding eligible nonreporters from a 0.50 draw; parameters/take_up/housing_assistance.yaml lines 12-15 say 0.50 was an ad hoc eCPS benchmark adjustment pending fuller calibration. Archived db/etl_housing_assistance.py lines 35-169 use a live HUD workbook to produce household-grain state counts, not SPM-unit recipient identities; that workbook is not pinned in the hermetic build and allocating its counts to SPM units would synthesize recipients. Microcosm never turns a reporter off and does not add unobserved take-up. Rent and both tenure enums are cloned unchanged." }, { "stage": "vehicle_assets", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index e33ffec84..49a2648ac 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf + sha256: 41e8194a6206ff84a287317ebb0fb39a083d2a8811ed013e22f4a21cc1255d18 stage_manifest: version: 1 country: us @@ -825,11 +825,12 @@ stages: - workers_compensation nonnegative_outputs: - workers_compensation - notes: Exact annual person-level WC_VAL carry from archived cps.py lines 1559-1571; code-1 DIS_VAL slots - remain excluded from the separate disability_benefits leaf rather than added here. The CPS-only workers_compensation - output is replaced on the PUF support half with the archived eight predictors and at most 5,000 ASEC - training people (extended_cps.py lines 135-194, 234-248, 639-745, and 1014-1073). Microcosm pins 100 - trees and typed person weights as reproducibility hardening. + notes: Exact annual person-level WC_VAL carry from archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe + datasets/cps/cps.py lines 1559-1571; code-1 DIS_VAL slots remain excluded from the separate disability_benefits + leaf rather than added here. The CPS-only workers_compensation output is replaced on the PUF support + half with the archived eight predictors and at most 5,000 ASEC training people (datasets/cps/extended_cps.py + lines 135-194, 234-248, 639-745, and 1014-1073). Microcosm pins 100 trees and typed person weights + as reproducibility hardening. - stage: weeks_unemployed_input survey: Census CPS ASEC source: https://www.census.gov/programs-surveys/cps.html @@ -907,18 +908,18 @@ stages: - weeks_unemployed nonnegative_outputs: - weeks_unemployed - notes: Restores the final retired eCPS person-year weeks_unemployed input. Archived cps.py lines 1438-1442 - carries Census ASEC LKWEEKS directly, mapping only the historical -1 not-in-universe sentinel to zero; - the SHA-pinned 2023 ASEC public-use archive supplies the omitted 2022-income-year source rows exactly. - Its 146,133 person records match the locked 2022 H5 one-for-one on PH_SEQ, P_SEQ, A_LINENO, A_AGE, - A_FNLWGT, UC_VAL, and WKSWORK. The source has 4,200 positive LKWEEKS rows, range 0--51, 3.1456094276% - A_FNLWGT-weighted positive support, and 172,563,622.04 weighted person-weeks; Census allocation flags - are retained because the retired direct carry did not filter them. Archived calibration/puf_impute.py - lines 703-785 replaces only the PUF support half with a QRF using age, sex, three tax-unit roles, - joint filing, and unemployment compensation, caps training at 5,000 ASEC people, clips predictions - to 0--52, and forces predictions to zero without positive unemployment compensation. Microcosm pins - 100 trees and typed person weights as reproducibility hardening, and rounds its interpolating QRF - draws back to the source's integer-week domain; no 2022 source value is filled statistically. + notes: Restores the final retired eCPS person-year weeks_unemployed input. Archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe + datasets/cps/cps.py lines 1438-1442 carries Census ASEC LKWEEKS directly, mapping only the historical + -1 not-in-universe sentinel to zero; the SHA-pinned 2023 ASEC public-use archive supplies the omitted + 2022-income-year source rows exactly. Its 146,133 person records match the locked 2022 H5 one-for-one + on PH_SEQ, P_SEQ, A_LINENO, A_AGE, A_FNLWGT, UC_VAL, and WKSWORK. The source has 4,200 positive LKWEEKS + rows, range 0--51, 3.1456094276% A_FNLWGT-weighted positive support, and 172,563,622.04 weighted person-weeks; + Census allocation flags are retained because the retired direct carry did not filter them. Archived + calibration/puf_impute.py lines 703-785 replaces only the PUF support half with a QRF using age, sex, + three tax-unit roles, joint filing, and unemployment compensation, caps training at 5,000 ASEC people, + clips predictions to 0--52, and forces predictions to zero without positive unemployment compensation. + Microcosm pins 100 trees and typed person weights as reproducibility hardening, and rounds its interpolating + QRF draws back to the source's integer-week domain; no 2022 source value is filled statistically. - stage: capital_gain_distributions survey: IRS SOI Sales of Capital Assets (TY2015) + Pub 1304 Table 1.4 source: https://www.irs.gov/statistics/soi-tax-stats-sales-of-capital-assets-reported-on-individual-tax-returns @@ -1242,16 +1243,16 @@ stages: the explicitly requested immutable pu2023.csv artifact replaces that commit''s later pu2024 default. The full SCF joins all five implicates to the summary extract on (y1, yy1). Archived commit 42ed5d45 utils/asset_imputation.py lines 15-19 maps summary-extract networth directly to the scf_net_worth - QRF anchor; calibration/source_impute.py lines 1146-1164 fits that anchor with these eight predictors - and calibration/source_impute.py lines 1324-1338 forces its construction-only balance-sheet components - to sum back to it before exporting net_worth. Microcosm therefore persists that signed household anchor - directly, including indebted households, without manufacturing or exporting the internal scf_* components. - Legacy auto_loan_balance preserves the retired pipeline''s name for original financed amounts, not - remaining principal. Auto outputs and net_worth stay Household-grain; only the three SSI liquid-asset - leaves are head-carried to Person. The OBBBA qualifying-interest proxy uses Treasury/IRS''s approximately - six million qualifying loans issued annually and is bounded by total auto-loan interest. Negative - amount/rate codes are floored at the source boundary for the nonnegative auto outputs; signed net_worth - is never clipped.' + QRF anchor; calibration/source_impute.py lines 1146-1164 add that anchor to the QRF outputs, lines + 1227-1240 fit it with these eight predictors (SCF_PREDICTORS, lines 168-177), and calibration/source_impute.py + lines 1324-1338 forces its construction-only balance-sheet components to sum back to it before exporting + net_worth. Microcosm therefore persists that signed household anchor directly, including indebted + households, without manufacturing or exporting the internal scf_* components. Legacy auto_loan_balance + preserves the retired pipeline''s name for original financed amounts, not remaining principal. Auto + outputs and net_worth stay Household-grain; only the three SSI liquid-asset leaves are head-carried + to Person. The OBBBA qualifying-interest proxy uses Treasury/IRS''s approximately six million qualifying + loans issued annually and is bounded by total auto-loan interest. Negative amount/rate codes are floored + at the source boundary for the nonnegative auto outputs; signed net_worth is never clipped.' - stage: ssi_disability_criteria survey: Census SIPP source: https://www.census.gov/programs-surveys/sipp.html @@ -1597,23 +1598,25 @@ stages: outputs: - takes_up_ssi_if_eligible notes: Restores the final retired eCPS SSI take-up export from source-backed evidence at archived commit - 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe. datasets/cps/cps.py line 584 maps the direct ASEC SSI_VAL - reporter signal, lines 650-657 derive the take-up leaf, and lines 1497-1499 export it; datasets/cps/takeup.py - lines 10-35 define the retired stable random assignment. The retired all-age 0.50 rate has no full-population - administrative source and is not reused. utils/ssi_targets.py lines 41-74 preserve the archived age-target - method and cite SSA SSI Monthly Statistics, December 2024, Table 1, row Total with—Federal payment, - by the three published age bands; those recipient counts enter the build only as ledger-fed calibration - registry targets (role ssa_ssi_age_band_recipients, microcosm#469 and microcosm#470), never as count-matching - goals in this stage. Direct SSI_VAL reporters remain true. Every other source-person identity draws - once against the documented band prior — the anchored-mass-corrected count-truthful threshold (band - target minus basis reporter floor) over (basis candidate capacity minus basis reporter floor) on weighted - uncapped_ssi > 0 candidates, zero once the reporter floor meets the target, falling back to the basis - reporter share of capacity when capacity cannot subsample the target — seeded and stable per person_source_id, - with the decision fanned identically to both support clones. The capacity basis is this frame's weights - by default, or a prior attempt's delivered-weight us_ssi_take_up.json diagnostics (microcosm#507/#508); - either way the draw happens exactly once, with no reconcile loop. Delivered recipients on release - weights are gated per enforced band (18_64, 65_plus) within a documented relative tolerance; the under-18 - band is fenced pending microcosm#453/#509 and its gap ships in the release scorecard. + 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe. datasets/cps/cps.py lines 1497-1499 map the direct ASEC + SSI_VAL reporter signal to the temporary ssi_reported anchor, line 584 loads the retired SSI rate, + lines 650-657 derive the take-up leaf from both, and lines 759-766 drop the anchor and save the leaf; + datasets/cps/takeup.py lines 10-35 define the retired stable random assignment. The retired all-age + 0.50 rate has no full-population administrative source and is not reused. utils/ssi_targets.py lines + 41-74 preserve the archived age-target method and cite SSA SSI Monthly Statistics, December 2024, + Table 1, row Total with—Federal payment, by the three published age bands; those recipient counts + enter the build only as ledger-fed calibration registry targets (role ssa_ssi_age_band_recipients, + microcosm#469 and microcosm#470), never as count-matching goals in this stage. Direct SSI_VAL reporters + remain true. Every other source-person identity draws once against the documented band prior — the + anchored-mass-corrected count-truthful threshold (band target minus basis reporter floor) over (basis + candidate capacity minus basis reporter floor) on weighted uncapped_ssi > 0 candidates, zero once + the reporter floor meets the target, falling back to the basis reporter share of capacity when capacity + cannot subsample the target — seeded and stable per person_source_id, with the decision fanned identically + to both support clones. The capacity basis is this frame's weights by default, or a prior attempt's + delivered-weight us_ssi_take_up.json diagnostics (microcosm#507/#508); either way the draw happens + exactly once, with no reconcile loop. Delivered recipients on release weights are gated per enforced + band (18_64, 65_plus) within a documented relative tolerance; the under-18 band is fenced pending + microcosm#453/#509 and its gap ships in the release scorecard. - stage: sipp_tips survey: Census SIPP source: https://www.census.gov/programs-surveys/sipp.html @@ -1879,9 +1882,10 @@ stages: files on PERIDNUM, requires I_ERNVAL == I_SEVAL == 0 on the prior record, treats only -1/-9999 as unavailable, records availability before falling back to current WSAL_VAL/SEMP_VAL, and preserves signed self-employment losses. After support cloning, datasets/cps/extended_cps.py lines 140-194, - 639-745, and 1014-1073 jointly QRF-imputes both earnings leaves onto the PUF half from the exact eight - predictors with a 5,000-person cap. Microcosm adds typed design weights. The archived finalizer drops - formula-owned employment_income_last_year; the signed self-employment leaf and availability flag persist. + 234-248, 639-745, and 1014-1073 jointly QRF-imputes both earnings leaves onto the PUF half from the + exact eight predictors with a 5,000-person cap. Microcosm adds typed design weights. The archived + finalizer drops formula-owned employment_income_last_year; the signed self-employment leaf and availability + flag persist. - stage: immigration_status survey: CPS ASEC + published unauthorized-population estimates source: https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/ @@ -2330,14 +2334,15 @@ stages: outputs: - takes_up_wic_if_eligible notes: 'The retired eCPS derives PolicyEngine-US wic_category_str and assigns a stable seeded Bernoulli - at category-specific rates (archived cps.py lines 684-691, wic_takeup.yaml lines 1-33, and randomness.py - lines 5-28). The archived rates exactly match USDA FNS CY2022 average-month coverage: pregnant 45.6%, - all postpartum 68.9%, breastfeeding 66.3%, infants 78.4%, and children 46.0%. Microcosm deliberately - runs this stage after pregnancy; the retired file computed WIC categories before its later pregnancy - seed, which accidentally made the pregnant rate unreachable. The hermetic sources carry no breastfeeding - assessment, so breastfeeding remains unavailable and mothers of infants use FNS''s all-postpartum - rate rather than an invented breastfeeding status. Stable source-identity hashes preserve paired support-clone - outcomes. The separate is_wic_at_nutritional_risk input remains a reviewed source-unavailability exclusion.' + at category-specific rates (archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/cps/cps.py + lines 684-691, parameters/take_up/wic_takeup.yaml lines 1-33, and utils/randomness.py lines 5-28). + The archived rates exactly match USDA FNS CY2022 average-month coverage: pregnant 45.6%, all postpartum + 68.9%, breastfeeding 66.3%, infants 78.4%, and children 46.0%. Microcosm deliberately runs this stage + after pregnancy; the retired file computed WIC categories before its later pregnancy seed, which accidentally + made the pregnant rate unreachable. The hermetic sources carry no breastfeeding assessment, so breastfeeding + remains unavailable and mothers of infants use FNS''s all-postpartum rate rather than an invented + breastfeeding status. Stable source-identity hashes preserve paired support-clone outcomes. The separate + is_wic_at_nutritional_risk input remains a reviewed source-unavailability exclusion.' - stage: snap_abawd_discretionary_exemption survey: Census CPS ASEC + statutory exemption cap (7 U.S.C. 2015(o)(6)) source: https://www.law.cornell.edu/uscode/text/7/2015#o_6 @@ -2560,13 +2565,14 @@ stages: - pre_subsidy_rent notes: Exact CPS H_TENURE and SPM carries plus the archived 10-predictor ACS rent QRF. receives_housing_assistance receives the retired second-stage CPS-to-PUF QRF, and takes_up_housing_assistance_if_eligible is kept - exactly equal to that measured/imputed receipt anchor. Archived cps.py lines 664-682 preserved reporters - before adding eligible nonreporters from a 0.50 draw; parameters/take_up/housing_assistance.yaml lines - 12-15 say 0.50 was an ad hoc eCPS benchmark adjustment pending fuller calibration. Archived db/etl_housing_assistance.py - lines 35-169 use a live HUD workbook to produce household-grain state counts, not SPM-unit recipient - identities; that workbook is not pinned in the hermetic build and allocating its counts to SPM units - would synthesize recipients. Microcosm never turns a reporter off and does not add unobserved take-up. - Rent and both tenure enums are cloned unchanged. + exactly equal to that measured/imputed receipt anchor. Archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe + datasets/cps/cps.py lines 664-682 preserved reporters before adding eligible nonreporters from a 0.50 + draw; parameters/take_up/housing_assistance.yaml lines 12-15 say 0.50 was an ad hoc eCPS benchmark + adjustment pending fuller calibration. Archived db/etl_housing_assistance.py lines 35-169 use a live + HUD workbook to produce household-grain state counts, not SPM-unit recipient identities; that workbook + is not pinned in the hermetic build and allocating its counts to SPM units would synthesize recipients. + Microcosm never turns a reporter off and does not add unobserved take-up. Rent and both tenure enums + are cloned unchanged. - stage: vehicle_assets survey: Census SIPP source: https://www.census.gov/programs-surveys/sipp.html diff --git a/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json b/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json index 904f8bf51..f8968caee 100644 --- a/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json @@ -195,7 +195,7 @@ "evidence": "irs_soi.historic_table_2 (compiled) carries the tax-return interest; sample fact bea_nipa.cy2023.personal_interest_income.a064rc.amount", "fence": { "origin": "us-data PR #994 (added) -> PR #1059 (removed)", - "purpose": "\"BEA personal interest/dividends include imputed interest, pension-plan dividends, and trust flows, so those macro totals should not directly calibrate tax/CPS interest and dividend variables\" (loss.py:67-70, added by PR #1059)", + "purpose": "\"BEA personal interest/dividends include imputed interest, pension-plan dividends, and trust flows, so those macro totals should not directly calibrate tax/CPS interest and dividend variables\" (utils/loss.py:67-70 at 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, added by PR #1059)", "verdict_basis": "us-data itself declined it as a direct target (PR #1059): a macro benchmark, not a close microdata concept. Tax-return interest is calibrated from IRS SOI + CBO (compiled)." } }, diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py new file mode 100644 index 000000000..5460cd312 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -0,0 +1,633 @@ +"""Country spec packages cite sources by symbol or at a commit, never drifting. + +A line number names text only at a fixed commit, so a citation such as +``asec_pool.py lines 61-75`` silently points at other code after an unrelated +edit. The validators live in ``test_support.microcosm_build. +spec_source_citations``; these tests state the invariants over every country +spec package (every build directory with a ``country_package.json``) and check +each refusal branch with Hypothesis, so a vacuous pass on today's data cannot +hide a broken check: + +- no resource cites a line of a file in this repository, pinned or not + (``in_repository_line_citations``): this repository's sources are cited by + symbol; +- no resource cites a line of any other source unless the citation's own + string names a commit, or the resource pins that file in a + ``path``/``commit`` mapping (``unpinned_line_citations``); +- every dotted ``microcosm.*`` name in a resource resolves through import + plus getattr (``resolve_dotted_symbol``). + +Differential checks compare two records of one fact: the UK citations' commits +with the incumbent commit each resource declares, and the US known gaps' +symbol citations with the code they name (the Census person-column restore +list, the relationship fallback, and the PUF support tool's functions). +""" + +from __future__ import annotations + +import ast +import inspect + +import pandas as pd +import pytest +from hypothesis import assume, given, settings +from hypothesis import strategies as st + +from microcosm.build.us_runtime import asec_pool +from microcosm.build.us_runtime.asec_census_person_columns import ( + ASEC_CENSUS_PERSON_COLUMN_NAMES, + ASEC_CENSUS_PERSON_COLUMNS, +) +from test_support.microcosm_build.spec_source_citations import ( + COUNTRY_PACKAGE_ROOT, + REPOSITORY_ROOT, + LineCitation, + commit_pinned_sources, + country_packages, + dotted_microcosm_symbols, + in_repository_line_citations, + line_citations, + named_commits, + names_a_commit, + package_payloads, + resolve_dotted_symbol, + resolves_in_repository, + unpinned_line_citations, +) + +PACKAGES = ("am", "be", "nz", "uk", "us") + + +# --------------------------------------------------------------------------- +# Committed resources +# --------------------------------------------------------------------------- + + +def test_every_country_package_is_scanned() -> None: + assert tuple(country_packages()) == PACKAGES + for country in PACKAGES: + assert package_payloads(country) + + +@pytest.mark.parametrize("country", PACKAGES) +def test_no_resource_cites_a_line_it_does_not_pin(country: str) -> None: + offenders = { + name: hits + for name, payload in package_payloads(country).items() + if (hits := unpinned_line_citations(payload)) + } + assert offenders == {} + + +@pytest.mark.parametrize("country", PACKAGES) +def test_no_resource_cites_a_line_of_this_repository(country: str) -> None: + offenders = { + name: hits + for name, payload in package_payloads(country).items() + if (hits := in_repository_line_citations(payload)) + } + assert offenders == {} + + +@pytest.mark.parametrize( + ("country", "resource", "cited"), + [ + ("nz", "as_rate_bridge.json", {"run.py"}), + ("uk", "uk_data_target_parity.json", {"obr.py", "voa_council_tax.py"}), + ("uk", "uk_population_targets.json", {"income.py", "hmrc_salary_sacrifice.py"}), + ("us", "source_stages.json", {"cps.py", "census_cps.py", "sipp.py"}), + ("us", "spec/sources.yaml", {"cps.py", "census_cps.py", "sipp.py"}), + ("us", "ecps_parity_known_gaps.json", {"cps.py", "census_cps.py"}), + ("us", "target_parity_manifest.json", {"loss.py"}), + ], +) +def test_the_pinned_line_citations_are_scanned( + country: str, resource: str, cited: set[str] +) -> None: + payload = package_payloads(country)[resource] + assert cited <= {citation.name for citation in line_citations(payload)} + + +@pytest.mark.parametrize("country", PACKAGES) +def test_every_dotted_microcosm_symbol_resolves(country: str) -> None: + named = set().union( + *map(dotted_microcosm_symbols, package_payloads(country).values()) + ) + unresolved = [] + for dotted in sorted(named): + try: + resolve_dotted_symbol(dotted) + except (ImportError, AttributeError) as error: + unresolved.append(f"{dotted}: {error}") + assert unresolved == [] + + +@pytest.mark.parametrize( + ("dotted", "error"), + [ + ("microcosm.graph.executor._no_such_symbol", AttributeError), + ("microcosm.no_such_shard.module.symbol", ModuleNotFoundError), + ], +) +def test_an_unresolvable_dotted_symbol_raises( + dotted: str, error: type[Exception] +) -> None: + with pytest.raises(error): + resolve_dotted_symbol(dotted) + + +# --------------------------------------------------------------------------- +# Validator properties +# --------------------------------------------------------------------------- + + +_STEM = st.from_regex(r"[a-z_][a-z0-9_]{0,15}", fullmatch=True) +_SUFFIX = st.sampled_from(("py", "yaml", "json", "sh", "csv", "toml")) +#: Directories that do not exist in this checkout, so a path under one is +#: another repository's (an empty directory leaves a bare file name). +_FOREIGN_DIRECTORY = st.sampled_from( + ( + "", + "archived_data/datasets/cps/", + "targets/sources/", + "nz-lane/emtr_reproduction/", + ) +) +#: Line-locator spellings: {file} is the cited source, {a} and {b} lines. +_LINE_LOCATORS = ( + "{file}:{a}", + "{file}:{a}-{b}", + "{file}:{a},{b}", + "{file}#L{a}", + "{file}#L{a}-L{b}", + "{file} L{a}", + "{file} line {a}", + "{file} lines {a}-{b}", + "{file}, lines {a}-{b}", + "{file} (lines {a}-{b})", + "Lines {a}-{b} of {file}", + "line {a} in {file}", +) +#: Symbol-citation spellings, which carry no line number. +_SYMBOL_LOCATORS = ( + "{file} {symbol}", + '{file} _pointer("{symbol}")', + '{file}, the "# --- Relationships" block', + "{file}: {symbol}", + "{symbol} in {file} at main", +) +#: Inline-commit spellings. +_INLINE_PINS = ( + "commit {sha}", + "Commit {sha}", + "at {sha}", + "At {sha}", + "uk-data@{sha}", + "https://github.com/PolicyEngine/archived-data/blob/{sha}/x", +) +#: Prose around a citation. It has no digit, so it can neither extend a +#: planted line number nor name a commit of its own. +_PROSE = st.text( + alphabet="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ,;()'\"-_", + max_size=40, +) +_HEX = st.text(alphabet="0123456789abcdef", min_size=7, max_size=40) +_BRANCH_NAMES = ("main", "HEAD", "launch/ga-path-execution-gate", "ba6e7d", "") +#: Files in this checkout, in each path form a citation can use. +_IN_REPOSITORY_FILES = ( + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py", + "microcosm-frame/src/microcosm/frame/concepts.py", + "microcosm/graph/executor.py", + "tools/build_us_puf_support_base.py", + "experiments/build_j_recert/buildj_base.sh", + "experiments/build_j_recert/base_j.summary.json", +) + + +def _plant(text: str, depth: int, as_key: bool) -> object: + planted: object = {text: "cited"} if as_key else text + for level in range(depth): + planted = {"level": level, "note": [planted]} + return planted + + +def _is_commit_like(sha: str) -> bool: + return any(c.isdigit() for c in sha) and any(c.isalpha() for c in sha) + + +@settings(max_examples=300, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + directory=_FOREIGN_DIRECTORY, + locator=st.sampled_from(_LINE_LOCATORS), + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + prefix=_PROSE, + tail=_PROSE, + depth=st.integers(0, 3), + as_key=st.booleans(), +) +def test_an_unpinned_line_citation_is_refused_at_any_depth( + stem: str, + suffix: str, + directory: str, + locator: str, + a: int, + b: int, + prefix: str, + tail: str, + depth: int, + as_key: bool, +) -> None: + file = f"{directory}{stem}.{suffix}" + text = f"{prefix} {locator.format(file=file, a=a, b=b)} {tail}" + hits = unpinned_line_citations(_plant(text, depth, as_key)) + assert LineCitation(file, text) in hits + assert not resolves_in_repository(file) + + +@settings(max_examples=200, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + directory=_FOREIGN_DIRECTORY, + pin_directory=_FOREIGN_DIRECTORY, + locator=st.sampled_from(_LINE_LOCATORS), + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + depth=st.integers(0, 3), + as_key=st.booleans(), + commit=_HEX, +) +def test_a_line_citation_of_a_file_pinned_at_a_commit_passes( + stem: str, + suffix: str, + directory: str, + pin_directory: str, + locator: str, + a: int, + b: int, + depth: int, + as_key: bool, + commit: str, +) -> None: + citation = locator.format(file=f"{directory}{stem}.{suffix}", a=a, b=b) + document = { + "source": { + "repository": "TheAxiomFoundation/ops", + "path": f"{pin_directory}{stem}.{suffix}", + "commit": commit, + }, + "notes": [_plant(citation, depth, as_key)], + } + assert line_citations(document) + assert unpinned_line_citations(document) == [] + + +@settings(max_examples=300, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + directory=_FOREIGN_DIRECTORY, + locator=st.sampled_from(_LINE_LOCATORS), + pin=st.sampled_from(_INLINE_PINS), + sha=_HEX, + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + prefix=_PROSE, + middle=_PROSE, + tail=_PROSE, + pin_first=st.booleans(), + depth=st.integers(0, 3), + as_key=st.booleans(), +) +def test_a_commit_named_in_the_citing_string_pins_it( + stem: str, + suffix: str, + directory: str, + locator: str, + pin: str, + sha: str, + a: int, + b: int, + prefix: str, + middle: str, + tail: str, + pin_first: bool, + depth: int, + as_key: bool, +) -> None: + assume(_is_commit_like(sha)) + citation = locator.format(file=f"{directory}{stem}.{suffix}", a=a, b=b) + named = pin.format(sha=sha) + first, second = (named, citation) if pin_first else (citation, named) + text = f"{prefix} {first} {middle} {second} {tail}" + document = _plant(text, depth, as_key) + assert names_a_commit(text) + assert line_citations(document) + assert unpinned_line_citations(document) == [] + + +@settings(max_examples=200, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + locator=st.sampled_from(_LINE_LOCATORS), + pin=st.sampled_from(_INLINE_PINS), + sha=_HEX, + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), +) +def test_a_commit_named_in_another_string_does_not_pin_a_citation( + stem: str, + suffix: str, + locator: str, + pin: str, + sha: str, + a: int, + b: int, +) -> None: + assume(_is_commit_like(sha)) + citation = locator.format(file=f"{stem}.{suffix}", a=a, b=b) + document = {"provenance": pin.format(sha=sha), "note": citation} + assert LineCitation(f"{stem}.{suffix}", citation) in unpinned_line_citations( + document + ) + + +@settings(max_examples=200, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + locator=st.sampled_from(_LINE_LOCATORS), + keyword=st.sampled_from(("commit ", "at ", "At ", "@")), + not_a_commit=st.one_of( + st.text(alphabet="0123456789", min_size=7, max_size=40), + st.text(alphabet="abcdef", min_size=7, max_size=40), + st.text(alphabet="0123456789abcdef", min_size=1, max_size=6), + st.text(alphabet="0123456789abcdef", min_size=41, max_size=64), + st.sampled_from(_BRANCH_NAMES), + ), + a=st.integers(1, 99_999), +) +def test_a_decimal_a_word_a_digest_or_a_branch_names_no_commit( + stem: str, + suffix: str, + locator: str, + keyword: str, + not_a_commit: str, + a: int, +) -> None: + citation = locator.format(file=f"{stem}.{suffix}", a=a, b=a) + text = f"{citation} {keyword}{not_a_commit}" + assert not names_a_commit(f"{keyword}{not_a_commit}") + assert LineCitation(f"{stem}.{suffix}", text) in unpinned_line_citations( + {"note": text} + ) + + +@settings(max_examples=200, deadline=None) +@given( + stem=_STEM, + other=_STEM, + suffix=_SUFFIX, + locator=st.sampled_from(_LINE_LOCATORS), + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + commit=_HEX, + branch=st.sampled_from(_BRANCH_NAMES), + pin_other_file=st.booleans(), +) +def test_a_pin_of_another_file_or_at_a_branch_does_not_cover_a_citation( + stem: str, + other: str, + suffix: str, + locator: str, + a: int, + b: int, + commit: str, + branch: str, + pin_other_file: bool, +) -> None: + if pin_other_file: + assume(other != stem) + pin = {"path": f"{other}.{suffix}", "commit": commit} + else: + pin = {"path": f"{stem}.{suffix}", "commit": branch} + citation = locator.format(file=f"{stem}.{suffix}", a=a, b=b) + hits = unpinned_line_citations({"source": pin, "note": citation}) + assert LineCitation(f"{stem}.{suffix}", citation) in hits + assert commit_pinned_sources({"source": pin}).isdisjoint({f"{stem}.{suffix}"}) + + +@settings(max_examples=200, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + directory=_FOREIGN_DIRECTORY, + locator=st.sampled_from(_SYMBOL_LOCATORS), + symbol=_STEM, + prefix=_PROSE, + tail=_PROSE, +) +def test_a_symbol_citation_is_not_a_line_citation( + stem: str, + suffix: str, + directory: str, + locator: str, + symbol: str, + prefix: str, + tail: str, +) -> None: + citation = locator.format(file=f"{directory}{stem}.{suffix}", symbol=symbol) + assert line_citations({"rule": f"{prefix} {citation} {tail}"}) == [] + + +@settings(max_examples=200, deadline=None) +@given( + file=st.sampled_from(_IN_REPOSITORY_FILES), + locator=st.sampled_from(_LINE_LOCATORS), + pin=st.sampled_from(_INLINE_PINS), + sha=_HEX, + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + pinned_inline=st.booleans(), + pinned_by_path=st.booleans(), +) +def test_a_line_citation_of_this_repository_is_refused_even_when_pinned( + file: str, + locator: str, + pin: str, + sha: str, + a: int, + b: int, + pinned_inline: bool, + pinned_by_path: bool, +) -> None: + assume(_is_commit_like(sha)) + text = locator.format(file=file, a=a, b=b) + if pinned_inline: + text = f"{text} {pin.format(sha=sha)}" + document: dict[str, object] = {"note": text} + if pinned_by_path: + document["source"] = {"path": file, "commit": sha} + assert resolves_in_repository(file) + assert LineCitation(file, text) in in_repository_line_citations(document) + + +def test_in_repository_paths_resolve_from_the_root_packages_or_a_shard() -> None: + for file in _IN_REPOSITORY_FILES: + assert resolves_in_repository(file), file + for file in ( + "concepts.py", + "archived_data/datasets/cps/cps.py", + "targets/sources/obr.py", + "../microcosm/tools/build_us_puf_support_base.py", + "/tools/build_us_puf_support_base.py", + "microcosm/graph", + ): + assert not resolves_in_repository(file), file + assert (REPOSITORY_ROOT / _IN_REPOSITORY_FILES[0]).is_file() + assert COUNTRY_PACKAGE_ROOT.is_dir() + + +@settings(max_examples=200, deadline=None) +@given( + pin=st.sampled_from(_INLINE_PINS), + sha=_HEX, + prefix=_PROSE, + tail=_PROSE, +) +def test_named_commits_returns_the_bare_sha( + pin: str, sha: str, prefix: str, tail: str +) -> None: + assume(_is_commit_like(sha)) + assert named_commits(f"{prefix} {pin.format(sha=sha)} {tail}") == [sha] + + +# --------------------------------------------------------------------------- +# Differential: citations against the records and code they name +# --------------------------------------------------------------------------- + + +def _line_citing_commits(payload: object) -> set[str]: + texts = {citation.text for citation in line_citations(payload)} + assert texts + return {commit for text in texts for commit in named_commits(text)} + + +def test_uk_data_parity_citations_name_the_incumbent_commit() -> None: + parity = package_payloads("uk")["uk_data_target_parity.json"] + incumbent = parity["reference"]["incumbent_commit"] + commits = _line_citing_commits(parity) + assert commits == {"8629dbbfe82727278d47ee5ff09fd1da4acefa38"} + assert all(commit.startswith(incumbent) for commit in commits) + + +def test_uk_population_citations_name_the_registry_pinned_ref() -> None: + contract = package_payloads("uk")["uk_population_targets.json"] + pinned_ref = contract["registry_parity"]["pinned_ref"] + commits = _line_citing_commits(contract) + assert pinned_ref in commits + assert all(pinned_ref.startswith(commit) for commit in commits) + + +def _known_gaps() -> dict[str, dict[str, object]]: + return package_payloads("us")["ecps_parity_known_gaps.json"]["known_gaps"] + + +#: Each US known gap whose build contract names Census person columns that +#: the reviewed restore leaves out. +_UNRESTORED_GAP_COLUMNS = { + "employer_sponsored_insurance_premiums": ( + "NOW_OWNGRP", + "NOW_HIPAID", + "NOW_GRPFTYP", + ), + "financial_assistance": ("FIN_VAL", "FIN_YN", "I_FINVAL"), + "is_unmarried_partner_of_household_head": ("PERRP", "PECOHAB", "A_FAMREL"), + "survivor_benefits": ("SRVS_VAL",), +} + + +@pytest.mark.parametrize(("gap", "columns"), sorted(_UNRESTORED_GAP_COLUMNS.items())) +def test_a_known_gap_names_columns_the_census_restore_leaves_out( + gap: str, columns: tuple[str, ...] +) -> None: + contract = _known_gaps()[gap]["evidence"]["hermetic_build_contract"] + assert "microcosm.build.us_runtime.asec_pool._prepare_year_input" in contract + for column in columns: + assert column in contract + assert column not in ASEC_CENSUS_PERSON_COLUMN_NAMES + + +def test_the_restore_runs_before_the_relationship_fallback() -> None: + contract = _known_gaps()["is_unmarried_partner_of_household_head"]["evidence"][ + "hermetic_build_contract" + ] + assert "restores the 2022 and 2023 inputs' missing A_EXPRRP" in contract + (exprrp,) = [ + column for column in ASEC_CENSUS_PERSON_COLUMNS if column.name == "A_EXPRRP" + ] + assert exprrp.domain is not None and 13 in exprrp.domain + source = inspect.getsource(asec_pool._prepare_year_input) + restore = source.index("restore_asec_census_person_columns(") + assert restore < source.index("_with_relationship_recode(person)") + + +@settings(max_examples=100, deadline=None) +@given( + households=st.lists( + st.lists( + st.tuples( + st.integers(0, 3), + st.integers(1, 2), + st.integers(0, 3), + st.integers(0, 3), + ), + min_size=1, + max_size=4, + ), + min_size=1, + max_size=4, + ) +) +def test_the_relationship_fallback_never_emits_code_13( + households: list[list[tuple[int, int, int, int]]], +) -> None: + rows = [ + { + "PH_SEQ": household, + "A_LINENO": line, + "A_SPOUSE": spouse, + "A_SEX": sex, + "PEPAR1": parent1, + "PEPAR2": parent2, + } + for household, members in enumerate(households, start=1) + for line, (spouse, sex, parent1, parent2) in enumerate(members, start=1) + ] + person, source = asec_pool._with_relationship_recode(pd.DataFrame(rows)) + assert source == "derived:line_spouse_parent" + assert 13 not in set(person["A_EXPRRP"]) + with_source = person.assign(A_EXPRRP=13) + kept, kept_source = asec_pool._with_relationship_recode(with_source) + assert kept is with_source + assert kept_source == "source:A_EXPRRP" + + +def test_the_known_gaps_name_functions_the_puf_support_tool_defines() -> None: + tool = REPOSITORY_ROOT / "tools/build_us_puf_support_base.py" + module = ast.parse(tool.read_text(encoding="utf-8")) + defined = { + node.name: node for node in module.body if isinstance(node, ast.FunctionDef) + } + named = ("_parse_args", "_pooled_asec_sources_from_args", "_asec_sources_from_args") + for gap in ("employer_sponsored_insurance_premiums", "financial_assistance"): + contract = _known_gaps()[gap]["evidence"]["hermetic_build_contract"] + assert all(name in contract for name in named), gap + assert set(named) <= set(defined) + assert '"--asec-h5"' in ast.get_source_segment( + tool.read_text(encoding="utf-8"), defined["_parse_args"] + ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py index a780c4814..ddd18faa5 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py @@ -25,7 +25,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf", + "sha256": "41e8194a6206ff84a287317ebb0fb39a083d2a8811ed013e22f4a21cc1255d18", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_financial_assistance_exclusion.py b/packages/microcosm-build/tests/engine_free/us/test_us_financial_assistance_exclusion.py index f37819fd1..73e8b3352 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_financial_assistance_exclusion.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_financial_assistance_exclusion.py @@ -81,8 +81,17 @@ def test_exclusion_pins_all_sha_locked_hermetic_inputs_and_grain() -> None: build_script = (ROOT / "experiments/build_j_recert/buildj_base.sh").read_text() for year in (2022, 2023, 2024): assert f'--asec-h5 {year}="$USD/census_cps_{year}.h5"' in build_script - assert "buildj_base.sh lines 65-69" in evidence["hermetic_build_contract"] - assert "base_j.summary.json lines 55-75" in evidence["hermetic_build_contract"] + contract = evidence["hermetic_build_contract"] + assert "--asec-h5 2024=, 2023=, and 2022= arguments" in contract + assert "base_source.sources[].sha256" in contract + summary = json.loads( + (ROOT / "experiments/build_j_recert/base_j.summary.json").read_text() + ) + assert [len(source["sha256"]) for source in summary["base_source"]["sources"]] == [ + 64, + 64, + 64, + ] asec_pool = ( ROOT / "packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py" ).read_text() diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_other_health_insurance.py b/packages/microcosm-build/tests/engine_free/us/test_us_other_health_insurance.py index bd3d3c228..5cdd38164 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_other_health_insurance.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_other_health_insurance.py @@ -607,8 +607,17 @@ def test_esi_exclusion_pins_complete_hermetic_source_unavailability_evidence() - "NOW_HIPAID", "NOW_GRPFTYP", ] - assert "buildj_base.sh lines 65-69" in evidence["hermetic_build_contract"] - assert "base_j.summary.json lines 55-75" in evidence["hermetic_build_contract"] + contract = evidence["hermetic_build_contract"] + assert "--asec-h5 2024=, 2023=, and 2022= arguments" in contract + assert "base_source.sources[].sha256" in contract + summary = json.loads( + (ROOT / "experiments/build_j_recert/base_j.summary.json").read_text() + ) + assert [len(source["sha256"]) for source in summary["base_source"]["sources"]] == [ + 64, + 64, + 64, + ] build_script = (ROOT / "experiments/build_j_recert/buildj_base.sh").read_text() for year in (2022, 2023, 2024): assert f'--asec-h5 {year}="$USD/census_cps_{year}.h5"' in build_script diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_survivor_benefits_exclusion.py b/packages/microcosm-build/tests/engine_free/us/test_us_survivor_benefits_exclusion.py index 03a12b4a2..725fdd28f 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_survivor_benefits_exclusion.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_survivor_benefits_exclusion.py @@ -77,8 +77,17 @@ def test_exclusion_pins_all_sha_locked_hermetic_inputs() -> None: build_script = (ROOT / "experiments/build_j_recert/buildj_base.sh").read_text() for year in (2022, 2023, 2024): assert f'--asec-h5 {year}="$USD/census_cps_{year}.h5"' in build_script - assert "buildj_base.sh lines 65-69" in evidence["hermetic_build_contract"] - assert "base_j.summary.json lines 55-75" in evidence["hermetic_build_contract"] + contract = evidence["hermetic_build_contract"] + assert "--asec-h5 2024=, 2023=, and 2022= arguments" in contract + assert "base_source.sources[].sha256" in contract + summary = json.loads( + (ROOT / "experiments/build_j_recert/base_j.summary.json").read_text() + ) + assert [len(source["sha256"]) for source in summary["base_source"]["sources"]] == [ + 64, + 64, + 64, + ] def test_generated_release_manifest_preserves_the_evidenced_exclusion() -> None: diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_wic_nutritional_risk_exclusion.py b/packages/microcosm-build/tests/engine_free/us/test_us_wic_nutritional_risk_exclusion.py index dfc90e49e..832d82b0c 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_wic_nutritional_risk_exclusion.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_wic_nutritional_risk_exclusion.py @@ -123,8 +123,17 @@ def test_exclusion_pins_all_sha_locked_hermetic_inputs() -> None: build_script = (ROOT / "experiments/build_j_recert/buildj_base.sh").read_text() for year in (2022, 2023, 2024): assert f'--asec-h5 {year}="$USD/census_cps_{year}.h5"' in build_script - assert "buildj_base.sh lines 65-69" in evidence["hermetic_build_contract"] - assert "base_j.summary.json lines 55-75" in evidence["hermetic_build_contract"] + contract = evidence["hermetic_build_contract"] + assert "--asec-h5 2024=, 2023=, and 2022= arguments" in contract + assert "base_source.sources[].sha256" in contract + summary = json.loads( + (ROOT / "experiments/build_j_recert/base_j.summary.json").read_text() + ) + assert [len(source["sha256"]) for source in summary["base_source"]["sources"]] == [ + 64, + 64, + 64, + ] asec_pool = ( ROOT / "packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py" ).read_text() diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index ca014679b..d32e5b2a0 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -338,7 +338,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "d1df6b316e4937e809f5f020ea2d3a7bd38bccc1845f3ae6e104f7eed6a73323", + "spec_sha256": "5df3ce86a59c61b45ca76cb90cf87aa80818d89fd7f835440a7dafb3f66ff578", }, } diff --git a/test_support/microcosm_build/spec_source_citations.py b/test_support/microcosm_build/spec_source_citations.py new file mode 100644 index 000000000..4aae81650 --- /dev/null +++ b/test_support/microcosm_build/spec_source_citations.py @@ -0,0 +1,243 @@ +"""Source-citation validators shared by the country spec package tests. + +A line number names text only at a fixed commit. A country spec resource +therefore cites this repository's own sources by symbol (a dotted +``microcosm.*`` name, a function or class name, a ``# ---`` section header, +or a key or flag name), because their line numbers move under unrelated +edits. It cites another repository's source by line only when the citation +is pinned at a commit, in one of two ways: + +- the same string names the commit inline: ``commit ``, ``at ``, + ``@`` or a GitHub ``/blob//`` permalink, where ```` is 7-40 + lowercase hex digits with at least one digit and one letter, so a decimal + amount ("at 2500000") or a word ("at defaced") pins nothing; +- the resource holds a mapping, at any depth, whose ``path`` names the file + and whose ``commit`` is 7-40 hex digits (``source``-style pins, matched by + file name). + +An inline commit pins every citation in its string. A file that resolves in +this checkout is in-repository, and a line citation of it is refused even +when pinned; a bare file name cannot be resolved, so it must be pinned. +""" + +from __future__ import annotations + +import importlib +import json +import re +from collections.abc import Iterator, Mapping +from pathlib import Path, PurePosixPath +from typing import NamedTuple + +from microcosm.build.spec_engine import load_yaml12 +from test_support.paths import REPOSITORY_ROOT + +#: Every country spec package: a build directory with a country_package.json. +COUNTRY_PACKAGE_ROOT = REPOSITORY_ROOT / "packages/microcosm-build/src/microcosm/build" + +#: Text source kinds a line citation can name. +_SOURCE_SUFFIXES = ( + "py|pyi|ipynb|yaml|yml|jsonl|json|toml|cfg|ini|sh|md|rst|txt|csv|sql" + "|rs|tsx|ts|mjs|js" +) +#: A source file named in prose: ``concepts.py``, or a path ending in one. +_SOURCE = ( + r"(?P[A-Za-z0-9_./-]*[A-Za-z0-9_]\.(?:" + _SOURCE_SUFFIXES + r"))" + r"(?![A-Za-z0-9_])" +) +#: A line locator into a source, in each spelling the packages have used or +#: GitHub produces: ``x.py:12``, ``x.py:12-19``, ``x.py:12,19``, ``x.py#L12``, +#: ``x.py L12``, ``x.py line 12``, ``x.py, lines 12-19``, +#: ``x.py (lines 12-19)``, and the reverse ``lines 12-19 of x.py`` and +#: ``line 12 in x.py``. +_LINE_CITATIONS = ( + re.compile(_SOURCE + r"(?::|#L|\s+L|,?\s*\(?\s*[Ll]ines?\s+)\d"), + re.compile( + r"\b[Ll]ines?\s+\d+(?:\s*[-–]\s*\d+)?(?:\s*,\s*\d+)*\s+(?:of|in)\s+" + _SOURCE + ), +) +#: A commit pin's value in a ``path``/``commit`` mapping. A branch name such +#: as ``main`` pins nothing. +_COMMIT_PIN = re.compile(r"[0-9a-fA-F]{7,40}") +#: A commit named inline. The hex needs a digit and a letter, and must not +#: run on into a file name (``at a1b2c3d.py:12`` names a file, not a commit). +INLINE_COMMIT = re.compile( + r"(?:(?i:\bcommit\s+|\bat\s+)|@|/(?:blob|tree)/)" + r"(?=[0-9a-f]*[0-9])(?=[0-9a-f]*[a-f])[0-9a-f]{7,40}" + r"(?![0-9A-Za-z_]|\.[0-9A-Za-z_])" +) +#: A dotted Microcosm symbol named in prose, such as +#: ``microcosm.graph.executor._structural_columns``. +DOTTED_MICROCOSM_SYMBOL = re.compile( + r"(? str: + return PurePosixPath(self.file).name + + +def nested_strings(value: object) -> Iterator[str]: + """Every string in a payload, mapping keys included, at any depth.""" + + if isinstance(value, Mapping): + for key, child in value.items(): + yield str(key) + yield from nested_strings(child) + elif isinstance(value, list): + for child in value: + yield from nested_strings(child) + elif isinstance(value, str): + yield value + + +def line_citations(value: object) -> list[LineCitation]: + """Each line citation of a source in a payload, in string order.""" + + return [ + LineCitation(match["file"], text) + for text in nested_strings(value) + for pattern in _LINE_CITATIONS + for match in pattern.finditer(text) + ] + + +def names_a_commit(text: str) -> bool: + """Whether a string names a commit inline.""" + + return INLINE_COMMIT.search(text) is not None + + +def named_commits(text: str) -> list[str]: + """The commits a string names inline, in order.""" + + return [ + re.sub(r"^(?:(?i:commit|at)\s+|@|/(?:blob|tree)/)", "", match.group(0)) + for match in INLINE_COMMIT.finditer(text) + ] + + +def commit_pinned_sources(value: object) -> set[str]: + """File names of the sources a payload pins in ``path``/``commit`` maps.""" + + pinned: set[str] = set() + if isinstance(value, Mapping): + path, commit = value.get("path"), value.get("commit") + if ( + isinstance(path, str) + and isinstance(commit, str) + and _COMMIT_PIN.fullmatch(commit) + ): + pinned.add(PurePosixPath(path).name) + for child in value.values(): + pinned |= commit_pinned_sources(child) + elif isinstance(value, list): + for child in value: + pinned |= commit_pinned_sources(child) + return pinned + + +def unpinned_line_citations(document: object) -> list[LineCitation]: + """Line citations that neither their string nor the document pins.""" + + pinned = commit_pinned_sources(document) + return [ + citation + for citation in line_citations(document) + if citation.name not in pinned and not names_a_commit(citation.text) + ] + + +def resolves_in_repository(file: str, root: Path = REPOSITORY_ROOT) -> bool: + """Whether a cited path names a file in this checkout. + + A path resolves from the repository root (``tools/x.py``, + ``packages/microcosm-build/src/...``), from ``packages/``, or, for a + ``microcosm/...`` module path, from any shard's ``src/``. + """ + + relative = PurePosixPath(file) + if relative.is_absolute() or ".." in relative.parts or len(relative.parts) < 2: + return False + if (root / relative).is_file() or (root / "packages" / relative).is_file(): + return True + return relative.parts[0] == "microcosm" and any( + (shard / "src" / relative).is_file() for shard in (root / "packages").iterdir() + ) + + +def in_repository_line_citations( + document: object, root: Path = REPOSITORY_ROOT +) -> list[LineCitation]: + """Line citations of this repository's files, pinned or not.""" + + return [ + citation + for citation in line_citations(document) + if resolves_in_repository(citation.file, root) + ] + + +def dotted_microcosm_symbols(value: object) -> set[str]: + """Every dotted ``microcosm.*`` name in a payload's strings.""" + + return { + match + for text in nested_strings(value) + for match in DOTTED_MICROCOSM_SYMBOL.findall(text) + } + + +def resolve_dotted_symbol(dotted: str) -> object: + """Import the longest module prefix of a dotted name, then get the rest.""" + + parts = dotted.split(".") + for cut in range(len(parts), 0, -1): + name = ".".join(parts[:cut]) + try: + target: object = importlib.import_module(name) + except ModuleNotFoundError as error: + if error.name != name: + raise + continue + for attribute in parts[cut:]: + target = getattr(target, attribute) + return target + raise ModuleNotFoundError(dotted, name=dotted) + + +def country_packages() -> list[str]: + """Each country spec package directory name, sorted.""" + + return sorted( + path.parent.name for path in COUNTRY_PACKAGE_ROOT.glob("*/country_package.json") + ) + + +def package_payloads(country: str) -> dict[str, object]: + """Every resource in a country package, parsed, by relative path. + + JSON and YAML resources are parsed; any other file is one string. + """ + + root = COUNTRY_PACKAGE_ROOT / country + payloads: dict[str, object] = {} + for path in sorted(root.rglob("*")): + if not path.is_file() or "__pycache__" in path.parts: + continue + name = path.relative_to(root).as_posix() + text = path.read_text(encoding="utf-8") + if path.suffix == ".json": + payloads[name] = json.loads(text) + elif path.suffix in {".yaml", ".yml"}: + payloads[name] = load_yaml12(text, source=name) + else: + payloads[name] = text + return payloads diff --git a/test_support/microcosm_build/us_spec_bundle.py b/test_support/microcosm_build/us_spec_bundle.py index ef4bc8868..f028a1b17 100644 --- a/test_support/microcosm_build/us_spec_bundle.py +++ b/test_support/microcosm_build/us_spec_bundle.py @@ -195,7 +195,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf" + "41e8194a6206ff84a287317ebb0fb39a083d2a8811ed013e22f4a21cc1255d18" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/build_us_target_parity_manifest.py b/tools/build_us_target_parity_manifest.py index 57d3238ba..f5eae5577 100644 --- a/tools/build_us_target_parity_manifest.py +++ b/tools/build_us_target_parity_manifest.py @@ -305,7 +305,8 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: '"BEA personal interest/dividends include imputed interest, ' "pension-plan dividends, and trust flows, so those macro totals " "should not directly calibrate tax/CPS interest and dividend " - 'variables" (loss.py:67-70, added by PR #1059)' + 'variables" (utils/loss.py:67-70 at ' + "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, added by PR #1059)" ), verdict_basis=( "us-data itself declined it as a direct target (PR #1059): a " diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 5dc45835d..07742a653 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf" + "41e8194a6206ff84a287317ebb0fb39a083d2a8811ed013e22f4a21cc1255d18" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" From 9c361ed7efebe8373dabce7045ff4e5d00f7bb06 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 8 Oct 2026 09:25:28 -0400 Subject: [PATCH 3/5] Fold #1141's NZ citation validators into the shared guard test_nz_spec_package.py now imports nested_strings, line_citations, commit_pinned_sources and unpinned_line_citations from test_support/microcosm_build/spec_source_citations.py and keeps only its NZ-specific checks: the harness pin, the five replaced citations as regression vectors, and the relationship-pointer differential. The package-wide scan, the Hypothesis properties and the dotted-symbol resolution now run once for every country in test_spec_source_citations.py, which also asserts that the NZ and US dotted symbol citations are scanned. Co-Authored-By: Claude Opus 5.5 --- .../shared/test_nz_spec_package.py | 344 +----------------- .../shared/test_spec_source_citations.py | 27 ++ 2 files changed, 44 insertions(+), 327 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py b/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py index 640ba21b2..926e42b70 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py @@ -20,10 +20,11 @@ - every scenario names exactly one method-card row and a tier from {entitlement, calibration}, and changes exactly one knob from the centre (``scenario_errors``); -- no resource cites a line number of a Python source unless the same - resource pins that source at a commit (``unpinned_python_line_citations``): - this repository's sources are cited by symbol, because their line numbers - move under unrelated edits. +- the source-citation invariants every country package shares (no line + citation of this repository's files, no unpinned line citation of another + repository's, every dotted ``microcosm.*`` name resolves) are stated and + tested once, in ``test_spec_source_citations.py``; this file keeps the NZ + regression vectors and the harness pin. Differential checks compare two records of one fact: the loader's target references with a direct parse, the rules bindings' module digests with the @@ -37,13 +38,11 @@ import copy import dataclasses import hashlib -import importlib import inspect import json import math import re from collections.abc import Callable, Iterator, Mapping, Sequence -from pathlib import PurePosixPath from typing import Any import pytest @@ -58,6 +57,12 @@ ) from microcosm.build.spec_engine import load_yaml12 from microcosm.frame import concepts as frame_concepts +from test_support.microcosm_build.spec_source_citations import ( + commit_pinned_sources, + line_citations, + nested_strings, + unpinned_line_citations, +) from test_support.paths import paths_for _BUILD = paths_for("microcosm-build") @@ -106,20 +111,6 @@ def _nested_keys(value: object) -> Iterator[str]: yield from _nested_keys(child) -def _nested_strings(value: object) -> Iterator[str]: - """Every string in a payload, mapping keys included, at any depth.""" - - if isinstance(value, Mapping): - for key, child in value.items(): - yield str(key) - yield from _nested_strings(child) - elif isinstance(value, list): - for child in value: - yield from _nested_strings(child) - elif isinstance(value, str): - yield value - - def _package_payloads() -> dict[str, object]: payloads: dict[str, object] = {} for path in sorted(NZ_ROOT.rglob("*")): @@ -452,102 +443,6 @@ def scenario_errors(document: Mapping[str, Any]) -> list[str]: return errors -#: A Python source named in prose: ``concepts.py``, or a path ending in one. -_PY_SOURCE = r"(?P[A-Za-z0-9_./-]*[A-Za-z0-9_]\.py)(?![A-Za-z0-9_])" -#: A line locator into a Python source, in each spelling the package has used -#: or GitHub produces: ``x.py:12``, ``x.py:12-19``, ``x.py#L12``, ``x.py L12``, -#: ``x.py line 12``, ``x.py, lines 12-19``, ``x.py (lines 12-19)``, and the -#: reverse ``lines 12-19 of x.py``. -_PY_LINE_CITATIONS = ( - re.compile(_PY_SOURCE + r"(?:\s*:\s*|#L|\s+L|,?\s*\(?\s*[Ll]ines?\s+)\d"), - re.compile( - r"\b[Ll]ines?\s+\d+(?:\s*[-–]\s*\d+)?(?:\s*,\s*\d+)*\s+(?:of|in)\s+" - + _PY_SOURCE - ), -) -#: A commit pin. A branch name such as ``main`` pins nothing. -_COMMIT_PIN = re.compile(r"[0-9a-fA-F]{7,40}") - - -def python_line_citations(value: object) -> list[tuple[str, str]]: - """Each (file name, string) where a string cites a line of a Python source.""" - - return [ - (PurePosixPath(match["file"]).name, text) - for text in _nested_strings(value) - for pattern in _PY_LINE_CITATIONS - for match in pattern.finditer(text) - ] - - -def commit_pinned_python_sources(value: object) -> set[str]: - """File names of the Python sources a payload pins at a commit. - - A pin is a mapping, at any depth, whose ``path`` names a ``.py`` file and - whose ``commit`` is 7-40 hex digits. - """ - - pinned: set[str] = set() - if isinstance(value, Mapping): - path, commit = value.get("path"), value.get("commit") - if ( - isinstance(path, str) - and path.endswith(".py") - and isinstance(commit, str) - and _COMMIT_PIN.fullmatch(commit) - ): - pinned.add(PurePosixPath(path).name) - for child in value.values(): - pinned |= commit_pinned_python_sources(child) - elif isinstance(value, list): - for child in value: - pinned |= commit_pinned_python_sources(child) - return pinned - - -def unpinned_python_line_citations(document: object) -> list[tuple[str, str]]: - """Line citations of Python sources that the document does not pin. - - A line number names code only at a fixed commit. This repository's sources - shift under unrelated edits, so a resource cites them by symbol - (``concepts.py _pointer("parent_1_person_id")``). A resource cites another - repository's source by line only when it also pins that source at a - commit, as as_rate_bridge.json's ``source`` does. Pins match by file name. - """ - - pinned = commit_pinned_python_sources(document) - return [ - (name, text) - for name, text in python_line_citations(document) - if name not in pinned - ] - - -#: A dotted Microcosm symbol named in prose, such as -#: ``microcosm.graph.executor._structural_columns``. -_DOTTED_MICROCOSM_SYMBOL = re.compile( - r"(? object: - """Import the longest module prefix of a dotted name, then get the rest.""" - - parts = dotted.split(".") - for cut in range(len(parts), 0, -1): - name = ".".join(parts[:cut]) - try: - target: object = importlib.import_module(name) - except ModuleNotFoundError as error: - if error.name != name: - raise - continue - for attribute in parts[cut:]: - target = getattr(target, attribute) - return target - raise ModuleNotFoundError(dotted, name=dotted) - - # --------------------------------------------------------------------------- # Reference sets # --------------------------------------------------------------------------- @@ -1598,68 +1493,14 @@ def test_every_refusal_branch_names_the_broken_scenario( # --------------------------------------------------------------------------- -_PY_STEM = st.from_regex(r"[a-z_][a-z0-9_]{0,15}", fullmatch=True) -_PY_DIRECTORY = st.sampled_from( - ( - "", - "packages/microcosm-frame/src/microcosm/frame/", - "microcosm/graph/", - "nz-lane/emtr_reproduction/", - ) -) -#: Line-locator spellings: {file} is the cited source, {a} and {b} lines. -_LINE_LOCATORS = ( - "{file}:{a}", - "{file}:{a}-{b}", - "{file}:{a},{b}", - "{file}#L{a}", - "{file}#L{a}-L{b}", - "{file} L{a}", - "{file} line {a}", - "{file} lines {a}-{b}", - "{file}, lines {a}-{b}", - "{file} (lines {a}-{b})", - "Lines {a}-{b} of {file}", - "line {a} in {file}", -) -#: Symbol-citation spellings, which carry no line number. -_SYMBOL_LOCATORS = ( - "{file} {symbol}", - '{file} _pointer("{symbol}")', - '{file}, the "# --- Relationships" block', - "{file}: {symbol}", - "{symbol} in {file} at main", -) -#: Prose around a citation. It has no digit, so it can neither extend a -#: planted line number nor plant a citation of its own. -_PROSE = st.text( - alphabet="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ,;()'\"-_", - max_size=40, -) -_BRANCH_NAMES = ("main", "HEAD", "launch/ga-path-execution-gate", "ba6e7d", "") - - -def _plant(text: str, depth: int, as_key: bool) -> object: - planted: object = {text: "cited"} if as_key else text - for level in range(depth): - planted = {"level": level, "note": [planted]} - return planted - - class TestSourceCitations: - def test_no_resource_cites_a_python_line_it_does_not_pin(self) -> None: - offenders = { - name: hits - for name, payload in _package_payloads().items() - if (hits := unpinned_python_line_citations(payload)) - } - assert offenders == {} - def test_the_harness_line_citations_are_scanned_and_pinned(self) -> None: bridge = _load("as_rate_bridge.json") - assert {name for name, _ in python_line_citations(bridge)} == {"run.py"} - assert commit_pinned_python_sources(bridge) == {"run.py"} + assert {citation.name for citation in line_citations(bridge)} == {"run.py"} + assert "run.py" in commit_pinned_sources(bridge) + assert bridge["source"]["path"].endswith("/run.py") assert re.fullmatch(r"[0-9a-f]{40}", bridge["source"]["commit"]) + assert unpinned_line_citations(bridge) == [] @pytest.mark.parametrize( "replaced", @@ -1675,126 +1516,7 @@ def test_the_harness_line_citations_are_scanned_and_pinned(self) -> None: def test_the_line_citations_the_package_carried_are_refused( self, replaced: str ) -> None: - assert unpinned_python_line_citations({"rule": replaced}) - - @settings(max_examples=300, deadline=None) - @given( - stem=_PY_STEM, - directory=_PY_DIRECTORY, - locator=st.sampled_from(_LINE_LOCATORS), - a=st.integers(1, 99_999), - b=st.integers(1, 99_999), - prefix=_PROSE, - suffix=_PROSE, - depth=st.integers(0, 3), - as_key=st.booleans(), - ) - def test_an_unpinned_line_citation_is_refused_at_any_depth( - self, - stem: str, - directory: str, - locator: str, - a: int, - b: int, - prefix: str, - suffix: str, - depth: int, - as_key: bool, - ) -> None: - citation = locator.format(file=f"{directory}{stem}.py", a=a, b=b) - text = f"{prefix} {citation} {suffix}" - hits = unpinned_python_line_citations(_plant(text, depth, as_key)) - assert (f"{stem}.py", text) in hits - - @settings(max_examples=200, deadline=None) - @given( - stem=_PY_STEM, - directory=_PY_DIRECTORY, - pin_directory=_PY_DIRECTORY, - locator=st.sampled_from(_LINE_LOCATORS), - a=st.integers(1, 99_999), - b=st.integers(1, 99_999), - depth=st.integers(0, 3), - as_key=st.booleans(), - commit=_HEX40_TEXT, - length=st.integers(7, 40), - ) - def test_a_line_citation_of_a_source_pinned_at_a_commit_passes( - self, - stem: str, - directory: str, - pin_directory: str, - locator: str, - a: int, - b: int, - depth: int, - as_key: bool, - commit: str, - length: int, - ) -> None: - citation = locator.format(file=f"{directory}{stem}.py", a=a, b=b) - document = { - "source": { - "repository": "TheAxiomFoundation/ops", - "path": f"{pin_directory}{stem}.py", - "commit": commit[:length], - }, - "notes": [_plant(citation, depth, as_key)], - } - assert python_line_citations(document) - assert unpinned_python_line_citations(document) == [] - - @settings(max_examples=200, deadline=None) - @given( - stem=_PY_STEM, - other=_PY_STEM, - locator=st.sampled_from(_LINE_LOCATORS), - a=st.integers(1, 99_999), - b=st.integers(1, 99_999), - commit=_HEX40_TEXT, - branch=st.sampled_from(_BRANCH_NAMES), - pin_other_file=st.booleans(), - ) - def test_a_pin_of_another_file_or_at_a_branch_does_not_cover_a_citation( - self, - stem: str, - other: str, - locator: str, - a: int, - b: int, - commit: str, - branch: str, - pin_other_file: bool, - ) -> None: - if pin_other_file: - assume(other != stem) - pin = {"path": f"{other}.py", "commit": commit} - else: - pin = {"path": f"{stem}.py", "commit": branch} - citation = locator.format(file=f"{stem}.py", a=a, b=b) - hits = unpinned_python_line_citations({"source": pin, "note": citation}) - assert (f"{stem}.py", citation) in hits - - @settings(max_examples=200, deadline=None) - @given( - stem=_PY_STEM, - directory=_PY_DIRECTORY, - locator=st.sampled_from(_SYMBOL_LOCATORS), - symbol=_PY_STEM, - prefix=_PROSE, - suffix=_PROSE, - ) - def test_a_symbol_citation_is_not_a_line_citation( - self, - stem: str, - directory: str, - locator: str, - symbol: str, - prefix: str, - suffix: str, - ) -> None: - citation = locator.format(file=f"{directory}{stem}.py", symbol=symbol) - assert python_line_citations({"rule": f"{prefix} {citation} {suffix}"}) == [] + assert unpinned_line_citations({"rule": replaced}) def test_benefit_unit_rule_cites_the_relationship_pointers_by_symbol( self, @@ -1823,7 +1545,7 @@ def test_benefit_unit_rule_cites_the_relationship_pointers_by_symbol( ) cited = { name - for text in _nested_strings(rule) + for text in nested_strings(rule) for name in re.findall(r'_pointer\("(\w+)"\)', text) } assert cited == { @@ -1840,38 +1562,6 @@ def test_benefit_unit_rule_cites_the_relationship_pointers_by_symbol( assert "concepts.py _check_pointers" in child["rule"] assert callable(frame_concepts._check_pointers) - def test_every_dotted_microcosm_symbol_in_the_package_resolves(self) -> None: - named = { - match - for payload in _package_payloads().values() - for text in _nested_strings(payload) - for match in _DOTTED_MICROCOSM_SYMBOL.findall(text) - } - assert { - "microcosm.graph.executor._structural_columns", - "microcosm.graph.executor._validate_population_declaration", - } <= named - unresolved = [] - for dotted in sorted(named): - try: - resolve_dotted_symbol(dotted) - except (ImportError, AttributeError) as error: - unresolved.append(f"{dotted}: {error}") - assert unresolved == [] - - @pytest.mark.parametrize( - ("dotted", "error"), - [ - ("microcosm.graph.executor._no_such_symbol", AttributeError), - ("microcosm.no_such_shard.module.symbol", ModuleNotFoundError), - ], - ) - def test_an_unresolvable_dotted_symbol_raises( - self, dotted: str, error: type[Exception] - ) -> None: - with pytest.raises(error): - resolve_dotted_symbol(dotted) - # --------------------------------------------------------------------------- # Remaining resources diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index 5460cd312..092aa7a00 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -122,6 +122,33 @@ def test_every_dotted_microcosm_symbol_resolves(country: str) -> None: assert unresolved == [] +@pytest.mark.parametrize( + ("country", "cited"), + [ + ( + "nz", + { + "microcosm.graph.executor._structural_columns", + "microcosm.graph.executor._validate_population_declaration", + }, + ), + ( + "us", + { + "microcosm.build.us_runtime.asec_pool.load_asec_h5_tables", + "microcosm.build.us_runtime.asec_pool._prepare_year_input", + "microcosm.build.us_runtime.asec_pool._with_relationship_recode", + }, + ), + ], +) +def test_the_dotted_symbol_citations_are_scanned(country: str, cited: set[str]) -> None: + named = set().union( + *map(dotted_microcosm_symbols, package_payloads(country).values()) + ) + assert cited <= named + + @pytest.mark.parametrize( ("dotted", "error"), [ From eb4e26b77d8d1745f4d480b3e1bf4f94664dafac Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 8 Oct 2026 09:30:55 -0400 Subject: [PATCH 4/5] Scan only microcosm. names as dotted symbol citations #1122 adds an NZ resource whose format id is microcosm.axiom_input_closure.v1, which the dotted-symbol scan read as a Python symbol and failed to import. A dotted name now counts as a symbol citation only when its second component is a shard in this checkout (build, calibrate, data, diagnostics, fit, frame, graph). Every dotted name the packages carry today is under a shard, so the scanned sets are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../shared/test_spec_source_citations.py | 13 +++++++++++++ .../microcosm_build/spec_source_citations.py | 11 ++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index 092aa7a00..7019d1f60 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -40,6 +40,7 @@ ) from test_support.microcosm_build.spec_source_citations import ( COUNTRY_PACKAGE_ROOT, + MICROCOSM_SHARDS, REPOSITORY_ROOT, LineCitation, commit_pinned_sources, @@ -149,6 +150,18 @@ def test_the_dotted_symbol_citations_are_scanned(country: str, cited: set[str]) assert cited <= named +def test_only_names_under_a_shard_are_symbol_citations() -> None: + assert {"build", "frame", "graph"} <= MICROCOSM_SHARDS + text = ( + "format microcosm.axiom_input_closure.v1, read by " + "microcosm.frame.concepts.concept and microcosm.build.us/source_stages.json" + ) + assert dotted_microcosm_symbols({"note": text}) == { + "microcosm.frame.concepts.concept", + "microcosm.build.us", + } + + @pytest.mark.parametrize( ("dotted", "error"), [ diff --git a/test_support/microcosm_build/spec_source_citations.py b/test_support/microcosm_build/spec_source_citations.py index 4aae81650..cff05d868 100644 --- a/test_support/microcosm_build/spec_source_citations.py +++ b/test_support/microcosm_build/spec_source_citations.py @@ -71,6 +71,14 @@ DOTTED_MICROCOSM_SYMBOL = re.compile( r"(?`` namespaces in this checkout. A dotted name under +#: any other second component, such as the ``microcosm.axiom_input_closure.v1`` +#: format id, names no Python symbol. +MICROCOSM_SHARDS = frozenset( + path.name + for path in REPOSITORY_ROOT.glob("packages/microcosm-*/src/microcosm/*") + if path.is_dir() and path.name != "__pycache__" +) class LineCitation(NamedTuple): @@ -186,12 +194,13 @@ def in_repository_line_citations( def dotted_microcosm_symbols(value: object) -> set[str]: - """Every dotted ``microcosm.*`` name in a payload's strings.""" + """Every dotted ``microcosm..*`` name in a payload's strings.""" return { match for text in nested_strings(value) for match in DOTTED_MICROCOSM_SYMBOL.findall(text) + if match.split(".")[1] in MICROCOSM_SHARDS } From 0a841e4483083d9b833ae6e771d23e8cd3d6b35c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 8 Oct 2026 10:02:27 -0400 Subject: [PATCH 5/5] Fix issues from review round 1: spellings, partial paths, pin strictness - Recognise a quoted or backticked file name and the `x.py: lines N`, `x.py, LN` and `x.py (LN)` locators; the Hypothesis spellings cover them. - A cited path of two or more components is this repository's when it is the tail of a source file here (`us_runtime/asec_pool.py`), not only when it resolves from the root, so a partial in-repo path pinned inline is still refused. - A path/commit mapping now needs the same sha form as an inline commit (7-40 lowercase hex with a digit and a letter); a decimal, a word or an upper-case value pins nothing. - The unmarried-partner build contract says Build J predates #720, and a test checks the summary's per-year relationship_recode_source. No package resource gains or loses a citation hit under the wider rules. Co-Authored-By: Claude Opus 5.5 --- .../build/us/ecps_parity_known_gaps.json | 2 +- .../shared/test_spec_source_citations.py | 54 +++++++++++- .../microcosm_build/spec_source_citations.py | 83 +++++++++++++------ 3 files changed, 110 insertions(+), 29 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json b/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json index 31ae0ab0f..11f46cc81 100644 --- a/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json +++ b/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json @@ -176,7 +176,7 @@ "derived_A_EXPRRP": "microcosm.build.us_runtime.asec_pool._with_relationship_recode derives relationship codes from line, spouse, sex, and parent pointers only for a person table without A_EXPRRP, such as the raw 2022 and 2023 H5 inputs, and that fallback never emits code 13; since microcosm #720 the pooling tools bind each year's pinned Census person file and microcosm.build.us_runtime.asec_pool._prepare_year_input restores Census A_EXPRRP, whose reviewed domain includes code 13, before the recode, so pooled 2022 and 2023 persons no longer take the fallback.", "rejection": "Using only the 2024 alternatives and filling the equally weighted 2022-2023 source years as false, or splitting the combined partner/roommate category, would synthesize person statuses absent from the locked inputs." }, - "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh passes only the three prebuilt 2022-2024 artifacts via its --asec-h5 2024=, 2023=, and 2022= arguments; experiments/build_j_recert/base_j.summary.json base_source.sources[].sha256 and base_source.metadata.sources[].share record their SHA-256 digests and equal source-year shares; microcosm.build.us_runtime.asec_pool.load_asec_h5_tables reads those tables, and since microcosm #720 microcosm.build.us_runtime.asec_pool._prepare_year_input restores the 2022 and 2023 inputs' missing A_EXPRRP from pinned Census person files but not PERRP, PECOHAB, or A_FAMREL." + "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh passes only the three prebuilt 2022-2024 artifacts via its --asec-h5 2024=, 2023=, and 2022= arguments; experiments/build_j_recert/base_j.summary.json base_source.sources[].sha256 and base_source.metadata.sources[].share record their SHA-256 digests and equal source-year shares; microcosm.build.us_runtime.asec_pool.load_asec_h5_tables reads those tables, and since microcosm #720 microcosm.build.us_runtime.asec_pool._prepare_year_input restores the 2022 and 2023 inputs' missing A_EXPRRP from pinned Census person files but not PERRP, PECOHAB, or A_FAMREL. Build J predates #720: base_j.summary.json base_source.metadata.sources[].relationship_recode_source records derived:line_spouse_parent for 2022 and 2023 and source:A_EXPRRP for 2024." } }, "is_wic_at_nutritional_risk": { diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index 7019d1f60..8f7c04616 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -9,8 +9,10 @@ hide a broken check: - no resource cites a line of a file in this repository, pinned or not - (``in_repository_line_citations``): this repository's sources are cited by - symbol; + (``in_repository_line_citations``), where a cited path is this + repository's when it resolves here or is the tail of two or more + components of a file here: this repository's sources are cited by symbol + (a bare file name cannot be resolved and falls to the next rule); - no resource cites a line of any other source unless the citation's own string names a commit, or the resource pins that file in a ``path``/``commit`` mapping (``unpinned_line_citations``); @@ -27,6 +29,7 @@ import ast import inspect +import json import pandas as pd import pytest @@ -207,6 +210,11 @@ def test_an_unresolvable_dotted_symbol_raises( "{file} (lines {a}-{b})", "Lines {a}-{b} of {file}", "line {a} in {file}", + "`{file}` lines {a}-{b}", + "{file}: lines {a}-{b}", + "{file}, L{a}", + "{file} (L{a})", + "line {a} of `{file}`", ) #: Symbol-citation spellings, which carry no line number. _SYMBOL_LOCATORS = ( @@ -236,6 +244,8 @@ def test_an_unresolvable_dotted_symbol_raises( #: Files in this checkout, in each path form a citation can use. _IN_REPOSITORY_FILES = ( "packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py", + "us_runtime/asec_pool.py", + "graph/executor.py", "microcosm-frame/src/microcosm/frame/concepts.py", "microcosm/graph/executor.py", "tools/build_us_puf_support_base.py", @@ -312,6 +322,7 @@ def test_a_line_citation_of_a_file_pinned_at_a_commit_passes( as_key: bool, commit: str, ) -> None: + assume(_is_commit_like(commit)) citation = locator.format(file=f"{directory}{stem}.{suffix}", a=a, b=b) document = { "source": { @@ -461,6 +472,31 @@ def test_a_pin_of_another_file_or_at_a_branch_does_not_cover_a_citation( assert commit_pinned_sources({"source": pin}).isdisjoint({f"{stem}.{suffix}"}) +@settings(max_examples=200, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + locator=st.sampled_from(_LINE_LOCATORS), + not_a_commit=st.one_of( + st.text(alphabet="0123456789", min_size=7, max_size=40), + st.text(alphabet="abcdef", min_size=7, max_size=40), + st.text(alphabet="0123456789ABCDEF", min_size=7, max_size=40).filter( + lambda value: any(c.isalpha() for c in value) + ), + ), + a=st.integers(1, 99_999), +) +def test_a_path_pin_whose_commit_is_no_sha_pins_nothing( + stem: str, suffix: str, locator: str, not_a_commit: str, a: int +) -> None: + pin = {"path": f"{stem}.{suffix}", "commit": not_a_commit} + citation = locator.format(file=f"{stem}.{suffix}", a=a, b=a) + assert commit_pinned_sources({"source": pin}) == set() + assert LineCitation(f"{stem}.{suffix}", citation) in unpinned_line_citations( + {"source": pin, "note": citation} + ) + + @settings(max_examples=200, deadline=None) @given( stem=_STEM, @@ -521,6 +557,8 @@ def test_in_repository_paths_resolve_from_the_root_packages_or_a_shard() -> None assert resolves_in_repository(file), file for file in ( "concepts.py", + "datasets/cps/cps.py", + "src/lib.rs", "archived_data/datasets/cps/cps.py", "targets/sources/obr.py", "../microcosm/tools/build_us_puf_support_base.py", @@ -614,6 +652,18 @@ def test_the_restore_runs_before_the_relationship_fallback() -> None: source = inspect.getsource(asec_pool._prepare_year_input) restore = source.index("restore_asec_census_person_columns(") assert restore < source.index("_with_relationship_recode(person)") + assert "Build J predates #720" in contract + summary = json.loads( + (REPOSITORY_ROOT / "experiments/build_j_recert/base_j.summary.json").read_text() + ) + assert { + source["year"]: source["relationship_recode_source"] + for source in summary["base_source"]["metadata"]["sources"] + } == { + 2022: "derived:line_spouse_parent", + 2023: "derived:line_spouse_parent", + 2024: "source:A_EXPRRP", + } @settings(max_examples=100, deadline=None) diff --git a/test_support/microcosm_build/spec_source_citations.py b/test_support/microcosm_build/spec_source_citations.py index cff05d868..6b96963a2 100644 --- a/test_support/microcosm_build/spec_source_citations.py +++ b/test_support/microcosm_build/spec_source_citations.py @@ -12,20 +12,28 @@ lowercase hex digits with at least one digit and one letter, so a decimal amount ("at 2500000") or a word ("at defaced") pins nothing; - the resource holds a mapping, at any depth, whose ``path`` names the file - and whose ``commit`` is 7-40 hex digits (``source``-style pins, matched by - file name). - -An inline commit pins every citation in its string. A file that resolves in -this checkout is in-repository, and a line citation of it is refused even -when pinned; a bare file name cannot be resolved, so it must be pinned. + and whose ``commit`` is a sha of the same form (``source``-style pins, + matched by file name across the whole resource). + +An inline commit pins every citation in its string. A cited path is +in-repository when it resolves in this checkout, or when it has at least two +components and is the tail of a file here (``us_runtime/asec_pool.py``); a +line citation of it is refused even when pinned. A bare file name cannot be +told apart from another repository's, so it must be pinned. + +The recognised spellings are the ones ``_LINE_CITATIONS`` documents. A symbol +between the file and its lines (``x.py some_function, lines 12-19``) is not +recognised. """ from __future__ import annotations import importlib import json +import os import re from collections.abc import Iterator, Mapping +from functools import cache from pathlib import Path, PurePosixPath from typing import NamedTuple @@ -47,25 +55,35 @@ ) #: A line locator into a source, in each spelling the packages have used or #: GitHub produces: ``x.py:12``, ``x.py:12-19``, ``x.py:12,19``, ``x.py#L12``, -#: ``x.py L12``, ``x.py line 12``, ``x.py, lines 12-19``, -#: ``x.py (lines 12-19)``, and the reverse ``lines 12-19 of x.py`` and -#: ``line 12 in x.py``. +#: ``x.py L12``, ``x.py, L12``, ``x.py (L12)``, ``x.py line 12``, +#: ``x.py, lines 12-19``, ``x.py: lines 12-19``, ``x.py (lines 12-19)``, and +#: the reverse ``lines 12-19 of x.py`` and ``line 12 in x.py``. The file name +#: may be quoted or backticked (```x.py` lines 12-19``). _LINE_CITATIONS = ( - re.compile(_SOURCE + r"(?::|#L|\s+L|,?\s*\(?\s*[Ll]ines?\s+)\d"), re.compile( - r"\b[Ll]ines?\s+\d+(?:\s*[-–]\s*\d+)?(?:\s*,\s*\d+)*\s+(?:of|in)\s+" + _SOURCE + _SOURCE + r"[`'\"]?(?::|#L|,?\s*\(?\s*L|,?\s*:?\s*\(?\s*[Ll]ines?\s+)(?=\d)" + ), + re.compile( + r"\b[Ll]ines?\s+\d+(?:\s*[-–]\s*\d+)?(?:\s*,\s*\d+)*\s+(?:of|in)\s+[`'\"]?" + + _SOURCE ), ) -#: A commit pin's value in a ``path``/``commit`` mapping. A branch name such -#: as ``main`` pins nothing. -_COMMIT_PIN = re.compile(r"[0-9a-fA-F]{7,40}") -#: A commit named inline. The hex needs a digit and a letter, and must not -#: run on into a file name (``at a1b2c3d.py:12`` names a file, not a commit). +#: A commit sha: 7-40 lowercase hex with at least one digit and one letter, +#: so a decimal amount ("2500000"), a word ("defaced") or a branch name such +#: as ``main`` is not one. +_SHA = r"(?=[0-9a-f]*[0-9])(?=[0-9a-f]*[a-f])[0-9a-f]{7,40}" +#: A commit pin's value in a ``path``/``commit`` mapping. +_COMMIT_PIN = re.compile(_SHA) +#: A commit named inline. The sha must not run on into a file name +#: (``at a1b2c3d.py:12`` names a file, not a commit). INLINE_COMMIT = re.compile( r"(?:(?i:\bcommit\s+|\bat\s+)|@|/(?:blob|tree)/)" - r"(?=[0-9a-f]*[0-9])(?=[0-9a-f]*[a-f])[0-9a-f]{7,40}" - r"(?![0-9A-Za-z_]|\.[0-9A-Za-z_])" + + _SHA + + r"(?![0-9A-Za-z_]|\.[0-9A-Za-z_])" ) +#: Top-level directories whose files are this repository's sources. +_SOURCE_ROOTS = ("packages", "tools", "experiments", "test_support", "docs") +_SKIPPED_DIRECTORIES = {"__pycache__", ".venv", "node_modules", ".git"} #: A dotted Microcosm symbol named in prose, such as #: ``microcosm.graph.executor._structural_columns``. DOTTED_MICROCOSM_SYMBOL = re.compile( @@ -163,22 +181,35 @@ def unpinned_line_citations(document: object) -> list[LineCitation]: ] +@cache +def _repository_path_tails(root: Path) -> frozenset[str]: + """Every tail of two or more components of a source file in ``root``.""" + + tails: set[str] = set() + for top in _SOURCE_ROOTS: + for directory, subdirectories, files in os.walk(root / top): + subdirectories[:] = [ + name for name in subdirectories if name not in _SKIPPED_DIRECTORIES + ] + parts = PurePosixPath(Path(directory).relative_to(root).as_posix()).parts + for name in files: + path = (*parts, name) + tails.update("/".join(path[cut:]) for cut in range(len(path) - 1)) + return frozenset(tails) + + def resolves_in_repository(file: str, root: Path = REPOSITORY_ROOT) -> bool: """Whether a cited path names a file in this checkout. - A path resolves from the repository root (``tools/x.py``, - ``packages/microcosm-build/src/...``), from ``packages/``, or, for a - ``microcosm/...`` module path, from any shard's ``src/``. + A path of two or more components resolves when it is the tail of a source + file here: ``tools/x.py``, ``packages/microcosm-build/src/...``, + ``microcosm/graph/executor.py`` and ``us_runtime/asec_pool.py`` all do. """ relative = PurePosixPath(file) if relative.is_absolute() or ".." in relative.parts or len(relative.parts) < 2: return False - if (root / relative).is_file() or (root / "packages" / relative).is_file(): - return True - return relative.parts[0] == "microcosm" and any( - (shard / "src" / relative).is_file() for shard in (root / "packages").iterdir() - ) + return relative.as_posix() in _repository_path_tails(root) def in_repository_line_citations(