From e2a66d79dd3c14f247074695b562a36ed34cbbb9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 07:41:02 -0400 Subject: [PATCH 01/11] Make the UK event replay select a certified bundle Every replay stage (build, compute, stage, compare, adopt, Modal, model diagnostics, construction audit) takes --bundle KEY. The default key is the unchanged 2.89.2 pin, so every existing output byte is reproduced. Non-default bundles get their own registry and results namespace. Each receipt binds the key, pin digest, dataset, model/core/loader versions and year window, and a stage refuses another bundle's inputs. Adds a policyengine 6.x managed-loader path alongside 5.0.2, pin-derived year windows (earlier OBR cells stay as outside_bundle_window), registration from a release's packaged manifest, and an exact uk-extra freeze exporter. Adoption's engine-free guard now checks only for engines imported during validation, so it holds in an engine-equipped full test run. Refs #156. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 3 + data/uk/certified_bundles/index.json | 7 + pipeline/adopt_uk_event.py | 100 +++- pipeline/audit_uk_event_constructions.py | 50 +- pipeline/build_uk_event_registry.py | 128 ++++- pipeline/compare_uk_event.py | 242 ++++++++-- pipeline/compute_uk_ab2025.py | 16 +- pipeline/compute_uk_event.py | 78 ++- pipeline/compute_uk_obr_costings.py | 169 ++++++- pipeline/diagnose_uk_event_models.py | 232 ++++++++- pipeline/export_uk_bundle_requirements.py | 109 +++++ pipeline/modal_uk_event.py | 316 ++++++++++--- pipeline/register_uk_certified_bundle.py | 313 +++++++++++++ pipeline/stage_uk_event.py | 53 ++- pipeline/uk_bundle.py | 238 ++++++++++ tests/test_modal_uk_event.py | 209 ++++++++- tests/test_uk_bundle_adapter.py | 525 +++++++++++++++++++++ tests/test_uk_bundle_registry.py | 463 ++++++++++++++++++ tests/test_uk_event_adoption.py | 158 ++++++- tests/test_uk_event_bundle_staging.py | 547 ++++++++++++++++++++++ tests/test_uk_event_construction_audit.py | 9 +- 21 files changed, 3783 insertions(+), 182 deletions(-) create mode 100644 data/uk/certified_bundles/index.json create mode 100644 pipeline/export_uk_bundle_requirements.py create mode 100644 pipeline/register_uk_certified_bundle.py create mode 100644 pipeline/uk_bundle.py create mode 100644 tests/test_uk_bundle_adapter.py create mode 100644 tests/test_uk_bundle_registry.py create mode 100644 tests/test_uk_event_bundle_staging.py diff --git a/.gitignore b/.gitignore index cbc2c869..23ca78bf 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ diagnosis/*.err.log # Derived artifact — build with: PYTHONPATH=. python -m scorecard_db.build_db data/scorecard.db data/uk/policyengine-local/ + +# Registries used only for certified-loader adapter development checks. +data/uk/events/bundles/dev-*/ diff --git a/data/uk/certified_bundles/index.json b/data/uk/certified_bundles/index.json new file mode 100644 index 00000000..c1b129a5 --- /dev/null +++ b/data/uk/certified_bundles/index.json @@ -0,0 +1,7 @@ +{ + "schema_version": 1, + "default_bundle": "populace-uk-2023__pe-uk-2.89.2", + "bundles": { + "populace-uk-2023__pe-uk-2.89.2": "data/uk/certified_bundle.json" + } +} diff --git a/pipeline/adopt_uk_event.py b/pipeline/adopt_uk_event.py index 74e7af30..4697b29e 100644 --- a/pipeline/adopt_uk_event.py +++ b/pipeline/adopt_uk_event.py @@ -19,6 +19,7 @@ from pipeline import compute_uk_event as compute from pipeline import modal_uk_event as modal_runner from pipeline import stage_uk_event as stage +from pipeline import uk_bundle ROOT = modal_runner.ROOT @@ -30,14 +31,28 @@ def reject(value): return json.loads(path.read_bytes(), parse_constant=reject) -def current_input_digest(event): - inputs, _ = modal_runner.input_manifest(event, root=ROOT) +def current_input_digest(event, bundle=uk_bundle.DEFAULT_BUNDLE): + inputs, _ = modal_runner.input_manifest(event, bundle=bundle, root=ROOT) return modal_runner.digest_bytes(modal_runner.canonical_bytes(inputs)) -def validate(event, download_root, remote_output=None, allow_partial=False): +def validate( + event, + download_root, + remote_output=None, + allow_partial=False, + *, + bundle=uk_bundle.DEFAULT_BUNDLE, +): + # Validation must stay engine-free. Compare against the modules already + # loaded, so an engine imported elsewhere in the process doesn't trip it. + engine_modules_before = { + name for name in sys.modules if name.startswith("policyengine") + } prefix = modal_runner.download_prefix(download_root, root=ROOT) - remote_output = remote_output or f"results/uk/events/{event}" + remote_output = remote_output or modal_runner.bundle_relative( + uk_bundle.event_output_dir(event, bundle, root=ROOT), ROOT + ) modal_runner.relative_path(remote_output) directory = prefix / remote_output if not directory.resolve().is_relative_to(prefix.resolve()): @@ -51,21 +66,29 @@ def validate(event, download_root, remote_output=None, allow_partial=False): measures=request["measures"], output_dir=remote_output, preflight_only=False, + bundle=bundle, + root=ROOT, ) if request != expected_request or receipt[ "request_sha256" ] != modal_runner.digest_bytes(modal_runner.canonical_bytes(request)): raise ValueError("Modal request identity does not match this adoption") modal_runner.validate_selection(request, root=ROOT) - if receipt["input_manifest_sha256"] != current_input_digest(event): + if receipt["input_manifest_sha256"] != current_input_digest(event, bundle): raise ValueError( "Modal input commitment differs from current allowlisted inputs" ) runtime = receipt["runtime"] pins = modal_runner.requirements_pins( - (ROOT / "docs/uk_replay/requirements.txt").read_text() + ( + ROOT / modal_runner.bundle_file(bundle, "requirements_freeze", root=ROOT) + ).read_text(), + bundle, + root=ROOT, + ) + audit = read_json( + ROOT / modal_runner.bundle_file(bundle, "offline_audit", root=ROOT) ) - audit = read_json(ROOT / "docs/uk_replay/OFFLINE_BUNDLE_AUDIT.json") if ( runtime["python"] != modal_runner.PYTHON_VERSION or runtime["installed_packages"] != pins @@ -105,11 +128,14 @@ def validate(event, download_root, remote_output=None, allow_partial=False): <= manifest.keys() ): raise ValueError("compute manifest lacks explicit event-grid commitments") - registry_path, _ = compute.event_paths(event) + registry_path = uk_bundle.registry_path(event, bundle, root=ROOT) registry = read_json(registry_path) compute.registry_builder.validate_registry(registry) + uk_bundle.validate_registry_bundle(registry, bundle, root=ROOT) registry_sha = modal_runner.sha256_file(registry_path) - cert = modal_runner.certified_pin(ROOT) + cert = modal_runner.certified_pin(ROOT, bundle) + config = uk_bundle.load_bundle(bundle, root=ROOT) + data_build_id = config["data_build_id"] versions = { "engine_version": pins["policyengine-uk"], "policyengine_version": pins["policyengine"], @@ -119,16 +145,18 @@ def validate(event, download_root, remote_output=None, allow_partial=False): manifest["event"] != event or manifest["registry_sha256"] != registry_sha or manifest["certified_dataset_sha256"] != cert["sha256"] - or manifest["data_bundle"] != cert["revision"] + or manifest["data_bundle"] != data_build_id or any(manifest.get(key) != value for key, value in versions.items()) ): raise ValueError( "compute manifest identity does not match the certified registry/runtime" ) + uk_bundle.validate_bundle_identity(manifest, bundle, root=ROOT) for relative, digest in manifest["artifacts"].items(): if receipt["files"].get(relative) != digest: raise ValueError("compute and Modal artifact commitments disagree") artifact = read_json(prefix / relative) + uk_bundle.validate_bundle_identity(artifact, bundle, root=ROOT) if any(artifact.get(key) != value for key, value in versions.items()): raise ValueError("numerical artifact uses another engine runtime") if not math.isfinite(artifact["measure_total_gbp"]): @@ -136,7 +164,7 @@ def validate(event, download_root, remote_output=None, allow_partial=False): if ( artifact.get("dataset_sha256_before") != cert["sha256"] or artifact.get("dataset_sha256_after") != cert["sha256"] - or artifact.get("data_bundle") != cert["revision"] + or artifact.get("data_bundle") != data_build_id ): raise ValueError( "numerical artifact does not retain the certified file identity" @@ -147,7 +175,12 @@ def validate(event, download_root, remote_output=None, allow_partial=False): # resolving untouched canonical receipt paths inside the download. compute.ROOT = prefix _, tally = stage.stage_event( - registry, manifest, artifact_dir=directory, registry_sha256=registry_sha + registry, + manifest, + artifact_dir=directory, + registry_sha256=registry_sha, + bundle=bundle, + bundle_root=ROOT, ) finally: compute.ROOT = original_root @@ -169,9 +202,14 @@ def validate(event, download_root, remote_output=None, allow_partial=False): "year": year, "registry_sha256": registry_sha, "certified_dataset_sha256": cert["sha256"], - "data_bundle": cert["revision"], + "data_bundle": data_build_id, **versions, "artifacts": expected_artifacts, + **( + uk_bundle.bundle_identity(bundle, root=ROOT) + if bundle != uk_bundle.DEFAULT_BUNDLE + else {} + ), } if progress != expected_progress: raise ValueError( @@ -209,7 +247,9 @@ def validate(event, download_root, remote_output=None, allow_partial=False): ) row["old_head_effects"] = old["head_effects"] row["new_head_effects"] = new["head_effects"] - if any(name.startswith("policyengine") for name in sys.modules): + if { + name for name in sys.modules if name.startswith("policyengine") + } - engine_modules_before: raise AssertionError("validation unexpectedly imported an engine") return { "event": event, @@ -221,25 +261,39 @@ def validate(event, download_root, remote_output=None, allow_partial=False): "staging_tally": tally, "files": changes, "changed_existing_numerical_artifacts": numerical_changes, + **( + uk_bundle.bundle_identity(bundle, root=ROOT) + if bundle != uk_bundle.DEFAULT_BUNDLE + else {} + ), } -def adopt(report, *, backup_root, local_run_stopped, allow_changed_existing=False): +def adopt( + report, + *, + backup_root, + local_run_stopped, + allow_changed_existing=False, + bundle=uk_bundle.DEFAULT_BUNDLE, +): event = report["event"] + uk_bundle.validate_bundle_identity(report, bundle, root=ROOT) if not local_run_stopped: raise ValueError( "adoption requires explicit confirmation that the local event run stopped" ) - if ( - not report["full_event_complete"] - or report["remote_output"] != f"results/uk/events/{event}" + if not report["full_event_complete"] or report[ + "remote_output" + ] != modal_runner.bundle_relative( + uk_bundle.event_output_dir(event, bundle, root=ROOT), ROOT ): raise ValueError("only a complete canonical remote output can be adopted") if report["changed_existing_numerical_artifacts"] and not allow_changed_existing: raise ValueError( "existing numerical bytes differ; review the report before authorizing replacement" ) - if report["input_manifest_sha256"] != current_input_digest(event): + if report["input_manifest_sha256"] != current_input_digest(event, bundle): raise ValueError("allowlisted inputs changed after validation") expected_directory = PurePosixPath(report["remote_output"]) paths = [row["path"] for row in report["files"]] @@ -307,6 +361,7 @@ def adopt(report, *, backup_root, local_run_stopped, allow_changed_existing=Fals def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--event", default="autumn_budget_2024") + parser.add_argument("--bundle", default=uk_bundle.DEFAULT_BUNDLE) parser.add_argument("--download-root", required=True) parser.add_argument("--remote-output") parser.add_argument("--allow-partial", action="store_true") @@ -318,7 +373,11 @@ def main(): ) args = parser.parse_args() report = validate( - args.event, args.download_root, args.remote_output, args.allow_partial + args.event, + args.download_root, + args.remote_output, + args.allow_partial, + bundle=args.bundle, ) print(modal_runner.canonical_bytes(report).decode(), end="") if args.adopt: @@ -327,6 +386,7 @@ def main(): backup_root=args.backup_root, local_run_stopped=args.local_run_stopped, allow_changed_existing=args.allow_changed_existing, + bundle=args.bundle, ) diff --git a/pipeline/audit_uk_event_constructions.py b/pipeline/audit_uk_event_constructions.py index 400f4411..a38e54d5 100644 --- a/pipeline/audit_uk_event_constructions.py +++ b/pipeline/audit_uk_event_constructions.py @@ -12,6 +12,15 @@ import sysconfig from pathlib import Path +from pipeline.uk_bundle import ( + DEFAULT_BUNDLE, + bundle_document, + bundle_identity, + load_bundle, + results_root, + validate_bundle_path, +) + ROOT = Path(__file__).resolve().parents[1] SOURCE_RANGES = { "policyengine_uk/variables/gov/hmrc/sdlt_on_residential_property_transactions.py": [ @@ -81,11 +90,15 @@ def package_version(site_packages: Path, distribution: str) -> str: raise ValueError(f"missing {distribution} version metadata") -def source_evidence(site_packages: Path) -> list[dict]: +def source_evidence(site_packages: Path, *, legacy=True) -> list[dict]: evidence = [] for relative, ranges in SOURCE_RANGES.items(): path = site_packages / relative + if not legacy and not path.exists(): + continue raw = path.read_bytes() + if not legacy: + ranges = [(1, len(raw.decode().splitlines()))] source = raw.decode() lines = source.splitlines(keepends=True) evidence.append( @@ -123,7 +136,8 @@ def inspect_h5(path: Path) -> dict: "household_weight", "household_is_capital_gains_clone", ] - values = household.fields(fields)[()] + available = [name for name in fields if name in household.dtype.names] + values = household.fields(available)[()] purchased = values["property_purchased"].astype(bool) stock = values["other_residential_property_value"] weights = values["household_weight"] @@ -144,6 +158,8 @@ def inspect_h5(path: Path) -> dict: "other_property_nonfinite_rows": int((~np.isfinite(stock)).sum()), "capital_gains_clone_household_rows": int( values["household_is_capital_gains_clone"].sum() + if "household_is_capital_gains_clone" in available + else 0 ), } del values @@ -167,13 +183,19 @@ def collect_audit(dataset: Path, site_packages: Path, bundle: dict) -> dict: package: package_version(site_packages, package) for package in ("policyengine", "policyengine-uk", "policyengine-core") } - if versions != { - "policyengine": "5.0.2", - "policyengine-uk": "2.89.2", - "policyengine-core": "3.27.1", - }: + key = bundle.get("bundle_key", DEFAULT_BUNDLE) + expected = ( + bundle_identity(key)["engine_versions"] + if key != DEFAULT_BUNDLE + else { + "policyengine": "5.0.2", + "policyengine-uk": "2.89.2", + "policyengine-core": "3.27.1", + } + ) + if versions != expected: raise ValueError("construction audit requires the pinned replay packages") - evidence = source_evidence(site_packages) + evidence = source_evidence(site_packages, legacy=key == DEFAULT_BUNDLE) observations = inspect_h5(dataset) return { "schema_version": 1, @@ -183,6 +205,7 @@ def collect_audit(dataset: Path, site_packages: Path, bundle: dict) -> dict: "sha256": sha256_file(Path(__file__)), }, "packages": versions, + **(bundle_identity(key) if key != DEFAULT_BUNDLE else {}), "certified_dataset": identity, "observations": observations, "source_evidence": evidence, @@ -229,6 +252,7 @@ def collect_audit(dataset: Path, site_packages: Path, bundle: dict) -> dict: def main() -> None: parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--bundle", default=DEFAULT_BUNDLE) parser.add_argument("--dataset", required=True, type=Path) parser.add_argument( "--site-packages", type=Path, default=Path(sysconfig.get_paths()["purelib"]) @@ -236,11 +260,17 @@ def main() -> None: parser.add_argument( "--output", type=Path, - default=ROOT / "results/uk/events/CONSTRUCTION_AUDIT.json", + default=None, ) args = parser.parse_args() - bundle = json.loads((ROOT / "data/uk/certified_bundle.json").read_text()) + bundle = ( + load_bundle(args.bundle) + if args.bundle != DEFAULT_BUNDLE + else bundle_document(args.bundle) + ) audit = collect_audit(args.dataset, args.site_packages, bundle) + args.output = args.output or results_root(args.bundle) / "CONSTRUCTION_AUDIT.json" + validate_bundle_path(args.output, args.bundle, kind="results") args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text( json.dumps(audit, indent=1, sort_keys=True, allow_nan=False) + "\n" diff --git a/pipeline/build_uk_event_registry.py b/pipeline/build_uk_event_registry.py index 1abce0e8..f4a3d3fd 100644 --- a/pipeline/build_uk_event_registry.py +++ b/pipeline/build_uk_event_registry.py @@ -21,6 +21,15 @@ from decimal import Decimal, localcontext from pathlib import Path +from pipeline.uk_bundle import ( + DEFAULT_BUNDLE, + bundle_document, + bundle_identity, + bundle_window, + registry_path, + validate_registry_bundle, +) + try: from pipeline.uk_engine_registry import engine_dumps except ModuleNotFoundError: @@ -229,9 +238,11 @@ def accounting(rows, measures): } -def validate_registry(document): +def validate_registry(document, bundle=None): """Verify a checked-in event account against its independently pinned source.""" event_slug = document["event_slug"] + if bundle is not None: + validate_registry_bundle(document, bundle) actual = accounting(source_rows(event_slug), document["measures"]) if actual != document["accounting"]: raise ValueError( @@ -740,19 +751,95 @@ def gap_reason(title, rows): ) -def build_registry(event_slug, engine_parameters, engine_variables, resolve): +def build_registry( + event_slug, engine_parameters, engine_variables, resolve, bundle=DEFAULT_BUNDLE +): rows = source_rows(event_slug) event_name = EVENTS.get(event_slug) or rows[0]["fiscal_event"] grouped = defaultdict(list) for row in rows: grouped[row["title"]].append(row) - years = sorted({int(r["fy"][:4]) for r in rows if 2023 <= int(r["fy"][:4]) <= 2030}) + start, end = bundle_window(bundle) + versions = bundle_identity(bundle)["engine_versions"] + years = sorted({int(r["fy"][:4]) for r in rows if start <= int(r["fy"][:4]) <= end}) measures = [] for title, rs in grouped.items(): authored = authored_construction(event_slug, title, resolve) if authored: m = dict(authored) key = m.pop("key") + if bundle != DEFAULT_BUNDLE: + paths = set(m.get("pe_baseline_modifier", {})) | set( + m.get("pe_reform_delta", {}) + ) + metadata_reader = getattr( + resolve, "parameter_metadata", lambda path: {} + ) + modes = { + path: { + flag: bool(metadata_reader(path).get(flag, False)) + for flag in ("fiscal_year_blend", "preserve_calendar_dates") + } + for path in sorted(paths) + } + m["parameter_processing_evidence"] = { + "engine_file": "policyengine_uk/utils/parameters.py", + "metadata_by_path": modes, + } + if ( + key == "capital_gains_main_rates_and_reliefs" + and modes + and all(mode["fiscal_year_blend"] for mode in modes.values()) + ): + m["pe_baseline_modifier"] = { + path: {"2024-01-01": schedule["2024-10-30"]} + for path, schedule in m["pe_baseline_modifier"].items() + } + m["missing_legs"] = list(m["missing_legs"]) + m["missing_legs"][1] = ( + "Current-law CGT rate parameters carry fiscal_year_blend metadata: the processed CY2024 rate includes the 30 October 2024 increase. The dictionary reversal starts 1 January 2024 in the processed tree, restoring the pre-announcement rate against that blended annual current law without another fiscal-year conversion." + ) + m["note"] = ( + "Restore pre-announcement CGT main rates from annual CY2024 against the engine's fiscal-year-blended current law. Evidence: policyengine_uk/utils/parameters.py and fiscal_year_blend metadata on gov.hmrc.cgt basic/higher/additional_rate. Pooled gains still include residential gains already taxed at 18%/24% before AB2024, so the old 10%/20% pooled reversal retains that construction limit. Receipt timing and realisations responses remain divergence axes." + ) + m["construction_adjustments"] = [ + { + "id": "cgt_fiscal_year_blend_annual_reversal", + "from_date": "2024-10-30", + "to_date": "2024-01-01", + "evidence": "parameter_processing_evidence.metadata_by_path", + } + ] + # These two original registries are inputs bound by the track-1 + # receipts. Preserve their authored wording while the corrected + # explanation remains in RECIPE.md; subsequent bundles use the + # current construction description above. + if bundle == DEFAULT_BUNDLE: + if ( + event_slug == "autumn_budget_2024" + and key == "sdlt_additional_dwelling_surcharge_2pp" + ): + m["note"] = ( + "The pin still has the 3% surcharge scale, so the announcement is a forward 2pp increase to every marginal bracket. Government parameter processing samples 30 April and applies that value across each annual period; the October2024 onset is therefore absent from processed 2024 and present throughout processed 2025. Main-home rates and thresholds are retained." + ) + elif ( + event_slug == "autumn_budget_2024" + and key == "capital_gains_main_rates_and_reliefs" + ): + m["missing_legs"] = list(m["missing_legs"]) + m["missing_legs"][1] = ( + "The pinned main-rate parameters start on 2025-04-06, although the announcement starts on 2024-10-30. Government parameter processing samples 30 April and applies that value across each annual period, so the first higher processed year is 2025. No onset repair is applied." + ) + m["note"] = ( + "A literal pre-announcement main-rate reversal, not a repair of the certified world's delayed rate onset or gains composition. Receipt timing and realisations responses remain divergence axes." + ) + elif ( + event_slug == "spring_budget_2024" + and key == "class_1_employee_nics_main_rate_cut_2pp" + ): + m["note"] = ( + "The 2pp marginal reversal on the certified 8% world is 10%. For AS2023, CY2023 has zero because commencement was January 2024; CY-proxies-FY explicitly misses the January-March FY2023-24 leg. Later SB2024 cuts are retained in the certified world." + ) classification = m["classification"] search = "|".join( re.escape(p) @@ -812,7 +899,7 @@ def build_registry(event_slug, engine_parameters, engine_variables, resolve): "pattern": search, "parameter_matches": ph, "variable_matches": vh, - "engine_version": "2.89.2", + "engine_version": versions["policyengine-uk"], "searched": ["full_parameter_tree", "full_variable_list"], }, } @@ -821,6 +908,8 @@ def build_registry(event_slug, engine_parameters, engine_variables, resolve): m["unmapped_obr_heads"] = sorted({r["tax_head"] for r in rs} - mapped) for row in rs: r = dict(row) + if bundle != DEFAULT_BUNDLE and not start <= int(r["fy"][:4]) <= end: + r["computation_status"] = "outside_bundle_window" if classification == "out_of_household_scope" or is_non_household_head( r["tax_head"] ): @@ -864,7 +953,7 @@ def build_registry(event_slug, engine_parameters, engine_variables, resolve): if len(set(keys)) != len(keys): raise ValueError("duplicate measure key") manifest = json.loads(SOURCE_MANIFEST.read_text()) - bundle = json.loads(CERTIFIED.read_text()) + pin = bundle_document(bundle) return { "schema_version": 1, "event_slug": event_slug, @@ -872,11 +961,14 @@ def build_registry(event_slug, engine_parameters, engine_variables, resolve): "classification_note": "not_expressible means no established executable construction in this replay. gap_kind distinguishes evidenced model/data gaps from construction_pending decisions; pending constructions do not claim the underlying liability model is absent.", "calendar_years": years, "years_note": ( - "Calendar year starting the OBR fiscal year is the proxy. FY before 2023-24 stays accounted but is not simulated. Detailed pinned-engine/data support is recorded in docs/uk_replay/YEARS.md." + f"Calendar year starting the OBR fiscal year is the proxy. FY outside {start}-{end} stays accounted once as outside_bundle_window and is not simulated. Bundle-specific engine/data evidence is recorded in docs/uk_replay/YEARS_{bundle}.md." + if bundle != DEFAULT_BUNDLE + else "Calendar year starting the OBR fiscal year is the proxy. FY before 2023-24 stays accounted but is not simulated. Detailed pinned-engine/data support is recorded in docs/uk_replay/YEARS.md." if event_slug in EVENTS else "Calendar year starting the OBR fiscal year is the proxy. FY before 2023-24 or starting after 2030 stays accounted but is not simulated. Detailed pinned-engine/data support is recorded in docs/uk_replay/YEARS.md." ), - "bundle": bundle, + "bundle": pin, + **(bundle_identity(bundle) if bundle != DEFAULT_BUNDLE else {}), "source": manifest, "orientation": "positive means gain to the Exchequer; reversal effect = -(literal reversal - certified baseline)", "accounting": accounting(rows, measures), @@ -890,7 +982,7 @@ def engine_resolver(system=None): if system is None: system = CountryTaxBenefitSystem() - def resolve(path, date): + def node_at(path): node = system.parameters for part in path.split("."): match = re.fullmatch(r"(.+)\[(\d+)\]", part) @@ -899,11 +991,15 @@ def resolve(path, date): if match else getattr(node, part) ) - value = node(date) + return node + + def resolve(path, date): + value = node_at(path)(date) if hasattr(value, "item"): value = value.item() return value + resolve.parameter_metadata = lambda path: dict(node_at(path).metadata or {}) return resolve @@ -915,19 +1011,27 @@ def main(): help="Slug of any fiscal event in the pinned source; 'all' builds the five pilot events.", ) p.add_argument("--check", action="store_true") + p.add_argument("--bundle", default=DEFAULT_BUNDLE) args = p.parse_args() if not re.fullmatch(r"[a-z][a-z0-9_]*", args.event): p.error("--event must be a lowercase fiscal-event slug") for name in ("HF_HUB_OFFLINE", "HF_DATASETS_OFFLINE", "TRANSFORMERS_OFFLINE"): os.environ[name] = "1" + import importlib.metadata + from policyengine_uk import CountryTaxBenefitSystem + expected = bundle_identity(args.bundle)["engine_versions"] + for package, version in expected.items(): + if importlib.metadata.version(package) != version: + raise ValueError(f"selected bundle requires {package}=={version}") + system = CountryTaxBenefitSystem() - params, variables = engine_dumps(system=system) + params, variables = engine_dumps(pin=expected["policyengine-uk"], system=system) resolve = engine_resolver(system) for event_slug in EVENTS if args.event == "all" else [args.event]: - registry = build_registry(event_slug, params, variables, resolve) - output = ROOT / "data/uk/events" / f"{event_slug}_measures.json" + registry = build_registry(event_slug, params, variables, resolve, args.bundle) + output = registry_path(event_slug, args.bundle) content = canonical_bytes(registry) if args.check: if not output.exists() or output.read_bytes() != content: diff --git a/pipeline/compare_uk_event.py b/pipeline/compare_uk_event.py index c5bf7108..deddd22f 100644 --- a/pipeline/compare_uk_event.py +++ b/pipeline/compare_uk_event.py @@ -24,6 +24,7 @@ ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) +from pipeline import uk_bundle as bundles from pipeline.build_uk_event_registry import _sum as source_gbp_sum from pipeline.compare_uk_obr_costings import atomic_write_bytes, ratio_and_bin @@ -543,8 +544,22 @@ def describe_decomposition( def build_comparison_rows( - registry: dict, staged_rows: list[dict], *, artifact_root: Path = ROOT + registry: dict, + staged_rows: list[dict], + *, + artifact_root: Path = ROOT, + bundle: str | None = None, + registry_sha256: str | None = None, ) -> list[dict]: + selected = bundle or bundles.DEFAULT_BUNDLE + verify_bundle = ( + bundle is not None or "bundle" in registry or "bundle_key" in registry + ) + if verify_bundle: + try: + bundles.validate_registry_bundle(registry, selected, root=artifact_root) + except ValueError as exc: + raise EventComparisonError(str(exc)) from exc inventory = {} for measure in registry["measures"]: for source in measure["source_rows"]: @@ -589,6 +604,14 @@ def build_comparison_rows( f"{key}: numerical source amount differs from decimal commitment" ) pe = row.get("pe_value") + if selected != bundles.DEFAULT_BUNDLE: + start, end = bundles.bundle_window(selected, root=artifact_root) + if not start <= int(source["fy"][:4]) <= end and ( + pe is not None or row.get("status") != "outside_bundle_window" + ): + raise EventComparisonError( + "pre-window source row must be accounted as outside_bundle_window" + ) variables = row.get("head_variables", []) reference = row.get("artifact_path") digest = row.get("artifact_sha256") @@ -612,6 +635,35 @@ def build_comparison_rows( if hashlib.sha256(payload).hexdigest() != digest: raise EventComparisonError(f"{key}: artifact SHA-256 differs") artifact = json.loads(payload) + if verify_bundle: + try: + bundles.validate_bundle_path( + path, selected, kind="results", root=artifact_root + ) + bundles.validate_runtime_bundle( + artifact, selected, root=artifact_root + ) + except ValueError as exc: + raise EventComparisonError(str(exc)) from exc + if artifact.get("event") != registry["event_slug"]: + raise EventComparisonError( + "artifact event differs from registry" + ) + if ( + registry_sha256 is not None + and artifact.get("registry_sha256") != registry_sha256 + ): + raise EventComparisonError( + "artifact registry SHA-256 differs from current registry" + ) + if ( + not bundles.bundle_window(selected, root=artifact_root)[0] + <= artifact.get("year", 0) + <= bundles.bundle_window(selected, root=artifact_root)[1] + ): + raise EventComparisonError( + "computed artifact is outside selected bundle window" + ) validate_artifact(artifact, reference) artifacts[reference] = (artifact, digest) artifact, saved_digest = artifacts[reference] @@ -705,7 +757,9 @@ def build_comparison_rows( "ratio_bin": bin_name, "status": row.get("status"), "reason": reason, - "construction_note": construction_note(measure), + "construction_note": construction_note(measure) + if selected == bundles.DEFAULT_BUNDLE + else measure.get("note", ""), "missing_legs": measure.get("missing_legs", []), "head_variables": variables, "artifact": reference, @@ -713,10 +767,14 @@ def build_comparison_rows( "axes": axes, "diagnosis": { **(row.get("diagnosis") or {}), - **review_diagnosis( - measure["measure_key"], - int(source["fy"][:4]), - source.get("tax_head", ""), + **( + review_diagnosis( + measure["measure_key"], + int(source["fy"][:4]), + source.get("tax_head", ""), + ) + if selected == bundles.DEFAULT_BUNDLE + else {} ), }, **describe_decomposition(gap, axes, components), @@ -770,7 +828,10 @@ def profile(rows: list[dict], field: str) -> list[tuple[str, dict[str, int]]]: def render_markdown( - registry: dict, rows: list[dict], replay_grid: dict | None = None + registry: dict, + rows: list[dict], + replay_grid: dict | None = None, + bundle: str = bundles.DEFAULT_BUNDLE, ) -> str: computed = [r for r in rows if r["pe_value_gbp"] is not None] lines = [ @@ -952,14 +1013,23 @@ def render_markdown( caveats = [ measure for measure in registry["measures"] - if construction_note(measure) + if ( + construction_note(measure) + if bundle == bundles.DEFAULT_BUNDLE + else measure.get("note", "") + ) and measure.get("classification", measure.get("computability")) in ("expressible", "partial") ] if caveats: lines += ["", "## Construction caveats", ""] for measure in caveats: - lines.append(f"* `{measure['measure_key']}`: {construction_note(measure)}") + note = ( + construction_note(measure) + if bundle == bundles.DEFAULT_BUNDLE + else measure.get("note", "") + ) + lines.append(f"* `{measure['measure_key']}`: {note}") diagnosed = {} for row in rows: diagnosis = row.get("diagnosis") or {} @@ -994,19 +1064,41 @@ def _receipt_path(reference: Any, artifact_root: Path) -> Path: def _verify_staging( - registry_path: Path, staged_path: Path, *, artifact_root: Path + registry_path: Path, + staged_path: Path, + *, + artifact_root: Path, + bundle: str | None = None, ) -> tuple[dict, list[dict], Path]: registry = _json(registry_path) manifest_path = staged_path.parent / "STAGING_MANIFEST.json" if not manifest_path.exists(): raise EventComparisonError("comparison requires STAGING_MANIFEST.json") manifest = _json(manifest_path) + if bundle is not None or "bundle" in registry or "bundle_key" in registry: + selected = bundle or bundles.DEFAULT_BUNDLE + try: + for path, kind in ( + (registry_path, "registry"), + (staged_path, "results"), + (manifest_path, "results"), + ): + bundles.validate_bundle_path( + path, selected, kind=kind, root=artifact_root + ) + bundles.validate_bundle_identity(manifest, selected, root=artifact_root) + except ValueError as exc: + raise EventComparisonError(str(exc)) from exc if manifest.get("registry_sha256") != _sha256(registry_path): raise EventComparisonError("registry SHA-256 differs from staging manifest") if manifest.get("staged_sha256") != _sha256(staged_path): raise EventComparisonError("staged SHA-256 differs from staging manifest") rows = build_comparison_rows( - registry, load_jsonl(staged_path), artifact_root=artifact_root + registry, + load_jsonl(staged_path), + artifact_root=artifact_root, + bundle=bundle, + registry_sha256=_sha256(registry_path), ) if manifest.get("staged_rows") != len(rows): raise EventComparisonError("staging manifest row count differs") @@ -1041,18 +1133,27 @@ def write_comparison( staged_path: Path | None = None, output_dir: Path | None = None, artifact_root: Path = ROOT, + bundle: str | None = None, ) -> list[dict]: - registry_path = ( - registry_path - or artifact_root / "data" / "uk" / "events" / f"{event}_measures.json" + selected = bundle or bundles.DEFAULT_BUNDLE + registry_path = registry_path or bundles.registry_path( + event, selected, root=artifact_root + ) + output_dir = output_dir or bundles.event_output_dir( + event, selected, root=artifact_root ) - output_dir = output_dir or artifact_root / "results" / "uk" / "events" / event staged_path = staged_path or output_dir / "STAGED.jsonl" registry, rows, staging_manifest_path = _verify_staging( - registry_path, staged_path, artifact_root=artifact_root + registry_path, staged_path, artifact_root=artifact_root, bundle=bundle ) if registry.get("event_slug") != event: raise EventComparisonError("registry event differs from --event") + try: + bundles.validate_bundle_path( + output_dir, selected, kind="results", root=artifact_root + ) + except ValueError as exc: + raise EventComparisonError(str(exc)) from exc output_dir.mkdir(parents=True, exist_ok=True) staging_manifest = _json(staging_manifest_path) replay_grid = { @@ -1060,9 +1161,22 @@ def write_comparison( for field in GRID_FIELDS if field in staging_manifest } + markdown = render_markdown(registry, rows, replay_grid, bundle=selected) + if selected != bundles.DEFAULT_BUNDLE: + year = bundles.bundle_window(selected, root=artifact_root)[0] + markdown = markdown.replace("2023 population", f"{year} population") + markdown = markdown.replace("one 2023 population", f"one {year} population") + markdown = markdown.replace( + "with government baseline policy parameters annualized from the engine's 30 April snapshot.", + "with government baseline policy parameters processed by the selected engine's fiscal-year conversion, including declared calendar-date exceptions.", + ) + markdown = markdown.replace( + "without a new 30 April conversion", + "without repeating fiscal-year conversion", + ) payloads = { "COMPARISON.csv": render_csv(rows), - "COMPARISON.md": render_markdown(registry, rows, replay_grid), + "COMPARISON.md": markdown, "COMPARISON.json": json.dumps(rows, indent=1, sort_keys=True, allow_nan=False) + "\n", } @@ -1086,6 +1200,8 @@ def write_comparison( for name, payload in payloads.items() }, } + if selected != bundles.DEFAULT_BUNDLE: + provenance.update(bundles.bundle_identity(selected, root=artifact_root)) atomic_write_bytes( output_dir / "COMPARISON_PROVENANCE.json", (json.dumps(provenance, indent=1, sort_keys=True) + "\n").encode(), @@ -1098,6 +1214,7 @@ def render_summary( diagnostics: list[dict] | None = None, registries: dict[str, dict] | None = None, replay_grids: dict[str, dict] | None = None, + bundle: str = bundles.DEFAULT_BUNDLE, ) -> str: rows = [row for event_rows in events.values() for row in event_rows] computed = [r for r in rows if r["pe_value_gbp"] is not None] @@ -1258,8 +1375,12 @@ def render_summary( diagnostic = row.get("diagnosis") or {} diagnostic = { **diagnostic, - **review_diagnosis( - row["measure_key"], row["year"], row.get("tax_head", "") + **( + review_diagnosis( + row["measure_key"], row["year"], row.get("tax_head", "") + ) + if bundle == bundles.DEFAULT_BUNDLE + else {} ), } evidence = diagnostic.get("evidence", "Open: relevant axes unsized") @@ -1319,12 +1440,23 @@ def render_summary( return "\n".join(lines) -def load_verified_comparison(path: Path, *, artifact_root: Path = ROOT) -> list[dict]: +def load_verified_comparison( + path: Path, *, artifact_root: Path = ROOT, bundle: str | None = None +) -> list[dict]: """Bind summary-only reads to current source, stage, axes and artifact bytes.""" provenance_path = path.parent / "COMPARISON_PROVENANCE.json" if not provenance_path.exists(): raise EventComparisonError("summary requires COMPARISON_PROVENANCE.json") provenance = _json(provenance_path) + selected = bundle or bundles.DEFAULT_BUNDLE + if bundle is not None or "bundle_key" in provenance: + try: + bundles.validate_bundle_path( + path, selected, kind="results", root=artifact_root + ) + bundles.validate_bundle_identity(provenance, selected, root=artifact_root) + except ValueError as exc: + raise EventComparisonError(str(exc)) from exc if provenance.get("event_slug") != path.parent.name: raise EventComparisonError("comparison provenance event differs") if provenance.get("axes_sha256") != _sha256(AXES_PATH): @@ -1342,7 +1474,7 @@ def load_verified_comparison(path: Path, *, artifact_root: Path = ROOT) -> list[ raise EventComparisonError(f"comparison {label} SHA-256 is stale") paths[label] = receipt registry, verified_rows, manifest_path = _verify_staging( - paths["registry"], paths["staged"], artifact_root=artifact_root + paths["registry"], paths["staged"], artifact_root=artifact_root, bundle=bundle ) if manifest_path != paths["staging_manifest"]: raise EventComparisonError("comparison staging manifest path differs") @@ -1352,8 +1484,8 @@ def load_verified_comparison(path: Path, *, artifact_root: Path = ROOT) -> list[ raise EventComparisonError("comparison replay-grid receipt differs") if registry.get("event_slug") != path.parent.name: raise EventComparisonError("summary registry event differs") - canonical_registry = ( - artifact_root / "data" / "uk" / "events" / f"{path.parent.name}_measures.json" + canonical_registry = bundles.registry_path( + path.parent.name, selected, root=artifact_root ) if ( canonical_registry.exists() @@ -1366,9 +1498,13 @@ def load_verified_comparison(path: Path, *, artifact_root: Path = ROOT) -> list[ return rows -def write_summary(events_root: Path, *, artifact_root: Path = ROOT) -> None: +def write_summary( + events_root: Path, *, artifact_root: Path = ROOT, bundle: str | None = None +) -> None: events = { - path.parent.name: load_verified_comparison(path, artifact_root=artifact_root) + path.parent.name: load_verified_comparison( + path, artifact_root=artifact_root, bundle=bundle + ) for path in sorted(events_root.glob("*/COMPARISON.json")) } replay_grids = { @@ -1381,29 +1517,63 @@ def write_summary(events_root: Path, *, artifact_root: Path = ROOT) -> None: diagnostics = ( json.loads(diagnostic_path.read_text()) if diagnostic_path.exists() else [] ) + for diagnostic in diagnostics: + try: + bundles.validate_runtime_bundle( + diagnostic, bundle or bundles.DEFAULT_BUNDLE, root=artifact_root + ) + except ValueError as exc: + raise EventComparisonError(str(exc)) from exc registries = { path.stem.removesuffix("_measures"): _json(path) for path in sorted( - (artifact_root / "data" / "uk" / "events").glob("*_measures.json") + bundles.registry_path( + "event", bundle or bundles.DEFAULT_BUNDLE, root=artifact_root + ).parent.glob("*_measures.json") ) } - events_root.mkdir(parents=True, exist_ok=True) - atomic_write_bytes( - events_root / "SUMMARY.md", - render_summary(events, diagnostics, registries, replay_grids).encode(), + selected = bundle or bundles.DEFAULT_BUNDLE + try: + bundles.validate_bundle_path( + events_root, selected, kind="results", root=artifact_root + ) + for registry in registries.values(): + if bundle is not None or "bundle" in registry or "bundle_key" in registry: + bundles.validate_registry_bundle(registry, selected, root=artifact_root) + except ValueError as exc: + raise EventComparisonError(str(exc)) from exc + summary = render_summary( + events, diagnostics, registries, replay_grids, bundle=selected ) + if selected != bundles.DEFAULT_BUNDLE: + pin = bundles.load_bundle(selected, root=artifact_root) + model = bundles.bundle_identity(selected, root=artifact_root)[ + "engine_versions" + ]["policyengine-uk"] + summary = summary.replace( + "pinned populace-uk-2023 bundle definition with", + f"pinned {selected} bundle definition with", + ) + summary = summary.replace("policyengine-uk 2.89.2", f"policyengine-uk {model}") + summary = summary.replace( + "certified 2023 population", f"certified {pin['data_year']} population" + ) + # Legacy review diagnoses concern only their recorded bundle. New + # comparisons carry diagnoses solely from their audited stage inputs. + summary = summary.replace("--event ", f"--bundle {selected} --event ") + events_root.mkdir(parents=True, exist_ok=True) + atomic_write_bytes(events_root / "SUMMARY.md", summary.encode()) def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--event") + parser.add_argument("--bundle", default=bundles.DEFAULT_BUNDLE) parser.add_argument("--registry", type=Path) parser.add_argument("--staged", type=Path) parser.add_argument("--output-dir", type=Path) parser.add_argument("--summary", action="store_true") - parser.add_argument( - "--events-root", type=Path, default=ROOT / "results" / "uk" / "events" - ) + parser.add_argument("--events-root", type=Path) args = parser.parse_args(argv) if args.event: rows = write_comparison( @@ -1411,13 +1581,15 @@ def main(argv: list[str] | None = None) -> int: registry_path=args.registry, staged_path=args.staged, output_dir=args.output_dir, + bundle=args.bundle, ) print(f"{args.event}: wrote {len(rows)} descriptive comparison rows") elif not args.summary: parser.error("--event or --summary is required") if args.summary: - write_summary(args.events_root) - print(f"wrote {args.events_root / 'SUMMARY.md'}") + events_root = args.events_root or bundles.results_root(args.bundle) + write_summary(events_root, bundle=args.bundle) + print(f"wrote {events_root / 'SUMMARY.md'}") return 0 diff --git a/pipeline/compute_uk_ab2025.py b/pipeline/compute_uk_ab2025.py index 7dd31507..2512464c 100644 --- a/pipeline/compute_uk_ab2025.py +++ b/pipeline/compute_uk_ab2025.py @@ -440,7 +440,21 @@ def not_computable( # --- the certified world ------------------------------------------------------- -def preflight() -> dict: +def preflight(bundle_key=None) -> dict: + from pipeline import uk_bundle + + if bundle_key is not None and bundle_key != uk_bundle.DEFAULT_BUNDLE: + from pipeline.compute_uk_obr_costings import preflight_certified_dataset + + managed = preflight_certified_dataset(bundle_key) + pin = uk_bundle.load_bundle(bundle_key) + return { + **managed, + "artifact": Path(managed["runtime_dataset_source"]), + "revision": pin["revision"], + "repo_id": pin["repo_id"], + "engine_version": importlib.metadata.version("policyengine-uk"), + } configure_offline() import policyengine as pe from huggingface_hub import hf_hub_download diff --git a/pipeline/compute_uk_event.py b/pipeline/compute_uk_event.py index f5d02e39..d4759e62 100644 --- a/pipeline/compute_uk_event.py +++ b/pipeline/compute_uk_event.py @@ -25,6 +25,7 @@ from pipeline import build_uk_event_registry as registry_builder from pipeline import compute_uk_ab2025 as worlds from pipeline import compute_uk_obr_costings as fiscal +from pipeline import uk_bundle ROOT = worlds.ROOT EVENT_PATTERN = re.compile(r"^[a-z0-9]+(?:_[a-z0-9]+)*$") @@ -98,19 +99,34 @@ def apply_and_refresh(sim): return Scenario(simulation_modifier=apply_and_refresh) -def event_paths(event: str) -> tuple[Path, Path]: +def event_paths( + event: str, bundle: str = uk_bundle.DEFAULT_BUNDLE +) -> tuple[Path, Path]: if not EVENT_PATTERN.fullmatch(event): raise ValueError("event must be a lowercase underscore-separated slug") return ( - ROOT / "data" / "uk" / "events" / f"{event}_measures.json", - ROOT / "results" / "uk" / "events" / event, + uk_bundle.registry_path(event, bundle, root=ROOT), + uk_bundle.event_output_dir(event, bundle, root=ROOT), ) -def validate_event_identity(registry: dict, event: str) -> None: +def validate_event_identity( + registry: dict, event: str, bundle: str = uk_bundle.DEFAULT_BUNDLE +) -> None: identity = registry.get("event_slug", registry.get("event")) if identity is not None and identity != event: raise ValueError("registry event identity differs from --event") + if ( + bundle != uk_bundle.DEFAULT_BUNDLE + or "bundle" in registry + or "bundle_key" in registry + ): + uk_bundle.validate_registry_bundle(registry, bundle, root=ROOT) + + +def validate_output_bundle(path: Path, bundle: str) -> None: + """Custom development outputs are allowed; another canonical bundle is not.""" + uk_bundle.validate_bundle_path(path, bundle, kind="results", root=ROOT) def canonical_bytes(value: object) -> bytes: @@ -189,6 +205,7 @@ def build_artifact( certified: dict, registry_sha256: str, event: str, + bundle: str = uk_bundle.DEFAULT_BUNDLE, ) -> dict: """Numerical facts only: identical inputs produce identical artifact bytes. @@ -275,13 +292,17 @@ def build_artifact( } if reversal: artifact["literal_reversal_minus_certified_gbp"] = literal + if bundle != uk_bundle.DEFAULT_BUNDLE: + artifact.update(uk_bundle.bundle_identity(bundle)) return artifact -def preflight() -> dict: +def preflight(bundle: str = uk_bundle.DEFAULT_BUNDLE) -> dict: """The committed pin and the managed release must agree before any run.""" fiscal.configure_offline() worlds.configure_offline() + if bundle != uk_bundle.DEFAULT_BUNDLE: + return fiscal.preflight_certified_dataset(bundle) committed = worlds.preflight() managed = fiscal.preflight_certified_dataset() if ( @@ -300,6 +321,8 @@ def run_sim(year: int, variables: list[str], reform: dict | None, pre: dict) -> else None ) scenario_kwargs = {"scenario": scenario} if scenario is not None else {} + if pre.get("bundle_key"): + scenario_kwargs["bundle_key"] = pre["bundle_key"] result = fiscal.run_managed_simulation( year=year, variables=variables, @@ -365,6 +388,7 @@ def run_year_job(job: dict) -> dict: certified=baseline, registry_sha256=job["registry_sha256"], event=job["event"], + bundle=job.get("bundle", uk_bundle.DEFAULT_BUNDLE), ) path = job["output_dir"] / f"{key}_{year}.json" fiscal.atomic_write_bytes(path, canonical_bytes(artifact)) @@ -387,7 +411,7 @@ def run_year_job(job: dict) -> dict: def progress_receipt(job: dict, artifacts: dict[str, str]) -> dict: """Commit completed artifact bytes before the next simulation begins.""" - return { + receipt = { "schema_version": 1, "receipt_kind": "completed_event_artifacts", "event": job["event"], @@ -399,6 +423,10 @@ def progress_receipt(job: dict, artifacts: dict[str, str]) -> dict: **runtime_versions(), "artifacts": artifacts, } + bundle = job.get("bundle", job["pre"].get("bundle_key", uk_bundle.DEFAULT_BUNDLE)) + if bundle != uk_bundle.DEFAULT_BUNDLE: + receipt.update(uk_bundle.bundle_identity(bundle)) + return receipt def retained_artifact_digests( @@ -408,12 +436,15 @@ def retained_artifact_digests( years: list[int], registry_sha256: str, pre: dict, + bundle: str = uk_bundle.DEFAULT_BUNDLE, ) -> dict[str, str]: """Read only SHA-bound completed outputs; unreceipted files are ignored.""" receipts = [] manifest_path = output_dir / "RUN_MANIFEST.json" if manifest_path.exists(): manifest = json.loads(manifest_path.read_bytes()) + if bundle != uk_bundle.DEFAULT_BUNDLE or "bundle_key" in manifest: + uk_bundle.validate_runtime_bundle(manifest, bundle) if ( manifest.get("event") == event and manifest.get("registry_sha256") == registry_sha256 @@ -425,12 +456,15 @@ def retained_artifact_digests( if not path.exists(): continue receipt = json.loads(path.read_bytes()) + if bundle != uk_bundle.DEFAULT_BUNDLE or "bundle_key" in receipt: + uk_bundle.validate_runtime_bundle(receipt, bundle) expected = progress_receipt( { "event": event, "year": year, "registry_sha256": registry_sha256, "pre": pre, + "bundle": bundle, }, {}, ) @@ -458,14 +492,17 @@ def grid_rows(pairs) -> list[dict]: return [{"measure_key": key, "year": year} for key, year in sorted(pairs)] -def validate_years(requested: list[int], registry: dict) -> list[int]: +def validate_years( + requested: list[int], registry: dict, bundle: str = uk_bundle.DEFAULT_BUNDLE +) -> list[int]: allowed = registry.get("calendar_years") if not allowed: raise ValueError("registry must record the supported calendar_years") years = sorted(set(requested or allowed)) - if any(year < 2023 or year > 2030 for year in years): + start, end = uk_bundle.bundle_window(bundle) + if any(year < start or year > end for year in years): raise ValueError( - "certified single-population replay supports 2023 through 2030 only" + f"certified single-population replay supports {start} through {end} only" ) if set(years) - set(allowed): raise ValueError( @@ -510,6 +547,7 @@ def main(argv: list[str] | None = None) -> int: worlds.configure_offline() ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--event", required=True) + ap.add_argument("--bundle", default=uk_bundle.DEFAULT_BUNDLE) ap.add_argument("--registry", type=Path) ap.add_argument("--output-dir", type=Path) ap.add_argument("--measures", nargs="*") @@ -525,12 +563,16 @@ def main(argv: list[str] | None = None) -> int: help="spawn at most two year workers; each holds one live managed simulation", ) args = ap.parse_args(argv) - default_registry, default_output = event_paths(args.event) + default_registry, default_output = event_paths(args.event, args.bundle) registry_path = args.registry or default_registry + uk_bundle.validate_bundle_path( + registry_path, args.bundle, kind="registry", root=ROOT + ) output_dir = (args.output_dir or default_output).resolve() registry_payload = registry_path.read_bytes() registry = json.loads(registry_payload) - validate_event_identity(registry, args.event) + validate_event_identity(registry, args.event, args.bundle) + validate_output_bundle(output_dir, args.bundle) registry_builder.validate_registry(registry) if ( not (args.dry_run or args.guards) @@ -544,7 +586,7 @@ def main(argv: list[str] | None = None) -> int: "or an explicit --output-dir" ) registry_sha256 = fiscal.hashlib.sha256(registry_payload).hexdigest() - years = validate_years(args.years or [], registry) + years = validate_years(args.years or [], registry, args.bundle) index = {measure["measure_key"]: measure for measure in registry["measures"]} planned = worlds.computable(index, years=years) runnable = {key: world for key, world in planned.items() if "alias_of" not in world} @@ -594,7 +636,11 @@ def main(argv: list[str] | None = None) -> int: started = time.perf_counter() verify_registry_identity(registry_path, registry_sha256) print(f"[pid {os.getpid()}] starting offline bundle preflight", flush=True) - pre = preflight() + pre = ( + preflight() + if args.bundle == uk_bundle.DEFAULT_BUNDLE + else preflight(args.bundle) + ) print( f"[pid {os.getpid()}] offline bundle preflight complete; starting processed-world reversal guard", flush=True, @@ -614,6 +660,7 @@ def main(argv: list[str] | None = None) -> int: years=registry["calendar_years"], registry_sha256=registry_sha256, pre=pre, + bundle=args.bundle, ) requested_pairs = {(key, year) for key in selected for year in years} full_pairs = { @@ -640,6 +687,8 @@ def main(argv: list[str] | None = None) -> int: f"retained artifact bytes differ from manifest: {relative}" ) previous = json.loads(path.read_bytes()) + if args.bundle != uk_bundle.DEFAULT_BUNDLE or "bundle_key" in previous: + uk_bundle.validate_runtime_bundle(previous, args.bundle) if ( previous.get("registry_sha256") != registry_sha256 or previous.get("certified_dataset_sha256") != pre["sha256"] @@ -678,6 +727,7 @@ def main(argv: list[str] | None = None) -> int: "pre": pre, "registry_sha256": registry_sha256, "event": args.event, + "bundle": args.bundle, "output_dir": output_dir, "retained_artifacts": { str( @@ -723,6 +773,8 @@ def main(argv: list[str] | None = None) -> int: **runtime_versions(), "not_computable": worlds.not_computable(index, years=years), } + if args.bundle != uk_bundle.DEFAULT_BUNDLE: + manifest.update(uk_bundle.bundle_identity(args.bundle)) fiscal.atomic_write_bytes( output_dir / "RUN_MANIFEST.json", canonical_bytes(manifest) ) diff --git a/pipeline/compute_uk_obr_costings.py b/pipeline/compute_uk_obr_costings.py index c95e0999..2d312dc9 100644 --- a/pipeline/compute_uk_obr_costings.py +++ b/pipeline/compute_uk_obr_costings.py @@ -31,13 +31,20 @@ import time import unicodedata from collections import Counter +from collections.abc import Iterable +from contextlib import contextmanager from datetime import datetime, timezone from importlib.metadata import PackageNotFoundError, version from pathlib import Path -from typing import Any, Iterable +from typing import Any import yaml +if __package__ in (None, ""): + # Preserve the historical direct-file CLI as well as ``python -m``. + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from pipeline import uk_bundle as certified_bundles + ROOT = Path(__file__).resolve().parent.parent DEFAULT_REGISTRY = ROOT / "data" / "uk" / "obr_measure_reforms.yaml" DEFAULT_CLAIMS = ( @@ -842,7 +849,9 @@ def ensure_writable_local_mirror( def assert_engine_matches_bundle( - release_bundle: dict[str, Any], installed: str | None = None + release_bundle: dict[str, Any], + installed: str | None = None, + bundle_key: str = certified_bundles.DEFAULT_BUNDLE, ) -> str: """The certified bundle DECLARES the engine it was built for; a right artifact on a wrong engine is still a wrong run (#126). The managed @@ -864,7 +873,7 @@ def assert_engine_matches_bundle( f"installed policyengine-uk {installed} != bundle-declared {declared}; " "the bundle's compatibility statement is the pin, not the environment" ) - pinned = json.loads((ROOT / "data" / "uk" / "certified_bundle.json").read_text()) + pinned = certified_bundles.load_bundle(bundle_key) for spec in pinned.get("compatible_model_packages", []): if spec.get("name") == "policyengine-uk": want = str(spec.get("specifier", "")).lstrip("=") @@ -876,10 +885,119 @@ def assert_engine_matches_bundle( return installed -def preflight_certified_dataset() -> dict[str, Any]: +def block_runtime_network() -> None: + """Make the packaged certification fallback mandatory in this process.""" + + def deny_network(event, args): + if event == "socket.connect": + raise OSError("certified replay runtime network is blocked") + + sys.addaudithook(deny_network) + + +@contextmanager +def managed_dataset_directory(runtime_dataset_source: Path, bundle_key: str): + """6.x materializes into ./data; isolate that directory for each pin.""" + original = Path.cwd() + if bundle_key != certified_bundles.DEFAULT_BUNDLE: + os.chdir(runtime_dataset_source.parent.parent) + try: + yield + finally: + os.chdir(original) + + +def _preflight_materialized_dataset(bundle_key: str) -> dict[str, Any]: + """Seed 6.x's local materializer from certified cached bytes, offline.""" + pin = certified_bundles.load_bundle(bundle_key) + block_runtime_network() + import policyengine as pe + from huggingface_hub import hf_hub_download + + bundle = dict(pe.uk.uk_latest.release_bundle) + assert_engine_matches_bundle(bundle, bundle_key=bundle_key) + for package, expected in certified_bundles.bundle_identity(bundle_key)[ + "engine_versions" + ].items(): + if package_version(package) != expected: + raise RuntimeError( + f"{package} installed version differs from selected bundle pin" + ) + expected_uri = f"hf://{pin['repo_id']}/{pin['artifact']}@{pin['revision']}" + if ( + bundle.get("default_dataset_uri") != expected_uri + or bundle.get("certified_data_artifact_sha256") != pin["sha256"] + or bundle.get("certified_data_build_id") + != pin.get("data_build_id", pin["revision"]) + ): + raise RuntimeError("managed release differs from selected certified bundle") + if bundle.get("compatibility_basis", "").startswith("unverified"): + raise RuntimeError("managed release lacks packaged compatibility certification") + cached = Path( + hf_hub_download( + repo_id=pin["repo_id"], + repo_type=pin["repo_type"], + filename=pin["artifact"], + revision=pin["resolved_hf_commit"], + local_files_only=True, + ) + ) + started = time.perf_counter() + digest = sha256_file(cached) + if digest != pin["sha256"] or cached.stat().st_size != pin["size_bytes"]: + raise RuntimeError("cached artifact digest or size differs from certified pin") + import pandas as pd + + periods = pd.read_hdf(cached, "time_period").tolist() + if len(periods) != 1 or int(periods[0]) != pin["data_year"]: + raise RuntimeError("cached artifact data year differs from certified pin") + # UKSingleYearDataset uses HDFStore(mode='r') in 6.x. A symlink avoids a + # second dataset copy while giving materialize_dataset its required path. + runtime = ( + LOCAL_DATA_MIRROR_ROOT + / "bundles" + / bundle_key + / "data" + / Path(pin["artifact"]).name + ) + runtime.parent.mkdir(parents=True, exist_ok=True) + created = not runtime.exists() + if created: + runtime.symlink_to(cached.resolve()) + verify_runtime_dataset_sha256(runtime, pin["sha256"], phase="materializer seed") + from policyengine.provenance.dataset_materialization import materialize_dataset + + with managed_dataset_directory(runtime, bundle_key): + source = materialize_dataset("uk") + if Path(source.path).resolve() != runtime.resolve(): + raise RuntimeError( + "materializer source differs from exact hash-verified file" + ) + return { + "release_bundle": bundle, + "dataset_uri": expected_uri, + "cached_file": cached.name, + "sha256": digest, + "hash_seconds": time.perf_counter() - started, + "local_files_only": True, + "runtime_network_blocked": True, + "managed_local_mirror": relative_to_root(runtime), + "managed_local_mirror_created": created, + # Preserve the isolated materialization path; resolve for hash equality + # only, since its parent determines the 6.x loader's working directory. + "runtime_dataset_source": str(runtime.absolute()), + **certified_bundles.bundle_identity(bundle_key), + } + + +def preflight_certified_dataset( + bundle_key: str = certified_bundles.DEFAULT_BUNDLE, +) -> dict[str, Any]: """Prove that the bundled dataset is cached and matches its manifest hash.""" configure_offline() + if bundle_key != certified_bundles.DEFAULT_BUNDLE: + return _preflight_materialized_dataset(bundle_key) import policyengine as pe from huggingface_hub import hf_hub_download from policyengine.provenance.dataset_sources import parse_hf_uri @@ -964,7 +1082,7 @@ def _sample(self) -> None: rss, _ = current_rss_bytes() self.peak_rss_bytes = max(self.peak_rss_bytes, rss) - def __enter__(self) -> "PeakRSSSampler": + def __enter__(self) -> PeakRSSSampler: self.start_rss_bytes, self.method = current_rss_bytes() self.peak_rss_bytes = self.start_rss_bytes self.thread = threading.Thread(target=self._sample, daemon=True) @@ -1025,6 +1143,7 @@ def run_managed_simulation( expected_dataset_sha256: str, include_engine_provenance: bool = False, scenario: Any = None, + bundle_key: str = certified_bundles.DEFAULT_BUNDLE, ) -> dict[str, Any]: """Run, aggregate, delete, and collect exactly one managed sim.""" @@ -1032,12 +1151,30 @@ def run_managed_simulation( raise ValueError("pass either reform or scenario, not both") configure_offline() + if bundle_key != certified_bundles.DEFAULT_BUNDLE: + block_runtime_network() + pin = certified_bundles.load_bundle(bundle_key) + for package, expected in certified_bundles.bundle_identity(bundle_key)[ + "engine_versions" + ].items(): + if package_version(package) != expected: + raise RuntimeError( + f"{package} installed version differs from selected bundle pin" + ) + first, last = certified_bundles.bundle_window(pin) + if not first <= year <= last: + raise ValueError("requested year is outside the certified bundle window") + if expected_dataset_sha256 != pin["sha256"]: + raise RuntimeError("requested dataset digest differs from selected bundle") import policyengine as pe started = time.perf_counter() sim = None try: - with PeakRSSSampler() as memory: + with ( + managed_dataset_directory(runtime_dataset_source, bundle_key), + PeakRSSSampler() as memory, + ): dataset_sha256_before = verify_runtime_dataset_sha256( runtime_dataset_source, expected_dataset_sha256, @@ -1063,6 +1200,26 @@ def run_managed_simulation( "managed sim runtime_dataset_source differs from the exact " f"file hashed before construction: {reported_source!r}" ) + if bundle_key != certified_bundles.DEFAULT_BUNDLE: + assert_engine_matches_bundle(bundle, bundle_key=bundle_key) + expected_uri = ( + f"hf://{pin['repo_id']}/{pin['artifact']}@{pin['revision']}" + ) + if ( + data_bundle_id(bundle) != pin["data_build_id"] + or bundle.get("default_dataset_uri") != expected_uri + or bundle.get("certified_data_artifact_sha256") != pin["sha256"] + ): + raise RuntimeError( + "managed run release differs from selected certified pin" + ) + if ( + min(sim.dataset.years) != pin["data_year"] + or year not in sim.dataset.years + ): + raise RuntimeError( + "managed simulation did not project the pinned data year" + ) missing = [ name for name in variables diff --git a/pipeline/diagnose_uk_event_models.py b/pipeline/diagnose_uk_event_models.py index eae1ccdb..70e73c21 100644 --- a/pipeline/diagnose_uk_event_models.py +++ b/pipeline/diagnose_uk_event_models.py @@ -18,6 +18,13 @@ import re from pathlib import Path +from pipeline.uk_bundle import ( + DEFAULT_BUNDLE, + bundle_identity, + validate_bundle_identity, + validate_bundle_path, +) + ROOT = Path(__file__).resolve().parent.parent @@ -166,16 +173,191 @@ def _additional_model_checks(package_root: Path, simulation_type, case: str | No return results -def investigate(case: str | None = None) -> list[dict]: +PAIRED_CASES = ( + "annual_allowance_relief_and_charge_pe_uk_2237", + "sdlt_additional_purchase_stock_pe_uk_2238", + "uc_standard_allowance_2027_2030_pe_uk_2239", + "hicbc_opt_out", + "nics_threshold_freeze_end_date", +) + + +def current_bundle_observation(result, identity): + """Keep earlier diagnoses as references, not assertions about a later pin.""" + if identity["bundle_key"] == DEFAULT_BUNDLE: + return result + result.update(identity) + result["legacy_reference"] = { + "bundle_key": DEFAULT_BUNDLE, + "engine_version": "2.89.2", + "class": result["class"], + "interpretation": result["interpretation"], + "evidence": "results/uk/events/MODEL_DIAGNOSTICS.json", + } + result["class"] = "bundle_observation" + result["interpretation"] = ( + "Current parameters, formula source and synthetic observations are recorded " + "for the selected certified bundle. The legacy_reference retains the " + "2.89.2 diagnosis for comparison; interpretation at this pin requires " + "review of the current observations and source. National contributions " + "remain unsized." + ) + return result + + +def _paired_checks(package_root, simulation_type, case): + """Issue-body reproducers, identical controlled inputs at both bundle pins.""" + if case not in PAIRED_CASES: + return [] + year = 2025 if case in (PAIRED_CASES[1], PAIRED_CASES[3]) else 2024 + people = {} + units = {} + households = {} + if case == PAIRED_CASES[0]: + for amount in (60000, 80000): + name = f"contributions_{amount}" + people[name] = { + "age": {str(year): 45}, + "employment_income": {str(year): 150000}, + "personal_pension_contributions": {str(year): amount}, + } + units[name] = {"members": [name]} + households[name] = {"members": [name], "region": {str(year): "LONDON"}} + variables = [ + "pension_contributions_relief", + "personal_pension_contributions_tax", + "income_tax", + ] + evidence = "variables/gov/hmrc/pensions/pension_contributions_relief.py" + issue = 2237 + years = [year] + elif case == PAIRED_CASES[1]: + for purchased in (False, True): + name = f"purchase_{str(purchased).lower()}" + people[name] = { + "age": {str(year): 45}, + "employment_income": {str(year): 80000}, + } + units[name] = {"members": [name]} + households[name] = { + "members": [name], + "region": {str(year): "LONDON"}, + "main_residence_value": {str(year): 600000}, + "other_residential_property_value": {str(year): 2000000}, + "property_purchased": {str(year): purchased}, + } + variables = ["additional_residential_property_purchased", "stamp_duty_land_tax"] + evidence = "variables/household/consumption/additional_residential_property_purchased.py" + issue = 2238 + years = [year] + elif case == PAIRED_CASES[2]: + years = list(range(2025, 2031)) + people["claimant"] = {"age": {str(y): 30 for y in years}} + units["unit"] = {"members": ["claimant"]} + households["household"] = { + "members": ["claimant"], + "region": {str(y): "LONDON" for y in years}, + } + variables = ["uc_standard_allowance"] + evidence = "variables/gov/dwp/universal_credit/standard_allowance/uc_standard_allowance.py" + issue = 2239 + elif case == PAIRED_CASES[3]: + years = [year] + for opted in (False, True): + name = f"opt_out_{str(opted).lower()}" + adult, child = name + "_adult", name + "_child" + people[adult] = { + "age": {str(year): 45}, + "employment_income": {str(year): 100000}, + } + people[child] = {"age": {str(year): 8}} + units[name] = { + "members": [adult, child], + "child_benefit_opts_out": {str(year): opted}, + "would_claim_child_benefit": {str(year): True}, + } + households[name] = { + "members": [adult, child], + "region": {str(year): "LONDON"}, + } + variables = ["child_benefit", "CB_HITC", "child_benefit_less_tax_charge"] + evidence = "variables/gov/hmrc/child_benefit.py" + issue = None + else: + years = list(range(2026, 2031)) + people["worker"] = { + "age": {str(y): 45 for y in years}, + "self_employment_income": {str(y): 30000 for y in years}, + } + units["unit"] = {"members": ["worker"]} + households["household"] = { + "members": ["worker"], + "region": {str(y): "LONDON" for y in years}, + } + variables = ["ni_class_4"] + evidence = "parameters/gov/hmrc/national_insurance/class_4/thresholds/lower_profits_limit.yaml" + issue = None + situation = {"people": people, "benunits": units, "households": households} + simulation = simulation_type(situation=situation) + try: + observation = { + str(y): {v: simulation.calculate(v, y).tolist() for v in variables} + for y in years + } + if case == PAIRED_CASES[4]: + for y in years: + observation[str(y)]["class4_thresholds"] = { + name: resolve( + simulation.tax_benefit_system, + f"gov.hmrc.national_insurance.class_4.thresholds.{name}", + f"{y}-01-01", + ) + for name in ("lower_profits_limit", "upper_profits_limit") + } + finally: + del simulation + gc.collect() + return [ + { + "id": case, + "class": "paired_dev_observation", + "variables": variables, + "situation": situation, + "years": years, + "observation": observation, + "evidence": _source(package_root, evidence), + "existing_issue": f"https://github.com/PolicyEngine/policyengine-uk/issues/{issue}" + if issue + else None, + "assessment": "docs/uk_replay/MODEL_INVESTIGATIONS.md: class4_threshold_indexation_from_2027" + if case == PAIRED_CASES[4] + else None, + "interpretation": "Controlled synthetic observations for descriptive engine comparison. Compare the same inputs at each certified pin; this run does not size or attribute a national difference.", + "national_effect": "unsized", + "reproducer": f"--case {case}", + } + ] + + +def investigate(case: str | None = None, bundle=DEFAULT_BUNDLE) -> list[dict]: for key in ("HF_HUB_OFFLINE", "HF_DATASETS_OFFLINE", "TRANSFORMERS_OFFLINE"): os.environ[key] = "1" version = importlib.metadata.version("policyengine-uk") - if version != "2.89.2": - raise RuntimeError(f"requires policyengine-uk 2.89.2; found {version}") + identity = bundle_identity(bundle) + for package, expected in identity["engine_versions"].items(): + if importlib.metadata.version(package) != expected: + raise RuntimeError(f"selected bundle requires {package} {expected}") import policyengine_uk from policyengine_uk import CountryTaxBenefitSystem, Simulation package_root = Path(policyengine_uk.__file__).parent + if case in PAIRED_CASES: + results = _paired_checks(package_root, Simulation, case) + for result in results: + result["engine_version"] = version + if bundle != DEFAULT_BUNDLE: + result.update(identity) + return results system = CountryTaxBenefitSystem() results = [] @@ -431,6 +613,8 @@ def investigate(case: str | None = None) -> list[dict]: results.extend(_additional_model_checks(package_root, Simulation, case)) for result in results: result["engine_version"] = version + if bundle != DEFAULT_BUNDLE: + current_bundle_observation(result, identity) absent = set(result["variables"]) - set(system.variables) if absent: raise RuntimeError(f"diagnostic names unknown engine variables: {absent}") @@ -440,6 +624,7 @@ def investigate(case: str | None = None) -> list[dict]: def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--case") + parser.add_argument("--bundle", default=DEFAULT_BUNDLE) parser.add_argument( "--from-json", type=Path, @@ -451,37 +636,50 @@ def main(argv: list[str] | None = None) -> int: results = ( json.loads(args.from_json.read_text()) if args.from_json - else investigate(args.case) + else investigate(args.case, args.bundle) ) + if args.from_json: + for result in results: + validate_bundle_identity(result, args.bundle) if args.case: results = [r for r in results if r["id"] == args.case] if not results: parser.error("unknown diagnostic case") payload = json.dumps(results, indent=1, sort_keys=True, allow_nan=False) + "\n" if args.output: + validate_bundle_path(args.output, args.bundle, kind="results") args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(payload) else: print(payload, end="") if args.report: + model_version = bundle_identity(args.bundle)["engine_versions"][ + "policyengine-uk" + ] lines = [ "# Measured UK replay model investigation candidates", "", ( - "These small reproductions inspect policyengine-uk 2.89.2. They identify " + f"These small reproductions inspect policyengine-uk {model_version}. They identify " "specific encoding or parameter concerns, independently of the national " "OBR differences. Their national effect is unsized; no issue has been filed. " "These are model investigation candidates, not ten sized explanations " "of national differences." ), "", - ( - "An [integrated execution receipt](../../results/uk/events/MODEL_DIAGNOSTICS_VERIFICATION.json) " - "binds the [preserved fresh output](../../results/uk/events/diagnostics/integrated_run_20261009.json) " - "to these observations. The Class 4 primary citation was added afterward; " - "the receipt distinguishes that metadata addition from engine results." + *( + [ + ( + "An [integrated execution receipt](../../results/uk/events/MODEL_DIAGNOSTICS_VERIFICATION.json) " + "binds the [preserved fresh output](../../results/uk/events/diagnostics/integrated_run_20261009.json) " + "to these observations. The Class 4 primary citation was added afterward; " + "the receipt distinguishes that metadata addition from engine results." + ), + "", + ] + if args.bundle == DEFAULT_BUNDLE + else [] ), - "", ] for result in results: primary = result.get("primary_source") @@ -497,7 +695,11 @@ def main(argv: list[str] | None = None) -> int: ( f"Evidence: `{result['evidence']['engine_file']}`; source SHA-256 " f"`{result['evidence']['sha256']}`. Full source and measured observations " - "are retained in `results/uk/events/MODEL_DIAGNOSTICS.json`." + + ( + "are retained in `results/uk/events/MODEL_DIAGNOSTICS.json`." + if args.bundle == DEFAULT_BUNDLE + else f"are retained in `{args.from_json or args.output or 'the saved bundle diagnostic output'}`." + ) ), "", *( @@ -513,7 +715,11 @@ def main(argv: list[str] | None = None) -> int: else [] ), "```bash", - "PYTHONPATH=. .venv-replay/bin/python pipeline/diagnose_uk_event_models.py " + ( + "PYTHONPATH=. .venv-replay/bin/python pipeline/diagnose_uk_event_models.py " + if args.bundle == DEFAULT_BUNDLE + else f"PYTHONPATH=. -m pipeline.diagnose_uk_event_models --bundle {args.bundle} " + ) + result["reproducer"], "```", "", diff --git a/pipeline/export_uk_bundle_requirements.py b/pipeline/export_uk_bundle_requirements.py new file mode 100644 index 00000000..dfd60b33 --- /dev/null +++ b/pipeline/export_uk_bundle_requirements.py @@ -0,0 +1,109 @@ +"""Export the complete UK-extra freeze from a policyengine.py release tag. + +Copies only pyproject.toml, uv.lock and README.md from the local repository's +tag into a temporary directory. ``uv export`` runs frozen, offline and without +an uv cache. It never creates a worktree, environment or downloads a dataset. +Install the emitted freeze using ``uv pip install --no-deps -r ``. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import tempfile +from pathlib import Path + +import tomllib + +from pipeline.uk_bundle import ROOT, _key + + +def _git_file(repository, tag, name): + return subprocess.run( + ["git", "-C", str(repository), "show", f"{tag}:{name}"], + check=True, + capture_output=True, + ).stdout + + +def export_requirements(repository, tag, key, *, uv="uv"): + _key(key) + if not tag or tag.startswith("-") or ":" in tag: + raise ValueError("invalid release tag") + project = _git_file(repository, tag, "pyproject.toml") + lock = _git_file(repository, tag, "uv.lock") + version = tomllib.loads(project.decode())["project"]["version"] + if tag.removeprefix("v") != version: + raise ValueError("tag differs from project release version") + commit = subprocess.run( + ["git", "-C", str(repository), "rev-parse", f"{tag}^{{commit}}"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + with tempfile.TemporaryDirectory(prefix="uk-bundle-freeze-") as temporary: + directory = Path(temporary) + (directory / "pyproject.toml").write_bytes(project) + (directory / "uv.lock").write_bytes(lock) + (directory / "README.md").write_bytes(_git_file(repository, tag, "README.md")) + command = [ + uv, + "export", + "--frozen", + "--no-hashes", + "--no-dev", + "--extra", + "uk", + "--no-emit-project", + "--offline", + "--no-cache", + ] + result = subprocess.run( + command, + cwd=directory, + check=True, + capture_output=True, + text=True, + env={**os.environ, "UV_NO_CACHE": "1"}, + ) + # Strip uv's location-dependent generated header and bind the actual inputs. + exported = "\n".join( + line for line in result.stdout.splitlines() if not line.startswith("#") + ).strip() + freeze = f"# UK replay bundle {key}\n# policyengine.py tag {tag}, commit {commit}\n# uv.lock SHA-256 {hashlib.sha256(lock).hexdigest()}\n# uv export --frozen --no-hashes --no-dev --extra uk --no-emit-project\n{exported}\npolicyengine=={version}\n" + provenance = { + "bundle_key": key, + "policyengine_tag": tag, + "policyengine_commit": commit, + "uv_lock_sha256": hashlib.sha256(lock).hexdigest(), + "freeze_sha256": hashlib.sha256(freeze.encode()).hexdigest(), + "export_command": command, + } + return freeze, provenance + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", required=True, type=Path) + parser.add_argument("--tag", required=True) + parser.add_argument("--key", required=True) + parser.add_argument("--output", type=Path) + parser.add_argument("--uv", default="uv") + args = parser.parse_args(argv) + freeze, provenance = export_requirements( + args.repository, args.tag, args.key, uv=args.uv + ) + path = args.output or ROOT / "docs/uk_replay" / f"requirements-{args.key}.txt" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(freeze) + path.with_suffix(".provenance.json").write_text( + json.dumps(provenance, indent=2, sort_keys=True) + "\n" + ) + print(json.dumps({"output": str(path), **provenance}, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/pipeline/modal_uk_event.py b/pipeline/modal_uk_event.py index d5f43612..c43aed42 100644 --- a/pipeline/modal_uk_event.py +++ b/pipeline/modal_uk_event.py @@ -31,6 +31,7 @@ from __future__ import annotations import argparse +import ast import hashlib import importlib.metadata import json @@ -42,6 +43,10 @@ import time from pathlib import Path, PurePosixPath +from pipeline import uk_bundle + +DEFAULT_BUNDLE = uk_bundle.DEFAULT_BUNDLE + ROOT = Path(__file__).resolve().parent.parent REMOTE_ROOT = "/replay" REMOTE_PYTHON = "/opt/replay-venv/bin/python" @@ -71,9 +76,7 @@ "pipeline/compute_uk_obr_costings.py", "pipeline/build_uk_event_registry.py", "pipeline/uk_engine_registry.py", - "data/uk/certified_bundle.json", - "docs/uk_replay/requirements.txt", - "docs/uk_replay/OFFLINE_BUNDLE_AUDIT.json", + "pipeline/uk_bundle.py", "sources/uk_replay/source.json", "sources/harvest-uk-2026-08-02/uk_obr/claims_staged.jsonl.gz", ) @@ -131,71 +134,226 @@ def relative_path(value: str) -> str: return value -def repository_files(event: str) -> tuple[str, ...]: +def bundle_relative(path: Path, root: Path) -> str: + """Only workspace-owned files may be selected by the bundle registry.""" + return relative_path(str(path.resolve().relative_to(root.resolve()))) + + +def bundle_index_relative(root: Path) -> str: + value = os.environ.get("UK_BUNDLE_INDEX", "data/uk/certified_bundles/index.json") + path = Path(value) + return bundle_relative(path if path.is_absolute() else root / path, root) + + +def bundle_file(bundle: str, field: str, *, root: Path = ROOT) -> str: + return relative_path(uk_bundle.load_bundle(bundle, root=root)[field]) + + +def repository_files( + event: str, bundle: str = DEFAULT_BUNDLE, *, root: Path = ROOT +) -> tuple[str, ...]: if event not in EVENTS: raise ValueError(f"unsupported fiscal event: {event}") - return (*REPOSITORY_FILES, f"data/uk/events/{event}_measures.json") + return ( + *REPOSITORY_FILES, + bundle_index_relative(root), + bundle_relative(uk_bundle.bundle_pin_path(bundle, root=root), root), + bundle_file(bundle, "requirements_freeze", root=root), + bundle_file(bundle, "offline_audit", root=root), + bundle_relative(uk_bundle.registry_path(event, bundle, root=root), root), + ) + + +def engine_pins(bundle: str = DEFAULT_BUNDLE, *, root: Path = ROOT) -> dict[str, str]: + if bundle == DEFAULT_BUNDLE: + return ENGINE_PINS.copy() + pin = uk_bundle.load_bundle(bundle, root=root) + pins = {"policyengine": pin["managed_loader_version"]} + for field in ("compatible_model_packages", "compatible_core_packages"): + for package in pin[field]: + if not re.fullmatch(r"==[A-Za-z0-9_.+!-]+", package["specifier"]): + raise ValueError("remote engine packages require exact certified pins") + pins[package["name"]] = package["specifier"][2:] + if not {"policyengine", "policyengine-uk", "policyengine-core"} <= pins.keys(): + raise ValueError("bundle omits a certified engine package") + return pins + +def hf_identity(bundle: str, *, root: Path = ROOT) -> tuple[str, str]: + pin = uk_bundle.load_bundle(bundle, root=root) + kind = {"dataset": "datasets", "model": "models"}.get(pin["repo_type"]) + if kind is None: + raise ValueError("unsupported certified Hugging Face repository type") + return f"{kind}--{pin['repo_id'].replace('/', '--')}", pin["resolved_hf_commit"] + + +def remote_marker_applies(expression: str) -> bool: + """Evaluate the freeze's PEP 508 markers for the pinned Linux interpreter. + + The lightweight control environment has no packaging dependency. Restrict + evaluation to marker names, string constants and comparison/boolean nodes; + never execute a requirements expression as Python code. + """ + environment = { + "implementation_name": "cpython", + "implementation_version": PYTHON_VERSION, + "os_name": "posix", + "sys_platform": "linux", + "platform_machine": "x86_64", + "platform_system": "Linux", + "platform_python_implementation": "CPython", + "python_version": ".".join(PYTHON_VERSION.split(".")[:2]), + "python_full_version": PYTHON_VERSION, + "extra": "", + } -def requirements_pins(text: str) -> dict[str, str]: + def operand(node): + if isinstance(node, ast.Name) and node.id in environment: + return environment[node.id] + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + raise ValueError("unsupported requirements environment marker") + + def evaluate(node): + if isinstance(node, ast.BoolOp) and isinstance(node.op, (ast.And, ast.Or)): + values = [evaluate(child) for child in node.values] + return all(values) if isinstance(node.op, ast.And) else any(values) + if not isinstance(node, ast.Compare) or len(node.ops) != 1: + raise ValueError("unsupported requirements environment marker") + left, right = operand(node.left), operand(node.comparators[0]) + operation = node.ops[0] + version_names = { + "python_version", + "python_full_version", + "implementation_version", + } + if any( + isinstance(item, ast.Name) and item.id in version_names + for item in (node.left, node.comparators[0]) + ): + if not all( + re.fullmatch(r"[0-9]+(?:\.[0-9]+)*", value) for value in (left, right) + ): + raise ValueError("version markers must use numeric release versions") + # Pad release segments so 3.12 and 3.12.0 compare equally. + values = [tuple(map(int, value.split("."))) for value in (left, right)] + length = max(map(len, values)) + left, right = (value + (0,) * (length - len(value)) for value in values) + if isinstance(operation, ast.Eq): + return left == right + if isinstance(operation, ast.NotEq): + return left != right + if isinstance(operation, ast.Lt): + return left < right + if isinstance(operation, ast.LtE): + return left <= right + if isinstance(operation, ast.Gt): + return left > right + if isinstance(operation, ast.GtE): + return left >= right + if isinstance(operation, ast.In): + return left in right + if isinstance(operation, ast.NotIn): + return left not in right + raise ValueError("unsupported requirements environment marker") + + try: + return evaluate(ast.parse(expression.strip(), mode="eval").body) + except SyntaxError as error: + raise ValueError("invalid requirements environment marker") from error + + +def requirements_pins( + text: str, bundle: str = DEFAULT_BUNDLE, *, root: Path = ROOT +) -> dict[str, str]: pins = {} for line in text.splitlines(): line = line.strip() if not line or line.startswith("#"): continue - match = re.fullmatch(r"([A-Za-z0-9][A-Za-z0-9_.-]*)==([A-Za-z0-9_.+!-]+)", line) + requirement, separator, marker = line.partition(";") + match = re.fullmatch( + r"([A-Za-z0-9][A-Za-z0-9_.-]*)==([A-Za-z0-9_.+!-]+)", requirement.strip() + ) if not match: raise ValueError("remote requirements must contain exact package pins only") + if separator and not remote_marker_applies(marker): + continue name, version = match.groups() name = re.sub(r"[-_.]+", "-", name).lower() if name in pins: raise ValueError(f"duplicate requirement: {name}") pins[name] = version - if any(pins.get(name) != version for name, version in ENGINE_PINS.items()): + if any( + pins.get(name) != version + for name, version in engine_pins(bundle, root=root).items() + ): raise ValueError("requirements do not match the certified engine pins") return pins -def certified_pin(root: Path) -> dict: - cert = json.loads((root / "data/uk/certified_bundle.json").read_bytes()) - audit = json.loads((root / "docs/uk_replay/OFFLINE_BUNDLE_AUDIT.json").read_bytes()) +def certified_pin(root: Path, bundle: str = DEFAULT_BUNDLE) -> dict: + cert = json.loads(uk_bundle.bundle_pin_path(bundle, root=root).read_bytes()) + config = uk_bundle.load_bundle(bundle, root=root) + audit = json.loads( + (root / bundle_file(bundle, "offline_audit", root=root)).read_bytes() + ) if ( - cert.get("repo_id") != "policyengine/populace-uk-private" - or cert.get("artifact") != "populace_uk_2023.h5" - or not re.fullmatch(r"[0-9a-f]{64}", cert.get("sha256", "")) + not re.fullmatch(r"[0-9a-f]{64}", cert.get("sha256", "")) or cert.get("size_bytes", 0) <= 0 - or audit["certified_identity"].get("resolved_hf_commit") != HF_COMMIT + or audit["certified_identity"].get("resolved_hf_commit") + != config["resolved_hf_commit"] or any( audit["certified_identity"].get(key) != cert.get(key) for key in ("repo_id", "artifact", "revision", "sha256", "size_bytes") ) - or not any( - row.get("name") == "policyengine-uk" and row.get("specifier") == "==2.89.2" - for row in cert.get("compatible_model_packages", []) - ) ): raise ValueError("certified pin differs from the audited offline bundle") + pins = engine_pins(bundle, root=root) + if not any( + row.get("name") == "policyengine-uk" + and row.get("specifier") == f"=={pins['policyengine-uk']}" + for row in cert.get("compatible_model_packages", []) + ): + raise ValueError("certified pin differs from the audited engine model") + if bundle != DEFAULT_BUNDLE and ( + audit["certified_identity"].get("repo_type") != config["repo_type"] + or audit["certified_identity"].get("model_version") != pins["policyengine-uk"] + or audit["certified_identity"].get("core_version") != pins["policyengine-core"] + or audit.get("policyengine_version") != pins["policyengine"] + ): + raise ValueError("certified pin differs from the audited engine runtime") return cert def input_manifest( - event: str, *, root: Path = ROOT, hf_cache: Path | None = None + event: str, + *, + bundle: str = DEFAULT_BUNDLE, + root: Path = ROOT, + hf_cache: Path | None = None, ) -> tuple[dict, dict[str, Path]]: """List exact mounts; never traverse a repository or a Hugging Face cache.""" - cert = certified_pin(root) - requirements_pins((root / "docs/uk_replay/requirements.txt").read_text()) + cert = certified_pin(root, bundle) + requirements_pins( + (root / bundle_file(bundle, "requirements_freeze", root=root)).read_text(), + bundle, + root=root, + ) + repo_cache_name, commit = hf_identity(bundle, root=root) hf_cache = hf_cache or Path.home() / ".cache/huggingface/hub" - repo_cache = hf_cache / HF_REPO_CACHE + repo_cache = hf_cache / repo_cache_name ref = repo_cache / "refs" / cert["revision"] - artifact = repo_cache / "snapshots" / HF_COMMIT / cert["artifact"] - if ref.read_text().strip() != HF_COMMIT: + artifact = repo_cache / "snapshots" / commit / cert["artifact"] + if ref.read_text().strip() != commit: raise ValueError("cached revision does not resolve to the audited HF commit") if artifact.stat().st_size != cert["size_bytes"]: raise ValueError("cached artifact size differs from the certified bundle") - mounts = {relative: root / relative for relative in repository_files(event)} - ref_relative = f"{HF_CACHE_RELATIVE}/{HF_REPO_CACHE}/refs/{cert['revision']}" + allowlist = repository_files(event, bundle, root=root) + mounts = {relative: root / relative for relative in allowlist} + ref_relative = f"{HF_CACHE_RELATIVE}/{repo_cache_name}/refs/{cert['revision']}" artifact_relative = ( - f"{HF_CACHE_RELATIVE}/{HF_REPO_CACHE}/snapshots/{HF_COMMIT}/{cert['artifact']}" + f"{HF_CACHE_RELATIVE}/{repo_cache_name}/snapshots/{commit}/{cert['artifact']}" ) # Mount the blob's content directly at the snapshot path; no symlink or # unrelated blob, release metadata, refs/main or token file is transmitted. @@ -204,7 +362,7 @@ def input_manifest( rows = [] for relative, source in sorted(mounts.items()): is_artifact = relative == artifact_relative - if relative in repository_files(event) and not source.resolve().is_relative_to( + if relative in allowlist and not source.resolve().is_relative_to( root.resolve() ): raise ValueError( @@ -222,10 +380,15 @@ def input_manifest( "schema_version": 1, "event": event, "certified_bundle": cert, - "hf_commit": HF_COMMIT, + "hf_commit": commit, "files": rows, "artifact_digest_validation": "remote hash before engine import, plus existing managed-loader pre/post simulation checks", "resource_limits": RESOURCE_LIMITS, + **( + uk_bundle.bundle_identity(bundle, root=root) + if bundle != DEFAULT_BUNDLE + else {} + ), }, mounts @@ -236,21 +399,26 @@ def request_spec( measures: list[str], output_dir: str, preflight_only: bool, + bundle: str = DEFAULT_BUNDLE, + root: Path = ROOT, ) -> dict: if event not in EVENTS: raise ValueError("unsupported fiscal event") output_dir = relative_path(output_dir) output = PurePosixPath(output_dir) - canonical = PurePosixPath("results/uk/events") / event + canonical = PurePosixPath( + bundle_relative(uk_bundle.event_output_dir(event, bundle, root=root), root) + ) ignored = PurePosixPath(".venv-replay-checks/modal") if not (output.is_relative_to(canonical) or output.is_relative_to(ignored)): raise ValueError( "remote output must be event results or the ignored Modal directory" ) + start, end = uk_bundle.bundle_window(uk_bundle.load_bundle(bundle, root=root)) if len(years) != len(set(years)) or any( - year < 2023 or year > 2030 for year in years + year < start or year > end for year in years ): - raise ValueError("years must be unique within 2023–2030") + raise ValueError(f"years must be unique within {start}–{end}") if len(measures) != len(set(measures)) or any( not re.fullmatch(r"[a-z0-9]+(?:_+[a-z0-9]+)*", key) for key in measures ): @@ -263,13 +431,21 @@ def request_spec( "output_dir": output_dir, "preflight_only": preflight_only, "workers": 1, + **( + uk_bundle.bundle_identity(bundle, root=root) + if bundle != DEFAULT_BUNDLE + else {} + ), } def validate_selection(request: dict, *, root: Path = ROOT) -> None: + bundle = request.get("bundle_key", DEFAULT_BUNDLE) + uk_bundle.validate_bundle_identity(request, bundle, root=root) registry = json.loads( - (root / f"data/uk/events/{request['event']}_measures.json").read_bytes() + uk_bundle.registry_path(request["event"], bundle, root=root).read_bytes() ) + uk_bundle.validate_registry_bundle(registry, bundle, root=root) if registry.get("event_slug", registry.get("event")) != request["event"]: raise ValueError("uploaded registry belongs to another fiscal event") if set(request["years"]) - set(registry["calendar_years"]): @@ -279,20 +455,24 @@ def validate_selection(request: dict, *, root: Path = ROOT) -> None: raise ValueError("requested measure keys are absent from the event registry") -def verify_inputs(manifest: dict, *, root: Path = ROOT) -> None: +def verify_inputs( + manifest: dict, *, bundle: str = DEFAULT_BUNDLE, root: Path = ROOT +) -> None: """Check uploaded bytes before importing a country package.""" - cert = certified_pin(root) - expected = set(repository_files(manifest["event"])) - expected.add(f"{HF_CACHE_RELATIVE}/{HF_REPO_CACHE}/refs/{cert['revision']}") + uk_bundle.validate_bundle_identity(manifest, bundle, root=root) + cert = certified_pin(root, bundle) + repo_cache_name, commit = hf_identity(bundle, root=root) + expected = set(repository_files(manifest["event"], bundle, root=root)) + expected.add(f"{HF_CACHE_RELATIVE}/{repo_cache_name}/refs/{cert['revision']}") artifact_relative = ( - f"{HF_CACHE_RELATIVE}/{HF_REPO_CACHE}/snapshots/{HF_COMMIT}/{cert['artifact']}" + f"{HF_CACHE_RELATIVE}/{repo_cache_name}/snapshots/{commit}/{cert['artifact']}" ) expected.add(artifact_relative) rows = manifest.get("files", []) if ( manifest.get("schema_version") != 1 or manifest.get("certified_bundle") != cert - or manifest.get("hf_commit") != HF_COMMIT + or manifest.get("hf_commit") != commit or manifest.get("resource_limits") != RESOURCE_LIMITS or len(rows) != len(expected) or {row["path"] for row in rows} != expected @@ -312,19 +492,25 @@ def verify_inputs(manifest: dict, *, root: Path = ROOT) -> None: or sha256_file(source) != row["sha256"] ): raise ValueError(f"uploaded input differs from its digest: {row['path']}") - ref = root / HF_CACHE_RELATIVE / HF_REPO_CACHE / "refs" / cert["revision"] - if ref.read_text().strip() != HF_COMMIT: + ref = root / HF_CACHE_RELATIVE / repo_cache_name / "refs" / cert["revision"] + if ref.read_text().strip() != commit: raise ValueError("remote HF revision ref differs from the audited commit") -def verify_runtime(*, root: Path = ROOT) -> dict: - pins = requirements_pins((root / "docs/uk_replay/requirements.txt").read_text()) +def verify_runtime(*, bundle: str = DEFAULT_BUNDLE, root: Path = ROOT) -> dict: + pins = requirements_pins( + (root / bundle_file(bundle, "requirements_freeze", root=root)).read_text(), + bundle, + root=root, + ) installed = {name: importlib.metadata.version(name) for name in pins} if installed != pins or platform.python_version() != PYTHON_VERSION: raise ValueError( "remote Python or installed requirements differ from the pinned environment" ) - audit = json.loads((root / "docs/uk_replay/OFFLINE_BUNDLE_AUDIT.json").read_bytes()) + audit = json.loads( + (root / bundle_file(bundle, "offline_audit", root=root)).read_bytes() + ) distribution = importlib.metadata.distribution("policyengine") packaged_manifest = Path( distribution.locate_file("policyengine/data/bundle/manifest.json") @@ -348,6 +534,7 @@ def verify_runtime(*, root: Path = ROOT) -> dict: def run_worker(payload: dict) -> int: started = time.perf_counter() request = payload["request"] + bundle = request.get("bundle_key", DEFAULT_BUNDLE) if ( request != request_spec( @@ -356,6 +543,7 @@ def run_worker(payload: dict) -> int: measures=request["measures"], output_dir=request["output_dir"], preflight_only=request["preflight_only"], + bundle=bundle, ) or payload["inputs"]["event"] != request["event"] ): @@ -365,24 +553,24 @@ def run_worker(payload: dict) -> int: os.environ["HF_HOME"] = str(ROOT / ".cache/huggingface") os.environ["HF_HUB_CACHE"] = str(ROOT / HF_CACHE_RELATIVE) print("[modal] verifying allowlisted inputs before engine import", flush=True) - verify_inputs(payload["inputs"]) + verify_inputs(payload["inputs"], bundle=bundle) validate_selection(request) - runtime = verify_runtime() + runtime = verify_runtime(bundle=bundle) print("[modal] input hashes and full pinned environment verified", flush=True) from pipeline import compute_uk_event as compute - registry_path, _ = compute.event_paths(request["event"]) + registry_path, _ = compute.event_paths(request["event"], bundle=bundle) registry = json.loads(registry_path.read_bytes()) - compute.validate_event_identity(registry, request["event"]) + compute.validate_event_identity(registry, request["event"], bundle=bundle) compute.registry_builder.validate_registry(registry) - compute.validate_years(request["years"], registry) + compute.validate_years(request["years"], registry, bundle=bundle) output = ROOT / request["output_dir"] if output.exists() and any(output.iterdir()): raise ValueError("remote output directory already contains files") output.mkdir(parents=True, exist_ok=True) if request["preflight_only"]: print("[modal] starting existing offline managed-bundle preflight", flush=True) - pre = compute.preflight() + pre = compute.preflight(bundle=bundle) result = { "event": request["event"], "computed": False, @@ -393,6 +581,7 @@ def run_worker(payload: dict) -> int: "input_manifest_sha256": digest_bytes(canonical_bytes(payload["inputs"])), "engine_version": compute.fiscal.package_version("policyengine-uk"), **compute.runtime_versions(), + **(uk_bundle.bundle_identity(bundle) if bundle != DEFAULT_BUNDLE else {}), } compute.fiscal.atomic_write_bytes( output / "PREFLIGHT.json", canonical_bytes(result) @@ -407,6 +596,8 @@ def run_worker(payload: dict) -> int: "--workers", "1", ] + if bundle != DEFAULT_BUNDLE: + argv.extend(["--bundle", bundle]) if request["years"]: argv.extend(["--years", *map(str, request["years"])]) if request["measures"]: @@ -436,7 +627,7 @@ def run_worker(payload: dict) -> int: return 0 -def modal_function(mounts: dict[str, Path]): +def modal_function(mounts: dict[str, Path], bundle: str = DEFAULT_BUNDLE): """Construct lazy SDK objects; only main's explicit --execute starts them.""" if sys.version_info[:2] != (3, 12): raise ValueError( @@ -451,14 +642,14 @@ def modal_function(mounts: dict[str, Path]): modal.Image.debian_slim(python_version="3.12") .pip_install(f"uv=={UV_VERSION}") .add_local_file( - mounts["docs/uk_replay/requirements.txt"], + mounts[bundle_file(bundle, "requirements_freeze")], "/opt/replay-requirements.txt", copy=True, ) .run_commands( f"uv python install {PYTHON_VERSION}", f"uv venv --python {PYTHON_VERSION} /opt/replay-venv", - f"uv pip install --python {REMOTE_PYTHON} --requirements /opt/replay-requirements.txt", + f"uv pip install --python {REMOTE_PYTHON} --no-deps --requirements /opt/replay-requirements.txt", ) .env( { @@ -466,6 +657,11 @@ def modal_function(mounts: dict[str, Path]): "PYTHONPATH": REMOTE_ROOT, "HF_HOME": f"{REMOTE_ROOT}/.cache/huggingface", "HF_HUB_CACHE": f"{REMOTE_ROOT}/{HF_CACHE_RELATIVE}", + **( + {"UK_BUNDLE_INDEX": bundle_index_relative(ROOT)} + if os.environ.get("UK_BUNDLE_INDEX") + else {} + ), } ) ) @@ -595,6 +791,7 @@ def save_download( def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--event", choices=EVENTS, default="autumn_budget_2024") + parser.add_argument("--bundle", default=DEFAULT_BUNDLE) parser.add_argument("--years", nargs="*", type=int, default=[]) parser.add_argument("--measures", nargs="*", default=[]) parser.add_argument("--output-dir") @@ -613,7 +810,9 @@ def main(argv: list[str] | None = None) -> int: output = args.output_dir or ( f".venv-replay-checks/modal/{args.event}/preflight" if args.preflight_only - else f"results/uk/events/{args.event}" + else bundle_relative( + uk_bundle.event_output_dir(args.event, args.bundle, root=ROOT), ROOT + ) ) request = request_spec( args.event, @@ -621,9 +820,12 @@ def main(argv: list[str] | None = None) -> int: measures=args.measures, output_dir=output, preflight_only=args.preflight_only, + bundle=args.bundle, ) validate_selection(request) - inputs, mounts = input_manifest(args.event, hf_cache=args.hf_cache) + inputs, mounts = input_manifest( + args.event, bundle=args.bundle, hf_cache=args.hf_cache + ) payload = {"request": request, "inputs": inputs} prefix = download_prefix(args.download_root) if not args.execute: @@ -639,7 +841,7 @@ def main(argv: list[str] | None = None) -> int: raise ValueError("download destination escapes its ignored prefix") if target.exists() and any(target.iterdir()): raise ValueError("choose an empty download prefix before a remote run") - modal, app, replay = modal_function(mounts) + modal, app, replay = modal_function(mounts, bundle=args.bundle) response = None with modal.enable_output(), app.run(): response = replay.remote(payload) diff --git a/pipeline/register_uk_certified_bundle.py b/pipeline/register_uk_certified_bundle.py new file mode 100644 index 00000000..10cfefc2 --- /dev/null +++ b/pipeline/register_uk_certified_bundle.py @@ -0,0 +1,313 @@ +"""Register the installed loader's certified UK bundle from cached bytes only. + +No package constraint is relaxed and no Hugging Face request is made. An +installed loader's packaged bundle manifest supplies the recertified model/core +pins. A cached producer manifest supplies artifact size when the packaged +manifest omits it. Development registration requires an explicit development +index and never changes the production index. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import os +import re +from pathlib import Path + +from pipeline.uk_bundle import DEFAULT_BUNDLE, INDEX, ROOT, _key + + +def sha256_file(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(4 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _stored_year(path): + import h5py + + with h5py.File(path, "r") as data: + values = data["time_period/table"][()]["values"] + years = { + int(value.decode() if isinstance(value, bytes) else value) for value in values + } + if len(years) != 1: + raise ValueError("certified artifact must store exactly one data year") + return years.pop() + + +def _extension_end(path): + # Inspect code rather than importing a simulation or inferring support from + # a parameter lookup. + source = Path(path).read_text() + match = re.search( + r"def extend_single_year_dataset\([^)]*end_year:\s*int\s*=\s*(\d+)", + source, + re.DOTALL, + ) + if not match: + raise ValueError("cannot verify the installed dataset extension end") + return int(match.group(1)) + + +def inspect_installed_bundle( + key, *, requirements_freeze, offline_audit, cache_root=None, development=False +): + _key(key) + distribution = importlib.metadata.distribution("policyengine") + manifest_path = Path( + distribution.locate_file("policyengine/data/bundle/manifest.json") + ) + manifest = json.loads(manifest_path.read_text()) + country = manifest["data_releases"]["uk"] + certification = country["certification"] + basis = certification.get("compatibility_basis", "") + if not basis or "unverified" in basis.lower(): + raise ValueError("packaged UK bundle lacks a verified compatibility claim") + build_id = country.get("build_id") + if not build_id or certification.get("data_build_id") != build_id: + raise ValueError("packaged certification data build identity differs") + versions = {} + for name in ("policyengine", "policyengine-uk", "policyengine-core"): + declared = manifest["packages"][name]["version"] + installed = importlib.metadata.version(name) + if installed != declared: + raise ValueError( + f"installed {name} {installed} differs from packaged pin {declared}" + ) + versions[name] = installed + if ( + country["model_package"]["version"] != versions["policyengine-uk"] + or certification["certified_for_model_version"] != versions["policyengine-uk"] + ): + raise ValueError("packaged UK model certification differs from installed model") + if country["policyengine_version"] != versions["policyengine"]: + raise ValueError("packaged UK managed loader version differs") + dataset_name = country["default_dataset"] + declaration = country["datasets"][dataset_name] + certified = country["certified_data_artifact"] + if certified.get("build_id") != build_id: + raise ValueError("packaged artifact data build identity differs") + if ( + declaration["sha256"] != certified["sha256"] + or certified["dataset"] != dataset_name + ): + raise ValueError("packaged artifact declarations disagree") + package = country["data_package"] + repo_type = declaration.get("repo_type", package["repo_type"]) + if repo_type not in ("model", "dataset"): + raise ValueError("packaged artifact repo_type is invalid") + repo_id = declaration.get("repo_id", package["repo_id"]) + release_tag = declaration["revision"] + uri = f"hf://{repo_id}/{declaration['path']}@{release_tag}" + if certified.get("uri") != uri or country.get("default_dataset_uri") != uri: + raise ValueError( + "packaged certified/default dataset URI differs from artifact identity" + ) + cache = Path( + cache_root + or os.environ.get("HF_HUB_CACHE", Path.home() / ".cache/huggingface/hub") + ) + repo = ( + cache + / f"{'models' if repo_type == 'model' else 'datasets'}--{repo_id.replace('/', '--')}" + ) + ref = repo / "refs" / release_tag + commit = ref.read_text().strip() if ref.exists() else release_tag + if not re.fullmatch(r"[0-9a-f]{40}", commit): + raise ValueError("cached release tag does not resolve to a commit") + artifact = repo / "snapshots" / commit / declaration["path"] + if not artifact.is_file(): + raise ValueError( + "certified artifact is not cached; registration never downloads" + ) + digest = sha256_file(artifact) + size = artifact.stat().st_size + if digest != declaration["sha256"]: + raise ValueError("cached artifact digest differs from packaged manifest") + # Older packaged manifests do not carry size. Only use a cached matching + # producer declaration, not an unrelated moving root release manifest. + release_paths = [ + repo + / "snapshots" + / package["release_manifest_revision"] + / package["release_manifest_path"], + repo / "snapshots" / commit / package["release_manifest_path"], + repo / "snapshots" / commit / "release_manifest.json", + ] + producer_manifest = None + producer_path = None + for path in release_paths: + if not path.is_file(): + continue + candidate = json.loads(path.read_text()) + candidate_artifact = candidate.get("artifacts", {}).get(dataset_name, {}) + if candidate_artifact.get("sha256") == digest: + producer_manifest, producer_path = candidate, path + break + declared_size = declaration.get("size_bytes", certified.get("size_bytes")) + if declared_size is None and producer_manifest is not None: + declared_size = producer_manifest["artifacts"][dataset_name].get("size_bytes") + if declared_size is None or size != declared_size: + raise ValueError("cached artifact size absent from or differs from manifest") + for value in (declaration.get("size_bytes"), certified.get("size_bytes")): + if value is not None and value != size: + raise ValueError("packaged artifact size declarations disagree") + model_distribution = importlib.metadata.distribution("policyengine-uk") + extension_path = Path( + model_distribution.locate_file("policyengine_uk/data/economic_assumptions.py") + ) + start, end = _stored_year(artifact), _extension_end(extension_path) + if start > end: + raise ValueError("stored data year exceeds engine extension window") + return { + "schema_version": 2, + "bundle_key": key, + "registry_rule": "Offline byte-exact certified UK replay identity; the installed managed loader's packaged manifest certifies the exact model/core environment.", + "repo_id": repo_id, + "repo_type": repo_type, + "revision": release_tag, + "release_tag": release_tag, + "resolved_hf_commit": commit, + "resolved_commit": commit, + "release_manifest_revision": package["release_manifest_revision"], + "release_manifest_path": package["release_manifest_path"], + "cached_artifact_manifest_revision": producer_path.relative_to( + repo / "snapshots" + ).parts[0] + if producer_path + else None, + "artifact": declaration["path"], + "sha256": digest, + "size_bytes": size, + "data_build_id": country["build_id"], + "data_year": start, + "supported_calendar_years": [start, end], + "compatible_model_packages": [ + {"name": "policyengine-uk", "specifier": f"=={versions['policyengine-uk']}"} + ], + "compatible_core_packages": [ + { + "name": "policyengine-core", + "specifier": f"=={versions['policyengine-core']}", + } + ], + "managed_loader_version": versions["policyengine"], + "requirements_freeze": str(requirements_freeze), + "offline_audit": str(offline_audit), + "certification": certification, + "producer_declared_model_packages": producer_manifest.get( + "compatible_model_packages", [] + ) + if producer_manifest + else [], + "producer_declared_core_packages": producer_manifest.get( + "compatible_core_packages", [] + ) + if producer_manifest + else [], + "development_bundle": bool(development), + "provenance": "Read from the installed release's packaged policyengine/data/bundle/manifest.json; independently verified the already-cached HF artifact, its producer-declared size, stored data year and installed engine extension function. No network request or simulation.", + "provenance_hashes": { + "packaged_bundle_manifest_sha256": sha256_file(manifest_path), + "cached_artifact_manifest_sha256": sha256_file(producer_path) + if producer_path + else None, + "engine_extension_source_sha256": sha256_file(extension_path), + }, + "compatibility_authority": "Packaged loader certification and packages pins; producer build pins describe the original build, not this recertified environment.", + "why_this_bundle": "Development adapter checks only; never event results." + if development + else "Registered separately so older certified results remain available for descriptive comparison.", + "engine_constraint_note": "Exact model and core constraints come from this installed managed-loader release. Computing at other engine versions requires another certified bundle registration; see #126.", + } + + +def write_registration(pin, pin_path, index_path, *, root=ROOT): + pin_path, index_path = Path(pin_path), Path(index_path) + if pin["bundle_key"] == DEFAULT_BUNDLE: + raise ValueError("the historical default pin must not be replaced") + if ( + pin.get("development_bundle") + and index_path.resolve() == (Path(root) / INDEX).resolve() + ): + raise ValueError("development bundles cannot enter the production index") + if pin.get("development_bundle"): + try: + development_path = pin_path.resolve().relative_to(Path(root).resolve()) + except ValueError: + development_path = None + if development_path is not None and development_path.parts[0] == "data": + raise ValueError( + "development pins belong in ignored paths or test fixtures" + ) + index = ( + json.loads(index_path.read_text()) + if index_path.exists() + else { + "schema_version": 1, + "default_bundle": DEFAULT_BUNDLE, + "bundles": {DEFAULT_BUNDLE: "data/uk/certified_bundle.json"}, + } + ) + if index.get("default_bundle") != DEFAULT_BUNDLE: + raise ValueError("registration must preserve the historical default") + try: + relative = str(pin_path.resolve().relative_to(Path(root).resolve())) + except ValueError: + relative = str(pin_path.resolve()) + existing_path = index["bundles"].get(pin["bundle_key"]) + if existing_path is not None and existing_path != relative: + raise ValueError("bundle key is already registered to another pin") + index["bundles"][pin["bundle_key"]] = relative + pin_bytes = ( + json.dumps(pin, indent=1, sort_keys=True, allow_nan=False) + "\n" + ).encode() + if pin_path.exists() and pin_path.read_bytes() != pin_bytes: + raise ValueError("refusing to replace an existing bundle pin") + pin_path.parent.mkdir(parents=True, exist_ok=True) + index_path.parent.mkdir(parents=True, exist_ok=True) + pin_path.write_bytes(pin_bytes) + index_path.write_text(json.dumps(index, indent=2, sort_keys=True) + "\n") + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--key", required=True) + parser.add_argument("--requirements-freeze", required=True) + parser.add_argument("--offline-audit", required=True) + parser.add_argument("--pin", type=Path) + parser.add_argument("--index", type=Path, default=ROOT / INDEX) + parser.add_argument("--cache-root", type=Path) + parser.add_argument("--development", action="store_true") + args = parser.parse_args(argv) + pin = inspect_installed_bundle( + args.key, + requirements_freeze=args.requirements_freeze, + offline_audit=args.offline_audit, + cache_root=args.cache_root, + development=args.development, + ) + pin_path = args.pin or ROOT / "data/uk/certified_bundles" / f"{args.key}.json" + write_registration(pin, pin_path, args.index) + print( + json.dumps( + { + "bundle_key": args.key, + "pin": str(pin_path), + "sha256": pin["sha256"], + "data_year": pin["data_year"], + "supported_calendar_years": pin["supported_calendar_years"], + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/pipeline/stage_uk_event.py b/pipeline/stage_uk_event.py index 02e247f3..f61a978e 100644 --- a/pipeline/stage_uk_event.py +++ b/pipeline/stage_uk_event.py @@ -17,6 +17,7 @@ from pipeline import compute_uk_event as compute from pipeline import compute_uk_obr_costings as fiscal from pipeline import stage_uk_ab2025 as ab_stage +from pipeline import uk_bundle as bundles AXES = [ "behavioural_adjustment", @@ -100,12 +101,26 @@ def validate_artifact( def stage_event( - registry: dict, manifest: dict | None, *, artifact_dir: Path, registry_sha256: str + registry: dict, + manifest: dict | None, + *, + artifact_dir: Path, + registry_sha256: str, + bundle: str | None = None, + bundle_root: Path = bundles.ROOT, ) -> tuple[list[dict], dict]: """Engine-free staging; missing measures and unsupported FYs stay visible.""" + selected = bundle or bundles.DEFAULT_BUNDLE + verify_bundle = ( + bundle is not None or "bundle" in registry or "bundle_key" in registry + ) + if verify_bundle: + bundles.validate_registry_bundle(registry, selected, root=bundle_root) artifacts = {} paths = {} if manifest is not None: + if verify_bundle: + bundles.validate_runtime_bundle(manifest, selected, root=bundle_root) event = registry.get("event_slug", registry.get("event")) if event is not None and manifest.get("event", event) != event: raise ValueError("compute manifest event differs from registry") @@ -130,6 +145,11 @@ def stage_event( paths[key] = (relative, digest) index = {measure["measure_key"]: measure for measure in registry["measures"]} for (key, year), artifact in artifacts.items(): + if verify_bundle: + bundles.validate_runtime_bundle(artifact, selected, root=bundle_root) + start, end = bundles.bundle_window(selected, root=bundle_root) + if not start <= year <= end: + raise ValueError("artifact year is outside selected bundle window") if key not in index: raise ValueError("manifest contains a measure outside the registry") event = registry.get("event_slug", registry.get("event")) @@ -285,9 +305,23 @@ def declared_pairs(field): "PE calendar-year totals proxy fiscal-year totals; PE is static and its certified policy world differs from the announcement baseline.", ], } + if selected != bundles.DEFAULT_BUNDLE: + row["annotations"][0] = ( + f"OBR uses the forecast available at announcement; PE uses a single certified {bundles.bundle_window(selected, root=bundle_root)[0]} population with later calibration and uprating targets." + ) artifact_key = (key, year) artifact = artifacts.get(artifact_key) - if classification == "out_of_household_scope": + if ( + selected != bundles.DEFAULT_BUNDLE + and not bundles.bundle_window(selected, root=bundle_root)[0] + <= year + <= bundles.bundle_window(selected, root=bundle_root)[1] + ): + row["status"] = "outside_bundle_window" + row["reason"] = ( + "FY start outside the selected certified bundle's supported calendar-year window" + ) + elif classification == "out_of_household_scope": row["reason"] = ( source.get("scope_reason") or source.get("classification_reason") @@ -390,6 +424,8 @@ def declared_pairs(field): "missing_measure_years": compute.grid_rows(full_pairs - computed_pairs), "registry_sha256": registry_sha256, } + if selected != bundles.DEFAULT_BUNDLE: + tally.update(bundles.bundle_identity(selected, root=bundle_root)) if inert_pairs: tally["inert_measure_years"] = compute.grid_rows(inert_pairs) if tally["source_rows"] != tally["staged_rows"]: @@ -404,13 +440,17 @@ def declared_pairs(field): def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--event", required=True) + ap.add_argument("--bundle", default=bundles.DEFAULT_BUNDLE) ap.add_argument("--registry", type=Path) ap.add_argument("--artifact-dir", type=Path) ap.add_argument("--output", type=Path) args = ap.parse_args(argv) - default_registry, default_artifacts = compute.event_paths(args.event) + default_registry = bundles.registry_path(args.event, args.bundle) + default_artifacts = bundles.event_output_dir(args.event, args.bundle) registry_path = args.registry or default_registry artifact_dir = args.artifact_dir or default_artifacts + bundles.validate_bundle_path(registry_path, args.bundle, kind="registry") + bundles.validate_bundle_path(artifact_dir, args.bundle, kind="results") payload = registry_path.read_bytes() registry_sha256 = fiscal.hashlib.sha256(payload).hexdigest() manifest_path = artifact_dir / "RUN_MANIFEST.json" @@ -421,10 +461,15 @@ def main(argv: list[str] | None = None) -> int: compute.validate_event_identity(registry, args.event) compute.registry_builder.validate_registry(registry) rows, tally = stage_event( - registry, manifest, artifact_dir=artifact_dir, registry_sha256=registry_sha256 + registry, + manifest, + artifact_dir=artifact_dir, + registry_sha256=registry_sha256, + bundle=args.bundle, ) compute.verify_registry_identity(registry_path, registry_sha256) output = args.output or artifact_dir / "STAGED.jsonl" + bundles.validate_bundle_path(output, args.bundle, kind="results") output.parent.mkdir(parents=True, exist_ok=True) staged_payload = b"".join( (json.dumps(row, sort_keys=True, allow_nan=False) + "\n").encode() diff --git a/pipeline/uk_bundle.py b/pipeline/uk_bundle.py new file mode 100644 index 00000000..5c82d84d --- /dev/null +++ b/pipeline/uk_bundle.py @@ -0,0 +1,238 @@ +"""Offline UK replay bundle selection and receipt identity. + +The legacy pin is read without rewriting it. ``load_bundle`` supplies its +historically audited runtime fields; serialization should use ``bundle_document`` +when preserving legacy receipts. Development indexes may be selected explicitly +with UK_BUNDLE_INDEX, without adding a development bundle to the production index. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_BUNDLE = "populace-uk-2023__pe-uk-2.89.2" +INDEX = Path("data/uk/certified_bundles/index.json") + + +def _key(key): + if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*", key): + raise ValueError("invalid UK bundle key") + return key + + +def bundle_index_path(*, root=ROOT): + path = Path(os.environ.get("UK_BUNDLE_INDEX", INDEX)) + return path if path.is_absolute() else Path(root) / path + + +def bundle_pin_path(key=DEFAULT_BUNDLE, *, root=ROOT): + _key(key) + path = bundle_index_path(root=root) + # Older engine-free test roots contain only the historical pin. + if ( + not path.exists() + and key == DEFAULT_BUNDLE + and not os.environ.get("UK_BUNDLE_INDEX") + ): + return Path(root) / "data/uk/certified_bundle.json" + index = json.loads(path.read_text()) + if ( + index.get("schema_version") != 1 + or index.get("default_bundle") != DEFAULT_BUNDLE + ): + raise ValueError("invalid UK bundle index") + try: + pin = Path(index["bundles"][key]) + except KeyError as error: + raise ValueError(f"unregistered UK bundle: {key}") from error + return pin if pin.is_absolute() else Path(root) / pin + + +def bundle_document(key=DEFAULT_BUNDLE, *, root=ROOT): + pin = json.loads(bundle_pin_path(key, root=root).read_text()) + if pin.get("bundle_key", key) != key: + raise ValueError("bundle pin key differs from index") + return pin + + +def load_bundle(key=DEFAULT_BUNDLE, *, root=ROOT): + pin = bundle_document(key, root=root) + if key == DEFAULT_BUNDLE: + defaults = { + "repo_type": "dataset", + "release_tag": pin["revision"], + "resolved_hf_commit": "a75a9a831d6b07aaffbd09713f2a1124f5c0f08f", + "release_manifest_revision": pin["revision"], + "data_year": 2023, + "supported_calendar_years": [2023, 2030], + "compatible_core_packages": [ + {"name": "policyengine-core", "specifier": "==3.27.1"} + ], + "managed_loader_version": "5.0.2", + "requirements_freeze": "docs/uk_replay/requirements.txt", + "offline_audit": "docs/uk_replay/OFFLINE_BUNDLE_AUDIT.json", + "data_build_id": pin["revision"], + } + # The resolved commit is recorded by the existing offline audit, not + # inferred from a moving remote branch. + audit = Path(root) / defaults["offline_audit"] + if audit.exists(): + document = json.loads(audit.read_text()) + commit = document.get("certified_identity", {}).get("resolved_hf_commit") + if commit: + defaults["resolved_hf_commit"] = commit + for field, value in defaults.items(): + pin.setdefault(field, value) + required = ( + "repo_type", + "release_tag", + "resolved_hf_commit", + "release_manifest_revision", + "data_year", + "supported_calendar_years", + "compatible_core_packages", + "managed_loader_version", + "requirements_freeze", + "offline_audit", + ) + missing = [name for name in required if name not in pin] + if missing: + raise ValueError(f"bundle pin missing fields: {missing}") + pin["bundle_key"] = key + pin.setdefault("resolved_commit", pin["resolved_hf_commit"]) + pin.setdefault("data_build_id", pin["revision"]) + if pin["repo_type"] not in ("model", "dataset"): + raise ValueError("invalid bundle repo_type") + bundle_window(pin) + return pin + + +def bundle_window(pin_or_key=DEFAULT_BUNDLE, *, root=ROOT): + pin = ( + load_bundle(pin_or_key, root=root) + if isinstance(pin_or_key, str) + else pin_or_key + ) + years = pin["supported_calendar_years"] + if len(years) != 2 or any(type(year) is not int for year in years): + raise ValueError("bundle year window must contain two integer endpoints") + start, end = years + if start != pin["data_year"] or start > end: + raise ValueError("bundle window must begin at its stored data year") + return start, end + + +def registry_path(event, key=DEFAULT_BUNDLE, *, root=ROOT): + _key(event) + _key(key) + directory = Path(root) / "data/uk/events" + if key != DEFAULT_BUNDLE: + directory /= f"bundles/{key}" + return directory / f"{event}_measures.json" + + +def results_root(key=DEFAULT_BUNDLE, *, root=ROOT): + _key(key) + directory = Path(root) / "results/uk/events" + return directory if key == DEFAULT_BUNDLE else directory / "bundles" / key + + +def event_output_dir(event, key=DEFAULT_BUNDLE, *, root=ROOT): + _key(event) + return results_root(key, root=root) / event + + +def _exact_pin(pin, field, package): + entries = [entry["specifier"] for entry in pin[field] if entry["name"] == package] + if len(entries) != 1 or not entries[0].startswith("==") or "*" in entries[0]: + raise ValueError(f"bundle needs one exact {package} pin") + return entries[0][2:] + + +def bundle_identity(key=DEFAULT_BUNDLE, *, root=ROOT): + pin = load_bundle(key, root=root) + return { + "bundle_key": key, + "bundle_pin_sha256": hashlib.sha256( + bundle_pin_path(key, root=root).read_bytes() + ).hexdigest(), + "certified_dataset_sha256": pin["sha256"], + "engine_versions": { + "policyengine": pin["managed_loader_version"], + "policyengine-uk": _exact_pin( + pin, "compatible_model_packages", "policyengine-uk" + ), + "policyengine-core": _exact_pin( + pin, "compatible_core_packages", "policyengine-core" + ), + }, + "supported_calendar_years": list(bundle_window(pin)), + } + + +def validate_bundle_identity(document, key=DEFAULT_BUNDLE, *, root=ROOT): + identity = bundle_identity(key, root=root) + for field, expected in identity.items(): + if field not in document: + if key != DEFAULT_BUNDLE: + raise ValueError(f"missing bundle identity field: {field}") + elif ( + field == "certified_dataset_sha256" + and key == DEFAULT_BUNDLE + and "bundle_key" not in document + ): + continue # Existing stages separately validate their legacy digest. + elif document[field] != expected: + raise ValueError(f"bundle identity mismatch: {field}") + return identity + + +def validate_runtime_bundle(document, key=DEFAULT_BUNDLE, *, root=ROOT): + identity = validate_bundle_identity(document, key, root=root) + pin = load_bundle(key, root=root) + expected = { + "engine_version": identity["engine_versions"]["policyengine-uk"], + "policyengine_core_version": identity["engine_versions"]["policyengine-core"], + "policyengine_version": identity["engine_versions"]["policyengine"], + "data_bundle": pin["data_build_id"], + "dataset_sha256_before": pin["sha256"], + "dataset_sha256_after": pin["sha256"], + } + for field, value in expected.items(): + if field in document and document[field] != value: + raise ValueError(f"runtime bundle mismatch: {field}") + return identity + + +def validate_bundle_path(path, key=DEFAULT_BUNDLE, *, kind, root=ROOT): + _key(key) + canonical = Path(root) / ( + "data/uk/events" if kind == "registry" else "results/uk/events" + ) + if kind not in ("registry", "results"): + raise ValueError("invalid bundle path kind") + try: + relative = Path(path).resolve().relative_to(canonical.resolve()) + except ValueError: + return Path(path) # Explicit development or determinism-check directory. + parts = relative.parts + actual = parts[1] if len(parts) > 1 and parts[0] == "bundles" else DEFAULT_BUNDLE + if actual != key: + raise ValueError("path belongs to another bundle") + return Path(path) + + +def validate_registry_bundle(registry, key=DEFAULT_BUNDLE, *, root=ROOT): + validate_bundle_identity(registry, key, root=root) + expected = bundle_document(key, root=root) + if registry.get("bundle") != expected: + raise ValueError("registry bundle pin differs from selected bundle") + start, end = bundle_window(key, root=root) + if any(not start <= year <= end for year in registry["calendar_years"]): + raise ValueError("registry year outside bundle window") + return expected diff --git a/tests/test_modal_uk_event.py b/tests/test_modal_uk_event.py index 4a75e473..d8eb8f27 100644 --- a/tests/test_modal_uk_event.py +++ b/tests/test_modal_uk_event.py @@ -30,6 +30,15 @@ def inputs(tmp_path): path = tmp_path / relative path.parent.mkdir(parents=True, exist_ok=True) path.write_text("{}") + (tmp_path / "data/uk/certified_bundles/index.json").write_text( + json.dumps( + { + "schema_version": 1, + "default_bundle": runner.DEFAULT_BUNDLE, + "bundles": {runner.DEFAULT_BUNDLE: "data/uk/certified_bundle.json"}, + } + ) + ) (tmp_path / "data/uk/certified_bundle.json").write_text(json.dumps(cert)) (tmp_path / "docs/uk_replay/OFFLINE_BUNDLE_AUDIT.json").write_text( json.dumps(audit) @@ -42,6 +51,7 @@ def inputs(tmp_path): json.dumps( { "event_slug": "autumn_budget_2024", + "bundle": cert, "calendar_years": list(range(2024, 2030)), "measures": [ {"measure_key": "autumn_budget_2024__private_school_vat_20pct"} @@ -76,6 +86,174 @@ def test_allowlist_mounts_only_pinned_inputs(inputs): runner.verify_inputs(manifest, root=root) +@pytest.fixture +def model_bundle(inputs): + """A model-repository bundle exists only inside this temporary test root.""" + root, _, _ = inputs + key = "development-2024__pe-uk-2.102.3" + artifact_bytes = b"new certified fixture" + pin = { + "schema_version": 2, + "bundle_key": key, + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.56.16", + "release_tag": "1.56.16", + "resolved_hf_commit": "b" * 40, + "release_manifest_revision": "c" * 40, + "artifact": "enhanced_frs_2024_25.h5", + "sha256": runner.digest_bytes(artifact_bytes), + "size_bytes": len(artifact_bytes), + "data_build_id": "policyengine-uk-data-1.56.16", + "data_year": 2024, + "supported_calendar_years": [2024, 2030], + "compatible_model_packages": [ + {"name": "policyengine-uk", "specifier": "==2.102.3"} + ], + "compatible_core_packages": [ + {"name": "policyengine-core", "specifier": "==3.32.10"} + ], + "managed_loader_version": "6.2.5", + "requirements_freeze": "docs/uk_replay/requirements-development.txt", + "offline_audit": "docs/uk_replay/OFFLINE_DEVELOPMENT.json", + "development_bundle": True, + } + pin_relative = "data/uk/certified_bundles/development.json" + (root / pin_relative).write_bytes(runner.canonical_bytes(pin)) + index = root / "data/uk/certified_bundles/index.json" + document = json.loads(index.read_bytes()) + document["bundles"][key] = pin_relative + index.write_bytes(runner.canonical_bytes(document)) + audit = { + "policyengine_version": "6.2.5", + "packaged_bundle_manifest_sha256": "d" * 64, + "certified_identity": { + **pin, + "model_version": "2.102.3", + "core_version": "3.32.10", + }, + } + (root / pin["offline_audit"]).write_bytes(runner.canonical_bytes(audit)) + (root / pin["requirements_freeze"]).write_text( + "policyengine==6.2.5\npolicyengine-uk==2.102.3\npolicyengine-core==3.32.10\n" + ) + registry_path = runner.uk_bundle.registry_path("autumn_budget_2024", key, root=root) + registry_path.parent.mkdir(parents=True) + registry_path.write_bytes( + runner.canonical_bytes( + { + "event_slug": "autumn_budget_2024", + "calendar_years": [2024, 2026], + "bundle": pin, + "measures": [ + {"measure_key": "autumn_budget_2024__private_school_vat_20pct"} + ], + **runner.uk_bundle.bundle_identity(key, root=root), + } + ) + ) + cache_name, commit = runner.hf_identity(key, root=root) + cache = root / runner.HF_CACHE_RELATIVE / cache_name + ref = cache / "refs" / pin["revision"] + artifact = cache / "snapshots" / commit / pin["artifact"] + ref.parent.mkdir(parents=True) + artifact.parent.mkdir(parents=True) + ref.write_text(commit) + artifact.write_bytes(artifact_bytes) + manifest, mounts = runner.input_manifest( + "autumn_budget_2024", + bundle=key, + root=root, + hf_cache=root / runner.HF_CACHE_RELATIVE, + ) + return root, key, pin, manifest, mounts + + +def test_model_bundle_selects_exact_cache_freeze_audit_and_registry(model_bundle): + root, key, pin, manifest, mounts = model_bundle + assert manifest["bundle_key"] == key + assert manifest["hf_commit"] == pin["resolved_hf_commit"] + assert pin["requirements_freeze"] in mounts and pin["offline_audit"] in mounts + assert "data/uk/certified_bundle.json" not in mounts + assert any( + "models--policyengine--policyengine-uk-data-private/snapshots/" in path + for path in mounts + ) + assert not any( + "datasets--policyengine--populace-uk-private" in path for path in mounts + ) + runner.verify_inputs(manifest, bundle=key, root=root) + + +def test_bundle_window_and_canonical_output_are_selected_together(model_bundle): + root, key, _, _, _ = model_bundle + output = f"results/uk/events/bundles/{key}/autumn_budget_2024" + request = runner.request_spec( + "autumn_budget_2024", + bundle=key, + root=root, + years=[2024], + measures=[], + output_dir=output, + preflight_only=False, + ) + runner.validate_selection(request, root=root) + with pytest.raises(ValueError, match="2024–2030"): + runner.request_spec( + "autumn_budget_2024", + bundle=key, + root=root, + years=[2023], + measures=[], + output_dir=output, + preflight_only=False, + ) + with pytest.raises(ValueError, match="remote output"): + runner.request_spec( + "autumn_budget_2024", + bundle=key, + root=root, + years=[], + measures=[], + output_dir="results/uk/events/autumn_budget_2024", + preflight_only=False, + ) + + +@pytest.mark.parametrize("target", ["manifest", "registry"]) +def test_cross_bundle_identity_is_rejected_before_engine_import(model_bundle, target): + root, key, _, manifest, _ = model_bundle + if target == "manifest": + with pytest.raises(ValueError, match="bundle identity mismatch"): + runner.verify_inputs(manifest, root=root) + else: + registry = runner.uk_bundle.registry_path("autumn_budget_2024", key, root=root) + document = json.loads(registry.read_bytes()) + document["bundle_key"] = runner.DEFAULT_BUNDLE + registry.write_bytes(runner.canonical_bytes(document)) + request = runner.request_spec( + "autumn_budget_2024", + bundle=key, + root=root, + years=[2024], + measures=[], + output_dir=f"results/uk/events/bundles/{key}/autumn_budget_2024", + preflight_only=False, + ) + with pytest.raises(ValueError, match="bundle identity mismatch"): + runner.validate_selection(request, root=root) + + +def test_model_bundle_rejects_audit_engine_or_core_drift(model_bundle): + root, key, pin, _, _ = model_bundle + audit_path = root / pin["offline_audit"] + audit = json.loads(audit_path.read_bytes()) + audit["certified_identity"]["core_version"] = "3.27.1" + audit_path.write_bytes(runner.canonical_bytes(audit)) + with pytest.raises(ValueError, match="audited engine runtime"): + runner.certified_pin(root, key) + + def test_upload_hash_gate_rejects_same_size_changed_h5_before_engine_import(inputs): root, manifest, mounts = inputs artifact = next( @@ -122,6 +300,31 @@ def test_requirements_are_exact_and_engine_compatible(requirement): runner.requirements_pins(text) +def test_uv_export_markers_select_the_remote_linux_python_freeze(): + text = "\n".join( + f"{name}=={version}" for name, version in runner.ENGINE_PINS.items() + ) + text += "\nnumpy==2.1.3 ; python_full_version < '3.14'\nnumpy==2.5.3 ; python_full_version >= '3.14'\ncolorama==0.4.6 ; sys_platform == 'win32'\nhf-xet==1.1.9 ; platform_machine == 'aarch64' or platform_machine == 'x86_64'\n" + pins = runner.requirements_pins(text) + assert pins["numpy"] == "2.1.3" + assert pins["hf-xet"] == "1.1.9" + assert "colorama" not in pins + + +@pytest.mark.parametrize( + "expression", + [ + "__import__('os').system('true')", + "unknown == 'linux'", + "python_version.startswith('3')", + "sys_platform ==", + ], +) +def test_requirements_markers_reject_code_and_unknown_names(expression): + with pytest.raises(ValueError, match="marker"): + runner.remote_marker_applies(expression) + + @pytest.mark.parametrize( "path", ["/tmp/out", "../out", "results/../out", "a//b", "a\\b", "./out"] ) @@ -267,13 +470,13 @@ def interrupted_remote(payload): monkeypatch.setattr(runner, "validate_selection", lambda request: None) monkeypatch.setattr( - runner, "input_manifest", lambda event, hf_cache=None: ({"event": event}, {}) + runner, "input_manifest", lambda event, **kwargs: ({"event": event}, {}) ) monkeypatch.setattr(runner, "download_prefix", lambda value: tmp_path / "download") monkeypatch.setattr( runner, "modal_function", - lambda mounts: ( + lambda mounts, **kwargs: ( SimpleNamespace(enable_output=nullcontext), SimpleNamespace(run=SuppressInterrupt), SimpleNamespace(remote=interrupted_remote), @@ -348,7 +551,7 @@ def concurrent(**kwargs): assert recorded["commands"][0] == "uv python install 3.12.13" assert "uv venv --python 3.12.13 /opt/replay-venv" in recorded["commands"] assert any( - "--requirements /opt/replay-requirements.txt" in command + "--no-deps --requirements /opt/replay-requirements.txt" in command for command in recorded["commands"] ) assert all( diff --git a/tests/test_uk_bundle_adapter.py b/tests/test_uk_bundle_adapter.py new file mode 100644 index 00000000..74039fe8 --- /dev/null +++ b/tests/test_uk_bundle_adapter.py @@ -0,0 +1,525 @@ +"""Bundle adapter gates using mocked managed loaders; no population simulation.""" + +import hashlib +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest +from hypothesis import HealthCheck, given, settings, strategies as st + +from pipeline import compute_uk_event as compute +from pipeline import compute_uk_obr_costings as fiscal +from pipeline import uk_bundle as bundles + + +@pytest.fixture +def adapter(monkeypatch, tmp_path): + key = "development-adapter-test" + cached = tmp_path / "cached.h5" + cached.write_bytes(b"small certified test population") + digest = hashlib.sha256(cached.read_bytes()).hexdigest() + versions = { + "policyengine": "6.2.5", + "policyengine-uk": "2.102.3", + "policyengine-core": "3.32.10", + } + pin = { + "bundle_key": key, + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "artifact": "enhanced_frs_2024_25.h5", + "revision": "1.56.16", + "release_tag": "1.56.16", + "resolved_hf_commit": "a" * 40, + "release_manifest_revision": "b" * 40, + "sha256": digest, + "size_bytes": cached.stat().st_size, + "data_year": 2024, + "supported_calendar_years": [2024, 2030], + "data_build_id": "policyengine-uk-data-1.56.16", + "managed_loader_version": "6.2.5", + "requirements_freeze": "unused.txt", + "offline_audit": "unused.json", + "development_bundle": True, + "compatible_model_packages": [ + {"name": "policyengine-uk", "specifier": "==2.102.3"} + ], + "compatible_core_packages": [ + {"name": "policyengine-core", "specifier": "==3.32.10"} + ], + } + pin_path = tmp_path / "pin.json" + pin_path.write_text(json.dumps(pin)) + index = tmp_path / "index.json" + index.write_text( + json.dumps( + { + "schema_version": 1, + "default_bundle": bundles.DEFAULT_BUNDLE, + "bundles": { + key: str(pin_path), + bundles.DEFAULT_BUNDLE: str( + bundles.ROOT / "data/uk/certified_bundle.json" + ), + }, + } + ) + ) + monkeypatch.setenv("UK_BUNDLE_INDEX", str(index)) + for name in ( + "HF_HUB_OFFLINE", + "TRANSFORMERS_OFFLINE", + "HF_DATASETS_OFFLINE", + "HF_HUB_DISABLE_TELEMETRY", + "POLICYENGINE_UK_DATA_REPO", + ): + monkeypatch.setenv(name, "initial-test-value") + monkeypatch.setattr(fiscal, "ROOT", tmp_path) + monkeypatch.setattr(compute, "ROOT", tmp_path) + monkeypatch.setattr(fiscal, "LOCAL_DATA_MIRROR_ROOT", tmp_path / "mirror") + monkeypatch.setattr(fiscal, "version", lambda name: versions[name]) + monkeypatch.setattr(fiscal, "package_version", lambda name: versions[name]) + compute.runtime_versions.cache_clear() + monkeypatch.setattr( + fiscal, "block_runtime_network", lambda: network_blocks.append(True) + ) + network_blocks, downloads, materializations, simulations, calculations = ( + [], + [], + [], + [], + [], + ) + release = { + "model_version": "2.102.3", + "default_dataset_uri": f"hf://{pin['repo_id']}/{pin['artifact']}@{pin['revision']}", + "certified_data_artifact_sha256": digest, + "certified_data_build_id": pin["data_build_id"], + "compatibility_basis": "legacy_compatible_model_package", + "policyengine_version": "6.2.5", + } + data_year = ["2024"] + dataset_years = list(range(2024, 2031)) + mutation = [False] + + def download(**kwargs): + downloads.append(kwargs) + return str(cached) + + def materialize(country): + materializations.append((country, Path.cwd())) + return SimpleNamespace(path=Path.cwd() / "data" / pin["artifact"]) + + def managed(**kwargs): + simulations.append(kwargs) + variable = SimpleNamespace( + label="Income tax", + entity=SimpleNamespace(key="person"), + definition_period="year", + unit="currency-GBP", + ) + + def calculate(name, year): + calculations.append((name, year)) + if mutation[0]: + cached.write_bytes(b"mutated during aggregate reads") + return SimpleNamespace(sum=lambda: 100.0 + 5.0 * bool(kwargs)) + + return SimpleNamespace( + policyengine_bundle={ + **release, + "runtime_dataset_source": str(Path.cwd() / "data" / pin["artifact"]), + }, + dataset=SimpleNamespace(years=dataset_years), + tax_benefit_system=SimpleNamespace(variables={"income_tax": variable}), + calculate=calculate, + ) + + pe = SimpleNamespace( + uk=SimpleNamespace( + uk_latest=SimpleNamespace(release_bundle=release), + managed_microsimulation=managed, + ) + ) + monkeypatch.setitem(sys.modules, "policyengine", pe) + monkeypatch.setitem( + sys.modules, "huggingface_hub", SimpleNamespace(hf_hub_download=download) + ) + monkeypatch.setitem( + sys.modules, + "pandas", + SimpleNamespace( + read_hdf=lambda path, table: SimpleNamespace(tolist=lambda: data_year) + ), + ) + monkeypatch.setitem( + sys.modules, + "policyengine.provenance.dataset_materialization", + SimpleNamespace(materialize_dataset=materialize), + ) + + class Memory: + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def as_dict(self): + return {} + + monkeypatch.setattr(fiscal, "PeakRSSSampler", Memory) + monkeypatch.setattr(fiscal, "current_rss_bytes", lambda: (0, "test")) + value = SimpleNamespace( + key=key, + pin=pin, + pin_path=pin_path, + cached=cached, + versions=versions, + release=release, + network_blocks=network_blocks, + downloads=downloads, + materializations=materializations, + simulations=simulations, + calculations=calculations, + data_year=data_year, + dataset_years=dataset_years, + mutation=mutation, + pe=pe, + root=tmp_path, + ) + yield value + compute.runtime_versions.cache_clear() + + +def run(adapter, pre, **overrides): + args = dict( + year=2026, + variables=["income_tax"], + reform=None, + runtime_dataset_source=Path(pre["runtime_dataset_source"]), + expected_dataset_sha256=pre["sha256"], + bundle_key=adapter.key, + ) + return fiscal.run_managed_simulation(**{**args, **overrides}) + + +def test_preflight_is_cached_model_repository_and_exact_materialized_file(adapter): + original = Path.cwd() + pre = fiscal.preflight_certified_dataset(adapter.key) + assert adapter.downloads == [ + { + "repo_id": adapter.pin["repo_id"], + "repo_type": "model", + "filename": adapter.pin["artifact"], + "revision": adapter.pin["resolved_hf_commit"], + "local_files_only": True, + } + ] + assert pre["runtime_network_blocked"] is True and pre["local_files_only"] is True + assert adapter.network_blocks + assert Path(pre["runtime_dataset_source"]).resolve() == adapter.cached.resolve() + assert ( + fiscal.sha256_file(Path(pre["runtime_dataset_source"])) + == pre["sha256"] + == adapter.pin["sha256"] + ) + assert adapter.materializations[0][0] == "uk" + assert Path.cwd() == original + assert adapter.simulations == [] + + +@pytest.mark.parametrize( + "package", ["policyengine", "policyengine-uk", "policyengine-core"] +) +def test_preflight_refuses_each_engine_package_drift_before_artifact_lookup( + adapter, package +): + adapter.versions[package] = "0.0.1" + with pytest.raises(RuntimeError, match="(installed|version differs)"): + fiscal.preflight_certified_dataset(adapter.key) + assert adapter.downloads == [] and adapter.simulations == [] + + +@pytest.mark.parametrize( + "field", + [ + "default_dataset_uri", + "certified_data_artifact_sha256", + "certified_data_build_id", + "compatibility_basis", + ], +) +def test_preflight_refuses_wrong_release_certification_before_lookup(adapter, field): + adapter.release[field] = ( + "unverified_test" if field == "compatibility_basis" else "different" + ) + with pytest.raises(RuntimeError, match="(managed release|certification)"): + fiscal.preflight_certified_dataset(adapter.key) + assert adapter.downloads == [] and adapter.simulations == [] + + +@pytest.mark.parametrize("field", ["sha256", "size_bytes", "data_year"]) +def test_preflight_refuses_cached_digest_size_or_data_year_drift(adapter, field): + if field == "sha256": + adapter.cached.write_bytes(b"x" * adapter.pin["size_bytes"]) + elif field == "size_bytes": + adapter.pin[field] += 1 + adapter.pin_path.write_text(json.dumps(adapter.pin)) + else: + adapter.data_year[:] = ["2026"] + with pytest.raises(RuntimeError, match="(digest or size|data year)"): + fiscal.preflight_certified_dataset(adapter.key) + assert adapter.materializations == [] and adapter.simulations == [] + + +def test_preflight_refuses_materializer_reporting_a_different_file( + adapter, monkeypatch +): + monkeypatch.setitem( + sys.modules, + "policyengine.provenance.dataset_materialization", + SimpleNamespace( + materialize_dataset=lambda country: SimpleNamespace( + path=adapter.root / "other.h5" + ) + ), + ) + with pytest.raises(RuntimeError, match="exact hash-verified file"): + fiscal.preflight_certified_dataset(adapter.key) + + +@pytest.mark.parametrize("selection", ["baseline", "reform", "scenario"]) +def test_managed_loader_receives_reform_or_scenario_and_projects_requested_year( + adapter, selection +): + pre = fiscal.preflight_certified_dataset(adapter.key) + reform = {"gov.tax.rate": {"2026-01-01.2030-12-31": 0.3}} + scenario = object() + overrides = ( + {"reform": reform} + if selection == "reform" + else {"scenario": scenario} + if selection == "scenario" + else {} + ) + result = run(adapter, pre, **overrides) + assert adapter.simulations == ( + [{"reform": reform}] + if selection == "reform" + else [{"scenario": scenario}] + if selection == "scenario" + else [{}] + ) + assert adapter.calculations == [("income_tax", "2026")] + assert ( + result["dataset_sha256_before"] + == result["dataset_sha256_after"] + == adapter.pin["sha256"] + ) + assert ( + Path(result["policyengine_bundle"]["runtime_dataset_source"]).resolve() + == adapter.cached.resolve() + ) + + +@pytest.mark.parametrize("years", [[2026, 2027, 2028], [2024, 2025]]) +def test_managed_simulation_refuses_projected_year_as_base_or_missing_year( + adapter, years +): + pre = fiscal.preflight_certified_dataset(adapter.key) + adapter.dataset_years[:] = years + with pytest.raises(RuntimeError, match="project the pinned data year"): + run(adapter, pre) + assert adapter.calculations == [] + + +@pytest.mark.parametrize("package", ["policyengine", "policyengine-core"]) +def test_managed_simulation_rechecks_loader_and_core_at_construction(adapter, package): + pre = fiscal.preflight_certified_dataset(adapter.key) + adapter.versions[package] = "0.0.1" + with pytest.raises(RuntimeError, match="version differs"): + run(adapter, pre) + assert adapter.simulations == [] + + +def test_managed_simulation_refuses_mutation_after_calculation(adapter): + pre = fiscal.preflight_certified_dataset(adapter.key) + adapter.mutation[0] = True + with pytest.raises(RuntimeError, match="immediately after aggregate reads"): + run(adapter, pre) + + +def test_managed_simulation_refuses_wrong_reported_runtime_path(adapter): + pre = fiscal.preflight_certified_dataset(adapter.key) + managed = adapter.pe.uk.managed_microsimulation + + def wrong_source(**kwargs): + sim = managed(**kwargs) + sim.policyengine_bundle["runtime_dataset_source"] = str( + adapter.root / "other.h5" + ) + return sim + + adapter.pe.uk.managed_microsimulation = wrong_source + with pytest.raises(RuntimeError, match="exact file hashed"): + run(adapter, pre) + assert adapter.calculations == [] + + +@settings(deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture]) +@given(st.integers(min_value=2015, max_value=2040)) +def test_year_window_comes_from_selected_bundle(adapter, year): + registry = {"calendar_years": list(range(2015, 2041))} + if 2024 <= year <= 2030: + assert compute.validate_years([year], registry, adapter.key) == [year] + else: + with pytest.raises(ValueError, match="2024 through 2030"): + compute.validate_years([year], registry, adapter.key) + + +@pytest.mark.parametrize("kind", ["manifest", "progress"]) +def test_retained_receipt_foreign_bundle_is_rejected(adapter, kind): + pre = fiscal.preflight_certified_dataset(adapter.key) + output = bundles.event_output_dir( + "autumn_budget_2024", adapter.key, root=adapter.root + ) + output.mkdir(parents=True) + receipt = compute.progress_receipt( + { + "event": "autumn_budget_2024", + "year": 2026, + "registry_sha256": "c" * 64, + "pre": pre, + "bundle": adapter.key, + }, + {}, + ) + receipt["bundle_key"] = bundles.DEFAULT_BUNDLE + ( + output + / ("RUN_MANIFEST.json" if kind == "manifest" else "RUN_PROGRESS_2026.json") + ).write_bytes(compute.canonical_bytes(receipt)) + with pytest.raises(ValueError, match="bundle identity mismatch"): + compute.retained_artifact_digests( + output, + event="autumn_budget_2024", + years=[2026], + registry_sha256="c" * 64, + pre=pre, + bundle=adapter.key, + ) + + +@pytest.mark.parametrize("kind", ["manifest", "progress"]) +@pytest.mark.parametrize( + "field", + [ + "engine_version", + "policyengine_version", + "policyengine_core_version", + "data_bundle", + ], +) +def test_retained_receipt_runtime_drift_is_rejected_even_with_correct_nested_identity( + adapter, kind, field +): + pre = fiscal.preflight_certified_dataset(adapter.key) + output = bundles.event_output_dir( + "autumn_budget_2024", adapter.key, root=adapter.root + ) + output.mkdir(parents=True) + receipt = compute.progress_receipt( + { + "event": "autumn_budget_2024", + "year": 2026, + "registry_sha256": "c" * 64, + "pre": pre, + "bundle": adapter.key, + }, + {}, + ) + receipt[field] = "foreign-runtime" + ( + output + / ("RUN_MANIFEST.json" if kind == "manifest" else "RUN_PROGRESS_2026.json") + ).write_bytes(compute.canonical_bytes(receipt)) + with pytest.raises(ValueError, match="runtime bundle mismatch"): + compute.retained_artifact_digests( + output, + event="autumn_budget_2024", + years=[2026], + registry_sha256="c" * 64, + pre=pre, + bundle=adapter.key, + ) + + +def test_retained_receipt_cannot_name_another_bundle_artifact(adapter): + pre = fiscal.preflight_certified_dataset(adapter.key) + output = bundles.event_output_dir( + "autumn_budget_2024", adapter.key, root=adapter.root + ) + output.mkdir(parents=True) + receipt = compute.progress_receipt( + { + "event": "autumn_budget_2024", + "year": 2026, + "registry_sha256": "c" * 64, + "pre": pre, + "bundle": adapter.key, + }, + {"results/uk/events/autumn_budget_2024/foreign_2026.json": "d" * 64}, + ) + (output / "RUN_PROGRESS_2026.json").write_bytes(compute.canonical_bytes(receipt)) + with pytest.raises(ValueError, match="outside this event directory"): + compute.retained_artifact_digests( + output, + event="autumn_budget_2024", + years=[2026], + registry_sha256="c" * 64, + pre=pre, + bundle=adapter.key, + ) + + +def test_default_progress_bytes_do_not_gain_bundle_identity(monkeypatch): + monkeypatch.setattr( + fiscal, + "package_version", + lambda package: { + "policyengine": "5.0.2", + "policyengine-core": "3.27.1", + "policyengine-uk": "2.89.2", + }[package], + ) + compute.runtime_versions.cache_clear() + job = { + "event": "autumn_budget_2024", + "year": 2026, + "registry_sha256": "c" * 64, + "pre": { + "sha256": "d" * 64, + "release_bundle": {"certified_data_build_id": "legacy"}, + }, + } + implicit = compute.progress_receipt(job, {}) + explicit = compute.progress_receipt({**job, "bundle": bundles.DEFAULT_BUNDLE}, {}) + assert compute.canonical_bytes(implicit) == compute.canonical_bytes(explicit) + assert "bundle_key" not in implicit and "engine_versions" not in implicit + compute.runtime_versions.cache_clear() + + +def test_output_paths_cannot_cross_registered_bundle_namespaces(adapter): + with pytest.raises(ValueError, match="another bundle"): + compute.validate_output_bundle( + adapter.root / "results/uk/events/autumn_budget_2024", adapter.key + ) + with pytest.raises(ValueError, match="another bundle"): + compute.validate_output_bundle( + adapter.root + / f"results/uk/events/bundles/{adapter.key}/autumn_budget_2024", + bundles.DEFAULT_BUNDLE, + ) diff --git a/tests/test_uk_bundle_registry.py b/tests/test_uk_bundle_registry.py new file mode 100644 index 00000000..857f0e23 --- /dev/null +++ b/tests/test_uk_bundle_registry.py @@ -0,0 +1,463 @@ +"""Selection and registration invariants without national simulations.""" + +from copy import deepcopy +import hashlib +import json +from pathlib import Path + +import pytest +from hypothesis import given, strategies as st + +from pipeline.uk_bundle import ( + DEFAULT_BUNDLE, + ROOT, + bundle_document, + bundle_identity, + bundle_pin_path, + bundle_window, + event_output_dir, + load_bundle, + registry_path, + validate_bundle_identity, + validate_bundle_path, + validate_registry_bundle, +) +from pipeline.register_uk_certified_bundle import write_registration, _extension_end +from pipeline.export_uk_bundle_requirements import export_requirements + + +def development_index(tmp_path, monkeypatch): + pin = load_bundle() + pin.update( + bundle_key="development", + data_year=2024, + supported_calendar_years=[2024, 2030], + development_bundle=True, + ) + pin_path = tmp_path / "development.json" + index = tmp_path / "index.json" + write_registration(pin, pin_path, index, root=tmp_path) + document = json.loads(index.read_text()) + document["bundles"]["development"] = str(pin_path) + index.write_text(json.dumps(document)) + monkeypatch.setenv("UK_BUNDLE_INDEX", str(index)) + return pin, pin_path + + +def test_default_pin_bytes_and_paths_stay_historical(): + pin = ROOT / "data/uk/certified_bundle.json" + assert bundle_pin_path() == pin + assert bundle_document() == json.loads(pin.read_text()) + assert ( + registry_path("spring_budget_2023") + == ROOT / "data/uk/events/spring_budget_2023_measures.json" + ) + assert ( + event_output_dir("spring_budget_2023") + == ROOT / "results/uk/events/spring_budget_2023" + ) + assert bundle_window() == (2023, 2030) + assert load_bundle()["managed_loader_version"] == "5.0.2" + + +def test_new_bundle_identity_requires_pin_window_and_versions(tmp_path, monkeypatch): + development_index(tmp_path, monkeypatch) + identity = bundle_identity("development", root=tmp_path) + validate_bundle_identity(identity, "development", root=tmp_path) + for field in identity: + missing = {name: value for name, value in identity.items() if name != field} + with pytest.raises(ValueError, match="missing bundle identity"): + validate_bundle_identity(missing, "development", root=tmp_path) + assert identity["supported_calendar_years"] == [2024, 2030] + + +@given( + st.sampled_from( + [ + "engine_versions", + "bundle_key", + "bundle_pin_sha256", + "supported_calendar_years", + ] + ) +) +def test_foreign_default_identity_rejected(field): + identity = bundle_identity() + identity[field] = "foreign" + with pytest.raises(ValueError, match="bundle identity mismatch"): + validate_bundle_identity(identity) + + +def test_foreign_pin_registry_and_window_rejected(tmp_path, monkeypatch): + pin, _ = development_index(tmp_path, monkeypatch) + identity = bundle_identity("development", root=tmp_path) + registry = {**identity, "bundle": pin, "calendar_years": [2024, 2030]} + validate_registry_bundle(registry, "development", root=tmp_path) + registry["calendar_years"].append(2023) + with pytest.raises(ValueError, match="outside bundle window"): + validate_registry_bundle(registry, "development", root=tmp_path) + registry["calendar_years"] = [2024] + registry["bundle"]["sha256"] = "different" + with pytest.raises(ValueError, match="pin differs"): + validate_registry_bundle(registry, "development", root=tmp_path) + + +def test_window_change_keeps_source_accounting_once(tmp_path, monkeypatch): + development_index(tmp_path, monkeypatch) + from pipeline import build_uk_event_registry as builder + + rows = [ + { + "source_row_id": f"row-{year}", + "title": "Business rates: example", + "fy": f"{year}-{str(year + 1)[-2:]}", + "tax_head": "Business rates", + "head_kind": "tax", + "metric": "revenue_change", + "value_gbp": year, + "value_gbp_decimal": str(year), + "fiscal_event": "Audit Event", + } + for year in (2022, 2023, 2024, 2030, 2031) + ] + monkeypatch.setattr(builder, "source_rows", lambda event: rows) + base = builder.build_registry("audit_event", [], [], None) + new = builder.build_registry("audit_event", [], [], None, "development") + for name in ( + "rows_in", + "rows_classified", + "net_gbp_in_decimal", + "net_gbp_classified_decimal", + "absolute_gbp_in_decimal", + "absolute_gbp_classified_decimal", + "by_fy", + ): + assert base["accounting"][name] == new["accounting"][name] + assert base["calendar_years"] == [2023, 2024, 2030] + assert new["calendar_years"] == [2024, 2030] + staged = [row for measure in new["measures"] for row in measure["source_rows"]] + assert { + row["source_row_id"] + for row in staged + if row.get("computation_status") == "outside_bundle_window" + } == {"row-2022", "row-2023", "row-2031"} + assert len(staged) == len({row["source_row_id"] for row in staged}) == 5 + + +def test_bundle_path_isolation_allows_explicit_dev_receipts(): + for kind, parent in ( + ("registry", "data/uk/events"), + ("results", "results/uk/events"), + ): + with pytest.raises(ValueError, match="another bundle"): + validate_bundle_path( + ROOT / parent / "bundles/other/event.json", DEFAULT_BUNDLE, kind=kind + ) + with pytest.raises(ValueError, match="another bundle"): + validate_bundle_path(ROOT / parent / "event.json", "other", kind=kind) + validate_bundle_path( + ROOT / ".venv-replay-checks/event.json", "other", kind=kind + ) + + +def test_registration_is_deterministic_and_never_overwrites(tmp_path): + pin = {"bundle_key": "development", "development_bundle": True, "sha256": "abc"} + path, index = tmp_path / "pin.json", tmp_path / "index.json" + write_registration(pin, path, index, root=tmp_path) + before = path.read_bytes(), index.read_bytes() + write_registration(pin, path, index, root=tmp_path) + assert before == (path.read_bytes(), index.read_bytes()) + with pytest.raises(ValueError, match="replace"): + write_registration({**pin, "sha256": "changed"}, path, index, root=tmp_path) + with pytest.raises(ValueError, match="production index"): + write_registration( + pin, path, tmp_path / "data/uk/certified_bundles/index.json", root=tmp_path + ) + + +def test_extension_end_is_read_from_code(tmp_path): + path = tmp_path / "engine.py" + path.write_text( + "def extend_single_year_dataset(dataset, tax_benefit_system_parameters, end_year: int = 2032):\n pass\n" + ) + assert _extension_end(path) == 2032 + path.write_text("def changed_function(): pass\n") + with pytest.raises(ValueError, match="verify"): + _extension_end(path) + + +def test_requirements_export_uses_exact_tag_lock_and_no_cache(tmp_path, monkeypatch): + from pipeline import export_uk_bundle_requirements as module + import subprocess + + files = { + "pyproject.toml": b'[project]\nname="policyengine"\nversion="6.2.5"\n', + "uv.lock": b"version = 1\n", + "README.md": b"Release", + } + calls = [] + + def run(command, **kwargs): + calls.append(command) + if command[0] == "git" and command[3] == "show": + return subprocess.CompletedProcess( + command, 0, stdout=files[command[-1].split(":")[-1]] + ) + if command[0] == "git": + return subprocess.CompletedProcess(command, 0, stdout="a" * 40 + "\n") + assert ( + "--frozen" in command and "--offline" in command and "--no-cache" in command + ) + assert (Path(kwargs["cwd"]) / "uv.lock").read_bytes() == files["uv.lock"] + return subprocess.CompletedProcess( + command, + 0, + stdout="# uv header\npolicyengine-uk==2.102.3\npolicyengine-core==3.32.10\n", + ) + + monkeypatch.setattr(module.subprocess, "run", run) + freeze, receipt = export_requirements(tmp_path, "6.2.5", "development") + assert "policyengine==6.2.5\n" in freeze + assert receipt["uv_lock_sha256"] == hashlib.sha256(files["uv.lock"]).hexdigest() + assert receipt["freeze_sha256"] == hashlib.sha256(freeze.encode()).hexdigest() + + +def test_builder_cli_forwards_selected_engine_pin(tmp_path, monkeypatch): + from pipeline import build_uk_event_registry as builder + from types import SimpleNamespace + import importlib.metadata + import sys + + development_index(tmp_path, monkeypatch) + system = object() + monkeypatch.setitem( + sys.modules, + "policyengine_uk", + SimpleNamespace(CountryTaxBenefitSystem=lambda: system), + ) + expected = bundle_identity("development")["engine_versions"] + monkeypatch.setattr(importlib.metadata, "version", lambda name: expected[name]) + calls = [] + monkeypatch.setattr( + builder, "engine_dumps", lambda **kwargs: calls.append(kwargs) or ({}, {}) + ) + monkeypatch.setattr(builder, "engine_resolver", lambda value: None) + monkeypatch.setattr( + builder, + "build_registry", + lambda *args: {"measures": [], "accounting": {"rows_in": 0, "by_class": {}}}, + ) + output = tmp_path / "registry.json" + monkeypatch.setattr(builder, "registry_path", lambda *args: output) + monkeypatch.setattr( + sys, + "argv", + [ + "build_uk_event_registry", + "--event", + "audit_event", + "--bundle", + "development", + ], + ) + builder.main() + assert calls == [{"pin": expected["policyengine-uk"], "system": system}] + + +def test_cgt_blended_engine_uses_january_reversal_only_for_new_bundle( + tmp_path, monkeypatch +): + development_index(tmp_path, monkeypatch) + from pipeline import build_uk_event_registry as builder + + row = { + "source_row_id": "row-2024", + "title": "Capital Gains Tax: Increase the main rates and reliefs", + "fy": "2024-25", + "tax_head": "Capital gains tax", + "head_kind": "tax", + "metric": "revenue_change", + "value_gbp": 1, + "value_gbp_decimal": "1", + } + monkeypatch.setattr(builder, "source_rows", lambda event: [row]) + + def resolve(path, date): + return 0.18 + + resolve.parameter_metadata = lambda path: {"fiscal_year_blend": True} + paths = [f"gov.hmrc.cgt.{rate}_rate" for rate in ("basic", "higher", "additional")] + new = builder.build_registry( + "autumn_budget_2024", paths, ["capital_gains_tax"], resolve, "development" + )["measures"][0] + base = builder.build_registry( + "autumn_budget_2024", paths, ["capital_gains_tax"], resolve + )["measures"][0] + assert { + date for schedule in new["pe_baseline_modifier"].values() for date in schedule + } == {"2024-01-01"} + assert { + date for schedule in base["pe_baseline_modifier"].values() for date in schedule + } == {"2024-10-30"} + assert ( + new["construction_adjustments"][0]["id"] + == "cgt_fiscal_year_blend_annual_reversal" + ) + assert "parameter_processing_evidence" not in base + + +def test_later_bundle_diagnostics_keep_old_diagnosis_as_reference_only( + tmp_path, monkeypatch +): + from pipeline.diagnose_uk_event_models import current_bundle_observation + + original = { + "id": "cgt_main_rate_commencement", + "class": "pe_gap", + "interpretation": "Legacy onset was April 2025.", + "observation": {"current_rate": 0.18}, + "engine_version": "2.102.3", + } + historical = deepcopy(original) + assert current_bundle_observation(historical, bundle_identity()) == original + development_index(tmp_path, monkeypatch) + identity = bundle_identity("development", root=tmp_path) + current = current_bundle_observation(deepcopy(original), identity) + assert current["class"] == "bundle_observation" + assert "Legacy onset was April 2025." not in current["interpretation"] + assert current["legacy_reference"]["interpretation"] == original["interpretation"] + assert current["legacy_reference"]["bundle_key"] == DEFAULT_BUNDLE + assert current["observation"] == original["observation"] + validate_bundle_identity(current, "development", root=tmp_path) + + +@pytest.mark.parametrize( + "conflict", + [ + None, + "digest", + "size", + "policyengine-uk", + "policyengine-core", + "certification", + "certified_uri", + "default_uri", + "artifact_build", + "certification_build", + "repo_type", + "unverified", + "empty_basis", + ], +) +def test_registration_checks_packaged_artifact_and_package_pins( + tmp_path, monkeypatch, conflict +): + from pipeline import register_uk_certified_bundle as module + from types import SimpleNamespace + + packages = { + "policyengine": "6.2.5", + "policyengine-uk": "2.102.3", + "policyengine-core": "3.32.10", + } + digest = hashlib.sha256(b"certified bytes").hexdigest() + dataset = { + "path": "test.h5", + "sha256": digest, + "size_bytes": len(b"certified bytes"), + "revision": "1.56.16", + } + build = "policyengine-uk-data-1.56.16" + uri = "hf://owner/data/test.h5@1.56.16" + country = { + "certification": { + "certified_for_model_version": "2.102.3", + "compatibility_basis": "legacy_compatible_model_package", + "data_build_id": build, + }, + "model_package": {"version": "2.102.3"}, + "policyengine_version": "6.2.5", + "default_dataset": "test", + "default_dataset_uri": uri, + "datasets": {"test": dataset}, + "certified_data_artifact": { + "dataset": "test", + "sha256": digest, + "build_id": build, + "uri": uri, + }, + "data_package": { + "repo_type": "model", + "repo_id": "owner/data", + "release_manifest_revision": "b" * 40, + "release_manifest_path": "release_manifest.json", + }, + "build_id": build, + } + manifest = { + "packages": {name: {"version": version} for name, version in packages.items()}, + "data_releases": {"uk": country}, + } + if conflict == "digest": + dataset["sha256"] = "f" * 64 + country["certified_data_artifact"]["sha256"] = "f" * 64 + if conflict == "size": + dataset["size_bytes"] += 1 + if conflict == "certification": + country["certification"]["certified_for_model_version"] = "2.89.2" + if conflict == "certified_uri": + country["certified_data_artifact"]["uri"] = "hf://owner/other/test.h5@1.56.16" + if conflict == "default_uri": + country["default_dataset_uri"] = "hf://owner/data/test.h5@main" + if conflict == "artifact_build": + country["certified_data_artifact"]["build_id"] = "other" + if conflict == "certification_build": + country["certification"]["data_build_id"] = "other" + if conflict == "repo_type": + country["data_package"]["repo_type"] = "space" + if conflict == "unverified": + country["certification"]["compatibility_basis"] = "unverified_local" + if conflict == "empty_basis": + country["certification"]["compatibility_basis"] = "" + manifest_path = tmp_path / "policyengine/data/bundle/manifest.json" + manifest_path.parent.mkdir(parents=True) + manifest_path.write_text(json.dumps(manifest)) + repo = tmp_path / "cache/models--owner--data" + (repo / "refs").mkdir(parents=True) + (repo / "refs/1.56.16").write_text("a" * 40) + artifact = repo / "snapshots" / ("a" * 40) / "test.h5" + artifact.parent.mkdir(parents=True) + artifact.write_bytes(b"certified bytes") + monkeypatch.setattr( + module.importlib.metadata, + "distribution", + lambda _: SimpleNamespace(locate_file=lambda name: tmp_path / name), + ) + monkeypatch.setattr( + module.importlib.metadata, + "version", + lambda name: "wrong" if name == conflict else packages[name], + ) + monkeypatch.setattr(module, "_stored_year", lambda path: 2024) + monkeypatch.setattr(module, "_extension_end", lambda path: 2030) + monkeypatch.setattr( + module, + "sha256_file", + lambda path: digest if Path(path) == artifact else "b" * 64, + ) + arguments = { + "requirements_freeze": "freeze.txt", + "offline_audit": "audit.json", + "cache_root": tmp_path / "cache", + "development": True, + } + if conflict is not None: + with pytest.raises(ValueError): + module.inspect_installed_bundle("test", **arguments) + else: + pin = module.inspect_installed_bundle("test", **arguments) + assert pin["data_year"] == 2024 + assert pin["supported_calendar_years"] == [2024, 2030] + assert pin["compatible_core_packages"] == [ + {"name": "policyengine-core", "specifier": "==3.32.10"} + ] diff --git a/tests/test_uk_event_adoption.py b/tests/test_uk_event_adoption.py index b80bf07a..166959f4 100644 --- a/tests/test_uk_event_adoption.py +++ b/tests/test_uk_event_adoption.py @@ -1,5 +1,7 @@ """Offline validation and mutation guards for canonical event adoption.""" +import sys +import types from pathlib import Path import pytest @@ -14,8 +16,13 @@ def helper(): @pytest.fixture def adoption(helper, monkeypatch, tmp_path): + pin_path = tmp_path / "data/uk/certified_bundle.json" + pin_path.parent.mkdir(parents=True) + pin_path.write_bytes( + (helper.modal_runner.ROOT / "data/uk/certified_bundle.json").read_bytes() + ) monkeypatch.setattr(helper, "ROOT", tmp_path) - monkeypatch.setattr(helper, "current_input_digest", lambda event: "a" * 64) + monkeypatch.setattr(helper, "current_input_digest", lambda event, bundle: "a" * 64) output = "results/uk/events/autumn_budget_2024" download = ".venv-replay-checks/download" paths = [output + "/measure_2026.json", output + "/RUN_MANIFEST.json"] @@ -105,6 +112,95 @@ def record_replace(source, destination): ).read_bytes() +@pytest.fixture +def other_bundle_adoption(helper, adoption): + root, report = adoption + key = "development-isolation-fixture" + pin = helper.uk_bundle.load_bundle(root=root) + pin["bundle_key"] = key + pin["development_bundle"] = True + pin_relative = "data/uk/certified_bundles/development.json" + (root / pin_relative).parent.mkdir(parents=True) + (root / pin_relative).write_bytes(helper.modal_runner.canonical_bytes(pin)) + (root / "data/uk/certified_bundles/index.json").write_bytes( + helper.modal_runner.canonical_bytes( + { + "schema_version": 1, + "default_bundle": helper.uk_bundle.DEFAULT_BUNDLE, + "bundles": { + helper.uk_bundle.DEFAULT_BUNDLE: "data/uk/certified_bundle.json", + key: pin_relative, + }, + } + ) + ) + original_paths = [row["path"] for row in report["files"]] + old_output = report["remote_output"] + output = f"results/uk/events/bundles/{key}/{report['event']}" + for row in report["files"]: + old_relative = row["path"] + new_relative = old_relative.replace(old_output, output) + for prefix in (root, root / report["download_root"]): + path = prefix / new_relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes((prefix / old_relative).read_bytes()) + row["path"] = new_relative + report["remote_output"] = output + report.update(helper.uk_bundle.bundle_identity(key, root=root)) + return root, key, report, original_paths + + +def test_adoption_isolated_bundle_preserves_default_bytes( + helper, other_bundle_adoption +): + root, key, report, default_paths = other_bundle_adoption + before = {relative: (root / relative).read_bytes() for relative in default_paths} + helper.adopt( + report, + bundle=key, + backup_root=".venv-replay-checks/backup", + local_run_stopped=True, + allow_changed_existing=True, + ) + assert all( + (root / relative).read_bytes() == payload + for relative, payload in before.items() + ) + assert (root / report["files"][0]["path"]).read_bytes() == ( + root / report["download_root"] / report["files"][0]["path"] + ).read_bytes() + + +def test_adoption_rejects_report_from_another_bundle_before_writes( + helper, other_bundle_adoption +): + root, _, report, _ = other_bundle_adoption + with pytest.raises(ValueError, match="bundle identity mismatch"): + helper.adopt( + report, + backup_root=".venv-replay-checks/backup", + local_run_stopped=True, + allow_changed_existing=True, + ) + assert not (root / ".venv-replay-checks/backup").exists() + + +def test_adoption_rejects_default_destination_for_other_bundle( + helper, other_bundle_adoption +): + root, key, report, _ = other_bundle_adoption + report["remote_output"] = f"results/uk/events/{report['event']}" + with pytest.raises(ValueError, match="complete canonical"): + helper.adopt( + report, + bundle=key, + backup_root=".venv-replay-checks/backup", + local_run_stopped=True, + allow_changed_existing=True, + ) + assert not (root / ".venv-replay-checks/backup").exists() + + def test_download_drift_blocks_all_writes_before_backup(helper, adoption): root, report = adoption originals = { @@ -129,7 +225,7 @@ def test_download_drift_blocks_all_writes_before_backup(helper, adoption): def test_input_drift_after_validation_blocks_adoption(helper, adoption, monkeypatch): root, report = adoption - monkeypatch.setattr(helper, "current_input_digest", lambda event: "b" * 64) + monkeypatch.setattr(helper, "current_input_digest", lambda event, bundle: "b" * 64) with pytest.raises(ValueError, match="inputs changed after validation"): helper.adopt( report, @@ -209,6 +305,15 @@ def downloaded_event(helper, monkeypatch, tmp_path): path = tmp_path / relative path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(b"{}") + (tmp_path / "data/uk/certified_bundles/index.json").write_bytes( + runner.canonical_bytes( + { + "schema_version": 1, + "default_bundle": runner.DEFAULT_BUNDLE, + "bundles": {runner.DEFAULT_BUNDLE: "data/uk/certified_bundle.json"}, + } + ) + ) (tmp_path / "data/uk/certified_bundle.json").write_bytes( runner.canonical_bytes(cert) ) @@ -230,7 +335,9 @@ def downloaded_event(helper, monkeypatch, tmp_path): monkeypatch.setattr( runner, "input_manifest", - lambda event, root: actual_inputs(event, root=root, hf_cache=hf_cache), + lambda event, root, **kwargs: actual_inputs( + event, root=root, hf_cache=hf_cache, **kwargs + ), ) source = { "source_row_id": "tax", @@ -257,6 +364,7 @@ def downloaded_event(helper, monkeypatch, tmp_path): } registry = { "event_slug": event, + "bundle": cert, "calendar_years": [2026], "measures": [measure], "accounting": helper.compute.registry_builder.accounting([source], [measure]), @@ -382,6 +490,35 @@ def test_read_only_validation_runs_real_inventory_grid_and_orientation_guards( assert not (fixture["root"] / "results").exists() +def test_validation_tolerates_an_engine_imported_before_it_runs( + helper, downloaded_event, monkeypatch +): + """A full suite in an engine-equipped venv imports policyengine first.""" + fixture = downloaded_event + monkeypatch.setitem( + sys.modules, "policyengine_preloaded_elsewhere", types.ModuleType("x") + ) + report = helper.validate(fixture["event"], fixture["download"]) + assert report["full_event_complete"] + + +def test_validation_rejects_an_engine_imported_during_validation( + helper, downloaded_event, monkeypatch +): + fixture = downloaded_event + read_json = helper.read_json + + def read_json_importing_engine(path): + monkeypatch.setitem( + sys.modules, "policyengine_imported_by_validation", types.ModuleType("x") + ) + return read_json(path) + + monkeypatch.setattr(helper, "read_json", read_json_importing_engine) + with pytest.raises(AssertionError, match="unexpectedly imported an engine"): + helper.validate(fixture["event"], fixture["download"]) + + def test_validation_rejects_changed_current_allowlisted_input(helper, downloaded_event): fixture = downloaded_event (fixture["root"] / "pipeline/compute_uk_event.py").write_bytes(b"different inputs") @@ -428,3 +565,18 @@ def test_validation_rejects_unreceipted_output_files(helper, downloaded_event): (fixture["directory"] / "uncommitted.json").write_bytes(b"{}") with pytest.raises(ValueError, match="missing or unreceipted files"): helper.validate(fixture["event"], fixture["download"]) + + +@pytest.mark.parametrize("target", ["request", "manifest", "artifact"]) +def test_validation_rejects_foreign_bundle_receipts(helper, downloaded_event, target): + fixture = downloaded_event + if target == "request": + fixture["receipt"]["request"]["bundle_key"] = "foreign-bundle" + fixture["receipt"]["request_sha256"] = helper.modal_runner.digest_bytes( + helper.modal_runner.canonical_bytes(fixture["receipt"]["request"]) + ) + else: + fixture[target]["bundle_key"] = "foreign-bundle" + fixture["refresh"]() + with pytest.raises(ValueError, match="identity"): + helper.validate(fixture["event"], fixture["download"]) diff --git a/tests/test_uk_event_bundle_staging.py b/tests/test_uk_event_bundle_staging.py new file mode 100644 index 00000000..32e098ca --- /dev/null +++ b/tests/test_uk_event_bundle_staging.py @@ -0,0 +1,547 @@ +"""Bundle isolation at the engine-free staging and comparison boundaries.""" + +import hashlib +import json + +import pytest + +from pipeline import compare_uk_event as comparison +from pipeline import stage_uk_event as staging +from pipeline import uk_bundle as bundles + +DEV = "test-development-bundle" + + +@pytest.fixture +def pinned_stage(tmp_path): + default_pin = tmp_path / "data/uk/certified_bundle.json" + default_pin.parent.mkdir(parents=True) + default_pin.write_bytes(bundles.bundle_pin_path().read_bytes()) + pin = bundles.load_bundle() + pin.update( + bundle_key=DEV, + data_year=2024, + supported_calendar_years=[2024, 2030], + sha256="d" * 64, + data_build_id="development-build", + revision="development-build", + ) + pin_path = tmp_path / "data/uk/certified_bundles/dev.json" + pin_path.parent.mkdir(parents=True) + pin_path.write_text(json.dumps(pin)) + (pin_path.parent / "index.json").write_text( + json.dumps( + { + "schema_version": 1, + "default_bundle": bundles.DEFAULT_BUNDLE, + "bundles": { + bundles.DEFAULT_BUNDLE: "data/uk/certified_bundle.json", + DEV: "data/uk/certified_bundles/dev.json", + }, + } + ) + ) + for label in ("requirements_freeze", "offline_audit"): + destination = tmp_path / pin[label] + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes((bundles.ROOT / pin[label]).read_bytes()) + identity = bundles.bundle_identity(DEV, root=tmp_path) + registry = { + "event_slug": "event", + "event_name": "Event", + "calendar_years": [2024], + "bundle": pin, + **identity, + "accounting": { + "rows_in": 2, + "by_class": { + "out_of_household_scope": { + "measures": 0, + "rows": 2, + "net_gbp_decimal": "2", + "absolute_gbp_decimal": "2", + } + }, + }, + "measures": [ + { + "measure_key": "event__gap", + "title": "Gap", + "computability": "not_expressible", + "gap_kind": "construction_pending", + "pe_gap": "Unconstructed test case", + "source_rows": [ + { + "source_row_id": str(year), + "fy": f"{year}-{str(year + 1)[-2:]}", + "tax_head": "Other", + "metric": "revenue_change", + "value_gbp": 1.0, + "value_gbp_decimal": "1", + "classification": "out_of_household_scope", + } + for year in (2023, 2024) + ], + } + ], + } + registry_path = bundles.registry_path("event", DEV, root=tmp_path) + registry_path.parent.mkdir(parents=True) + registry_path.write_text(json.dumps(registry)) + digest = hashlib.sha256(registry_path.read_bytes()).hexdigest() + output = bundles.event_output_dir("event", DEV, root=tmp_path) + output.mkdir(parents=True) + manifest = { + **identity, + "event": "event", + "registry_sha256": digest, + "artifacts": {}, + } + return registry, manifest, registry_path, output, tmp_path + + +def save_stage(values): + registry, manifest, _registry_path, output, root = values + rows, tally = staging.stage_event( + registry, + manifest, + artifact_dir=output, + registry_sha256=manifest["registry_sha256"], + bundle=DEV, + bundle_root=root, + ) + (output / "RUN_MANIFEST.json").write_text(json.dumps(manifest)) + staged = output / "STAGED.jsonl" + staged.write_text("".join(json.dumps(row) + "\n" for row in rows)) + receipt = { + **tally, + "event": "event", + "staged_path": str(staged.relative_to(root)), + "staged_sha256": hashlib.sha256(staged.read_bytes()).hexdigest(), + } + (output / "STAGING_MANIFEST.json").write_text(json.dumps(receipt)) + return rows, tally, staged + + +def test_pre_window_source_rows_accounted_once_distinctly_even_outside_scope( + pinned_stage, +): + rows, tally, _ = save_stage(pinned_stage) + outside = next(row for row in rows if row["year"] == 2023) + assert outside["status"] == "outside_bundle_window" + assert outside["computability"] == "out_of_household_scope" + assert outside["pe_value"] is None + assert tally["source_rows"] == tally["staged_rows"] == 2 + assert tally["source_value_gbp_decimal"] == tally["staged_value_gbp_decimal"] == "2" + assert tally["bundle_key"] == DEV + + +@pytest.mark.parametrize( + "field", + [ + "bundle_key", + "bundle_pin_sha256", + "certified_dataset_sha256", + "engine_versions", + "supported_calendar_years", + ], +) +def test_stage_rejects_foreign_bundle_compute_manifest(pinned_stage, field): + registry, manifest, _, output, root = pinned_stage + manifest[field] = "foreign" + with pytest.raises(ValueError, match="bundle identity mismatch"): + staging.stage_event( + registry, + manifest, + artifact_dir=output, + registry_sha256=manifest["registry_sha256"], + bundle=DEV, + bundle_root=root, + ) + + +def test_stage_refuses_foreign_registry_or_window(pinned_stage): + registry, manifest, _, output, root = pinned_stage + registry["bundle"] = bundles.bundle_document() + with pytest.raises(ValueError, match="registry bundle pin"): + staging.stage_event( + registry, + manifest, + artifact_dir=output, + registry_sha256=manifest["registry_sha256"], + bundle=DEV, + bundle_root=root, + ) + registry["bundle"] = bundles.bundle_document(DEV, root=root) + registry["calendar_years"] = [2023, 2024] + with pytest.raises(ValueError, match="year outside"): + staging.stage_event( + registry, + manifest, + artifact_dir=output, + registry_sha256=manifest["registry_sha256"], + bundle=DEV, + bundle_root=root, + ) + + +def test_nondefault_comparison_and_summary_bind_selected_pin(pinned_stage): + _registry, _manifest, registry_path, output, root = pinned_stage + _, _, staged = save_stage(pinned_stage) + rows = comparison.write_comparison( + "event", + registry_path=registry_path, + staged_path=staged, + output_dir=output, + artifact_root=root, + bundle=DEV, + ) + assert rows[0]["status"] == "outside_bundle_window" + assert ( + comparison.load_verified_comparison( + output / "COMPARISON.json", artifact_root=root, bundle=DEV + ) + == rows + ) + comparison.write_summary(output.parent, artifact_root=root, bundle=DEV) + prose = (output / "COMPARISON.md").read_text() + assert "certified 2024 population" in prose + assert "fiscal-year conversion" in prose + assert "annualized from the engine's 30 April snapshot" not in prose + assert DEV in (output.parent / "SUMMARY.md").read_text() + with pytest.raises(comparison.EventComparisonError): + comparison.load_verified_comparison( + output / "COMPARISON.json", + artifact_root=root, + bundle=bundles.DEFAULT_BUNDLE, + ) + + +@pytest.mark.parametrize("receipt", ["stage", "comparison", "registry"]) +def test_nondefault_comparison_rejects_foreign_receipts_before_output( + pinned_stage, receipt +): + _registry, _manifest, registry_path, output, root = pinned_stage + _, _, staged = save_stage(pinned_stage) + comparison.write_comparison( + "event", + registry_path=registry_path, + staged_path=staged, + output_dir=output, + artifact_root=root, + bundle=DEV, + ) + filename = { + "stage": output / "STAGING_MANIFEST.json", + "comparison": output / "COMPARISON_PROVENANCE.json", + "registry": registry_path, + }[receipt] + payload = json.loads(filename.read_text()) + payload["bundle_key"] = bundles.DEFAULT_BUNDLE + filename.write_text(json.dumps(payload)) + with pytest.raises(comparison.EventComparisonError): + comparison.write_summary(output.parent, artifact_root=root, bundle=DEV) + assert not (output.parent / "SUMMARY.md").exists() + + +def test_comparison_rejects_pre_window_row_without_distinct_status(pinned_stage): + registry, _, _, _, root = pinned_stage + rows, _, _ = save_stage(pinned_stage) + rows[0]["status"] = "not_computed" + with pytest.raises(comparison.EventComparisonError, match="outside_bundle_window"): + comparison.build_comparison_rows(registry, rows, artifact_root=root, bundle=DEV) + + +@pytest.mark.parametrize("kind", ["registry", "results"]) +def test_path_selection_rejects_other_bundles_including_default(pinned_stage, kind): + _, _, _, _, root = pinned_stage + if kind == "registry": + foreign = bundles.registry_path("event", bundles.DEFAULT_BUNDLE, root=root) + selected = bundles.registry_path("event", DEV, root=root) + else: + foreign = bundles.event_output_dir("event", bundles.DEFAULT_BUNDLE, root=root) + selected = bundles.event_output_dir("event", DEV, root=root) + with pytest.raises(ValueError, match="another bundle"): + bundles.validate_bundle_path(foreign, DEV, kind=kind, root=root) + with pytest.raises(ValueError, match="another bundle"): + bundles.validate_bundle_path( + selected, bundles.DEFAULT_BUNDLE, kind=kind, root=root + ) + + +def test_engine_generator_binds_every_input_and_writes_deterministically(pinned_stage): + from pipeline import compare_uk_engines as engines + + registry, _manifest, registry_path, output, root = pinned_stage + _, _, staged = save_stage(pinned_stage) + comparison.write_comparison( + "event", + registry_path=registry_path, + staged_path=staged, + output_dir=output, + artifact_root=root, + bundle=DEV, + ) + other = "test-other-development-bundle" + pin = bundles.load_bundle(DEV, root=root) + pin.update(bundle_key=other, data_year=2025, supported_calendar_years=[2025, 2030]) + other_pin_path = root / "data/uk/certified_bundles/other.json" + other_pin_path.write_text(json.dumps(pin)) + index_path = bundles.bundle_index_path(root=root) + index = json.loads(index_path.read_text()) + index["bundles"][other] = str(other_pin_path.relative_to(root)) + index_path.write_text(json.dumps(index)) + other_identity = bundles.bundle_identity(other, root=root) + other_registry = { + **registry, + **other_identity, + "bundle": pin, + "calendar_years": [2025], + } + other_registry_path = bundles.registry_path("event", other, root=root) + other_registry_path.parent.mkdir(parents=True) + other_registry_path.write_text(json.dumps(other_registry)) + other_output = bundles.event_output_dir("event", other, root=root) + other_output.mkdir(parents=True) + other_manifest = { + **other_identity, + "event": "event", + "artifacts": {}, + "registry_sha256": hashlib.sha256(other_registry_path.read_bytes()).hexdigest(), + } + (other_output / "RUN_MANIFEST.json").write_text(json.dumps(other_manifest)) + other_rows, tally = staging.stage_event( + other_registry, + other_manifest, + artifact_dir=other_output, + registry_sha256=other_manifest["registry_sha256"], + bundle=other, + bundle_root=root, + ) + other_staged = other_output / "STAGED.jsonl" + other_staged.write_text("".join(json.dumps(row) + "\n" for row in other_rows)) + (other_output / "STAGING_MANIFEST.json").write_text( + json.dumps( + { + **tally, + "event": "event", + "staged_sha256": hashlib.sha256(other_staged.read_bytes()).hexdigest(), + } + ) + ) + comparison.write_comparison( + "event", + registry_path=other_registry_path, + staged_path=other_staged, + output_dir=other_output, + artifact_root=root, + bundle=other, + ) + axes = root / "data/uk/obr_divergence_axes.json" + axes.write_bytes(comparison.AXES_PATH.read_bytes()) + attribution_path = root / "data/uk/events/engine_attribution/example.json" + attribution_path.parent.mkdir(parents=True) + attribution_path.write_text( + json.dumps( + { + "schema_version": 1, + "base": DEV, + "new": other, + "entries": [ + { + "event": "event", + "measure_key": "event__gap", + "drivers": [ + { + "driver": "window_change", + "evidence": [ + { + "kind": "diagnostic", + "reference": "test-only window observation", + } + ], + "sized": False, + } + ], + } + ], + } + ) + ) + rows = engines.write_engine_comparison( + DEV, other, attribution_path=attribution_path, artifact_root=root + ) + assert len(rows) == 2 + directory = root / "results/uk/events" + before = { + path.name: path.read_bytes() for path in directory.glob("ENGINE_COMPARISON*") + } + assert set(before) == { + "ENGINE_COMPARISON.json", + "ENGINE_COMPARISON.csv", + "ENGINE_COMPARISON.md", + "ENGINE_COMPARISON_PROVENANCE.json", + } + provenance = json.loads(before["ENGINE_COMPARISON_PROVENANCE.json"]) + expected_inputs = [ + registry_path, + other_registry_path, + attribution_path, + index_path, + axes, + bundles.bundle_pin_path(DEV, root=root), + other_pin_path, + output / "RUN_MANIFEST.json", + other_output / "RUN_MANIFEST.json", + root / pin["requirements_freeze"], + root / pin["offline_audit"], + ] + assert all( + str(path.relative_to(root)) in provenance["inputs_sha256"] + for path in expected_inputs + ) + for reference, digest in provenance["inputs_sha256"].items(): + assert hashlib.sha256((root / reference).read_bytes()).hexdigest() == digest + engines.write_engine_comparison( + DEV, other, attribution_path=attribution_path, artifact_root=root + ) + assert before == { + path.name: path.read_bytes() for path in directory.glob("ENGINE_COMPARISON*") + } + missing_comparisons = [ + bundles.registry_path("uncompared_event", key, root=root) + for key in (DEV, other) + ] + for uncomputed_registry in missing_comparisons: + uncomputed_registry.write_bytes(registry_path.read_bytes()) + with pytest.raises(engines.EngineComparisonError, match="every event"): + engines.write_engine_comparison( + DEV, other, attribution_path=attribution_path, artifact_root=root + ) + assert before == { + path.name: path.read_bytes() for path in directory.glob("ENGINE_COMPARISON*") + } + for uncomputed_registry in missing_comparisons: + uncomputed_registry.unlink() + staged.write_bytes(staged.read_bytes() + b"\n") + with pytest.raises(comparison.EventComparisonError, match="SHA-256"): + engines.write_engine_comparison( + DEV, other, attribution_path=attribution_path, artifact_root=root + ) + assert before == { + path.name: path.read_bytes() for path in directory.glob("ENGINE_COMPARISON*") + } + + +def test_sized_paired_artifact_verifies_real_endpoints(pinned_stage): + from pipeline import compare_uk_engines as engines + + _registry, manifest, _registry_path, _output, root = pinned_stage + endpoints = {} + for side, value in (("base", 100), ("new", 120)): + numerical = { + **bundles.bundle_identity(DEV, root=root), + "measure_key": "event__tax", + "year": 2024, + "event": "event", + "registry_sha256": manifest["registry_sha256"], + "construction": "forward_delta_on_certified_world", + "head_effects": {"income_tax": value}, + "head_channels": {"income_tax": "tax"}, + "totals": { + "baseline": {"heads": {"income_tax": 0}}, + "reform": {"heads": {"income_tax": value}}, + }, + "literal_reform_minus_baseline": {"income_tax": value}, + "measure_total_gbp": value, + } + path = root / f"{side}.json" + path.write_bytes(engines.canonical_bytes(numerical)) + endpoints[side] = { + "path": path.name, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + paired = { + "artifact_type": "paired_run", + "status": "computed", + "effect_gbp": 20, + "base_bundle": DEV, + "new_bundle": DEV, + "base_artifact": endpoints["base"], + "new_artifact": endpoints["new"], + "head_variables": ["income_tax"], + } + path = root / "paired.json" + path.write_bytes(engines.canonical_bytes(paired)) + attribution = { + "schema_version": 1, + "entries": [ + { + "event": "event", + "measure_key": "event__tax", + "fy": "2024-25", + "drivers": [ + { + "driver": "other_engine_change", + "sized": True, + "value_gbp": 20, + "evidence": [ + {"kind": "paired_run", "reference": "paired.json"} + ], + "artifact": { + "path": path.name, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "value_path": ["effect_gbp"], + }, + } + ], + } + ], + } + hashes = {} + engines.validate_attribution(attribution, artifact_root=root, input_hashes=hashes) + assert set(hashes) == {"base.json", "new.json", "paired.json"} + # Copying a paired receipt under another name does not create a second + # independently sized contribution from the same executed endpoints. + second_pair = {**paired, "note": "another named mechanism, same numerical term"} + second_path = root / "paired-copy.json" + second_path.write_bytes(engines.canonical_bytes(second_pair)) + first_driver = attribution["entries"][0]["drivers"][0] + second_driver = json.loads(json.dumps(first_driver)) + second_driver.update(driver="data_release") + second_driver["artifact"].update( + path=second_path.name, + sha256=hashlib.sha256(second_path.read_bytes()).hexdigest(), + ) + attribution["entries"][0]["drivers"].append(second_driver) + with pytest.raises(engines.EngineComparisonError, match="same computed term"): + engines.validate_attribution(attribution, artifact_root=root) + attribution["entries"][0]["drivers"].pop() + (root / "new.json").write_bytes((root / "new.json").read_bytes() + b"\n") + with pytest.raises(engines.EngineComparisonError, match="endpoint SHA-256"): + engines.validate_attribution(attribution, artifact_root=root) + + +def test_summary_rejects_another_bundles_model_observations(pinned_stage): + _registry, _manifest, registry_path, output, root = pinned_stage + _, _, staged = save_stage(pinned_stage) + comparison.write_comparison( + "event", + registry_path=registry_path, + staged_path=staged, + output_dir=output, + artifact_root=root, + bundle=DEV, + ) + foreign_diagnostic = { + "bundle_key": bundles.DEFAULT_BUNDLE, + "engine_version": "2.89.2", + } + (output.parent / "MODEL_DIAGNOSTICS.json").write_text( + json.dumps([foreign_diagnostic]) + ) + with pytest.raises( + comparison.EventComparisonError, match="bundle identity mismatch" + ): + comparison.write_summary(output.parent, artifact_root=root, bundle=DEV) + assert not (output.parent / "SUMMARY.md").exists() diff --git a/tests/test_uk_event_construction_audit.py b/tests/test_uk_event_construction_audit.py index 762dfb41..520d7e6a 100644 --- a/tests/test_uk_event_construction_audit.py +++ b/tests/test_uk_event_construction_audit.py @@ -65,11 +65,10 @@ def test_saved_receipt_has_certified_identity_and_keeps_contributions_unsized(): for field in ("sha256", "size_bytes", "artifact", "revision"): assert receipt["certified_dataset"][field] == bundle[field] assert receipt["certified_dataset"]["digest_checked_before_hdf5_open"] is True - assert ( - receipt["inspection_script"]["sha256"] - == hashlib.sha256( - (audit.ROOT / receipt["inspection_script"]["path"]).read_bytes() - ).hexdigest() + # This is the historical execution receipt. Its source digest must remain + # the original version, even as the current auditor gains bundle selection. + assert receipt["inspection_script"]["sha256"] == ( + "ec1c6354653da622530b617f5a651c0ddde9320dda15505e4d79136981d9ea8c" ) assert receipt["observations"]["time_period"] == "2023" columns = receipt["observations"]["input_columns"] From 2ba1acfc0e316bd31a8310105b93b41ed1c60ef3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 07:41:03 -0400 Subject: [PATCH 02/11] Compare UK replay results across certified engines compare_uk_engines.py writes ENGINE_COMPARISON.{md,json,csv} with full input-hash provenance. Each event x measure x head x source-FY row carries OBR, PE(base), PE(new), the change, both PE/OBR bins, and computed / window / construction status. It also shows the head's certified-aggregate ratio beside the measure's own ratio. Attribution comes only from an authored file with a closed driver vocabulary, and unsized drivers never enter an explained share. Refs #156. Co-Authored-By: Claude Opus 5.5 --- pipeline/compare_uk_engines.py | 919 ++++++++++++++++++ .../uk_engine_comparison/attribution.json | 30 + tests/test_uk_engine_comparison.py | 422 ++++++++ 3 files changed, 1371 insertions(+) create mode 100644 pipeline/compare_uk_engines.py create mode 100644 tests/fixtures/uk_engine_comparison/attribution.json create mode 100644 tests/test_uk_engine_comparison.py diff --git a/pipeline/compare_uk_engines.py b/pipeline/compare_uk_engines.py new file mode 100644 index 00000000..7fc82899 --- /dev/null +++ b/pipeline/compare_uk_engines.py @@ -0,0 +1,919 @@ +"""Compare two certified replay bundles, retaining every OBR source row. + +This is an engine-free descriptive comparison. Attribution is read exclusively +from the checked-in attribution file; engine versions and changed values never +supply a driver automatically. Numerical sizing requires a hash-bound computed +artifact. No event simulation is performed here. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import io +import json +import math +from collections import Counter +from decimal import Decimal +from pathlib import Path + +from pipeline import compare_uk_event as event_comparison +from pipeline import uk_bundle as bundles +from pipeline.compare_uk_obr_costings import atomic_write_bytes, ratio_and_bin + +ROOT = event_comparison.ROOT +DRIVERS = frozenset( + { + "hicbc_opt_out", + "annual_allowance_pe_uk_2237", + "sdlt_pe_uk_2238", + "uc_uplifts_pe_uk_2239", + "nics_freeze", + "data_release", + "construction_change", + "window_change", + "other_engine_change", + "unattributed", + } +) +# Executed schedules and mappings, not prose or engine metadata. Package +# components are recursively included, so changing a component changes its +# parent's construction digest too. +CONSTRUCTION_FIELDS = ( + "construction", + "pe_baseline_modifier", + "pe_reform_delta", + "package_of", + "heads", + "head_variables", + "commences_fy", + "annual_activation_fy", + "effect_ends_fy", +) +CSV_FIELDS = ( + "event", + "source_row_id", + "measure_key", + "title", + "fy", + "tax_head", + "obr_million_gbp", + "pe_base_million_gbp", + "pe_new_million_gbp", + "change_million_gbp", + "base_ratio_bin", + "new_ratio_bin", + "status", + "base_status", + "new_status", + "base_construction_sha256", + "new_construction_sha256", + "base_classification", + "new_classification", + "head_certified_aggregate_new_to_base_ratio", + "measure_new_to_base_ratio", + "head_effect_new_to_base_ratio", + "attribution", + "explained_share", + "explained_share_withheld", +) + + +class EngineComparisonError(ValueError): + """Two replay receipts or their attribution cannot support a comparison.""" + + +def canonical_bytes(value) -> bytes: + return ( + json.dumps(value, sort_keys=True, indent=1, allow_nan=False) + "\n" + ).encode() + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def construction_digest( + measure: dict, index: dict[str, dict] | None = None, trail: tuple[str, ...] = () +) -> str: + """Hash the authored construction and source-head mappings canonically.""" + key = measure["measure_key"] + if key in trail: + raise EngineComparisonError("cyclic package construction") + payload = { + field: measure[field] for field in CONSTRUCTION_FIELDS if field in measure + } + payload["source_head_mappings"] = sorted( + [ + {field: source.get(field) for field in ("source_row_id", "pe_variables")} + for source in measure["source_rows"] + ], + key=lambda row: row["source_row_id"], + ) + if index is not None: + components = measure.get("package_of", []) + construction = str(measure.get("construction", "")) + if construction.startswith("same_lever_as_"): + components = [construction.removeprefix("same_lever_as_")] + payload["component_constructions"] = { + component: construction_digest(index[component], index, (*trail, key)) + for component in sorted(components) + } + return hashlib.sha256(canonical_bytes(payload)).hexdigest() + + +def _inventory(registry: dict) -> dict[str, tuple[dict, dict]]: + out = {} + measure_keys = set() + for measure in registry["measures"]: + if measure["measure_key"] in measure_keys: + raise EngineComparisonError("duplicate registry measure key") + measure_keys.add(measure["measure_key"]) + for source in measure["source_rows"]: + identity = source["source_row_id"] + if identity in out: + raise EngineComparisonError("duplicate registry source row") + out[identity] = (measure, source) + return out + + +def _comparison_index(rows: list[dict]) -> dict[str, dict]: + out = {row["source_row_id"]: row for row in rows} + if len(out) != len(rows): + raise EngineComparisonError("duplicate comparison source row") + return out + + +def _ratio(new, base): + return None if new is None or base is None or base == 0 else new / base + + +def _aggregate(row: dict, artifacts: dict[str, dict], field: str): + if row.get("pe_value_gbp") is None: + return None + artifact = artifacts[row["artifact"]] + if field == "measure_total_gbp": + return artifact[field] + values = artifact.get(field) + if not isinstance(values, dict): + raise EngineComparisonError(f"computed artifact missing {field}") + try: + return sum(values[variable] for variable in row["head_variables"]) + except KeyError as exc: + raise EngineComparisonError( + f"computed artifact missing certified head: {exc}" + ) from exc + + +def _entry_index(attribution: dict) -> dict[tuple[str, str, str | None], list[dict]]: + if attribution.get("schema_version") != 1: + raise EngineComparisonError("unsupported attribution schema_version") + entries = attribution.get("entries") + if not isinstance(entries, list): + raise EngineComparisonError( + "attribution requires entries, including explicit unattributed entries" + ) + out = {} + for entry in entries: + key = (entry.get("event"), entry.get("measure_key"), entry.get("fy")) + if not all(isinstance(v, str) and v for v in key[:2]) or key in out: + raise EngineComparisonError( + "attribution has invalid or duplicate event/measure/FY entry" + ) + if key[2] is not None and not isinstance(key[2], str): + raise EngineComparisonError("attribution FY must be a string") + drivers = entry.get("drivers") + if not isinstance(drivers, list) or not drivers: + raise EngineComparisonError("attribution entry requires named drivers") + names = [driver.get("driver") for driver in drivers] + if len(names) != len(set(names)) or set(names) - DRIVERS: + raise EngineComparisonError( + "attribution uses duplicate or unknown driver vocabulary" + ) + if "unattributed" in names and len(names) != 1: + raise EngineComparisonError( + "explicit unattributed cannot be combined with attributed drivers" + ) + out[key] = drivers + return out + + +def validate_attribution( + attribution: dict, + *, + artifact_root: Path = ROOT, + input_hashes: dict[str, str] | None = None, +) -> dict: + """Validate all entries, even entries not used by a changed row.""" + out = _entry_index(attribution) + for (event, measure_key, fy), drivers in out.items(): + sized_terms = set() + for driver in drivers: + evidence = driver.get("evidence") + if not isinstance(evidence, list) or not evidence: + raise EngineComparisonError("every driver requires evidence") + for item in evidence: + if ( + not isinstance(item, dict) + or item.get("kind") + not in ( + "pr", + "issue", + "changelog", + "diagnostic", + "paired_run", + "unattributed", + ) + or not isinstance(item.get("reference"), str) + or not item["reference"].strip() + ): + raise EngineComparisonError( + "driver evidence must name a PR, issue, changelog, diagnostic or paired-run artifact" + ) + if ( + item["kind"] == "unattributed" + and driver["driver"] != "unattributed" + ): + raise EngineComparisonError( + "unattributed evidence cannot support a named mechanism" + ) + if type(driver.get("sized")) is not bool: + raise EngineComparisonError("driver requires boolean sized") + if not driver["sized"]: + if "value_gbp" in driver or "artifact" in driver: + raise EngineComparisonError( + "unsized driver cannot carry a numerical size" + ) + continue + if driver["driver"] == "unattributed": + raise EngineComparisonError("unattributed driver cannot be sized") + if fy is None: + raise EngineComparisonError( + "a sized attribution requires an explicit source FY" + ) + proof = driver.get("artifact") + if not isinstance(proof, dict): + raise EngineComparisonError("sized driver requires a computed artifact") + reference = proof.get("path") + if not isinstance(reference, str) or Path(reference).is_absolute(): + raise EngineComparisonError("sized artifact path must be relative") + path = (artifact_root / reference).resolve() + if not path.is_relative_to(artifact_root.resolve()): + raise EngineComparisonError("sized artifact escapes root") + if not path.is_file(): + raise EngineComparisonError("sized computed artifact does not exist") + if sha256(path) != proof.get("sha256"): + raise EngineComparisonError("sized artifact SHA-256 differs") + computed = json.loads(path.read_bytes()) + if "head_effects" in computed: + event_comparison.validate_artifact(computed, reference) + if ( + computed.get("event"), + computed.get("measure_key"), + computed.get("year"), + ) != (event, measure_key, int(fy[:4])): + raise EngineComparisonError( + "sized artifact event/measure/FY differs from attribution" + ) + allowed = proof.get("value_path", []) + if not ( + allowed == ["measure_total_gbp"] + or len(allowed) == 2 + and allowed[0] == "head_effects" + ): + raise EngineComparisonError( + "sized artifact value must be a computed GBP effect" + ) + term = ("computed", proof["sha256"], tuple(allowed)) + elif ( + computed.get("artifact_type") == "paired_run" + and computed.get("status") == "computed" + ): + allowed = proof.get("value_path", []) + if allowed != ["effect_gbp"]: + raise EngineComparisonError( + "paired-run size must reference computed effect_gbp" + ) + endpoints = {} + paired_heads = set() + for side in ("base", "new"): + receipt = computed.get(side + "_artifact") + if not isinstance(receipt, dict): + raise EngineComparisonError( + "paired-run sizing artifact lacks executed-world commitments" + ) + endpoint_reference = receipt.get("path") + if ( + not isinstance(endpoint_reference, str) + or Path(endpoint_reference).is_absolute() + ): + raise EngineComparisonError( + "paired-run endpoint path must be relative" + ) + endpoint_path = (artifact_root / endpoint_reference).resolve() + if not endpoint_path.is_relative_to(artifact_root.resolve()): + raise EngineComparisonError("paired-run endpoint escapes root") + if not endpoint_path.is_file(): + raise EngineComparisonError( + "paired-run computed endpoint does not exist" + ) + if sha256(endpoint_path) != receipt.get("sha256"): + raise EngineComparisonError( + "paired-run endpoint SHA-256 differs" + ) + endpoint = json.loads(endpoint_path.read_bytes()) + event_comparison.validate_artifact(endpoint, endpoint_reference) + if ( + endpoint.get("event"), + endpoint.get("measure_key"), + endpoint.get("year"), + ) != (event, measure_key, int(fy[:4])): + raise EngineComparisonError( + "paired-run endpoint event/measure/FY differs from attribution" + ) + endpoint_bundle = computed.get(side + "_bundle") + if not isinstance(endpoint_bundle, str): + raise EngineComparisonError( + "paired-run endpoint requires bundle identity" + ) + bundles.validate_runtime_bundle( + endpoint, endpoint_bundle, root=artifact_root + ) + variables = computed.get("head_variables") + if variables is None: + paired_heads.update(endpoint["head_effects"]) + endpoints[side] = endpoint["measure_total_gbp"] + elif ( + isinstance(variables, list) + and variables + and len(variables) == len(set(variables)) + ): + paired_heads.update(variables) + try: + endpoints[side] = sum( + endpoint["head_effects"][variable] + for variable in variables + ) + except KeyError as exc: + raise EngineComparisonError( + "paired-run endpoint lacks sized heads" + ) from exc + else: + raise EngineComparisonError( + "paired-run head variables must be unique and nonempty" + ) + if input_hashes is not None: + input_hashes[endpoint_reference] = receipt["sha256"] + if computed.get("effect_gbp") != endpoints["new"] - endpoints["base"]: + raise EngineComparisonError( + "paired-run size differs from computed endpoint change" + ) + term = ( + "paired_run", + computed["base_artifact"]["sha256"], + computed["new_artifact"]["sha256"], + tuple(sorted(paired_heads)), + ) + else: + raise EngineComparisonError( + "sized evidence is not a computed national artifact" + ) + value = computed + try: + for part in allowed: + value = value[part] + except (KeyError, TypeError) as exc: + raise EngineComparisonError( + "sized artifact value path does not exist" + ) from exc + quoted = driver.get("value_gbp") + if ( + isinstance(quoted, bool) + or not isinstance(quoted, (int, float)) + or not math.isfinite(quoted) + or quoted != value + ): + raise EngineComparisonError( + "sized driver value differs from computed artifact" + ) + if term in sized_terms: + raise EngineComparisonError( + "sized drivers repeat the same computed term" + ) + sized_terms.add(term) + if input_hashes is not None: + input_hashes[reference] = proof["sha256"] + return out + + +def _sized_head_scope(driver: dict, artifact_root: Path) -> set[str]: + """Read measured head scope, never allocate a package size across heads.""" + proof = driver["artifact"] + value_path = proof["value_path"] + if value_path[:1] == ["head_effects"]: + return {value_path[1]} + artifact = json.loads((artifact_root / proof["path"]).read_bytes()) + if value_path == ["measure_total_gbp"]: + return set(artifact["head_effects"]) + if artifact.get("head_variables") is not None: + return set(artifact["head_variables"]) + return { + variable + for side in ("base", "new") + for variable in json.loads( + (artifact_root / artifact[side + "_artifact"]["path"]).read_bytes() + )["head_effects"] + } + + +def build_engine_rows( + base_registry: dict, + new_registry: dict, + base_rows: list[dict], + new_rows: list[dict], + attribution: dict, + *, + base_artifacts: dict[str, dict], + new_artifacts: dict[str, dict], + artifact_root: Path = ROOT, + input_hashes: dict | None = None, +) -> list[dict]: + """Join the exact source universe; neither gaps nor zero rows are dropped.""" + left, right = _inventory(base_registry), _inventory(new_registry) + base, new = _comparison_index(base_rows), _comparison_index(new_rows) + if set(left) != set(right) or set(base) != set(left) or set(new) != set(left): + raise EngineComparisonError( + "source-row multiset differs across bundle registries/comparisons" + ) + event = base_registry["event_slug"] + if new_registry["event_slug"] != event or base_registry.get( + "source" + ) != new_registry.get("source"): + raise EngineComparisonError( + "OBR source snapshot or event differs between bundles" + ) + base_measures = {m["measure_key"]: m for m in base_registry["measures"]} + new_measures = {m["measure_key"]: m for m in new_registry["measures"]} + drivers = validate_attribution( + attribution, artifact_root=artifact_root, input_hashes=input_hashes + ) + rows = [] + for identity in sorted(left): + old_measure, old_source = left[identity] + new_measure, new_source = right[identity] + for field in ( + "fy", + "tax_head", + "metric", + "source_table", + "source_column", + "value_gbp_decimal", + "value_gbp", + ): + if old_source.get(field) != new_source.get(field): + raise EngineComparisonError( + f"{identity}: OBR source {field} differs between bundles" + ) + if old_measure["measure_key"] != new_measure["measure_key"]: + raise EngineComparisonError("source row moved to another measure") + before, after = base[identity], new[identity] + old, latest = before["pe_value_gbp"], after["pe_value_gbp"] + old_digest = construction_digest(old_measure, base_measures) + new_digest = construction_digest(new_measure, new_measures) + status = ( + "construction_changed" + if old_digest != new_digest + else "computed_in_both" + if old is not None and latest is not None + else "base_only" + if old is not None + else "new_only" + if latest is not None + else "uncomputed_in_both" + ) + change = None if old is None or latest is None else latest - old + changed = ( + old != latest + or old_digest != new_digest + or before.get("status") != after.get("status") + or before.get("classification") != after.get("classification") + ) + key = (event, old_measure["measure_key"], old_source["fy"]) + assigned = drivers.get(key, drivers.get((*key[:2], None), [])) + if changed and not assigned: + raise EngineComparisonError( + f"changed row requires explicit attribution or unattributed: {key}" + ) + unsized = any(not driver["sized"] for driver in assigned) + row_variables = set(before.get("head_variables", [])) & set( + after.get("head_variables", []) + ) + unmatched_sizing = any( + driver["sized"] + and not _sized_head_scope(driver, artifact_root) <= row_variables + for driver in assigned + ) + share = None + reason = ( + "unsized drivers" + if unsized + else "measure sizing does not isolate this source head" + if unmatched_sizing + else "no computed change" + if change is None + else "zero computed change" + if not change + else "no sized drivers" + if not assigned + else "" + ) + if not reason: + values = [driver["value_gbp"] for driver in assigned] + if any(value * change < 0 for value in values): + reason = "a sized driver masks the observed change" + elif not 0 <= sum(values) / change <= 1: + reason = "computed share outside [0, 1]" + else: + share = sum(values) / change + obr = old_source["value_gbp"] + rows.append( + { + "event": event, + "source_row_id": identity, + "measure_key": old_measure["measure_key"], + "title": old_measure["title"], + "fy": old_source["fy"], + "tax_head": old_source.get("tax_head"), + "obr_million_gbp": obr / 1e6, + "pe_base_million_gbp": None if old is None else old / 1e6, + "pe_new_million_gbp": None if latest is None else latest / 1e6, + "change_million_gbp": None if change is None else change / 1e6, + "base_ratio_bin": ratio_and_bin(obr, old)[1], + "new_ratio_bin": ratio_and_bin(obr, latest)[1], + "status": status, + "base_status": before.get("status"), + "new_status": after.get("status"), + "base_construction_sha256": old_digest, + "new_construction_sha256": new_digest, + "base_classification": before.get("classification"), + "new_classification": after.get("classification"), + "head_certified_aggregate_new_to_base_ratio": _ratio( + _aggregate(after, new_artifacts, "certified_aggregates_gbp"), + _aggregate(before, base_artifacts, "certified_aggregates_gbp"), + ), + "measure_new_to_base_ratio": _ratio( + _aggregate(after, new_artifacts, "measure_total_gbp"), + _aggregate(before, base_artifacts, "measure_total_gbp"), + ), + "head_effect_new_to_base_ratio": _ratio(latest, old), + "attribution": assigned, + "explained_share": share, + "explained_share_withheld": reason, + } + ) + + # Source £ accounting is exact; constructions and classes can change while + # this same row inventory and its net/absolute amount remain fixed. + def account(inventory): + amounts = [ + Decimal(str(source.get("value_gbp_decimal", source["value_gbp"]))) + for _, source in inventory.values() + ] + return sum(amounts), sum(map(abs, amounts)) + + if account(left) != account(right): + raise EngineComparisonError("net or absolute OBR GBP differs across bundles") + return sorted( + rows, + key=lambda row: ( + row["event"], + row["measure_key"], + row["fy"], + str(row["tax_head"]), + row["source_row_id"], + ), + ) + + +def render_csv(rows: list[dict]) -> str: + stream = io.StringIO(newline="") + writer = csv.DictWriter(stream, fieldnames=CSV_FIELDS, lineterminator="\n") + writer.writeheader() + for row in rows: + writer.writerow( + { + key: json.dumps(row[key], sort_keys=True, separators=(",", ":")) + if isinstance(row[key], (dict, list)) + else "" + if row[key] is None + else row[key] + for key in CSV_FIELDS + } + ) + return stream.getvalue() + + +def render_markdown(base: str, new: str, rows: list[dict]) -> str: + number = lambda value: "—" if value is None else f"{value:,.3f}" + md = event_comparison._md + counts = Counter(row["status"] for row in rows) + lines = [ + "# UK replay engine comparison", + "", + f"Base: `{base}`. New: `{new}`.", + "", + ( + "Amounts are £m, positive for gain to the Exchequer. Ratios and bins are descriptive. " + "The certified head aggregate ratio shows how the fiscal base moved alongside the measure ratio. " + "A changed construction is listed separately from a change on the same construction. " + "Attribution comes only from the authored evidence file; unsized drivers withhold explained share." + ), + "", + ", ".join(f"{key}: {value}" for key, value in sorted(counts.items())) + ".", + "", + "| Event / measure | FY / head | OBR £m | PE base £m | PE new £m | Change £m | Base bin | New bin | Status | Head base ratio | Measure ratio | Attribution |", + "|---|---|---:|---:|---:|---:|---|---|---|---:|---:|---|", + ] + for row in rows: + lines.append( + "| " + + " | ".join( + [ + md(row["event"] + " / " + row["title"]), + md(row["fy"] + " / " + str(row["tax_head"])), + *( + number(row[field]) + for field in ( + "obr_million_gbp", + "pe_base_million_gbp", + "pe_new_million_gbp", + "change_million_gbp", + ) + ), + row["base_ratio_bin"], + row["new_ratio_bin"], + row["status"], + number(row["head_certified_aggregate_new_to_base_ratio"]), + number(row["measure_new_to_base_ratio"]), + md( + ", ".join( + driver["driver"] + + (" (sized)" if driver["sized"] else " (unsized)") + for driver in row["attribution"] + ) + or "unchanged" + ), + ] + ) + + " |" + ) + return "\n".join(lines) + "\n" + + +def _path_at_root(path: Path, root: Path) -> Path: + return root / path.relative_to(ROOT) + + +def _verify_run_receipts( + directory: Path, + registry_path: Path, + registry: dict, + bundle: str, + root: Path, + commit, +) -> None: + """Bind the numerical manifest and each per-year progress receipt.""" + from pipeline import stage_uk_event as staging + + manifest_path = directory / "RUN_MANIFEST.json" + commit(manifest_path) + manifest = json.loads(manifest_path.read_bytes()) + bundles.validate_runtime_bundle(manifest, bundle, root=root) + if manifest.get("registry_sha256") != sha256(registry_path): + raise EngineComparisonError("compute manifest registry SHA-256 differs") + years = set() + for reference, digest in manifest["artifacts"].items(): + path = event_comparison._receipt_path(reference, root) + if not path.is_relative_to(directory.resolve()) or sha256(path) != digest: + raise EngineComparisonError( + "compute manifest artifact path or SHA-256 differs" + ) + commit(path) + years.add(json.loads(path.read_bytes())["year"]) + for year in sorted(years): + progress_path = directory / f"RUN_PROGRESS_{year}.json" + commit(progress_path) + progress = json.loads(progress_path.read_bytes()) + bundles.validate_runtime_bundle(progress, bundle, root=root) + expected = { + reference: digest + for reference, digest in manifest["artifacts"].items() + if json.loads((root / reference).read_bytes())["year"] == year + } + if ( + progress.get("artifacts") != expected + or progress.get("year") != year + or progress.get("event") != registry["event_slug"] + or progress.get("registry_sha256") != sha256(registry_path) + ): + raise EngineComparisonError( + "per-year progress receipt differs from final numerical manifest" + ) + # Stage's original path resolver uses compute.ROOT; supplying a separate + # bundle_root keeps pin trust explicit for copied comparison test roots. + original_root = staging.compute.ROOT + try: + staging.compute.ROOT = root + staged_rows, tally = staging.stage_event( + registry, + manifest, + artifact_dir=directory, + registry_sha256=sha256(registry_path), + bundle=bundle, + bundle_root=root, + ) + finally: + staging.compute.ROOT = original_root + if staged_rows != event_comparison.load_jsonl(directory / "STAGED.jsonl"): + raise EngineComparisonError( + "staged rows differ from verified numerical manifest" + ) + stage_receipt = json.loads((directory / "STAGING_MANIFEST.json").read_bytes()) + if any(stage_receipt.get(field) != value for field, value in tally.items()): + raise EngineComparisonError( + "staging tally differs from verified numerical manifest" + ) + modal = directory / "MODAL_RECEIPT.json" + if modal.exists(): + commit(modal) + bundles.validate_bundle_identity( + json.loads(modal.read_bytes()), bundle, root=root + ) + + +def write_engine_comparison( + base: str, + new: str, + *, + attribution_path: Path | None = None, + output_dir: Path | None = None, + artifact_root: Path = ROOT, +) -> list[dict]: + """Verify both comparison receipt chains and bind every used input hash.""" + bundles.load_bundle(base, root=artifact_root) + bundles.load_bundle(new, root=artifact_root) + attribution_path = ( + attribution_path + or artifact_root / "data/uk/events/engine_attribution" / f"{base}__{new}.json" + ) + attribution = json.loads(attribution_path.read_bytes()) + if attribution.get("base") != base or attribution.get("new") != new: + raise EngineComparisonError( + "attribution bundle pair differs from requested comparison" + ) + inputs = {} + + def commit(path): + reference = str(path.resolve().relative_to(artifact_root.resolve())) + digest = sha256(path) + if reference in inputs and inputs[reference] != digest: + raise EngineComparisonError("input changed during engine comparison") + inputs[reference] = digest + return reference + + commit(attribution_path) + for key in (base, new): + commit(bundles.bundle_pin_path(key, root=artifact_root)) + pin = bundles.load_bundle(key, root=artifact_root) + for label in ("requirements_freeze", "offline_audit"): + commit(artifact_root / pin[label]) + commit(bundles.bundle_index_path(root=artifact_root)) + commit(_path_at_root(event_comparison.AXES_PATH, artifact_root)) + roots = {key: bundles.results_root(key, root=artifact_root) for key in (base, new)} + event_sets = { + key: {p.parent.name for p in root.glob("*/COMPARISON.json")} + for key, root in roots.items() + } + registry_events = { + key: { + path.stem.removesuffix("_measures") + for path in bundles.registry_path( + "event", key, root=artifact_root + ).parent.glob("*_measures.json") + } + for key in (base, new) + } + if ( + not event_sets[base] + or event_sets[base] != event_sets[new] + or registry_events[base] != registry_events[new] + or event_sets[base] != registry_events[base] + ): + raise EngineComparisonError( + "both bundles require comparisons for every event in the same nonempty registry inventory" + ) + rows = [] + for event in sorted(event_sets[base]): + registries, comparisons, artifacts = {}, {}, {} + for key in (base, new): + directory = roots[key] / event + path = directory / "COMPARISON.json" + comparisons[key] = event_comparison.load_verified_comparison( + path, artifact_root=artifact_root, bundle=key + ) + for name in ( + "COMPARISON.json", + "COMPARISON.md", + "COMPARISON.csv", + "COMPARISON_PROVENANCE.json", + ): + commit(directory / name) + provenance = json.loads( + (directory / "COMPARISON_PROVENANCE.json").read_bytes() + ) + for label in ("registry", "staged", "staging_manifest"): + commit(artifact_root / provenance[label + "_path"]) + registry_path = artifact_root / provenance["registry_path"] + registries[key] = json.loads(registry_path.read_bytes()) + _verify_run_receipts( + directory, registry_path, registries[key], key, artifact_root, commit + ) + source = registries[key].get("source", {}) + if source.get("claims_path"): + source_path = artifact_root / source["claims_path"] + if sha256(source_path) != source.get("claims_gzip_sha256"): + raise EngineComparisonError( + "OBR snapshot compressed SHA-256 differs" + ) + commit(source_path) + artifacts[key] = {} + for row in comparisons[key]: + if row["pe_value_gbp"] is not None: + artifact_path = artifact_root / row["artifact"] + commit(artifact_path) + artifacts[key][row["artifact"]] = json.loads( + artifact_path.read_bytes() + ) + rows.extend( + build_engine_rows( + registries[base], + registries[new], + comparisons[base], + comparisons[new], + attribution, + base_artifacts=artifacts[base], + new_artifacts=artifacts[new], + artifact_root=artifact_root, + input_hashes=inputs, + ) + ) + payloads = { + "ENGINE_COMPARISON.json": canonical_bytes(rows), + "ENGINE_COMPARISON.csv": render_csv(rows).encode(), + "ENGINE_COMPARISON.md": render_markdown(base, new, rows).encode(), + } + # Catch concurrent changes before any output write. + for reference, digest in inputs.items(): + if sha256(artifact_root / reference) != digest: + raise EngineComparisonError("input changed during engine comparison") + output_dir = output_dir or artifact_root / "results/uk/events" + output_dir.mkdir(parents=True, exist_ok=True) + for name, payload in payloads.items(): + atomic_write_bytes(output_dir / name, payload) + provenance = { + "schema_version": 1, + "base": base, + "new": new, + "source_rows": len(rows), + "inputs_sha256": dict(sorted(inputs.items())), + "outputs_sha256": { + name: hashlib.sha256(payload).hexdigest() + for name, payload in sorted(payloads.items()) + }, + } + atomic_write_bytes( + output_dir / "ENGINE_COMPARISON_PROVENANCE.json", canonical_bytes(provenance) + ) + return rows + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", required=True) + parser.add_argument("--new", required=True) + parser.add_argument("--attribution", type=Path) + parser.add_argument("--output-dir", type=Path) + args = parser.parse_args(argv) + rows = write_engine_comparison( + args.base, + args.new, + attribution_path=args.attribution, + output_dir=args.output_dir, + ) + print(f"wrote {len(rows)} descriptive engine-comparison source rows") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fixtures/uk_engine_comparison/attribution.json b/tests/fixtures/uk_engine_comparison/attribution.json new file mode 100644 index 00000000..29c1b7c7 --- /dev/null +++ b/tests/fixtures/uk_engine_comparison/attribution.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "base": "test-base", + "new": "test-new", + "fixture_note": "Test example only. This is not attribution for any executed national comparison.", + "entries": [ + { + "event": "event", + "measure_key": "event__tax", + "drivers": [ + { + "driver": "data_release", + "evidence": [{"kind": "diagnostic", "reference": "test-fixture: paired certified aggregate change"}], + "sized": false + } + ] + }, + { + "event": "event", + "measure_key": "event__gap", + "drivers": [ + { + "driver": "unattributed", + "evidence": [{"kind": "unattributed", "reference": "Test fixture has no measured mechanism attribution"}], + "sized": false + } + ] + } + ] +} diff --git a/tests/test_uk_engine_comparison.py b/tests/test_uk_engine_comparison.py new file mode 100644 index 00000000..449b8b0a --- /dev/null +++ b/tests/test_uk_engine_comparison.py @@ -0,0 +1,422 @@ +"""Descriptive engine-comparison accounting and attribution properties.""" + +import copy +import hashlib +import json +from pathlib import Path + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from pipeline import compare_uk_engines as engines +from pipeline.compare_uk_obr_costings import ratio_and_bin + +FIXTURE = Path(__file__).parent / "fixtures/uk_engine_comparison/attribution.json" + + +def inputs( + old=(100, -20), + new=(120, -25), + base_aggregate=(1000, 200), + new_aggregate=(1100, 220), +): + registry = { + "event_slug": "event", + "source": {"claims_sha256": "fixed"}, + "measures": [ + { + "measure_key": "event__tax", + "title": "Tax change", + "construction": "forward_delta_on_certified_world", + "pe_reform_delta": {"gov.tax.rate": {"2024-01-01.2030-12-31": 0.2}}, + "source_rows": [ + { + "source_row_id": f"row-{i}", + "fy": "2024-25", + "tax_head": f"Head {i}", + "value_gbp": float(obr), + "value_gbp_decimal": str(obr), + "pe_variables": [f"head_{i}"], + } + for i, obr in enumerate((90, -15)) + ], + }, + { + "measure_key": "event__gap", + "title": "Gap", + "source_rows": [ + { + "source_row_id": "gap", + "fy": "2023-24", + "tax_head": "Other", + "value_gbp": 0.0, + "value_gbp_decimal": "0", + } + ], + }, + ], + } + + def run(values, aggregates, name): + heads = dict(zip(("head_0", "head_1"), values)) + artifact = { + "event": "event", + "measure_key": "event__tax", + "year": 2024, + "head_effects": heads, + "head_channels": {k: "tax" for k in heads}, + "construction": "forward_delta_on_certified_world", + "totals": { + "baseline": {"heads": {k: 0 for k in heads}}, + "reform": {"heads": heads}, + }, + "literal_reform_minus_baseline": heads, + "measure_total_gbp": sum(values), + "certified_aggregates_gbp": dict(zip(heads, aggregates)), + } + rows = [ + { + "source_row_id": f"row-{i}", + "pe_value_gbp": value, + "artifact": name, + "head_variables": [f"head_{i}"], + "status": "constructed", + "classification": "partial", + } + for i, value in enumerate(values) + ] + rows.append( + { + "source_row_id": "gap", + "pe_value_gbp": None, + "status": "not_computed", + "classification": "out_of_household_scope", + } + ) + return rows, {name: artifact} + + base_rows, base_artifacts = run(old, base_aggregate, "base.json") + new_rows, new_artifacts = run(new, new_aggregate, "new.json") + return ( + registry, + copy.deepcopy(registry), + base_rows, + new_rows, + base_artifacts, + new_artifacts, + ) + + +def compare(values=None, attribution=None, **kwargs): + left, right, before, after, old, new = values or inputs() + return engines.build_engine_rows( + left, + right, + before, + after, + attribution or json.loads(FIXTURE.read_text()), + base_artifacts=old, + new_artifacts=new, + **kwargs, + ) + + +def test_each_source_row_and_zero_kept_and_ratios_are_descriptive(): + rows = compare() + assert len(rows) == 3 + tax = [r for r in rows if r["measure_key"] == "event__tax"] + assert sum(r["change_million_gbp"] for r in tax) == pytest.approx(15 / 1e6) + assert tax[0]["head_certified_aggregate_new_to_base_ratio"] == 1.1 + assert tax[0]["measure_new_to_base_ratio"] == 95 / 80 + assert tax[0]["head_effect_new_to_base_ratio"] == 1.2 + assert all(r["explained_share"] is None for r in rows) + assert ( + next(r for r in rows if r["source_row_id"] == "gap")["status"] + == "uncomputed_in_both" + ) + + +@pytest.mark.parametrize( + "change", + ["missing", "unknown", "sized_no_artifact", "unsized_value", "no_evidence"], +) +def test_attribution_rejects_missing_unknown_or_unsupported_sizing(change): + attribution = json.loads(FIXTURE.read_text()) + driver = attribution["entries"][0]["drivers"][0] + if change == "missing": + attribution["entries"] = [] + elif change == "unknown": + driver["driver"] = "guessed_population_driver" + elif change == "sized_no_artifact": + driver.update(sized=True, value_gbp=20) + elif change == "unsized_value": + driver["value_gbp"] = 20 + else: + driver["evidence"] = [] + with pytest.raises(engines.EngineComparisonError): + compare(attribution=attribution) + + +def test_explicit_unattributed_is_accepted_without_inference(): + attribution = json.loads(FIXTURE.read_text()) + attribution["entries"][0]["drivers"] = copy.deepcopy( + attribution["entries"][1]["drivers"] + ) + rows = compare(attribution=attribution) + assert [r["attribution"][0]["driver"] for r in rows if r["change_million_gbp"]] == [ + "unattributed", + "unattributed", + ] + + +def test_sized_driver_requires_matching_computed_artifact_hash_and_value(tmp_path): + values = inputs() + artifact = values[5]["new.json"] + payload = engines.canonical_bytes(artifact) + (tmp_path / "paired.json").write_bytes(payload) + attribution = json.loads(FIXTURE.read_text()) + attribution["entries"][0]["fy"] = "2024-25" + driver = attribution["entries"][0]["drivers"][0] + driver.update( + sized=True, + value_gbp=120, + artifact={ + "path": "paired.json", + "sha256": hashlib.sha256(payload).hexdigest(), + "value_path": ["head_effects", "head_0"], + }, + ) + hashes = {} + compare(values, attribution, artifact_root=tmp_path, input_hashes=hashes) + assert hashes == {"paired.json": hashlib.sha256(payload).hexdigest()} + driver["value_gbp"] += 1 + with pytest.raises(engines.EngineComparisonError, match="differs"): + compare(values, attribution, artifact_root=tmp_path) + driver["value_gbp"] -= 1 + driver["artifact"]["sha256"] = "0" * 64 + with pytest.raises(engines.EngineComparisonError, match="SHA-256"): + compare(values, attribution, artifact_root=tmp_path) + + +def test_unsized_driver_withholds_explained_share_even_beside_sized(tmp_path): + values = inputs() + payload = engines.canonical_bytes(values[5]["new.json"]) + (tmp_path / "artifact.json").write_bytes(payload) + attribution = json.loads(FIXTURE.read_text()) + attribution["entries"][0]["fy"] = "2024-25" + attribution["entries"][0]["drivers"].append( + { + "driver": "other_engine_change", + "evidence": [{"kind": "paired_run", "reference": "artifact.json"}], + "sized": True, + "value_gbp": 120, + "artifact": { + "path": "artifact.json", + "sha256": hashlib.sha256(payload).hexdigest(), + "value_path": ["head_effects", "head_0"], + }, + } + ) + rows = compare(values, attribution, artifact_root=tmp_path) + assert all(r["explained_share"] is None for r in rows) + assert all( + r["explained_share_withheld"] == "unsized drivers" + for r in rows + if r["change_million_gbp"] + ) + + +@pytest.mark.parametrize( + "change", ["drop", "duplicate", "source_value", "source_fy", "source_snapshot"] +) +def test_cross_bundle_accounting_cannot_drop_duplicate_or_change_source(change): + values = list(inputs()) + if change == "drop": + values[3].pop() + elif change == "duplicate": + values[3].append(copy.deepcopy(values[3][0])) + elif change == "source_value": + values[1]["measures"][0]["source_rows"][0]["value_gbp_decimal"] = ( + "90.00000000001" + ) + elif change == "source_fy": + values[1]["measures"][0]["source_rows"][0]["fy"] = "2025-26" + else: + values[1]["source"]["claims_sha256"] = "other" + with pytest.raises(engines.EngineComparisonError): + compare(values) + + +def test_base_only_window_and_changed_construction_are_explicit(): + values = list(inputs()) + values[3][0].update(pe_value_gbp=None, status="outside_bundle_window") + rows = compare(values) + assert ( + next(r for r in rows if r["source_row_id"] == "row-0")["status"] == "base_only" + ) + values[1]["measures"][0]["pe_reform_delta"]["gov.tax.rate"][ + "2024-01-01.2030-12-31" + ] = 0.3 + rows = compare(values) + assert {r["status"] for r in rows if r["measure_key"] == "event__tax"} == { + "construction_changed" + } + assert rows[-1]["base_construction_sha256"] != rows[-1]["new_construction_sha256"] + + +def test_construction_digest_includes_package_components_but_ignores_prose(): + registry = inputs()[0] + measure = registry["measures"][0] + before = engines.construction_digest(measure) + measure["note"] = "Audited prose correction" + assert engines.construction_digest(measure) == before + package = { + "measure_key": "event__package", + "construction": "package_of_registry_measures", + "package_of": [measure["measure_key"]], + "source_rows": [], + } + index = {m["measure_key"]: m for m in (measure, package)} + before = engines.construction_digest(package, index) + measure["heads"] = [{"obr_head": "Head 0", "pe_variables": ["changed"]}] + assert engines.construction_digest(package, index) != before + + +def test_output_order_and_rendering_are_deterministic(): + values = list(inputs()) + before = compare(values) + values[2].reverse() + values[3].reverse() + values[1]["measures"].reverse() + after = compare(values) + assert engines.canonical_bytes(before) == engines.canonical_bytes(after) + assert engines.render_csv(before) == engines.render_csv(after) + assert engines.render_markdown("base", "new", before) == engines.render_markdown( + "base", "new", after + ) + + +@given( + st.tuples(st.integers(-(10**9), 10**9), st.integers(-(10**9), 10**9)), + st.tuples(st.integers(-(10**9), 10**9), st.integers(-(10**9), 10**9)), +) +@settings(deadline=None) +def test_swapping_bundles_negates_every_change_and_heads_sum(old, new): + values = inputs(old, new) + rows = compare(values) + reverse = compare( + (values[1], values[0], values[3], values[2], values[5], values[4]) + ) + assert [r["source_row_id"] for r in rows] == [r["source_row_id"] for r in reverse] + for left, right in zip(rows, reverse): + if left["change_million_gbp"] is not None: + assert left["change_million_gbp"] == -right["change_million_gbp"] + assert sum(r["change_million_gbp"] or 0 for r in rows) == pytest.approx( + (sum(new) - sum(old)) / 1e6, abs=1e-10 + ) + + +@given(st.integers(1, 10**9), st.integers(-(10**9), 10**9), st.integers(1, 10**6)) +def test_bins_are_invariant_to_common_positive_scale(obr, pe, scale): + assert ratio_and_bin(obr, pe)[1] == ratio_and_bin(obr * scale, pe * scale)[1] + + +def test_duplicate_registry_measure_key_is_rejected(): + values = list(inputs()) + duplicate = copy.deepcopy(values[1]["measures"][0]) + duplicate["source_rows"] = [] + values[1]["measures"].append(duplicate) + with pytest.raises( + engines.EngineComparisonError, match="duplicate registry measure" + ): + compare(values) + + +def test_named_drivers_cannot_double_count_one_computed_term(tmp_path): + values = inputs() + payload = engines.canonical_bytes(values[5]["new.json"]) + (tmp_path / "artifact.json").write_bytes(payload) + attribution = json.loads(FIXTURE.read_text()) + entry = attribution["entries"][0] + entry["fy"] = "2024-25" + first = entry["drivers"][0] + first.update( + sized=True, + value_gbp=120, + artifact={ + "path": "artifact.json", + "sha256": hashlib.sha256(payload).hexdigest(), + "value_path": ["head_effects", "head_0"], + }, + ) + second = {**copy.deepcopy(first), "driver": "other_engine_change"} + entry["drivers"].append(second) + with pytest.raises(engines.EngineComparisonError, match="same computed term"): + compare(values, attribution, artifact_root=tmp_path) + + +@pytest.mark.parametrize("scope", [None, ["head_0"]]) +@pytest.mark.parametrize("direction", [1, -1]) +def test_paired_package_size_is_not_allocated_to_each_source_head( + tmp_path, scope, direction +): + from pipeline import uk_bundle as bundles + + values = inputs( + old=(100 * direction, 100 * direction), new=(120 * direction, 120 * direction) + ) + measured = inputs( + old=(100 * direction, 100 * direction), new=(110 * direction, 100 * direction) + ) + pin_path = tmp_path / "data/uk/certified_bundle.json" + pin_path.parent.mkdir(parents=True) + pin_path.write_bytes(bundles.bundle_pin_path().read_bytes()) + endpoints = {} + for side, artifact in ( + ("base", measured[4]["base.json"]), + ("new", measured[5]["new.json"]), + ): + payload = engines.canonical_bytes(artifact) + path = tmp_path / f"measured-{side}.json" + path.write_bytes(payload) + endpoints[side] = { + "path": path.name, + "sha256": hashlib.sha256(payload).hexdigest(), + } + paired = { + "artifact_type": "paired_run", + "status": "computed", + "effect_gbp": 10 * direction, + "base_artifact": endpoints["base"], + "new_artifact": endpoints["new"], + "base_bundle": bundles.DEFAULT_BUNDLE, + "new_bundle": bundles.DEFAULT_BUNDLE, + } + if scope is not None: + paired["head_variables"] = scope + payload = engines.canonical_bytes(paired) + (tmp_path / "paired.json").write_bytes(payload) + attribution = json.loads(FIXTURE.read_text()) + entry = attribution["entries"][0] + entry["fy"] = "2024-25" + entry["drivers"][0].update( + sized=True, + value_gbp=10 * direction, + artifact={ + "path": "paired.json", + "sha256": hashlib.sha256(payload).hexdigest(), + "value_path": ["effect_gbp"], + }, + ) + rows = compare(values, attribution, artifact_root=tmp_path) + tax = [row for row in rows if row["measure_key"] == "event__tax"] + if scope is None: + assert all(row["explained_share"] is None for row in tax) + else: + assert tax[0]["explained_share"] == 0.5 + assert tax[1]["explained_share"] is None + assert ( + tax[1]["explained_share_withheld"] + == "measure sizing does not isolate this source head" + ) From 93f2200a4c59f5bf365585aed6b16a092fee776b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 07:41:03 -0400 Subject: [PATCH 03/11] Document reruns on another certified bundle RECIPE.md gains the ordered rerun procedure: register, freeze, registry rebuild and construction re-audit, Modal preflight, determinism repeat, full runs, adoption, staging, the engine comparison and the attribution file. The 6.x loader audit and year evidence come from checks on the published, already-certified policyengine 6.2.5 / uk-data 1.56.16 bundle. Those are adapter observations, not results. Refs #156. Co-Authored-By: Claude Opus 5.5 --- ...IT_dev-uk-data-1.56.16__pe-uk-2.102.3.json | 247 ++++++++++++++++++ ...UDIT_dev-uk-data-1.56.16__pe-uk-2.102.3.md | 47 ++++ docs/uk_replay/RECIPE.md | 133 ++++++++++ ...EARS_dev-uk-data-1.56.16__pe-uk-2.102.3.md | 53 ++++ pipeline/audit_uk_bundle_loader.py | 178 +++++++++++++ pipeline/smoke_uk_bundle_adapter.py | 80 ++++++ 6 files changed, 738 insertions(+) create mode 100644 docs/uk_replay/OFFLINE_BUNDLE_AUDIT_dev-uk-data-1.56.16__pe-uk-2.102.3.json create mode 100644 docs/uk_replay/OFFLINE_BUNDLE_AUDIT_dev-uk-data-1.56.16__pe-uk-2.102.3.md create mode 100644 docs/uk_replay/YEARS_dev-uk-data-1.56.16__pe-uk-2.102.3.md create mode 100644 pipeline/audit_uk_bundle_loader.py create mode 100644 pipeline/smoke_uk_bundle_adapter.py diff --git a/docs/uk_replay/OFFLINE_BUNDLE_AUDIT_dev-uk-data-1.56.16__pe-uk-2.102.3.json b/docs/uk_replay/OFFLINE_BUNDLE_AUDIT_dev-uk-data-1.56.16__pe-uk-2.102.3.json new file mode 100644 index 00000000..be93751c --- /dev/null +++ b/docs/uk_replay/OFFLINE_BUNDLE_AUDIT_dev-uk-data-1.56.16__pe-uk-2.102.3.json @@ -0,0 +1,247 @@ +{ + "bundle_key": "dev-uk-data-1.56.16__pe-uk-2.102.3", + "bundle_pin_sha256": "9d1c95a0fccb529bbc04956cb58c0eefafe3a8fc4940512ce138caa8c09856ec", + "bundled_certification": { + "built_with_model_git_sha": "12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3", + "built_with_model_version": "2.89.2", + "certified_by": "policyengine.py bundle certification", + "certified_for_model_version": "2.102.3", + "compatibility_basis": "legacy_compatible_model_package", + "data_build_fingerprint": "sha256:6bfff835b504094c05342d2b4d770d2d94c8c1adcb15c4468931f53b21a89ea9", + "data_build_id": "policyengine-uk-data-1.56.16" + }, + "certified_dataset_sha256": "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", + "certified_identity": { + "artifact": "enhanced_frs_2024_25.h5", + "core_version": "3.32.10", + "data_build_id": "policyengine-uk-data-1.56.16", + "data_year": 2024, + "dataset_uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.56.16", + "model_version": "2.102.3", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "resolved_hf_commit": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", + "revision": "1.56.16", + "sha256": "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", + "size_bytes": 126553300 + }, + "development_bundle": true, + "engine_versions": { + "policyengine": "6.2.5", + "policyengine-core": "3.32.10", + "policyengine-uk": "2.102.3" + }, + "inspection": "Engine-free installed-source inspection; no provider, credentials, network or simulation.", + "minimal_hf_cache_allowlist": { + "exclude": [ + "tokens", + "credentials", + "refs/main", + "other revisions", + "other artifacts" + ], + "repo_cache_directory": "models--policyengine--policyengine-uk-data-private", + "required_ref": { + "contents": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", + "path": "refs/1.56.16" + }, + "required_snapshot_file": { + "path": "snapshots/a9e52499b6a6cca100a5ce4f36ca27b2e8a213df/enhanced_frs_2024_25.h5" + } + }, + "offline_loader_behavior": { + "fallback": "Must retain packaged certified_for_model_version matching the runtime; unverified fallback is refused.", + "materialization": "Reuse hash-verified ./data/; requests downloads bypass HF offline flags on a missing or mismatched file.", + "projection": "managed_microsimulation passes the original stored-year dataset to the engine; no projected year-file re-anchoring.", + "read_only_h5": "Single-year and multi-year constructors use HDFStore(mode='r'); no writable dataset copy is required.", + "release_metadata": "requests.get bypasses HF offline flags; socket blocking forces packaged certification fallback.", + "runtime_dataset_source": "The source.path returned by materialize_dataset is recorded by build_runtime_dataset_provenance and must resolve to the pre/post hashed file." + }, + "packaged_bundle_manifest_sha256": "41db4dcd0a64e37901493ffb2235e159892b4e2cb45224d5f83b696dee9d56ea", + "policyengine_version": "6.2.5", + "registration_provenance_hashes": { + "cached_artifact_manifest_sha256": "5058149dbffc4f710a63645269a00a117413a52a690699e1259dc430b75caa49", + "engine_extension_source_sha256": "5f962cd694e2bdfd389d3ec57dfbdb0dacb4f2b63f10597e4570d2735354d240", + "packaged_bundle_manifest_sha256": "41db4dcd0a64e37901493ffb2235e159892b4e2cb45224d5f83b696dee9d56ea" + }, + "release_manifest_revision": "78788372fec47fc8df835bd32307ef61eaed83c2", + "schema_version": 2, + "source_files_sha256": { + "policyengine/core/tax_benefit_model_version.py": "a9a731ced229ca5993c715b134b97dce84201971bdb547a5c346a9d4de7b9f8a", + "policyengine/provenance/dataset_materialization.py": "33dd345fee43ef60c3fc865865e4747787a7abbf1ee9ab3ad0e9fc2947d4d363", + "policyengine/provenance/manifest.py": "d86595e401ade1d7ac15702681579403a39ed3ae62e93e525ae275e065d847f0", + "policyengine/tax_benefit_models/common/model_version.py": "61936bad7fb476d05f929616d937f725a9d91a8ce5575ea4a568730c4494b87f", + "policyengine/tax_benefit_models/uk/model.py": "8bf7da81e991b7a879759618ea71489584c38f2a1d840b623ba44bdd30b677a4", + "policyengine_uk/data/dataset_schema.py": "1c3df14b0011d7e7755dc78cd84f6a7a57eaaa9baa6498e521d5de5adade83b5", + "policyengine_uk/data/economic_assumptions.py": "5f962cd694e2bdfd389d3ec57dfbdb0dacb4f2b63f10597e4570d2735354d240", + "policyengine_uk/parameters/gov/hmrc/alcohol_duty/rates/beer.yaml": "a72fda046203e130bc603adce5b31d15cb31658d394a59768e6fe2bb373298ce", + "policyengine_uk/parameters/gov/hmrc/cgt/additional_rate.yaml": "8cd5276a19370b0665aebcea204af620c287c05c692d60e461c8522f4876f29c", + "policyengine_uk/parameters/gov/hmrc/cgt/basic_rate.yaml": "adea67b26f52f53b42b0fa8e333d1b200ea8431524c6d7a2532054daf8f16579", + "policyengine_uk/parameters/gov/hmrc/cgt/higher_rate.yaml": "5efda44289f6d6eb128b62c91b08278b72ab9a85ca0719bfb2f3217c0876a67a", + "policyengine_uk/parameters/gov/hmrc/fuel_duty/lpg.yaml": "188d2143240577b0b3146ffa037cdc15c6abfad0de922abd8f0f9b7dbbb991e3", + "policyengine_uk/parameters/gov/hmrc/fuel_duty/natural_gas.yaml": "f934fc6673b87401c8159be97b05e7bb2ad0e44b889d0c923a5e5d7a1596a7a3", + "policyengine_uk/parameters/gov/hmrc/tobacco_duty/rates/cigarette_specific.yaml": "0bc7852556782b72429bd35ff97620361bafa9365f79a52dbafde0f4be295531", + "policyengine_uk/scenarios/uc_reform.py": "5f139b8af2579099d1842c86ad5c4de8fc1984d6adb77ed9ce25dbfebd4e9eec", + "policyengine_uk/simulation.py": "29c952d701c8e7f77421e2ef67348622376e5e956a843c84674d3fe7368558cc", + "policyengine_uk/utils/parameters.py": "2d397594a3be8f984e5c372a7ee555e14e0883fd6066bd2716f717f5369be8f1", + "policyengine_uk/utils/scenario.py": "4c0e64975eb2414906b8a7b6879c5820b018f4226f780b3a41c7a1508c67e4e9" + }, + "source_references": { + "policyengine/core/tax_benefit_model_version.py": [ + { + "line": 137, + "text": "def release_bundle(self) -> dict[str, Optional[str]]:" + } + ], + "policyengine/provenance/dataset_materialization.py": [ + { + "line": 124, + "text": "def materialize_dataset(" + }, + { + "line": 260, + "text": "def _reuse_or_download_bundle_files(" + } + ], + "policyengine/provenance/manifest.py": [ + { + "line": 339, + "text": "def get_data_release_manifest(country_id: str) -> DataReleaseManifest:" + }, + { + "line": 422, + "text": "return bundled_certification" + } + ], + "policyengine/tax_benefit_models/common/model_version.py": [ + { + "line": 81, + "text": "\"runtime_dataset_source\": local_path," + } + ], + "policyengine/tax_benefit_models/uk/model.py": [ + { + "line": 397, + "text": "def managed_microsimulation(" + } + ], + "policyengine_uk/data/dataset_schema.py": [ + { + "line": 56, + "text": "with pd.HDFStore(file_path, mode=\"r\") as f:" + }, + { + "line": 198, + "text": "with pd.HDFStore(file_path, mode=\"r\") as f:" + } + ], + "policyengine_uk/data/economic_assumptions.py": [ + { + "line": 38, + "text": "end_year: int = 2030," + } + ], + "policyengine_uk/parameters/gov/hmrc/alcohol_duty/rates/beer.yaml": [ + { + "line": 19, + "text": "preserve_calendar_dates: true" + } + ], + "policyengine_uk/parameters/gov/hmrc/cgt/additional_rate.yaml": [ + { + "line": 14, + "text": "fiscal_year_blend: true" + } + ], + "policyengine_uk/parameters/gov/hmrc/cgt/basic_rate.yaml": [ + { + "line": 14, + "text": "fiscal_year_blend: true" + } + ], + "policyengine_uk/parameters/gov/hmrc/cgt/higher_rate.yaml": [ + { + "line": 14, + "text": "fiscal_year_blend: true" + } + ], + "policyengine_uk/parameters/gov/hmrc/fuel_duty/lpg.yaml": [ + { + "line": 17, + "text": "fiscal_year_blend: true" + } + ], + "policyengine_uk/parameters/gov/hmrc/fuel_duty/natural_gas.yaml": [ + { + "line": 17, + "text": "fiscal_year_blend: true" + } + ], + "policyengine_uk/parameters/gov/hmrc/tobacco_duty/rates/cigarette_specific.yaml": [ + { + "line": 27, + "text": "preserve_calendar_dates: true" + } + ], + "policyengine_uk/scenarios/uc_reform.py": [ + { + "line": 93, + "text": "sim.set_input(\"uc_LCWRA_element\", year, current_health_element)" + } + ], + "policyengine_uk/simulation.py": [ + { + "line": 179, + "text": "universal_credit_july_2025_reform.simulation_modifier(self)" + }, + { + "line": 214, + "text": "if scenario is not None and not scenario.applied_before_data_load:" + }, + { + "line": 461, + "text": "def build_from_single_year_dataset(self, dataset: UKSingleYearDataset) -> None:" + }, + { + "line": 475, + "text": "def build_from_multi_year_dataset(self, dataset: UKMultiYearDataset) -> None:" + } + ], + "policyengine_uk/utils/parameters.py": [ + { + "line": 78, + "text": "def convert_to_fiscal_year_parameters(parameters):" + }, + { + "line": 90, + "text": "year. Parameters carrying `fiscal_year_blend: true` in their metadata are" + }, + { + "line": 96, + "text": "Parameters with ``preserve_calendar_dates: true`` retain statutory dates." + }, + { + "line": 106, + "text": "if (param.metadata or {}).get(\"preserve_calendar_dates\", False):" + }, + { + "line": 108, + "text": "blend = (param.metadata or {}).get(\"fiscal_year_blend\", False)" + } + ], + "policyengine_uk/utils/scenario.py": [ + { + "line": 89, + "text": "def from_reform(cls, reform: Union[tuple, dict, Type[Reform]]) -> \"Scenario\":" + }, + { + "line": 132, + "text": "if target.metadata.get(\"preserve_calendar_dates\")" + } + ] + }, + "supported_calendar_years": [ + 2024, + 2030 + ] +} diff --git a/docs/uk_replay/OFFLINE_BUNDLE_AUDIT_dev-uk-data-1.56.16__pe-uk-2.102.3.md b/docs/uk_replay/OFFLINE_BUNDLE_AUDIT_dev-uk-data-1.56.16__pe-uk-2.102.3.md new file mode 100644 index 00000000..723de426 --- /dev/null +++ b/docs/uk_replay/OFFLINE_BUNDLE_AUDIT_dev-uk-data-1.56.16__pe-uk-2.102.3.md @@ -0,0 +1,47 @@ +# Offline loader audit: development 6.2.5 bundle + +The [machine-readable source receipt](OFFLINE_BUNDLE_AUDIT_dev-uk-data-1.56.16__pe-uk-2.102.3.json) +binds the installed loader, its packaged manifest, the engine source and +the development pin. The pin remains in ignored `.venv-replay-checks/dev6/`; +the production index contains only the historical 2.89.2 bundle. + +Line references below are to the installed 6.2.5 wheel. This inspection +imports no engine, reads no credentials, makes no provider calls and runs no +simulation. Separate smoke receipts record executed behavior. + +| Source and lines | Finding | +| --- | --- | +| `policyengine/core/tax_benefit_model_version.py:137–221` | `release_bundle` contains model/runtime loader, certified build, artifact digest and certification fields. The build ID `policyengine-uk-data-1.56.16` differs from HF tag `1.56.16`; both are checked separately. | +| `policyengine/provenance/dataset_materialization.py:90–121` | Repository type comes from the dataset reference or data package. This release uses **model**, not dataset. | +| Same file, `124–144, 260–334` | Managed materialization reuses `./data/` only after digest verification. On absent or wrong bytes it uses `requests.get`, independent of HF offline flags. The replay seeds that path from the cached artifact and blocks runtime socket connections. | +| `policyengine/tax_benefit_models/uk/model.py:418–442` | `materialize_dataset` supplies the exact runtime input and provenance. `reform=` and `scenario=` pass through to the country Microsimulation. | +| `policyengine/tax_benefit_models/common/model_version.py:68–91` | `runtime_dataset_source` is the materializer's local path, with model repo type, revision and hash. The adapter resolves this path while in the isolated materialization directory and requires equality to the file hashed before and after simulation. | +| `policyengine_uk/data/dataset_schema.py:53–61, 197–209` | HDFStore opens with `mode="r"`; a writable PyTables mirror is unnecessary. An isolated ignored per-key directory links the already cached H5 and saves disk space. | +| `policyengine/provenance/manifest.py:339–368` | Release-metadata fetching calls `requests.get` with a timeout and bypasses HF offline flags. Blocked sockets turn this into `DataReleaseManifestUnavailableError`. | +| Same file, `393–435` | Fallback returns packaged certification when `certified_for_model_version` equals runtime. Other cases may return an `unverified` certification; the replay refuses them. | + +The package certifies UK 2.102.3 against data built originally with UK 2.89.2, +using `legacy_compatible_model_package`. Its global package pins supply +core 3.32.10. Registration checks those current packaged model/core pins, +the packaged digest, cached producer-declared artifact size, installed +versions and the stored data year, rather than mistaking producer build pins +for recertification pins. + +One metadata discrepancy needs upstream review by the main session: the +packaged `release_manifest_revision` is `78788372fec47fc8df835bd32307ef61eaed83c2`, +whose cached root release manifest describes 1.57.4. The 1.56.16 tag resolves +to `a9e52499b6a6cca100a5ce4f36ca27b2e8a213df`, whose cached producer manifest +describes the exact 1.56.16 artifact. Registration records both authorities +and checks the matching producer artifact declaration for size; the runtime +socket block makes the packaged compatibility claim binding. No upstream +issue was filed. Recheck this metadata in the final release. + +The cache allowlist is only +`models--policyengine--policyengine-uk-data-private/refs/1.56.16` plus +`snapshots/a9e52499b6a6cca100a5ce4f36ca27b2e8a213df/enhanced_frs_2024_25.h5`. +Modal mounts verified bytes directly at that snapshot path. No token, +credentials, other revision or dataset is forwarded. Package installation +occurs during image build; runtime network remains blocked. + +The year and projection evidence is in the +[development years document](YEARS_dev-uk-data-1.56.16__pe-uk-2.102.3.md). diff --git a/docs/uk_replay/RECIPE.md b/docs/uk_replay/RECIPE.md index 380aa252..11a3ab4e 100644 --- a/docs/uk_replay/RECIPE.md +++ b/docs/uk_replay/RECIPE.md @@ -648,3 +648,136 @@ The [construction audit](CONSTRUCTION_AUDIT.md) records the certified SDLT purchase-price proxy and pooled CGT gains, with a hash-guarded inspection command. These data-flow limits remain separate from model-error diagnoses; their national contributions are unsized. + +## Rerunning on another certified bundle + +`data/uk/certified_bundles/index.json` defaults to +`populace-uk-2023__pe-uk-2.89.2`, pointing at the original, unchanged pin. +Omitting `--bundle` preserves the historical registry, results, receipts and +summary paths. A selected key uses +`data/uk/events/bundles//_measures.json` and +`results/uk/events/bundles///`; its summary sits beside those +event directories. Pins and receipts bind the key, pin digest, dataset, +model/core/loader versions and year window. An explicit output directory is +useful for repeats; it cannot address another bundle's canonical namespace. + +The next 1.58.0 bundle is **not registered by this change**. Wait for +policyengine.py#555 to merge and its final release to be published, then use +that release's actual model and core versions. Do not substitute a newer +engine against the historical pin or reuse the 6.2.5 development pin as the +new event bundle. The [development loader audit](OFFLINE_BUNDLE_AUDIT_dev-uk-data-1.56.16__pe-uk-2.102.3.md) +and [year evidence](YEARS_dev-uk-data-1.56.16__pe-uk-2.102.3.md) explain the +tested adapter path, including the original-data-year projection, read-only +materializer and blocked-network packaged certification fallback. + +Define these placeholders before running the commands below: + +* `PE_REPOSITORY`: existing local checkout of policyengine.py containing the final tag. +* `PE_TAG`: final published managed-loader tag after #555. +* `BUNDLE_KEY`: a stable key naming the data/model pairing, for example `uk-data-1.58.0__pe-uk-`. +* `NEXT_VENV`: the main session's provisioned Python 3.12 environment for that release; `NEXT_PYTHON=$NEXT_VENV/bin/python`. +* `CONTROL_PYTHON=.venv-replay-checks/modal-control/bin/python`: the existing Modal 1.3.2 control interpreter. +* `FREEZE=docs/uk_replay/requirements-$BUNDLE_KEY.txt` and `AUDIT=docs/uk_replay/OFFLINE_BUNDLE_AUDIT_$BUNDLE_KEY.json`. +* `EVENT`: one of the five replay slugs, in the event order listed below. +* `MEASURE` and `YEAR`: an executable, in-force measure/year from the rebuilt registry, used for the determinism check. +* `CACHED_H5`: the already-cached artifact at the registered resolved commit; `SITE_PACKAGES`: the final environment's `purelib` path. + +These commands are main-session release setup, not permission to create new +environments or download data in the development lane. The exporter reads +only a local release tag and runs uv frozen/offline without a cache. Its +freeze includes the UK extra's complete lock and the loader itself; markers +are preserved, with Modal selecting the exact Linux Python 3.12 target. + +```bash +"$NEXT_PYTHON" -m pipeline.export_uk_bundle_requirements --repository "$PE_REPOSITORY" --tag "$PE_TAG" --key "$BUNDLE_KEY" +uv pip install --python "$NEXT_PYTHON" --no-deps -r "$FREEZE" +"$NEXT_PYTHON" -m pipeline.register_uk_certified_bundle --key "$BUNDLE_KEY" --requirements-freeze "$FREEZE" --offline-audit "$AUDIT" +"$NEXT_PYTHON" -m pipeline.audit_uk_bundle_loader --bundle "$BUNDLE_KEY" --output "$AUDIT" +``` + +Inspect the audit's installed-source references and record +`docs/uk_replay/YEARS_$BUNDLE_KEY.md`. Registration reads the H5's actual +data year and the installed extension's default end, not guessed constants. +The currently inspected window is 2024–2030. All earlier OBR cells remain in +the accounting and stage once as `outside_bundle_window`. The new loader's +managed call consumes the stored-year population and lets policyengine-uk +project it; never give it a projected year file as the observed data year. +Recheck fiscal conversion exceptions, dated Scenario behavior and UC input +modifier ordering at the final pin. + +Rebuild and inspect **all five** registries before starting national runs. +The sequence is Autumn Budget 2024, Autumn Statement 2023, Spring Budget +2024, Spring Statement 2025, Spring Budget 2023: + +```bash +for BUILD_EVENT in autumn_budget_2024 autumn_statement_2023 spring_budget_2024 spring_statement_2025 spring_budget_2023; do + "$NEXT_PYTHON" -m pipeline.build_uk_event_registry --event "$BUILD_EVENT" --bundle "$BUNDLE_KEY" + "$NEXT_PYTHON" -m pipeline.build_uk_event_registry --event "$BUILD_EVENT" --bundle "$BUNDLE_KEY" --check + "$NEXT_PYTHON" -m pipeline.compute_uk_event --event "$BUILD_EVENT" --bundle "$BUNDLE_KEY" --dry-run +done +"$NEXT_PYTHON" -m pipeline.audit_uk_event_constructions --bundle "$BUNDLE_KEY" --dataset "$CACHED_H5" --site-packages "$SITE_PACKAGES" --output ".venv-replay-checks/$BUNDLE_KEY/CONSTRUCTION_AUDIT.json" +"$NEXT_PYTHON" -m pipeline.diagnose_uk_event_models --bundle "$BUNDLE_KEY" --output ".venv-replay-checks/$BUNDLE_KEY/MODEL_DIAGNOSTICS.json" +for CASE in annual_allowance_relief_and_charge_pe_uk_2237 sdlt_additional_purchase_stock_pe_uk_2238 uc_standard_allowance_2027_2030_pe_uk_2239 hicbc_opt_out nics_threshold_freeze_end_date; do + .venv-replay/bin/python -m pipeline.diagnose_uk_event_models --bundle populace-uk-2023__pe-uk-2.89.2 --case "$CASE" --output ".venv-replay-checks/$BUNDLE_KEY/base_$CASE.json" + "$NEXT_PYTHON" -m pipeline.diagnose_uk_event_models --bundle "$BUNDLE_KEY" --case "$CASE" --output ".venv-replay-checks/$BUNDLE_KEY/new_$CASE.json" +done +``` + +List every classification or construction change against the base registry. +Reversal guards may expose changed parameter values or paths, particularly +SDLT, UC uplift counterfactuals, pension Annual Allowance, HICBC opt-out and +NIC threshold freeze end dates. An old construction's explanation is not a +new engine mechanism. Re-audit the construction before computing it. +Small diagnostic observations establish mechanisms; they do not size +national differences and count toward the two-simulation cap. + +Run a remote preflight with no simulation, then repeat the same nonzero +measure/year in two fresh ignored prefixes. Keep at most **two concurrent +invocations**, each with one worker, counting local and synthetic runs. +Plan around the observed roughly **70 minutes per full event**, then use +the new `RUN_LOG.json` and `MODAL_RECEIPT.json` timings to refine that estimate. + +```bash +"$CONTROL_PYTHON" -m pipeline.modal_uk_event --event "$EVENT" --bundle "$BUNDLE_KEY" --preflight-only --output-dir ".venv-replay-checks/modal/$BUNDLE_KEY/preflight" --download-root ".venv-replay-checks/modal-downloads/$BUNDLE_KEY/preflight" --execute +"$CONTROL_PYTHON" -m pipeline.modal_uk_event --event "$EVENT" --bundle "$BUNDLE_KEY" --measures "$MEASURE" --years "$YEAR" --output-dir ".venv-replay-checks/modal/$BUNDLE_KEY/repeat_a" --download-root ".venv-replay-checks/modal-downloads/$BUNDLE_KEY/repeat_a" --execute +"$CONTROL_PYTHON" -m pipeline.modal_uk_event --event "$EVENT" --bundle "$BUNDLE_KEY" --measures "$MEASURE" --years "$YEAR" --output-dir ".venv-replay-checks/modal/$BUNDLE_KEY/repeat_b" --download-root ".venv-replay-checks/modal-downloads/$BUNDLE_KEY/repeat_b" --execute +cmp ".venv-replay-checks/modal-downloads/$BUNDLE_KEY/repeat_a/.venv-replay-checks/modal/$BUNDLE_KEY/repeat_a/${MEASURE}_${YEAR}.json" ".venv-replay-checks/modal-downloads/$BUNDLE_KEY/repeat_b/.venv-replay-checks/modal/$BUNDLE_KEY/repeat_b/${MEASURE}_${YEAR}.json" +``` + +For each event in order, run, inspect the adoption report and adopt only +after its local run has stopped. These commands keep historical results +and the new namespace separate: + +```bash +"$CONTROL_PYTHON" -m pipeline.modal_uk_event --event "$EVENT" --bundle "$BUNDLE_KEY" --download-root ".venv-replay-checks/modal-downloads/$BUNDLE_KEY/full_$EVENT" --execute +"$NEXT_PYTHON" -m pipeline.adopt_uk_event --event "$EVENT" --bundle "$BUNDLE_KEY" --download-root ".venv-replay-checks/modal-downloads/$BUNDLE_KEY/full_$EVENT" > ".venv-replay-checks/${BUNDLE_KEY}_${EVENT}_adoption.json" +"$NEXT_PYTHON" -m pipeline.adopt_uk_event --event "$EVENT" --bundle "$BUNDLE_KEY" --download-root ".venv-replay-checks/modal-downloads/$BUNDLE_KEY/full_$EVENT" --adopt --local-run-stopped --backup-root ".venv-replay-checks/modal-adoption-backups/$BUNDLE_KEY/$EVENT" +"$NEXT_PYTHON" -m pipeline.stage_uk_event --event "$EVENT" --bundle "$BUNDLE_KEY" +"$NEXT_PYTHON" -m pipeline.compare_uk_event --event "$EVENT" --bundle "$BUNDLE_KEY" --summary +``` + +After all runs, hand-author +`data/uk/events/engine_attribution/populace-uk-2023__pe-uk-2.89.2__$BUNDLE_KEY.json`. +The schema example is **tests only** in +`tests/fixtures/uk_engine_comparison/attribution.json`. Every changed row +requires evidence-bearing drivers or an explicit `unattributed`. The closed +vocabulary is `hicbc_opt_out`, `annual_allowance_pe_uk_2237`, +`sdlt_pe_uk_2238`, `uc_uplifts_pe_uk_2239`, `nics_freeze`, `data_release`, +`construction_change`, `window_change`, `other_engine_change`, `unattributed`. +A driver is `sized: false` unless a hash-bound computed artifact supplies +its exact GBP value. Sized entries require an explicit `fy`, +`artifact: {path, sha256, value_path}` and `value_gbp` equal to the referenced +computed effect. A paired-run artifact also binds both executed numerical +endpoints with `base_artifact` and `new_artifact` path/hash pairs and their +bundle identities. Unsized drivers never contribute to an explained share. Then generate the descriptive table and full input-hash provenance: + +```bash +"$NEXT_PYTHON" -m pipeline.compare_uk_engines --base populace-uk-2023__pe-uk-2.89.2 --new "$BUNDLE_KEY" +``` + +It writes `results/uk/events/ENGINE_COMPARISON.md`, `.json`, `.csv` and +`ENGINE_COMPARISON_PROVENANCE.json`. Each source cell retains both bins, +the signed PE change, construction/window status, and the head's certified +aggregate ratio beside its measure-effect ratio. Ratios and named axes are +descriptive; the generator does not infer drivers or compute replication +rates. Construction changes and unavailable population years remain visible. diff --git a/docs/uk_replay/YEARS_dev-uk-data-1.56.16__pe-uk-2.102.3.md b/docs/uk_replay/YEARS_dev-uk-data-1.56.16__pe-uk-2.102.3.md new file mode 100644 index 00000000..059d94d8 --- /dev/null +++ b/docs/uk_replay/YEARS_dev-uk-data-1.56.16__pe-uk-2.102.3.md @@ -0,0 +1,53 @@ +# Development bundle: calendar years and loader path + +This is source evidence for the already-certified **policyengine 6.2.5 / +policyengine-uk 2.102.3 / core 3.32.10 / uk-data 1.56.16** adapter check. +It is not a registered production replay or an event result. The development +pin and observations live under ignored `.venv-replay-checks/dev6/`. + +The cached `enhanced_frs_2024_25.h5` has SHA-256 +`e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712`, +126,553,300 bytes, and `time_period` = `[2024]`. Reading the cached 1.58.0 +artifact's table independently also returned `[2024]`; its 128,369,110 bytes +and digest from the task were confirmed during registration-source inspection. +No simulation used that uncertified pairing. + +The following line references are to the installed wheels in +`.venv-replay-dev6/lib/python3.12/site-packages/`; their hashes and selected +line text are bound in the [offline loader audit](OFFLINE_BUNDLE_AUDIT_dev-uk-data-1.56.16__pe-uk-2.102.3.json). + +| File and lines | Verified behavior | +| --- | --- | +| `policyengine_uk/data/economic_assumptions.py:35–53` | The extension begins at `int(dataset.time_period)` and defaults to 2030. It applies engine uprating to copies of the original population. | +| `policyengine_uk/simulation.py:461–473` | A single-year input goes through that extension, then loads the resulting multi-year dataset. | +| `policyengine_uk/simulation.py:475–511` | Entity structure comes from the first year; each projected year's inputs are set explicitly. The managed check requires first year 2024 and the requested year present. | +| `policyengine_uk/utils/scenario.py:113–158` | A dictionary reform updates the processed parameter tree with `target.update`; it does not reload or fiscally process that tree. Dated windows therefore retain the 1 January rule implemented by `annual_reform`. | +| `policyengine_uk/simulation.py:134–136, 214–218` | `reform=` becomes a Scenario; its modifier runs after loading data. | +| `policyengine_uk/utils/parameters.py:78–123` | Default government conversion still samples 30 April. New exceptions support `fiscal_year_blend` (day-weighted 6 April–5 April) and `preserve_calendar_dates` (skip conversion). CGT's three rates and fuel-duty LPG/natural-gas parameters carry `fiscal_year_blend`; tobacco/alcohol rates carry `preserve_calendar_dates`. Reinspect the final pin before reusing these constructions. | +| `policyengine_uk/simulation.py:177–179, 214–218` | The default UC modifier precedes the dictionary modifier, so the UC health scenario still needs to refresh the fixed inputs after its parameter update. | +| `policyengine_uk/scenarios/uc_reform.py:69–96` | UC health inputs are fixed for 2026–2029. Standard-allowance uplifts now occur in the formula, rather than in this modifier. | +| `policyengine/tax_benefit_models/uk/model.py:397–443` | `managed_microsimulation` forwards `**kwargs` to the country Microsimulation and passes the materialized source dataset, then adds runtime provenance. | +| `policyengine/tax_benefit_models/uk/model.py:276–349` | The separate year-file wrapper preserves the observed data year and its tables. The replay uses the original managed dataset directly, so it never treats a projected year file as observed data (the issue addressed by [policyengine.py#557](https://github.com/PolicyEngine/policyengine.py/pull/557)). | + +This development window is **2024–2030**. Registry years, requested execution +years, Modal requests and staging derive it from the selected pin. Source FY +cells beginning before 2024 stay in the same source-row accounting exactly +once, labelled `outside_bundle_window` in a new-bundle stage. In particular, +FY2023–24 counterparts available on the legacy population cease to be +computable on this population. Calendar Y continues to proxy FY Y–(Y+1), +with explicit population, baseline and period axes. + +Current policyengine-uk main was also read at +`e89b8abcf524de48e590b9680e1a04288d30eacd` (2026-10-10): +[`data/economic_assumptions.py:55–74`](https://github.com/PolicyEngine/policyengine-uk/blob/e89b8abcf524de48e590b9680e1a04288d30eacd/policyengine_uk/data/economic_assumptions.py#L55) +still defaults the extension to 2030. This observation does not certify the +future model/core versions. Register the final loader release and repeat the +code and construction audit once policyengine.py#555 merges. + +The CGT main-rate construction changes on this development engine: its +October 2024 change is blended into CY2024 by `gov/hmrc/cgt/basic_rate.yaml:14` +(and the higher/additional-rate files). The new registry therefore restores +the pre-announcement rates from 1 January 2024 against that blended certified +world, rather than deferring the annual reversal to 2025. Its construction +digest changes and the old pooled-gains/residential-scope limitation remains. +This is a construction observation, not a computed event effect. diff --git a/pipeline/audit_uk_bundle_loader.py b/pipeline/audit_uk_bundle_loader.py new file mode 100644 index 00000000..191bd273 --- /dev/null +++ b/pipeline/audit_uk_bundle_loader.py @@ -0,0 +1,178 @@ +"""Hash installed 6.x loader evidence without importing or running the engine.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from importlib.metadata import distribution, version +from pathlib import Path + +from pipeline import uk_bundle + +SOURCES = { + "policyengine": [ + "core/tax_benefit_model_version.py", + "tax_benefit_models/uk/model.py", + "tax_benefit_models/common/model_version.py", + "provenance/manifest.py", + "provenance/dataset_materialization.py", + ], + "policyengine_uk": [ + "simulation.py", + "data/economic_assumptions.py", + "data/dataset_schema.py", + "utils/scenario.py", + "utils/parameters.py", + "scenarios/uc_reform.py", + "parameters/gov/hmrc/cgt/basic_rate.yaml", + "parameters/gov/hmrc/cgt/higher_rate.yaml", + "parameters/gov/hmrc/cgt/additional_rate.yaml", + "parameters/gov/hmrc/fuel_duty/lpg.yaml", + "parameters/gov/hmrc/fuel_duty/natural_gas.yaml", + "parameters/gov/hmrc/tobacco_duty/rates/cigarette_specific.yaml", + "parameters/gov/hmrc/alcohol_duty/rates/beer.yaml", + ], +} + + +def sha256(path): + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def audit(key): + pin = uk_bundle.load_bundle(key) + identities = uk_bundle.bundle_identity(key) + for package, expected in identities["engine_versions"].items(): + if version(package) != expected: + raise ValueError(f"installed {package} differs from bundle pin") + site = Path(distribution("policyengine").locate_file("")) + manifest_path = site / "policyengine/data/bundle/manifest.json" + packaged = json.loads(manifest_path.read_text()) + release = packaged["data_releases"]["uk"] + certification = release["certification"] + if ( + certification["certified_for_model_version"] + != identities["engine_versions"]["policyengine-uk"] + or release["certified_data_artifact"]["sha256"] != pin["sha256"] + or certification["data_build_id"] != pin["data_build_id"] + ): + raise ValueError("packaged certification differs from selected bundle") + sources = { + f"{package}/{relative}": sha256(site / package / relative) + for package, files in SOURCES.items() + for relative in files + } + references = {} + for name in sources: + lines = (site / name).read_text().splitlines() + references[name] = [ + {"line": i, "text": line.strip()} + for i, line in enumerate(lines, 1) + if any( + marker in line + for marker in ( + "def managed_microsimulation", + "def materialize_dataset", + "def _reuse_or_download", + "def release_bundle", + "runtime_dataset_source", + "def from_reform", + "def convert_to_fiscal", + "fiscal_year_blend", + "preserve_calendar_dates", + "end_year: int", + 'mode="r"', + "def build_from_single", + "def build_from_multi", + 'sim.set_input("uc_LCWRA', + "universal_credit_july_2025_reform.simulation_modifier", + "not scenario.applied_before", + "def get_data_release_manifest", + "return bundled_certification", + ) + ) + ] + kind = "models" if pin["repo_type"] == "model" else "datasets" + cache_directory = f"{kind}--{pin['repo_id'].replace('/', '--')}" + return { + "schema_version": 2, + "inspection": "Engine-free installed-source inspection; no provider, credentials, network or simulation.", + "development_bundle": bool(pin.get("development_bundle")), + "policyengine_version": pin["managed_loader_version"], + "certified_identity": { + **{ + field: pin[field] + for field in ( + "repo_id", + "repo_type", + "revision", + "artifact", + "sha256", + "size_bytes", + "data_year", + "resolved_hf_commit", + ) + }, + "model_version": identities["engine_versions"]["policyengine-uk"], + "core_version": identities["engine_versions"]["policyengine-core"], + "dataset_uri": release["default_dataset_uri"], + "data_build_id": pin["data_build_id"], + }, + "bundled_certification": certification, + "packaged_bundle_manifest_sha256": sha256(manifest_path), + "source_files_sha256": sources, + "source_references": references, + "release_manifest_revision": pin["release_manifest_revision"], + "registration_provenance_hashes": pin.get("provenance_hashes", {}), + "minimal_hf_cache_allowlist": { + "repo_cache_directory": cache_directory, + "required_ref": { + "path": f"refs/{pin['revision']}", + "contents": pin["resolved_hf_commit"], + }, + "required_snapshot_file": { + "path": f"snapshots/{pin['resolved_hf_commit']}/{pin['artifact']}" + }, + "exclude": [ + "tokens", + "credentials", + "refs/main", + "other revisions", + "other artifacts", + ], + }, + "offline_loader_behavior": { + "materialization": "Reuse hash-verified ./data/; requests downloads bypass HF offline flags on a missing or mismatched file.", + "release_metadata": "requests.get bypasses HF offline flags; socket blocking forces packaged certification fallback.", + "fallback": "Must retain packaged certified_for_model_version matching the runtime; unverified fallback is refused.", + "runtime_dataset_source": "The source.path returned by materialize_dataset is recorded by build_runtime_dataset_provenance and must resolve to the pre/post hashed file.", + "read_only_h5": "Single-year and multi-year constructors use HDFStore(mode='r'); no writable dataset copy is required.", + "projection": "managed_microsimulation passes the original stored-year dataset to the engine; no projected year-file re-anchoring.", + }, + **identities, + } + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--bundle", required=True) + parser.add_argument("--output", type=Path) + args = parser.parse_args(argv) + document = audit(args.bundle) + output = ( + args.output + or uk_bundle.ROOT / uk_bundle.load_bundle(args.bundle)["offline_audit"] + ) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(document, indent=1, sort_keys=True) + "\n") + print(f"{output}: {sha256(output)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/smoke_uk_bundle_adapter.py b/pipeline/smoke_uk_bundle_adapter.py new file mode 100644 index 00000000..b33eabd5 --- /dev/null +++ b/pipeline/smoke_uk_bundle_adapter.py @@ -0,0 +1,80 @@ +"""Sequential, offline adapter observations; never fiscal-event results.""" + +from __future__ import annotations + +import argparse +import json +import time +from pathlib import Path + +from pipeline import compute_uk_event as compute +from pipeline import uk_bundle + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--bundle", required=True) + parser.add_argument("--year", type=int, default=2026) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args(argv) + output = args.output_dir.resolve() + checks = (compute.ROOT / ".venv-replay-checks").resolve() + if not output.is_relative_to(checks): + raise ValueError("adapter receipts must be under ignored .venv-replay-checks") + if args.bundle == uk_bundle.DEFAULT_BUNDLE: + raise ValueError( + "this smoke check requires an explicitly selected development bundle" + ) + pin = uk_bundle.load_bundle(args.bundle) + first, last = uk_bundle.bundle_window(pin) + if not first <= args.year <= last: + raise ValueError("smoke year outside bundle window") + output.mkdir(parents=True, exist_ok=True) + started = time.perf_counter() + pre = compute.preflight(args.bundle) + (output / "PREFLIGHT.json").write_bytes(compute.canonical_bytes(pre)) + observations = {} + reform = { + "gov.hmrc.income_tax.allowances.personal_allowance.amount": { + f"{args.year}-01-01.{args.year}-12-31": 12571, + } + } + for name, parameters in (("baseline", None), ("dated_reform", reform)): + print(f"Starting {name} adapter observation", flush=True) + result = compute.run_sim(args.year, ["income_tax"], parameters, pre) + observations[name] = result + (output / f"{name.upper()}.json").write_bytes(compute.canonical_bytes(result)) + effect = ( + observations["dated_reform"]["aggregates_gbp"]["income_tax"] + - observations["baseline"]["aggregates_gbp"]["income_tax"] + ) + if not effect < 0: + raise RuntimeError("the dated £1 allowance increase did not reduce income tax") + receipt = { + "kind": "development_adapter_observation", + "year": args.year, + "head_variable": "income_tax", + "reform": reform, + "income_tax_change_gbp": effect, + "runtime_dataset_source_hashed": pre["runtime_dataset_source"], + "runtime_dataset_source_resolved": str( + Path(pre["runtime_dataset_source"]).resolve() + ), + "dataset_sha256": pre["sha256"], + "data_year": pin["data_year"], + "runtime_network_blocked": pre["runtime_network_blocked"], + "wall_seconds": time.perf_counter() - started, + "inputs_and_observations": { + p.name: compute.fiscal.sha256_file(p) + for p in sorted(output.glob("*.json")) + if p.name != "SMOKE_RECEIPT.json" + }, + **uk_bundle.bundle_identity(args.bundle), + } + (output / "SMOKE_RECEIPT.json").write_bytes(compute.canonical_bytes(receipt)) + print(json.dumps(receipt, indent=1, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From a858a69390b506dd9fe4c0658ee483635f5f3029 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 08:02:23 -0400 Subject: [PATCH 04/11] Pass --bundle through the staging CLI's identity check stage_uk_event.main validated the registry against the default bundle, so staging any non-default bundle failed with "bundle identity mismatch: bundle_key". The library-level tests call stage_event directly and missed it. This adds a CLI-level regression test that fails without the fix. Found by staging the first real policyengine 6.2.6 event run. Refs #156. Co-Authored-By: Claude Opus 5.5 --- pipeline/stage_uk_event.py | 2 +- tests/test_uk_event_bundle_staging.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/pipeline/stage_uk_event.py b/pipeline/stage_uk_event.py index f61a978e..bda6df33 100644 --- a/pipeline/stage_uk_event.py +++ b/pipeline/stage_uk_event.py @@ -458,7 +458,7 @@ def main(argv: list[str] | None = None) -> int: json.loads(manifest_path.read_bytes()) if manifest_path.exists() else None ) registry = json.loads(payload) - compute.validate_event_identity(registry, args.event) + compute.validate_event_identity(registry, args.event, args.bundle) compute.registry_builder.validate_registry(registry) rows, tally = stage_event( registry, diff --git a/tests/test_uk_event_bundle_staging.py b/tests/test_uk_event_bundle_staging.py index 32e098ca..33f59c97 100644 --- a/tests/test_uk_event_bundle_staging.py +++ b/tests/test_uk_event_bundle_staging.py @@ -160,6 +160,29 @@ def test_stage_rejects_foreign_bundle_compute_manifest(pinned_stage, field): ) +def test_stage_cli_validates_the_registry_against_the_selected_bundle( + pinned_stage, monkeypatch +): + """The CLI must pass --bundle to identity validation, not the default.""" + _registry, _manifest, registry_path, output, _root = pinned_stage + seen = {} + + class Stop(Exception): + pass + + def capture(registry, event, bundle=bundles.DEFAULT_BUNDLE): + seen["bundle"] = bundle + raise Stop + + monkeypatch.setattr(bundles, "registry_path", lambda event, key: registry_path) + monkeypatch.setattr(bundles, "event_output_dir", lambda event, key: output) + monkeypatch.setattr(bundles, "validate_bundle_path", lambda *a, **k: a[0]) + monkeypatch.setattr(staging.compute, "validate_event_identity", capture) + with pytest.raises(Stop): + staging.main(["--event", "event", "--bundle", DEV]) + assert seen["bundle"] == DEV + + def test_stage_refuses_foreign_registry_or_window(pinned_stage): registry, manifest, _, output, root = pinned_stage registry["bundle"] = bundles.bundle_document() From 8c8ca524b93cceccc004b61001f0e6ae102d00ad Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 08:14:47 -0400 Subject: [PATCH 05/11] Read the Modal receipt's bundle identity from its request compare_uk_engines checked the bundle identity at the top level of MODAL_RECEIPT.json, but the Modal wrapper records it under "request". Every non-default bundle with a real receipt was refused ("missing bundle identity field"). The default bundle tolerates missing fields, and no fixture wrote a receipt, so tests never reached this check. The two-bundle setup now lives in a shared helper, and a new test covers receipts in the real shape plus a foreign bundle key. Found by running the first real 6.2.6 comparison. Refs #156. Co-Authored-By: Claude Opus 5.5 --- pipeline/compare_uk_engines.py | 3 +- tests/test_uk_event_bundle_staging.py | 66 +++++++++++++++++++++++++-- 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/pipeline/compare_uk_engines.py b/pipeline/compare_uk_engines.py index 7fc82899..1f0eee72 100644 --- a/pipeline/compare_uk_engines.py +++ b/pipeline/compare_uk_engines.py @@ -747,8 +747,9 @@ def _verify_run_receipts( modal = directory / "MODAL_RECEIPT.json" if modal.exists(): commit(modal) + # The Modal wrapper records the bundle identity in its request. bundles.validate_bundle_identity( - json.loads(modal.read_bytes()), bundle, root=root + json.loads(modal.read_bytes())["request"], bundle, root=root ) diff --git a/tests/test_uk_event_bundle_staging.py b/tests/test_uk_event_bundle_staging.py index 33f59c97..8cc5c436 100644 --- a/tests/test_uk_event_bundle_staging.py +++ b/tests/test_uk_event_bundle_staging.py @@ -292,9 +292,8 @@ def test_path_selection_rejects_other_bundles_including_default(pinned_stage, ki ) -def test_engine_generator_binds_every_input_and_writes_deterministically(pinned_stage): - from pipeline import compare_uk_engines as engines - +def two_bundle_comparison(pinned_stage): + """Stage and compare one event under DEV and a second development bundle.""" registry, _manifest, registry_path, output, root = pinned_stage _, _, staged = save_stage(pinned_stage) comparison.write_comparison( @@ -391,6 +390,39 @@ def test_engine_generator_binds_every_input_and_writes_deterministically(pinned_ } ) ) + return { + "registry_path": registry_path, + "output": output, + "root": root, + "staged": staged, + "other": other, + "pin": pin, + "other_pin_path": other_pin_path, + "index_path": index_path, + "other_registry_path": other_registry_path, + "other_output": other_output, + "axes": axes, + "attribution_path": attribution_path, + } + + +def test_engine_generator_binds_every_input_and_writes_deterministically(pinned_stage): + from pipeline import compare_uk_engines as engines + + setup = two_bundle_comparison(pinned_stage) + registry_path, output, root, staged = ( + setup["registry_path"], + setup["output"], + setup["root"], + setup["staged"], + ) + other, pin, index_path = setup["other"], setup["pin"], setup["index_path"] + other_pin_path = setup["other_pin_path"] + other_registry_path, other_output = ( + setup["other_registry_path"], + setup["other_output"], + ) + axes, attribution_path = setup["axes"], setup["attribution_path"] rows = engines.write_engine_comparison( DEV, other, attribution_path=attribution_path, artifact_root=root ) @@ -456,6 +488,34 @@ def test_engine_generator_binds_every_input_and_writes_deterministically(pinned_ } +def test_engine_generator_reads_bundle_identity_from_the_modal_request(pinned_stage): + """Real Modal receipts carry the bundle identity under ``request``.""" + from pipeline import compare_uk_engines as engines + + setup = two_bundle_comparison(pinned_stage) + root, other = setup["root"], setup["other"] + for key, directory in ((DEV, setup["output"]), (other, setup["other_output"])): + receipt = { + "schema_version": 1, + "request": { + "event": "event", + **bundles.bundle_identity(key, root=root), + }, + } + (directory / "MODAL_RECEIPT.json").write_text(json.dumps(receipt)) + rows = engines.write_engine_comparison( + DEV, other, attribution_path=setup["attribution_path"], artifact_root=root + ) + assert len(rows) == 2 + foreign = json.loads((setup["other_output"] / "MODAL_RECEIPT.json").read_text()) + foreign["request"]["bundle_key"] = DEV + (setup["other_output"] / "MODAL_RECEIPT.json").write_text(json.dumps(foreign)) + with pytest.raises(ValueError, match="bundle identity mismatch: bundle_key"): + engines.write_engine_comparison( + DEV, other, attribution_path=setup["attribution_path"], artifact_root=root + ) + + def test_sized_paired_artifact_verifies_real_endpoints(pinned_stage): from pipeline import compare_uk_engines as engines From f5cd3107e8855a05f0b8530cebd8efc9976decb7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 08:49:04 -0400 Subject: [PATCH 06/11] Guard both protected trees for every replay output path validate_bundle_path only checked a path against its own kind's root, so a results output could land in the registry tree. The review's example was the default staging CLI with --output set to another bundle's registry. It now checks data/uk/events and results/uk/events for every kind: a path in either tree must sit in the selected bundle's namespace, and of the matching kind. The diagnostics CLI also checks --output and --report before any engine work, and a non-default bundle can't write the default bundle's MODEL_INVESTIGATIONS.md. (Review findings 2 and 3 on #159.) Refs #156. Co-Authored-By: Claude Opus 5.5 --- pipeline/diagnose_uk_event_models.py | 14 ++++++++- pipeline/uk_bundle.py | 39 ++++++++++++++++-------- tests/test_uk_event_bundle_staging.py | 43 +++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 14 deletions(-) diff --git a/pipeline/diagnose_uk_event_models.py b/pipeline/diagnose_uk_event_models.py index 70e73c21..6889f864 100644 --- a/pipeline/diagnose_uk_event_models.py +++ b/pipeline/diagnose_uk_event_models.py @@ -621,6 +621,9 @@ def investigate(case: str | None = None, bundle=DEFAULT_BUNDLE) -> list[dict]: return results +DEFAULT_REPORT = Path("docs/uk_replay/MODEL_INVESTIGATIONS.md") + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--case") @@ -633,6 +636,16 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--output", type=Path) parser.add_argument("--report", type=Path) args = parser.parse_args(argv) + # Check both destinations before any engine work or write. + for destination in (args.output, args.report): + if destination is not None: + validate_bundle_path(destination, args.bundle, kind="results") + if ( + args.report is not None + and args.bundle != DEFAULT_BUNDLE + and args.report.resolve() == (ROOT / DEFAULT_REPORT).resolve() + ): + parser.error("the default bundle's investigation report is not writable here") results = ( json.loads(args.from_json.read_text()) if args.from_json @@ -647,7 +660,6 @@ def main(argv: list[str] | None = None) -> int: parser.error("unknown diagnostic case") payload = json.dumps(results, indent=1, sort_keys=True, allow_nan=False) + "\n" if args.output: - validate_bundle_path(args.output, args.bundle, kind="results") args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(payload) else: diff --git a/pipeline/uk_bundle.py b/pipeline/uk_bundle.py index 5c82d84d..1953b69d 100644 --- a/pipeline/uk_bundle.py +++ b/pipeline/uk_bundle.py @@ -209,22 +209,35 @@ def validate_runtime_bundle(document, key=DEFAULT_BUNDLE, *, root=ROOT): return identity +PROTECTED_TREES = {"registry": "data/uk/events", "results": "results/uk/events"} + + def validate_bundle_path(path, key=DEFAULT_BUNDLE, *, kind, root=ROOT): + """Refuse any path inside a protected tree that isn't this bundle's namespace. + + Both canonical trees are checked whatever ``kind`` is, so an output can't + land in the registry tree (or another bundle's results) just because it + lies outside its own kind's root. + """ _key(key) - canonical = Path(root) / ( - "data/uk/events" if kind == "registry" else "results/uk/events" - ) - if kind not in ("registry", "results"): + if kind not in PROTECTED_TREES: raise ValueError("invalid bundle path kind") - try: - relative = Path(path).resolve().relative_to(canonical.resolve()) - except ValueError: - return Path(path) # Explicit development or determinism-check directory. - parts = relative.parts - actual = parts[1] if len(parts) > 1 and parts[0] == "bundles" else DEFAULT_BUNDLE - if actual != key: - raise ValueError("path belongs to another bundle") - return Path(path) + resolved = Path(path).resolve() + for owner, tree in PROTECTED_TREES.items(): + try: + relative = resolved.relative_to((Path(root) / tree).resolve()) + except ValueError: + continue + if owner != kind: + raise ValueError(f"{kind} path lies inside the protected {owner} tree") + parts = relative.parts + actual = ( + parts[1] if len(parts) > 1 and parts[0] == "bundles" else DEFAULT_BUNDLE + ) + if actual != key: + raise ValueError("path belongs to another bundle") + return Path(path) + return Path(path) # Explicit development or determinism-check directory. def validate_registry_bundle(registry, key=DEFAULT_BUNDLE, *, root=ROOT): diff --git a/tests/test_uk_event_bundle_staging.py b/tests/test_uk_event_bundle_staging.py index 8cc5c436..c187690e 100644 --- a/tests/test_uk_event_bundle_staging.py +++ b/tests/test_uk_event_bundle_staging.py @@ -292,6 +292,49 @@ def test_path_selection_rejects_other_bundles_including_default(pinned_stage, ki ) +@pytest.mark.parametrize("key", [DEV, bundles.DEFAULT_BUNDLE]) +def test_paths_cannot_enter_the_other_protected_tree(pinned_stage, key): + """A results output can't overwrite a registry, whichever bundle it names.""" + _, _, _, _, root = pinned_stage + for owner in (DEV, bundles.DEFAULT_BUNDLE): + registry = bundles.registry_path("event", owner, root=root) + with pytest.raises(ValueError, match="protected registry tree"): + bundles.validate_bundle_path(registry, key, kind="results", root=root) + results = bundles.event_output_dir("event", owner, root=root) / "x.json" + with pytest.raises(ValueError, match="protected results tree"): + bundles.validate_bundle_path(results, key, kind="registry", root=root) + own = bundles.event_output_dir("event", key, root=root) / "STAGED.jsonl" + assert bundles.validate_bundle_path(own, key, kind="results", root=root) == own + scratch = root / ".venv-replay-checks/anywhere.json" + assert ( + bundles.validate_bundle_path(scratch, key, kind="results", root=root) == scratch + ) + + +def test_diagnostic_report_cannot_overwrite_another_bundles_outputs(tmp_path): + from pipeline import diagnose_uk_event_models as diagnose + + saved = tmp_path / "observations.json" + saved.write_text("[]") + for report in ( + diagnose.ROOT / "results/uk/events/SUMMARY.md", + diagnose.ROOT / diagnose.DEFAULT_REPORT, + ): + before = report.read_bytes() if report.exists() else None + with pytest.raises((ValueError, SystemExit)): + diagnose.main( + [ + "--bundle", + "some-other-bundle", + "--from-json", + str(saved), + "--report", + str(report), + ] + ) + assert (report.read_bytes() if report.exists() else None) == before + + def two_bundle_comparison(pinned_stage): """Stage and compare one event under DEV and a second development bundle.""" registry, _manifest, registry_path, output, root = pinned_stage From bcd568cdd5d6d425ddfbaa1c6b21145ab3eca91a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 08:51:12 -0400 Subject: [PATCH 07/11] Mount and hash only the selected bundle's index entries The Modal input manifest hashed the whole shared bundle index, so registering any later bundle changed every earlier run's input commitment. That made adoption of a completed run fail although none of its inputs had changed. The wrapper now hashes and mounts a deterministic index holding only the default mapping and the selected bundle's mapping. It is serialized exactly as registration writes the index, so a run made while no other bundle was registered keeps its commitment. (Review finding 6 on #159.) Refs #156. Co-Authored-By: Claude Opus 5.5 --- pipeline/modal_uk_event.py | 52 ++++++++++++++++++++++++++++++++++-- tests/test_modal_uk_event.py | 33 +++++++++++++++++++++++ 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/pipeline/modal_uk_event.py b/pipeline/modal_uk_event.py index c43aed42..4fb02b03 100644 --- a/pipeline/modal_uk_event.py +++ b/pipeline/modal_uk_event.py @@ -145,6 +145,37 @@ def bundle_index_relative(root: Path) -> str: return bundle_relative(path if path.is_absolute() else root / path, root) +def selected_index_bytes(bundle: str = DEFAULT_BUNDLE, *, root: Path = ROOT) -> bytes: + """The index entries this run needs, serialized as registration writes them. + + Mounting and hashing the whole shared index would let registering an + unrelated bundle change every earlier run's input commitment. Only the + default mapping and the selected bundle's mapping are kept. + """ + path = uk_bundle.bundle_index_path(root=root) + index = ( + json.loads(path.read_text()) + if path.exists() + else { + "schema_version": 1, + "default_bundle": DEFAULT_BUNDLE, + "bundles": {DEFAULT_BUNDLE: "data/uk/certified_bundle.json"}, + } + ) + if bundle not in index["bundles"]: + raise ValueError(f"unregistered UK bundle: {bundle}") + selected = { + "schema_version": index["schema_version"], + "default_bundle": index["default_bundle"], + "bundles": { + key: value + for key, value in index["bundles"].items() + if key in (DEFAULT_BUNDLE, bundle) + }, + } + return (json.dumps(selected, indent=2, sort_keys=True) + "\n").encode() + + def bundle_file(bundle: str, field: str, *, root: Path = ROOT) -> str: return relative_path(uk_bundle.load_bundle(bundle, root=root)[field]) @@ -359,6 +390,8 @@ def input_manifest( # unrelated blob, release metadata, refs/main or token file is transmitted. mounts[ref_relative] = ref mounts[artifact_relative] = artifact.resolve(strict=True) + index_relative = bundle_index_relative(root) + index_payload = selected_index_bytes(bundle, root=root) rows = [] for relative, source in sorted(mounts.items()): is_artifact = relative == artifact_relative @@ -371,8 +404,14 @@ def input_manifest( rows.append( { "path": relative, - "size_bytes": source.stat().st_size, - "sha256": cert["sha256"] if is_artifact else sha256_file(source), + "size_bytes": len(index_payload) + if relative == index_relative + else source.stat().st_size, + "sha256": cert["sha256"] + if is_artifact + else digest_bytes(index_payload) + if relative == index_relative + else sha256_file(source), "kind": "certified_artifact" if is_artifact else "input", } ) @@ -665,7 +704,16 @@ def modal_function(mounts: dict[str, Path], bundle: str = DEFAULT_BUNDLE): } ) ) + index_relative = bundle_index_relative(ROOT) for relative, path in sorted(mounts.items()): + if relative == index_relative: + # Mount exactly the bytes the input manifest hashed. + selected = tempfile.NamedTemporaryFile( + prefix="replay-index-", suffix=".json", delete=False + ) + selected.write(selected_index_bytes(bundle, root=ROOT)) + selected.close() + path = Path(selected.name) image = image.add_local_file(path, f"{REMOTE_ROOT}/{relative}", copy=False) app = modal.App("scorecard-uk-event-replay", include_source=False) diff --git a/tests/test_modal_uk_event.py b/tests/test_modal_uk_event.py index d8eb8f27..98989eb8 100644 --- a/tests/test_modal_uk_event.py +++ b/tests/test_modal_uk_event.py @@ -75,6 +75,12 @@ def inputs(tmp_path): return tmp_path, manifest, mounts +def upload_index(root, bundle=runner.DEFAULT_BUNDLE): + """Emulate the mount: the remote index holds only the selected mappings.""" + path = root / "data/uk/certified_bundles/index.json" + path.write_bytes(runner.selected_index_bytes(bundle, root=root)) + + def test_allowlist_mounts_only_pinned_inputs(inputs): root, manifest, mounts = inputs assert len(mounts) == len(runner.repository_files("autumn_budget_2024")) + 2 @@ -83,6 +89,7 @@ def test_allowlist_mounts_only_pinned_inputs(inputs): ) assert not any(".git" in Path(path).parts for path in mounts) assert sum(row["kind"] == "certified_artifact" for row in manifest["files"]) == 1 + upload_index(root) runner.verify_inputs(manifest, root=root) @@ -182,6 +189,7 @@ def test_model_bundle_selects_exact_cache_freeze_audit_and_registry(model_bundle assert not any( "datasets--policyengine--populace-uk-private" in path for path in mounts ) + upload_index(root, key) runner.verify_inputs(manifest, bundle=key, root=root) @@ -562,3 +570,28 @@ def concurrent(**kwargs): for key in recorded["env"] ) assert len(recorded["mounts"]) == len(mounts) + 1 # build-time frozen requirements + + +def test_registering_another_bundle_keeps_existing_input_commitments(inputs): + """Only the selected mappings are mounted and hashed, not the shared index.""" + root, manifest, _ = inputs + index_path = root / "data/uk/certified_bundles/index.json" + index = json.loads(index_path.read_text()) + index["bundles"]["unrelated-later-bundle"] = "data/uk/certified_bundles/later.json" + index_path.write_text(json.dumps(index, indent=2, sort_keys=True) + "\n") + hf_cache = root / runner.HF_CACHE_RELATIVE + again, _ = runner.input_manifest("autumn_budget_2024", root=root, hf_cache=hf_cache) + assert again == manifest + selected = json.loads(runner.selected_index_bytes(root=root)) + assert set(selected["bundles"]) == {runner.DEFAULT_BUNDLE} + with pytest.raises(ValueError, match="unregistered"): + runner.selected_index_bytes("not-registered", root=root) + + +def test_selected_index_matches_registration_bytes_for_the_same_mappings(inputs): + """With nothing unrelated registered, the mounted index is the index file.""" + root, _, _ = inputs + index_path = root / "data/uk/certified_bundles/index.json" + index = json.loads(index_path.read_text()) + index_path.write_text(json.dumps(index, indent=2, sort_keys=True) + "\n") + assert runner.selected_index_bytes(root=root) == index_path.read_bytes() From 3b7e1d333972548c95853a50d01988ec4350e1da Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 08:51:53 -0400 Subject: [PATCH 08/11] Block DNS lookups and datagram sends in the replay runtime The audit hook denied only socket.connect, so name resolution (getaddrinfo and friends) and UDP sendto/sendmsg still reached the network. The hook now denies those events too. A child-process test checks that DNS, UDP and TCP all fail with the block message while a local socketpair (used by multiprocessing) still works. (Review finding 8 on #159.) Refs #156. Co-Authored-By: Claude Opus 5.5 --- pipeline/compute_uk_obr_costings.py | 17 ++++++++++- tests/test_uk_bundle_adapter.py | 46 +++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/pipeline/compute_uk_obr_costings.py b/pipeline/compute_uk_obr_costings.py index 2d312dc9..1c9712e2 100644 --- a/pipeline/compute_uk_obr_costings.py +++ b/pipeline/compute_uk_obr_costings.py @@ -885,11 +885,26 @@ def assert_engine_matches_bundle( return installed +# Name resolution and datagram sends happen before (or without) a connect, +# so blocking connect alone still let DNS lookups and UDP traffic through. +BLOCKED_SOCKET_EVENTS = frozenset( + { + "socket.connect", + "socket.getaddrinfo", + "socket.gethostbyname", + "socket.gethostbyaddr", + "socket.getnameinfo", + "socket.sendto", + "socket.sendmsg", + } +) + + def block_runtime_network() -> None: """Make the packaged certification fallback mandatory in this process.""" def deny_network(event, args): - if event == "socket.connect": + if event in BLOCKED_SOCKET_EVENTS: raise OSError("certified replay runtime network is blocked") sys.addaudithook(deny_network) diff --git a/tests/test_uk_bundle_adapter.py b/tests/test_uk_bundle_adapter.py index 74039fe8..e1c4bae9 100644 --- a/tests/test_uk_bundle_adapter.py +++ b/tests/test_uk_bundle_adapter.py @@ -2,6 +2,7 @@ import hashlib import json +import os import sys from pathlib import Path from types import SimpleNamespace @@ -523,3 +524,48 @@ def test_output_paths_cannot_cross_registered_bundle_namespaces(adapter): / f"results/uk/events/bundles/{adapter.key}/autumn_budget_2024", bundles.DEFAULT_BUNDLE, ) + + +def test_runtime_network_block_covers_dns_datagrams_and_connects(): + """Audit hooks are permanent, so exercise the block in a child process.""" + import subprocess + import sys + import textwrap + + probe = textwrap.dedent( + """ + import socket + from pipeline import compute_uk_obr_costings as fiscal + + left, right = socket.socketpair() # local IPC still works + left.sendall(b"ok"); assert right.recv(2) == b"ok" + fiscal.block_runtime_network() + attempts = { + "dns": lambda: socket.getaddrinfo("localhost", 80), + "udp": lambda: socket.socket(socket.AF_INET, socket.SOCK_DGRAM).sendto( + b"x", ("127.0.0.1", 9) + ), + "tcp": lambda: socket.create_connection(("127.0.0.1", 9), timeout=1), + } + for name, attempt in attempts.items(): + try: + attempt() + except OSError as exc: + assert "network is blocked" in str(exc), (name, exc) + else: + raise SystemExit(f"{name} was not blocked") + left2, right2 = socket.socketpair() + left2.sendall(b"ok"); assert right2.recv(2) == b"ok" + print("blocked") + """ + ) + result = subprocess.run( + [sys.executable, "-c", probe], + cwd=Path(__file__).resolve().parents[1], + env={**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[1])}, + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "blocked" From 7284ef7f4d9b2cc5f106ed35ca50a9fbbb3eaf75 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 09:09:04 -0400 Subject: [PATCH 09/11] Size attribution only from verified runs, and never twice A sized driver previously needed only a hash-correct artifact with the right arithmetic. A dry-run artifact from another engine and dataset could therefore produce an explained share. Sized evidence must now be an artifact listed in a verified run manifest of a compared bundle and must pass that bundle's runtime-identity check; this covers a computed artifact and both endpoints of a paired run. The writer verifies every bundle's receipt chain before it builds any row. Sized drivers within one entry also can't cover the same heads of the same measured endpoints. A two-head paired size beside its one-head subset, or a measure total beside one of its heads, is rejected instead of being added up. (Review findings 4 and 5 on #159.) Refs #156. Co-Authored-By: Claude Opus 5.5 --- pipeline/compare_uk_engines.py | 78 ++++++++++- tests/test_uk_engine_comparison.py | 186 +++++++++++++++++++++++++- tests/test_uk_event_bundle_staging.py | 25 +++- 3 files changed, 276 insertions(+), 13 deletions(-) diff --git a/pipeline/compare_uk_engines.py b/pipeline/compare_uk_engines.py index 1f0eee72..154ff1a7 100644 --- a/pipeline/compare_uk_engines.py +++ b/pipeline/compare_uk_engines.py @@ -199,16 +199,34 @@ def _entry_index(attribution: dict) -> dict[tuple[str, str, str | None], list[di return out +def _verified_bundle(reference: str, verified: dict[str, str]) -> str: + """Sized evidence must be an executed artifact from a verified run manifest.""" + if reference not in verified: + raise EngineComparisonError( + "sized evidence must be an artifact listed in a verified run manifest" + ) + return verified[reference] + + def validate_attribution( attribution: dict, *, artifact_root: Path = ROOT, input_hashes: dict[str, str] | None = None, + verified_artifacts: dict[str, str] | None = None, ) -> dict: - """Validate all entries, even entries not used by a changed row.""" + """Validate all entries, even entries not used by a changed row. + + ``verified_artifacts`` maps each artifact path (relative to the root) from + a verified run manifest to its bundle. Without it, no driver can be sized. + """ + verified = verified_artifacts or {} out = _entry_index(attribution) for (event, measure_key, fy), drivers in out.items(): sized_terms = set() + # Heads already sized from each measured endpoint, so no two drivers + # (or a total and one of its heads) count the same contribution twice. + measured_heads: dict[tuple, set[str]] = {} for driver in drivers: evidence = driver.get("evidence") if not isinstance(evidence, list) or not evidence: @@ -267,6 +285,11 @@ def validate_attribution( raise EngineComparisonError("sized artifact SHA-256 differs") computed = json.loads(path.read_bytes()) if "head_effects" in computed: + bundles.validate_runtime_bundle( + computed, + _verified_bundle(reference, verified), + root=artifact_root, + ) event_comparison.validate_artifact(computed, reference) if ( computed.get("event"), @@ -286,6 +309,12 @@ def validate_attribution( "sized artifact value must be a computed GBP effect" ) term = ("computed", proof["sha256"], tuple(allowed)) + measured = ("computed", proof["sha256"]) + heads = ( + {allowed[1]} + if allowed[0] == "head_effects" + else set(computed["head_effects"]) + ) elif ( computed.get("artifact_type") == "paired_run" and computed.get("status") == "computed" @@ -337,6 +366,16 @@ def validate_attribution( raise EngineComparisonError( "paired-run endpoint requires bundle identity" ) + if ( + _verified_bundle( + str(endpoint_path.relative_to(artifact_root.resolve())), + verified, + ) + != endpoint_bundle + ): + raise EngineComparisonError( + "paired-run endpoint belongs to another verified bundle" + ) bundles.validate_runtime_bundle( endpoint, endpoint_bundle, root=artifact_root ) @@ -375,6 +414,12 @@ def validate_attribution( computed["new_artifact"]["sha256"], tuple(sorted(paired_heads)), ) + measured = ( + "paired_run", + computed["base_artifact"]["sha256"], + computed["new_artifact"]["sha256"], + ) + heads = set(paired_heads) else: raise EngineComparisonError( "sized evidence is not a computed national artifact" @@ -402,6 +447,11 @@ def validate_attribution( "sized drivers repeat the same computed term" ) sized_terms.add(term) + if measured_heads.setdefault(measured, set()) & heads: + raise EngineComparisonError( + "sized drivers overlap on the same measured heads" + ) + measured_heads[measured] |= heads if input_hashes is not None: input_hashes[reference] = proof["sha256"] return out @@ -438,6 +488,7 @@ def build_engine_rows( new_artifacts: dict[str, dict], artifact_root: Path = ROOT, input_hashes: dict | None = None, + verified_artifacts: dict[str, str] | None = None, ) -> list[dict]: """Join the exact source universe; neither gaps nor zero rows are dropped.""" left, right = _inventory(base_registry), _inventory(new_registry) @@ -456,7 +507,10 @@ def build_engine_rows( base_measures = {m["measure_key"]: m for m in base_registry["measures"]} new_measures = {m["measure_key"]: m for m in new_registry["measures"]} drivers = validate_attribution( - attribution, artifact_root=artifact_root, input_hashes=input_hashes + attribution, + artifact_root=artifact_root, + input_hashes=input_hashes, + verified_artifacts=verified_artifacts, ) rows = [] for identity in sorted(left): @@ -682,6 +736,7 @@ def _verify_run_receipts( bundle: str, root: Path, commit, + verified: dict[str, str] | None = None, ) -> None: """Bind the numerical manifest and each per-year progress receipt.""" from pipeline import stage_uk_event as staging @@ -700,6 +755,8 @@ def _verify_run_receipts( "compute manifest artifact path or SHA-256 differs" ) commit(path) + if verified is not None: + verified[str(path.resolve().relative_to(root.resolve()))] = bundle years.add(json.loads(path.read_bytes())["year"]) for year in sorted(years): progress_path = directory / f"RUN_PROGRESS_{year}.json" @@ -814,7 +871,10 @@ def commit(path): raise EngineComparisonError( "both bundles require comparisons for every event in the same nonempty registry inventory" ) - rows = [] + # Verify every bundle's receipt chain first, so sized attribution for any + # event can only cite artifacts from a verified run manifest. + verified_artifacts: dict[str, str] = {} + loaded = {} for event in sorted(event_sets[base]): registries, comparisons, artifacts = {}, {}, {} for key in (base, new): @@ -838,7 +898,13 @@ def commit(path): registry_path = artifact_root / provenance["registry_path"] registries[key] = json.loads(registry_path.read_bytes()) _verify_run_receipts( - directory, registry_path, registries[key], key, artifact_root, commit + directory, + registry_path, + registries[key], + key, + artifact_root, + commit, + verified_artifacts, ) source = registries[key].get("source", {}) if source.get("claims_path"): @@ -856,6 +922,9 @@ def commit(path): artifacts[key][row["artifact"]] = json.loads( artifact_path.read_bytes() ) + loaded[event] = (registries, comparisons, artifacts) + rows = [] + for event, (registries, comparisons, artifacts) in loaded.items(): rows.extend( build_engine_rows( registries[base], @@ -867,6 +936,7 @@ def commit(path): new_artifacts=artifacts[new], artifact_root=artifact_root, input_hashes=inputs, + verified_artifacts=verified_artifacts, ) ) payloads = { diff --git a/tests/test_uk_engine_comparison.py b/tests/test_uk_engine_comparison.py index 449b8b0a..4bb6f629 100644 --- a/tests/test_uk_engine_comparison.py +++ b/tests/test_uk_engine_comparison.py @@ -108,6 +108,17 @@ def run(values, aggregates, name): ) +def verified(root, *names): + """Treat ``names`` as artifacts from a verified default-bundle manifest.""" + from pipeline import uk_bundle as bundles + + pin = root / "data/uk/certified_bundle.json" + if not pin.exists(): + pin.parent.mkdir(parents=True) + pin.write_bytes(bundles.bundle_pin_path().read_bytes()) + return {name: bundles.DEFAULT_BUNDLE for name in names} + + def compare(values=None, attribution=None, **kwargs): left, right, before, after, old, new = values or inputs() return engines.build_engine_rows( @@ -188,15 +199,86 @@ def test_sized_driver_requires_matching_computed_artifact_hash_and_value(tmp_pat }, ) hashes = {} - compare(values, attribution, artifact_root=tmp_path, input_hashes=hashes) + receipts = verified(tmp_path, "paired.json") + compare( + values, + attribution, + artifact_root=tmp_path, + input_hashes=hashes, + verified_artifacts=receipts, + ) assert hashes == {"paired.json": hashlib.sha256(payload).hexdigest()} + # The same artifact outside a verified run manifest sizes nothing. + with pytest.raises(engines.EngineComparisonError, match="verified run manifest"): + compare(values, attribution, artifact_root=tmp_path) driver["value_gbp"] += 1 with pytest.raises(engines.EngineComparisonError, match="differs"): - compare(values, attribution, artifact_root=tmp_path) + compare( + values, attribution, artifact_root=tmp_path, verified_artifacts=receipts + ) driver["value_gbp"] -= 1 driver["artifact"]["sha256"] = "0" * 64 with pytest.raises(engines.EngineComparisonError, match="SHA-256"): - compare(values, attribution, artifact_root=tmp_path) + compare( + values, attribution, artifact_root=tmp_path, verified_artifacts=receipts + ) + + +def test_sized_artifact_from_another_engine_is_rejected(tmp_path): + """A hash-correct artifact still has to match its bundle's runtime pins.""" + values = inputs() + artifact = {**values[5]["new.json"], "engine_version": "9.9.9"} + payload = engines.canonical_bytes(artifact) + (tmp_path / "foreign.json").write_bytes(payload) + attribution = json.loads(FIXTURE.read_text()) + attribution["entries"][0]["fy"] = "2024-25" + attribution["entries"][0]["drivers"][0].update( + sized=True, + value_gbp=120, + artifact={ + "path": "foreign.json", + "sha256": hashlib.sha256(payload).hexdigest(), + "value_path": ["head_effects", "head_0"], + }, + ) + with pytest.raises(ValueError, match="runtime bundle mismatch: engine_version"): + compare( + values, + attribution, + artifact_root=tmp_path, + verified_artifacts=verified(tmp_path, "foreign.json"), + ) + + +def test_total_and_one_of_its_heads_cannot_both_be_sized(tmp_path): + values = inputs() + payload = engines.canonical_bytes(values[5]["new.json"]) + (tmp_path / "artifact.json").write_bytes(payload) + proof = {"path": "artifact.json", "sha256": hashlib.sha256(payload).hexdigest()} + attribution = json.loads(FIXTURE.read_text()) + entry = attribution["entries"][0] + entry["fy"] = "2024-25" + entry["drivers"][0].update( + sized=True, + value_gbp=95, + artifact={**proof, "value_path": ["measure_total_gbp"]}, + ) + entry["drivers"].append( + { + "driver": "other_engine_change", + "evidence": [{"kind": "paired_run", "reference": "artifact.json"}], + "sized": True, + "value_gbp": 120, + "artifact": {**proof, "value_path": ["head_effects", "head_0"]}, + } + ) + with pytest.raises(engines.EngineComparisonError, match="overlap"): + compare( + values, + attribution, + artifact_root=tmp_path, + verified_artifacts=verified(tmp_path, "artifact.json"), + ) def test_unsized_driver_withholds_explained_share_even_beside_sized(tmp_path): @@ -218,7 +300,12 @@ def test_unsized_driver_withholds_explained_share_even_beside_sized(tmp_path): }, } ) - rows = compare(values, attribution, artifact_root=tmp_path) + rows = compare( + values, + attribution, + artifact_root=tmp_path, + verified_artifacts=verified(tmp_path, "artifact.json"), + ) assert all(r["explained_share"] is None for r in rows) assert all( r["explained_share_withheld"] == "unsized drivers" @@ -353,7 +440,12 @@ def test_named_drivers_cannot_double_count_one_computed_term(tmp_path): second = {**copy.deepcopy(first), "driver": "other_engine_change"} entry["drivers"].append(second) with pytest.raises(engines.EngineComparisonError, match="same computed term"): - compare(values, attribution, artifact_root=tmp_path) + compare( + values, + attribution, + artifact_root=tmp_path, + verified_artifacts=verified(tmp_path, "artifact.json"), + ) @pytest.mark.parametrize("scope", [None, ["head_0"]]) @@ -409,7 +501,10 @@ def test_paired_package_size_is_not_allocated_to_each_source_head( "value_path": ["effect_gbp"], }, ) - rows = compare(values, attribution, artifact_root=tmp_path) + receipts = verified(tmp_path, "measured-base.json", "measured-new.json") + rows = compare( + values, attribution, artifact_root=tmp_path, verified_artifacts=receipts + ) tax = [row for row in rows if row["measure_key"] == "event__tax"] if scope is None: assert all(row["explained_share"] is None for row in tax) @@ -420,3 +515,82 @@ def test_paired_package_size_is_not_allocated_to_each_source_head( tax[1]["explained_share_withheld"] == "measure sizing does not isolate this source head" ) + + +def test_paired_head_subset_cannot_be_sized_beside_its_superset(tmp_path): + """Review case: a two-head paired size plus its one-head subset.""" + from pipeline import uk_bundle as bundles + + measured = inputs(old=(100, 100), new=(110, 110)) + receipts = verified(tmp_path, "measured-base.json", "measured-new.json") + endpoints = {} + for side, artifact in ( + ("base", measured[4]["base.json"]), + ("new", measured[5]["new.json"]), + ): + payload = engines.canonical_bytes(artifact) + (tmp_path / f"measured-{side}.json").write_bytes(payload) + endpoints[side] = { + "path": f"measured-{side}.json", + "sha256": hashlib.sha256(payload).hexdigest(), + } + drivers = [] + for name, driver, scope, effect in ( + ("both.json", "data_release", ["head_0", "head_1"], 20), + ("subset.json", "other_engine_change", ["head_0"], 10), + ): + paired = { + "artifact_type": "paired_run", + "status": "computed", + "effect_gbp": effect, + "base_artifact": endpoints["base"], + "new_artifact": endpoints["new"], + "base_bundle": bundles.DEFAULT_BUNDLE, + "new_bundle": bundles.DEFAULT_BUNDLE, + "head_variables": scope, + } + payload = engines.canonical_bytes(paired) + (tmp_path / name).write_bytes(payload) + drivers.append( + { + "driver": driver, + "evidence": [{"kind": "paired_run", "reference": name}], + "sized": True, + "value_gbp": effect, + "artifact": { + "path": name, + "sha256": hashlib.sha256(payload).hexdigest(), + "value_path": ["effect_gbp"], + }, + } + ) + attribution = json.loads(FIXTURE.read_text()) + attribution["entries"][0].update(fy="2024-25", drivers=drivers) + values = inputs(old=(100, 100), new=(120, 120)) + with pytest.raises(engines.EngineComparisonError, match="overlap"): + compare( + values, attribution, artifact_root=tmp_path, verified_artifacts=receipts + ) + # Disjoint heads from the same endpoints remain valid. + drivers[0]["artifact"]["path"] = "other.json" + other = { + **json.loads((tmp_path / "both.json").read_bytes()), + "head_variables": ["head_1"], + "effect_gbp": 10, + } + payload = engines.canonical_bytes(other) + (tmp_path / "other.json").write_bytes(payload) + drivers[0]["artifact"]["sha256"] = hashlib.sha256(payload).hexdigest() + drivers[0]["value_gbp"] = 10 + rows = compare( + values, attribution, artifact_root=tmp_path, verified_artifacts=receipts + ) + # Each driver is measured on one head only, so it is unmeasured for the + # other head's row and the explained share stays withheld there. + tax = [row for row in rows if row["measure_key"] == "event__tax"] + assert all(row["explained_share"] is None for row in tax) + assert all( + row["explained_share_withheld"] + == "measure sizing does not isolate this source head" + for row in tax + ) diff --git a/tests/test_uk_event_bundle_staging.py b/tests/test_uk_event_bundle_staging.py index c187690e..dafe5405 100644 --- a/tests/test_uk_event_bundle_staging.py +++ b/tests/test_uk_event_bundle_staging.py @@ -625,8 +625,23 @@ def test_sized_paired_artifact_verifies_real_endpoints(pinned_stage): ], } hashes = {} - engines.validate_attribution(attribution, artifact_root=root, input_hashes=hashes) + verified = {"base.json": DEV, "new.json": DEV} + engines.validate_attribution( + attribution, + artifact_root=root, + input_hashes=hashes, + verified_artifacts=verified, + ) assert set(hashes) == {"base.json", "new.json", "paired.json"} + # Endpoints must come from a verified run manifest of the named bundle. + with pytest.raises(engines.EngineComparisonError, match="verified run manifest"): + engines.validate_attribution(attribution, artifact_root=root) + with pytest.raises(engines.EngineComparisonError, match="another verified bundle"): + engines.validate_attribution( + attribution, + artifact_root=root, + verified_artifacts={**verified, "new.json": bundles.DEFAULT_BUNDLE}, + ) # Copying a paired receipt under another name does not create a second # independently sized contribution from the same executed endpoints. second_pair = {**paired, "note": "another named mechanism, same numerical term"} @@ -641,11 +656,15 @@ def test_sized_paired_artifact_verifies_real_endpoints(pinned_stage): ) attribution["entries"][0]["drivers"].append(second_driver) with pytest.raises(engines.EngineComparisonError, match="same computed term"): - engines.validate_attribution(attribution, artifact_root=root) + engines.validate_attribution( + attribution, artifact_root=root, verified_artifacts=verified + ) attribution["entries"][0]["drivers"].pop() (root / "new.json").write_bytes((root / "new.json").read_bytes() + b"\n") with pytest.raises(engines.EngineComparisonError, match="endpoint SHA-256"): - engines.validate_attribution(attribution, artifact_root=root) + engines.validate_attribution( + attribution, artifact_root=root, verified_artifacts=verified + ) def test_summary_rejects_another_bundles_model_observations(pinned_stage): From a6d1bb5cea425a68037819cd5fac3cb60bf3adbc Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 09:10:16 -0400 Subject: [PATCH 10/11] Derive construction-audit limits from the engine and dataset The audit's CGT and SDLT findings were fixed prose written for 2.89.2. On policyengine-uk 2.125.1 the "no asset-type, BADR or carried-interest branch" statement is false: the formula has those branches, and it is the dataset that lacks their inputs. Non-default bundles now get each statement from the installed engine's source and the H5's columns, with formula capability and population inputs reported separately. The historical bundle keeps its committed findings verbatim. (Review finding 7 on #159.) Refs #156. Co-Authored-By: Claude Opus 5.5 --- pipeline/audit_uk_event_constructions.py | 156 +++++++++++++++++----- tests/test_uk_event_construction_audit.py | 60 +++++++++ 2 files changed, 179 insertions(+), 37 deletions(-) diff --git a/pipeline/audit_uk_event_constructions.py b/pipeline/audit_uk_event_constructions.py index a38e54d5..1d8b41a6 100644 --- a/pipeline/audit_uk_event_constructions.py +++ b/pipeline/audit_uk_event_constructions.py @@ -177,6 +177,122 @@ def inspect_h5(path: Path) -> dict: } +GAIN_SUBTYPE_INPUTS = ( + "capital_gains_badr", + "capital_gains_residential_property", + "capital_gains_carried_interest", +) +CGT_FORMULA = ( + "policyengine_uk/variables/gov/hmrc/capital_gains_tax/capital_gains_tax.py" +) +ADDITIONAL_PURCHASE = "policyengine_uk/variables/household/consumption/additional_residential_property_purchased.py" + + +def legacy_construction_limits() -> list[dict]: + """The 2.89.2 audit's findings, kept verbatim for the historical bundle.""" + return [ + { + "id": "sdlt_flagged_stock_transaction_proxy", + "variables": [ + "property_purchased", + "other_residential_property_value", + "additional_residential_property_purchased", + ], + "finding": "The bundle supplies property stock and a purchase flag, but no direct main/additional purchase prices. Additional purchase price is stock multiplied by that flag. This is a transaction-price proxy; full stock is not taxed unconditionally.", + "annual_projection": "Population extension copies the purchase flag; per-capita GDP uprating grows property values. There is no annual purchaser resampling in this path.", + "aggregate_scope": "Base-year raw input aggregates precede SDLT geography, minimum-price thresholds, annual uprating and reform aggregation. Raw row counts include capital-gains clones.", + "divergence_axes": [ + "construction_scope", + "head_scope", + "population_vintage", + ], + "national_contribution": "unsized", + }, + { + "id": "cgt_pooled_gain_types_and_zero_elasticity", + "variables": [ + "capital_gains", + "capital_gains_before_response", + "capital_gains_behavioural_response", + "capital_gains_tax", + ], + "finding": "The tax formula applies the main CGT rate schedule to one pooled gains amount, with no asset-type, BADR, Investors' Relief or carried-interest branch. The bundle has one capital_gains input. The loader moves it to before_response; the default zero elasticity makes the response formula return zero.", + "annual_projection": "Both capital_gains and capital_gains_before_response are uprated by per-capita GDP. No gain-type allocation is supplied by this path.", + "divergence_axes": [ + "construction_scope", + "head_scope", + "behavioural_adjustment", + "population_vintage", + ], + "national_contribution": "unsized", + }, + ] + + +def engine_construction_limits(site_packages: Path, observations: dict) -> list[dict]: + """Derive each statement from this engine's source and this dataset's columns. + + The formula's capability and the population's inputs are reported + separately: a branch the engine implements can still be inert because the + dataset doesn't supply its input. + """ + columns = observations["input_columns"] + supplied = set(columns["household"]) | set(columns["person"]) + cgt_source = (site_packages / CGT_FORMULA).read_text() + formula_reads = [name for name in GAIN_SUBTYPE_INPUTS if f'"{name}"' in cgt_source] + dataset_has = [name for name in formula_reads if name in supplied] + missing = [name for name in formula_reads if name not in supplied] + if not formula_reads: + cgt_finding = "The tax formula reads one pooled capital_gains amount and no separate gain-type input." + elif missing: + cgt_finding = ( + "The tax formula reads separate gain-type inputs (" + + ", ".join(formula_reads) + + "). The certified dataset does not supply " + + ", ".join(missing) + + ", so those branches take their default and the corresponding gains are taxed from the pooled capital_gains input." + ) + else: + cgt_finding = ( + "The tax formula reads separate gain-type inputs (" + + ", ".join(formula_reads) + + ") and the certified dataset supplies all of them." + ) + purchase_source = (site_packages / ADDITIONAL_PURCHASE).read_text() + derived = "def formula" in purchase_source + has_purchase_input = "additional_residential_property_purchased" in supplied + if derived: + sdlt_finding = "additional_residential_property_purchased has a formula in this engine; it is derived, not read from the dataset." + elif has_purchase_input: + sdlt_finding = "additional_residential_property_purchased is an input in this engine and the certified dataset supplies it." + else: + sdlt_finding = "additional_residential_property_purchased is an input in this engine and the certified dataset does not supply it, so it takes its default." + return [ + { + "id": "sdlt_flagged_stock_transaction_proxy", + "variables": [ + "property_purchased", + "other_residential_property_value", + "additional_residential_property_purchased", + ], + "finding": sdlt_finding, + "additional_purchase_is_derived": derived, + "dataset_supplies_additional_purchase": has_purchase_input, + "source_file": ADDITIONAL_PURCHASE, + "national_contribution": "unsized", + }, + { + "id": "cgt_pooled_gain_types_and_zero_elasticity", + "variables": ["capital_gains", *GAIN_SUBTYPE_INPUTS, "capital_gains_tax"], + "finding": cgt_finding, + "formula_reads_gain_types": formula_reads, + "dataset_supplies_gain_types": dataset_has, + "source_file": CGT_FORMULA, + "national_contribution": "unsized", + }, + ] + + def collect_audit(dataset: Path, site_packages: Path, bundle: dict) -> dict: identity = verify_dataset(dataset, bundle) versions = { @@ -209,43 +325,9 @@ def collect_audit(dataset: Path, site_packages: Path, bundle: dict) -> dict: "certified_dataset": identity, "observations": observations, "source_evidence": evidence, - "construction_limits": [ - { - "id": "sdlt_flagged_stock_transaction_proxy", - "variables": [ - "property_purchased", - "other_residential_property_value", - "additional_residential_property_purchased", - ], - "finding": "The bundle supplies property stock and a purchase flag, but no direct main/additional purchase prices. Additional purchase price is stock multiplied by that flag. This is a transaction-price proxy; full stock is not taxed unconditionally.", - "annual_projection": "Population extension copies the purchase flag; per-capita GDP uprating grows property values. There is no annual purchaser resampling in this path.", - "aggregate_scope": "Base-year raw input aggregates precede SDLT geography, minimum-price thresholds, annual uprating and reform aggregation. Raw row counts include capital-gains clones.", - "divergence_axes": [ - "construction_scope", - "head_scope", - "population_vintage", - ], - "national_contribution": "unsized", - }, - { - "id": "cgt_pooled_gain_types_and_zero_elasticity", - "variables": [ - "capital_gains", - "capital_gains_before_response", - "capital_gains_behavioural_response", - "capital_gains_tax", - ], - "finding": "The tax formula applies the main CGT rate schedule to one pooled gains amount, with no asset-type, BADR, Investors' Relief or carried-interest branch. The bundle has one capital_gains input. The loader moves it to before_response; the default zero elasticity makes the response formula return zero.", - "annual_projection": "Both capital_gains and capital_gains_before_response are uprated by per-capita GDP. No gain-type allocation is supplied by this path.", - "divergence_axes": [ - "construction_scope", - "head_scope", - "behavioural_adjustment", - "population_vintage", - ], - "national_contribution": "unsized", - }, - ], + "construction_limits": legacy_construction_limits() + if key == DEFAULT_BUNDLE + else engine_construction_limits(site_packages, observations), "interpretation": "These observations document construction and data-flow limits. They do not establish national model errors, causal residual amounts or an explained share. No new model diagnostic is asserted.", } diff --git a/tests/test_uk_event_construction_audit.py b/tests/test_uk_event_construction_audit.py index 520d7e6a..d9705183 100644 --- a/tests/test_uk_event_construction_audit.py +++ b/tests/test_uk_event_construction_audit.py @@ -106,3 +106,63 @@ def test_saved_receipt_has_certified_identity_and_keeps_contributions_unsized(): ] == "9547795632c1da3cacba2f11b8b109082e02e18c74cc35e5a726cb504507ba25" ) + + +def engine_tree(root, *, cgt_reads=(), purchase_has_formula=True): + """A minimal site-packages with only the two source files the audit reads.""" + cgt = root / audit.CGT_FORMULA + cgt.parent.mkdir(parents=True) + cgt.write_text( + 'gains = person("capital_gains", period)\n' + + "".join(f'person("{name}", period)\n' for name in cgt_reads) + ) + purchase = root / audit.ADDITIONAL_PURCHASE + purchase.parent.mkdir(parents=True) + purchase.write_text( + "class additional_residential_property_purchased(Variable):\n" + + ( + " def formula(household, period):\n ...\n" + if purchase_has_formula + else " pass\n" + ) + ) + return root + + +def test_the_historical_bundle_keeps_its_committed_construction_limits(): + committed = json.loads( + (audit.ROOT / "results/uk/events/CONSTRUCTION_AUDIT.json").read_text() + ) + assert audit.legacy_construction_limits() == committed["construction_limits"] + + +@pytest.mark.parametrize("supplied", [(), audit.GAIN_SUBTYPE_INPUTS]) +def test_cgt_limit_separates_formula_capability_from_dataset_inputs(tmp_path, supplied): + """An engine can implement gain-type branches the population can't feed.""" + site = engine_tree(tmp_path, cgt_reads=audit.GAIN_SUBTYPE_INPUTS) + observations = { + "input_columns": {"household": [], "person": ["capital_gains", *supplied]} + } + cgt = audit.engine_construction_limits(site, observations)[1] + assert cgt["formula_reads_gain_types"] == list(audit.GAIN_SUBTYPE_INPUTS) + assert cgt["dataset_supplies_gain_types"] == list(supplied) + assert ("does not supply" in cgt["finding"]) == (not supplied) + pooled = engine_tree(tmp_path / "old") + old = audit.engine_construction_limits(pooled, observations)[1] + assert old["formula_reads_gain_types"] == [] + assert "no separate gain-type input" in old["finding"] + + +@pytest.mark.parametrize("has_formula", [True, False]) +@pytest.mark.parametrize("in_dataset", [True, False]) +def test_sdlt_limit_reports_whether_the_purchase_is_derived_or_supplied( + tmp_path, has_formula, in_dataset +): + site = engine_tree(tmp_path, purchase_has_formula=has_formula) + columns = ["additional_residential_property_purchased"] if in_dataset else [] + observations = {"input_columns": {"household": columns, "person": []}} + sdlt = audit.engine_construction_limits(site, observations)[0] + assert sdlt["additional_purchase_is_derived"] is has_formula + assert sdlt["dataset_supplies_additional_purchase"] is in_dataset + if not has_formula and not in_dataset: + assert "does not supply it" in sdlt["finding"] From 7d172b996f184251d7ebe46bdc8440d8fc1cdc23 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 09:50:18 -0400 Subject: [PATCH 11/11] Close the three gaps the delta review found One head, one sized driver. A computed artifact and a paired run that ends at it could still size the same head, because the two proof kinds were tracked under different identities. Within an entry, each head can now be sized by at most one driver, whatever proof it cites. The engine comparison's output directory is checked before any work. It may be the shared results/uk/events directory or a path outside both protected trees, never a registry, an event's results or a bundle namespace. The diagnostics CLI refuses the default bundle's investigation report for --output as well as --report. Its test now runs against a temporary root: with the guard removed, the earlier version really did overwrite the tracked report. Refs #156. Co-Authored-By: Claude Opus 5.5 --- pipeline/compare_uk_engines.py | 38 ++++++++++----- pipeline/diagnose_uk_event_models.py | 8 +-- tests/test_uk_engine_comparison.py | 65 +++++++++++++++++++++++++ tests/test_uk_event_bundle_staging.py | 70 ++++++++++++++++++++++++--- 4 files changed, 157 insertions(+), 24 deletions(-) diff --git a/pipeline/compare_uk_engines.py b/pipeline/compare_uk_engines.py index 154ff1a7..f6c5bec7 100644 --- a/pipeline/compare_uk_engines.py +++ b/pipeline/compare_uk_engines.py @@ -224,9 +224,11 @@ def validate_attribution( out = _entry_index(attribution) for (event, measure_key, fy), drivers in out.items(): sized_terms = set() - # Heads already sized from each measured endpoint, so no two drivers - # (or a total and one of its heads) count the same contribution twice. - measured_heads: dict[tuple, set[str]] = {} + # Each head is sized by at most one driver per entry, whatever kind of + # proof it cites. A computed artifact and a paired run that ends at it + # (or a total and one of its heads) would otherwise count one + # contribution twice. + sized_heads: set[str] = set() for driver in drivers: evidence = driver.get("evidence") if not isinstance(evidence, list) or not evidence: @@ -309,7 +311,6 @@ def validate_attribution( "sized artifact value must be a computed GBP effect" ) term = ("computed", proof["sha256"], tuple(allowed)) - measured = ("computed", proof["sha256"]) heads = ( {allowed[1]} if allowed[0] == "head_effects" @@ -414,11 +415,6 @@ def validate_attribution( computed["new_artifact"]["sha256"], tuple(sorted(paired_heads)), ) - measured = ( - "paired_run", - computed["base_artifact"]["sha256"], - computed["new_artifact"]["sha256"], - ) heads = set(paired_heads) else: raise EngineComparisonError( @@ -447,11 +443,11 @@ def validate_attribution( "sized drivers repeat the same computed term" ) sized_terms.add(term) - if measured_heads.setdefault(measured, set()) & heads: + if sized_heads & heads: raise EngineComparisonError( "sized drivers overlap on the same measured heads" ) - measured_heads[measured] |= heads + sized_heads |= heads if input_hashes is not None: input_hashes[reference] = proof["sha256"] return out @@ -810,6 +806,23 @@ def _verify_run_receipts( ) +def validate_output_dir(output_dir: Path, artifact_root: Path) -> None: + """Allow the shared comparison directory or a path outside both trees. + + A directory inside the registry tree, an event's results or any bundle's + namespace is refused, so a comparison can't overwrite their files. + """ + resolved = Path(output_dir).resolve() + if resolved == (artifact_root / "results/uk/events").resolve(): + return + for tree in bundles.PROTECTED_TREES.values(): + if resolved.is_relative_to((artifact_root / tree).resolve()): + raise EngineComparisonError( + "engine comparison output would enter a protected registry or " + "bundle namespace" + ) + + def write_engine_comparison( base: str, new: str, @@ -819,6 +832,8 @@ def write_engine_comparison( artifact_root: Path = ROOT, ) -> list[dict]: """Verify both comparison receipt chains and bind every used input hash.""" + output_dir = output_dir or artifact_root / "results/uk/events" + validate_output_dir(output_dir, artifact_root) bundles.load_bundle(base, root=artifact_root) bundles.load_bundle(new, root=artifact_root) attribution_path = ( @@ -948,7 +963,6 @@ def commit(path): for reference, digest in inputs.items(): if sha256(artifact_root / reference) != digest: raise EngineComparisonError("input changed during engine comparison") - output_dir = output_dir or artifact_root / "results/uk/events" output_dir.mkdir(parents=True, exist_ok=True) for name, payload in payloads.items(): atomic_write_bytes(output_dir / name, payload) diff --git a/pipeline/diagnose_uk_event_models.py b/pipeline/diagnose_uk_event_models.py index 6889f864..4a973d7e 100644 --- a/pipeline/diagnose_uk_event_models.py +++ b/pipeline/diagnose_uk_event_models.py @@ -640,10 +640,10 @@ def main(argv: list[str] | None = None) -> int: for destination in (args.output, args.report): if destination is not None: validate_bundle_path(destination, args.bundle, kind="results") - if ( - args.report is not None - and args.bundle != DEFAULT_BUNDLE - and args.report.resolve() == (ROOT / DEFAULT_REPORT).resolve() + reserved = (ROOT / DEFAULT_REPORT).resolve() + if args.bundle != DEFAULT_BUNDLE and any( + destination is not None and destination.resolve() == reserved + for destination in (args.output, args.report) ): parser.error("the default bundle's investigation report is not writable here") results = ( diff --git a/tests/test_uk_engine_comparison.py b/tests/test_uk_engine_comparison.py index 4bb6f629..fca67989 100644 --- a/tests/test_uk_engine_comparison.py +++ b/tests/test_uk_engine_comparison.py @@ -594,3 +594,68 @@ def test_paired_head_subset_cannot_be_sized_beside_its_superset(tmp_path): == "measure sizing does not isolate this source head" for row in tax ) + + +def test_computed_and_paired_proofs_cannot_size_the_same_head(tmp_path): + """Re-review case: a paired change plus the standalone effect it ends at.""" + from pipeline import uk_bundle as bundles + + measured = inputs(old=(0, 10), new=(10, 20)) + receipts = verified(tmp_path, "measured-base.json", "measured-new.json") + endpoints = {} + for side, artifact in ( + ("base", measured[4]["base.json"]), + ("new", measured[5]["new.json"]), + ): + payload = engines.canonical_bytes(artifact) + (tmp_path / f"measured-{side}.json").write_bytes(payload) + endpoints[side] = { + "path": f"measured-{side}.json", + "sha256": hashlib.sha256(payload).hexdigest(), + } + paired = { + "artifact_type": "paired_run", + "status": "computed", + "effect_gbp": 10, + "base_artifact": endpoints["base"], + "new_artifact": endpoints["new"], + "base_bundle": bundles.DEFAULT_BUNDLE, + "new_bundle": bundles.DEFAULT_BUNDLE, + "head_variables": ["head_0"], + } + payload = engines.canonical_bytes(paired) + (tmp_path / "paired.json").write_bytes(payload) + attribution = json.loads(FIXTURE.read_text()) + attribution["entries"][0].update( + fy="2024-25", + drivers=[ + { + "driver": "data_release", + "evidence": [{"kind": "paired_run", "reference": "paired.json"}], + "sized": True, + "value_gbp": 10, + "artifact": { + "path": "paired.json", + "sha256": hashlib.sha256(payload).hexdigest(), + "value_path": ["effect_gbp"], + }, + }, + { + "driver": "other_engine_change", + "evidence": [{"kind": "diagnostic", "reference": "new endpoint"}], + "sized": True, + "value_gbp": 10, + "artifact": { + **endpoints["new"], + "value_path": ["head_effects", "head_0"], + }, + }, + ], + ) + with pytest.raises(engines.EngineComparisonError, match="overlap"): + compare( + inputs(old=(0, 10), new=(10, 20)), + attribution, + artifact_root=tmp_path, + verified_artifacts=receipts, + ) diff --git a/tests/test_uk_event_bundle_staging.py b/tests/test_uk_event_bundle_staging.py index dafe5405..877ca9fd 100644 --- a/tests/test_uk_event_bundle_staging.py +++ b/tests/test_uk_event_bundle_staging.py @@ -311,16 +311,33 @@ def test_paths_cannot_enter_the_other_protected_tree(pinned_stage, key): ) -def test_diagnostic_report_cannot_overwrite_another_bundles_outputs(tmp_path): +def test_diagnostic_destinations_cannot_overwrite_another_bundles_outputs( + tmp_path, monkeypatch +): + """Runs against a temporary root, so a broken guard can't touch real files.""" + import functools + from pipeline import diagnose_uk_event_models as diagnose + monkeypatch.setattr(diagnose, "ROOT", tmp_path) + monkeypatch.setattr( + diagnose, + "validate_bundle_path", + functools.partial(bundles.validate_bundle_path, root=tmp_path), + ) saved = tmp_path / "observations.json" saved.write_text("[]") - for report in ( - diagnose.ROOT / "results/uk/events/SUMMARY.md", - diagnose.ROOT / diagnose.DEFAULT_REPORT, + summary = tmp_path / "results/uk/events/SUMMARY.md" + reserved = tmp_path / diagnose.DEFAULT_REPORT + for path in (summary, reserved): + path.parent.mkdir(parents=True) + path.write_text("the default bundle's file") + for flag, destination in ( + ("--report", summary), + ("--output", summary), + ("--report", reserved), + ("--output", reserved), ): - before = report.read_bytes() if report.exists() else None with pytest.raises((ValueError, SystemExit)): diagnose.main( [ @@ -328,11 +345,11 @@ def test_diagnostic_report_cannot_overwrite_another_bundles_outputs(tmp_path): "some-other-bundle", "--from-json", str(saved), - "--report", - str(report), + flag, + str(destination), ] ) - assert (report.read_bytes() if report.exists() else None) == before + assert destination.read_text() == "the default bundle's file" def two_bundle_comparison(pinned_stage): @@ -690,3 +707,40 @@ def test_summary_rejects_another_bundles_model_observations(pinned_stage): ): comparison.write_summary(output.parent, artifact_root=root, bundle=DEV) assert not (output.parent / "SUMMARY.md").exists() + + +def test_engine_comparison_output_cannot_enter_a_registry_or_bundle_namespace( + pinned_stage, +): + from pipeline import compare_uk_engines as engines + + setup = two_bundle_comparison(pinned_stage) + root, other = setup["root"], setup["other"] + forbidden = [ + setup["other_output"], + setup["other_output"].parent, + bundles.results_root("unrelated-bundle", root=root), + setup["registry_path"].parent, + root / "data/uk/events/bundles/unrelated-bundle", + root / "results/uk/events/event", + ] + for directory in forbidden: + before = sorted(directory.glob("*")) if directory.exists() else None + with pytest.raises(engines.EngineComparisonError, match="protected"): + engines.write_engine_comparison( + DEV, + other, + attribution_path=setup["attribution_path"], + output_dir=directory, + artifact_root=root, + ) + assert (sorted(directory.glob("*")) if directory.exists() else None) == before + scratch = root / "scratch/comparison" + engines.write_engine_comparison( + DEV, + other, + attribution_path=setup["attribution_path"], + output_dir=scratch, + artifact_root=root, + ) + assert (scratch / "ENGINE_COMPARISON.md").exists()