From 651501e76436a4aa5ce41a29d4cfb6bf597f744d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 22 Oct 2026 07:07:55 -0400 Subject: [PATCH 01/57] Bind UK gate signing to a committed key registry Every UK verifier trusted whichever key was in the environment, so any 32-byte key that signed a report also verified it. Register the keys that may sign (one per builder: HMAC means the verifier needs the signer's secret, so a cut is signed, certified and published by one key holder), and check the registry at every UK signer and verifier: - the publisher's schema-4 checks (battery, certification, dense report) accept only active or retired keys; - the gate battery, the certifier and the scoped re-signer sign only with an active key; - the national role refuses an unusable key before loading inputs, not after the solve (runbook F8); - the local release-candidate preflight names an unregistered key. The registry starts with the key that signed every UK cut on record (66bf0e29...). Tests keep their throwaway keys through a root-conftest admission; tests marked committed_gate_key_registry see the real one. Co-Authored-By: Claude Opus 5.5 --- .../uk-gate-signing-key-registry.added.md | 1 + conftest.py | 41 +++ .../src/microcosm/build/gate_battery.py | 5 +- .../build/uk_runtime/calibration_run.py | 2 + .../build/uk_runtime/full_build_cli.py | 1 + .../build/uk_runtime/national_role.py | 20 ++ .../build/uk_runtime/release_certification.py | 11 +- .../uk/test_uk_gate_signing_key_registry.py | 215 +++++++++++++ .../src/microcosm/data/contract.py | 17 +- .../src/microcosm/data/gate_signing_keys.py | 291 ++++++++++++++++++ .../shared/test_gate_signing_keys.py | 223 ++++++++++++++ pyproject.toml | 1 + tools/preflight_uk_local_release_candidate.py | 8 + 13 files changed, 832 insertions(+), 4 deletions(-) create mode 100644 changelog.d/uk-gate-signing-key-registry.added.md create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_gate_signing_key_registry.py create mode 100644 packages/microcosm-data/src/microcosm/data/gate_signing_keys.py create mode 100644 packages/microcosm-data/tests/engine_free/shared/test_gate_signing_keys.py diff --git a/changelog.d/uk-gate-signing-key-registry.added.md b/changelog.d/uk-gate-signing-key-registry.added.md new file mode 100644 index 0000000000..1aa86c968c --- /dev/null +++ b/changelog.d/uk-gate-signing-key-registry.added.md @@ -0,0 +1 @@ +UK gate reports, release-cut certifications and dense gate reports now verify only under a signing key registered in `microcosm.data.gate_signing_keys`, and every UK signer refuses an unregistered or retired key. The national build checks its key before loading any input, instead of failing after the solve. `python -m microcosm.data.gate_signing_keys check` reports whether the key in the environment may sign. diff --git a/conftest.py b/conftest.py index f434d79435..7a7c44ebf3 100644 --- a/conftest.py +++ b/conftest.py @@ -130,3 +130,44 @@ def unexpected_start(*args, **options): yield emitters for emitter in emitters: emitter.close() + + +@pytest.fixture(autouse=True) +def _admit_test_gate_signing_keys(request, monkeypatch): + """Let tests sign with their own UK gate keys. + + Production verifies only keys registered in + ``microcosm.data.gate_signing_keys``. Tests sign with throwaway keys, so + every unregistered key counts as an active test key here, while the + committed entries keep their real status. A test marked + ``committed_gate_key_registry`` sees the committed registry unchanged. + """ + + if request.node.get_closest_marker("committed_gate_key_registry"): + return + try: + from microcosm.data import gate_signing_keys + except ImportError: + return + committed = gate_signing_keys._registry + + class _AdmitTestKeys(dict): + def __missing__(self, key_sha256): + return gate_signing_keys.GateSigningKey( + key_sha256=key_sha256, + label="test-key", + status="active", + custodian="the test suite", + held_in="the test", + registered="test", + evidence="admitted by the root conftest", + ) + + def get(self, key_sha256, default=None): + return self[key_sha256] + + def registry(country): + entries = committed(country) + return None if entries is None else _AdmitTestKeys(entries) + + monkeypatch.setattr(gate_signing_keys, "_registry", registry) diff --git a/packages/microcosm-build/src/microcosm/build/gate_battery.py b/packages/microcosm-build/src/microcosm/build/gate_battery.py index 0a75277c90..bddda6c93a 100644 --- a/packages/microcosm-build/src/microcosm/build/gate_battery.py +++ b/packages/microcosm-build/src/microcosm/build/gate_battery.py @@ -62,6 +62,7 @@ weights_audit_gate, ) from microcosm.build.trace import compute_composition_fingerprint +from microcosm.data.gate_signing_keys import require_registered_gate_signing_key from microcosm.frame import Frame __all__ = [ @@ -155,7 +156,9 @@ def _signing_key(country: str) -> bytes: raise RuntimeError(f"{env} must be valid base64.") from exc if len(key) != 32: raise RuntimeError(f"{env} must decode to exactly 32 bytes.") - return key + # A key the committed registry does not let sign would produce reports + # no verifier accepts; refuse it here so the report says why. + return require_registered_gate_signing_key(country, key, purpose="sign") def _json_safe(value: object) -> object: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index f40ed607ea..27dc4eb25c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -51,6 +51,7 @@ uk_time_period, ) from microcosm.calibrate import TargetRegistry +from microcosm.data.gate_signing_keys import require_registered_gate_signing_key from microcosm.frame import Frame _REPOSITORY = Path(__file__).resolve().parents[6] @@ -992,6 +993,7 @@ def resign_uk_gate_report(payload: dict[str, object]) -> None: raise RuntimeError(f"{env} must be valid base64.") from exc if len(key) != 32: raise RuntimeError(f"{env} must decode to exactly 32 bytes.") + require_registered_gate_signing_key("uk", key, purpose="sign") attestation.pop("signing_error", None) attestation["signing_key_sha256"] = hashlib.sha256(key).hexdigest() attestation["signature"] = None diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py index 2e4b35b857..0bf0141d27 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py @@ -3606,6 +3606,7 @@ def _national_attempt( # These checks must precede input loading. The emitter is already # running so a refusal is visible as a failed attempt. refuse_occupied_national_output(args) + national_role.require_signing_key() preflight_staged_dataset(args) predecessor = resolve_predecessor(args.logbook_prev_row_digest) telemetry = create_staging_run_bundle(args, build_id=state.build_id) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_role.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_role.py index 5ffc89ce7b..e52e1b2656 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_role.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_role.py @@ -28,6 +28,7 @@ import numpy as np import pandas as pd +from microcosm.build.gate_battery import _signing_key from microcosm.build.logbook_adoption import atomic_write_json from microcosm.build.staging_dataset import ( SHA256SUMS_FILENAME, @@ -62,6 +63,7 @@ "national_dry_run", "national_manifest", "require_bound_input", + "require_signing_key", ] @@ -126,6 +128,24 @@ def require_bound_input(args: argparse.Namespace) -> None: _require_bound_input = require_bound_input +def require_signing_key() -> None: + """Refuse a national build whose gate reports could not be signed. + + The seam signs its terminal report only after the solve, so a missing, + malformed or unregistered key used to surface 25-40 minutes in, as + "unsigned full-scale calibration runs refuse to stage". Check it before + any input is read. + """ + + try: + _signing_key("uk") + except RuntimeError as error: + raise ValueError( + "--release-role national signs its gate reports and needs a " + f"registered UK gate signing key: {error}" + ) from error + + def national_dry_run( args: argparse.Namespace, *, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py index 7bece76f04..dce4d31eb2 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py @@ -61,6 +61,10 @@ uk_cgt_projection_read_from_engine, ) from microcosm.build.uk_runtime.terminal_gates import uk_export_candidate_columns +from microcosm.data.gate_signing_keys import ( + GateSigningKeyError, + require_registered_gate_signing_key, +) __all__ = [ "UK_RELEASE_CERTIFICATION_KIND", @@ -575,7 +579,12 @@ def _require_signing_key() -> bytes: raise UKReleaseCertificationError( f"{env_name} must decode to exactly 32 bytes." ) - return key + # The certifier verifies every part and signs the certification with + # this one key, so it must be a registered key that may still sign. + try: + return require_registered_gate_signing_key("uk", key, purpose="sign") + except GateSigningKeyError as exc: + raise UKReleaseCertificationError(f"{env_name}: {exc}") from exc def _verify_part( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_gate_signing_key_registry.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_gate_signing_key_registry.py new file mode 100644 index 0000000000..2bbb74b2be --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_gate_signing_key_registry.py @@ -0,0 +1,215 @@ +"""Every UK signer and verifier consults the committed signing key registry.""" + +from __future__ import annotations + +import base64 +import json + +import pytest + +from microcosm.build import gate_battery +from microcosm.build.uk_runtime import ( + calibration_run, + national_role, + release_certification, +) +from microcosm.data import contract +from microcosm.data import gate_signing_keys as registry +from microcosm.data.gate_signing_keys import ( + UK_GATE_SIGNING_KEY_ENV, + GateSigningKey, + GateSigningKeyError, + gate_signing_key_fingerprint, +) + +ACTIVE = b"\x41" * 32 +RETIRED = b"\x52" * 32 +REVOKED = b"\x56" * 32 +UNREGISTERED = b"\x55" * 32 + + +def _entry(key: bytes, status: str) -> GateSigningKey: + return GateSigningKey( + key_sha256=gate_signing_key_fingerprint(key), + label=f"test-{status}", + status=status, # type: ignore[arg-type] + custodian="test", + held_in="test", + registered="2026-10-22", + evidence="test", + retired_on=None if status == "active" else "2026-10-22", + ) + + +@pytest.fixture +def strict_registry(monkeypatch): + """A registry with one key of each status, and nothing else admitted.""" + + table = { + entry.key_sha256: entry + for entry in ( + _entry(ACTIVE, "active"), + _entry(RETIRED, "retired"), + _entry(REVOKED, "revoked"), + ) + } + monkeypatch.setattr( + registry, "_registry", lambda country: table if country == "uk" else None + ) + return table + + +def _use_key(monkeypatch, key: bytes) -> None: + monkeypatch.setenv(UK_GATE_SIGNING_KEY_ENV, base64.b64encode(key).decode()) + + +def test_every_layer_reads_one_variable(): + assert gate_battery.gate_signing_key_env("uk") == UK_GATE_SIGNING_KEY_ENV + assert contract._UK_GATE_BATTERY_SIGNING_KEY_ENV == UK_GATE_SIGNING_KEY_ENV + + +def test_a_registry_refusal_is_a_signing_error_not_a_crash(): + """The battery turns a RuntimeError into an unsigned, unshippable report; + the registry's refusal must be one.""" + + assert issubclass(GateSigningKeyError, RuntimeError) + + +@pytest.mark.parametrize( + ("key", "signs"), + [(ACTIVE, True), (RETIRED, False), (REVOKED, False), (UNREGISTERED, False)], +) +def test_the_battery_signs_only_with_an_active_key( + strict_registry, monkeypatch, key, signs +): + _use_key(monkeypatch, key) + if signs: + assert gate_battery._signing_key("uk") == key + else: + with pytest.raises(GateSigningKeyError): + gate_battery._signing_key("uk") + + +@pytest.mark.parametrize( + ("key", "certifies"), + [(ACTIVE, True), (RETIRED, False), (REVOKED, False), (UNREGISTERED, False)], +) +def test_the_certifier_needs_an_active_key( + strict_registry, monkeypatch, key, certifies +): + _use_key(monkeypatch, key) + if certifies: + assert release_certification._require_signing_key() == key + else: + with pytest.raises(release_certification.UKReleaseCertificationError): + release_certification._require_signing_key() + + +def test_the_national_role_refuses_an_unusable_key_before_loading_inputs( + strict_registry, monkeypatch +): + _use_key(monkeypatch, ACTIVE) + national_role.require_signing_key() + for key in (RETIRED, UNREGISTERED): + _use_key(monkeypatch, key) + with pytest.raises(ValueError, match="registered UK gate signing key"): + national_role.require_signing_key() + monkeypatch.delenv(UK_GATE_SIGNING_KEY_ENV) + with pytest.raises(ValueError, match=UK_GATE_SIGNING_KEY_ENV): + national_role.require_signing_key() + + +def test_resigning_a_scoped_report_needs_an_active_key(strict_registry, monkeypatch): + payload = {"attestation": {"signing_error": "unsigned at battery time"}} + _use_key(monkeypatch, UNREGISTERED) + with pytest.raises(GateSigningKeyError, match="not registered"): + calibration_run.resign_uk_gate_report(payload) + assert payload["attestation"] == {"signing_error": "unsigned at battery time"} + + +def test_the_local_preflight_names_an_unregistered_key( + strict_registry, tmp_path, monkeypatch +): + import importlib.util + from pathlib import Path + + path = ( + Path(__file__).resolve().parents[5] + / "tools/preflight_uk_local_release_candidate.py" + ) + spec = importlib.util.spec_from_file_location("preflight_uk_local", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + failures = module.check_env( + pins_path=tmp_path / "missing-pins", + spine_h5=tmp_path / "spine.h5", + ladder_npz=tmp_path / "ladder.npz", + ledger_dir=tmp_path, + environ={module.SIGNING_KEY_ENV: base64.b64encode(UNREGISTERED).decode()}, + verify_digests=False, + ) + assert any("not registered" in failure for failure in failures) + failures = module.check_env( + pins_path=tmp_path / "missing-pins", + spine_h5=tmp_path / "spine.h5", + ladder_npz=tmp_path / "ladder.npz", + ledger_dir=tmp_path, + environ={module.SIGNING_KEY_ENV: base64.b64encode(ACTIVE).decode()}, + verify_digests=False, + ) + assert not any(module.SIGNING_KEY_ENV in failure for failure in failures) + + +def test_a_signed_report_verifies_until_its_key_is_revoked( + strict_registry, tmp_path, monkeypatch +): + """Round trip: the battery signs under the active key and the publisher's + dense check accepts it; retiring keeps it verifiable; revoking does not.""" + + from microcosm.build.uk_runtime.graph_national import replay_uk_dense_gate_battery + from test_support.microcosm_build.uk_full_build_cli import STEM, gate_payload + + _use_key(monkeypatch, ACTIVE) + report = replay_uk_dense_gate_battery( + json.loads(gate_payload("terminal", release_candidate=True)), + report_path=tmp_path / f"{STEM}.local_gates.json", + release_id="uk-local-candidate-f100-s0-test", + release_candidate=True, + ) + assert report["shippable"] is True + assert report["attestation"]["signing_key_sha256"] == ( + gate_signing_key_fingerprint(ACTIVE) + ) + + def failures() -> list[str]: + found: list[str] = [] + contract._check_uk_dense_gate_report( + report, failures=found, attempt_id="uk-local-candidate-f100-s0-test" + ) + return found + + assert failures() == [] + for status, verifies in (("retired", True), ("revoked", False)): + strict_registry[gate_signing_key_fingerprint(ACTIVE)] = _entry(ACTIVE, status) + assert (failures() == []) is verifies + # An unregistered key that signed is refused even though the HMAC is valid. + strict_registry.pop(gate_signing_key_fingerprint(ACTIVE)) + assert any("not registered" in line for line in failures()) + + +def test_an_unregistered_signer_writes_an_unshippable_report( + strict_registry, tmp_path, monkeypatch +): + from microcosm.build.uk_runtime.graph_national import replay_uk_dense_gate_battery + from test_support.microcosm_build.uk_full_build_cli import STEM, gate_payload + + _use_key(monkeypatch, UNREGISTERED) + report = replay_uk_dense_gate_battery( + json.loads(gate_payload("terminal", release_candidate=True)), + report_path=tmp_path / f"{STEM}.local_gates.json", + release_id="uk-local-candidate-f100-s0-test", + release_candidate=True, + ) + assert report["shippable"] is False + assert report["attestation"]["signature"] is None + assert "not registered" in report["attestation"]["signing_error"] diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index 1ec3939296..fe272cb3f1 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -49,6 +49,7 @@ from pydantic import ValidationError from microcosm.data.denied_pools import denied_pool_publication_for +from microcosm.data.gate_signing_keys import unregistered_gate_signing_key_problem from microcosm.data.stored_inputs import is_model_named from microcosm.data.us_critical_targets import ( US_CRITICAL_TARGET_FIT_REQUIREMENTS as _US_CRITICAL_TARGET_FIT_REQUIREMENTS, @@ -1081,9 +1082,21 @@ def _uk_terminal_verification_key(failures: list[str]) -> bytes | None: def _uk_gate_battery_verification_key(failures: list[str]) -> bytes | None: - """The shared executor's trust root (schema-4 reports).""" + """The shared executor's trust root (schema-4 reports). - return _uk_release_key_from_env(_UK_GATE_BATTERY_SIGNING_KEY_ENV, failures) + The key in the environment must also be registered (active or retired) + in ``microcosm.data.gate_signing_keys``: without that, any key that + signed a report would verify it. + """ + + key = _uk_release_key_from_env(_UK_GATE_BATTERY_SIGNING_KEY_ENV, failures) + if key is None: + return None + problem = unregistered_gate_signing_key_problem("uk", key, purpose="verify") + if problem is not None: + failures.append(f"{_UK_GATE_BATTERY_SIGNING_KEY_ENV}: {problem}") + return None + return key def _reject_json_constant(token: str) -> None: diff --git a/packages/microcosm-data/src/microcosm/data/gate_signing_keys.py b/packages/microcosm-data/src/microcosm/data/gate_signing_keys.py new file mode 100644 index 0000000000..a875c08334 --- /dev/null +++ b/packages/microcosm-data/src/microcosm/data/gate_signing_keys.py @@ -0,0 +1,291 @@ +"""Committed registry of the keys allowed to sign UK gate-battery reports. + +Gate reports, release-cut certifications and dense gate reports are signed +with HMAC-SHA256 under the key in ``MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY``. +Before this registry, every verifier trusted whichever key was in the +environment: a report signed by any 32-byte key verified, provided the +verifier held the same key. Verification now also requires the key's SHA-256 +fingerprint to be registered here, so a cut can only be certified, +assembled or published under a key the repository names. + +HMAC is symmetric: verifying a report needs the secret that signed it. So a +cut is signed, certified and published end to end by one key holder, and +the registry names one key per builder rather than one shared secret. A new +builder gets their own key and a reviewed entry; nobody has to share a key. + +Statuses: + +* ``active``: may sign new reports and verify. +* ``retired``: verifies reports already signed, but signs nothing new. + Use it for a routine rotation. +* ``revoked``: neither signs nor verifies. Use it if a key leaks; every cut + it signed must then be re-signed or withdrawn. + +It lives in ``microcosm-data`` because the publisher verifies, and the +build shard imports it for its signing-side checks. Entries are +source-coded; add, retire or revoke one only in a reviewed pull request +(the procedure is in ``docs/uk-cut-anyone.md``). + +Run ``python -m microcosm.data.gate_signing_keys check`` to see whether the +key in your environment may sign. +""" + +from __future__ import annotations + +import argparse +import base64 +import binascii +import hashlib +import json +import os +import secrets +import sys +from collections.abc import Mapping, Sequence +from dataclasses import asdict, dataclass +from types import MappingProxyType +from typing import Literal + +__all__ = [ + "GATE_SIGNING_KEY_STATUSES", + "UK_GATE_SIGNING_KEYS", + "UK_GATE_SIGNING_KEY_ENV", + "GateSigningKey", + "GateSigningKeyError", + "decode_gate_signing_key", + "gate_signing_key_fingerprint", + "registered_gate_signing_key", + "require_registered_gate_signing_key", + "unregistered_gate_signing_key_problem", +] + +#: The environment variable every UK signer and verifier reads. Lockstep with +#: ``microcosm.build.gate_battery.gate_signing_key_env("uk")``. +UK_GATE_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" + +GateSigningKeyStatus = Literal["active", "retired", "revoked"] +GateSigningKeyPurpose = Literal["sign", "verify"] +GATE_SIGNING_KEY_STATUSES: frozenset[str] = frozenset({"active", "retired", "revoked"}) +_PURPOSE_STATUSES: Mapping[str, frozenset[str]] = MappingProxyType( + {"sign": frozenset({"active"}), "verify": frozenset({"active", "retired"})} +) + + +class GateSigningKeyError(RuntimeError): + """The key in the environment is absent, malformed or not registered.""" + + +@dataclass(frozen=True) +class GateSigningKey: + """One registered signing key, identified by its fingerprint only. + + The fingerprint is the SHA-256 of the 32 decoded key bytes: the value + every report records as ``attestation.signing_key_sha256``. The secret + itself is never committed. + """ + + key_sha256: str + label: str + status: GateSigningKeyStatus + custodian: str + held_in: str + registered: str + evidence: str + retired_on: str | None = None + + +UK_GATE_SIGNING_KEYS: Mapping[str, GateSigningKey] = MappingProxyType( + { + "66bf0e29dbdabc2e3d7a79f21f43ac28d17b592faaf91e67c9bcf9b30c09a6ff": ( + GateSigningKey( + key_sha256=( + "66bf0e29dbdabc2e3d7a79f21f43ac28d17b592faaf91e67c9bcf9b30c09a6ff" + ), + label="uk-gate-juaristi22-2026", + status="active", + custodian="María Juaristi (GitHub juaristi22)", + held_in="her build machine; not in a shared store", + registered="2026-10-22", + evidence=( + "Signed every UK gate report on record from 2026-08-18 " + "(uk/national_staging_build_record.json, #635) through the " + "2026-10-19 and 2026-10-21 national cuts and the " + "2026-10-19 dense cut." + ), + ) + ), + } +) + + +def _registry(country: str) -> Mapping[str, GateSigningKey] | None: + """The registry for a country, or ``None`` when the country has none. + + Countries without a registry keep the old behaviour (any well-formed key + in the environment). Tests replace this function to admit their keys. + """ + + return UK_GATE_SIGNING_KEYS if country == "uk" else None + + +def gate_signing_key_fingerprint(key: bytes) -> str: + """The SHA-256 a report records for the key that signed it.""" + + return hashlib.sha256(key).hexdigest() + + +def decode_gate_signing_key(encoded: str | None, *, env: str) -> bytes: + """Decode a base64 signing key, refusing anything but 32 bytes.""" + + if not encoded: + raise GateSigningKeyError(f"{env} must contain a base64-encoded 32-byte key.") + try: + key = base64.b64decode(encoded.strip(), validate=True) + except (binascii.Error, ValueError) as exc: + raise GateSigningKeyError(f"{env} must be valid base64.") from exc + if len(key) != 32: + raise GateSigningKeyError(f"{env} must decode to exactly 32 bytes.") + return key + + +def registered_gate_signing_key(country: str, key_sha256: str) -> GateSigningKey | None: + """The registry entry for a fingerprint, or ``None`` if unregistered.""" + + registry = _registry(country) + if registry is None: + return None + return registry.get(key_sha256) + + +def unregistered_gate_signing_key_problem( + country: str, key: bytes, *, purpose: GateSigningKeyPurpose +) -> str | None: + """Why ``key`` may not sign or verify for ``country``, or ``None`` if it may. + + Returns ``None`` for a country without a registry. + """ + + if purpose not in _PURPOSE_STATUSES: + raise ValueError(f"purpose must be 'sign' or 'verify', got {purpose!r}.") + registry = _registry(country) + if registry is None: + return None + fingerprint = gate_signing_key_fingerprint(key) + entry = registry.get(fingerprint) + if entry is None: + return ( + f"the {country.upper()} gate signing key with fingerprint " + f"{fingerprint} is not registered in " + "microcosm.data.gate_signing_keys; a key signs or verifies " + "release evidence only after a reviewed pull request registers it." + ) + if entry.status not in _PURPOSE_STATUSES[purpose]: + return ( + f"the {country.upper()} gate signing key {entry.label} " + f"({fingerprint}) is {entry.status}; a {entry.status} key may not " + f"{purpose}." + ) + return None + + +def require_registered_gate_signing_key( + country: str, key: bytes, *, purpose: GateSigningKeyPurpose +) -> bytes: + """Return ``key`` if the registry lets it ``purpose``; raise otherwise.""" + + problem = unregistered_gate_signing_key_problem(country, key, purpose=purpose) + if problem is not None: + raise GateSigningKeyError(problem) + return key + + +# --------------------------------------------------------------------------- +# Operator CLI +# --------------------------------------------------------------------------- + + +def _entry_payload(entry: GateSigningKey) -> dict[str, object]: + return asdict(entry) + + +def _check(args: argparse.Namespace) -> int: + env = args.env or UK_GATE_SIGNING_KEY_ENV + try: + key = decode_gate_signing_key(os.environ.get(env), env=env) + except GateSigningKeyError as exc: + print(json.dumps({"ok": False, "env": env, "problem": str(exc)}, indent=2)) + return 1 + fingerprint = gate_signing_key_fingerprint(key) + problem = unregistered_gate_signing_key_problem( + args.country, key, purpose=args.purpose + ) + entry = registered_gate_signing_key(args.country, fingerprint) + print( + json.dumps( + { + "ok": problem is None, + "env": env, + "country": args.country, + "purpose": args.purpose, + "key_sha256": fingerprint, + "registered": _entry_payload(entry) if entry else None, + "problem": problem, + }, + indent=2, + ) + ) + return 0 if problem is None else 1 + + +def _list(args: argparse.Namespace) -> int: + registry = _registry(args.country) or {} + print(json.dumps([_entry_payload(entry) for entry in registry.values()], indent=2)) + return 0 + + +def _generate(args: argparse.Namespace) -> int: + key = secrets.token_bytes(32) + # The key goes to stdout for the operator to pipe into a secret store; + # the fingerprint goes to stderr so it can be pasted into a pull request. + print(base64.b64encode(key).decode()) + print(f"key_sha256 {gate_signing_key_fingerprint(key)}", file=sys.stderr) + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="python -m microcosm.data.gate_signing_keys", + description=( + "Inspect the committed UK gate signing key registry and the key " + "in the environment." + ), + ) + sub = parser.add_subparsers(dest="command", required=True) + check = sub.add_parser( + "check", + help="exit 0 when the key in the environment may sign (or verify)", + ) + check.add_argument("--country", default="uk") + check.add_argument("--purpose", choices=("sign", "verify"), default="sign") + check.add_argument( + "--env", + default=None, + help=f"variable to read (default {UK_GATE_SIGNING_KEY_ENV})", + ) + check.set_defaults(func=_check) + listing = sub.add_parser("list", help="print the registered keys") + listing.add_argument("--country", default="uk") + listing.set_defaults(func=_list) + generate = sub.add_parser( + "generate", + help=( + "print a fresh base64 key on stdout and its fingerprint on stderr " + "(store the key, then register the fingerprint)" + ), + ) + generate.set_defaults(func=_generate) + args = parser.parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/microcosm-data/tests/engine_free/shared/test_gate_signing_keys.py b/packages/microcosm-data/tests/engine_free/shared/test_gate_signing_keys.py new file mode 100644 index 0000000000..4d0d7df795 --- /dev/null +++ b/packages/microcosm-data/tests/engine_free/shared/test_gate_signing_keys.py @@ -0,0 +1,223 @@ +"""The committed UK gate signing key registry binds verification to named keys.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import re + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from microcosm.data import contract +from microcosm.data import gate_signing_keys as registry +from microcosm.data.gate_signing_keys import ( + GATE_SIGNING_KEY_STATUSES, + UK_GATE_SIGNING_KEY_ENV, + UK_GATE_SIGNING_KEYS, + GateSigningKey, + GateSigningKeyError, + decode_gate_signing_key, + gate_signing_key_fingerprint, + require_registered_gate_signing_key, + unregistered_gate_signing_key_problem, +) + +#: The key that signed every UK cut on record (T19, T21, the 10/19 dense cut). +CONTINUITY_FINGERPRINT = ( + "66bf0e29dbdabc2e3d7a79f21f43ac28d17b592faaf91e67c9bcf9b30c09a6ff" +) +UNREGISTERED_KEY = b"\x11" * 32 + + +def _entry(key: bytes, status: str) -> GateSigningKey: + return GateSigningKey( + key_sha256=gate_signing_key_fingerprint(key), + label=f"test-{status}", + status=status, # type: ignore[arg-type] + custodian="test", + held_in="test", + registered="2026-10-22", + evidence="test", + retired_on="2026-10-22" if status != "active" else None, + ) + + +def _use_registry(monkeypatch, *entries: GateSigningKey) -> None: + table = {entry.key_sha256: entry for entry in entries} + monkeypatch.setattr( + registry, "_registry", lambda country: table if country == "uk" else None + ) + + +@pytest.mark.committed_gate_key_registry +def test_the_committed_registry_is_well_formed(): + assert UK_GATE_SIGNING_KEYS, "the UK registry must name at least one key" + labels = set() + for fingerprint, entry in UK_GATE_SIGNING_KEYS.items(): + assert re.fullmatch(r"[0-9a-f]{64}", fingerprint) + assert entry.key_sha256 == fingerprint + assert entry.status in GATE_SIGNING_KEY_STATUSES + assert (entry.retired_on is None) == (entry.status == "active") + assert entry.label not in labels + labels.add(entry.label) + for field in ("custodian", "held_in", "registered", "evidence"): + assert getattr(entry, field).strip() + assert any(entry.status == "active" for entry in UK_GATE_SIGNING_KEYS.values()) + + +@pytest.mark.committed_gate_key_registry +def test_the_key_that_signed_every_cut_on_record_still_verifies(): + """Continuity: re-registering must never strand the published cuts.""" + + entry = UK_GATE_SIGNING_KEYS[CONTINUITY_FINGERPRINT] + assert entry.status in {"active", "retired"} + + +@pytest.mark.committed_gate_key_registry +def test_an_unregistered_key_neither_signs_nor_verifies(): + for purpose in ("sign", "verify"): + problem = unregistered_gate_signing_key_problem( + "uk", UNREGISTERED_KEY, purpose=purpose + ) + assert problem is not None + assert gate_signing_key_fingerprint(UNREGISTERED_KEY) in problem + with pytest.raises(GateSigningKeyError, match="not registered"): + require_registered_gate_signing_key("uk", UNREGISTERED_KEY, purpose=purpose) + + +@pytest.mark.committed_gate_key_registry +def test_countries_without_a_registry_keep_the_old_behaviour(): + assert ( + unregistered_gate_signing_key_problem("us", UNREGISTERED_KEY, purpose="sign") + is None + ) + assert registry.registered_gate_signing_key("us", "0" * 64) is None + + +@given( + key=st.binary(min_size=32, max_size=32), + status=st.sampled_from(sorted(GATE_SIGNING_KEY_STATUSES)), +) +def test_status_decides_purpose(key, status): + """For every key and status: active signs and verifies, retired only + verifies, revoked does neither, and signing implies verifying.""" + + entry = _entry(key, status) + with pytest.MonkeyPatch.context() as patch: + _use_registry(patch, entry) + may_sign = ( + unregistered_gate_signing_key_problem("uk", key, purpose="sign") is None + ) + may_verify = ( + unregistered_gate_signing_key_problem("uk", key, purpose="verify") is None + ) + assert may_sign == (status == "active") + assert may_verify == (status in {"active", "retired"}) + assert not may_sign or may_verify + + +@given( + key=st.binary(min_size=32, max_size=32), + other=st.binary(min_size=32, max_size=32), +) +def test_registration_is_by_exact_fingerprint(key, other): + """A registered key admits only itself: no other key verifies under it.""" + + with pytest.MonkeyPatch.context() as patch: + _use_registry(patch, _entry(key, "active")) + other_problem = unregistered_gate_signing_key_problem( + "uk", other, purpose="verify" + ) + assert (other_problem is None) == (other == key) + + +@given(key=st.binary(min_size=32, max_size=32)) +def test_the_fingerprint_is_the_attested_sha256(key): + assert gate_signing_key_fingerprint(key) == hashlib.sha256(key).hexdigest() + encoded = base64.b64encode(key).decode() + assert decode_gate_signing_key(encoded, env="E") == key + + +@given(raw=st.binary(max_size=64).filter(lambda value: len(value) != 32)) +def test_a_key_of_any_other_length_is_refused(raw): + with pytest.raises(GateSigningKeyError, match="exactly 32 bytes|must contain"): + decode_gate_signing_key(base64.b64encode(raw).decode(), env="E") + + +def test_malformed_base64_is_refused(): + with pytest.raises(GateSigningKeyError, match="valid base64"): + decode_gate_signing_key("not base64!", env="E") + + +# --------------------------------------------------------------------------- +# The publisher's verification key +# --------------------------------------------------------------------------- + + +def _set_key(monkeypatch, key: bytes) -> None: + monkeypatch.setenv(UK_GATE_SIGNING_KEY_ENV, base64.b64encode(key).decode()) + + +@pytest.mark.parametrize( + ("status", "verifies"), [("active", True), ("retired", True), ("revoked", False)] +) +def test_the_publisher_verifies_only_registered_keys(monkeypatch, status, verifies): + key = b"\x22" * 32 + _set_key(monkeypatch, key) + _use_registry(monkeypatch, _entry(key, status)) + failures: list[str] = [] + assert (contract._uk_gate_battery_verification_key(failures) == key) is verifies + assert (failures == []) is verifies + if not verifies: + assert "revoked" in failures[0] + + +def test_the_publisher_refuses_an_unregistered_key(monkeypatch): + _set_key(monkeypatch, UNREGISTERED_KEY) + _use_registry(monkeypatch, _entry(b"\x22" * 32, "active")) + failures: list[str] = [] + assert contract._uk_gate_battery_verification_key(failures) is None + assert len(failures) == 1 and "not registered" in failures[0] + + +# --------------------------------------------------------------------------- +# The operator CLI +# --------------------------------------------------------------------------- + + +def test_check_reports_the_fingerprint_and_status(monkeypatch, capsys): + key = b"\x33" * 32 + _set_key(monkeypatch, key) + _use_registry(monkeypatch, _entry(key, "retired")) + assert registry.main(["check", "--purpose", "verify"]) == 0 + report = json.loads(capsys.readouterr().out) + assert report["ok"] is True + assert report["key_sha256"] == gate_signing_key_fingerprint(key) + assert report["registered"]["status"] == "retired" + assert registry.main(["check"]) == 1 # a retired key may not sign + assert "retired" in json.loads(capsys.readouterr().out)["problem"] + + +def test_check_fails_without_a_key(monkeypatch, capsys): + monkeypatch.delenv(UK_GATE_SIGNING_KEY_ENV, raising=False) + assert registry.main(["check"]) == 1 + assert json.loads(capsys.readouterr().out)["ok"] is False + + +def test_generate_prints_a_fresh_key_and_its_fingerprint(capsys): + assert registry.main(["generate"]) == 0 + captured = capsys.readouterr() + key = decode_gate_signing_key(captured.out.strip(), env="E") + assert captured.err.strip() == f"key_sha256 {gate_signing_key_fingerprint(key)}" + assert registry.main(["generate"]) == 0 + assert capsys.readouterr().out.strip() != captured.out.strip() + + +@pytest.mark.committed_gate_key_registry +def test_list_prints_the_committed_entries(capsys): + assert registry.main(["list"]) == 0 + listed = json.loads(capsys.readouterr().out) + assert {row["key_sha256"] for row in listed} == set(UK_GATE_SIGNING_KEYS) diff --git a/pyproject.toml b/pyproject.toml index 7f2c4aadbb..d6dcdd8aa6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,4 +63,5 @@ markers = [ "requires_us: needs policyengine-us (the us extra); auto-skipped when absent", "requires_uk: needs policyengine-uk (the uk extra); auto-skipped when absent", "integration: requires explicit --run-integration collection", + "committed_gate_key_registry: run against the committed UK gate signing key registry, not the test-key admission", ] diff --git a/tools/preflight_uk_local_release_candidate.py b/tools/preflight_uk_local_release_candidate.py index dcea3f811c..a3e39cd8a5 100644 --- a/tools/preflight_uk_local_release_candidate.py +++ b/tools/preflight_uk_local_release_candidate.py @@ -36,6 +36,8 @@ from datetime import date from pathlib import Path +from microcosm.data.gate_signing_keys import unregistered_gate_signing_key_problem + SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" PIN_NAMES = ("spine", "ladder", "facts", "manifest") RELEASE_BLOCKING = "release_blocking" @@ -73,6 +75,12 @@ def check_env( f"{SIGNING_KEY_ENV} decodes to {len(raw)} bytes; the gate " "battery signs only with exactly 32." ) + else: + problem = unregistered_gate_signing_key_problem( + "uk", raw, purpose="sign" + ) + if problem is not None: + failures.append(f"{SIGNING_KEY_ENV}: {problem}") except Exception: failures.append(f"{SIGNING_KEY_ENV} is not valid base64.") if not pins_path.exists(): From a093b36b2b9f2b22932110e1efea35f9364e69fd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 22 Oct 2026 07:10:45 -0400 Subject: [PATCH 02/57] Give the national-build tests the test signing key The national role now checks its key before loading inputs, so the tests that drive a national build through stubs need one, as the others do. Co-Authored-By: Claude Opus 5.5 --- .../tests/engine_free/uk/test_uk_full_build_cli.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py index 98db163cee..0db37b28a7 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py @@ -261,6 +261,7 @@ def test_national_role_is_validated_then_built_through_the_graph(tmp_path, monke attempt id, the Hub pre-flight, the Logbook and the staging telemetry live in the driver's national envelope. """ + monkeypatch.setenv(SIGNING_KEY_ENV, TEST_SIGNING_KEY) argv = _national_argv(tmp_path) national = cli.parse_args(argv) assert national._posture is cli.uk_rowwise_posture("national") @@ -2036,6 +2037,7 @@ def test_national_interrupt_records_a_discarded_row_and_re_raises( one: the seam's terminal-disposition contract, kept by the driver (the failure sidecar names the interrupt, the row says ``discarded``, the interrupt propagates).""" + monkeypatch.setenv(SIGNING_KEY_ENV, TEST_SIGNING_KEY) argv = _national_argv(tmp_path) monkeypatch.setattr(cli, "preflight_staged_dataset", lambda args: None) monkeypatch.setattr( @@ -2103,6 +2105,7 @@ def test_dense_validation_result_matches_emitted_run_status( def test_build_lifecycle_handles_returned_errors_and_exceptions( tmp_path, monkeypatch, fake_telemetry_emitters, role, outcome ): + monkeypatch.setenv(SIGNING_KEY_ENV, TEST_SIGNING_KEY) args = ( arguments(tmp_path) if role == "dense" @@ -2429,6 +2432,7 @@ def test_a_sigterm_during_the_national_build_records_a_discarded_row( tmp_path, monkeypatch ): """The national line routes a SIGTERM through its interrupt arm too.""" + monkeypatch.setenv(SIGNING_KEY_ENV, TEST_SIGNING_KEY) argv = _national_argv(tmp_path) monkeypatch.setattr(cli, "preflight_staged_dataset", lambda args: None) monkeypatch.setattr( From fee05bdc8fa1e28a431f73c40b2f5f35c23f95ab Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 22 Oct 2026 07:32:19 -0400 Subject: [PATCH 03/57] Add a one-command, resumable UK cut runner with pinned inputs tools/run_uk_cut.py runs fetch -> preflight -> spine -> national (dry run first) -> score (only when the in-build evaluation did not pass) -> certify -> stage-1 comparison (diagnostic) -> assemble -> inspection publish, from a clean checkout of main. Each step has a done-check over its real outputs, so `all` resumes where it stopped and never rebuilds over a spine, overwrites a calibration run or re-assembles a release. Publication is opt-in (--publish) and always --no-latest --tag-only, so main's files and the latest-*.json pointers never move (runbook Q12/F14). Inputs come from tools/uk_cut_inputs.json, which says only where each input lives; every digest is read from the pins the build itself checks (source_stages.json, frs_release.json, chronicle_feed.json, gates.json). FRS, SPI, WAS and ETB come byte-exact from the gated UKDS zips already in policyengine-uk-data-private; NTS 2002-2024 and LCFS 2024-25 are declared there pending a one-time upload. The Chronicle artifact and the input-mass reference come from the new private dataset policyengine/microcosm-uk-build-inputs. tools/modal_rebuild_uk_chronicle_artifact.py rebuilds the Chronicle UK artifact from the public repo on Modal and uploads it only if its digests reproduce the pin. Co-Authored-By: Claude Opus 5.5 --- changelog.d/uk-cut-runner.added.md | 1 + .../tests/engine_free/uk/test_run_uk_cut.py | 551 ++++++ tools/modal_rebuild_uk_chronicle_artifact.py | 298 +++ tools/run_uk_cut.py | 1590 +++++++++++++++++ tools/uk_cut_inputs.json | 158 ++ 5 files changed, 2598 insertions(+) create mode 100644 changelog.d/uk-cut-runner.added.md create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_run_uk_cut.py create mode 100644 tools/modal_rebuild_uk_chronicle_artifact.py create mode 100644 tools/run_uk_cut.py create mode 100644 tools/uk_cut_inputs.json diff --git a/changelog.d/uk-cut-runner.added.md b/changelog.d/uk-cut-runner.added.md new file mode 100644 index 0000000000..82f7057128 --- /dev/null +++ b/changelog.d/uk-cut-runner.added.md @@ -0,0 +1 @@ +`tools/run_uk_cut.py` builds, signs and assembles a Microcosm UK national cut in one resumable command, fetching every input from Hugging Face by the pins the build already enforces (`tools/uk_cut_inputs.json` says only where each input lives). `tools/modal_rebuild_uk_chronicle_artifact.py` rebuilds the pinned Chronicle UK consumer artifact on Modal and hosts it privately. diff --git a/packages/microcosm-build/tests/engine_free/uk/test_run_uk_cut.py b/packages/microcosm-build/tests/engine_free/uk/test_run_uk_cut.py new file mode 100644 index 0000000000..c8d1e0449d --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_run_uk_cut.py @@ -0,0 +1,551 @@ +"""The one-command UK cut runner: pins, fetching, argv and resumability.""" + +from __future__ import annotations + +import hashlib +import io +import json +import zipfile +from contextlib import redirect_stdout +from pathlib import Path + +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from tools import run_uk_cut as cut + +REPO = Path(cut.__file__).resolve().parents[1] + + +def _sha(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +# --------------------------------------------------------------------------- # +# Pins: the runner accepts exactly what the build checks # +# --------------------------------------------------------------------------- # + + +def test_every_declared_input_resolves_to_a_committed_pin(): + sources = {source.key: source for source in cut.load_input_sources()} + assert set(sources) == { + "frs_2024_25", + "spi_2022_23", + "was_round_8", + "etb_1977_2024", + "nts_2002_2024", + "lcfs_2024_25", + "hmrc_personal_incomes_ods", + "chronicle_uk", + "input_mass_reference", + "incumbent_efrs", + } + for source in sources.values(): + assert source.files + for pin in source.files: + assert len(pin.sha256) == 64 and int(pin.sha256, 16) >= 0 + + +def test_the_runner_pins_are_the_build_pins(): + """Lockstep with the files the build itself reads its pins from.""" + + sources = {source.key: source for source in cut.load_input_sources()} + feed = json.loads((cut.UK_DIR / "chronicle_feed.json").read_text()) + chronicle = {pin.name: pin.sha256 for pin in sources["chronicle_uk"].files} + assert chronicle == { + "manifest.json": feed["manifest_sha256"], + "consumer_facts.jsonl": feed["facts_sha256"], + } + assert sources["chronicle_uk"].path_in_repo.endswith(feed["source_commit"]) + frs = json.loads((cut.UK_DIR / "frs_release.json").read_text())["acquisition"] + assert sources["frs_2024_25"].revision == frs["huggingface_revision"] + assert sources["frs_2024_25"].archive_sha256 == frs["zip_sha256"] + assert sources["frs_2024_25"].filename == frs["zip_filename"] + identity = cut._gate_reference_identity(cut.UK_DIR) + incumbent = sources["incumbent_efrs"] + assert incumbent.revision == identity["revision"] + assert incumbent.files[0].sha256 == identity["sha256"] + stage_pins = cut.committed_source_pins() + for key in ("frs_2024_25", "spi_2022_23", "was_round_8", "etb_1977_2024"): + for pin in sources[key].files: + assert pin.sha256 == stage_pins[pin.name]["sha256"] + assert pin.size_bytes == stage_pins[pin.name]["size_bytes"] + + +def test_the_hosted_mass_reference_is_the_one_the_gate_binds(): + from microcosm.build.uk_runtime.weighted_integrity import ( + UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, + ) + + manifest = json.loads(cut.CUT_INPUTS.read_text()) + pin = manifest["file_pins"]["uk_input_mass_reference.json"] + assert pin["canonical_evidence_sha256"] == UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 + source = next(s for s in manifest["sources"] if s["key"] == "input_mass_reference") + assert source["path_in_repo"].endswith(UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256) + + +def test_every_spine_tab_flag_has_a_pinned_input(): + """The spine's licensed tabs are all fetched, under the names it reads.""" + + paths = cut.CutPaths(Path("/w"), "t", "0" * 40) + argv = cut.spine_argv(paths, python="py", staging_local_only=True) + sources = {s.key: s for s in cut.load_input_sources()} + fetched = { + str(paths.input_file(s.directory, pin.name)) + for s in sources.values() + for pin in s.files + } + values = [a for a in argv[2:] if a.startswith("/w/inputs/")] + assert values + for value in values: + assert value in fetched or value == str(paths.inputs / "frs_2024_25") + + +# --------------------------------------------------------------------------- # +# Every argv is accepted by the tool's own parser # +# --------------------------------------------------------------------------- # + + +def _paths(tmp_path: Path) -> cut.CutPaths: + return cut.CutPaths(tmp_path, "test", "a" * 40) + + +def test_spine_argv_parses(): + from microcosm.build.uk_runtime import spine_build + + paths = cut.CutPaths(Path("/w"), "t", "a" * 40) + args = spine_build._parse_args( + cut.spine_argv(paths, python="py", staging_local_only=True)[2:] + ) + assert args.release_candidate is True + assert str(args.spine_h5) == str(paths.spine_h5) + + +@pytest.mark.parametrize("dry_run", [True, False]) +def test_national_argv_parses(tmp_path, dry_run): + from microcosm.build.uk_runtime import full_build_cli + + sources = {s.key: s for s in cut.load_input_sources()} + argv = cut.national_argv( + _paths(tmp_path), + python="py", + spine_sha256="b" * 64, + sources=sources, + dry_run=dry_run, + logbook_prev_row_digest="c" * 64, + ) + args = full_build_cli.parse_args(argv[2:]) + assert args.release_role == "national" + assert args.dry_run is dry_run + assert args.input_sha256 == "b" * 64 + + +def _tool(name: str): + import importlib + + return importlib.import_module(f"tools.{name}") + + +def test_certify_and_assemble_argv_parse(tmp_path): + sources = {s.key: s for s in cut.load_input_sources()} + paths = _paths(tmp_path) + certify = _tool("certify_uk_release_cut")._parse_args( + cut.certify_argv( + paths, + python="py", + candidate_sha256="d" * 64, + spine_sha256="e" * 64, + sources=sources, + )[2:] + ) + assert certify.release_id == cut.RELEASE_ID + assemble = _tool("assemble_uk_release_dir")._parse_args( + cut.assemble_argv(paths, python="py")[2:] + ) + assert str(assemble.out_dir) == "releases" + + +def test_score_argv_names_only_flags_the_scorer_declares(tmp_path, capsys): + sources = {s.key: s for s in cut.load_input_sources()} + argv = cut.score_argv( + _paths(tmp_path), python="py", candidate_sha256="f" * 64, sources=sources + ) + with pytest.raises(SystemExit): + _tool("score_uk_national_candidate").main(["--help"]) + help_text = capsys.readouterr().out + for flag in (a for a in argv[2:] if a.startswith("--")): + assert flag in help_text + + +@given(tag=st.text(min_size=1, max_size=80)) +def test_publication_is_always_inspection_only(tag): + """Whatever the tag, the publisher is told not to move main or a pointer.""" + + paths = cut.CutPaths(Path("/w"), "t", "a" * 40) + for preflight in (True, False): + argv = cut.publish_argv(paths, python="py", tag=tag, preflight_only=preflight) + assert "--no-latest" in argv and "--tag-only" in argv + assert "--promote-line" not in argv + assert argv[argv.index("--repo-id") + 1] == cut.PUBLISH_REPO + assert argv[argv.index("--tag-name") + 1] == tag + + +def test_publication_argv_uses_only_declared_flags(capsys): + from microcosm.data import publish_cli + + paths = cut.CutPaths(Path("/w"), "t", "a" * 40) + argv = cut.publish_argv(paths, python="py", tag="x", preflight_only=True) + assert argv[1:3] == ["-m", "microcosm.data.publish_cli"] + assert argv[3] == str(paths.release_dir) + with pytest.raises(SystemExit): + publish_cli.main(["--help"]) + help_text = capsys.readouterr().out + for flag in (a for a in argv[4:] if a.startswith("--")): + assert flag in help_text + + +# --------------------------------------------------------------------------- # +# Fetching: verified or absent, and idempotent # +# --------------------------------------------------------------------------- # + + +class FakeHub: + """Serves files and zips from memory and counts downloads.""" + + def __init__(self, root: Path, files: dict[str, bytes]) -> None: + self.root = root + self.files = files + self.downloads = 0 + + def hf_file(self, *, repo_id, repo_type, filename, revision): + self.downloads += 1 + if filename not in self.files: + raise FileNotFoundError(f"404 {filename}") + path = self.root / "hub" / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(self.files[filename]) + return path + + def url(self, url, dest): + self.downloads += 1 + dest.write_bytes(self.files[url]) + return dest + + +def _zip(members: dict[str, bytes]) -> bytes: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as bundle: + for name, data in members.items(): + bundle.writestr(f"UKDA-0000-tab/tab/{name}", data) + return buffer.getvalue() + + +def _zip_source(members: dict[str, bytes], *, pins=None) -> cut.InputSource: + pins = pins or {name: _sha(data) for name, data in members.items()} + return cut.InputSource( + key="survey", + kind="hf_zip_members", + directory="survey", + files=tuple(cut.PinnedFile(name, pins[name]) for name in members), + repo_id="org/repo", + repo_type="model", + revision="r", + filename="survey.zip", + ) + + +member_maps = st.dictionaries( + st.from_regex(r"[a-z]{1,8}\.tab", fullmatch=True), + st.binary(max_size=256), + min_size=1, + max_size=4, +) + + +@settings(suppress_health_check=[HealthCheck.function_scoped_fixture], deadline=None) +@given(members=member_maps) +def test_a_matching_archive_installs_once(tmp_path_factory, members): + root = tmp_path_factory.mktemp("fetch") + hub = FakeHub(root, {"survey.zip": _zip(members)}) + source = _zip_source(members) + first = cut.fetch_inputs([source], root / "inputs", downloader=hub) + assert [r.status for r in first] == ["fetched"] + for name, data in members.items(): + assert (root / "inputs/survey" / name).read_bytes() == data + downloads = hub.downloads + second = cut.fetch_inputs([source], root / "inputs", downloader=hub) + assert [r.status for r in second] == ["present"] + assert hub.downloads == downloads + + +@settings(suppress_health_check=[HealthCheck.function_scoped_fixture], deadline=None) +@given(members=member_maps, data=st.data()) +def test_a_file_that_misses_its_pin_is_never_installed(tmp_path_factory, members, data): + root = tmp_path_factory.mktemp("fetch") + victim = data.draw(st.sampled_from(sorted(members))) + pins = {name: _sha(value) for name, value in members.items()} + pins[victim] = _sha(members[victim] + b"tampered") + hub = FakeHub(root, {"survey.zip": _zip(members)}) + result = cut.fetch_inputs( + [_zip_source(members, pins=pins)], root / "inputs", downloader=hub + ) + assert result[0].status == "refused" + assert not (root / "inputs/survey" / victim).exists() + assert not list((root / "inputs").rglob("*.partial")) + + +def test_a_corrupted_installed_file_is_replaced_not_trusted(tmp_path): + members = {"a.tab": b"alpha", "b.tab": b"beta"} + hub = FakeHub(tmp_path, {"survey.zip": _zip(members)}) + source = _zip_source(members) + cut.fetch_inputs([source], tmp_path / "inputs", downloader=hub) + installed = tmp_path / "inputs/survey/a.tab" + installed.write_bytes(b"ALPHA") # same size, different bytes and mtime + result = cut.fetch_inputs([source], tmp_path / "inputs", downloader=hub) + assert result[0].status == "fetched" + assert installed.read_bytes() == b"alpha" + + +def test_a_missing_upload_is_reported_with_its_note(tmp_path): + source = cut.InputSource( + key="nts", + kind="hf_zip_members", + directory="nts", + files=(cut.PinnedFile("trip.tab", "0" * 64),), + repo_id="org/repo", + repo_type="model", + filename="nts.zip", + upload_note="upload nts.zip", + ) + result = cut.fetch_inputs( + [source], tmp_path / "inputs", downloader=FakeHub(tmp_path, {}) + ) + assert result[0].status == "missing" + assert "upload nts.zip" in result[0].detail + + +def test_the_archive_digest_is_checked_when_pinned(tmp_path): + members = {"a.tab": b"alpha"} + source = _zip_source(members) + pinned = cut.InputSource(**{**source.__dict__, "archive_sha256": "0" * 64}) + hub = FakeHub(tmp_path, {"survey.zip": _zip(members)}) + result = cut.fetch_inputs([pinned], tmp_path / "inputs", downloader=hub) + assert result[0].status == "refused" + assert "survey.zip hashes to" in result[0].detail + + +# --------------------------------------------------------------------------- # +# Resumability: each step runs once, and nothing is overwritten # +# --------------------------------------------------------------------------- # + + +class FakeBuild: + """Stands in for the five tools: writes what each step writes.""" + + def __init__(self, paths: cut.CutPaths, fail: set[str] | None = None) -> None: + self.paths = paths + self.calls: list[str] = [] + self.fail = fail or set() + + def __call__(self, argv: list[str], cwd: Path, log_path: Path) -> int: + tool = Path(argv[1]).name if argv[1] != "-m" else argv[2] + step = { + "build_uk_frs_spine.py": "spine", + "build_uk_full.py": "national-dry-run" + if "--dry-run" in argv + else "national", + "score_uk_national_candidate.py": "score", + "certify_uk_release_cut.py": "certify", + "assemble_uk_release_dir.py": "assemble", + "microcosm.data.publish_cli": "publish", + }[tool] + self.calls.append(step) + if step in self.fail: + return 1 + getattr(self, step.replace("-", "_"))(log_path) + return 0 + + def spine(self, log_path): + h5 = self.paths.spine_h5 + h5.write_bytes(b"spine") + h5.with_name(h5.name + ".sha256").write_text(f"{_sha(b'spine')} {h5.name}\n") + h5.with_suffix(".build.json").write_text("{}") + h5.with_suffix(".spine_gates.json").write_text('{"shippable": true}') + + def national_dry_run(self, log_path): + log_path.write_text('{"dry_run": true}') + + def national(self, log_path): + run = self.paths.run_dir + run.mkdir(parents=True) + self.paths.candidate_h5.write_bytes(b"candidate") + for name in ( + "calibration_diagnostics.json", + f"{cut.CANDIDATE_NAME}.terminal_gates.json", + "national_target_registry.json", + "national_contract_registry.json", + ): + (run / name).write_text("{}") + (run / "build_record.json").write_text( + json.dumps({"artifacts": {"staging_h5": {"sha256": _sha(b"candidate")}}}) + ) + + def score(self, log_path): + (self.paths.run_dir / "score_vs_incumbent.json").write_text( + json.dumps( + { + "artifacts": {"candidate": {"sha256": _sha(b"candidate")}}, + "evaluation": {"verdict": "passed"}, + } + ) + ) + + def certify(self, log_path): + ( + self.paths.run_dir / f"{cut.CANDIDATE_NAME}.release_certification.json" + ).write_text( + json.dumps({"shippable": True, "candidate": {"sha256": _sha(b"candidate")}}) + ) + + def assemble(self, log_path): + self.paths.release_dir.mkdir(parents=True) + (self.paths.release_dir / "release_manifest.json").write_text("{}") + log_path.write_text( + "assembling...\n" + + json.dumps({"cut_tag": "microcosm-uk-2024-25-national-x", "n": 1}) + ) + + def publish(self, log_path): + pass + + +def _runner(tmp_path, fake=None): + paths = _paths(tmp_path) + fake = fake or FakeBuild(paths) + sources = {s.key: s for s in cut.load_input_sources()} + runner = cut.Runner( + paths=paths, sources=sources, python="py", execute=fake, log=lambda _: None + ) + return runner, fake + + +def _all(runner): + cut.run_spine(runner, staging_local_only=True) + cut.run_national(runner, retry=False, logbook_prev_row_digest=None) + cut.run_score(runner) + cut.run_certify(runner, logbook_prev_row_digest=None) + return cut.run_assemble(runner) + + +def test_a_complete_run_executes_each_step_once_and_resumes_to_nothing(tmp_path): + runner, fake = _runner(tmp_path) + summary = _all(runner) + assert summary["tag"] == "microcosm-uk-2024-25-national-x" + assert fake.calls == [ + "spine", + "national-dry-run", + "national", + "score", + "certify", + "assemble", + ] + _all(runner) # every done-check now passes + assert len(fake.calls) == 6 + state = json.loads(runner.paths.state.read_text()) + assert {step: s["status"] for step, s in state["steps"].items()} == { + step: "done" + for step in ( + "spine", + "national-dry-run", + "national", + "score", + "certify", + "assemble", + ) + } + + +@pytest.mark.parametrize( + "failing", ["spine", "national", "score", "certify", "assemble"] +) +def test_a_failed_step_stops_the_run_and_a_rerun_resumes_there(tmp_path, failing): + runner, fake = _runner(tmp_path) + fake.fail = {failing} + with pytest.raises(cut.CutError, match=f"{failing} exited 1"): + _all(runner) + assert json.loads(runner.paths.state.read_text())["steps"][failing]["status"] == ( + "failed" + ) + before = list(fake.calls) + fake.fail = set() + if failing == "national": + # The tool refuses an occupied --out; the runner moves the failed + # attempt aside only when asked. + runner.paths.run_dir.mkdir(parents=True, exist_ok=True) + with pytest.raises(cut.CutError, match="--retry-national"): + _all(runner) + cut.run_national(runner, retry=True, logbook_prev_row_digest=None) + _all(runner) + assert fake.calls.count(failing) == before.count(failing) + 1 + # Steps that finished before the failure are not repeated. + for done in fake.calls[: before.index(failing)]: + assert fake.calls.count(done) == 1 + + +def test_an_incomplete_spine_is_never_rebuilt_over(tmp_path): + runner, fake = _runner(tmp_path) + runner.paths.spine_dir.mkdir(parents=True) + runner.paths.spine_h5.write_bytes(b"half a spine") + with pytest.raises(cut.CutError, match="would delete it first"): + cut.run_spine(runner, staging_local_only=True) + assert fake.calls == [] + assert runner.paths.spine_h5.read_bytes() == b"half a spine" + + +def test_retrying_the_national_run_keeps_the_failed_attempt(tmp_path): + runner, fake = _runner(tmp_path) + cut.run_spine(runner, staging_local_only=True) + runner.paths.run_dir.mkdir(parents=True) + (runner.paths.run_dir / "evidence.json").write_text("kept") + cut.run_national(runner, retry=True, logbook_prev_row_digest=None) + aside = list(runner.paths.cut.glob("national.failed-*")) + assert len(aside) == 1 and (aside[0] / "evidence.json").read_text() == "kept" + + +def test_a_losing_candidate_is_not_certified(tmp_path): + runner, fake = _runner(tmp_path) + + def failing_score(log_path): + (runner.paths.run_dir / "score_vs_incumbent.json").write_text( + json.dumps( + { + "artifacts": {"candidate": {"sha256": _sha(b"candidate")}}, + "evaluation": {"verdict": "failed"}, + } + ) + ) + + fake.score = failing_score + cut.run_spine(runner, staging_local_only=True) + cut.run_national(runner, retry=False, logbook_prev_row_digest=None) + with pytest.raises(cut.CutError, match="does not pass the incumbent score"): + cut.run_score(runner) + assert "certify" not in fake.calls + + +@given( + prefix=st.lists(st.text(alphabet=st.characters(blacklist_characters="{}"))), + tag=st.from_regex(r"[a-z0-9-]{1,40}", fullmatch=True), +) +def test_the_assembly_summary_is_found_after_any_log_noise(prefix, tag): + text = "\n".join(prefix) + "\n" + json.dumps({"cut_tag": tag, "nested": {"a": 1}}) + assert cut._last_json_object(text)["cut_tag"] == tag + + +def test_status_reads_without_running_anything(tmp_path): + paths = _paths(tmp_path) + out = io.StringIO() + with redirect_stdout(out): + cut.status(paths) + assert "todo spine" in out.getvalue() diff --git a/tools/modal_rebuild_uk_chronicle_artifact.py b/tools/modal_rebuild_uk_chronicle_artifact.py new file mode 100644 index 0000000000..c15d2acfc7 --- /dev/null +++ b/tools/modal_rebuild_uk_chronicle_artifact.py @@ -0,0 +1,298 @@ +"""Modal app: rebuild the pinned Chronicle UK consumer artifact off this machine. + +The UK national build pins one Chronicle consumer artifact by digest +(``packages/microcosm-build/src/microcosm/build/uk/chronicle_feed.json``): +``build-bundle --suite uk`` then ``build-consumer-artifact`` at one Chronicle +commit. Chronicle is public and its UK suite builds from committed source +files plus two DESNZ workbooks fetched from the publisher by pinned sha256 +(``LEDGER_SOURCE_ARTIFACT_FETCH=1``), so anyone can rebuild it. The bundle +takes tens of gigabytes of disk, more than a busy laptop has spare, so this +runs it in a Modal container and keeps only the artifact. + +The container clones Chronicle at the pinned commit, syncs its own +``uv.lock``, builds the bundle and the artifact, and hashes both artifact +files against the pin. Results go to the ``microcosm-uk-chronicle`` Modal +volume under ``//``, with a receipt beside them. + +Check the pin and print the plan (free):: + + modal run tools/modal_rebuild_uk_chronicle_artifact.py + +Rebuild (paid, a few hours):: + + modal run --detach tools/modal_rebuild_uk_chronicle_artifact.py --build + +Rebuild and upload to the private Hugging Face repo when the digests match +the pin (``HF_TOKEN`` must be exported locally with write access to +``policyengine/microcosm-uk-build-inputs``; it is passed to this run only, +never stored as a Modal secret):: + + modal run --detach tools/modal_rebuild_uk_chronicle_artifact.py --build --upload + +An artifact whose digests differ from the pin is never uploaded under the +pinned path; with ``--upload`` its receipt and bundle reports go to +``diagnostic/`` so the difference can be traced. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import resource +import shutil +import subprocess +import time +from datetime import UTC, datetime +from pathlib import Path + +import modal + +APP_NAME = "microcosm-uk-chronicle-rebuild" +VOLUME_NAME = "microcosm-uk-chronicle" +VOLUME_MOUNT = "/artifacts" +HF_REPO = "policyengine/microcosm-uk-build-inputs" +CHRONICLE_URL = "https://github.com/PolicyEngine/chronicle.git" +PYTHON_VERSION = "3.14" +UV_VERSION = "0.11.7" +HF_HUB_VERSION = "1.18.0" +FEED_PIN = Path(__file__).resolve().parents[1] / ( + "packages/microcosm-build/src/microcosm/build/uk/chronicle_feed.json" +) + + +def _load_pin() -> dict: + pin = json.loads(FEED_PIN.read_text()) + if pin.get("country") != "uk" or pin.get("source_repo") != "PolicyEngine/chronicle": + raise SystemExit(f"{FEED_PIN} is not the UK Chronicle feed pin.") + return pin + + +PIN = _load_pin() if modal.is_local() else json.loads(os.environ["CHRONICLE_PIN"]) + + +def _secrets() -> list[modal.Secret]: + if not modal.is_local(): + return [] + token = os.environ.get("HF_TOKEN") + # An ephemeral secret for this run only; nothing is stored in the + # workspace. Absent locally, the build runs and simply cannot upload. + return [modal.Secret.from_dict({"HF_TOKEN": token})] if token else [] + + +image = ( + modal.Image.debian_slim(python_version=PYTHON_VERSION) + .apt_install("git", "curl", "time") + .pip_install(f"uv=={UV_VERSION}", f"huggingface_hub=={HF_HUB_VERSION}") + .run_commands(f"uv python install {PYTHON_VERSION}") + .env( + { + "CHRONICLE_PIN": json.dumps(PIN, sort_keys=True), + "UV_LINK_MODE": "copy", + "PYTHONDONTWRITEBYTECODE": "1", + } + ) +) +app = modal.App(APP_NAME) +volume = modal.Volume.from_name(VOLUME_NAME, create_if_missing=True) + + +def _now() -> str: + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 22), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _run(argv: list[str], *, cwd: Path, env: dict[str, str], log: Path) -> dict: + started = time.monotonic() + with log.open("ab") as handle: + handle.write(f"\n$ {' '.join(argv)}\n".encode()) + handle.flush() + completed = subprocess.run( + ["/usr/bin/time", "-v", *argv], + cwd=cwd, + env=env, + stdout=handle, + stderr=subprocess.STDOUT, + ) + if completed.returncode != 0: + tail = log.read_text(errors="replace")[-4000:] + raise RuntimeError(f"{argv[:4]} exited {completed.returncode}:\n{tail}") + return {"argv": argv, "seconds": round(time.monotonic() - started, 1)} + + +@app.function( + image=image, + volumes={VOLUME_MOUNT: volume}, + secrets=_secrets(), + cpu=8.0, + memory=98304, + ephemeral_disk=512 * 1024, # Modal's floor; the bundle needs tens of GB + timeout=8 * 3600, +) +def rebuild(upload: bool) -> dict: + """Clone, build, hash and (optionally) upload; return the receipt.""" + + commit = PIN["source_commit"] + work = Path("/work") + repo = work / "chronicle" + bundle = work / "bundle" + artifact = work / "artifact" + work.mkdir(parents=True, exist_ok=True) + log = work / "build.log" + receipt: dict = { + "schema_version": 1, + "kind": "microcosm.uk_chronicle_artifact_rebuild", + "started_at": _now(), + "pin": PIN, + "chronicle_commit": commit, + "steps": [], + } + env = { + **os.environ, + "LEDGER_SOURCE_ARTIFACT_FETCH": "1", + "UV_PYTHON": PYTHON_VERSION, + "UV_PROJECT_ENVIRONMENT": str(work / "venv"), + } + env.pop("HF_TOKEN", None) # the build itself never needs the Hub + subprocess.run(["git", "clone", "--quiet", CHRONICLE_URL, str(repo)], check=True) + subprocess.run(["git", "-C", str(repo), "checkout", "--quiet", commit], check=True) + head = subprocess.check_output( + ["git", "-C", str(repo), "rev-parse", "HEAD"], text=True + ).strip() + if head != commit: + raise RuntimeError(f"checked out {head}, not the pinned {commit}") + for argv in ( + ["uv", "sync", "--locked"], + [ + "uv", + "run", + "--no-sync", + "chronicle", + "build-bundle", + "--suite", + "uk", + "--out", + str(bundle), + "--replace", + ], + [ + "uv", + "run", + "--no-sync", + "chronicle", + "build-consumer-artifact", + "--facts", + str(bundle), + "--out", + str(artifact), + "--replace", + ], + ): + receipt["steps"].append(_run(argv, cwd=repo, env=env, log=log)) + receipt["environment"] = { + "python": subprocess.check_output( + ["uv", "run", "--no-sync", "python", "--version"], + cwd=repo, + env=env, + text=True, + ).strip(), + "uv_lock_sha256": _sha256(repo / "uv.lock"), + "peak_child_rss_kib": resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss, + } + facts = artifact / "consumer_facts.jsonl" + manifest = artifact / "manifest.json" + observed = { + "facts_sha256": _sha256(facts), + "manifest_sha256": _sha256(manifest), + "facts_size_bytes": facts.stat().st_size, + "manifest": json.loads(manifest.read_text()), + } + matches = { + "facts_sha256": observed["facts_sha256"] == PIN["facts_sha256"], + "manifest_sha256": observed["manifest_sha256"] == PIN["manifest_sha256"], + "fact_row_count": observed["manifest"].get("fact_row_count") + == PIN["fact_row_count"], + } + receipt.update(observed=observed, matches=matches, reproduced=all(matches.values())) + + # Keep the artifact and the small bundle reports; drop the bundle. + out = Path(VOLUME_MOUNT) / commit / observed["facts_sha256"] + out.mkdir(parents=True, exist_ok=True) + for name in ("manifest.json", "consumer_facts.jsonl"): + shutil.copyfile(artifact / name, out / name) + reports = out / "bundle-reports" + reports.mkdir(exist_ok=True) + for name in ("coverage.json", "source_packages.json", "reports/build_bundle.json"): + source = bundle / name + if source.exists(): + shutil.copyfile(source, reports / Path(name).name) + shutil.copyfile(log, out / "build.log") + receipt["volume_path"] = f"{VOLUME_NAME}:{out.relative_to(VOLUME_MOUNT)}" + + if upload: + receipt["upload"] = _upload(out, receipt) + receipt["finished_at"] = _now() + (out / "receipt.json").write_text(json.dumps(receipt, indent=2, sort_keys=True)) + volume.commit() + return receipt + + +def _upload(out: Path, receipt: dict) -> dict: + from huggingface_hub import HfApi + + if not os.environ.get("HF_TOKEN"): + return {"uploaded": False, "reason": "HF_TOKEN was not exported locally"} + api = HfApi(token=os.environ["HF_TOKEN"]) + commit = receipt["chronicle_commit"] + if receipt["reproduced"]: + prefix = f"chronicle-uk/{commit}" + allow = ["manifest.json", "consumer_facts.jsonl"] + message = f"Add the Chronicle UK consumer artifact rebuilt at {commit}" + else: + # Never under the pinned path: only the evidence for a diff. + prefix = ( + f"diagnostic/chronicle-uk/{commit}/{receipt['observed']['facts_sha256']}" + ) + allow = ["manifest.json", "bundle-reports/*", "build.log"] + message = ( + f"Diagnostic: Chronicle UK rebuild at {commit} did not reproduce the pin" + ) + (out / "receipt.json").write_text(json.dumps(receipt, indent=2, sort_keys=True)) + allow.append("receipt.json") + result = api.upload_folder( + repo_id=HF_REPO, + repo_type="dataset", + folder_path=str(out), + path_in_repo=prefix, + allow_patterns=allow, + commit_message=message, + ) + return {"uploaded": True, "path_in_repo": prefix, "commit": result.oid} + + +@app.local_entrypoint() +def main(build: bool = False, upload: bool = False) -> None: + plan = { + "chronicle_commit": PIN["source_commit"], + "expected_facts_sha256": PIN["facts_sha256"], + "expected_manifest_sha256": PIN["manifest_sha256"], + "expected_rows": PIN["fact_row_count"], + "volume": VOLUME_NAME, + "upload_repo": HF_REPO if upload else None, + "hf_token_present": bool(os.environ.get("HF_TOKEN")), + } + print(json.dumps(plan, indent=2)) + if not build: + print("Plan only; pass --build to rebuild (paid).") + return + if upload and not os.environ.get("HF_TOKEN"): + raise SystemExit("--upload needs HF_TOKEN exported locally.") + receipt = rebuild.remote(upload) + print(json.dumps(receipt, indent=2, sort_keys=True)) diff --git a/tools/run_uk_cut.py b/tools/run_uk_cut.py new file mode 100644 index 0000000000..b6c8ddf3fc --- /dev/null +++ b/tools/run_uk_cut.py @@ -0,0 +1,1590 @@ +"""Build, sign and assemble a Microcosm UK national cut in one resumable command. + +A cut used to be five commands run by hand on one machine (spine, national +calibration, certify, assemble, publish; ``docs/uk-national-release-assembly-runbook-806.md``), +with inputs gathered from wherever they happened to be. This command runs +them in order from a clean checkout, with every input fetched and checked +against the pins the build itself enforces, so anyone with the credentials +can cut at the policyengine-uk version their checkout locks:: + + uv run --no-sync python tools/run_uk_cut.py status --label sat24 + uv run --no-sync python tools/run_uk_cut.py fetch + uv run --no-sync python tools/run_uk_cut.py preflight --label sat24 + uv run --no-sync python tools/run_uk_cut.py all --label sat24 + uv run --no-sync python tools/run_uk_cut.py all --label sat24 --publish + +``all`` is resumable: each step first checks whether its outputs already +exist and verify (the done-check), and skips if so. A step that fails leaves +its outputs for inspection; nothing here deletes a spine, a calibration run +or a release directory. Re-run ``all`` after fixing the cause; a failed +national run is moved aside only with ``--retry-national``. + +Outward-facing steps are opt-in. ``--publish`` publishes for inspection +only, always with ``--no-latest --tag-only``, so the repository's main +branch and its ``latest-*.json`` pointers never move. Promotion and the +policyengine.py certification are separate, deliberate calls; this command +prints them and runs neither. + +Credentials come from the environment: ``HF_TOKEN`` (write access to +``policyengine/populace-uk-private`` and ``policyengine/populace-uk-staging``, +read access to the input repositories) and +``MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY``, which must be registered in +``microcosm.data.gate_signing_keys``. See ``docs/uk-cut-anyone.md``. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import shlex +import shutil +import subprocess +import sys +import tempfile +import time +import zipfile +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from datetime import UTC, date, datetime +from pathlib import Path +from typing import Any, Protocol + +REPO_ROOT = Path(__file__).resolve().parents[1] +UK_DIR = REPO_ROOT / "packages/microcosm-build/src/microcosm/build/uk" +CUT_INPUTS = REPO_ROOT / "tools/uk_cut_inputs.json" +DEFAULT_WORK_DIR = Path("~/microcosm-uk-work").expanduser() +SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" +RELEASE_ID = "microcosm-uk-2024-25-national" +CANDIDATE_NAME = "microcosm_uk_2024_25" +SPINE_NAME = "uk_frs_spine_2024_25.h5" +PUBLISH_REPO = "policyengine/populace-uk-private" +TELEMETRY_REPO = "policyengine/populace-uk-staging" +STAGING_BRANCH = f"release-staging/{RELEASE_ID}" +DEFAULT_RESERVATION_TOOL = Path("~/reviews/fleet-ops/tools/reservation.py").expanduser() + + +class CutError(RuntimeError): + """A refusal the operator must act on; the message says how.""" + + +# --------------------------------------------------------------------------- # +# Pins and input sources # +# --------------------------------------------------------------------------- # + + +@dataclass(frozen=True) +class PinnedFile: + """One input file and the digest the build checks it against.""" + + name: str + sha256: str + size_bytes: int | None = None + + +@dataclass(frozen=True) +class InputSource: + """Where one group of input files comes from (``tools/uk_cut_inputs.json``).""" + + key: str + kind: str + directory: str + files: tuple[PinnedFile, ...] + repo_id: str | None = None + repo_type: str | None = None + revision: str | None = None + filename: str | None = None + path_in_repo: str | None = None + archive_sha256: str | None = None + url: str | None = None + upload_note: str | None = None + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 22), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _walk_artifacts(node: object): + if isinstance(node, Mapping): + if "sha256" in node and (node.get("filename") or node.get("locator")): + yield node + for value in node.values(): + yield from _walk_artifacts(value) + elif isinstance(node, list): + for value in node: + yield from _walk_artifacts(value) + + +def committed_source_pins(uk_dir: Path = UK_DIR) -> dict[str, dict[str, Any]]: + """Every file pin in ``source_stages.json``, by file name. + + Tabs are named by ``filename`` or ``locator``; the HMRC workbook by its + URL, whose last segment is its file name. A name pinned twice must carry + the same digest. + """ + + stages = json.loads((uk_dir / "source_stages.json").read_text()) + pins: dict[str, dict[str, Any]] = {} + for artifact in _walk_artifacts(stages): + raw = str(artifact.get("filename") or artifact.get("locator")) + name = raw.rsplit("/", 1)[-1] + pin = { + "sha256": str(artifact["sha256"]), + "size_bytes": artifact.get("size_bytes"), + "url": raw if raw.startswith("https://") else None, + } + previous = pins.get(name) + if previous is not None and previous["sha256"] != pin["sha256"]: + raise CutError(f"source_stages.json pins {name} to two digests.") + pins[name] = previous or pin + return pins + + +def _gate_reference_identity(uk_dir: Path) -> Mapping[str, str]: + gates = json.loads((uk_dir / "gates.json").read_text()) + entries = gates["gates"] if isinstance(gates, Mapping) else gates + for entry in entries: + if entry.get("gate") == "input_mass_parity": + params = entry["parameters"] + return params["reference_registry"][params["reference"]]["identity"] + raise CutError("gates.json declares no input_mass_parity reference identity.") + + +def load_input_sources( + manifest_path: Path = CUT_INPUTS, uk_dir: Path = UK_DIR +) -> list[InputSource]: + """Resolve each declared source to files and the digests the build checks.""" + + manifest = json.loads(manifest_path.read_text()) + if manifest.get("version") != 1: + raise CutError(f"{manifest_path}: version must be 1.") + stage_pins = committed_source_pins(uk_dir) + feed = json.loads((uk_dir / "chronicle_feed.json").read_text()) + frs_release = json.loads((uk_dir / "frs_release.json").read_text()) + reference = _gate_reference_identity(uk_dir) + extra = manifest.get("file_pins", {}) + sources: list[InputSource] = [] + for raw in manifest["sources"]: + key = raw["key"] + revision = raw.get("revision") + archive_sha256 = raw.get("archive_sha256") + url = None + files: list[PinnedFile] = [] + for member in raw["members"]: + if key == "chronicle_uk": + digest = { + "manifest.json": feed["manifest_sha256"], + "consumer_facts.jsonl": feed["facts_sha256"], + }[member] + files.append(PinnedFile(member, digest)) + elif key == "incumbent_efrs": + if member != reference["filename"]: + raise CutError( + f"incumbent member {member} is not the gate's " + f"reference {reference['filename']}." + ) + files.append(PinnedFile(member, reference["sha256"])) + elif member in extra: + files.append(PinnedFile(member, extra[member]["sha256"])) + elif member in stage_pins: + pin = stage_pins[member] + files.append(PinnedFile(member, pin["sha256"], pin["size_bytes"])) + url = url or pin["url"] + else: + raise CutError( + f"{key}: no committed pin for {member}; every input must be " + "pinned in source_stages.json, chronicle_feed.json or gates.json." + ) + if raw.get("archive_sha256_pin") == "frs_release": + acquisition = frs_release["acquisition"] + archive_sha256 = acquisition["zip_sha256"] + if acquisition["huggingface_revision"] != revision: + raise CutError( + "frs_2024_25 revision disagrees with uk/frs_release.json." + ) + if raw.get("revision_pin") == "gates:input_mass_parity": + revision = reference["revision"] + if raw["kind"] == "url" and not url: + raise CutError(f"{key}: the pin carries no https URL.") + sources.append( + InputSource( + key=key, + kind=raw["kind"], + directory=raw["directory"], + files=tuple(files), + repo_id=raw.get("repo_id"), + repo_type=raw.get("repo_type"), + revision=revision, + filename=raw.get("filename"), + path_in_repo=raw.get("path_in_repo"), + archive_sha256=archive_sha256, + url=url if raw["kind"] == "url" else None, + upload_note=raw.get("upload_note"), + ) + ) + return sources + + +# --------------------------------------------------------------------------- # +# Fetching # +# --------------------------------------------------------------------------- # + + +class Downloader(Protocol): + def hf_file( + self, *, repo_id: str, repo_type: str, filename: str, revision: str | None + ) -> Path: ... + + def url(self, url: str, dest: Path) -> Path: ... + + +class HubDownloader: + """Download through the Hugging Face cache (resumable, deduplicated).""" + + def __init__(self, cache_dir: Path) -> None: + self.cache_dir = cache_dir + + def hf_file( + self, *, repo_id: str, repo_type: str, filename: str, revision: str | None + ) -> Path: + from huggingface_hub import hf_hub_download + + return Path( + hf_hub_download( + repo_id, + filename, + repo_type=repo_type, + revision=revision, + cache_dir=str(self.cache_dir), + ) + ) + + def url(self, url: str, dest: Path) -> Path: + import urllib.request + + tmp = dest.with_name(dest.name + ".partial") + with ( + urllib.request.urlopen(url, timeout=120) as response, + tmp.open("wb") as handle, + ): + shutil.copyfileobj(response, handle) + tmp.replace(dest) + return dest + + +class VerifiedFiles: + """Remembers files already hashed, keyed by size and mtime. + + A file is re-hashed when either changes, or always with ``reverify``; + a cache entry never vouches for a file it did not hash. + """ + + def __init__(self, path: Path, *, reverify: bool = False) -> None: + self.path = path + self.reverify = reverify + try: + self.entries: dict[str, dict[str, Any]] = json.loads(path.read_text()) + except (FileNotFoundError, json.JSONDecodeError): + self.entries = {} + + def matches(self, file: Path, pin: PinnedFile) -> bool: + if not file.is_file(): + return False + stat = file.stat() + if pin.size_bytes is not None and stat.st_size != pin.size_bytes: + return False + key = str(file) + cached = self.entries.get(key) + if ( + not self.reverify + and cached is not None + and cached.get("size") == stat.st_size + and cached.get("mtime_ns") == stat.st_mtime_ns + ): + return cached.get("sha256") == pin.sha256 + digest = _sha256_file(file) + self.entries[key] = { + "size": stat.st_size, + "mtime_ns": stat.st_mtime_ns, + "sha256": digest, + } + return digest == pin.sha256 + + def save(self) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + tmp = self.path.with_name(self.path.name + ".tmp") + tmp.write_text(json.dumps(self.entries, indent=2, sort_keys=True)) + tmp.replace(self.path) + + +def _place(source: Path, dest: Path, pin: PinnedFile) -> None: + """Copy (clone where the filesystem can) and verify before the rename.""" + + dest.parent.mkdir(parents=True, exist_ok=True) + tmp = dest.with_name(dest.name + ".partial") + shutil.copyfile(source, tmp) + digest = _sha256_file(tmp) + if digest != pin.sha256: + tmp.unlink() + raise CutError( + f"{source} hashes to {digest}, not the pinned {pin.sha256} for " + f"{pin.name}; it was not installed." + ) + tmp.replace(dest) + + +def _extract_member(archive: Path, pin: PinnedFile, dest: Path) -> None: + with zipfile.ZipFile(archive) as bundle: + matches = [ + info + for info in bundle.infolist() + if Path(info.filename).name == pin.name + and not info.filename.startswith("__MACOSX") + and not info.is_dir() + ] + if len(matches) != 1: + raise CutError( + f"{archive.name} holds {len(matches)} members named {pin.name}; " + "expected exactly one." + ) + dest.parent.mkdir(parents=True, exist_ok=True) + tmp = dest.with_name(dest.name + ".partial") + digest = hashlib.sha256() + with bundle.open(matches[0]) as member, tmp.open("wb") as handle: + for chunk in iter(lambda: member.read(1 << 22), b""): + digest.update(chunk) + handle.write(chunk) + if digest.hexdigest() != pin.sha256: + tmp.unlink() + raise CutError( + f"{archive.name}:{pin.name} hashes to {digest.hexdigest()}, not the " + f"pinned {pin.sha256}; it was not installed." + ) + tmp.replace(dest) + + +@dataclass +class FetchResult: + key: str + status: str # present | fetched | missing | refused + detail: str = "" + files: dict[str, str] = field(default_factory=dict) + + +def fetch_inputs( + sources: Sequence[InputSource], + inputs_dir: Path, + *, + downloader: Downloader, + reverify: bool = False, + only: set[str] | None = None, +) -> list[FetchResult]: + """Install every pinned input under ``inputs_dir`` and verify it. + + Idempotent: a file already present with its pinned digest is left alone. + Nothing unverified is ever left at an input's final path. + """ + + verified = VerifiedFiles(inputs_dir / ".verified.json", reverify=reverify) + results: list[FetchResult] = [] + try: + for source in sources: + if only and source.key not in only: + continue + target = inputs_dir / source.directory + paths = {pin.name: target / pin.name for pin in source.files} + if all(verified.matches(paths[pin.name], pin) for pin in source.files): + results.append( + FetchResult( + source.key, + "present", + files={n: str(p) for n, p in paths.items()}, + ) + ) + continue + try: + _fetch_source(source, paths, downloader=downloader, verified=verified) + except CutError as error: + results.append(FetchResult(source.key, "refused", str(error))) + continue + except Exception as error: # the Hub's 404/401, a dropped connection + status = "missing" if _looks_missing(error) else "refused" + detail = f"{type(error).__name__}: {error}" + if status == "missing" and source.upload_note: + detail += f" | needs upload: {source.upload_note}" + results.append(FetchResult(source.key, status, detail)) + continue + results.append( + FetchResult( + source.key, "fetched", files={n: str(p) for n, p in paths.items()} + ) + ) + finally: + verified.save() + return results + + +def _looks_missing(error: Exception) -> bool: + name = type(error).__name__ + return name in {"EntryNotFoundError", "RemoteEntryNotFoundError"} or ( + "404" in str(error) + ) + + +def _fetch_source( + source: InputSource, + paths: Mapping[str, Path], + *, + downloader: Downloader, + verified: VerifiedFiles, +) -> None: + pending = [ + pin for pin in source.files if not verified.matches(paths[pin.name], pin) + ] + if source.kind == "hf_zip_members": + archive = downloader.hf_file( + repo_id=source.repo_id, + repo_type=source.repo_type, + filename=source.filename, + revision=source.revision, + ) + if source.archive_sha256 is not None: + digest = _sha256_file(archive) + if digest != source.archive_sha256: + raise CutError( + f"{source.filename} hashes to {digest}, not the pinned " + f"{source.archive_sha256}." + ) + for pin in pending: + _extract_member(archive, pin, paths[pin.name]) + elif source.kind == "hf_files": + for pin in pending: + name = ( + f"{source.path_in_repo}/{pin.name}" if source.path_in_repo else pin.name + ) + local = downloader.hf_file( + repo_id=source.repo_id, + repo_type=source.repo_type, + filename=name, + revision=source.revision, + ) + _place(local, paths[pin.name], pin) + elif source.kind == "url": + for pin in pending: + with tempfile.TemporaryDirectory() as scratch: + local = downloader.url(source.url, Path(scratch) / pin.name) + _place(local, paths[pin.name], pin) + else: + raise CutError(f"{source.key}: unknown source kind {source.kind!r}.") + for pin in pending: + if not verified.matches(paths[pin.name], pin): + raise CutError(f"{paths[pin.name]} did not verify after install.") + + +# --------------------------------------------------------------------------- # +# The cut's layout # +# --------------------------------------------------------------------------- # + + +@dataclass(frozen=True) +class CutPaths: + """Every path a cut reads or writes, derived from the work dir, label and commit.""" + + work_dir: Path + label: str + commit: str + + @property + def inputs(self) -> Path: + return self.work_dir / "inputs" + + @property + def cut(self) -> Path: + return self.work_dir / "cuts" / f"{self.label}-{self.commit[:9]}" + + @property + def spine_dir(self) -> Path: + return self.cut / "spine" + + @property + def spine_h5(self) -> Path: + return self.spine_dir / SPINE_NAME + + @property + def run_dir(self) -> Path: + return self.cut / "national" + + @property + def candidate_h5(self) -> Path: + return self.run_dir / f"{CANDIDATE_NAME}.h5" + + @property + def release_parent(self) -> Path: + return self.cut / "release" + + @property + def release_dir(self) -> Path: + return self.release_parent / "releases" / RELEASE_ID + + @property + def logs(self) -> Path: + return self.cut / "logs" + + @property + def diagnostics(self) -> Path: + return self.cut / "diagnostics" + + @property + def state(self) -> Path: + return self.cut / "state.json" + + @property + def receipt(self) -> Path: + return self.cut / "cut_receipt.json" + + def input_file(self, directory: str, name: str) -> Path: + return self.inputs / directory / name + + +def spine_argv(paths: CutPaths, *, python: str, staging_local_only: bool) -> list[str]: + """The spine command, every input taken from the verified inputs dir.""" + + def tab(directory: str, name: str) -> str: + return str(paths.input_file(directory, name)) + + argv = [ + python, + str(REPO_ROOT / "tools/build_uk_frs_spine.py"), + "--frs-raw-dir", str(paths.inputs / "frs_2024_25"), + "--spi-tab", tab("spi_2022_23", "put2223uk.tab"), + "--hmrc-ods", tab("hmrc", "Collated_Tables_3_1_to_3_11_2324.ods"), + "--was-tab", tab("was_round_8", "was_round_8_hhold_eul_may_2025_230525.tab"), + "--was-person-tab", + tab("was_round_8", "was_round_8_person_eul_may_2025_230525.tab"), + "--nts-household-tab", tab("nts_2002_2024", "household_eul_2002-2024.tab"), + "--nts-individual-tab", tab("nts_2002_2024", "individual_eul_2002-2024.tab"), + "--nts-trip-tab", tab("nts_2002_2024", "trip_eul_2002-2024.tab"), + "--nts-stage-tab", tab("nts_2002_2024", "stage_eul_2002-2024.tab"), + "--nts-ticket-tab", tab("nts_2002_2024", "ticket_eul_2002-2024.tab"), + "--lcfs-hh-tab", tab("lcfs_2024_25", "dvhh_uk_anon_v3_2024.tab"), + "--lcfs-person-tab", tab("lcfs_2024_25", "dvper_uk_anon_2024.tab"), + "--etb-tab", tab("etb_1977_2024", "householdv2_1977-2024.tab"), + "--release-candidate", + "--spine-h5", str(paths.spine_h5), + ] # fmt: skip + if staging_local_only: + argv.append("--staging-local-only") + return argv + + +def national_argv( + paths: CutPaths, + *, + python: str, + spine_sha256: str, + sources: Mapping[str, InputSource], + dry_run: bool, + logbook_prev_row_digest: str | None = None, +) -> list[str]: + chronicle = {pin.name: pin.sha256 for pin in sources["chronicle_uk"].files} + incumbent = sources["incumbent_efrs"].files[0] + argv = [ + python, + str(REPO_ROOT / "tools/build_uk_full.py"), + "--release-role", "national", + "--input-h5", str(paths.spine_h5), + "--input-sha256", spine_sha256, + "--ledger-facts", str(paths.inputs / "chronicle_uk"), + "--ledger-facts-sha256", chronicle["consumer_facts.jsonl"], + "--ledger-manifest-sha256", chronicle["manifest.json"], + "--incumbent-h5", str(paths.input_file("incumbent", incumbent.name)), + "--incumbent-sha256", incumbent.sha256, + "--out", str(paths.run_dir), + ] # fmt: skip + if logbook_prev_row_digest: + argv += ["--logbook-prev-row-digest", logbook_prev_row_digest] + if dry_run: + argv.append("--dry-run") + return argv + + +def score_argv( + paths: CutPaths, *, python: str, candidate_sha256: str, sources: Mapping +) -> list[str]: + incumbent = sources["incumbent_efrs"].files[0] + return [ + python, + str(REPO_ROOT / "tools/score_uk_national_candidate.py"), + "--candidate-h5", str(paths.candidate_h5), + "--candidate-sha256", candidate_sha256, + "--incumbent-h5", str(paths.input_file("incumbent", incumbent.name)), + "--incumbent-sha256", incumbent.sha256, + "--registry-json", str(paths.run_dir / "national_target_registry.json"), + "--band-edge-registry-json", + str(paths.run_dir / "national_contract_registry.json"), + "--output-json", str(paths.run_dir / "score_vs_incumbent.json"), + ] # fmt: skip + + +def certify_argv( + paths: CutPaths, + *, + python: str, + candidate_sha256: str, + spine_sha256: str, + sources: Mapping, + logbook_prev_row_digest: str | None = None, +) -> list[str]: + chronicle = {pin.name: pin.sha256 for pin in sources["chronicle_uk"].files} + reference = sources["input_mass_reference"].files[0] + argv = [ + python, + str(REPO_ROOT / "tools/certify_uk_release_cut.py"), + "--candidate-h5", str(paths.candidate_h5), + "--candidate-sha256", candidate_sha256, + "--candidate-name", CANDIDATE_NAME, + "--spine-h5", str(paths.spine_h5), + "--spine-sha256", spine_sha256, + "--diagnostics-json", str(paths.run_dir / "calibration_diagnostics.json"), + "--build-record-json", str(paths.run_dir / "build_record.json"), + "--seam-gate-report", + str(paths.run_dir / f"{CANDIDATE_NAME}.terminal_gates.json"), + "--ledger-facts", str(paths.inputs / "chronicle_uk"), + "--ledger-facts-sha256", chronicle["consumer_facts.jsonl"], + "--ledger-manifest-sha256", chronicle["manifest.json"], + "--input-mass-reference", + str(paths.input_file("input_mass_reference", reference.name)), + "--score-receipt", str(paths.run_dir / "score_vs_incumbent.json"), + "--release-id", RELEASE_ID, + ] # fmt: skip + if logbook_prev_row_digest: + argv += ["--logbook-prev-row-digest", logbook_prev_row_digest] + return argv + + +def assemble_argv(paths: CutPaths, *, python: str) -> list[str]: + return [ + python, + str(REPO_ROOT / "tools/assemble_uk_release_dir.py"), + "--candidate-h5", str(paths.candidate_h5), + "--spine-h5", str(paths.spine_h5), + "--certification-json", + str(paths.run_dir / f"{CANDIDATE_NAME}.release_certification.json"), + "--build-record-json", str(paths.run_dir / "build_record.json"), + "--diagnostics-json", str(paths.run_dir / "calibration_diagnostics.json"), + "--seam-gate-report", + str(paths.run_dir / f"{CANDIDATE_NAME}.terminal_gates.json"), + "--release-cut-gate-json", + str(paths.run_dir / f"{CANDIDATE_NAME}.release_cut_gates.json"), + "--score-receipt", str(paths.run_dir / "score_vs_incumbent.json"), + "--out-dir", "releases", + ] # fmt: skip + + +def publish_argv( + paths: CutPaths, *, python: str, tag: str, preflight_only: bool +) -> list[str]: + """Inspection publication: never the default line, never main's files.""" + + argv = [ + python, "-m", "microcosm.data.publish_cli", str(paths.release_dir), + "--repo-id", PUBLISH_REPO, + "--artifact-root", str(paths.run_dir), + "--no-latest", "--tag-only", + "--tag-name", tag, + ] # fmt: skip + if preflight_only: + argv.append("--preflight-only") + return argv + + +def promote_command(paths: CutPaths, tag: str) -> str: + return shlex.join( + [ + "uv", "run", "--no-sync", "python", "-m", "microcosm.data.publish_cli", + str(paths.release_dir), "--repo-id", PUBLISH_REPO, + "--artifact-root", str(paths.run_dir), + "--promote-line", "national", "--tag-name", tag, + ] + ) # fmt: skip + + +# --------------------------------------------------------------------------- # +# Done-checks # +# --------------------------------------------------------------------------- # + + +def _read_json(path: Path) -> dict[str, Any] | None: + try: + value = json.loads(path.read_text()) + except (FileNotFoundError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def spine_sha256(paths: CutPaths) -> str | None: + sidecar = paths.spine_h5.with_name(paths.spine_h5.name + ".sha256") + try: + return sidecar.read_text().split()[0] + except (FileNotFoundError, IndexError): + return None + + +def spine_done(paths: CutPaths) -> tuple[bool, str]: + """A complete spine: H5, sidecars, matching digest, shippable gate report.""" + + h5 = paths.spine_h5 + if not h5.exists(): + return False, "no spine yet" + recorded = spine_sha256(paths) + for suffix in (".build.json", ".spine_gates.json"): + if not h5.with_suffix(suffix).exists(): + return False, f"spine sidecar {h5.with_suffix(suffix).name} is missing" + if recorded is None: + return False, "spine .sha256 sidecar is missing" + if _sha256_file(h5) != recorded: + return False, "spine H5 does not match its .sha256 sidecar" + gates = _read_json(h5.with_suffix(".spine_gates.json")) or {} + if gates.get("shippable") is not True: + return False, "spine gate report is not shippable" + return True, f"spine {recorded}" + + +def national_done(paths: CutPaths) -> tuple[bool, str]: + record = _read_json(paths.run_dir / "build_record.json") + if record is None: + return False, "no build_record.json" + for name in ( + f"{CANDIDATE_NAME}.h5", + "calibration_diagnostics.json", + f"{CANDIDATE_NAME}.terminal_gates.json", + "national_target_registry.json", + "national_contract_registry.json", + ): + if not (paths.run_dir / name).exists(): + return False, f"{name} is missing" + expected = (record.get("artifacts") or {}).get("staging_h5", {}).get("sha256") + if not expected or _sha256_file(paths.candidate_h5) != expected: + return False, "candidate H5 does not match build_record.json" + return True, f"candidate {expected}" + + +def candidate_sha256(paths: CutPaths) -> str: + record = _read_json(paths.run_dir / "build_record.json") or {} + return str(record["artifacts"]["staging_h5"]["sha256"]) + + +def score_done(paths: CutPaths) -> tuple[bool, str]: + """The fields the certifier reads: the scored candidate and the verdict.""" + + receipt = _read_json(paths.run_dir / "score_vs_incumbent.json") + if receipt is None: + return False, "no score receipt" + scored = ((receipt.get("artifacts") or {}).get("candidate") or {}).get("sha256") + if scored != candidate_sha256(paths): + return False, "score receipt names another candidate" + verdict = (receipt.get("evaluation") or {}).get("verdict") + if verdict != "passed": + return False, f"score verdict is {verdict!r}" + return True, "score passed" + + +def certify_done(paths: CutPaths) -> tuple[bool, str]: + cert = _read_json(paths.run_dir / f"{CANDIDATE_NAME}.release_certification.json") + if cert is None: + return False, "no release certification" + if cert.get("shippable") is not True: + return False, "certification is not shippable" + if (cert.get("candidate") or {}).get("sha256") != candidate_sha256(paths): + return False, "certification names another candidate" + return True, "certified shippable" + + +def assemble_done(paths: CutPaths) -> tuple[bool, str]: + manifest = paths.release_dir / "release_manifest.json" + summary = _read_json(paths.cut / "assemble_summary.json") + if not manifest.exists(): + return False, "no release directory" + if summary is None or not summary.get("tag"): + return False, "release directory exists but its assembly summary is missing" + return True, f"assembled {summary['tag']}" + + +# --------------------------------------------------------------------------- # +# Preflight # +# --------------------------------------------------------------------------- # + + +@dataclass +class Check: + name: str + ok: bool + detail: str + blocking: bool = True + + +def _git(*argv: str, cwd: Path = REPO_ROOT) -> str: + return subprocess.check_output(["git", *argv], cwd=cwd, text=True).strip() + + +def git_checks(*, allow_off_main: str | None) -> list[Check]: + checks = [] + dirty = _git("status", "--porcelain", "--untracked-files=no") + checks.append(Check("clean checkout", not dirty, dirty or "clean")) + head = _git("rev-parse", "HEAD") + subprocess.run( + ["git", "fetch", "--quiet", "origin", "main"], cwd=REPO_ROOT, check=False + ) + on_main = ( + subprocess.run( + ["git", "merge-base", "--is-ancestor", head, "origin/main"], + cwd=REPO_ROOT, + check=False, + ).returncode + == 0 + ) + if on_main: + checks.append(Check("commit on origin/main", True, head)) + else: + pushed = bool(_git("branch", "-r", "--contains", head)) + checks.append( + Check( + "commit on origin/main", + bool(allow_off_main) and pushed, + ( + f"{head} is not on origin/main" + + ( + f"; allowed ({allow_off_main})" + if allow_off_main and pushed + else "; build from main, or pass --allow-off-main " + "'' for a pushed commit" + ) + ), + ) + ) + return checks + + +def environment_checks() -> list[Check]: + checks = [] + import microcosm.build as build_package + + location = Path(build_package.__file__).resolve() + checks.append( + Check( + "env built from this checkout", + REPO_ROOT in location.parents, + f"microcosm.build from {location}", + ) + ) + try: + from microcosm.build.uk_runtime.hmrc_uprating import ( + assert_installed_engine_is_pinned, + ) + + version = assert_installed_engine_is_pinned() + checks.append(Check("policyengine-uk pinned", True, str(version))) + except Exception as error: + checks.append(Check("policyengine-uk pinned", False, str(error))) + return checks + + +def signing_key_check() -> Check: + from microcosm.data.gate_signing_keys import ( + GateSigningKeyError, + decode_gate_signing_key, + gate_signing_key_fingerprint, + require_registered_gate_signing_key, + ) + + try: + key = decode_gate_signing_key( + os.environ.get(SIGNING_KEY_ENV), env=SIGNING_KEY_ENV + ) + require_registered_gate_signing_key("uk", key, purpose="sign") + except GateSigningKeyError as error: + return Check("signing key registered", False, str(error)) + return Check("signing key registered", True, gate_signing_key_fingerprint(key)) + + +def hub_checks() -> list[Check]: + checks = [] + if not os.environ.get("HF_TOKEN"): + return [ + Check("HF_TOKEN", False, "export HF_TOKEN (write to the release repos)") + ] + try: + from huggingface_hub import HfApi + + from microcosm.build.staging_storage import HuggingFaceDatasetStorage + + api = HfApi() + for repo in (PUBLISH_REPO, TELEMETRY_REPO): + storage = HuggingFaceDatasetStorage(repo, api=api) + storage.head_revision() + can_write = storage.credential_can_write() + checks.append( + Check( + f"write {repo}", + can_write is not False, + "write scope confirmed" + if can_write + else "scope unreadable; the upload will prove it", + ) + ) + refs = api.list_repo_refs(PUBLISH_REPO, repo_type="dataset") + stale = [b.name for b in refs.branches if b.name == STAGING_BRANCH] + checks.append( + Check( + "no leftover release-staging branch", + not stale, + f"delete {STAGING_BRANCH} after checking no publish is running" + if stale + else "none", + ) + ) + except Exception as error: + checks.append(Check("Hugging Face reachable", False, f"{type(error).__name__}")) + return checks + + +def register_expiry_check(today: date, *, warn_days: int = 7) -> Check: + earliest: tuple[str, str] | None = None + for path in sorted(UK_DIR.glob("*.json")): + text = path.read_text() + if "expires_on" not in text: + continue + for match in _expiries(json.loads(text)): + if earliest is None or match < earliest[0]: + earliest = (match, path.name) + if earliest is None: + return Check("register expiries", True, "none") + days = (date.fromisoformat(earliest[0]) - today).days + detail = f"earliest {earliest[0]} in {earliest[1]} ({days} days)" + if days < 0: + return Check("register expiries", False, detail + ": expired") + # An entry expiring soon still builds today; say so without blocking. + return Check("register expiries", days >= warn_days, detail, blocking=False) + + +def _expiries(node: object): + if isinstance(node, Mapping): + value = node.get("expires_on") + if isinstance(value, str) and len(value) == 10: + yield value + for child in node.values(): + yield from _expiries(child) + elif isinstance(node, list): + for child in node: + yield from _expiries(child) + + +def host_checks( + work_dir: Path, *, min_free_gb: float, reservation_caller: str | None +) -> list[Check]: + checks = [] + usage = shutil.disk_usage(work_dir if work_dir.exists() else work_dir.parent) + free_gb = usage.free / 1e9 + checks.append( + Check( + "free disk", + free_gb >= min_free_gb, + f"{free_gb:.1f} GB free; floor {min_free_gb:.0f} GB", + ) + ) + if platform.system() == "Darwin": + power = subprocess.run( + ["pmset", "-g", "batt"], capture_output=True, text=True, check=False + ).stdout + on_ac = "AC Power" in power.splitlines()[0] if power else True + checks.append( + Check( + "on AC power", + on_ac, + "AC" if on_ac else "on battery: a closed lid stops long runs", + blocking=False, + ) + ) + load5 = os.getloadavg()[1] + checks.append( + Check( + "load", + True, + f"load5 {load5:.1f} on {os.cpu_count()} CPUs", + blocking=False, + ) + ) + if reservation_caller and DEFAULT_RESERVATION_TOOL.exists(): + allowed = subprocess.run( + [ + sys.executable, + str(DEFAULT_RESERVATION_TOOL), + "allowed", + reservation_caller, + ], + capture_output=True, + text=True, + check=False, + ) + checks.append( + Check( + "host reservation", + allowed.returncode == 0, + (allowed.stdout or allowed.stderr).strip()[:200] or "allowed", + ) + ) + return checks + + +def contract_pin_check(python: str) -> Check: + tests = [ + "packages/microcosm-build/tests/engine_free/shared/test_gate_battery_contract_pins.py", + "packages/microcosm-data/tests/engine_free/shared/test_contract.py", + ] + run = subprocess.run( + [python, "-m", "pytest", "-q", "-p", "no:cacheprovider", "-x", *tests], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + tail = (run.stdout or run.stderr).strip().splitlines()[-1:] or [""] + return Check("contract pins agree with the gate spec", run.returncode == 0, tail[0]) + + +def surfaces_check(python: str, chronicle_dir: Path) -> Check: + """The feed-gated surface tests that skip in CI (runbook: stale receipts).""" + + env = {**os.environ, "CHRONICLE_UK_FACTS": str(chronicle_dir)} + run = subprocess.run( + [ + python, "-m", "pytest", "-q", "-p", "no:cacheprovider", + "packages/microcosm-build/tests/engine_free/uk/test_uk_target_references.py", + ], + cwd=REPO_ROOT, + capture_output=True, + text=True, + env=env, + check=False, + ) # fmt: skip + tail = (run.stdout or run.stderr).strip().splitlines()[-1:] or [""] + return Check( + "target surfaces current against the pinned feed", run.returncode == 0, tail[0] + ) + + +# --------------------------------------------------------------------------- # +# Running steps # +# --------------------------------------------------------------------------- # + + +@dataclass +class Runner: + paths: CutPaths + sources: Mapping[str, InputSource] + python: str = sys.executable + execute: Callable[[list[str], Path, Path], int] | None = None + log: Callable[[str], None] = print + + def __post_init__(self) -> None: + if self.execute is None: + self.execute = _execute + + def state(self) -> dict[str, Any]: + return _read_json(self.paths.state) or { + "label": self.paths.label, + "commit": self.paths.commit, + "steps": {}, + } + + def record(self, step: str, **fields: Any) -> None: + state = self.state() + state["steps"].setdefault(step, {}).update(fields) + self.paths.cut.mkdir(parents=True, exist_ok=True) + tmp = self.paths.state.with_name("state.json.tmp") + tmp.write_text(json.dumps(state, indent=2, sort_keys=True)) + tmp.replace(self.paths.state) + + def run(self, step: str, argv: list[str], *, cwd: Path = REPO_ROOT) -> None: + self.paths.logs.mkdir(parents=True, exist_ok=True) + log_path = self.paths.logs / f"{step}.log" + started = time.monotonic() + self.record(step, status="running", started_at=_now(), argv=argv) + self.log(f"[{step}] {shlex.join(argv)}") + code = self.execute(argv, cwd, log_path) + seconds = round(time.monotonic() - started, 1) + if code != 0: + self.record(step, status="failed", exit_code=code, seconds=seconds) + raise CutError(f"{step} exited {code}; see {log_path}") + self.record(step, status="done", finished_at=_now(), seconds=seconds) + + +def _now() -> str: + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _execute(argv: list[str], cwd: Path, log_path: Path) -> int: + """Run one step, teeing its output to the console and its log.""" + + with log_path.open("a") as log: + log.write(f"\n# {_now()} $ {shlex.join(argv)}\n") + log.flush() + process = subprocess.Popen( + argv, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + assert process.stdout is not None + for line in process.stdout: + sys.stdout.write(line) + log.write(line) + return process.wait() + + +def run_spine(runner: Runner, *, staging_local_only: bool) -> None: + paths = runner.paths + done, detail = spine_done(paths) + if done: + runner.log(f"[spine] done: {detail}") + return + if paths.spine_h5.exists(): + raise CutError( + f"a spine exists at {paths.spine_h5} but is incomplete ({detail}). A " + "rerun at this path would delete it first; inspect it, then use a " + "new --label." + ) + paths.spine_dir.mkdir(parents=True, exist_ok=True) + runner.run( + "spine", + spine_argv(paths, python=runner.python, staging_local_only=staging_local_only), + ) + done, detail = spine_done(paths) + if not done: + raise CutError(f"the spine finished but is not usable: {detail}") + + +def run_national( + runner: Runner, *, retry: bool, logbook_prev_row_digest: str | None +) -> None: + paths = runner.paths + done, detail = national_done(paths) + if done: + runner.log(f"[national] done: {detail}") + return + spine = spine_sha256(paths) + if spine is None: + raise CutError("the national step needs a finished spine.") + if paths.run_dir.exists(): + if not retry: + raise CutError( + f"{paths.run_dir} holds an unfinished national run ({detail}); " + "pass --retry-national to move it aside and start a fresh one." + ) + aside = paths.run_dir.with_name( + f"national.failed-{datetime.now(UTC):%Y%m%dT%H%M%SZ}" + ) + paths.run_dir.rename(aside) + runner.log(f"[national] moved the unfinished run to {aside}") + common = dict( + python=runner.python, + spine_sha256=spine, + sources=runner.sources, + logbook_prev_row_digest=logbook_prev_row_digest, + ) + plan_log = paths.logs / "national-dry-run.log" + if not plan_log.exists() or "national-dry-run" not in runner.state()["steps"]: + runner.run("national-dry-run", national_argv(paths, dry_run=True, **common)) + runner.run("national", national_argv(paths, dry_run=False, **common)) + done, detail = national_done(paths) + if not done: + raise CutError(f"the national run finished but is not usable: {detail}") + + +def run_score(runner: Runner) -> None: + paths = runner.paths + done, detail = score_done(paths) + if done: + runner.log(f"[score] done: {detail}") + return + runner.log(f"[score] re-scoring the candidate ({detail})") + runner.run( + "score", + score_argv( + paths, + python=runner.python, + candidate_sha256=candidate_sha256(paths), + sources=runner.sources, + ), + ) + done, detail = score_done(paths) + if not done: + raise CutError(f"the candidate does not pass the incumbent score: {detail}") + + +def run_certify(runner: Runner, *, logbook_prev_row_digest: str | None) -> None: + paths = runner.paths + done, detail = certify_done(paths) + if done: + runner.log(f"[certify] done: {detail}") + return + runner.run( + "certify", + certify_argv( + paths, + python=runner.python, + candidate_sha256=candidate_sha256(paths), + spine_sha256=spine_sha256(paths), + sources=runner.sources, + logbook_prev_row_digest=logbook_prev_row_digest, + ), + ) + done, detail = certify_done(paths) + if not done: + raise CutError(f"the release cut is not certified: {detail}") + + +def run_stage1(runner: Runner, previous_h5: Path | None) -> None: + paths = runner.paths + output = paths.diagnostics / "stage1_comparison.json" + if output.exists(): + runner.log(f"[stage1] done: {output}") + return + if previous_h5 is None: + runner.record("stage1", status="skipped", reason="no --previous-h5") + runner.log( + "[stage1] skipped: pass --previous-h5 for the " + "interim stage-1 comparison (diagnostic; microcosm#1276)" + ) + return + paths.diagnostics.mkdir(parents=True, exist_ok=True) + runner.run( + "stage1", + [ + runner.python, + str(REPO_ROOT / "tools/measure_uk_weighted_integrity_baselines.py"), + "--h5", f"candidate={paths.candidate_h5}", + "--h5", f"previous={previous_h5}", + "--output", str(output), + ], + ) # fmt: skip + + +def run_assemble(runner: Runner) -> dict[str, Any]: + paths = runner.paths + done, detail = assemble_done(paths) + if done: + runner.log(f"[assemble] done: {detail}") + return _read_json(paths.cut / "assemble_summary.json") or {} + if paths.release_dir.exists(): + raise CutError( + f"{paths.release_dir} exists without an assembly summary; inspect it. " + "Assembly refuses an existing release directory." + ) + paths.release_parent.mkdir(parents=True, exist_ok=True) + runner.run( + "assemble", assemble_argv(paths, python=runner.python), cwd=paths.release_parent + ) + summary = _last_json_object((paths.logs / "assemble.log").read_text()) + tag = summary.get("tag") or summary.get("cut_tag") or summary.get("tag_name") + if not tag: + raise CutError("assembly printed no cut tag; see logs/assemble.log") + summary["tag"] = tag + (paths.cut / "assemble_summary.json").write_text( + json.dumps(summary, indent=2, sort_keys=True) + ) + return summary + + +def _last_json_object(text: str) -> dict[str, Any]: + """The last top-level JSON object a tool printed (its summary).""" + + decoder = json.JSONDecoder() + found: dict[str, Any] = {} + index = 0 + while (start := text.find("{", index)) != -1: + try: + value, end = decoder.raw_decode(text, start) + except json.JSONDecodeError: + index = start + 1 + continue + if isinstance(value, dict): + found = value + index = end + return found + + +def tag_published(tag: str) -> bool: + from huggingface_hub import HfApi + + refs = HfApi().list_repo_refs(PUBLISH_REPO, repo_type="dataset") + return any(ref.name == tag for ref in refs.tags) + + +def run_publish(runner: Runner, tag: str, *, publish: bool) -> None: + paths = runner.paths + runner.run( + "publish-preflight", + publish_argv(paths, python=runner.python, tag=tag, preflight_only=True), + cwd=paths.release_parent, + ) + if not publish: + runner.log( + "[publish] not published: pass --publish to publish this cut for " + "inspection (tag only; main and latest-*.json do not move)" + ) + return + if tag_published(tag): + runner.log(f"[publish] done: tag {tag} exists") + return + runner.run( + "publish", + publish_argv(paths, python=runner.python, tag=tag, preflight_only=False), + cwd=paths.release_parent, + ) + + +def write_receipt(runner: Runner, extra: Mapping[str, Any]) -> None: + from microcosm.data.gate_signing_keys import gate_signing_key_fingerprint + + paths = runner.paths + key = os.environ.get(SIGNING_KEY_ENV) + import base64 + + receipt = { + "schema_version": 1, + "kind": "microcosm.uk_cut_receipt", + "release_id": RELEASE_ID, + "label": paths.label, + "code": {"repository": "PolicyEngine/microcosm", "commit": paths.commit}, + "engine": _engine_version(), + "signing_key_sha256": gate_signing_key_fingerprint(base64.b64decode(key)) + if key + else None, + "inputs": { + source.key: { + "repo_id": source.repo_id, + "revision": source.revision, + "archive": source.filename, + "url": source.url, + "files": {pin.name: pin.sha256 for pin in source.files}, + } + for source in runner.sources.values() + }, + "spine_sha256": spine_sha256(paths), + "candidate_sha256": candidate_sha256(paths) + if (paths.run_dir / "build_record.json").exists() + else None, + "steps": runner.state()["steps"], + **extra, + } + paths.receipt.write_text(json.dumps(receipt, indent=2, sort_keys=True)) + + +def _engine_version() -> str | None: + try: + from importlib.metadata import version + + return version("policyengine-uk") + except Exception: + return None + + +# --------------------------------------------------------------------------- # +# Command line # +# --------------------------------------------------------------------------- # + + +def _parse(argv: Sequence[str] | None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog="run_uk_cut.py", description=__doc__.split("\n\n")[0] + ) + parser.add_argument( + "command", + choices=("status", "fetch", "preflight", "all", "inputs"), + help=( + "status: where each step stands; fetch: install and verify every " + "input; preflight: every check, nothing built; all: fetch, " + "preflight, then each step not already done; inputs: print the " + "resolved input pins" + ), + ) + parser.add_argument("--label", help="names the cut's directory, e.g. sat24") + parser.add_argument("--work-dir", type=Path, default=DEFAULT_WORK_DIR) + parser.add_argument("--min-free-gb", type=float, default=60.0) + parser.add_argument( + "--reservation-caller", + default="uk-hub", + help="the caller name checked against the Fleet ops host reservation", + ) + parser.add_argument( + "--allow-off-main", + metavar="REASON", + help="build a pushed commit that is not on origin/main (say who agreed)", + ) + parser.add_argument("--only", action="append", help="fetch only these sources") + parser.add_argument("--reverify", action="store_true", help="re-hash every input") + parser.add_argument("--skip-surface-check", action="store_true") + parser.add_argument("--retry-national", action="store_true") + parser.add_argument("--previous-h5", type=Path, help="for the stage-1 comparison") + parser.add_argument("--logbook-prev-row-digest") + parser.add_argument( + "--spine-telemetry", + action="store_true", + help="send the spine's telemetry to the staging repo (default: local only)", + ) + parser.add_argument( + "--publish", + action="store_true", + help="publish for inspection (--no-latest --tag-only); outward-facing", + ) + return parser.parse_args(argv) + + +def _cut_paths(args: argparse.Namespace) -> CutPaths: + if not args.label: + raise CutError("--label is required for this command.") + return CutPaths(args.work_dir.expanduser(), args.label, _git("rev-parse", "HEAD")) + + +def _print_checks(checks: Sequence[Check]) -> bool: + ok = True + for check in checks: + mark = "ok " if check.ok else ("FAIL" if check.blocking else "warn") + print(f" {mark} {check.name}: {check.detail}") + ok = ok and (check.ok or not check.blocking) + return ok + + +def _print_fetch(results: Sequence[FetchResult]) -> bool: + ok = True + for result in results: + good = result.status in {"present", "fetched"} + ok = ok and good + print( + f" {'ok ' if good else 'FAIL'} {result.key}: {result.status} {result.detail}" + ) + return ok + + +def preflight(args: argparse.Namespace, paths: CutPaths) -> list[Check]: + checks = git_checks(allow_off_main=args.allow_off_main) + checks += environment_checks() + checks.append(signing_key_check()) + checks += hub_checks() + checks.append(register_expiry_check(date.today())) + checks += host_checks( + paths.work_dir, + min_free_gb=args.min_free_gb, + reservation_caller=args.reservation_caller, + ) + checks.append(contract_pin_check(sys.executable)) + if not args.skip_surface_check: + chronicle = paths.inputs / "chronicle_uk" + if (chronicle / "consumer_facts.jsonl").exists(): + checks.append(surfaces_check(sys.executable, chronicle)) + else: + checks.append( + Check("target surfaces", False, "fetch the Chronicle artifact first") + ) + return checks + + +def status(paths: CutPaths) -> None: + print(f"cut {paths.cut}") + for name, check in ( + ("spine", spine_done), + ("national", national_done), + ("score", score_done), + ("certify", certify_done), + ("assemble", assemble_done), + ): + try: + done, detail = check(paths) + except Exception as error: # an earlier step is missing + done, detail = False, f"not reached ({type(error).__name__})" + print(f" {'done' if done else 'todo'} {name}: {detail}") + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parse(argv) + try: + sources = {source.key: source for source in load_input_sources()} + if args.command == "inputs": + print( + json.dumps( + { + key: { + "kind": s.kind, + "repo_id": s.repo_id, + "revision": s.revision, + "files": {p.name: p.sha256 for p in s.files}, + } + for key, s in sources.items() + }, + indent=2, + ) + ) + return 0 + work = args.work_dir.expanduser() + if args.command in {"fetch", "all"}: + print("inputs:") + results = fetch_inputs( + list(sources.values()), + work / "inputs", + downloader=HubDownloader(work / "cache" / "hf"), + reverify=args.reverify, + only=set(args.only) if args.only else None, + ) + if not _print_fetch(results): + print("Some inputs are not available; see docs/uk-cut-anyone.md.") + return 1 + if args.command == "fetch": + return 0 + paths = _cut_paths(args) + if args.command == "status": + status(paths) + return 0 + print("preflight:") + if not _print_checks(preflight(args, paths)): + print("Preflight failed; nothing was built.") + return 1 + if args.command == "preflight": + return 0 + runner = Runner(paths=paths, sources=sources) + run_spine(runner, staging_local_only=not args.spine_telemetry) + run_national( + runner, + retry=args.retry_national, + logbook_prev_row_digest=args.logbook_prev_row_digest, + ) + run_score(runner) + run_certify(runner, logbook_prev_row_digest=args.logbook_prev_row_digest) + run_stage1(runner, args.previous_h5) + summary = run_assemble(runner) + tag = summary["tag"] + run_publish(runner, tag, publish=args.publish) + write_receipt(runner, {"tag": tag, "published": args.publish}) + print(f"\ncut {tag} assembled; receipt {paths.receipt}") + print( + "Next (each a separate, deliberate call):\n" + f" promote: {promote_command(paths, tag)}\n" + " certify in policyengine.py: tools/certify_uk_cut_in_policyengine_py.py " + f"--national-tag {tag} ..." + ) + return 0 + except CutError as error: + print(f"REFUSED: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/uk_cut_inputs.json b/tools/uk_cut_inputs.json new file mode 100644 index 0000000000..55b127e70b --- /dev/null +++ b/tools/uk_cut_inputs.json @@ -0,0 +1,158 @@ +{ + "version": 1, + "about": "Where tools/run_uk_cut.py fetches each input of a Microcosm UK national cut. This file says only where; every expected sha256 and size is read from the committed pins (uk/source_stages.json, uk/frs_release.json, uk/chronicle_feed.json, uk/gates.json, weighted_integrity.py), so a file is accepted only if it is byte-identical to what the build itself will check. revision null means not yet uploaded: the runner then reads the repository's main branch and still verifies every file against its pin (docs/uk-cut-anyone.md).", + "sources": [ + { + "key": "frs_2024_25", + "kind": "hf_zip_members", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "filename": "frs_2024_25.zip", + "archive_sha256_pin": "frs_release", + "directory": "frs_2024_25", + "members": [ + "accounts.tab", + "adult.tab", + "benefits.tab", + "benunit.tab", + "child.tab", + "chldcare.tab", + "extchild.tab", + "househol.tab", + "job.tab", + "maint.tab", + "mortgage.tab", + "oddjob.tab", + "penprov.tab", + "pension.tab" + ] + }, + { + "key": "spi_2022_23", + "kind": "hf_zip_members", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "filename": "spi_2022_23.zip", + "archive_sha256": "2c7a779797ed3accfc04d2c6401867665af216f810141f7639f3e2164edc4517", + "directory": "spi_2022_23", + "members": [ + "put2223uk.tab" + ] + }, + { + "key": "was_round_8", + "kind": "hf_zip_members", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "filename": "was_2006_22.zip", + "archive_sha256": "df5e8be49e51aa70f4bf686b98ab44ef11ee1eb260cbaf51308f89dc62449ae1", + "directory": "was_round_8", + "members": [ + "was_round_8_hhold_eul_may_2025_230525.tab", + "was_round_8_person_eul_may_2025_230525.tab" + ] + }, + { + "key": "etb_1977_2024", + "kind": "hf_zip_members", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "filename": "etb_1977_24.zip", + "archive_sha256": "96fff4868745a2a9bb1169dcecdaf9958c1179d81f9ef67494c1f29622c7d405", + "directory": "etb_1977_2024", + "members": [ + "householdv2_1977-2024.tab" + ] + }, + { + "key": "nts_2002_2024", + "kind": "hf_zip_members", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": null, + "filename": "nts_2002_24.zip", + "archive_sha256": null, + "directory": "nts_2002_2024", + "members": [ + "household_eul_2002-2024.tab", + "individual_eul_2002-2024.tab", + "trip_eul_2002-2024.tab", + "stage_eul_2002-2024.tab", + "ticket_eul_2002-2024.tab" + ], + "upload_note": "UKDS SN 5340, 19th edition, TAB format. Upload the UKDS download zip unchanged as nts_2002_24.zip; any folder layout inside it is fine, the members are found by file name and checked against their pins." + }, + { + "key": "lcfs_2024_25", + "kind": "hf_zip_members", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": null, + "filename": "lcfs_2024_25.zip", + "archive_sha256": null, + "directory": "lcfs_2024_25", + "members": [ + "dvhh_uk_anon_v3_2024.tab", + "dvper_uk_anon_2024.tab" + ], + "upload_note": "UKDS SN 9603, TAB format. Upload the UKDS download zip unchanged as lcfs_2024_25.zip, alongside lcfs_2021_22.zip and lcfs_2023_24.zip." + }, + { + "key": "hmrc_personal_incomes_ods", + "kind": "url", + "url_pin": "source_stages:Collated_Tables_3_1_to_3_11_2324.ods", + "directory": "hmrc", + "members": [ + "Collated_Tables_3_1_to_3_11_2324.ods" + ] + }, + { + "key": "chronicle_uk", + "kind": "hf_files", + "repo_id": "policyengine/microcosm-uk-build-inputs", + "repo_type": "dataset", + "revision": null, + "path_in_repo": "chronicle-uk/abf0921b896d29ee0745ed2aefde8d6337ed7164", + "directory": "chronicle_uk", + "members": [ + "manifest.json", + "consumer_facts.jsonl" + ] + }, + { + "key": "input_mass_reference", + "kind": "hf_files", + "repo_id": "policyengine/microcosm-uk-build-inputs", + "repo_type": "dataset", + "revision": "95e0cb20fb37811c3f9009854d882a7415194926", + "path_in_repo": "input-mass-reference/fd41cb5f6cf6c4ef812320f21d1942173d49ce6f8725b21fbc9d9ca5423d298c", + "directory": "input_mass_reference", + "members": [ + "uk_input_mass_reference.json" + ] + }, + { + "key": "incumbent_efrs", + "kind": "hf_files", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision_pin": "gates:input_mass_parity", + "path_in_repo": "", + "directory": "incumbent", + "members": [ + "enhanced_frs_2024_25.h5" + ] + } + ], + "file_pins": { + "uk_input_mass_reference.json": { + "sha256": "b3490f5bbc46581f2b5f81560a249b8983e37f80c67f00988cd3ed960a7f81c4", + "canonical_evidence_sha256": "fd41cb5f6cf6c4ef812320f21d1942173d49ce6f8725b21fbc9d9ca5423d298c", + "note": "Byte pin of the hosted file; the build's own loader checks the canonical evidence digest. Regenerated at microcosm 0eac9bbbd (policyengine-uk 2.89.0); see the receipt beside it." + } + } +} From a1790efc3cfe51ddd5f65640fb4fdb46bafb741c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 22 Oct 2026 07:38:40 -0400 Subject: [PATCH 04/57] Script the policyengine.py certification handoff for a UK cut Does what policyengine.py#685 did by hand, from one command, on a fresh branch from policyengine.py's main: one-writer check, regional-route pre-check (refuses a dense cut on another policyengine-uk before anything is written, naming the routes), certify-data at the cut's own model and core, uv lock + release_lock, generate --include-tros, policyengine.py's own certified-constant rewrite plus the local-area tag, snapshot refresh, check --published-spm --include-tros --strict-tros, the UC floor and the tests. It commits; it pushes and opens a draft only when asked, and never merges or releases. The PR body lists the judgment calls (UC floor acceptance, dropped overlays, warnings, failing tests) as a checklist. Co-Authored-By: Claude Opus 5.5 --- .../uk-cut-pepy-certification.added.md | 1 + .../test_certify_uk_cut_in_policyengine_py.py | 194 +++++++ tools/certify_uk_cut_in_policyengine_py.py | 547 ++++++++++++++++++ tools/modal_rebuild_uk_chronicle_artifact.py | 6 +- 4 files changed, 745 insertions(+), 3 deletions(-) create mode 100644 changelog.d/uk-cut-pepy-certification.added.md create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_certify_uk_cut_in_policyengine_py.py create mode 100644 tools/certify_uk_cut_in_policyengine_py.py diff --git a/changelog.d/uk-cut-pepy-certification.added.md b/changelog.d/uk-cut-pepy-certification.added.md new file mode 100644 index 0000000000..112ce7b17c --- /dev/null +++ b/changelog.d/uk-cut-pepy-certification.added.md @@ -0,0 +1 @@ +`tools/certify_uk_cut_in_policyengine_py.py` certifies a published UK cut into policyengine.py on a fresh branch: it refuses when another pull request is already certifying the cut, checks the regional route before writing anything, runs certify-data, the lock, TRO, constant and snapshot regeneration and the strict bundle check, measures the UC floor and runs the tests, then drafts the pull request with every judgment call left as a checklist. diff --git a/packages/microcosm-build/tests/engine_free/uk/test_certify_uk_cut_in_policyengine_py.py b/packages/microcosm-build/tests/engine_free/uk/test_certify_uk_cut_in_policyengine_py.py new file mode 100644 index 0000000000..2c1dd03307 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_certify_uk_cut_in_policyengine_py.py @@ -0,0 +1,194 @@ +"""The policyengine.py certification handoff: routing, guards and rewrites.""" + +from __future__ import annotations + +import argparse + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from tools import certify_uk_cut_in_policyengine_py as handoff + +NATIONAL = "microcosm-uk-2024-25-national-20261021T195620Z-a858c100" +DENSE = "microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe" +NEW_DENSE = "microcosm-uk-2024-25-dense-20261026T010203Z-0badc0de" + + +def test_only_cut_tags_are_certified(): + assert handoff.check_tag(NATIONAL, "national") == NATIONAL + for bad in ("main", NATIONAL + "x", DENSE, "microcosm-uk-2024-25-national"): + with pytest.raises(handoff.CertifyError, match="never @main"): + handoff.check_tag(bad, "national") + + +@pytest.mark.parametrize( + ("model", "claims", "version", "covered"), + [ + ("2.134.4", ("==2.134.4",), "2.134.4", True), + ("2.132.6", ("==2.132.6",), "2.134.4", False), + ("2.132.6", ("==2.132.6", "==2.134.4"), "2.134.4", True), + ], +) +def test_coverage_is_built_with_or_an_exact_claim(model, claims, version, covered): + cut = handoff.CutVersions(tag=DENSE, model=model, core="3.33.3", claims=claims) + assert handoff.covers(cut, version) is covered + + +def _args(**kwargs): + base = dict( + dense_tag=None, + keep_regional_release=False, + area_coded_national_uri=None, + drop_regional_release=False, + ) + return argparse.Namespace(**{**base, **kwargs}) + + +def test_each_route_produces_its_certify_data_flags(): + record = {"regional_source_manifest_uri": handoff.manifest_uri(DENSE, "x")} + dense = handoff.regional_flags(_args(dense_tag=NEW_DENSE), record) + assert dense[0] == "--regional-manifest-uri" and f"@{NEW_DENSE}/" in dense[1] + assert handoff.regional_tag(dense) == NEW_DENSE + kept = handoff.regional_flags(_args(keep_regional_release=True), record) + assert kept == ["--regional-manifest-uri", record["regional_source_manifest_uri"]] + area = handoff.regional_flags(_args(area_coded_national_uri="hf://model/x@1/r"), {}) + assert area[-2:] == ["--regional-release-kind", "area_coded_national"] + assert handoff.regional_tag(area) is None + dropped = handoff.regional_flags(_args(drop_regional_release=True), record) + assert dropped == ["--drop-regional-release"] + with pytest.raises(handoff.CertifyError, match="no regional release"): + handoff.regional_flags(_args(keep_regional_release=True), {}) + + +def test_routes_are_mutually_exclusive_and_required(): + with pytest.raises(SystemExit): + handoff.parse(["--national-tag", NATIONAL]) + with pytest.raises(SystemExit): + handoff.parse( + [ + "--national-tag", + NATIONAL, + "--dense-tag", + DENSE, + "--drop-regional-release", + ] + ) + with pytest.raises(SystemExit): + handoff.parse( + ["--national-tag", NATIONAL, "--drop-regional-release", "--open-pr"] + ) + + +def test_the_regional_tag_is_rewritten_only_as_a_literal(tmp_path): + for relative in handoff.CONSTANT_FILES: + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + f'A = "{DENSE}"\nB = "hf://x@{DENSE}/releases/r.json"\n' + f"# {DENSE} in a comment stays\nC = 'x{DENSE}'\n" + ) + changed = handoff.rewrite_regional_constants(tmp_path, DENSE, NEW_DENSE) + assert sorted(changed) == sorted(handoff.CONSTANT_FILES) + text = (tmp_path / handoff.CONSTANT_FILES[0]).read_text() + assert f'A = "{NEW_DENSE}"' in text + assert f"@{NEW_DENSE}/releases" in text + assert f"# {DENSE} in a comment stays" in text + assert handoff.rewrite_regional_constants(tmp_path, DENSE, DENSE) == [] + + +@given( + old=st.from_regex(r"[a-z0-9.\-]{3,20}", fullmatch=True), + new=st.from_regex(r"[a-z0-9.\-]{3,20}", fullmatch=True), +) +def test_the_rewrite_is_a_bijection_on_literals(tmp_path_factory, old, new): + """Rewriting old->new then new->old restores a file of quoted literals.""" + + if old == new or old in new or new in old: + return + root = tmp_path_factory.mktemp("constants") + original = f'X = "{old}"\nY = "@{old}/"\n' + for relative in handoff.CONSTANT_FILES: + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(original) + handoff.rewrite_regional_constants(root, old, new) + handoff.rewrite_regional_constants(root, new, old) + for relative in handoff.CONSTANT_FILES: + assert (root / relative).read_text() == original + + +def test_dropped_overlays_are_reported(): + old = {"dataset_overlays": {"uk": {"frs_2024_25": {}, "enhanced_frs_2024_25": {}}}} + new = {"dataset_overlays": {"uk": {"frs_2024_25": {}}}} + assert handoff.overlay_diff(old, new) == ["enhanced_frs_2024_25"] + assert handoff.overlay_diff({}, {}) == [] + + +def test_the_pr_body_leaves_every_judgment_to_a_person(): + body = handoff.pr_body( + { + "national_tag": NATIONAL, + "model": "2.134.4", + "core": "3.33.3", + "regional_summary": "`--drop-regional-release`", + "basis": "built_with_model_package", + "microcosm_commit": "a" * 40, + "commands": ["python scripts/bundle.py certify-data ..."], + "dropped_overlays": ["enhanced_frs_2024_25"], + "warnings": ["regional release: certifying on a claim"], + "tests_failed": ["tests/test_models.py::test_x"], + } + ) + for needed in ( + "UC floor acceptance", + "enhanced_frs_2024_25", + "Certification warnings", + "tests/test_models.py::test_x", + "One writer", + "- [ ]", + ): + assert needed in body + + +def test_a_dry_run_refuses_an_uncovered_dense_cut_before_writing(monkeypatch, tmp_path): + cuts = { + NATIONAL: handoff.CutVersions(NATIONAL, "2.134.4", "3.33.3", ("==2.134.4",)), + DENSE: handoff.CutVersions(DENSE, "2.132.6", "3.33.3", ("==2.132.6",)), + } + monkeypatch.setattr(handoff, "read_cut", lambda tag, release: cuts[tag]) + monkeypatch.setattr(handoff, "writer_check", lambda tag, repo: []) + monkeypatch.setattr( + handoff.subprocess, + "run", + lambda *a, **k: handoff.subprocess.CompletedProcess(a, 0, "", ""), + ) + bundle = ( + '{"data_releases": {"uk": {"regional_source_manifest_uri": ' + f'"{handoff.manifest_uri(DENSE, handoff.DENSE_RELEASE)}"}}}}}}' + ) + real_check_output = handoff.subprocess.check_output + monkeypatch.setattr( + handoff.subprocess, + "check_output", + lambda argv, **k: ( + bundle if argv[:2] == ["git", "show"] else real_check_output(argv, **k) + ), + ) + args = handoff.parse( + ["--national-tag", NATIONAL, "--keep-regional-release", "--dry-run", + "--pepy-repo", str(tmp_path)] + ) # fmt: skip + with pytest.raises( + handoff.CertifyError, match="built with policyengine-uk 2.132.6" + ): + handoff.certify(args) + args.accept_regional_model_mismatch = "area receipts are uncertified (#685)" + assert handoff.certify(args) == 0 + another = handoff.parse( + ["--national-tag", NATIONAL, "--dense-tag", DENSE, "--dry-run", + "--pepy-repo", str(tmp_path)] + ) # fmt: skip + monkeypatch.setattr(handoff, "writer_check", lambda tag, repo: ["#700 open"]) + with pytest.raises(handoff.CertifyError, match="already being certified"): + handoff.certify(another) diff --git a/tools/certify_uk_cut_in_policyengine_py.py b/tools/certify_uk_cut_in_policyengine_py.py new file mode 100644 index 0000000000..c75c81d5a7 --- /dev/null +++ b/tools/certify_uk_cut_in_policyengine_py.py @@ -0,0 +1,547 @@ +"""Certify a published Microcosm UK cut into policyengine.py, on a fresh branch. + +policyengine.py serves the UK cut its bundle manifest certifies; moving that +pin is a policyengine.py pull request and release. This script does the +mechanical part of that pull request the way policyengine.py#685 did it by +hand (``docs/engineering/skills/data-certification.md`` there), in a new +worktree branched from policyengine.py's ``origin/main``: + +1. refuse if someone is already certifying this cut (one writer per cut); +2. read the national (and regional) release manifests at their tags and + check, before anything is written, that the regional route can certify; +3. ``scripts/bundle.py certify-data`` with the national tag and the chosen + regional route, at the cut's own policyengine-uk and core; +4. ``uv lock`` and ``scripts/release_lock.py``; +5. ``bundle.py generate --include-tros``, the certified-constant rewrite of + policyengine.py's own qualification library, the household snapshots, + then ``bundle.py check --published-spm --include-tros --strict-tros``; +6. the UK Universal Credit floor (weighted national totals only) and the + test suite; +7. a commit, and a pull request body with every judgment left to a person + (the UC floor acceptance, dropped dataset overlays, failing tests). + +It pushes and opens a draft pull request only with ``--push`` and +``--open-pr``. It never merges and never releases. + +Regional routes (pick one): + +``--dense-tag TAG`` a Microcosm local-area cut (ship together) +``--keep-regional-release`` the bundle's current regional release +``--area-coded-national-uri`` an enhanced-FRS release that claims the version +``--drop-regional-release`` no area routing (sim-api's UK default then + lacks constituency and local-authority runs) + +A regional release built on another policyengine-uk is refused by +certification unless policyengine.py accepts a recorded mismatch; pass +``--accept-regional-model-mismatch REASON`` only where that option exists +(see ``docs/uk-cut-anyone.md``). + +Needs ``HF_TOKEN`` (read ``policyengine/populace-uk-private``), PyPI, and +``gh`` for the writer check and the pull request. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shlex +import subprocess +import sys +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +PEPY_REPO = "PolicyEngine/policyengine.py" +CUT_REPO = "policyengine/populace-uk-private" +NATIONAL_RELEASE = "microcosm-uk-2024-25-national" +DENSE_RELEASE = "microcosm-uk-2024-25-dense" +PEPY_PYTHON = "3.13" # certify-uk-release.yaml's interpreter +TAG_PATTERN = re.compile( + r"^microcosm-uk-2024-25-(national|dense)-\d{8}T\d{6}Z-[0-9a-f]{8}$" +) +CONSTANT_PATTERN = r'(?P
["@/-]){old}(?P["/])'
+CONSTANT_FILES = (
+    "tests/test_models.py",
+    "tests/test_release_manifests.py",
+    "tests/test_uk_regions.py",
+)
+
+
+class CertifyError(RuntimeError):
+    """A refusal the operator must act on."""
+
+
+def manifest_uri(tag: str, release: str) -> str:
+    return f"hf://dataset/{CUT_REPO}@{tag}/releases/{release}/release_manifest.json"
+
+
+def check_tag(tag: str, kind: str) -> str:
+    match = TAG_PATTERN.match(tag)
+    if not match or match.group(1) != kind:
+        raise CertifyError(
+            f"{tag!r} is not a {kind} cut tag; certify cut tags only, never @main."
+        )
+    return tag
+
+
+@dataclass(frozen=True)
+class CutVersions:
+    tag: str
+    model: str
+    core: str
+    claims: tuple[str, ...]
+
+
+def read_cut(tag: str, release: str) -> CutVersions:
+    from huggingface_hub import hf_hub_download
+
+    path = hf_hub_download(
+        CUT_REPO,
+        f"releases/{release}/release_manifest.json",
+        repo_type="dataset",
+        revision=tag,
+    )
+    manifest = json.loads(Path(path).read_text())
+    build = manifest.get("build") or {}
+    if build.get("cut_tag") != tag:
+        raise CertifyError(f"{tag}: the manifest names cut {build.get('cut_tag')!r}.")
+    return CutVersions(
+        tag=tag,
+        model=build["built_with_model_package"]["version"],
+        core=build["built_with_core_package"]["version"],
+        claims=tuple(
+            claim["specifier"]
+            for claim in manifest.get("compatible_model_packages", ())
+            if claim.get("name") == "policyengine-uk"
+        ),
+    )
+
+
+def covers(cut: CutVersions, version: str) -> bool:
+    """policyengine.py's rule: built with the version, or an exact claim of it."""
+
+    return cut.model == version or f"=={version}" in cut.claims
+
+
+def regional_flags(args: argparse.Namespace, bundle_record: dict) -> list[str]:
+    if args.dense_tag:
+        return [
+            "--regional-manifest-uri",
+            manifest_uri(check_tag(args.dense_tag, "dense"), DENSE_RELEASE),
+        ]
+    if args.keep_regional_release:
+        uri = bundle_record.get("regional_source_manifest_uri")
+        if not uri:
+            raise CertifyError("the bundle routes no regional release to keep.")
+        return ["--regional-manifest-uri", uri]
+    if args.area_coded_national_uri:
+        return [
+            "--regional-manifest-uri",
+            args.area_coded_national_uri,
+            "--regional-release-kind",
+            "area_coded_national",
+        ]
+    return ["--drop-regional-release"]
+
+
+def regional_tag(flags: list[str]) -> str | None:
+    if "--regional-manifest-uri" not in flags:
+        return None
+    uri = flags[flags.index("--regional-manifest-uri") + 1]
+    match = re.search(r"@([^/]+)/", uri)
+    return (
+        match.group(1) if match and match.group(1).startswith(DENSE_RELEASE) else None
+    )
+
+
+class Shell:
+    """Runs commands in the policyengine.py worktree and records each one."""
+
+    def __init__(self, cwd: Path, log: Path, *, dry_run: bool = False) -> None:
+        self.cwd = cwd
+        self.log = log
+        self.dry_run = dry_run
+        self.commands: list[str] = []
+
+    def __call__(
+        self, argv: list[str], *, env: dict[str, str] | None = None, check=True
+    ) -> subprocess.CompletedProcess:
+        line = shlex.join(argv)
+        self.commands.append(line)
+        print(f"$ {line}", flush=True)
+        if self.dry_run:
+            return subprocess.CompletedProcess(argv, 0, "", "")
+        with self.log.open("a") as handle:
+            handle.write(f"\n# {datetime.now(UTC):%Y-%m-%dT%H:%M:%SZ} $ {line}\n")
+            handle.flush()
+            result = subprocess.run(
+                argv,
+                cwd=self.cwd,
+                env={**os.environ, **(env or {})},
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                check=False,
+            )
+            handle.write(result.stdout)
+        if check and result.returncode != 0:
+            tail = "\n".join(result.stdout.splitlines()[-25:])
+            raise CertifyError(f"{line} exited {result.returncode}:\n{tail}")
+        return result
+
+
+def writer_check(national_tag: str, pepy_repo: Path) -> list[str]:
+    """Open pull requests or branches already certifying this cut."""
+
+    found = []
+    result = subprocess.run(
+        [
+            "gh", "pr", "list", "--repo", PEPY_REPO, "--state", "open",
+            "--search", national_tag, "--json", "number,title,headRefName",
+        ],
+        capture_output=True,
+        text=True,
+        check=False,
+    )  # fmt: skip
+    if result.returncode != 0:
+        raise CertifyError(
+            "could not list policyengine.py pull requests to check for another "
+            f"writer ({result.stderr.strip()[:200]}); retry, or ask the UK hub."
+        )
+    for pr in json.loads(result.stdout or "[]"):
+        found.append(f"#{pr['number']} {pr['title']} ({pr['headRefName']})")
+    branch = f"certify-uk-{national_tag.rsplit('-', 1)[-1]}"
+    remote = subprocess.run(
+        ["git", "ls-remote", "--heads", "origin", branch],
+        cwd=pepy_repo,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if remote.stdout.strip():
+        found.append(f"branch {branch} exists on origin")
+    return found
+
+
+def rewrite_regional_constants(
+    root: Path, old: str | None, new: str | None
+) -> list[str]:
+    """Replace the old local-area cut tag in the certified-constant files."""
+
+    if not old or not new or old == new:
+        return []
+    pattern = re.compile(CONSTANT_PATTERN.format(old=re.escape(old)))
+    changed = []
+    for relative in CONSTANT_FILES:
+        path = root / relative
+        text = path.read_text()
+        updated = pattern.sub(lambda m: m.group("pre") + new + m.group("post"), text)
+        if updated != text:
+            path.write_text(updated)
+            changed.append(relative)
+    return changed
+
+
+def overlay_diff(old_bundle: dict, new_bundle: dict) -> list[str]:
+    old = set((old_bundle.get("dataset_overlays") or {}).get("uk", {}) or {})
+    new = set((new_bundle.get("dataset_overlays") or {}).get("uk", {}) or {})
+    return sorted(old - new)
+
+
+def pr_body(context: dict[str, Any]) -> str:
+    lines = [
+        f"Certifies Microcosm UK national cut `{context['national_tag']}` "
+        f"(policyengine-uk {context['model']}, core {context['core']}).",
+        "",
+        f"- **Regional route:** {context['regional_summary']}",
+        f"- **Compatibility basis:** `{context.get('basis')}`",
+        f"- **Made by** `tools/certify_uk_cut_in_policyengine_py.py` in "
+        f"PolicyEngine/microcosm at `{context['microcosm_commit'][:9]}`.",
+        "",
+        "## How it was made",
+        "",
+        "```bash",
+        *context["commands"],
+        "```",
+        "",
+        "## Receipts",
+        "",
+        f"- UC floor (weighted national totals, `scripts/check_uk_uc_floor.py`): "
+        f"{context.get('uc_floor', 'not run')}",
+        f"- Tests: {context.get('tests', 'not run')}",
+        f"- Certified-constant files rewritten: {context.get('constants') or 'none'}",
+        "",
+        "## Needs a person before this leaves draft",
+        "",
+        "- [ ] **UC floor acceptance.** A later Microcosm cut with the same gap "
+        "needs the same explicit acceptance in its pull request "
+        "(data-certification.md, recorded exceptions). Accepted by: ___",
+    ]
+    if context.get("dropped_overlays"):
+        lines.append(
+            "- [ ] **Dataset overlays dropped by certification:** "
+            f"{', '.join(context['dropped_overlays'])}. Re-add any that must stay "
+            "loadable from the previous bundle (as #685 did)."
+        )
+    if context.get("warnings"):
+        lines.append(
+            "- [ ] **Certification warnings** (from `certify-data`):\n  "
+            + "\n  ".join(f"- {w}" for w in context["warnings"])
+        )
+    if context.get("tests_failed"):
+        lines.append(
+            "- [ ] **Failing tests** to update for the new pair: "
+            + ", ".join(context["tests_failed"][:20])
+        )
+    lines += [
+        "- [ ] One writer: the UK hub and María know this PR exists.",
+        "",
+        "🤖 Generated with [Claude Code](https://claude.com/claude-code)",
+    ]
+    return "\n".join(lines) + "\n"
+
+
+def parse(argv: list[str] | None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    parser.add_argument("--national-tag", required=True)
+    route = parser.add_mutually_exclusive_group(required=True)
+    route.add_argument("--dense-tag")
+    route.add_argument("--keep-regional-release", action="store_true")
+    route.add_argument("--area-coded-national-uri")
+    route.add_argument("--drop-regional-release", action="store_true")
+    parser.add_argument(
+        "--accept-regional-model-mismatch",
+        metavar="REASON",
+        help="forwarded to certify-data where policyengine.py supports it",
+    )
+    parser.add_argument("--model-version", help="default: the cut's build-time model")
+    parser.add_argument("--core-version", help="default: the cut's build-time core")
+    parser.add_argument(
+        "--pepy-repo",
+        type=Path,
+        default=Path("~/PolicyEngine/policyengine.py").expanduser(),
+    )
+    parser.add_argument("--worktree", type=Path, help="default: beside the repo")
+    parser.add_argument("--skip-writer-check", metavar="WHY")
+    parser.add_argument("--skip-tests", action="store_true")
+    parser.add_argument("--test-workers", type=int, default=4)
+    parser.add_argument("--push", action="store_true")
+    parser.add_argument("--open-pr", action="store_true", help="draft; needs --push")
+    parser.add_argument("--dry-run", action="store_true", help="print the plan only")
+    args = parser.parse_args(argv)
+    if args.open_pr and not args.push:
+        parser.error("--open-pr needs --push.")
+    check_tag(args.national_tag, "national")
+    return args
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse(argv)
+    try:
+        return certify(args)
+    except CertifyError as error:
+        print(f"REFUSED: {error}", file=sys.stderr)
+        return 2
+
+
+def certify(args: argparse.Namespace) -> int:
+    microcosm_commit = subprocess.check_output(
+        ["git", "rev-parse", "HEAD"], cwd=Path(__file__).resolve().parents[1], text=True
+    ).strip()
+    repo = args.pepy_repo
+    subprocess.run(["git", "fetch", "--quiet", "origin", "main"], cwd=repo, check=True)
+
+    if args.skip_writer_check:
+        print(f"writer check skipped: {args.skip_writer_check}")
+    else:
+        others = writer_check(args.national_tag, repo)
+        if others:
+            raise CertifyError(
+                "this cut is already being certified: "
+                + "; ".join(others)
+                + ". Coordinate with its owner instead of opening a second PR."
+            )
+
+    national = read_cut(args.national_tag, NATIONAL_RELEASE)
+    model = args.model_version or national.model
+    core = args.core_version or national.core
+    if not covers(national, model):
+        raise CertifyError(
+            f"the national cut was built with policyengine-uk {national.model} and "
+            f"claims {list(national.claims)}; it cannot be certified for {model}."
+        )
+    old_bundle = json.loads(
+        subprocess.check_output(
+            ["git", "show", "origin/main:src/policyengine/data/bundle/manifest.json"],
+            cwd=repo,
+            text=True,
+        )
+    )
+    record = old_bundle["data_releases"]["uk"]
+    flags = regional_flags(args, record)
+    new_regional = regional_tag(flags)
+    regional_summary = shlex.join(flags)
+    if new_regional is not None:
+        dense = read_cut(new_regional, DENSE_RELEASE)
+        if not covers(dense, model):
+            if not args.accept_regional_model_mismatch:
+                raise CertifyError(
+                    f"the regional cut {new_regional} was built with policyengine-uk "
+                    f"{dense.model} and claims {list(dense.claims)}, so certification "
+                    f"for {model} will refuse it. Routes: a dense cut on {model} "
+                    "(--dense-tag), --drop-regional-release, an enhanced-FRS release "
+                    "that claims the version (--area-coded-national-uri), or an "
+                    "accepted, recorded mismatch (--accept-regional-model-mismatch) "
+                    "where policyengine.py supports one."
+                )
+            flags += [
+                "--accept-regional-model-mismatch",
+                args.accept_regional_model_mismatch,
+            ]
+            regional_summary += f" (regional built on {dense.model}; mismatch accepted)"
+    old_regional = regional_tag(
+        ["--regional-manifest-uri", record.get("regional_source_manifest_uri") or ""]
+    )
+
+    suffix = args.national_tag.rsplit("-", 1)[-1]
+    branch = f"certify-uk-{suffix}"
+    worktree = args.worktree or repo.parent / "_worktrees" / f"pepy-{branch}"
+    plan = {
+        "national_tag": args.national_tag,
+        "model": model,
+        "core": core,
+        "regional": regional_summary,
+        "branch": branch,
+        "worktree": str(worktree),
+    }
+    print(json.dumps(plan, indent=2))
+    if args.dry_run:
+        return 0
+    if worktree.exists():
+        raise CertifyError(f"{worktree} exists; remove it or pass --worktree.")
+    subprocess.run(
+        [
+            "git",
+            "worktree",
+            "add",
+            "--quiet",
+            "-b",
+            branch,
+            str(worktree),
+            "origin/main",
+        ],
+        cwd=repo,
+        check=True,
+    )
+    log = worktree.parent / f"{worktree.name}.certify.log"
+    run = Shell(worktree, log)
+    python = str(worktree / ".venv/bin/python")
+    sync = ["uv", "sync", "--frozen", "--extra", "uk", "--extra", "dev",
+            "--python", PEPY_PYTHON]  # fmt: skip
+    run(sync)
+    certify_result = run(
+        [
+            python, "scripts/bundle.py", "certify-data", "--country", "uk",
+            "--data-producer", "populace",
+            "--manifest-uri", manifest_uri(args.national_tag, NATIONAL_RELEASE),
+            *flags,
+            "--model-version", model, "--core-version", core,
+        ]
+    )  # fmt: skip
+    warnings = [
+        line.strip()
+        for line in certify_result.stdout.splitlines()
+        if "warning" in line.lower()
+    ]
+    run(
+        ["uv", "lock", "--no-config", "--no-sources", "--default-index",
+         "https://pypi.org/simple", "--python", python]
+    )  # fmt: skip
+    run(sync)
+    run([python, "scripts/release_lock.py"])
+    run([python, "scripts/bundle.py", "generate", "--include-tros"])
+    new_bundle = json.loads(
+        (worktree / "src/policyengine/data/bundle/manifest.json").read_text()
+    )
+    new_record = new_bundle["data_releases"]["uk"]
+    rewrite = run(
+        [
+            python, "-c",
+            "import json, sys; from pathlib import Path; "
+            "from policyengine.provenance.uk_qualification import "
+            "certified_constant_replacements as r, rewrite_certified_constants as w; "
+            "old, new = json.loads(sys.argv[1]), json.loads(sys.argv[2]); "
+            "print(json.dumps(w(Path('.'), r(old, new))))",
+            json.dumps(record), json.dumps(new_record),
+        ]
+    )  # fmt: skip
+    constants = json.loads(rewrite.stdout.strip().splitlines()[-1] or "[]")
+    constants += rewrite_regional_constants(worktree, old_regional, new_regional)
+    run(
+        [python, "-m", "pytest", "tests/test_household_calculator_snapshot.py", "-q"],
+        env={"PE_UPDATE_SNAPSHOTS": "1"},
+    )
+    run(
+        [python, "scripts/bundle.py", "check", "--published-spm", "--include-tros",
+         "--strict-tros"]
+    )  # fmt: skip
+    uc_path = worktree.parent / f"{worktree.name}.uc_floor.json"
+    uc = run(
+        [python, "scripts/check_uk_uc_floor.py", "--dataset",
+         new_record["default_dataset"], "--year", "2025", "--output", str(uc_path)],
+        check=False,
+    )  # fmt: skip
+    uc_summary = (
+        uc_path.read_text().strip()[:600]
+        if uc_path.exists()
+        else f"exit {uc.returncode}"
+    )
+    tests_summary, tests_failed = "skipped", []
+    if not args.skip_tests:
+        tests = run(
+            [python, "-m", "pytest", "-n", str(args.test_workers), "-q",
+             "-p", "no:cacheprovider"],
+            check=False,
+        )  # fmt: skip
+        tests_failed = re.findall(r"^FAILED (\S+)", tests.stdout, flags=re.MULTILINE)
+        tests_summary = (tests.stdout.strip().splitlines() or ["?"])[-1]
+    context = {
+        "national_tag": args.national_tag,
+        "model": model,
+        "core": core,
+        "regional_summary": f"`{regional_summary}`",
+        "basis": (new_record.get("certification") or {}).get("compatibility_basis"),
+        "microcosm_commit": microcosm_commit,
+        "commands": run.commands,
+        "uc_floor": f"`{uc_summary}`",
+        "tests": tests_summary,
+        "tests_failed": tests_failed,
+        "constants": ", ".join(constants),
+        "dropped_overlays": overlay_diff(old_bundle, new_bundle),
+        "warnings": warnings,
+    }
+    body = worktree.parent / f"{worktree.name}.pr-body.md"
+    body.write_text(pr_body(context))
+    run(["git", "add", "-A"])
+    run(
+        ["git", "commit", "-q", "-m",
+         f"Certify Microcosm UK cut {args.national_tag}\n\n"
+         "Co-Authored-By: Claude Opus 5.5 "]
+    )  # fmt: skip
+    print(f"\ncommitted on {branch} in {worktree}; PR body draft: {body}")
+    if args.push:
+        run(["git", "push", "-u", "origin", branch])
+    if args.open_pr:
+        run(
+            ["gh", "pr", "create", "--draft", "--repo", PEPY_REPO, "--base", "main",
+             "--head", branch, "--title",
+             f"Certify Microcosm UK cut {args.national_tag}", "--body-file", str(body)]
+        )  # fmt: skip
+    return 0 if not tests_failed else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/tools/modal_rebuild_uk_chronicle_artifact.py b/tools/modal_rebuild_uk_chronicle_artifact.py
index c15d2acfc7..709b6191de 100644
--- a/tools/modal_rebuild_uk_chronicle_artifact.py
+++ b/tools/modal_rebuild_uk_chronicle_artifact.py
@@ -72,11 +72,11 @@ def _load_pin() -> dict:
 
 
 def _secrets() -> list[modal.Secret]:
-    if not modal.is_local():
-        return []
-    token = os.environ.get("HF_TOKEN")
     # An ephemeral secret for this run only; nothing is stored in the
     # workspace. Absent locally, the build runs and simply cannot upload.
+    # The container must declare the same dependencies as the launcher, and
+    # there the secret itself supplies HF_TOKEN, so both sides agree.
+    token = os.environ.get("HF_TOKEN")
     return [modal.Secret.from_dict({"HF_TOKEN": token})] if token else []
 
 

From 762c6e475dd615c028437a098dae9d21c016513c Mon Sep 17 00:00:00 2001
From: Max Ghenis 
Date: Thu, 22 Oct 2026 07:41:47 -0400
Subject: [PATCH 05/57] Document building a UK cut without the original
 builder's machine

docs/uk-cut-anyone.md: the one command, what it never does, where every
input lives and how each missing one gets hosted (NTS/LCFS upload, the
Chronicle rebuild, the mass-reference recipe), the signing-key registry
with rotation and revocation, the policyengine.py handoff, and the four
regional routes. CLAUDE.md points at it from the release boundary.

Co-Authored-By: Claude Opus 5.5 
---
 docs/uk-cut-anyone.md | 280 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 280 insertions(+)
 create mode 100644 docs/uk-cut-anyone.md

diff --git a/docs/uk-cut-anyone.md b/docs/uk-cut-anyone.md
new file mode 100644
index 0000000000..36bfc2eb48
--- /dev/null
+++ b/docs/uk-cut-anyone.md
@@ -0,0 +1,280 @@
+# Building a UK national cut without the original builder's machine
+
+Any team member or agent session with the credentials below can build, sign
+and publish a Microcosm UK national cut, then open its policyengine.py
+certification pull request. Each input comes from Hugging Face and is checked
+against the pins the build itself enforces.
+
+This is the short path. The step-by-step reference behind it stays in
+[uk-national-release-assembly-runbook-806.md](uk-national-release-assembly-runbook-806.md).
+The UK hub's runbook of the hand-run build is at
+`~/reviews/uk-hub/jobs/microcosm-cut-runbook/RUNBOOK.md` on Max's machine.
+
+## What you need
+
+| Need | Where it comes from | Check |
+|---|---|---|
+| A clean checkout of `main` locked at the policyengine-uk you want | a "Lock policyengine-uk X" PR merged to main (runbook §2) | `tools/run_uk_cut.py preflight` |
+| `HF_TOKEN` | write access to `policyengine/populace-uk-private` and `policyengine/populace-uk-staging`, and read access to the input repos below | preflight |
+| `MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY` | a key registered **active** in `microcosm.data.gate_signing_keys` (see [Signing keys](#signing-keys)) | `python -m microcosm.data.gate_signing_keys check` |
+| About 60 GB of free disk, AC power, a quiet host | the machine | preflight |
+
+## One command
+
+```bash
+cd 
+uv sync --all-packages --locked --extra uk
+export HF_TOKEN=…   MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY=…
+uv run --no-sync python tools/run_uk_cut.py all --label sat24              # build, certify, assemble
+uv run --no-sync python tools/run_uk_cut.py all --label sat24 --publish    # and publish for inspection
+```
+
+Agent sessions on Max's build Mac read the team key with
+`agent-secret get microcosm/uk-gate-signing-key-team maxghenis`. The key can
+sign once #1350 registers it, after María agrees. `HF_TOKEN` comes from
+`agent-secret get HUGGING_FACE_TOKEN_MAX maxghenis`.
+
+`all` runs these steps in order:
+
+1. `fetch`
+2. `preflight`
+3. spine
+4. national dry run, then the national calibration
+5. re-score, only when the in-build evaluation did not pass
+6. certify
+7. stage-1 comparison (with `--previous-h5`)
+8. assemble
+9. publish preflight
+10. publish (with `--publish`)
+
+Every step first checks whether its outputs already exist and verify, and
+skips itself if they do. So when a run stops, fix the cause and run the same
+command again.
+
+Nothing is overwritten:
+
+- A half-built spine is left for you to inspect. Rebuilding at the same path
+  would delete it, so pick a new `--label` instead.
+- An unfinished national run is moved aside only with `--retry-national`.
+- An existing release directory is never re-assembled.
+
+Other subcommands:
+
+- `status --label L`: where each step stands.
+- `fetch`: install and verify the inputs only.
+- `preflight --label L`: every check, with nothing built.
+- `inputs`: the resolved pins.
+
+Outputs go to `~/microcosm-uk-work`, outside any repository, because they
+derive from licensed data:
+
+| Path | Contents |
+|---|---|
+| `inputs/` | shared verified inputs |
+| `cuts/