From 652cf20aac86f866132f3ee45280abf91b26baa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 20 Sep 2026 17:24:48 +0100 Subject: [PATCH 1/6] Add per-line release pointers for the UK dataset lines (#823) Max's ruling on microcosm#823: latest.json stays the frozen repository- global pointer on the June 2023 release, and each promotable UK line gets its own schema-1 pointer, latest-.json, whose payload adds the discriminator `line` and the immutable cut tag as `revision` so the loader can follow a per-cut tag without a schema bump. - release.py: line_pointer_path / line_pointer_payload / latest_line_release beside the latest.json and latest-evidence.json families; LatestPointer gains `revision` (defaults to the release id) and `line`; latest_release refuses `line` and `revision` fields; prepare_release(line=...) refuses the evidence tier, --no-latest and --tag-only, refuses a line that is not the release id's, reserves every latest*.json root path, and narrows the #823 refusal to "latest.json may not name a per-cut tag; a line pointer may"; _publish_atomic writes the chosen pointer last; notify_release announces the promoted line and revision. - contract.py: line_for_release_id (national, local-k) and the shared per-cut tag grammar keyed on the line, so the per-cut revision allowance covers every promotable line. - publish_cli.py: --promote-line LINE (requires --tag-name; refuses --no-latest, --tag-only, --evidence). Implemented by Codex from the reviewed plan; reviewed and verified here (722 microcosm-data tests green). Co-Authored-By: Claude Fable 5.1 --- changelog.d/823-line-pointers.added.md | 1 + .../src/microcosm/data/contract.py | 38 +- .../src/microcosm/data/publish_cli.py | 18 + .../src/microcosm/data/release.py | 219 +++++++++-- .../src/microcosm/data/slack.py | 16 +- .../microcosm-data/tests/test_contract.py | 13 + .../tests/test_publish_guard.py | 80 +++++ packages/microcosm-data/tests/test_release.py | 339 +++++++++++++++++- 8 files changed, 677 insertions(+), 47 deletions(-) create mode 100644 changelog.d/823-line-pointers.added.md diff --git a/changelog.d/823-line-pointers.added.md b/changelog.d/823-line-pointers.added.md new file mode 100644 index 000000000..bba2acfd1 --- /dev/null +++ b/changelog.d/823-line-pointers.added.md @@ -0,0 +1 @@ +Added schema-1 per-line release pointers, publisher and CLI guards, and line-specific alerts so UK national and local cuts can be promoted without changing `latest.json`. diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index c0b644c0e..74dbc6206 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -71,6 +71,7 @@ "US_SOURCE_COVERAGE_DIAGNOSTICS_FILE", "ReleaseContractError", "compatibility_claim_declarer_error", + "line_for_release_id", "release_dataset_role", "required_release_files", "validate_evidence_release_dir", @@ -615,6 +616,21 @@ def compatibility_claim_declarer_error(declared_by: object) -> str | None: # microcosm.build.uk_runtime.release_identity.UK_NATIONAL_RELEASE_ID (the # data shard cannot import the build shard); lockstep-tested. _UK_NATIONAL_RELEASE_ID = "microcosm-uk-2024-25-national" +_UK_LOCAL_LINE_RELEASE_ID_RE = re.compile( + r"^microcosm-uk-2024-25-local-k(?P[1-9][0-9]*)$" +) + + +def line_for_release_id(release_id: str) -> str | None: + """Return the promotable UK publication line for ``release_id``.""" + if release_id == _UK_NATIONAL_RELEASE_ID: + return "national" + match = _UK_LOCAL_LINE_RELEASE_ID_RE.fullmatch(release_id) + if match is not None: + return f"local-k{match.group('n')}" + return None + + # The dense joint national + local line (microcosm#762 A18, ruling # 2026-09-03): the same constant-id approach, published on the inspect lane # under the non-default local-area role. Mirrored from @@ -682,6 +698,17 @@ def compatibility_claim_declarer_error(declared_by: object) -> str | None: # a hand-edited or stale revision cannot claim a cut the attempt chain never # produced. _UK_NATIONAL_REVISION_SUFFIX_RE = re.compile(r"[0-9]{8}T[0-9]{6}Z-[0-9a-f]{8}") + + +def _uk_line_cut_tag_re(release_id: str) -> re.Pattern[str] | None: + """Return the immutable cut-tag grammar for a promotable UK line.""" + if line_for_release_id(release_id) is None: + return None + return re.compile( + re.escape(release_id) + "-" + _UK_NATIONAL_REVISION_SUFFIX_RE.pattern + ) + + # The canonical release-dir filenames of the evidence the certification signs # (tools/assemble_uk_release_dir.py copies each byte-for-byte). Validation # binds every local file to its signed digest: a certification whose evidence @@ -1276,6 +1303,7 @@ def _check_release_manifest( "release_manifest.json must declare a non-empty 'artifacts' mapping." ) else: + line_cut_tag_re = _uk_line_cut_tag_re(release_id) diagnostics_artifact = artifacts.get("calibration_diagnostics") if not isinstance(diagnostics_artifact, Mapping): failures.append( @@ -1303,13 +1331,9 @@ def _check_release_manifest( revision == release_id or (annual_revision is not None and revision == annual_revision) or ( - release_id == _UK_NATIONAL_RELEASE_ID + line_cut_tag_re is not None and isinstance(revision, str) - and revision.startswith(release_id + "-") - and _UK_NATIONAL_REVISION_SUFFIX_RE.fullmatch( - revision[len(release_id) + 1 :] - ) - is not None + and line_cut_tag_re.fullmatch(revision) is not None ) ) # A present-but-non-string revision must fail here rather than @@ -1322,7 +1346,7 @@ def _check_release_manifest( expected = ( f"the release id {release_id!r} or a " f"'{release_id}--' per-cut tag" - if release_id == _UK_NATIONAL_RELEASE_ID + if line_cut_tag_re is not None else f"the release id {release_id!r}" ) failures.append( diff --git a/packages/microcosm-data/src/microcosm/data/publish_cli.py b/packages/microcosm-data/src/microcosm/data/publish_cli.py index 04ec7e383..c000cda13 100644 --- a/packages/microcosm-data/src/microcosm/data/publish_cli.py +++ b/packages/microcosm-data/src/microcosm/data/publish_cli.py @@ -195,6 +195,14 @@ def main(argv: list[str] | None = None) -> int: "--tag-name", help="Optional tag name override; defaults to the release id.", ) + parser.add_argument( + "--promote-line", + metavar="LINE", + help=( + "Promote the release through latest-.json. Requires " + "--tag-name and never writes latest.json." + ), + ) parser.add_argument( "--extra-file", action="append", @@ -255,6 +263,15 @@ def main(argv: list[str] | None = None) -> int: ) args = parser.parse_args(argv) + if args.promote_line is not None: + if args.no_latest: + parser.error("--promote-line cannot be combined with --no-latest.") + if args.tag_only: + parser.error("--promote-line cannot be combined with --tag-only.") + if args.evidence: + parser.error("--promote-line cannot be combined with --evidence.") + if not args.tag_name: + parser.error("--promote-line requires --tag-name.") if args.tag_only and not args.no_latest: parser.error("--tag-only requires --no-latest.") if args.tag_only and not args.create_tag: @@ -306,6 +323,7 @@ def main(argv: list[str] | None = None) -> int: update_latest=not args.no_latest, tag_only=args.tag_only, evidence=args.evidence, + line=args.promote_line, ) from microcosm.data.source_enrichment import recorded_narrowed_claims diff --git a/packages/microcosm-data/src/microcosm/data/release.py b/packages/microcosm-data/src/microcosm/data/release.py index 5bc8164f6..b0a673ae0 100644 --- a/packages/microcosm-data/src/microcosm/data/release.py +++ b/packages/microcosm-data/src/microcosm/data/release.py @@ -1,4 +1,4 @@ -"""Publish a release and point ``latest.json`` at it. +"""Publish a release and move its approved repository pointer. The Hub repo publishes builds under ``releases//``, but nothing identified which release is *current*: a consumer had to list the tree and @@ -19,6 +19,10 @@ returns the typed pointer, the one-call answer to "which release is current?" for dashboards and scorers. +Approved UK lines coexist through ``latest-.json``. Those schema-1 +pointers add ``line`` and ``revision`` so a line can name an immutable cut tag +without changing the frozen repository-global ``latest.json``. + The EVIDENCE tier (microcosm#506) publishes through the same producer with ``evidence=True``: identical immutable-tag mechanics, but the pointer that moves is ``latest-evidence.json`` — structurally never ``latest.json`` — @@ -34,6 +38,7 @@ import hashlib import json import posixpath +import re from collections.abc import Mapping from dataclasses import dataclass from datetime import UTC, datetime @@ -43,6 +48,8 @@ from microcosm.data.contract import ( EVIDENCE_RELEASE_ID_SEGMENT, NATIONAL_DEFAULT_DATASET_ROLE, + _uk_line_cut_tag_re, + line_for_release_id, release_dataset_role, required_release_files, validate_evidence_release_dir, @@ -54,13 +61,17 @@ "LATEST_EVIDENCE_POINTER_PATH", "LATEST_POINTER_PATH", "LATEST_POINTER_SCHEMA_VERSION", + "LINE_POINTER_PATH_TEMPLATE", "RELEASE_TIER_CERTIFIED", "RELEASE_TIER_EVIDENCE", "LatestPointer", "PreparedRelease", "prepare_release", "latest_evidence_pointer_payload", + "latest_line_release", "latest_pointer_payload", + "line_pointer_path", + "line_pointer_payload", "publish_release", "latest_release", "latest_evidence_release", @@ -70,6 +81,12 @@ #: directory: the pointer is repo state, not release state. LATEST_POINTER_PATH = "latest.json" +#: Per-line pointers coexist with the frozen repository-global pointer. +LINE_POINTER_PATH_TEMPLATE = "latest-{line}.json" + +_LINE_RE = re.compile(r"^[a-z]+(-k[1-9][0-9]*)?$") +_RESERVED_POINTER_PATH_RE = re.compile(r"^latest(-[a-z0-9-]+)?\.json$") + #: The evidence-tier pointer (microcosm#506): which evidence release is the #: best *current* one. A separate file at the repo root, so certified #: consumers reading ``latest.json`` (and the pe.py certification path built @@ -102,12 +119,21 @@ class LatestPointer: :data:`RELEASE_TIER_CERTIFIED` for ``latest.json`` (whose payload predates tiers and carries no field), or :data:`RELEASE_TIER_EVIDENCE` for ``latest-evidence.json``. + revision: Immutable tag to fetch. Repository-global and evidence + pointers default to ``release_id``; a line pointer may name a cut tag. + line: Publication line for ``latest-.json``; otherwise ``None``. """ release_id: str updated_at: str paths: dict[str, str] tier: str = RELEASE_TIER_CERTIFIED + revision: str | None = None + line: str | None = None + + def __post_init__(self) -> None: + if self.revision is None: + object.__setattr__(self, "revision", self.release_id) def latest_pointer_payload(release_id: str, *, updated_at: str | None = None) -> dict: @@ -134,6 +160,49 @@ def latest_pointer_payload(release_id: str, *, updated_at: str | None = None) -> } +def line_pointer_path(line: str) -> str: + """Return the reserved root pointer path for a publication line.""" + if _LINE_RE.fullmatch(line) is None: + raise ValueError( + f"invalid release line {line!r}; expected ^[a-z]+(-k[1-9][0-9]*)?$" + ) + return LINE_POINTER_PATH_TEMPLATE.format(line=line) + + +def line_pointer_payload( + release_id: str, + *, + line: str, + revision: str, + updated_at: str | None = None, +) -> dict: + """Build a schema-1 per-line pointer naming an immutable revision.""" + line_pointer_path(line) + expected_line = line_for_release_id(release_id) + if line != expected_line: + raise ValueError( + f"release {release_id!r} belongs to line {expected_line!r}, not {line!r}." + ) + cut_tag_re = _uk_line_cut_tag_re(release_id) + if ( + not isinstance(revision, str) + or not revision + or not ( + revision == release_id + or (cut_tag_re is not None and cut_tag_re.fullmatch(revision) is not None) + ) + ): + raise ValueError( + f"revision {revision!r} is not the release id {release_id!r} or " + "a per-cut tag in that release's line family." + ) + return { + **latest_pointer_payload(release_id, updated_at=updated_at), + "line": line, + "revision": revision, + } + + def latest_evidence_pointer_payload( release_id: str, *, updated_at: str | None = None ) -> dict: @@ -170,6 +239,8 @@ class PreparedRelease: tag: str filenames: list[str] root_artifacts: dict[str, str] + line: str | None + pointer_path: str def prepare_release( @@ -184,6 +255,7 @@ def prepare_release( update_latest: bool = True, tag_only: bool = False, evidence: bool = False, + line: str | None = None, ) -> PreparedRelease: """Run every local publisher guard and collect the files to upload. @@ -221,7 +293,23 @@ def prepare_release( validate_release_dir(release_dir, artifact_root=artifact_root) release_id = release_dir.name role = release_dataset_role(release_dir) - if role != NATIONAL_DEFAULT_DATASET_ROLE and update_latest: + if line is not None: + pointer_path = line_pointer_path(line) + if evidence: + raise ValueError("line promotion cannot publish at the evidence tier.") + if not update_latest: + raise ValueError("line promotion requires update_latest=True.") + if tag_only: + raise ValueError("line promotion cannot use tag_only=True.") + expected_line = line_for_release_id(release_id) + if line != expected_line: + raise ValueError( + f"release {release_id!r} belongs to line {expected_line!r}, " + f"not {line!r}." + ) + else: + pointer_path = LATEST_EVIDENCE_POINTER_PATH if evidence else LATEST_POINTER_PATH + if role != NATIONAL_DEFAULT_DATASET_ROLE and update_latest and line is None: # microcosm#398 defense in depth beyond --no-latest: a non-default # release can never move the global default pointer, even if a # caller asks. @@ -302,14 +390,15 @@ def prepare_release( "is not a clean relative POSIX path; refusing to upload it." ) pointer_collisions = sorted( - {LATEST_POINTER_PATH, LATEST_EVIDENCE_POINTER_PATH} & set(root_artifacts) + path + for path in root_artifacts + if _RESERVED_POINTER_PATH_RE.fullmatch(path) is not None ) if pointer_collisions: raise ValueError( "release_manifest.json declares root artifact(s) at reserved " - f"pointer path(s) {pointer_collisions}; latest.json and " - "latest-evidence.json are written only by the publisher itself, " - "never as release artifacts." + f"pointer path(s) {pointer_collisions}; latest*.json pointers are " + "written only by the publisher itself, never as release artifacts." ) artifact_revisions, unreadable_revisions = _release_manifest_artifact_revisions( release_dir @@ -341,15 +430,14 @@ def prepare_release( "release_manifest.json pins artifacts to revisions; tag_name must " "match the release id or uniform per-cut artifact revision." ) - # A pointer may only ever name a release whose immutable tag IS the - # release id: per-cut tags are the inspect lane, and the pointer/loader - # contract cannot follow them. Refusing here makes that a publisher - # invariant rather than a runbook convention. - if update_latest and tag != release_id: + # The repository-global pointer may only name a release whose immutable tag + # IS the release id. Approved line pointers have their own reader contract + # and may follow a per-cut tag; latest.json still may not. + if update_latest and tag != release_id and line is None: raise ValueError( - "per-cut tags publish inspect-only: the release pointer cannot " - f"name tag {tag!r} for release {release_id!r}. Pass " - "update_latest=False (--no-latest)." + f"latest.json may not name per-cut tag {tag!r} for release " + f"{release_id!r}; without an approved line pointer, per-cut tags " + "publish inspect-only. Pass update_latest=False (--no-latest)." ) if root_artifacts and artifact_root is None: raise ValueError( @@ -378,6 +466,8 @@ def prepare_release( tag=tag, filenames=filenames, root_artifacts=root_artifacts, + line=line, + pointer_path=pointer_path, ) @@ -397,6 +487,7 @@ def publish_release( tag_only: bool = False, notify: bool = True, evidence: bool = False, + line: str | None = None, ) -> dict: """Publish a release directory and optionally point ``latest.json`` at it. @@ -460,11 +551,14 @@ def publish_release( the certified default or feed pe.py certification. Tag and upload mechanics are otherwise identical. ``update_latest`` then governs the evidence pointer. + line: Promote an approved release line through ``latest-.json``. + Line promotion requires ``update_latest=True``, a matching line + release id, and artifact revisions pinned to the selected tag. Returns: - The release's pointer payload (``latest.json`` shape, plus a ``tier`` - field at the evidence tier). It is uploaded only when - ``update_latest=True``. + The release's pointer payload (``latest.json`` shape, plus ``tier`` at + the evidence tier or ``line`` and ``revision`` for a line promotion). + It is uploaded only when ``update_latest=True``. Raises: ReleaseContractError: If the release directory violates its tier's @@ -482,6 +576,7 @@ def publish_release( update_latest=update_latest, tag_only=tag_only, evidence=evidence, + line=line, ) release_id = prepared.release_id @@ -489,8 +584,18 @@ def publish_release( api = _hf_api() if evidence: payload = latest_evidence_pointer_payload(release_id, updated_at=updated_at) + pointer_label = "evidence release" + elif prepared.line is not None: + payload = line_pointer_payload( + release_id, + line=prepared.line, + revision=prepared.tag, + updated_at=updated_at, + ) + pointer_label = f"{prepared.line} line" else: payload = latest_pointer_payload(release_id, updated_at=updated_at) + pointer_label = "release" if create_tag and not callable(getattr(api, "create_tag", None)): raise TypeError( "publish_release requires a Hub backend with create_tag support; " @@ -515,7 +620,8 @@ def publish_release( create_tag=create_tag, update_latest=update_latest, tag_only=tag_only, - evidence=evidence, + pointer_path=prepared.pointer_path, + pointer_label=pointer_label, ) # The pointer is live: announce it. Best-effort and coupled to the promotion # so every publish path alerts; warn (don't fail) if the webhook is unset. @@ -525,6 +631,9 @@ def publish_release( if evidence: # The alert must never read as a certified release announcement. notify_kwargs["tier"] = RELEASE_TIER_EVIDENCE + if prepared.line is not None: + notify_kwargs["line"] = prepared.line + notify_kwargs["revision"] = prepared.tag notify_release(repo_id, release_id, payload.get("updated_at"), **notify_kwargs) return payload @@ -605,7 +714,8 @@ def _publish_atomic( create_tag: bool, update_latest: bool = True, tag_only: bool = False, - evidence: bool = False, + pointer_path: str, + pointer_label: str, ) -> None: staging_branch = f"release-staging/{release_id}" main_revision = _repo_revision(api, repo_id=repo_id) @@ -632,7 +742,7 @@ def _publish_atomic( if immutable_revision is None: raise RuntimeError( "Hub create_commit returned no revision for immutable release " - f"{release_id!r}; refusing to update main or latest.json." + f"{release_id!r}; refusing to update main or {pointer_path}." ) if create_tag: _create_release_tag( @@ -652,16 +762,11 @@ def _publish_atomic( # candidates deliberately stop here: neither canonical root artifacts # nor release-directory copies are written to main. return - # The evidence tier writes ONLY its own pointer file: the certified - # ``latest.json`` path never appears in an evidence commit, so no bug in - # flag-plumbing can promote an evidence artifact to certified default. - tier_label = "evidence release" if evidence else "release" - pointer_path = LATEST_EVIDENCE_POINTER_PATH if evidence else LATEST_POINTER_PATH if update_latest: - message = f"Update latest {tier_label} to {release_id}" + message = f"Update latest {pointer_label} to {release_id}" pointer = json.dumps(payload, indent=1).encode() else: - message = f"Publish non-default {tier_label} {release_id}" + message = f"Publish non-default {pointer_label} {release_id}" pointer = None api.create_commit( repo_id=repo_id, @@ -887,10 +992,9 @@ def latest_release(repo_id: str, *, api=None) -> LatestPointer: Raises: ValueError: If the pointer is malformed, its schema version is newer - than this library understands, or it carries a ``tier`` field at - all — the certified payload predates tiers and no certified - producer writes one, so any tier field (even ``"certified"``) is - foreign and must never be consumed as the certified default. + than this library understands, or it carries a ``tier``, ``line``, + or ``revision`` field. Those fields belong to other pointer + families and must never be consumed as the certified default. """ payload = _read_pointer(repo_id, api, pointer_path=LATEST_POINTER_PATH) if "tier" in payload: @@ -899,11 +1003,64 @@ def latest_release(repo_id: str, *, api=None) -> LatestPointer: f"({payload.get('tier')!r}); the certified pointer never does — " f"evidence releases live at {LATEST_EVIDENCE_POINTER_PATH}." ) + for field in ("line", "revision"): + if field in payload: + raise ValueError( + f"{LATEST_POINTER_PATH} in {repo_id} carries a {field!r} field; " + "the repository-global certified pointer never does." + ) return LatestPointer( release_id=str(payload["release_id"]), updated_at=str(payload.get("updated_at", "")), paths={str(k): str(v) for k, v in payload["paths"].items()}, tier=RELEASE_TIER_CERTIFIED, + revision=str(payload["release_id"]), + ) + + +def latest_line_release(repo_id: str, *, line: str, api=None) -> LatestPointer: + """Read and validate the schema-1 pointer for ``line``.""" + pointer_path = line_pointer_path(line) + payload = _read_pointer(repo_id, api, pointer_path=pointer_path) + if "tier" in payload: + raise ValueError( + f"{pointer_path} in {repo_id} carries a 'tier' field; line pointers " + "are certified pointers, not evidence pointers." + ) + pointer_line = payload.get("line") + if pointer_line != line: + raise ValueError( + f"{pointer_path} in {repo_id} declares line {pointer_line!r}, " + f"expected {line!r}." + ) + release_id = str(payload["release_id"]) + revision = payload.get("revision") + expected_line = line_for_release_id(release_id) + if line != expected_line: + raise ValueError( + f"{pointer_path} in {repo_id} names release {release_id!r}, which " + f"belongs to line {expected_line!r}, not {line!r}." + ) + cut_tag_re = _uk_line_cut_tag_re(release_id) + if ( + not isinstance(revision, str) + or not revision + or not ( + revision == release_id + or (cut_tag_re is not None and cut_tag_re.fullmatch(revision) is not None) + ) + ): + raise ValueError( + f"{pointer_path} in {repo_id} has revision {revision!r}, which is " + f"outside release {release_id!r}'s line family." + ) + return LatestPointer( + release_id=release_id, + updated_at=str(payload.get("updated_at", "")), + paths={str(k): str(v) for k, v in payload["paths"].items()}, + tier=RELEASE_TIER_CERTIFIED, + revision=revision, + line=line, ) diff --git a/packages/microcosm-data/src/microcosm/data/slack.py b/packages/microcosm-data/src/microcosm/data/slack.py index b9fd01646..59aabb9d5 100644 --- a/packages/microcosm-data/src/microcosm/data/slack.py +++ b/packages/microcosm-data/src/microcosm/data/slack.py @@ -50,6 +50,8 @@ def notify_release( post: Callable[[str, dict[str, Any]], None] | None = None, warn_if_unset: bool = False, tier: str = "certified", + line: str | None = None, + revision: str | None = None, ) -> bool: """Announce a published release to the country's Slack channel. @@ -59,7 +61,8 @@ def notify_release( returning silently, so a release that publishes without an alert is visible in the log rather than a mystery. ``tier="evidence"`` labels the alert as an evidence-tier publish (microcosm#506) so it can never read as a new - certified release; the certified message is unchanged. + certified release. ``line`` and ``revision`` label an approved line + promotion; the existing certified and evidence messages are unchanged. """ country = country_for_repo(repo_id) url = webhook or os.environ.get(CHANNEL_ENV[country]) @@ -87,7 +90,16 @@ def notify_release( ) if part ) - if is_evidence: + if line is not None: + promoted_revision = revision or release_id + text = ( + f"Microcosm {country.upper()} line promoted: {line} -> {promoted_revision}" + ) + header = ( + f":rocket: *Microcosm {label} line promoted*\n" + f"`{line}` → `{promoted_revision}`" + ) + elif is_evidence: text = f"New Microcosm {country.upper()} EVIDENCE release: {release_id}" header = f":warning: *New Microcosm {label} EVIDENCE release*\n`{release_id}`" else: diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index 9daf01574..b18928ff4 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -2163,6 +2163,19 @@ def test_real_june_release_validates_with_legacy_schema_and_selector_shapes( ) +@pytest.mark.parametrize( + ("release_id", "line"), + [ + (UK_NATIONAL_RELEASE_ID, "national"), + ("microcosm-uk-2024-25-local-k55000", "local-k55000"), + ("microcosm-uk-2024-25-dense", None), + (UK_RELEASE_ID, None), + ], +) +def test_line_for_release_id(release_id: str, line: str | None) -> None: + assert contract.line_for_release_id(release_id) == line + + def test_uk_national_release_dir_validates(tmp_path: Path) -> None: validate_release_dir(_write_uk_national_release_dir(tmp_path)) diff --git a/packages/microcosm-data/tests/test_publish_guard.py b/packages/microcosm-data/tests/test_publish_guard.py index 4540f4c1d..a61890267 100644 --- a/packages/microcosm-data/tests/test_publish_guard.py +++ b/packages/microcosm-data/tests/test_publish_guard.py @@ -428,3 +428,83 @@ def test_publish_cli_defaults_to_the_certified_tier(tmp_path, monkeypatch): rc = main([str(tmp_path)]) assert rc == 0 assert calls[0][2]["evidence"] is False + + +def test_publish_cli_forwards_line_promotion_and_cut_tag(tmp_path, monkeypatch): + cut_tag = "microcosm-uk-2024-25-national-20260920T120000Z-deadbeef" + calls = _capture_publish(monkeypatch) + + rc = main( + [ + str(tmp_path), + "--promote-line", + "national", + "--tag-name", + cut_tag, + ] + ) + + assert rc == 0 + assert len(calls) == 1 + assert calls[0][2]["line"] == "national" + assert calls[0][2]["tag_name"] == cut_tag + assert calls[0][2]["update_latest"] is True + + +@pytest.mark.parametrize( + ("arguments", "message"), + [ + ( + [ + "--promote-line", + "national", + "--tag-name", + "cut-tag", + "--no-latest", + ], + "--promote-line cannot be combined with --no-latest", + ), + ( + [ + "--promote-line", + "national", + "--tag-name", + "cut-tag", + "--tag-only", + ], + "--promote-line cannot be combined with --tag-only", + ), + ( + [ + "--promote-line", + "national", + "--tag-name", + "cut-tag", + "--evidence", + ], + "--promote-line cannot be combined with --evidence", + ), + ( + ["--promote-line", "national"], + "--promote-line requires --tag-name", + ), + ], +) +def test_publish_cli_refuses_incompatible_line_promotion_modes( + tmp_path, capsys, monkeypatch, arguments, message +): + import microcosm.data.publish_cli as cli + + called = False + + def unexpected_publish(*args, **kwargs): + nonlocal called + called = True + + monkeypatch.setattr(cli, "publish_release", unexpected_publish) + + with pytest.raises(SystemExit, match="2"): + cli.main([str(tmp_path), *arguments]) + + assert message in capsys.readouterr().err + assert called is False diff --git a/packages/microcosm-data/tests/test_release.py b/packages/microcosm-data/tests/test_release.py index 2a0027863..43b622a3f 100644 --- a/packages/microcosm-data/tests/test_release.py +++ b/packages/microcosm-data/tests/test_release.py @@ -27,8 +27,11 @@ LATEST_POINTER_SCHEMA_VERSION, latest_evidence_pointer_payload, latest_evidence_release, + latest_line_release, latest_pointer_payload, latest_release, + line_pointer_path, + line_pointer_payload, publish_release, ) @@ -42,6 +45,10 @@ def _no_slack_webhook(monkeypatch): RELEASE_ID = "populace-us-2024-9f1260b-20260611" +JUNE_UK_RELEASE_ID = "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z" +UK_NATIONAL_RELEASE_ID = "microcosm-uk-2024-25-national" +UK_NATIONAL_CUT_TAG = f"{UK_NATIONAL_RELEASE_ID}-20260920T120000Z-deadbeef" +UK_LOCAL_RELEASE_ID = "microcosm-uk-2024-25-local-k55000" GIT_COMMIT = "5fa48f07436a806ad75ff76fd22cfb8613bddbe0" DATASET_SHA = "cfe0edd307e479920c6a177b316f944bc27839f89e081ede5218a32d6b6b16d8" CALIBRATION_SHA = "ac31f2be76a0f8dc4da89b6935aa4b8b1b2e1bd4eb3d03b809333084f25b376e" @@ -376,7 +383,8 @@ def _source_coverage_diagnostics() -> dict: class FakeHub: """Model atomic Hub commits, refs, and downloads with an ordered event log.""" - def __init__(self) -> None: + def __init__(self, repo_id: str = "policyengine/populace-us") -> None: + self.repo_id = repo_id self.uploads: list[tuple[str, bytes]] = [] self.tags: list[dict[str, str | None]] = [] self.events: list[tuple[str, dict]] = [] @@ -393,7 +401,7 @@ def _content(path_or_fileobj) -> bytes: def upload_file(self, *, path_or_fileobj, path_in_repo, repo_id, repo_type) -> None: assert repo_type == "dataset" - assert repo_id == "policyengine/populace-us" + assert repo_id == self.repo_id content = self._content(path_or_fileobj) self.uploads.append((path_in_repo, content)) self._commit_number += 1 @@ -415,7 +423,7 @@ def create_branch( exist_ok=False, ) -> None: assert repo_type == "dataset" - assert repo_id == "policyengine/populace-us" + assert repo_id == self.repo_id if branch in self._refs and not exist_ok: raise ValueError(f"branch exists: {branch}") base = revision or "main" @@ -424,7 +432,7 @@ def create_branch( def repo_info(self, *, repo_id, repo_type, revision=None) -> dict[str, str]: assert repo_type == "dataset" - assert repo_id == "policyengine/populace-us" + assert repo_id == self.repo_id ref = revision or "main" return {"sha": self._refs[ref]} @@ -439,7 +447,7 @@ def create_commit( parent_commit=None, ): assert repo_type == "dataset" - assert repo_id == "policyengine/populace-us" + assert repo_id == self.repo_id ref = revision or "main" current_commit = self._refs[ref] if parent_commit is not None: @@ -477,7 +485,7 @@ def create_tag( self, *, repo_id, tag, repo_type, revision=None, exist_ok=False ) -> None: assert repo_type == "dataset" - assert repo_id == "policyengine/populace-us" + assert repo_id == self.repo_id if tag in self._refs and not exist_ok: raise ValueError(f"tag exists: {tag}") self._refs[tag] = revision or self._refs["main"] @@ -486,7 +494,7 @@ def create_tag( def delete_branch(self, *, repo_id, branch, repo_type) -> None: assert repo_type == "dataset" - assert repo_id == "policyengine/populace-us" + assert repo_id == self.repo_id del self._refs[branch] self.events.append(("delete_branch", {"branch": branch})) @@ -496,6 +504,7 @@ def seed_main_file(self, path: str, content: bytes) -> None: def hf_hub_download(self, *, repo_id, filename, repo_type, revision=None) -> str: assert repo_type == "dataset" + assert repo_id == self.repo_id ref = revision or "main" commit = self._refs.get(ref, ref) try: @@ -647,6 +656,35 @@ def artifact_root(tmp_path: Path) -> Path: return directory +def _as_uk_national_line_release(release_dir: Path) -> Path: + """Retarget the publisher fixture to the constant UK national line id.""" + national = release_dir.with_name(UK_NATIONAL_RELEASE_ID) + release_dir.rename(national) + + build_path = national / "build_manifest.json" + build = json.loads(build_path.read_text()) + build["build_id"] = UK_NATIONAL_RELEASE_ID + build_path.write_text(json.dumps(build)) + + certification_path = national / "release_certification.json" + certification_path.write_text("{}") + manifest_path = national / "release_manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["build"]["build_id"] = UK_NATIONAL_RELEASE_ID + for artifact in manifest["artifacts"].values(): + artifact["repo_id"] = "policyengine/populace-uk-private" + artifact["revision"] = UK_NATIONAL_CUT_TAG + manifest["artifacts"]["release_certification"] = { + "kind": "diagnostics", + "path": certification_path.name, + "repo_id": "policyengine/populace-uk-private", + "revision": UK_NATIONAL_CUT_TAG, + "sha256": _sha256(certification_path), + } + manifest_path.write_text(json.dumps(manifest)) + return national + + @pytest.fixture def annual_release(release_dir, artifact_root): from .test_annual_projections import _h5, add_annual_extension @@ -752,6 +790,216 @@ def test_pointer_payload_names_every_contract_file() -> None: ) +@pytest.mark.parametrize( + ("release_id", "line", "revision"), + [ + (UK_NATIONAL_RELEASE_ID, "national", UK_NATIONAL_CUT_TAG), + (UK_LOCAL_RELEASE_ID, "local-k55000", UK_LOCAL_RELEASE_ID), + ], +) +def test_line_pointer_payload_round_trips_for_each_line_path( + hub: FakeHub, release_id: str, line: str, revision: str +) -> None: + payload = line_pointer_payload( + release_id, + line=line, + revision=revision, + updated_at="2026-09-20T12:00:00+00:00", + ) + pointer_path = line_pointer_path(line) + hub.seed_main_file(pointer_path, json.dumps(payload).encode()) + + pointer = latest_line_release("policyengine/populace-us", line=line, api=hub) + + assert pointer.release_id == release_id + assert pointer.revision == revision + assert pointer.line == line + assert pointer.updated_at == "2026-09-20T12:00:00+00:00" + assert payload["schema_version"] == LATEST_POINTER_SCHEMA_VERSION + assert pointer_path == f"latest-{line}.json" + + +@pytest.mark.parametrize( + "line", + ["", "National", "local-k0", "local-k01", "national/other"], +) +def test_line_pointer_path_refuses_invalid_lines(line: str) -> None: + with pytest.raises(ValueError, match="invalid release line"): + line_pointer_path(line) + + +@pytest.mark.parametrize( + "revision", + [ + "", + UK_NATIONAL_RELEASE_ID + "-hotfix", + UK_NATIONAL_RELEASE_ID + "-20260920t120000Z-deadbeef", + UK_NATIONAL_RELEASE_ID + "-20260920T120000Z-DEADBEEF", + ], +) +def test_line_pointer_payload_refuses_revisions_outside_the_cut_family( + revision: str, +) -> None: + with pytest.raises(ValueError, match="per-cut tag"): + line_pointer_payload( + UK_NATIONAL_RELEASE_ID, + line="national", + revision=revision, + ) + + +def test_line_pointer_payload_refuses_a_line_id_mismatch() -> None: + with pytest.raises(ValueError, match="belongs to line 'national'"): + line_pointer_payload( + UK_NATIONAL_RELEASE_ID, + line="local-k55000", + revision=UK_NATIONAL_CUT_TAG, + ) + + +@pytest.mark.parametrize( + ("options", "message"), + [ + ({"update_latest": False}, "requires update_latest=True"), + ({"tag_only": True}, "cannot use tag_only=True"), + ({"evidence": True}, "cannot publish at the evidence tier"), + ], +) +def test_prepare_release_refuses_incompatible_line_modes( + release_dir: Path, monkeypatch, options: dict, message: str +) -> None: + monkeypatch.setattr( + release_module, "validate_release_dir", lambda _path, **_kwargs: None + ) + monkeypatch.setattr( + release_module, + "validate_evidence_release_dir", + lambda _path, **_kwargs: None, + ) + + with pytest.raises(ValueError, match=message): + release_module.prepare_release(release_dir, line="national", **options) + + +def test_prepare_release_refuses_a_line_id_mismatch( + release_dir: Path, monkeypatch +) -> None: + monkeypatch.setattr( + release_module, "validate_release_dir", lambda _path, **_kwargs: None + ) + + with pytest.raises(ValueError, match="belongs to line None"): + release_module.prepare_release(release_dir, line="national") + + +def test_national_cut_promotes_only_its_line_pointer( + release_dir: Path, artifact_root: Path, tmp_path: Path, monkeypatch +) -> None: + national = _as_uk_national_line_release(release_dir) + monkeypatch.setattr( + release_module, "validate_release_dir", lambda _path, **_kwargs: None + ) + alerts: list = [] + monkeypatch.setattr( + release_module, + "notify_release", + lambda repo_id, release_id, updated_at, **kwargs: alerts.append( + (repo_id, release_id, updated_at, kwargs) + ), + ) + uk_hub = FakeHub("policyengine/populace-uk-private") + uk_hub._download_dir = tmp_path / "uk-hub-cache" + june_pointer = latest_pointer_payload( + JUNE_UK_RELEASE_ID, updated_at="2026-06-19T02:38:00+00:00" + ) + june_pointer_bytes = json.dumps(june_pointer, indent=1).encode() + uk_hub.seed_main_file(LATEST_POINTER_PATH, june_pointer_bytes) + + published = publish_release( + national, + "policyengine/populace-uk-private", + api=uk_hub, + artifact_root=artifact_root, + tag_name=UK_NATIONAL_CUT_TAG, + line="national", + updated_at="2026-09-20T12:00:00+00:00", + ) + + pointer_path = line_pointer_path("national") + main_files = uk_hub._commits[uk_hub._refs["main"]] + assert main_files[LATEST_POINTER_PATH] == june_pointer_bytes + assert { + path + for path in main_files + if path.startswith("latest") and path.endswith(".json") + } == {LATEST_POINTER_PATH, pointer_path} + assert uk_hub.events[-1][1]["paths"][-1] == pointer_path + assert uk_hub.uploads[-1][0] == pointer_path + assert uk_hub.tags == [{"tag": UK_NATIONAL_CUT_TAG, "revision": "commit-1"}] + assert published["revision"] == UK_NATIONAL_CUT_TAG + assert published["line"] == "national" + assert ( + latest_line_release( + "policyengine/populace-uk-private", line="national", api=uk_hub + ).revision + == UK_NATIONAL_CUT_TAG + ) + assert alerts == [ + ( + "policyengine/populace-uk-private", + UK_NATIONAL_RELEASE_ID, + "2026-09-20T12:00:00+00:00", + { + "warn_if_unset": True, + "line": "national", + "revision": UK_NATIONAL_CUT_TAG, + }, + ) + ] + + +def test_local_k_role_moves_only_its_own_line_pointer(tmp_path: Path) -> None: + from .test_local_area_contract import _write_local_bundle + + release_dir = _write_local_bundle(tmp_path, release_id=UK_LOCAL_RELEASE_ID) + uk_hub = FakeHub("policyengine/populace-uk-private") + uk_hub._download_dir = tmp_path / "uk-local-hub-cache" + + published = publish_release( + release_dir, + "policyengine/populace-uk-private", + api=uk_hub, + artifact_root=tmp_path, + line="local-k55000", + notify=False, + ) + + pointer_path = line_pointer_path("local-k55000") + assert published["revision"] == UK_LOCAL_RELEASE_ID + assert uk_hub.tags == [{"tag": UK_LOCAL_RELEASE_ID, "revision": "commit-1"}] + assert uk_hub.events[-1][1]["paths"][-1] == pointer_path + assert all(path != LATEST_POINTER_PATH for path, _ in uk_hub.uploads) + assert all(path != LATEST_EVIDENCE_POINTER_PATH for path, _ in uk_hub.uploads) + + +def test_line_promotion_alert_names_the_line_and_revision() -> None: + from microcosm.data.slack import notify_release + + sent: list = [] + assert notify_release( + "policyengine/populace-uk-private", + UK_NATIONAL_RELEASE_ID, + webhook="https://hooks.slack.test/uk", + post=lambda url, payload: sent.append((url, payload)), + line="national", + revision=UK_NATIONAL_CUT_TAG, + ) + headline = sent[0][1]["blocks"][0]["text"]["text"] + assert "line promoted" in headline + assert "national" in headline + assert UK_NATIONAL_CUT_TAG in headline + + def test_publish_release_announces_after_pointer( hub: FakeHub, release_dir: Path, artifact_root: Path, monkeypatch ) -> None: @@ -1403,10 +1651,58 @@ def test_publish_then_resolve_round_trips( ) pointer = latest_release("policyengine/populace-us", api=hub) assert pointer.release_id == RELEASE_ID + assert pointer.revision == RELEASE_ID + assert pointer.line is None assert pointer.updated_at == "2026-06-11T13:53:15+00:00" assert pointer.paths == published["paths"] +@pytest.mark.parametrize("field", ["line", "revision"]) +def test_latest_release_refuses_line_pointer_fields(hub: FakeHub, field: str) -> None: + payload = latest_pointer_payload(RELEASE_ID) + payload[field] = None + hub.seed_main_file(LATEST_POINTER_PATH, json.dumps(payload).encode()) + + with pytest.raises(ValueError, match=field): + latest_release("policyengine/populace-us", api=hub) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("line", "local-k55000", "declares line"), + ("revision", None, "revision"), + ("revision", "", "revision"), + ("revision", UK_NATIONAL_RELEASE_ID + "-hotfix", "revision"), + ("tier", "certified", "tier"), + ], +) +def test_latest_line_release_refuses_foreign_pointer_fields( + hub: FakeHub, field: str, value: object, message: str +) -> None: + payload = line_pointer_payload( + UK_NATIONAL_RELEASE_ID, + line="national", + revision=UK_NATIONAL_CUT_TAG, + ) + payload[field] = value + hub.seed_main_file(line_pointer_path("national"), json.dumps(payload).encode()) + + with pytest.raises(ValueError, match=message): + latest_line_release("policyengine/populace-us", line="national", api=hub) + + +def test_latest_line_release_refuses_a_release_from_another_line( + hub: FakeHub, +) -> None: + payload = latest_pointer_payload(UK_LOCAL_RELEASE_ID) + payload.update(line="national", revision=UK_LOCAL_RELEASE_ID) + hub.seed_main_file(line_pointer_path("national"), json.dumps(payload).encode()) + + with pytest.raises(ValueError, match="belongs to line 'local-k55000'"): + latest_line_release("policyengine/populace-us", line="national", api=hub) + + def test_future_pointer_schema_is_refused(hub: FakeHub) -> None: hub.seed_main_file( LATEST_POINTER_PATH, @@ -1637,6 +1933,8 @@ def test_publish_then_latest_evidence_release_round_trips( ) pointer = latest_evidence_release("policyengine/populace-us", api=hub) assert pointer.release_id == EVIDENCE_RELEASE_ID + assert pointer.revision == EVIDENCE_RELEASE_ID + assert pointer.line is None assert pointer.tier == "evidence" assert pointer.updated_at == "2026-07-22T13:53:15+00:00" assert ( @@ -1753,6 +2051,33 @@ def test_publish_refuses_root_artifacts_at_pointer_paths( assert hub.uploads == [] +@pytest.mark.parametrize( + "pointer_path", + [ + "latest-national.json", + "latest-local-k55000.json", + "latest-future-line-123.json", + ], +) +def test_publish_refuses_root_artifacts_at_every_reserved_line_pointer_path( + hub: FakeHub, + release_dir: Path, + artifact_root: Path, + pointer_path: str, +) -> None: + _declare_root_artifact(release_dir, key="smuggled_line_pointer", path=pointer_path) + + with pytest.raises(ValueError, match="reserved pointer path"): + publish_release( + release_dir, + "policyengine/populace-us", + api=hub, + artifact_root=artifact_root, + ) + + assert hub.uploads == [] + + def test_publish_refuses_unclean_root_artifact_paths( hub: FakeHub, evidence_release_dir: Path, artifact_root: Path ) -> None: From 01ea484ffb646b55c0d2e1dab0c8aa59374d3509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 20 Sep 2026 17:44:50 +0100 Subject: [PATCH 2/6] Register the UK 2024-25 national line as the 2025 default and follow line pointers in the loader (#823) - registry.py: DatasetSpec.pointer_path (default latest.json, the literal the release module pins); ("uk", 2025) becomes the default variant on microcosm_uk_2024_25.h5 through latest-national.json, so resolve("uk") lands on the new line by latest_year while ("uk", 2023) keeps latest.json; the dense entry is microcosm_uk_2024_25_dense.h5 and stays tag-only; the local-area line is reserved in a comment for #898. - loader.py: the pointer is read through the spec's pointer_path (latest_release for latest.json, latest_line_release for a latest-.json, anything else refused); the release manifest is fetched at the pointer's immutable revision and every artifact must be pinned to it; artifact selection follows the manifest role (default -> default_datasets.national; non_default_local_area only through a line pointer, with empty default_datasets and exactly one microdata artifact); available() lists default-variant keys only. Implemented by Codex from the reviewed plan (stacked on the line-pointer increment); reviewed and verified here (735 microcosm-data tests green). Co-Authored-By: Claude Fable 5.1 --- changelog.d/823-uk-2025-registry.changed.md | 1 + .../src/microcosm/data/loader.py | 178 +++++++-- .../src/microcosm/data/registry.py | 22 ++ packages/microcosm-data/tests/test_loader.py | 362 +++++++++++++++++- .../microcosm-data/tests/test_registry.py | 40 +- 5 files changed, 563 insertions(+), 40 deletions(-) create mode 100644 changelog.d/823-uk-2025-registry.changed.md diff --git a/changelog.d/823-uk-2025-registry.changed.md b/changelog.d/823-uk-2025-registry.changed.md new file mode 100644 index 000000000..24cb526be --- /dev/null +++ b/changelog.d/823-uk-2025-registry.changed.md @@ -0,0 +1 @@ +Changed the UK 2025 registry default to the 2024–25 national dataset and taught the certified loader to follow per-line pointers at immutable revisions. Non-default local-area manifests now load only through their line pointers. diff --git a/packages/microcosm-data/src/microcosm/data/loader.py b/packages/microcosm-data/src/microcosm/data/loader.py index b885160fa..c2f95246a 100644 --- a/packages/microcosm-data/src/microcosm/data/loader.py +++ b/packages/microcosm-data/src/microcosm/data/loader.py @@ -1,11 +1,12 @@ """Resolve, verify, download, and load a certified microcosm release. -The normal path follows the repository's release certificate: resolve -``latest.json``, read that release's manifest at its immutable release tag, -download the manifest-selected dataset at the same tag, verify its SHA-256, -and enforce the certified model/Core versions before constructing the engine -dataset. Mutable root artifacts are available only through the explicitly -unsafe ``load(..., unverified_root=True)`` escape hatch. +The normal path follows the repository's release certificate: resolve the +registry-selected pointer, read that release's manifest at the pointer's +immutable revision, download the manifest-selected dataset at the same +revision, verify its SHA-256, and enforce the certified model/Core versions +before constructing the engine dataset. Mutable root artifacts are available +only through the explicitly unsafe ``load(..., unverified_root=True)`` escape +hatch. """ from __future__ import annotations @@ -25,9 +26,20 @@ from packaging.specifiers import InvalidSpecifier, SpecifierSet from packaging.version import InvalidVersion, Version -from microcosm.data.contract import RELEASE_MANIFEST_SCHEMA_VERSION +from microcosm.data.contract import ( + NATIONAL_DEFAULT_DATASET_ROLE, + NON_DEFAULT_LOCAL_AREA_DATASET_ROLE, + RELEASE_MANIFEST_SCHEMA_VERSION, +) from microcosm.data.registry import DEFAULT_VARIANT, REGISTRY, DatasetSpec -from microcosm.data.release import latest_release +from microcosm.data.release import ( + LATEST_POINTER_PATH, + LINE_POINTER_PATH_TEMPLATE, + LatestPointer, + latest_line_release, + latest_release, + line_pointer_path, +) __all__ = [ "available", @@ -39,9 +51,25 @@ ] _SHA256_PATTERN = re.compile(r"[0-9a-fA-F]{64}") +_LINE_POINTER_PATH_PATTERN = re.compile( + re.escape(LINE_POINTER_PATH_TEMPLATE).replace(re.escape("{line}"), r"(?P.+)") +) _HubDownload = Callable[..., str] +def _line_from_pointer_path(pointer_path: str) -> str | None: + """Invert the shared line-pointer template and validate its line.""" + match = _LINE_POINTER_PATH_PATTERN.fullmatch(pointer_path) + if match is None: + return None + line = match.group("line") + try: + rendered_path = line_pointer_path(line) + except ValueError: + return None + return line if rendered_path == pointer_path else None + + @dataclass(frozen=True) class _PackageCertification: name: str @@ -65,8 +93,14 @@ class _CertifiedPackageCompatibilityError(ValueError): def available() -> list[tuple[str, int]]: - """Every published ``(country, year)``, sorted without variant duplicates.""" - return sorted({(country, year) for country, year, _ in REGISTRY}) + """Every default-variant ``(country, year)``, sorted.""" + return sorted( + { + (country, year) + for country, year, variant in REGISTRY + if variant == DEFAULT_VARIANT + } + ) def available_variants() -> list[tuple[str, int, str]]: @@ -229,6 +263,8 @@ def _certified_release_from_manifest( spec: DatasetSpec, release_id: str, manifest: Mapping[str, Any], + *, + expected_revision: str, ) -> _CertifiedRelease: """Validate and select ``spec`` from one release manifest.""" schema_version = manifest.get("schema_version") @@ -241,28 +277,79 @@ def _certified_release_from_manifest( build = _required_mapping(manifest, "build", release_id=release_id) if build.get("build_id") != release_id: raise ValueError( - f"latest.json points to release {release_id!r}, but its release manifest " - f"declares build_id {build.get('build_id')!r}." + f"{spec.pointer_path} points to release {release_id!r}, but its release " + f"manifest declares build_id {build.get('build_id')!r}." ) + dataset_role = manifest.get("dataset_role", NATIONAL_DEFAULT_DATASET_ROLE) + if dataset_role not in { + NATIONAL_DEFAULT_DATASET_ROLE, + NON_DEFAULT_LOCAL_AREA_DATASET_ROLE, + }: + raise ValueError( + f"Release {release_id!r} has unknown dataset_role {dataset_role!r}." + ) default_datasets = _required_mapping( manifest, "default_datasets", release_id=release_id ) - artifact_key = default_datasets.get("national") - if not isinstance(artifact_key, str) or not artifact_key: + if dataset_role == NATIONAL_DEFAULT_DATASET_ROLE: + artifact_key = default_datasets.get("national") + if not isinstance(artifact_key, str) or not artifact_key: + raise ValueError( + f"Release {release_id!r} has no default_datasets.national artifact." + ) + artifact_label = "default" + else: + if _line_from_pointer_path(spec.pointer_path) is None: + raise ValueError( + f"Release {release_id!r} declares dataset_role " + f"{NON_DEFAULT_LOCAL_AREA_DATASET_ROLE!r}; it may be selected only " + f"through a {LINE_POINTER_PATH_TEMPLATE!r} line pointer, not " + f"{spec.pointer_path!r}." + ) + if default_datasets != {}: + raise ValueError( + f"Release {release_id!r} declares dataset_role " + f"{NON_DEFAULT_LOCAL_AREA_DATASET_ROLE!r} but default_datasets " + "is not empty." + ) + artifact_label = "local-area" + + artifacts = _required_mapping(manifest, "artifacts", release_id=release_id) + mismatched_revisions = { + str(key): candidate.get("revision") if isinstance(candidate, Mapping) else None + for key, candidate in artifacts.items() + if not isinstance(candidate, Mapping) + or candidate.get("revision") != expected_revision + } + if mismatched_revisions: raise ValueError( - f"Release {release_id!r} has no default_datasets.national artifact." + f"Release {release_id!r} has artifacts not pinned to expected revision " + f"{expected_revision!r}: {mismatched_revisions}." ) - artifacts = _required_mapping(manifest, "artifacts", release_id=release_id) + if dataset_role == NON_DEFAULT_LOCAL_AREA_DATASET_ROLE: + microdata_keys = [ + key + for key, candidate in artifacts.items() + if isinstance(candidate, Mapping) and candidate.get("kind") == "microdata" + ] + if len(microdata_keys) != 1: + raise ValueError( + f"Release {release_id!r} declares dataset_role " + f"{NON_DEFAULT_LOCAL_AREA_DATASET_ROLE!r} but has " + f"{len(microdata_keys)} microdata artifacts; expected exactly one." + ) + artifact_key = microdata_keys[0] + artifact = artifacts.get(artifact_key) if not isinstance(artifact, Mapping): raise ValueError( - f"Release {release_id!r} default dataset {artifact_key!r} is missing " + f"Release {release_id!r} {artifact_label} dataset {artifact_key!r} is missing " "from release_manifest.json artifacts." ) if artifact.get("kind") != "microdata": raise ValueError( - f"Release {release_id!r} default artifact {artifact_key!r} has kind " + f"Release {release_id!r} {artifact_label} artifact {artifact_key!r} has kind " f"{artifact.get('kind')!r}, not 'microdata'." ) @@ -272,24 +359,20 @@ def _certified_release_from_manifest( artifact_sha256 = artifact.get("sha256") if artifact_repo_id != spec.hf_repo: raise ValueError( - f"Release {release_id!r} default artifact repo {artifact_repo_id!r} " + f"Release {release_id!r} {artifact_label} artifact repo " + f"{artifact_repo_id!r} " f"does not match registry repo {spec.hf_repo!r}." ) if artifact_path != spec.filename: raise ValueError( - f"Release {release_id!r} default artifact path {artifact_path!r} " + f"Release {release_id!r} {artifact_label} artifact path {artifact_path!r} " f"does not match registry filename {spec.filename!r}." ) - if artifact_revision != release_id: - raise ValueError( - f"Release {release_id!r} default artifact revision " - f"{artifact_revision!r} is not pinned to the release id." - ) if not isinstance(artifact_sha256, str) or not _SHA256_PATTERN.fullmatch( artifact_sha256 ): raise ValueError( - f"Release {release_id!r} default artifact has no valid sha256." + f"Release {release_id!r} {artifact_label} artifact has no valid sha256." ) model = _package_certification( @@ -325,22 +408,42 @@ def _sha256(path: Path) -> str: return digest.hexdigest() +def _pointer_for( + spec: DatasetSpec, *, hub_download: _HubDownload | None = None +) -> LatestPointer: + """Read the release pointer configured for ``spec``.""" + if hub_download is None: + hub_download = _hub_download() + api = SimpleNamespace(hf_hub_download=hub_download) + if spec.pointer_path == LATEST_POINTER_PATH: + return latest_release(spec.hf_repo, api=api) + line = _line_from_pointer_path(spec.pointer_path) + if line is not None: + return latest_line_release( + spec.hf_repo, + line=line, + api=api, + ) + raise ValueError( + f"Unsupported release pointer path {spec.pointer_path!r} for {spec.key}; " + f"expected {LATEST_POINTER_PATH!r} or the " + f"{LINE_POINTER_PATH_TEMPLATE!r} line-pointer form." + ) + + def _resolve_certified_release( spec: DatasetSpec, *, hub_download: _HubDownload | None = None ) -> _CertifiedRelease: """Resolve and validate the release pointer and its pinned manifest.""" if hub_download is None: hub_download = _hub_download() - pointer = latest_release( - spec.hf_repo, - api=SimpleNamespace(hf_hub_download=hub_download), - ) + pointer = _pointer_for(spec, hub_download=hub_download) manifest_path = Path( hub_download( repo_id=spec.hf_repo, filename=pointer.paths["release_manifest"], repo_type="dataset", - revision=pointer.release_id, + revision=pointer.revision, ) ) manifest = _load_manifest( @@ -348,7 +451,12 @@ def _resolve_certified_release( repo_id=spec.hf_repo, release_id=pointer.release_id, ) - return _certified_release_from_manifest(spec, pointer.release_id, manifest) + return _certified_release_from_manifest( + spec, + pointer.release_id, + manifest, + expected_revision=pointer.revision, + ) def _verified_download(spec: DatasetSpec) -> tuple[Path, _CertifiedRelease]: @@ -390,9 +498,9 @@ def download( ) -> Path: """Download and SHA-verify the latest certified release artifact. - The mutable ``latest.json`` pointer selects a release; both its manifest - and dataset artifact are then fetched at ``revision=``. Uses - the Hugging Face cache, so repeated calls do not re-download unchanged + The registry-selected mutable pointer selects a release; both its manifest + and dataset artifact are then fetched at the pointer's immutable revision. + Uses the Hugging Face cache, so repeated calls do not re-download unchanged files. Raises: diff --git a/packages/microcosm-data/src/microcosm/data/registry.py b/packages/microcosm-data/src/microcosm/data/registry.py index 73898089e..4cc2851b5 100644 --- a/packages/microcosm-data/src/microcosm/data/registry.py +++ b/packages/microcosm-data/src/microcosm/data/registry.py @@ -46,6 +46,8 @@ class DatasetSpec: engine_package: The pip package providing the engine, e.g. ``"policyengine-us"`` — named in the install extra and in the ImportError when the engine is absent. + pointer_path: The repository-root release pointer followed by the + certified loader. """ country: str @@ -56,6 +58,7 @@ class DatasetSpec: engine_class: str engine_package: str variant: str = DEFAULT_VARIANT + pointer_path: str = "latest.json" @property def key(self) -> tuple[str, int, str]: @@ -108,6 +111,20 @@ def register(spec: DatasetSpec) -> DatasetSpec: ) ) +register( + DatasetSpec( + country="uk", + year=2025, + variant=DEFAULT_VARIANT, + hf_repo="policyengine/populace-uk-private", + filename="microcosm_uk_2024_25.h5", + engine_module="policyengine_uk.data", + engine_class="UKSingleYearDataset", + engine_package="policyengine-uk", + pointer_path="latest-national.json", + ) +) + register( DatasetSpec( # microcosm#762 A18: the dense joint national + local UK build is @@ -123,6 +140,11 @@ def register(spec: DatasetSpec) -> DatasetSpec: engine_package="policyengine-uk", ) ) + +# microcosm#898 reserves the local-area line as variant "local", with pointer +# latest-local-k.json and artifact microcosm_uk_2024_25_local_k.h5. Add +# its registry entry only once a certified artifact exists. + register( DatasetSpec( country="us", diff --git a/packages/microcosm-data/tests/test_loader.py b/packages/microcosm-data/tests/test_loader.py index 9cfbcc033..a50b3734b 100644 --- a/packages/microcosm-data/tests/test_loader.py +++ b/packages/microcosm-data/tests/test_loader.py @@ -6,6 +6,7 @@ import json import os import re +from dataclasses import replace from importlib import metadata from pathlib import Path from types import SimpleNamespace @@ -16,7 +17,17 @@ import microcosm.data.loader as loader from microcosm.data import DEFAULT_VARIANT, download, latest_year, load, resolve -from microcosm.data.release import LATEST_POINTER_PATH, latest_pointer_payload +from microcosm.data.contract import ( + NATIONAL_DEFAULT_DATASET_ROLE, + NON_DEFAULT_LOCAL_AREA_DATASET_ROLE, +) +from microcosm.data.registry import DatasetSpec +from microcosm.data.release import ( + LATEST_POINTER_PATH, + latest_pointer_payload, + line_pointer_path, + line_pointer_payload, +) #: Release id served by the hub fixture. Any well-formed id works: nothing in #: this suite may assert which release the live Hub currently points at, @@ -24,6 +35,14 @@ RELEASE_ID = "populace-us-2024-buildi-sparse-rmloss100-6e8e929-20260709T034135Z" TAGGED_ARTIFACT = b"certified release artifact" MUTABLE_ROOT_ARTIFACT = b"uncertified in-flight root artifact" +UK_REPO_ID = "policyengine/populace-uk-private" +UK_2023_RELEASE_ID = "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z" +UK_NATIONAL_RELEASE_ID = "microcosm-uk-2024-25-national" +UK_NATIONAL_CUT_TAG = f"{UK_NATIONAL_RELEASE_ID}-20260920T120000Z-deadbeef" +UK_LOCAL_LINE = "local-k15" +UK_LOCAL_RELEASE_ID = "microcosm-uk-2024-25-local-k15" +UK_LOCAL_CUT_TAG = f"{UK_LOCAL_RELEASE_ID}-20260920T120000Z-cafebabe" +UK_TAGGED_ARTIFACT = b"certified UK line artifact" def test_resolve_defaults_to_the_latest_year() -> None: @@ -103,6 +122,79 @@ def _release_manifest( } +def _uk_release_manifest( + *, + release_id: str = UK_NATIONAL_RELEASE_ID, + revision: str = UK_NATIONAL_CUT_TAG, + filename: str = "microcosm_uk_2024_25.h5", + artifact_key: str = "microcosm_uk_2024_25", + dataset_role: str | None = NATIONAL_DEFAULT_DATASET_ROLE, + default_datasets: dict[str, str] | None = None, + artifact_content: bytes = UK_TAGGED_ARTIFACT, +) -> dict: + if default_datasets is None: + default_datasets = {"national": artifact_key} + manifest = { + "schema_version": 1, + "data_package": {"name": "microcosm-data", "version": "0.1.0"}, + "default_datasets": default_datasets, + "build": { + "build_id": release_id, + "built_with_core_package": { + "name": "policyengine-core", + "version": "3.27.1", + }, + "built_with_model_package": { + "name": "policyengine-uk", + "version": "2.89.2", + }, + }, + "compatible_core_packages": [ + {"name": "policyengine-core", "specifier": "==3.27.1"} + ], + "compatible_model_packages": [ + {"name": "policyengine-uk", "specifier": "==2.89.2"} + ], + "artifacts": { + artifact_key: { + "kind": "microdata", + "path": filename, + "repo_id": UK_REPO_ID, + "revision": revision, + "sha256": hashlib.sha256(artifact_content).hexdigest(), + } + }, + } + if dataset_role is not None: + manifest["dataset_role"] = dataset_role + return manifest + + +def _uk_local_spec(*, pointer_path: str | None = None) -> DatasetSpec: + return DatasetSpec( + country="uk", + year=2025, + variant="local", + hf_repo=UK_REPO_ID, + filename="microcosm_uk_2024_25_local_k15.h5", + engine_module="policyengine_uk.data", + engine_class="UKSingleYearDataset", + engine_package="policyengine-uk", + pointer_path=pointer_path or line_pointer_path(UK_LOCAL_LINE), + ) + + +def _uk_local_manifest() -> dict: + return _uk_release_manifest( + release_id=UK_LOCAL_RELEASE_ID, + revision=UK_LOCAL_CUT_TAG, + filename="microcosm_uk_2024_25_local_k15.h5", + artifact_key="microcosm_uk_2024_25_local_k15", + dataset_role=NON_DEFAULT_LOCAL_AREA_DATASET_ROLE, + default_datasets={}, + ) + + class FakeHubDownload: """Serve mutable and release-tagged files while recording every request.""" @@ -175,6 +267,58 @@ def __call__( ) from exc +def _hub_with_pointer( + tmp_path: Path, + *, + repo_id: str, + pointer_path: str, + pointer: dict, + manifest: dict, +) -> FakeHubDownload: + hub = FakeHubDownload(tmp_path) + hub.calls.clear() + hub.files.clear() + revision = str(pointer.get("revision", pointer["release_id"])) + hub._add( + tmp_path, + repo_id, + pointer_path, + None, + json.dumps(pointer).encode(), + ) + hub._add( + tmp_path, + repo_id, + pointer["paths"]["release_manifest"], + revision, + json.dumps(manifest).encode(), + ) + return hub + + +def _line_pointer_hub( + tmp_path: Path, + *, + line: str, + release_id: str, + revision: str, + manifest: dict, +) -> FakeHubDownload: + pointer = line_pointer_payload( + release_id, + line=line, + revision=revision, + updated_at="2026-09-20T12:00:00+00:00", + ) + return _hub_with_pointer( + tmp_path, + repo_id=UK_REPO_ID, + pointer_path=line_pointer_path(line), + pointer=pointer, + manifest=manifest, + ) + + def _patch_engine( monkeypatch: pytest.MonkeyPatch, *, @@ -201,6 +345,217 @@ def __init__(self, *, file_path: str) -> None: return constructed +def test_resolve_uk_defaults_to_2025_through_national_line_pointer( + tmp_path: Path, +) -> None: + spec = resolve("uk") + manifest = _uk_release_manifest() + hub = _line_pointer_hub( + tmp_path, + line="national", + release_id=UK_NATIONAL_RELEASE_ID, + revision=UK_NATIONAL_CUT_TAG, + manifest=manifest, + ) + + certified = loader._resolve_certified_release(spec, hub_download=hub) + + manifest_path = f"releases/{UK_NATIONAL_RELEASE_ID}/release_manifest.json" + assert spec.key == ("uk", 2025, DEFAULT_VARIANT) + assert spec.pointer_path == "latest-national.json" + assert certified.release_id == UK_NATIONAL_RELEASE_ID + assert certified.artifact_revision == UK_NATIONAL_CUT_TAG + assert hub.calls == [ + (UK_REPO_ID, "latest-national.json", None), + (UK_REPO_ID, manifest_path, UK_NATIONAL_CUT_TAG), + ] + + +def test_resolve_uk_2023_still_uses_latest_json(tmp_path: Path) -> None: + spec = resolve("uk", 2023) + manifest = _uk_release_manifest( + release_id=UK_2023_RELEASE_ID, + revision=UK_2023_RELEASE_ID, + filename="populace_uk_2023.h5", + artifact_key="populace_uk_2023", + dataset_role=None, + ) + pointer = latest_pointer_payload( + UK_2023_RELEASE_ID, + updated_at="2026-06-19T03:00:00+00:00", + ) + hub = _hub_with_pointer( + tmp_path, + repo_id=UK_REPO_ID, + pointer_path=LATEST_POINTER_PATH, + pointer=pointer, + manifest=manifest, + ) + + certified = loader._resolve_certified_release(spec, hub_download=hub) + + manifest_path = f"releases/{UK_2023_RELEASE_ID}/release_manifest.json" + assert spec.pointer_path == LATEST_POINTER_PATH + assert certified.release_id == UK_2023_RELEASE_ID + assert certified.artifact_revision == UK_2023_RELEASE_ID + assert hub.calls == [ + (UK_REPO_ID, LATEST_POINTER_PATH, None), + (UK_REPO_ID, manifest_path, UK_2023_RELEASE_ID), + ] + + +@pytest.mark.parametrize("pointer_path", ["current.json", "latest-local-k0.json"]) +def test_resolve_refuses_an_unsupported_pointer_path(pointer_path: str) -> None: + spec = replace(resolve("us", 2024), pointer_path=pointer_path) + + def unexpected_download(**_kwargs) -> str: + pytest.fail("an unsupported pointer path must fail before a Hub request") + + with pytest.raises(ValueError, match="Unsupported release pointer path"): + loader._resolve_certified_release(spec, hub_download=unexpected_download) + + +def test_line_pointer_refuses_any_artifact_revision_mismatch(tmp_path: Path) -> None: + manifest = _uk_release_manifest() + manifest["artifacts"]["calibration_report"] = { + "kind": "diagnostics", + "path": "calibration_diagnostics.json", + "repo_id": UK_REPO_ID, + "revision": UK_NATIONAL_RELEASE_ID, + "sha256": "0" * 64, + } + hub = _line_pointer_hub( + tmp_path, + line="national", + release_id=UK_NATIONAL_RELEASE_ID, + revision=UK_NATIONAL_CUT_TAG, + manifest=manifest, + ) + + with pytest.raises( + ValueError, + match=r"artifacts not pinned to expected revision.*calibration_report", + ): + loader._resolve_certified_release(resolve("uk"), hub_download=hub) + + +def test_local_area_line_selects_its_only_microdata_artifact(tmp_path: Path) -> None: + manifest = _uk_local_manifest() + manifest["artifacts"]["calibration_report"] = { + "kind": "diagnostics", + "path": "calibration_diagnostics.json", + "repo_id": UK_REPO_ID, + "revision": UK_LOCAL_CUT_TAG, + "sha256": "0" * 64, + } + hub = _line_pointer_hub( + tmp_path, + line=UK_LOCAL_LINE, + release_id=UK_LOCAL_RELEASE_ID, + revision=UK_LOCAL_CUT_TAG, + manifest=manifest, + ) + + certified = loader._resolve_certified_release( + _uk_local_spec(), + hub_download=hub, + ) + + manifest_path = f"releases/{UK_LOCAL_RELEASE_ID}/release_manifest.json" + assert certified.release_id == UK_LOCAL_RELEASE_ID + assert certified.artifact_path == "microcosm_uk_2024_25_local_k15.h5" + assert certified.artifact_revision == UK_LOCAL_CUT_TAG + assert hub.calls == [ + (UK_REPO_ID, "latest-local-k15.json", None), + (UK_REPO_ID, manifest_path, UK_LOCAL_CUT_TAG), + ] + + +def test_latest_json_refuses_a_local_area_manifest(tmp_path: Path) -> None: + manifest = _uk_local_manifest() + local_artifact = manifest["artifacts"]["microcosm_uk_2024_25_local_k15"] + local_artifact["revision"] = UK_LOCAL_RELEASE_ID + pointer = latest_pointer_payload( + UK_LOCAL_RELEASE_ID, + updated_at="2026-09-20T12:00:00+00:00", + ) + hub = _hub_with_pointer( + tmp_path, + repo_id=UK_REPO_ID, + pointer_path=LATEST_POINTER_PATH, + pointer=pointer, + manifest=manifest, + ) + spec = replace(_uk_local_spec(), pointer_path=LATEST_POINTER_PATH) + + with pytest.raises( + ValueError, + match=r"non_default_local_area.*line pointer.*latest\.json", + ): + loader._resolve_certified_release(spec, hub_download=hub) + + +def test_local_area_line_refuses_nonempty_default_datasets(tmp_path: Path) -> None: + manifest = _uk_local_manifest() + manifest["default_datasets"] = {"national": "microcosm_uk_2024_25_local_k15"} + hub = _line_pointer_hub( + tmp_path, + line=UK_LOCAL_LINE, + release_id=UK_LOCAL_RELEASE_ID, + revision=UK_LOCAL_CUT_TAG, + manifest=manifest, + ) + + with pytest.raises(ValueError, match=r"default_datasets is not empty"): + loader._resolve_certified_release(_uk_local_spec(), hub_download=hub) + + +@pytest.mark.parametrize("microdata_count", [0, 2]) +def test_local_area_line_requires_exactly_one_microdata_artifact( + tmp_path: Path, + microdata_count: int, +) -> None: + manifest = _uk_local_manifest() + artifact = manifest["artifacts"]["microcosm_uk_2024_25_local_k15"] + if microdata_count == 0: + artifact["kind"] = "diagnostics" + else: + manifest["artifacts"]["second_microdata"] = { + "kind": "microdata", + "path": "second.h5", + "repo_id": UK_REPO_ID, + "revision": UK_LOCAL_CUT_TAG, + "sha256": "1" * 64, + } + hub = _line_pointer_hub( + tmp_path, + line=UK_LOCAL_LINE, + release_id=UK_LOCAL_RELEASE_ID, + revision=UK_LOCAL_CUT_TAG, + manifest=manifest, + ) + + with pytest.raises( + ValueError, + match=rf"{microdata_count} microdata artifacts; expected exactly one", + ): + loader._resolve_certified_release(_uk_local_spec(), hub_download=hub) + + +def test_loader_refuses_an_unknown_dataset_role(tmp_path: Path) -> None: + manifest = _uk_release_manifest(dataset_role="research_preview") + hub = _line_pointer_hub( + tmp_path, + line="national", + release_id=UK_NATIONAL_RELEASE_ID, + revision=UK_NATIONAL_CUT_TAG, + manifest=manifest, + ) + + with pytest.raises(ValueError, match="unknown dataset_role 'research_preview'"): + loader._resolve_certified_release(resolve("uk"), hub_download=hub) + + def test_download_uses_pointer_manifest_and_release_revision( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -445,9 +800,10 @@ def _is_offline_error(exc: Exception) -> bool: @pytest.mark.skipif(_hf_offline(), reason="Hugging Face offline mode is enabled") -@pytest.mark.parametrize("country", ["us", "uk"]) +@pytest.mark.parametrize(("country", "year"), [("us", 2024), ("uk", 2023)]) def test_live_latest_pointer_resolves_to_a_coherent_certified_release( country: str, + year: int, ) -> None: """The real pointer and pinned manifest resolve without downloading large H5s. @@ -459,7 +815,7 @@ def test_live_latest_pointer_resolves_to_a_coherent_certified_release( release *identity* belongs to policyengine.py's certification fixtures, which are updated atomically with each certification PR. """ - spec = resolve(country) + spec = resolve(country, year) try: certified = loader._resolve_certified_release(spec) except Exception as exc: diff --git a/packages/microcosm-data/tests/test_registry.py b/packages/microcosm-data/tests/test_registry.py index 1d68af314..97da1209c 100644 --- a/packages/microcosm-data/tests/test_registry.py +++ b/packages/microcosm-data/tests/test_registry.py @@ -4,6 +4,7 @@ import pytest +import microcosm.data.loader as loader_module import microcosm.data.registry as registry_module from microcosm.data import ( DEFAULT_VARIANT, @@ -13,6 +14,7 @@ available_variants, register, ) +from microcosm.data.release import LATEST_POINTER_PATH def test_us_2024_is_registered_with_a_complete_spec() -> None: @@ -41,9 +43,33 @@ def test_uk_2023_compact_is_registered_with_a_complete_spec() -> None: assert spec.engine_package == "policyengine-uk" -def test_available_lists_registered_keys_sorted() -> None: +def test_dataset_spec_default_pointer_path_matches_release_default() -> None: + assert ( + DatasetSpec.__dataclass_fields__["pointer_path"].default == LATEST_POINTER_PATH + ) + + +def test_available_lists_default_variant_keys_only_sorted(monkeypatch) -> None: + dense_only = DatasetSpec( + country="zz", + year=2099, + variant="dense", + hf_repo="policyengine/populace-zz", + filename="populace_zz_2099_dense.h5", + engine_module="policyengine_zz.data", + engine_class="ZZSingleYearDataset", + engine_package="policyengine-zz", + ) + monkeypatch.setattr( + loader_module, + "REGISTRY", + {**REGISTRY, dense_only.key: dense_only}, + ) + keys = available() assert ("us", 2024) in keys + assert ("uk", 2025) in keys + assert ("zz", 2099) not in keys assert keys == sorted(keys) @@ -118,4 +144,14 @@ def test_uk_2025_dense_is_registered_off_the_default_variant() -> None: assert spec.filename == "microcosm_uk_2024_25_dense.h5" assert spec.engine_class == "UKSingleYearDataset" assert spec.engine_package == "policyengine-uk" - assert ("uk", 2025, DEFAULT_VARIANT) not in REGISTRY + assert ("uk", 2025, DEFAULT_VARIANT) in REGISTRY + + +def test_uk_2025_compact_is_registered_as_the_national_line() -> None: + spec = REGISTRY[("uk", 2025, DEFAULT_VARIANT)] + assert spec.variant == DEFAULT_VARIANT + assert spec.hf_repo == "policyengine/populace-uk-private" + assert spec.filename == "microcosm_uk_2024_25.h5" + assert spec.engine_class == "UKSingleYearDataset" + assert spec.engine_package == "policyengine-uk" + assert spec.pointer_path == "latest-national.json" From 939b6ed0bc2f83a88a0bcb27974a06b54d39c9c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 20 Sep 2026 19:46:59 +0100 Subject: [PATCH 3/6] Give the UK national assembler its promotion command and pin its dataset name (#823) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tools/assemble_uk_release_dir.py: `_DATASET_KEY = "microcosm_uk_2024_25"` replaces the key derived from the candidate stem; a candidate not named microcosm_uk_2024_25.h5 (the name tools/build_uk_rowwise_candidate.py --release-role national writes) is refused by name; default_datasets, build_manifest.dataset.filename, the calibration NPZ and the release manifest artifact key follow the constant, and a test pins it to the registry's ("uk", 2025, "compact") filename so the assembler and the loader stay on one name. The JSON summary keeps `publish_command` (the inspect publication, --no-latest --tag-name ) and adds `promote_command` (--promote-line national --tag-name , without --no-latest). Runbook 806 §4 shows both commands and §6 becomes "Promote the reviewed cut" (only latest-national.json moves; latest.json stays on June 2023; resolve("uk") then lands on 2025; the alert names the line and revision); README "Releasing & alerts" and CLAUDE.md name line promotion. Implemented by Codex from the reviewed, narrowed plan; reviewed and verified here (assembler, certification, scorer and registry suites: 54 green). Co-Authored-By: Claude Fable 5.1 --- CLAUDE.md | 5 +- README.md | 6 ++ changelog.d/823-national-promotion.added.md | 1 + ...k-national-release-assembly-runbook-806.md | 42 ++++++++++---- .../tests/test_uk_release_assembler.py | 55 ++++++++++++++++++- .../tests/uk_certification_fixtures.py | 4 +- tools/assemble_uk_release_dir.py | 48 +++++++++++----- 7 files changed, 130 insertions(+), 31 deletions(-) create mode 100644 changelog.d/823-national-promotion.added.md diff --git a/CLAUDE.md b/CLAUDE.md index 1de84ee2b..d69a3ccfe 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -96,8 +96,9 @@ Builds, calibrations, and releases run outside PR CI, need gated Hugging Face data and credentials, and cannot run from forks. Release publication is a deliberate human step (`tools/publish_release.sh` → `microcosm-publish-release`), gated by `tools/preflight_us_release_gates.py`; -see README "Releasing & alerts". Publication also refuses a release whose -build recorded staging telemetry that never reached its repo +reviewed line promotion is a separate deliberate call to the same CLI with +`--promote-line`. See README "Releasing & alerts". Publication also refuses a +release whose build recorded staging telemetry that never reached its repo (`--allow-missing-staging` overrides); a build that declared `--no-staging` publishes without the flag. Never publish or promote artifacts as a side effect of another task. A UK rowwise run's **staged** bundle diff --git a/README.md b/README.md index fa5599a96..bbb2508f0 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,12 @@ tools/publish_release.sh releases/ --repo-id policyengine/populace-us publish CLI posts a release alert to Slack — `#populace-us` or `#populace-uk`, chosen from the repo id. +Promotable UK release lines use pointers named `latest-.json`. Publish a +cut for inspection with `--no-latest --tag-name `, then promote the +reviewed cut with `--promote-line --tag-name `. Promotion moves +only that line pointer; the UK repository-global `latest.json` remains frozen +on the June 2023 release. + US exact-k ladder candidates use a tag-only lane. Run `tools/build_us_exact_k_ladder_release.py`, then execute the `publish_command` recorded in `package_result.json`. That command includes `--create-tag`, diff --git a/changelog.d/823-national-promotion.added.md b/changelog.d/823-national-promotion.added.md new file mode 100644 index 000000000..e5eb5920a --- /dev/null +++ b/changelog.d/823-national-promotion.added.md @@ -0,0 +1 @@ +Add separate inspect and `latest-national.json` promotion commands to UK national release assembly, pin its candidate to `microcosm_uk_2024_25.h5`, and document the reviewed-cut workflow. diff --git a/docs/uk-national-release-assembly-runbook-806.md b/docs/uk-national-release-assembly-runbook-806.md index 2e676d89b..11075028c 100644 --- a/docs/uk-national-release-assembly-runbook-806.md +++ b/docs/uk-national-release-assembly-runbook-806.md @@ -138,7 +138,7 @@ as an operator cross-check but refuses to replace one. ## 4. Publish for inspection -Run the command printed by the assembler. Its shape is: +Run the assembler's `publish_command` first. Its shape is: ```bash uv run python -m microcosm.data.publish_cli \ @@ -149,11 +149,24 @@ uv run python -m microcosm.data.publish_cli \ --tag-name microcosm-uk-2024-25-national-- ``` +The assembler also prints this `promote_command`; keep it for §6 and run it +only after the cut passes the review in §5: + +```bash +uv run python -m microcosm.data.publish_cli \ + releases/microcosm-uk-2024-25-national \ + --repo-id policyengine/populace-uk-private \ + --artifact-root \ + --promote-line national \ + --tag-name microcosm-uk-2024-25-national-- +``` + Do not omit `--tag-name`, and do not pass `--no-create-tag`: every artifact in the manifest is pinned to that immutable per-cut tag. `--no-latest` is -mandatory for this inspect lane — and enforced: publication refuses to move -`latest.json` for any tag that is not the release id itself, so omitting the -flag fails closed instead of promoting an inspect cut. +mandatory for the inspect publication. The promotion command replaces it with +`--promote-line national`; the CLI requires that flag to accompany +`--tag-name` and refuses to combine it with `--no-latest`. A per-cut tag without +either flag cannot move the repository-global `latest.json`. If tag creation returns HTTP 409 after the staging commit, publication can leave the constant branch @@ -173,12 +186,17 @@ Adjudicate the release using the copied certification, scoped gate reports, calibration diagnostics, and score receipt. The release remains inspect-only until that review is complete. -## Promotion is a separate change +## 6. Promote the reviewed cut + +After completing the review in §5, run the assembler's `promote_command` shown +in §4. The promotion moves only `latest-national.json`, recording `national` as +the line and the reviewed cut tag as its `revision`. It never moves the +repository-global `latest.json`, which remains on the June 2023 release. + +After promotion, `microcosm.data.resolve("uk")` lands on the 2025 registry +entry, whose certified loader follows `latest-national.json` to the reviewed +revision. An explicit `microcosm.data.load("uk", 2023)` continues to follow +`latest.json`. -Do not write `latest.json` for this line yet. The current pointer cannot name a -per-cut tag, while the certified loader expects the artifact revision to equal -the release id and fetches the manifest from a tag named by that id. Promotion -needs the loader/pointer design tracked in microcosm#823 before a reviewed cut -can become the default; publication enforces this by refusing a pointer move -for any per-cut tag. Until then, publish every national cut with `--no-latest` -and its explicit per-cut tag. +With `SLACK_WEBHOOK_POPULACE_UK` configured, successful promotion sends an +alert naming the `national` line and the promoted revision. diff --git a/packages/microcosm-build/tests/test_uk_release_assembler.py b/packages/microcosm-build/tests/test_uk_release_assembler.py index cfe48ea32..efb2b4b91 100644 --- a/packages/microcosm-build/tests/test_uk_release_assembler.py +++ b/packages/microcosm-build/tests/test_uk_release_assembler.py @@ -22,6 +22,7 @@ ) from microcosm.build.uk_runtime.release_identity import UK_NATIONAL_RELEASE_ID from microcosm.data.contract import validate_release_dir +from microcosm.data.registry import REGISTRY from microcosm.frame import WeightKind @@ -69,6 +70,14 @@ def test_uk_release_publisher_labels_come_from_target_contract() -> None: assert driver._uk_publisher_labels() == expected +def test_national_dataset_filename_mirrors_registry() -> None: + driver = _load_driver_module() + + assert driver._DATASET_KEY == "microcosm_uk_2024_25" + assert driver._DATASET_FILENAME == f"{driver._DATASET_KEY}.h5" + assert REGISTRY[("uk", 2025, "compact")].filename == driver._DATASET_FILENAME + + def _frame(weights: list[float], *, weight_kind: WeightKind): ids = np.arange(len(weights), dtype="int64") return uk_national_frame( @@ -330,7 +339,7 @@ def test_assemble_green_release_dir(assembler_inputs, capsys) -> None: validate_release_dir(release_dir) calibration_path = assembler_inputs["candidate"].with_name( - "microcosm_uk_2024_calibration.npz" + "microcosm_uk_2024_25_calibration.npz" ) with np.load(calibration_path) as calibration: np.testing.assert_array_equal( @@ -348,6 +357,7 @@ def test_assemble_green_release_dir(assembler_inputs, capsys) -> None: build_manifest = json.loads((release_dir / "build_manifest.json").read_text()) release_manifest = json.loads((release_dir / "release_manifest.json").read_text()) + assert build_manifest["dataset"]["filename"] == driver._DATASET_FILENAME assert build_manifest["staging"] == { "contract_version": 2, "enabled": False, @@ -362,6 +372,14 @@ def test_assemble_green_release_dir(assembler_inputs, capsys) -> None: } assert build_manifest["attempt_id"] == _ATTEMPT_ID assert build_manifest["cut_tag"] == _CUT_TAG + assert release_manifest["default_datasets"] == {"national": driver._DATASET_KEY} + assert release_manifest["artifacts"][driver._DATASET_KEY]["path"] == ( + driver._DATASET_FILENAME + ) + assert ( + release_manifest["artifacts"][f"{driver._DATASET_KEY}_calibration"]["path"] + == f"{driver._DATASET_KEY}_calibration.npz" + ) assert {entry["revision"] for entry in release_manifest["artifacts"].values()} == { _CUT_TAG } @@ -391,6 +409,24 @@ def test_assemble_green_release_dir(assembler_inputs, capsys) -> None: "--tag-name", _CUT_TAG, ] + promote_command = shlex.split(summary["promote_command"]) + assert promote_command == [ + "uv", + "run", + "python", + "-m", + "microcosm.data.publish_cli", + str(release_dir), + "--repo-id", + "policyengine/populace-uk-private", + "--artifact-root", + str(assembler_inputs["candidate"].parent), + "--promote-line", + "national", + "--tag-name", + _CUT_TAG, + ] + assert "--no-latest" not in promote_command def test_assemble_preserves_successful_version_2_delivery( @@ -432,6 +468,23 @@ def test_assemble_refuses_candidate_sha_mismatch(assembler_inputs) -> None: _load_driver_module().main(assembler_inputs["argv"]) +def test_assemble_refuses_candidate_with_wrong_filename(assembler_inputs) -> None: + candidate = assembler_inputs["candidate"] + wrongly_named_candidate = candidate.with_name("microcosm_uk_2024.h5") + wrongly_named_candidate.write_bytes(candidate.read_bytes()) + argv = list(assembler_inputs["argv"]) + argv[argv.index("--candidate-h5") + 1] = str(wrongly_named_candidate) + + with pytest.raises( + SystemExit, + match=( + r"national line's candidate is microcosm_uk_2024_25\.h5 as written by " + r"tools/build_uk_rowwise_candidate\.py --release-role national" + ), + ): + _load_driver_module().main(argv) + + def test_assemble_refuses_unshippable_certification(assembler_inputs) -> None: certification = json.loads(assembler_inputs["certification"].read_text()) certification["shippable"] = False diff --git a/packages/microcosm-build/tests/uk_certification_fixtures.py b/packages/microcosm-build/tests/uk_certification_fixtures.py index 8f7124d82..0e7d7a1d9 100644 --- a/packages/microcosm-build/tests/uk_certification_fixtures.py +++ b/packages/microcosm-build/tests/uk_certification_fixtures.py @@ -167,7 +167,7 @@ def green_score_receipt(candidate_sha: str) -> dict: def green_certification_inputs(tmp_path: Path): """Three green signed parts plus a closed identity join.""" - candidate = tmp_path / "microcosm_uk_2024.h5" + candidate = tmp_path / "microcosm_uk_2024_25.h5" candidate.write_bytes(b"candidate-bytes") candidate_sha = sha256(candidate) diagnostics = tmp_path / "calibration_diagnostics.json" @@ -248,7 +248,7 @@ def green_certification_inputs(tmp_path: Path): ) return { "release_id": "uk-757-first-certified-cut", - "candidate_name": "microcosm_uk_2024", + "candidate_name": "microcosm_uk_2024_25", "candidate_path": candidate, "candidate_sha256": candidate_sha, "spine_sha256": GREEN_PARENT_SPINE_SHA256, diff --git a/tools/assemble_uk_release_dir.py b/tools/assemble_uk_release_dir.py index 3c08d888a..96a09aae1 100644 --- a/tools/assemble_uk_release_dir.py +++ b/tools/assemble_uk_release_dir.py @@ -34,6 +34,8 @@ _ATTEMPT_SUFFIX = re.compile(r"(?P\d{8}T\d{6}Z)-(?P[0-9a-f]{8})") _GIT_COMMIT = re.compile(r"[0-9a-f]{40}") _REPO_ID = "policyengine/populace-uk-private" +_DATASET_KEY = "microcosm_uk_2024_25" +_DATASET_FILENAME = f"{_DATASET_KEY}.h5" _RUNTIME_PACKAGES = ( "python", "policyengine-core", @@ -92,6 +94,13 @@ def main(argv: list[str] | None = None) -> int: def _assemble(args: argparse.Namespace) -> dict[str, object]: + if args.candidate_h5.name != _DATASET_FILENAME: + raise SystemExit( + "error: the national line's candidate is " + f"{_DATASET_FILENAME} as written by " + "tools/build_uk_rowwise_candidate.py --release-role national; " + f"--candidate-h5 was {args.candidate_h5.name}" + ) _refuse_non_release_smoke_spine(args.spine_h5) certification_bytes = args.certification_json.read_bytes() certification = _load_json_bytes(certification_bytes, label="--certification-json") @@ -235,11 +244,7 @@ def _assemble(args: argparse.Namespace) -> dict[str, object]: cut_tag = _cut_tag(attempt_id, args.cut_tag) runtime = _runtime_versions(build_block, dict(args.runtime_version)) code_pin = _diagnostics_code_pin(diagnostics) - candidate_filename = candidate.get("filename") - if not isinstance(candidate_filename, str) or not candidate_filename: - raise SystemExit("error: certification.candidate.filename must be non-empty") - dataset_key = Path(candidate_filename).stem - calibration_filename = f"{dataset_key}_calibration.npz" + calibration_filename = f"{_DATASET_KEY}_calibration.npz" # Nothing is written in place: assembly stages into a private directory, # validates there, and only then atomically renames into empty # destinations — a late failure can never leave a plausible partial @@ -311,8 +316,6 @@ def _assemble(args: argparse.Namespace) -> dict[str, object]: household_weight=household_weight, initial_household_weight=initial_household_weight, measured=measured, - candidate_filename=candidate_filename, - dataset_key=dataset_key, cut_tag=cut_tag, attempt_id=attempt_id, runtime=runtime, @@ -341,8 +344,6 @@ def _stage_and_finalize( household_weight: np.ndarray, initial_household_weight: np.ndarray, measured: Mapping[str, str], - candidate_filename: str, - dataset_key: str, cut_tag: str, attempt_id: str, runtime: Mapping[str, str], @@ -375,7 +376,7 @@ def _stage_and_finalize( "build_sha": code_pin[:7], "runtime": runtime, "dataset": { - "filename": candidate_filename, + "filename": _DATASET_FILENAME, "sha256": measured["candidate"], }, "calibration": { @@ -442,7 +443,7 @@ def artifact(kind: str, path: str, sha256: str) -> dict[str, str]: "name": "microcosm-data", "version": runtime["microcosm-data"], }, - "default_datasets": {"national": dataset_key}, + "default_datasets": {"national": _DATASET_KEY}, "build": { "build_id": UK_NATIONAL_RELEASE_ID, "built_at": created_at, @@ -470,10 +471,10 @@ def artifact(kind: str, path: str, sha256: str) -> dict[str, str]: } ], "artifacts": { - dataset_key: artifact( - "microdata", candidate_filename, measured["candidate"] + _DATASET_KEY: artifact( + "microdata", _DATASET_FILENAME, measured["candidate"] ), - f"{dataset_key}_calibration": artifact( + f"{_DATASET_KEY}_calibration": artifact( "calibration", calibration_filename, calibration_sha ), **{ @@ -563,6 +564,24 @@ def artifact(kind: str, path: str, sha256: str) -> dict[str, str]: cut_tag, ] ) + promote_command = shlex.join( + [ + "uv", + "run", + "python", + "-m", + "microcosm.data.publish_cli", + str(destination), + "--repo-id", + _REPO_ID, + "--artifact-root", + str(args.candidate_h5.parent), + "--promote-line", + "national", + "--tag-name", + cut_tag, + ] + ) return { "release_id": UK_NATIONAL_RELEASE_ID, "release_dir": str(destination), @@ -585,6 +604,7 @@ def artifact(kind: str, path: str, sha256: str) -> dict[str, str]: }, "evidence": evidence_summary, "publish_command": publish_command, + "promote_command": promote_command, } From 7053b9173f5f25accb8d4c5bbed39191d1a733d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 23 Sep 2026 17:13:46 +0100 Subject: [PATCH 4/6] Address Vahid's review on #966: promotion reuses the cut tag, the national line waits for its pointer, one role per line Blocking 1. publish_release always re-ran the staging commit and create_tag, so the documented inspect -> promote sequence died on the existing tag and no pointer was written; a retry after a failure between tag creation and the pointer commit died the same way. _publish_atomic now recognises an existing tag before touching the staging branch: when the tagged release_manifest.json is byte-identical to the local one the immutable revision already exists, so no staging branch, no immutable commit, no create_tag, and only the main commit carrying the pointer is written; a tag that describes another release refuses. Tested: the two-step sequence, the retry after an injected pointer-commit failure, the foreign-tag refusal both ways. Blocking 2. resolve("uk") is a registry lookup, so registering the 2025 national entry as the default would have sent every default load to a pointer that does not exist until the first promotion. The entry is registered as variant "national" (Max's ruling keys the line as a variant): load("uk", 2025, variant="national") follows latest-national.json, resolve("uk") and available() stay on 2023, and the flip to the default variant is a one-line follow-up after promotion. Should-fix 3-5. The live pointer test covers the national variant (skipping until the pointer exists, asserting the cut-tag revision shape); the loader states why every artifact must pin the pointer's revision and the latest.json path is tested for it; dataset_role_for_line binds national -> national_default and local-k -> non_default_local_area, refused by the publisher and the loader. Nits. line_pointer_path refuses "evidence" at the grammar; the assembler binds certification.candidate.filename to the line's dataset filename again; runbook 806 and the README carry the sequencing. Co-Authored-By: Claude Fable 5.1 --- README.md | 2 +- changelog.d/823-line-role-rule.fixed.md | 1 + ...national-variant-until-promoted.changed.md | 1 + .../823-promotion-reuses-the-cut-tag.fixed.md | 1 + ...k-national-release-assembly-runbook-806.md | 29 ++- .../tests/test_uk_release_assembler.py | 2 +- .../src/microcosm/data/contract.py | 17 ++ .../src/microcosm/data/loader.py | 18 ++ .../src/microcosm/data/registry.py | 9 +- .../src/microcosm/data/release.py | 157 +++++++++++++---- packages/microcosm-data/tests/test_loader.py | 109 +++++++++++- .../microcosm-data/tests/test_registry.py | 22 ++- packages/microcosm-data/tests/test_release.py | 165 ++++++++++++++++++ tools/assemble_uk_release_dir.py | 5 + 14 files changed, 476 insertions(+), 62 deletions(-) create mode 100644 changelog.d/823-line-role-rule.fixed.md create mode 100644 changelog.d/823-national-variant-until-promoted.changed.md create mode 100644 changelog.d/823-promotion-reuses-the-cut-tag.fixed.md diff --git a/README.md b/README.md index bbb2508f0..9d2cba429 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ Promotable UK release lines use pointers named `latest-.json`. Publish a cut for inspection with `--no-latest --tag-name `, then promote the reviewed cut with `--promote-line --tag-name `. Promotion moves only that line pointer; the UK repository-global `latest.json` remains frozen -on the June 2023 release. +on the June 2023 release. Promotion reuses the immutable cut tag the inspect publication created (it checks the tagged manifest is byte-identical) and writes only the pointer commit, so the two-step sequence and a retry after a failed pointer commit both work. A line's registry entry is registered off the default variant until its first promotion; the default flips in a follow-up after the pointer exists. US exact-k ladder candidates use a tag-only lane. Run `tools/build_us_exact_k_ladder_release.py`, then execute the `publish_command` diff --git a/changelog.d/823-line-role-rule.fixed.md b/changelog.d/823-line-role-rule.fixed.md new file mode 100644 index 000000000..ca4cf295e --- /dev/null +++ b/changelog.d/823-line-role-rule.fixed.md @@ -0,0 +1 @@ +A UK publication line carries one dataset role: the publisher refuses a line promotion whose manifest declares another role (`national` publishes `national_default` releases, `local-k` lines `non_default_local_area`), and the loader refuses a manifest that reached a line pointer with the wrong role. The release-line grammar refuses `evidence` (it would collide with the evidence-tier pointer), the loader states why every artifact must pin the pointer's revision, and the national assembler again binds `certification.candidate.filename` to the line's dataset filename. diff --git a/changelog.d/823-national-variant-until-promoted.changed.md b/changelog.d/823-national-variant-until-promoted.changed.md new file mode 100644 index 000000000..c492e22e4 --- /dev/null +++ b/changelog.d/823-national-variant-until-promoted.changed.md @@ -0,0 +1 @@ +The UK 2024-25 national line is registered as variant `national` (`microcosm.data.load("uk", 2025, variant="national")`), not as the 2025 default: `resolve("uk")` stays on the June 2023 line until the line's first cut is promoted and a follow-up flips the entry to the default variant, so a default load never chases a pointer that does not exist yet (review of #966; Max's #823 ruling keys the line as a variant). diff --git a/changelog.d/823-promotion-reuses-the-cut-tag.fixed.md b/changelog.d/823-promotion-reuses-the-cut-tag.fixed.md new file mode 100644 index 000000000..eccd14c55 --- /dev/null +++ b/changelog.d/823-promotion-reuses-the-cut-tag.fixed.md @@ -0,0 +1 @@ +Promoting a UK release line reuses the immutable cut tag an inspect publication already created: the publisher recognises an existing tag whose `release_manifest.json` is byte-identical to the local release, creates no second immutable revision, and writes only the main commit carrying the line pointer; a tag that describes another release refuses. The documented inspect-then-promote sequence and a retry after a failure between tag creation and the pointer commit both work (review of #966). diff --git a/docs/uk-national-release-assembly-runbook-806.md b/docs/uk-national-release-assembly-runbook-806.md index 11075028c..692b08989 100644 --- a/docs/uk-national-release-assembly-runbook-806.md +++ b/docs/uk-national-release-assembly-runbook-806.md @@ -168,11 +168,18 @@ mandatory for the inspect publication. The promotion command replaces it with `--tag-name` and refuses to combine it with `--no-latest`. A per-cut tag without either flag cannot move the repository-global `latest.json`. -If tag creation returns HTTP 409 after the staging commit, publication can -leave the constant branch -`release-staging/microcosm-uk-2024-25-national` behind. Delete that branch -manually in the private Hugging Face repository before retrying the same cut. -Do not delete the immutable cut tag. +The two commands share the cut tag by design: the promotion recognises the +immutable tag the inspect publication created, checks that the tagged +`release_manifest.json` is byte-identical to the local one, creates no second +immutable revision, and writes only the main commit carrying +`latest-national.json`. The same recognition makes a retry safe after a +failure between tag creation and the pointer commit: rerun the same command. +A tag that describes another release refuses; publish under a fresh cut tag. + +Only a failure before the tag exists (the staging commit itself) can leave +the constant branch `release-staging/microcosm-uk-2024-25-national` behind. +Delete that branch manually in the private Hugging Face repository before +retrying. Never delete an immutable cut tag. ## 5. Inspect on the dashboard @@ -193,10 +200,14 @@ in §4. The promotion moves only `latest-national.json`, recording `national` as the line and the reviewed cut tag as its `revision`. It never moves the repository-global `latest.json`, which remains on the June 2023 release. -After promotion, `microcosm.data.resolve("uk")` lands on the 2025 registry -entry, whose certified loader follows `latest-national.json` to the reviewed -revision. An explicit `microcosm.data.load("uk", 2023)` continues to follow -`latest.json`. +The 2025 national entry is registered off the default variant until this +first promotion: `microcosm.data.load("uk", 2025, variant="national")` follows +`latest-national.json` as soon as the pointer exists, while +`microcosm.data.resolve("uk")` stays on the June 2023 line. The default moves +in a one-line follow-up PR after promotion (the registry entry's `variant` +becomes the default), with its test; sequencing it this way means a default +load never chases a pointer that does not exist yet. An explicit +`microcosm.data.load("uk", 2023)` continues to follow `latest.json`. With `SLACK_WEBHOOK_POPULACE_UK` configured, successful promotion sends an alert naming the `national` line and the promoted revision. diff --git a/packages/microcosm-build/tests/test_uk_release_assembler.py b/packages/microcosm-build/tests/test_uk_release_assembler.py index efb2b4b91..730250a6d 100644 --- a/packages/microcosm-build/tests/test_uk_release_assembler.py +++ b/packages/microcosm-build/tests/test_uk_release_assembler.py @@ -75,7 +75,7 @@ def test_national_dataset_filename_mirrors_registry() -> None: assert driver._DATASET_KEY == "microcosm_uk_2024_25" assert driver._DATASET_FILENAME == f"{driver._DATASET_KEY}.h5" - assert REGISTRY[("uk", 2025, "compact")].filename == driver._DATASET_FILENAME + assert REGISTRY[("uk", 2025, "national")].filename == driver._DATASET_FILENAME def _frame(weights: list[float], *, weight_kind: WeightKind): diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index 74dbc6206..8c148c6b3 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -631,6 +631,23 @@ def line_for_release_id(release_id: str) -> str | None: return None +def dataset_role_for_line(line: str) -> str: + """The one dataset role a publication line may carry. + + The national line publishes national-default releases; every local-area + line publishes non-default local-area releases. The publisher refuses a + line promotion whose manifest declares another role, and the loader + refuses a manifest that reached a line pointer with the wrong role, so a + local-area release can never ride the national pointer (or the reverse) + on the strength of its release id alone. + """ + if line == "national": + return NATIONAL_DEFAULT_DATASET_ROLE + if re.fullmatch(r"local-k[1-9][0-9]*", line): + return NON_DEFAULT_LOCAL_AREA_DATASET_ROLE + raise ValueError(f"{line!r} is not a publication line.") + + # The dense joint national + local line (microcosm#762 A18, ruling # 2026-09-03): the same constant-id approach, published on the inspect lane # under the non-default local-area role. Mirrored from diff --git a/packages/microcosm-data/src/microcosm/data/loader.py b/packages/microcosm-data/src/microcosm/data/loader.py index c2f95246a..9f5c63cc2 100644 --- a/packages/microcosm-data/src/microcosm/data/loader.py +++ b/packages/microcosm-data/src/microcosm/data/loader.py @@ -30,6 +30,7 @@ NATIONAL_DEFAULT_DATASET_ROLE, NON_DEFAULT_LOCAL_AREA_DATASET_ROLE, RELEASE_MANIFEST_SCHEMA_VERSION, + dataset_role_for_line, ) from microcosm.data.registry import DEFAULT_VARIANT, REGISTRY, DatasetSpec from microcosm.data.release import ( @@ -289,6 +290,17 @@ def _certified_release_from_manifest( raise ValueError( f"Release {release_id!r} has unknown dataset_role {dataset_role!r}." ) + pointer_line = _line_from_pointer_path(spec.pointer_path) + if pointer_line is not None: + # A line pointer carries one role: the manifest it names must + # declare it, whatever its release id says. + expected_role = dataset_role_for_line(pointer_line) + if dataset_role != expected_role: + raise ValueError( + f"Release {release_id!r} declares dataset_role {dataset_role!r}; " + f"{spec.pointer_path!r} follows line {pointer_line!r}, which " + f"publishes only {expected_role!r} releases." + ) default_datasets = _required_mapping( manifest, "default_datasets", release_id=release_id ) @@ -316,6 +328,12 @@ def _certified_release_from_manifest( artifact_label = "local-area" artifacts = _required_mapping(manifest, "artifacts", release_id=release_id) + # One revision per pointer, by intent: the loader follows a pointer to a + # single immutable revision and downloads every artifact of the release + # from it, so every artifact must be pinned to exactly that revision. + # This is stricter than the publisher contract's annual-revision + # allowance for mixed-revision releases; a release that mixes revisions + # is publishable for inspection but never resolvable as a default here. mismatched_revisions = { str(key): candidate.get("revision") if isinstance(candidate, Mapping) else None for key, candidate in artifacts.items() diff --git a/packages/microcosm-data/src/microcosm/data/registry.py b/packages/microcosm-data/src/microcosm/data/registry.py index 4cc2851b5..e16e9f45f 100644 --- a/packages/microcosm-data/src/microcosm/data/registry.py +++ b/packages/microcosm-data/src/microcosm/data/registry.py @@ -113,9 +113,16 @@ def register(spec: DatasetSpec) -> DatasetSpec: register( DatasetSpec( + # microcosm#823: the UK 2024-25 national line, followed through its + # own pointer. Registered OFF the default variant until the first + # promoted cut exists on the Hub: resolve("uk") picks the latest + # default-variant year at import time, so a default entry here would + # send every default load to a pointer that does not exist yet. The + # flip to DEFAULT_VARIANT is a one-line follow-up after promotion + # (review of #966; Max's #823 ruling keys the line as a variant). country="uk", year=2025, - variant=DEFAULT_VARIANT, + variant="national", hf_repo="policyengine/populace-uk-private", filename="microcosm_uk_2024_25.h5", engine_module="policyengine_uk.data", diff --git a/packages/microcosm-data/src/microcosm/data/release.py b/packages/microcosm-data/src/microcosm/data/release.py index b0a673ae0..d39b86b71 100644 --- a/packages/microcosm-data/src/microcosm/data/release.py +++ b/packages/microcosm-data/src/microcosm/data/release.py @@ -49,6 +49,7 @@ EVIDENCE_RELEASE_ID_SEGMENT, NATIONAL_DEFAULT_DATASET_ROLE, _uk_line_cut_tag_re, + dataset_role_for_line, line_for_release_id, release_dataset_role, required_release_files, @@ -166,6 +167,13 @@ def line_pointer_path(line: str) -> str: raise ValueError( f"invalid release line {line!r}; expected ^[a-z]+(-k[1-9][0-9]*)?$" ) + if line == "evidence": + # The grammar would render latest-evidence.json, the evidence-tier + # pointer: refused at the grammar, not only downstream. + raise ValueError( + "release line 'evidence' collides with the evidence-tier pointer " + f"{LATEST_EVIDENCE_POINTER_PATH}; it is not a publication line." + ) return LINE_POINTER_PATH_TEMPLATE.format(line=line) @@ -307,6 +315,12 @@ def prepare_release( f"release {release_id!r} belongs to line {expected_line!r}, " f"not {line!r}." ) + expected_role = dataset_role_for_line(line) + if role != expected_role: + raise ValueError( + f"release {release_id!r} declares dataset_role {role!r}; line " + f"{line!r} publishes only {expected_role!r} releases." + ) else: pointer_path = LATEST_EVIDENCE_POINTER_PATH if evidence else LATEST_POINTER_PATH if role != NATIONAL_DEFAULT_DATASET_ROLE and update_latest and line is None: @@ -553,7 +567,14 @@ def publish_release( the evidence pointer. line: Promote an approved release line through ``latest-.json``. Line promotion requires ``update_latest=True``, a matching line - release id, and artifact revisions pinned to the selected tag. + release id, the line's dataset role, and artifact revisions + pinned to the selected tag. A cut published for inspection + earlier (``update_latest=False`` under the same tag) is promoted + by the same call: the publisher recognises the existing tag that + already describes this release, creates no second immutable + revision, and writes only the main commit carrying the pointer. + The same recognition makes a retry after a failure between tag + creation and the pointer commit safe. Returns: The release's pointer payload (``latest.json`` shape, plus ``tier`` at @@ -719,43 +740,61 @@ def _publish_atomic( ) -> None: staging_branch = f"release-staging/{release_id}" main_revision = _repo_revision(api, repo_id=repo_id) - api.create_branch( - repo_id=repo_id, - branch=staging_branch, - repo_type="dataset", - revision=main_revision, - ) - immutable_commit = api.create_commit( - repo_id=repo_id, - repo_type="dataset", - revision=staging_branch, - commit_message=f"Publish immutable release {release_id}", - operations=_commit_operations( - release_dir=release_dir, - artifact_root=artifact_root, - release_id=release_id, - filenames=filenames, - root_artifacts=root_artifacts, - ), + existing_revision = ( + _tag_revision(api, repo_id=repo_id, tag=tag) if create_tag else None ) - immutable_revision = _commit_revision(immutable_commit) - if immutable_revision is None: - raise RuntimeError( - "Hub create_commit returned no revision for immutable release " - f"{release_id!r}; refusing to update main or {pointer_path}." - ) - if create_tag: - _create_release_tag( + if existing_revision is not None: + # The immutable revision already exists under this tag (an inspect + # publication of the same cut, or a run that failed after tagging). + # It must describe exactly this release; then there is nothing + # immutable left to write, and publication continues with the main + # commit alone. A tag that describes another release refuses. + _require_tag_describes_release( api, repo_id=repo_id, + release_id=release_id, tag=tag, - revision=immutable_revision, + release_dir=release_dir, + ) + immutable_revision = existing_revision + else: + api.create_branch( + repo_id=repo_id, + branch=staging_branch, + repo_type="dataset", + revision=main_revision, + ) + immutable_commit = api.create_commit( + repo_id=repo_id, + repo_type="dataset", + revision=staging_branch, + commit_message=f"Publish immutable release {release_id}", + operations=_commit_operations( + release_dir=release_dir, + artifact_root=artifact_root, + release_id=release_id, + filenames=filenames, + root_artifacts=root_artifacts, + ), + ) + immutable_revision = _commit_revision(immutable_commit) + if immutable_revision is None: + raise RuntimeError( + "Hub create_commit returned no revision for immutable release " + f"{release_id!r}; refusing to update main or {pointer_path}." + ) + if create_tag: + _create_release_tag( + api, + repo_id=repo_id, + tag=tag, + revision=immutable_revision, + ) + api.delete_branch( + repo_id=repo_id, + branch=staging_branch, + repo_type="dataset", ) - api.delete_branch( - repo_id=repo_id, - branch=staging_branch, - repo_type="dataset", - ) if tag_only: # The release-id tag points directly at immutable_revision, so deleting # the temporary branch does not make the candidate unreachable. Exact-k @@ -926,6 +965,58 @@ def _commit_revision(commit_info: Any) -> str | None: return None +def _tag_revision(api: object, *, repo_id: str, tag: str) -> str | None: + """The revision an existing tag points at, or None when the tag is absent.""" + repo_info = getattr(api, "repo_info", None) + if not callable(repo_info): + return None + try: + info = repo_info(repo_id=repo_id, repo_type="dataset", revision=tag) + except Exception: # noqa: BLE001 - an absent revision is the common case + return None + value = info.get("sha") if isinstance(info, Mapping) else getattr(info, "sha", None) + return str(value) if value else None + + +def _require_tag_describes_release( + api: object, + *, + repo_id: str, + release_id: str, + tag: str, + release_dir: Path, +) -> None: + """Refuse an existing tag unless its release manifest is byte-identical. + + The manifest pins every artifact's digest and revision, so equal bytes + mean the tagged revision is this release; anything else (another cut, + a manifest that never landed) must not be promoted under this tag. + """ + manifest_path = f"releases/{release_id}/release_manifest.json" + try: + remote = Path( + api.hf_hub_download( + repo_id=repo_id, + filename=manifest_path, + repo_type="dataset", + revision=tag, + ) + ).read_bytes() + except Exception as exc: # noqa: BLE001 - the refusal names the cause + raise ValueError( + f"tag {tag!r} already exists in {repo_id} but carries no readable " + f"{manifest_path} ({exc}); it does not describe this release, so " + "it cannot be reused. Publish under a fresh cut tag." + ) from exc + local = (release_dir / "release_manifest.json").read_bytes() + if remote != local: + raise ValueError( + f"tag {tag!r} already exists in {repo_id} and describes another " + f"release (its {manifest_path} differs from the local one); " + "refusing to promote it. Publish under a fresh cut tag." + ) + + def _create_release_tag(api: object, *, repo_id: str, tag: str, revision: str | None): kwargs = { "repo_id": repo_id, diff --git a/packages/microcosm-data/tests/test_loader.py b/packages/microcosm-data/tests/test_loader.py index a50b3734b..44e7667be 100644 --- a/packages/microcosm-data/tests/test_loader.py +++ b/packages/microcosm-data/tests/test_loader.py @@ -345,10 +345,23 @@ def __init__(self, *, file_path: str) -> None: return constructed -def test_resolve_uk_defaults_to_2025_through_national_line_pointer( +def test_resolve_uk_default_stays_on_2023_until_the_national_line_is_promoted(): + """The national line is registered off the default variant until its + pointer exists on the Hub, so a default load never chases an absent + pointer (review of #966); the flip is a one-line follow-up.""" + assert latest_year("uk") == 2023 + assert resolve("uk").key == ("uk", 2023, DEFAULT_VARIANT) + assert resolve("uk", 2025, variant="national").pointer_path == ( + "latest-national.json" + ) + with pytest.raises(ValueError, match="published years"): + resolve("uk", 2025) + + +def test_resolve_uk_national_variant_follows_the_line_pointer( tmp_path: Path, ) -> None: - spec = resolve("uk") + spec = resolve("uk", 2025, variant="national") manifest = _uk_release_manifest() hub = _line_pointer_hub( tmp_path, @@ -361,7 +374,7 @@ def test_resolve_uk_defaults_to_2025_through_national_line_pointer( certified = loader._resolve_certified_release(spec, hub_download=hub) manifest_path = f"releases/{UK_NATIONAL_RELEASE_ID}/release_manifest.json" - assert spec.key == ("uk", 2025, DEFAULT_VARIANT) + assert spec.key == ("uk", 2025, "national") assert spec.pointer_path == "latest-national.json" assert certified.release_id == UK_NATIONAL_RELEASE_ID assert certified.artifact_revision == UK_NATIONAL_CUT_TAG @@ -436,7 +449,9 @@ def test_line_pointer_refuses_any_artifact_revision_mismatch(tmp_path: Path) -> ValueError, match=r"artifacts not pinned to expected revision.*calibration_report", ): - loader._resolve_certified_release(resolve("uk"), hub_download=hub) + loader._resolve_certified_release( + resolve("uk", 2025, variant="national"), hub_download=hub + ) def test_local_area_line_selects_its_only_microdata_artifact(tmp_path: Path) -> None: @@ -553,7 +568,9 @@ def test_loader_refuses_an_unknown_dataset_role(tmp_path: Path) -> None: ) with pytest.raises(ValueError, match="unknown dataset_role 'research_preview'"): - loader._resolve_certified_release(resolve("uk"), hub_download=hub) + loader._resolve_certified_release( + resolve("uk", 2025, variant="national"), hub_download=hub + ) def test_download_uses_pointer_manifest_and_release_revision( @@ -799,11 +816,29 @@ def _is_offline_error(exc: Exception) -> bool: ) +def _is_missing_pointer_error(exc: BaseException) -> bool: + names = {cls.__name__ for cls in type(exc).__mro__} + text = str(exc) + return bool( + names & {"EntryNotFoundError", "FileNotFoundError"} + or "404" in text + or "Entry Not Found" in text + ) + + @pytest.mark.skipif(_hf_offline(), reason="Hugging Face offline mode is enabled") -@pytest.mark.parametrize(("country", "year"), [("us", 2024), ("uk", 2023)]) +@pytest.mark.parametrize( + ("country", "year", "variant"), + [ + ("us", 2024, DEFAULT_VARIANT), + ("uk", 2023, DEFAULT_VARIANT), + ("uk", 2025, "national"), + ], +) def test_live_latest_pointer_resolves_to_a_coherent_certified_release( country: str, year: int, + variant: str, ) -> None: """The real pointer and pinned manifest resolve without downloading large H5s. @@ -815,12 +850,16 @@ def test_live_latest_pointer_resolves_to_a_coherent_certified_release( release *identity* belongs to policyengine.py's certification fixtures, which are updated atomically with each certification PR. """ - spec = resolve(country, year) + spec = resolve(country, year, variant=variant) try: certified = loader._resolve_certified_release(spec) except Exception as exc: if _is_offline_error(exc): pytest.skip(f"Hugging Face metadata unavailable offline: {exc}") + if variant != DEFAULT_VARIANT and _is_missing_pointer_error(exc): + # A line pointer exists only once its first cut is promoted; + # until then the line is registered but not yet live. + pytest.skip(f"line pointer {spec.pointer_path} not yet promoted: {exc}") if isinstance(exc, (RepositoryNotFoundError, GatedRepoError)) or "401" in str( exc ): @@ -836,6 +875,60 @@ def test_live_latest_pointer_resolves_to_a_coherent_certified_release( ) assert certified.artifact_repo_id == spec.hf_repo assert certified.artifact_path == spec.filename - assert certified.artifact_revision == certified.release_id + if variant == DEFAULT_VARIANT: + assert certified.artifact_revision == certified.release_id + else: + # A line pointer names the immutable cut tag of its constant id. + assert certified.artifact_revision.startswith(f"{certified.release_id}-") assert re.fullmatch(r"[0-9a-f]{64}", certified.artifact_sha256) assert certified.model.name == spec.engine_package + + +def test_line_pointer_refuses_a_manifest_with_another_lines_role(tmp_path: Path): + """A line pointer carries one role: the national pointer cannot select a + local-area manifest, nor a local-area pointer a national one.""" + spec = resolve("uk", 2025, variant="national") + manifest = _uk_release_manifest() + manifest["dataset_role"] = "non_default_local_area" + manifest["default_datasets"] = {} + hub = _line_pointer_hub( + tmp_path, + line="national", + release_id=UK_NATIONAL_RELEASE_ID, + revision=UK_NATIONAL_CUT_TAG, + manifest=manifest, + ) + with pytest.raises(ValueError, match="publishes only 'national_default'"): + loader._resolve_certified_release(spec, hub_download=hub) + + +def test_latest_pointer_refuses_any_artifact_revision_mismatch(tmp_path: Path): + """One revision per pointer, by intent: the loader is stricter than the + publisher contract's annual-revision allowance (review of #966).""" + spec = resolve("uk", 2023) + manifest = _uk_release_manifest( + release_id=UK_2023_RELEASE_ID, + revision=UK_2023_RELEASE_ID, + filename="populace_uk_2023.h5", + artifact_key="populace_uk_2023", + dataset_role=None, + ) + manifest["artifacts"]["calibration_report"] = { + "kind": "diagnostics", + "path": f"releases/{UK_2023_RELEASE_ID}/calibration_diagnostics.json", + "repo_id": UK_REPO_ID, + "revision": f"{UK_2023_RELEASE_ID}-annual-2027", + "sha256": "0" * 64, + } + pointer = latest_pointer_payload( + UK_2023_RELEASE_ID, updated_at="2026-06-19T03:00:00+00:00" + ) + hub = _hub_with_pointer( + tmp_path, + repo_id=UK_REPO_ID, + pointer_path=LATEST_POINTER_PATH, + pointer=pointer, + manifest=manifest, + ) + with pytest.raises(ValueError, match="not pinned to expected revision"): + loader._resolve_certified_release(spec, hub_download=hub) diff --git a/packages/microcosm-data/tests/test_registry.py b/packages/microcosm-data/tests/test_registry.py index 97da1209c..eae3a397a 100644 --- a/packages/microcosm-data/tests/test_registry.py +++ b/packages/microcosm-data/tests/test_registry.py @@ -68,7 +68,9 @@ def test_available_lists_default_variant_keys_only_sorted(monkeypatch) -> None: keys = available() assert ("us", 2024) in keys - assert ("uk", 2025) in keys + assert ("uk", 2023) in keys + # The national line is off the default variant until promoted. + assert ("uk", 2025) not in keys assert ("zz", 2099) not in keys assert keys == sorted(keys) @@ -144,14 +146,16 @@ def test_uk_2025_dense_is_registered_off_the_default_variant() -> None: assert spec.filename == "microcosm_uk_2024_25_dense.h5" assert spec.engine_class == "UKSingleYearDataset" assert spec.engine_package == "policyengine-uk" - assert ("uk", 2025, DEFAULT_VARIANT) in REGISTRY + assert ("uk", 2025, "national") in REGISTRY -def test_uk_2025_compact_is_registered_as_the_national_line() -> None: - spec = REGISTRY[("uk", 2025, DEFAULT_VARIANT)] - assert spec.variant == DEFAULT_VARIANT - assert spec.hf_repo == "policyengine/populace-uk-private" - assert spec.filename == "microcosm_uk_2024_25.h5" - assert spec.engine_class == "UKSingleYearDataset" - assert spec.engine_package == "policyengine-uk" +def test_uk_2025_national_is_registered_off_the_default_variant_until_promoted(): + """resolve("uk") must not reach a pointer that does not exist yet: the + line's pointer appears on the Hub at its first promotion, and only the + follow-up flip to the default variant moves the default (review of + #966; Max's #823 ruling keys the line as a variant).""" + spec = REGISTRY[("uk", 2025, "national")] + assert spec.variant == "national" != DEFAULT_VARIANT assert spec.pointer_path == "latest-national.json" + assert spec.filename == "microcosm_uk_2024_25.h5" + assert ("uk", 2025, DEFAULT_VARIANT) not in REGISTRY diff --git a/packages/microcosm-data/tests/test_release.py b/packages/microcosm-data/tests/test_release.py index 43b622a3f..dbbeab2d3 100644 --- a/packages/microcosm-data/tests/test_release.py +++ b/packages/microcosm-data/tests/test_release.py @@ -2264,3 +2264,168 @@ def no_hub(*args, **kwargs): publish_cli.main([*cli_args, "--preflight-only"]) assert hub.events == [] assert hub.uploads == [] + + +def _uk_hub(tmp_path: Path) -> FakeHub: + hub = FakeHub("policyengine/populace-uk-private") + hub._download_dir = tmp_path / "uk-hub-cache" + june_pointer = latest_pointer_payload( + JUNE_UK_RELEASE_ID, updated_at="2026-06-19T02:38:00+00:00" + ) + hub.seed_main_file(LATEST_POINTER_PATH, json.dumps(june_pointer, indent=1).encode()) + return hub + + +def test_inspect_then_promote_reuses_the_immutable_tag( + release_dir: Path, artifact_root: Path, tmp_path: Path, monkeypatch +) -> None: + """The documented two-step sequence: publish for inspection under the cut + tag, then promote the same cut. The second call recognises the existing + tag that describes this release, writes no second immutable revision + and only the main commit carrying the pointer (review of #966).""" + national = _as_uk_national_line_release(release_dir) + monkeypatch.setattr( + release_module, "validate_release_dir", lambda _path, **_kwargs: None + ) + monkeypatch.setattr(release_module, "notify_release", lambda *a, **k: None) + hub = _uk_hub(tmp_path) + june_pointer_bytes = hub._commits[hub._refs["main"]][LATEST_POINTER_PATH] + + publish_release( + national, + "policyengine/populace-uk-private", + api=hub, + artifact_root=artifact_root, + tag_name=UK_NATIONAL_CUT_TAG, + update_latest=False, + ) + assert line_pointer_path("national") not in hub._commits[hub._refs["main"]] + events_after_inspect = len(hub.events) + + promoted = publish_release( + national, + "policyengine/populace-uk-private", + api=hub, + artifact_root=artifact_root, + tag_name=UK_NATIONAL_CUT_TAG, + line="national", + updated_at="2026-09-23T12:00:00+00:00", + ) + + kinds = [event for event, _ in hub.events[events_after_inspect:]] + assert kinds == ["create_commit"], kinds + assert hub.tags == [{"tag": UK_NATIONAL_CUT_TAG, "revision": "commit-1"}] + main_files = hub._commits[hub._refs["main"]] + assert main_files[LATEST_POINTER_PATH] == june_pointer_bytes + assert json.loads(main_files[line_pointer_path("national")])["revision"] == ( + UK_NATIONAL_CUT_TAG + ) + assert promoted["revision"] == UK_NATIONAL_CUT_TAG + assert ( + latest_line_release( + "policyengine/populace-uk-private", line="national", api=hub + ).revision + == UK_NATIONAL_CUT_TAG + ) + + +def test_retry_after_a_failed_pointer_commit_reuses_the_tag( + release_dir: Path, artifact_root: Path, tmp_path: Path, monkeypatch +) -> None: + """A failure between tag creation and the pointer commit leaves the tag; + the retry stands on it instead of dying on create_tag.""" + national = _as_uk_national_line_release(release_dir) + monkeypatch.setattr( + release_module, "validate_release_dir", lambda _path, **_kwargs: None + ) + monkeypatch.setattr(release_module, "notify_release", lambda *a, **k: None) + hub = _uk_hub(tmp_path) + hub.fail_main_commit = True + with pytest.raises(RuntimeError, match="injected main commit failure"): + publish_release( + national, + "policyengine/populace-uk-private", + api=hub, + artifact_root=artifact_root, + tag_name=UK_NATIONAL_CUT_TAG, + line="national", + ) + assert hub.tags == [{"tag": UK_NATIONAL_CUT_TAG, "revision": "commit-1"}] + assert line_pointer_path("national") not in hub._commits[hub._refs["main"]] + + hub.fail_main_commit = False + publish_release( + national, + "policyengine/populace-uk-private", + api=hub, + artifact_root=artifact_root, + tag_name=UK_NATIONAL_CUT_TAG, + line="national", + ) + + assert hub.tags == [{"tag": UK_NATIONAL_CUT_TAG, "revision": "commit-1"}] + assert line_pointer_path("national") in hub._commits[hub._refs["main"]] + + +def test_promotion_refuses_a_tag_that_describes_another_release( + release_dir: Path, artifact_root: Path, tmp_path: Path, monkeypatch +) -> None: + national = _as_uk_national_line_release(release_dir) + monkeypatch.setattr( + release_module, "validate_release_dir", lambda _path, **_kwargs: None + ) + monkeypatch.setattr(release_module, "notify_release", lambda *a, **k: None) + hub = _uk_hub(tmp_path) + # A tag already pointing at a revision with no manifest for this release. + hub._refs[UK_NATIONAL_CUT_TAG] = hub._refs["main"] + with pytest.raises(ValueError, match="does not describe this release"): + publish_release( + national, + "policyengine/populace-uk-private", + api=hub, + artifact_root=artifact_root, + tag_name=UK_NATIONAL_CUT_TAG, + line="national", + ) + # A tag whose manifest is another cut's. + hub._commits["other"] = { + f"releases/{UK_NATIONAL_RELEASE_ID}/release_manifest.json": b"{}" + } + hub._refs[UK_NATIONAL_CUT_TAG] = "other" + with pytest.raises(ValueError, match="describes another release"): + publish_release( + national, + "policyengine/populace-uk-private", + api=hub, + artifact_root=artifact_root, + tag_name=UK_NATIONAL_CUT_TAG, + line="national", + ) + assert line_pointer_path("national") not in hub._commits[hub._refs["main"]] + + +def test_line_pointer_path_refuses_the_evidence_line() -> None: + with pytest.raises(ValueError, match="collides with the evidence-tier pointer"): + line_pointer_path("evidence") + + +def test_line_promotion_refuses_a_release_whose_role_is_not_the_lines( + release_dir: Path, artifact_root: Path, monkeypatch +) -> None: + """The national line carries national-default releases only; a + local-area manifest cannot ride the national pointer on its id.""" + national = _as_uk_national_line_release(release_dir) + manifest_path = national / "release_manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["dataset_role"] = "non_default_local_area" + manifest_path.write_text(json.dumps(manifest)) + monkeypatch.setattr( + release_module, "validate_release_dir", lambda _path, **_kwargs: None + ) + with pytest.raises(ValueError, match="publishes only 'national_default'"): + release_module.prepare_release( + national, + artifact_root=artifact_root, + tag_name=UK_NATIONAL_CUT_TAG, + line="national", + ) diff --git a/tools/assemble_uk_release_dir.py b/tools/assemble_uk_release_dir.py index 96a09aae1..240afe808 100644 --- a/tools/assemble_uk_release_dir.py +++ b/tools/assemble_uk_release_dir.py @@ -160,6 +160,11 @@ def _assemble(args: argparse.Namespace) -> dict[str, object]: build_record.get("input_posture"), "build_record.input_posture" ) + _require_equal( + "certification.candidate.filename vs the national line's dataset filename", + candidate.get("filename"), + _DATASET_FILENAME, + ) _require_equal( "candidate bytes vs certification.candidate.sha256", measured["candidate"], From 72d6094cb31bdd98371a1b3184cf597064e1cdee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:58:20 +0100 Subject: [PATCH 5/6] Only absent answers are swallowed on the tag-reuse path; narrow the live skip; fix the stale registry fragment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vahid's post-approval notes on #966. _tag_revision treated any repo_info failure as "no such tag", so a transient failure with the tag in place re-entered the create path and died on create_tag with the staging branch left behind, which runbook §4 now says cannot happen; _require_tag_describes_release treated any download failure as "describes another release" and sent the operator to a fresh cut tag, orphaning a good one. Both now swallow only the Hub's (or the fake's) "does not exist" answers (RevisionNotFoundError / KeyError, EntryNotFoundError / FileNotFoundError) and let everything else propagate; tested with injected transient failures on both calls. The live pointer test skipped "not yet promoted" on any 404 or EntryNotFoundError, before the credentials branch, so an anonymous RepositoryNotFoundError or a post-promotion manifest 404 skipped too; the skip now requires an EntryNotFoundError naming the pointer path and runs after the credentials branch. The 823-uk-2025-registry fragment still announced the 2025 default and neither fragment nor the data README recorded that available() lists default-variant pairs only; both say so now. Co-Authored-By: Claude Fable 5.1 --- changelog.d/823-uk-2025-registry.changed.md | 2 +- packages/microcosm-data/README.md | 5 +- .../src/microcosm/data/release.py | 33 ++++++++++-- packages/microcosm-data/tests/test_loader.py | 24 +++++---- packages/microcosm-data/tests/test_release.py | 53 +++++++++++++++++++ 5 files changed, 99 insertions(+), 18 deletions(-) diff --git a/changelog.d/823-uk-2025-registry.changed.md b/changelog.d/823-uk-2025-registry.changed.md index 24cb526be..2c9cc54ef 100644 --- a/changelog.d/823-uk-2025-registry.changed.md +++ b/changelog.d/823-uk-2025-registry.changed.md @@ -1 +1 @@ -Changed the UK 2025 registry default to the 2024–25 national dataset and taught the certified loader to follow per-line pointers at immutable revisions. Non-default local-area manifests now load only through their line pointers. +Registered the UK 2024-25 national line in the data registry as variant `national` on `microcosm_uk_2024_25.h5`, followed through `latest-national.json`, and taught the certified loader to follow per-line pointers at immutable revisions; non-default local-area manifests load only through their line pointers. `available()` now lists default-variant `(country, year)` pairs only (`available_variants()` lists every registered variant, the dense line included). The UK default stays on the June 2023 line until the national line's first promotion and the follow-up flip of its entry to the default variant. diff --git a/packages/microcosm-data/README.md b/packages/microcosm-data/README.md index cf6551e75..6aa7d14a3 100644 --- a/packages/microcosm-data/README.md +++ b/packages/microcosm-data/README.md @@ -23,8 +23,9 @@ sim.calculate("household_net_income", 2024).sum() `latest.json`, reads the selected release manifest at the immutable release tag, verifies the artifact SHA-256, and refuses model or Core versions outside the release's certified compatibility specifiers. `available()` lists -published `(country, year)` pairs; `available_variants()` lists every published -`(country, year, variant)`. +the default-variant `(country, year)` pairs (the ones a bare ``load(country, year)`` +resolves); `available_variants()` lists every published +`(country, year, variant)`, non-default lines included. The old mutable-root behavior is available only as an unsafe escape hatch: `load("us", 2024, unverified_root=True)`. It emits a runtime warning because it diff --git a/packages/microcosm-data/src/microcosm/data/release.py b/packages/microcosm-data/src/microcosm/data/release.py index d39b86b71..f0d9974fb 100644 --- a/packages/microcosm-data/src/microcosm/data/release.py +++ b/packages/microcosm-data/src/microcosm/data/release.py @@ -972,12 +972,32 @@ def _tag_revision(api: object, *, repo_id: str, tag: str) -> str | None: return None try: info = repo_info(repo_id=repo_id, repo_type="dataset", revision=tag) - except Exception: # noqa: BLE001 - an absent revision is the common case - return None + except Exception as exc: + # Only "no such revision" means the tag is absent. Anything else (a + # transient network failure, an auth error) must propagate: treating + # it as absent would re-enter the create path and die on create_tag + # with the staging branch left behind. + if _is_absent_revision_error(exc): + return None + raise value = info.get("sha") if isinstance(info, Mapping) else getattr(info, "sha", None) return str(value) if value else None +_ABSENT_REVISION_ERRORS = frozenset({"RevisionNotFoundError", "KeyError"}) +_ABSENT_ENTRY_ERRORS = frozenset({"EntryNotFoundError", "FileNotFoundError"}) + + +def _is_absent_revision_error(exc: BaseException) -> bool: + """A Hub (or fake) answer that the revision does not exist, nothing else.""" + return bool({cls.__name__ for cls in type(exc).__mro__} & _ABSENT_REVISION_ERRORS) + + +def _is_absent_entry_error(exc: BaseException) -> bool: + """A Hub (or fake) answer that the file does not exist at that revision.""" + return bool({cls.__name__ for cls in type(exc).__mro__} & _ABSENT_ENTRY_ERRORS) + + def _require_tag_describes_release( api: object, *, @@ -1002,9 +1022,14 @@ def _require_tag_describes_release( revision=tag, ) ).read_bytes() - except Exception as exc: # noqa: BLE001 - the refusal names the cause + except Exception as exc: + # Only an absent manifest means the tag describes another release; a + # transient failure must propagate rather than send the operator to a + # fresh cut tag and orphan a good one. + if not _is_absent_entry_error(exc): + raise raise ValueError( - f"tag {tag!r} already exists in {repo_id} but carries no readable " + f"tag {tag!r} already exists in {repo_id} but carries no " f"{manifest_path} ({exc}); it does not describe this release, so " "it cannot be reused. Publish under a fresh cut tag." ) from exc diff --git a/packages/microcosm-data/tests/test_loader.py b/packages/microcosm-data/tests/test_loader.py index 44e7667be..5b3aff2f0 100644 --- a/packages/microcosm-data/tests/test_loader.py +++ b/packages/microcosm-data/tests/test_loader.py @@ -816,14 +816,13 @@ def _is_offline_error(exc: Exception) -> bool: ) -def _is_missing_pointer_error(exc: BaseException) -> bool: +def _is_missing_pointer_error(exc: BaseException, pointer_path: str) -> bool: + """The pointer file itself is absent at the repo root: an + ``EntryNotFoundError`` naming that path (the Hub's answer for a file that + does not exist at the revision). Repository, auth and manifest errors are + not this, and must not skip as "not yet promoted".""" names = {cls.__name__ for cls in type(exc).__mro__} - text = str(exc) - return bool( - names & {"EntryNotFoundError", "FileNotFoundError"} - or "404" in text - or "Entry Not Found" in text - ) + return "EntryNotFoundError" in names and pointer_path in str(exc) @pytest.mark.skipif(_hf_offline(), reason="Hugging Face offline mode is enabled") @@ -856,16 +855,19 @@ def test_live_latest_pointer_resolves_to_a_coherent_certified_release( except Exception as exc: if _is_offline_error(exc): pytest.skip(f"Hugging Face metadata unavailable offline: {exc}") - if variant != DEFAULT_VARIANT and _is_missing_pointer_error(exc): - # A line pointer exists only once its first cut is promoted; - # until then the line is registered but not yet live. - pytest.skip(f"line pointer {spec.pointer_path} not yet promoted: {exc}") if isinstance(exc, (RepositoryNotFoundError, GatedRepoError)) or "401" in str( exc ): pytest.skip( f"repo requires credentials this environment lacks (private): {exc}" ) + if variant != DEFAULT_VARIANT and _is_missing_pointer_error( + exc, spec.pointer_path + ): + # A line pointer exists only once its first cut is promoted; + # until then the line is registered but not yet live. Anything + # else on a promoted line (a manifest 404, a bad pin) fails. + pytest.skip(f"line pointer {spec.pointer_path} not yet promoted: {exc}") raise assert certified.release_id.startswith( diff --git a/packages/microcosm-data/tests/test_release.py b/packages/microcosm-data/tests/test_release.py index dbbeab2d3..fbb54f488 100644 --- a/packages/microcosm-data/tests/test_release.py +++ b/packages/microcosm-data/tests/test_release.py @@ -2429,3 +2429,56 @@ def test_line_promotion_refuses_a_release_whose_role_is_not_the_lines( tag_name=UK_NATIONAL_CUT_TAG, line="national", ) + + +def test_transient_errors_while_checking_an_existing_tag_propagate( + release_dir: Path, artifact_root: Path, tmp_path: Path, monkeypatch +) -> None: + """Only "absent" answers are swallowed: a transient failure looking up + the tag or its manifest propagates, so a good cut is never orphaned + and the create path is never re-entered with the tag in place.""" + national = _as_uk_national_line_release(release_dir) + monkeypatch.setattr( + release_module, "validate_release_dir", lambda _path, **_kwargs: None + ) + monkeypatch.setattr(release_module, "notify_release", lambda *a, **k: None) + hub = _uk_hub(tmp_path) + original_repo_info = hub.repo_info + + def flaky_repo_info(*, repo_id, repo_type, revision=None): + if revision == UK_NATIONAL_CUT_TAG: + raise RuntimeError("injected transient Hub failure") + return original_repo_info( + repo_id=repo_id, repo_type=repo_type, revision=revision + ) + + monkeypatch.setattr(hub, "repo_info", flaky_repo_info) + with pytest.raises(RuntimeError, match="injected transient Hub failure"): + publish_release( + national, + "policyengine/populace-uk-private", + api=hub, + artifact_root=artifact_root, + tag_name=UK_NATIONAL_CUT_TAG, + line="national", + ) + assert hub.tags == [] + assert not [event for event, _ in hub.events if event == "create_branch"] + + monkeypatch.setattr(hub, "repo_info", original_repo_info) + hub._refs[UK_NATIONAL_CUT_TAG] = hub._refs["main"] + + def flaky_download(*, repo_id, filename, repo_type, revision=None): + raise RuntimeError("injected transient download failure") + + monkeypatch.setattr(hub, "hf_hub_download", flaky_download) + with pytest.raises(RuntimeError, match="injected transient download failure"): + publish_release( + national, + "policyengine/populace-uk-private", + api=hub, + artifact_root=artifact_root, + tag_name=UK_NATIONAL_CUT_TAG, + line="national", + ) + assert line_pointer_path("national") not in hub._commits[hub._refs["main"]] From 3bd0ed2fd95b08cdcbf3cf618cf5a1c868db32ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 24 Sep 2026 14:23:20 +0100 Subject: [PATCH 6/6] Give the synthetic smoke refusal test the time a cold import needs test_synthetic_smoke_command_refuses_release_options runs the spine builder and expects argparse's refusal within 30 s; reaching argparse imports the build package and the engine, and the 3.14 CI runner took longer than that on #966 while the 3.13 lane passed. 120 s, as #1006 sets it, is what a cold import needs; the assertion is unchanged. Co-Authored-By: Claude Fable 5.1 --- .../microcosm-build/tests/test_uk_staging_integration.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/microcosm-build/tests/test_uk_staging_integration.py b/packages/microcosm-build/tests/test_uk_staging_integration.py index 71346d87b..6116976ae 100644 --- a/packages/microcosm-build/tests/test_uk_staging_integration.py +++ b/packages/microcosm-build/tests/test_uk_staging_integration.py @@ -147,7 +147,10 @@ def test_synthetic_smoke_command_refuses_release_options(tmp_path: Path) -> None cwd=ROOT, text=True, capture_output=True, - timeout=30, + # The refusal comes from argparse, but reaching it imports the build + # package and the engine; the 3.14 CI runner has taken over 30 s for + # that alone (a flake seen on #966), so allow what a cold import needs. + timeout=120, ) assert result.returncode == 2