diff --git a/changelog.d/build-outcomes-blocked-status.changed.md b/changelog.d/build-outcomes-blocked-status.changed.md new file mode 100644 index 000000000..9ea59d4aa --- /dev/null +++ b/changelog.d/build-outcomes-blocked-status.changed.md @@ -0,0 +1 @@ +UK builds now record how they ended the same way everywhere. Staging run documents move to schema version 3: a build whose gates refused its candidate closes with the new `blocked` status and a `block` record (gate phase, blocking gate ids, count) instead of `completed`, and a build that raises closes `failed` with a classified error code and `failure_class` (`INTERRUPTED`, `OUT_OF_MEMORY`, `GRAPH_NODE_FAILED`, `BUILD_FAILED`) instead of a uniform `BUILD_FAILED`. A dense build refused at the preflight gates is now recorded as blocked at phase `preflight` (it used to close as a pass), with Logbook receipts that resolve in the preflight gate document. The hosted emitter gains the same `blocked` run event and error codes. One classification (`microcosm.build.run_outcome`) drives the staging bundle, the emitter and the Logbook disposition, so they cannot disagree. The delivery summary keeps contract version 2; version 2 documents stay readable, and the version 2 fixtures are frozen beside the new version 3 set. Readers must accept version 3 first (PolicyEngine/calibration-diagnostics#206). A block whose details the staging content policy refuses closes the run `failed` with `GATE_BLOCK_UNRECORDED` rather than leaving it `running`; `blocking_gate_ids` is empty, never a placeholder, when the refusal named no gate; a raised refusal's `blocked` event carries the gate statuses when its report could be read. diff --git a/changelog.d/build-outcomes-collector-rejections.fixed.md b/changelog.d/build-outcomes-collector-rejections.fixed.md new file mode 100644 index 000000000..a864f4ccd --- /dev/null +++ b/changelog.d/build-outcomes-collector-rejections.fixed.md @@ -0,0 +1 @@ +The hosted telemetry emitter no longer retries a settled collector rejection forever. A 4xx other than a timeout (408) or a rate limit (429) on a run's events or registration (an event shape the collector does not accept, say) now makes that run local-only with a recorded reason and one warning; the build continues and later contract changes cannot wedge delivery. diff --git a/changelog.d/build-outcomes-dense-gate-signing.fixed.md b/changelog.d/build-outcomes-dense-gate-signing.fixed.md new file mode 100644 index 000000000..e7836fc59 --- /dev/null +++ b/changelog.d/build-outcomes-dense-gate-signing.fixed.md @@ -0,0 +1 @@ +A graph-built dense candidate can now be assembled. Its `*.local_gates.json` used to hold the graph's unsigned 26-gate document, which the dense release contract, the release preflight and the size evaluation could not read. The build now writes the signed six-gate local battery report there: the local outcomes are projected from the graph's terminal document (nothing is re-evaluated), replayed through the gate battery and signed with the attempt's Logbook build id as `release_id`. The report is written before any refusal, so a blocked candidate leaves it too, and it stays unsigned (never shippable) without the signing key or a Logbook attempt. The full document keeps its graph evidence name (`uk.full.gates.calibrated.gate_report.json`, `outputs.full_gate_report`) and is what the package binds. Gate receipts now resolve: local gates in the signed report, the rest in the full document. `--release-candidate` now refuses to start without a 32-byte signing key or with `--target-geographies` that select no local targets; a filtered build writes no local report and says why. The release preflight requires exactly 32 bytes of key. diff --git a/changelog.d/build-outcomes-resume-lineage.added.md b/changelog.d/build-outcomes-resume-lineage.added.md new file mode 100644 index 000000000..b07731417 --- /dev/null +++ b/changelog.d/build-outcomes-resume-lineage.added.md @@ -0,0 +1 @@ +UK full builds (dense and national) now record how a resumed attempt built on earlier ones. Each attempt's `request.json` names its Logbook build id and staging run id; the rowwise manifest gains an `execution` block (graph store, attempt directory, nodes reused from the store versus computed, and the earlier attempt directories on the same store with their ids); and the staging run gets the counts as a `graph_execution` stage event. The block sits outside the run parameters, so the candidate identity is unchanged. diff --git a/changelog.d/build-outcomes-sigterm.changed.md b/changelog.d/build-outcomes-sigterm.changed.md new file mode 100644 index 000000000..89351809f --- /dev/null +++ b/changelog.d/build-outcomes-sigterm.changed.md @@ -0,0 +1 @@ +UK spine, dense and national builds now record a SIGTERM (a supervisor, a budget stop, `kill`) instead of dying silently: the staging run closes `failed` with `TERMINATED`, the attempt gets a `discarded` Logbook row like Ctrl-C, and the command exits with status 143. The spine build now also records Ctrl-C (it used to leave the run `running` with no row). A second SIGTERM still kills the process at once; SIGKILL and out-of-memory kills stay covered only by the hosted emitter's heartbeat. diff --git a/changelog.d/build-outcomes-spine-output-sha256.added.md b/changelog.d/build-outcomes-spine-output-sha256.added.md new file mode 100644 index 000000000..e0bbead10 --- /dev/null +++ b/changelog.d/build-outcomes-spine-output-sha256.added.md @@ -0,0 +1 @@ +The UK spine build now records its output H5's file sha256, measured after the last write (smoke marking included): in the build sidecar (`output {filename, sha256, size_bytes}`), in a `
.sha256` file beside the H5, in the `spine_h5_creation` stage event and in the Logbook row's `pipeline` verdict (`artifact_sha256`). A full build binding that spine refuses an input H5 whose digest differs from the sidecar's `output.sha256`; sidecars written before carry no such key and still bind by content identity. diff --git a/docs/uk-full-build-graph.md b/docs/uk-full-build-graph.md index 9d76af74c..0cedd56e4 100644 --- a/docs/uk-full-build-graph.md +++ b/docs/uk-full-build-graph.md @@ -24,7 +24,7 @@ uv run --no-sync python tools/build_uk_full.py --release-role dense \ The three atomic-area supports are the artifacts pinned in `uk/uk_atomic_area_supports.provenance.json` and `uk/spec/sources.yaml` (built by `tools/build_uk_atomic_area_supports.py` from the published ONS, NRS and NISRA lookups; publisher registration PolicyEngine/chronicle#269). `--geography-assignment atomic` is the default and requires all three; `--atomic-support-sha256-{ew,scotland,ni}` pin them like `--ladder-sha256`, and a release candidate requires all four pins. `--geography-assignment legacy` keeps the previous sequential ladder draw for measurement builds, takes no supports, and is refused by `--release-candidate`. -The checkpoint must bind the exact frame content, the current spine stage roster and the gate-report bytes. The bound-spine node compares the checkpoint's gate report digests with the branch's own gate declarations and refuses a spine whose gate manifest differs from them, so `--input-h5` needs a spine built by a branch with the same declarations; every acceptance spine on disk when this registration landed predates them and is not admitted. Historical candidate H5 files and reviewed-bypass sidecars are not alternate build sources. Chronicle facts and manifest must match the independently reviewed national and local feed declarations; filtering targets does not relax source validation. +The checkpoint must bind the exact frame content, the current spine stage roster and the gate-report bytes. A spine build records its H5's file digest in the sidecar (`output {filename, sha256, size_bytes}`), in a `
.sha256` file beside it, in its `spine_h5_creation` stage event and in its Logbook `pipeline` verdict (`artifact_sha256`); when the sidecar carries `output.sha256`, the full build refuses an `--input-h5` whose measured digest differs (older sidecars without the key bind by content identity alone). The bound-spine node compares the checkpoint's gate report digests with the branch's own gate declarations and refuses a spine whose gate manifest differs from them, so `--input-h5` needs a spine built by a branch with the same declarations; every acceptance spine on disk when this registration landed predates them and is not admitted. Historical candidate H5 files and reviewed-bypass sidecars are not alternate build sources. Chronicle facts and manifest must match the independently reviewed national and local feed declarations; filtering targets does not relax source validation. The target registry binds Census household and demographic rows from the reviewed Chronicle feed, including the approved Northern Ireland constituency geography. Geography is assigned after expansion by the shared atomic-geography operators (microcosm#931): `uk.full.identity` keys every household with `household_draw_key` from the spine's explicit lineage (source household id, SPI support channel and clone index, CGT clone and donor flags) and the pool clone index; `uk.full.geography.assign` draws one atomic area per household (E&W 2021 Output Area, Scotland 2022 Output Area, NI 2021 Data Zone) by census household count within the household's FRS region, with a keyed `sha256-u53-v1` stream so a household's draw never depends on row order, K or any other household; `uk.full.geography.derive` reads every larger geography off the support's versioned mappings; `uk.full.geography.local_authority` resolves the engine's `local_authority` key from the derived authority code (the same resolver the ladder path uses); the shared `uk.full.geography.gate` and the UK distribution gate `uk.full.geography_gate` follow, and `uk.full.pool` refuses unless the shared gate passed. The OA ladder still supplies the constituency and local-authority rosters, household dispersion and lookup support for target compilation; its household counts are not a second source of calibration targets. Source receipts retain the Chronicle identity, the paired ladder digest and the geography binding (assignment mode, definition sha256, support pins, identity column, stream), so target values and the geography used to assign households can be audited separately. @@ -99,15 +99,17 @@ The full target compiler preserves the unreduced band-edge register, reference-p The shared store defaults to `/.graph-store`. `--graph-store` can reuse another store. `--resume require` requires completed numerical nodes and evidence to be available; output materialization and byte readback still verify the recreated files. `--resume-size-checkpoint` imports a legacy size-search checkpoint only after validating invocation identity, ordered target/household axes, initial weights and recomputed losses. It skips the saved dense solve and search. New runs persist their intermediates as graph artifacts before drawing. -Each attempt also keeps its checkpoint manifests and small stage, gate and provenance reports under `/uk-full-attempts/`. A failed run's `failure.json` links to this evidence, including a persisted spine gate verdict before downstream admission stops execution. A previously completed output bundle remains intact. +Each attempt also keeps its checkpoint manifests and small stage, gate and provenance reports under `/uk-full-attempts/`. Its `request.json` carries the request bindings and the attempt's Logbook build id and staging run id (`attempt`). The rowwise manifest's `execution` block records the graph store, the attempt directory, how many graph nodes the attempt reused from the store rather than computed (the first run to reach a node decides), and the earlier attempts on the same store that ran the same request (their directories and ids, the twenty most recent, beside counts of every earlier attempt on the store and of the matching ones), so a resumed build names the attempts whose work it reused; the same counts reach the staging run as a `graph_execution` stage event. The block sits outside the run parameters, so it does not change the candidate identity. A failed run's `failure.json` links to this evidence, including a persisted spine gate verdict before downstream admission stops execution. A previously completed output bundle remains intact. -The output bundle is named from the role's posture and the FRS release vintage: `microcosm_uk_2024_25_local.h5`, its signed gate report `microcosm_uk_2024_25_local.local_gates.json`, and the `.diagnostics.json`, `.targets.csv`, `.area_support.csv`, `.holdout.json` and `.target_selection.json` siblings on the same stem, beside `graph.json`, `operations.json`, the stored source/stage/gate evidence and the graph manifests. `candidate.json` is the immutable graph package inventory: it binds the dataset, its evidence, the target selector and the independent K/k request. Comparison inputs refer to this candidate identity. Adding comparison evidence does not rewrite the candidate package. `rowwise_candidate_manifest.json` is projected from the stored artifacts in the schema-4 shape the rowwise tool wrote (`graph_terminal.rowwise_candidate_manifest_from_graph`), so the dense release pre-flight and assembler read a graph build as they read a rowwise-tool build; it records the release role, the release verdict, `staging_delivery` and `staged_dataset`. +The output bundle is named from the role's posture and the FRS release vintage: `microcosm_uk_2024_25_local.h5`, its local gate report `microcosm_uk_2024_25_local.local_gates.json`, and the `.diagnostics.json`, `.targets.csv`, `.area_support.csv`, `.holdout.json` and `.target_selection.json` siblings on the same stem, beside `graph.json`, `operations.json`, the stored source/stage/gate evidence and the graph manifests. `candidate.json` is the immutable graph package inventory: it binds the dataset, its evidence, the target selector and the independent K/k request. Comparison inputs refer to this candidate identity. Adding comparison evidence does not rewrite the candidate package. `rowwise_candidate_manifest.json` is projected from the stored artifacts in the schema-4 shape the rowwise tool wrote (`graph_terminal.rowwise_candidate_manifest_from_graph`), so the dense release pre-flight and assembler read a graph build as they read a rowwise-tool build; it records the release role, the release verdict, `staging_delivery` and `staged_dataset`. + +The graph evaluates every declared UK gate once; `uk.full.gates.calibrated.gate_report.json` is that full terminal document, the one the package inventory binds (`outputs.full_gate_report`). The local gate report is the six-gate local battery the dense release contract verifies: the driver projects the six `UK_LOCAL_GATE_SCOPE` outcomes from the full document (nothing is re-evaluated; the projection checks the declarations are the same), replays them through the gate battery without raising (the graph's own enforcement still sets the exit status), and writes the report before any refusal, so a blocked candidate leaves it too. Its `release_id` is the Logbook attempt's build id. It is signed when the attempt is bound and `MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY` holds a 32-byte key; otherwise it is written unsigned, with `signing_error`, and is never shippable. A build whose `--target-geographies` select no local targets drops the local fit claim: it writes no local gate report (`outputs.local_gate_report` is null and `local_gate_report_absence` says why) rather than recording the excluded local gates as not applicable. The Logbook receipts of the six local gates resolve in the local report (`#/gates/`); the other gates' receipts resolve in the full document (`#/report/outcomes/`). The terminal graph node writes unsigned `certification.json` from those identified artifacts and any declared native or matched-size comparisons. `build.json` is the completion marker that binds both `candidate.json` and the certification artifact. Physical output bytes are checked against their declared artifacts. Bundle publication writes the completion marker last and rolls back handled failures or interrupts. A process kill or power loss can leave an absent completion marker; a directory without a valid bound marker is not a completed build. -A non-dry dense run is wrapped in the rowwise tool's operational envelope: the Logbook attempt (a `uk-local-candidate` row spooled under `/logbook-spool` on every terminal outcome, chained through `--logbook-prev-row-digest` or `POPULACE_LOGBOOK_PREV_ROW_DIGEST`, with an error receipt on failure), version 2 staging telemetry with the run's sampling evidence (`sample: {"mode": "full"}` on the f100 rung, null below it, judged on the effective fraction), stage events around each graph phase and per-epoch `calibration_progress` rows from the dense solve and, with `--dataset-households`, the size search and refit (rows tagged with their `phase`), and the staged-dataset delivery of the published bundle under `staged//` in the private repository. `--staging-local-only`, `--no-staging`, `--staging-read-back` and `--no-staged-dataset` behave as in [UK staging operations](uk-staging-operations.md). Dry runs plan without solving or writing and record no Logbook row. +A non-dry dense run is wrapped in the rowwise tool's operational envelope: the Logbook attempt (a `uk-local-candidate` row spooled under `/logbook-spool` on every terminal outcome, chained through `--logbook-prev-row-digest` or `POPULACE_LOGBOOK_PREV_ROW_DIGEST`, with an error receipt on failure), version 3 staging telemetry with the run's sampling evidence (`sample: {"mode": "full"}` on the f100 rung, null below it, judged on the effective fraction), stage events around each graph phase and per-epoch `calibration_progress` rows from the dense solve and, with `--dataset-households`, the size search and refit (rows tagged with their `phase`), and the staged-dataset delivery of the published bundle under `staged//` in the private repository. `--staging-local-only`, `--no-staging`, `--staging-read-back` and `--no-staged-dataset` behave as in [UK staging operations](uk-staging-operations.md). Dry runs plan without solving or writing and record no Logbook row. -Structural failures stop export. The maintained local statistical failure policy may still export an unreleasable diagnostic candidate with a nonzero process status. Missing evidence remains explicit. `--release-candidate` applies the maintained strictness and solve settings; it does not publish, sign or authorize a release. Fixture acceptance proves graph behavior. Native certification additionally requires measured incumbent comparison evidence supplied with `--native-scorecard`; exact-count promotion also requires the measured comparison at the requested k through `--matched-size-scorecard`. The certification node verifies their candidate and output identities before assessing readiness. +Structural failures stop export. The maintained local statistical failure policy may still export an unreleasable diagnostic candidate with a nonzero process status. Missing evidence remains explicit. `--release-candidate` applies the maintained strictness and solve settings and requires the UK gate signing key and local targets, because it signs the local gate report; it does not publish or authorize a release. Fixture acceptance proves graph behavior. Native certification additionally requires measured incumbent comparison evidence supplied with `--native-scorecard`; exact-count promotion also requires the measured comparison at the requested k through `--matched-size-scorecard`. The certification node verifies their candidate and output identities before assessing readiness. `tools/build_uk_rowwise_dataset.py` stays the separate driver it was (its tests load it by path, and it still serves `--candidate-clone-counts`). `tools/calibrate_uk_national_dataset.py` was retired by microcosm#823 and does not forward. diff --git a/docs/uk-staging-operations.md b/docs/uk-staging-operations.md index 4c6b33541..a71bb3de3 100644 --- a/docs/uk-staging-operations.md +++ b/docs/uk-staging-operations.md @@ -53,8 +53,11 @@ repositories accept the new fixture manifest in their respective feature branches. Version 1 fixtures describe current US output and remain fixed during the UK -implementation. Version 2 fixtures include successful spine, calibration, -sanitized failure, delivery-failure, and incompatible-version cases. +implementation. Version 2 fixtures (successful spine, calibration, sanitized +failure, delivery-failure, and incompatible-version cases) are frozen: the UK +writer now emits version 3, and the version 2 bytes stay only to prove old runs +still read. Version 3 fixtures add a `blocked` run and a failure that carries +its `failure_class`. The implementation separates versioned file serialization from country configuration. `microcosm.build.staging_storage` owns repository access and @@ -66,7 +69,8 @@ shared modules contain no UK repository or environment defaults. The two serializers remain separate because version 1 and version 2 intentionally write different files. -Regenerate and verify the canonical version 2 bytes with: +Regenerate and verify the canonical version 3 bytes with (the generator no +longer writes version 2): ```bash uv run python tools/generate_staging_contract_fixtures.py @@ -120,12 +124,12 @@ The two UK commands (`tools/build_uk_frs_spine.py`, a shim over in either release role, `national` or `dense`; `tools/build_uk_rowwise_candidate.py` is a stub over the same driver) support these staging modes: -- Default: local version 2 files plus best-effort delivery to +- Default: local version 3 files plus best-effort delivery to `policyengine/populace-uk-staging`. - `--staging-local-only`: the same validated files without constructing a remote client. -- `--no-staging`: no telemetry files, with a version 2 opt-out object written - into build evidence. +- `--no-staging`: no telemetry files, with an opt-out delivery object (delivery + contract version 2) written into build evidence. Common options are `--staging-dir`, `--staging-repo-id`, `--staging-run-id`, `--staging-candidate-id`, @@ -133,7 +137,7 @@ Common options are `--staging-dir`, `--staging-repo-id`, repository identifier is invalid in remote mode. Authenticated read-back is valid only in remote mode. -Version 2 storage uses the fixed repository prefix `runs/`. Individual runs +Version 2 and 3 storage use the fixed repository prefix `runs/`. Individual runs cannot select another prefix, so consumers can enumerate that subtree without traversing the rest of the repository. @@ -146,8 +150,36 @@ runs//events.ndjson runs//calibration_progress.json # calibration runs only ``` -Every JSON document and event declares its schema name and version 2. Unknown -schema names or versions are incompatible data. Only reviewed aggregate JSON +Every JSON document and event declares its schema name and version (3 for +new runs; version 2 runs stay valid, but one run never mixes versions). Unknown +schema names or versions are incompatible data. + +A run ends in one of three terminal statuses: + +- `completed`: the build finished and its gates passed; +- `blocked` (version 3): the build reached a gate decision and the gates refused + the candidate. The run carries `block` (`phase`; `blocking_failure_count`, at + least 1; `blocking_gate_ids`, empty when the refusal named no gate, never a + placeholder) and ends with a `blocked` event that also lists the gate statuses + when the refusing report could be read. A dense build refused at the preflight + gates is blocked at phase `preflight`. A block whose details the staging + content policy refuses closes the run `failed` with `GATE_BLOCK_UNRECORDED` + instead, so no run is left `running`; +- `failed`: the build raised, or returned without a recorded block. `failure` + carries an `error_code` (`INTERRUPTED`, `TERMINATED`, `OUT_OF_MEMORY`, + `GRAPH_NODE_FAILED`, `BUILD_FAILED`, `BUILD_REFUSED`, `RUNG_ABORTED`, + `GATE_BLOCK_UNRECORDED`) and, in version 3, a `failure_class`. + +A stopped build is a failed run, not a running one. Ctrl-C closes it with +`INTERRUPTED`; a SIGTERM (a supervisor, a budget stop, `kill`) closes it with +`TERMINATED`, and the command then exits with status 143. Both record a +`discarded` Logbook row. The first SIGTERM is handled once: a second one kills +the process at once, even while the graph store is still settling. SIGKILL and +out-of-memory kills cannot be caught; for those the hosted emitter's heartbeat +reports `unexpected_process_exit`, and no Logbook row is written. + +The delivery summary (`staging_delivery`) keeps contract version 2: its shape +did not change, and publication and the release assemblers pin it. Only reviewed aggregate JSON artifacts are permitted; population H5 files, NumPy archives, source survey tables, row-level extracts, archives, credentials, and environment data are rejected before remote storage is called. diff --git a/packages/microcosm-build/src/microcosm/build/gate_battery.py b/packages/microcosm-build/src/microcosm/build/gate_battery.py index e476787a5..0a75277c9 100644 --- a/packages/microcosm-build/src/microcosm/build/gate_battery.py +++ b/packages/microcosm-build/src/microcosm/build/gate_battery.py @@ -847,10 +847,19 @@ class GateBatteryBlockedError(RuntimeError): including the block itself — is on disk. """ - def __init__(self, phase: str, failures: Sequence[str], report_path: Path) -> None: + def __init__( + self, + phase: str, + failures: Sequence[str], + report_path: Path, + *, + blocking_gate_ids: Sequence[str] = (), + ) -> None: self.phase = phase self.failures = tuple(failures) self.report_path = report_path + #: The ids of the gates that blocked (empty when the raiser did not say). + self.blocking_gate_ids = tuple(blocking_gate_ids) lines = "\n".join(f" - {line}" for line in self.failures) super().__init__( f"Gate battery blocked at phase {phase!r} (report: {report_path}):\n{lines}" @@ -1045,7 +1054,12 @@ def enforce(self, phase: str, *, mode: BlockingMode) -> bool: for outcome in blocking: if outcome.status is GateStatus.EVIDENCE_ABSENT: failures.append(f"[{outcome.entry.id}] {outcome.reason}") - raise GateBatteryBlockedError(phase, failures, self._report_path) + raise GateBatteryBlockedError( + phase, + failures, + self._report_path, + blocking_gate_ids=[outcome.entry.id for outcome in blocking], + ) return True # -- report assembly ---------------------------------------------------- diff --git a/packages/microcosm-build/src/microcosm/build/run_outcome.py b/packages/microcosm-build/src/microcosm/build/run_outcome.py new file mode 100644 index 000000000..58087c0e8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/run_outcome.py @@ -0,0 +1,224 @@ +"""How a build ended, decided once and copied into every record. + +The staging run bundle, the hosted telemetry and the Logbook row each describe a +build's end in their own vocabulary. This module is the single place that maps one +build's end onto all three, so they cannot disagree: + +* a build whose gates refused its candidate is ``blocked`` (a staging status of its + own since contract version 3), wherever the refusal surfaced — a phase report the + build checked itself, or a :class:`GateBatteryBlockedError` / + ``SpineGateBlockedError`` raised (possibly wrapped by the graph executor); +* a build that raised is ``failed`` with an error code and a failure class drawn + from the labels the calibration dashboard renders; +* the Logbook keeps its own vocabulary, derived here by :func:`logbook_disposition`. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + +from microcosm.build.gate_battery import GateBatteryBlockedError + + +class BuildRefusedError(RuntimeError): + """A build returned a non-zero status without raising and without a gate block. + + No such path exists today (every refusal is a recorded gate block); this is the + guard that keeps a future one from closing as ``completed``. + """ + + +class RunOutcome(Enum): + """How a build attempt ended.""" + + COMPLETED = "completed" + BLOCKED = "blocked" + FAILED = "failed" + INTERRUPTED = "interrupted" + TERMINATED = "terminated" + ABORTED = "aborted" + + +#: The Logbook disposition each outcome records. A blocked candidate is a failed +#: attempt to the Logbook; an operator stop or a rung abort discards the attempt. +_LOGBOOK_DISPOSITIONS = { + RunOutcome.COMPLETED: "iterating", + RunOutcome.BLOCKED: "failed", + RunOutcome.FAILED: "failed", + RunOutcome.INTERRUPTED: "discarded", + RunOutcome.TERMINATED: "discarded", + RunOutcome.ABORTED: "discarded", +} + + +def logbook_disposition(outcome: RunOutcome) -> str: + """The Logbook ``disposition`` for one outcome.""" + + return _LOGBOOK_DISPOSITIONS[outcome] + + +@dataclass(frozen=True) +class GateBlock: + """Which gate phase refused the candidate, and which gates.""" + + phase: str + #: The gates that refused; empty when the raiser named none (the count + #: still says how many failures there were). Never a placeholder id. + blocking_gate_ids: tuple[str, ...] + blocking_failure_count: int + #: Every gate's status in the refusing report, when that report could be + #: read; ``None`` otherwise. + gate_statuses: Mapping[str, str] | None = None + + @classmethod + def of( + cls, + phase: str, + blocking_gate_ids: Sequence[str], + *, + blocking_failure_count: int | None = None, + gate_statuses: Mapping[str, str] | None = None, + ) -> GateBlock: + ids = tuple(str(gate_id) for gate_id in blocking_gate_ids) + count = len(ids) if blocking_failure_count is None else blocking_failure_count + return cls( + phase=str(phase), + blocking_gate_ids=ids, + blocking_failure_count=max(1, int(count)), + gate_statuses=None + if gate_statuses is None + else { + str(gate_id): str(status) for gate_id, status in gate_statuses.items() + }, + ) + + +@dataclass(frozen=True) +class Classified: + """One build's end, ready for the staging bundle, the emitter and the Logbook.""" + + outcome: RunOutcome + error_code: str | None = None + failure_class: str | None = None + block: GateBlock | None = None + + @property + def disposition(self) -> str: + return logbook_disposition(self.outcome) + + +#: A run whose gate block could not be recorded (the staging contract refused +#: the block's details) closes ``failed`` with this code and class instead, so no +#: path leaves a run ``running``. +UNRECORDED_GATE_BLOCK = Classified( + RunOutcome.FAILED, "GATE_BLOCK_UNRECORDED", "unrecorded_gate_block" +) + + +def _cause_chain(error: BaseException) -> Iterator[BaseException]: + seen: set[int] = set() + current: BaseException | None = error + while current is not None and id(current) not in seen: + seen.add(id(current)) + yield current + current = current.__cause__ or current.__context__ + + +def _written_gate_statuses(report_path: object) -> dict[str, str] | None: + """The gate statuses a battery wrote beside its block, if the file reads.""" + + try: + gates = json.loads(Path(report_path).read_text(encoding="utf-8")).get("gates") + except (OSError, TypeError, ValueError, AttributeError): + return None + if not isinstance(gates, Mapping): + return None + return { + str(gate_id): str(entry.get("status")) + for gate_id, entry in gates.items() + if isinstance(entry, Mapping) + } + + +def _phase_report_gate_statuses(report: object) -> dict[str, str] | None: + """The gate statuses of an in-memory phase report (a spine refusal's).""" + + outcomes = getattr(report, "outcomes", None) + if outcomes is None: + return None + try: + return { + str(outcome.entry.id): str(getattr(outcome.status, "value", outcome.status)) + for outcome in outcomes + } + except AttributeError: + return None + + +def _gate_block(error: BaseException) -> GateBlock | None: + """A gate refusal anywhere in the cause chain, as a :class:`GateBlock`.""" + + for link in _cause_chain(error): + if isinstance(link, GateBatteryBlockedError): + ids = link.blocking_gate_ids or tuple( + line.split("]", 1)[0].lstrip("[") + for line in link.failures + if line.startswith("[") and "]" in line + ) + return GateBlock.of( + link.phase, + ids, + blocking_failure_count=max(len(link.failures), len(ids), 1), + gate_statuses=_written_gate_statuses(link.report_path), + ) + # SpineGateBlockedError lives in the UK runtime; match it structurally so + # this module stays country-agnostic. + if type(link).__name__ == "SpineGateBlockedError": + ids = tuple(getattr(link, "blocking_gate_ids", ()) or ()) + report = getattr(link, "report", None) + phase = getattr(link, "phase", None) or getattr(report, "phase", "spine") + return GateBlock.of( + phase, ids, gate_statuses=_phase_report_gate_statuses(report) + ) + return None + + +def classify_failure(error: BaseException) -> Classified: + """Classify a raised build error: a gate block, an operator stop, or a failure.""" + + block = _gate_block(error) + if block is not None: + return Classified(RunOutcome.BLOCKED, block=block) + for link in _cause_chain(error): + if type(link).__name__ == "BuildTerminatedError": + return Classified(RunOutcome.TERMINATED, "TERMINATED", "terminated") + for link in _cause_chain(error): + if isinstance(link, KeyboardInterrupt): + return Classified(RunOutcome.INTERRUPTED, "INTERRUPTED", "interrupted") + if isinstance(error, BuildRefusedError): + return Classified(RunOutcome.FAILED, "BUILD_REFUSED", "refused") + for link in _cause_chain(error): + if isinstance(link, MemoryError): + return Classified(RunOutcome.FAILED, "OUT_OF_MEMORY", "out_of_memory") + for link in _cause_chain(error): + if type(link).__name__ in {"NodeRejectedError", "NodeRejected"}: + return Classified(RunOutcome.FAILED, "GRAPH_NODE_FAILED", "error") + return Classified(RunOutcome.FAILED, "BUILD_FAILED", "error") + + +def classify_return(status: int, block: GateBlock | None) -> Classified: + """Classify a build that returned rather than raised. + + A non-zero return with no recorded block is a failure: no path that refuses a + candidate may close as ``completed``. + """ + + if block is not None: + return Classified(RunOutcome.BLOCKED, block=block) + if status == 0: + return Classified(RunOutcome.COMPLETED) + return Classified(RunOutcome.FAILED, "BUILD_REFUSED", "refused") diff --git a/packages/microcosm-build/src/microcosm/build/staging_v2.py b/packages/microcosm-build/src/microcosm/build/staging_v2.py index 105c668b6..aeff31e80 100644 --- a/packages/microcosm-build/src/microcosm/build/staging_v2.py +++ b/packages/microcosm-build/src/microcosm/build/staging_v2.py @@ -26,7 +26,14 @@ HuggingFaceDatasetStorage, ) +#: The delivery summary's contract version. Its shape has not changed since version 2, +#: and publishers and release assemblers pin it, so it does not follow the documents. STAGING_CONTRACT_VERSION = 2 +#: The schema version of the run documents this writer emits. Version 3 adds the +#: ``blocked`` run status with its ``block`` details, and ``failure_class`` on failures. +STAGING_DOCUMENT_VERSION = 3 +#: Document versions this module reads: every historical run stays valid. +SUPPORTED_DOCUMENT_VERSIONS = (2, 3) DEFAULT_STAGING_PREFIX = "runs" RUN_MANIFEST_SCHEMA = "microcosm.staging.run-manifest" @@ -35,7 +42,7 @@ EVENT_SCHEMA = "microcosm.staging.event" DeliveryMode = Literal["local_and_remote", "local_only", "disabled"] -LifecycleStatus = Literal["running", "completed", "failed"] +LifecycleStatus = Literal["running", "completed", "blocked", "failed"] _SAFE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") _SHA256 = re.compile(r"^[0-9a-f]{64}$") @@ -207,12 +214,12 @@ def _normalized_content_key(value: object) -> str: return re.sub(r"[^a-z0-9]+", "_", with_word_boundaries.lower()).strip("_") -def _schema_identity(name: str) -> dict[str, Any]: +def _schema_identity(name: str, version: int) -> dict[str, Any]: return { "type": "object", "properties": { "schema_name": {"const": name}, - "schema_version": {"const": STAGING_CONTRACT_VERSION}, + "schema_version": {"const": version}, }, "required": ["schema_name", "schema_version"], } @@ -297,6 +304,34 @@ def _schema_identity(name: str) -> dict[str, Any]: "additionalProperties": False, } +_FAILURE_SCHEMA_V3: dict[str, Any] = { + **_FAILURE_SCHEMA, + "properties": { + **_FAILURE_SCHEMA["properties"], + "failure_class": { + "type": ["string", "null"], + "pattern": "^[a-z][a-z0-9_]*$", + }, + }, + "required": [*_FAILURE_SCHEMA["required"], "failure_class"], +} + +#: Why a run's gates refused its candidate (version 3). ``phase`` names the gate +#: phase that refused it (``preflight``, ``terminal``, a spine phase). +_BLOCK_SCHEMA: dict[str, Any] = { + "type": ["object", "null"], + "properties": { + "phase": _SAFE_ID_SCHEMA, + "blocking_failure_count": {"type": "integer", "minimum": 1}, + "blocking_gate_ids": { + "type": "array", + "items": {"type": "string", "minLength": 1, "maxLength": 200}, + }, + }, + "required": ["phase", "blocking_failure_count", "blocking_gate_ids"], + "additionalProperties": False, +} + _PIPELINE_SCHEMA: dict[str, Any] = { "type": "object", "properties": { @@ -307,6 +342,15 @@ def _schema_identity(name: str) -> dict[str, Any]: "additionalProperties": False, } +_RUN_STATUSES: dict[int, list[str]] = { + 2: ["running", "completed", "failed"], + 3: ["running", "completed", "blocked", "failed"], +} +_EVENT_STATUSES: dict[int, list[str]] = { + 2: ["started", "completed", "failed", "progress"], + 3: ["started", "completed", "failed", "progress", "blocked"], +} + _RUN_FIELDS: dict[str, Any] = { "run_id": _SAFE_ID_SCHEMA, "country_code": {"type": "string", "pattern": "^[A-Z]{2}$"}, @@ -328,8 +372,9 @@ def _object_schema( name: str, properties: Mapping[str, Any], required: list[str], + version: int, ) -> dict[str, Any]: - identity = _schema_identity(name) + identity = _schema_identity(name, version) return { "type": "object", "properties": {**identity["properties"], **properties}, @@ -338,111 +383,149 @@ def _object_schema( } -SCHEMAS: dict[str, dict[str, Any]] = { - RUN_MANIFEST_SCHEMA: _object_schema( - RUN_MANIFEST_SCHEMA, - { - **_RUN_FIELDS, - "sample": _SAMPLE_SCHEMA, - "delivery": _DELIVERY_SCHEMA, - "artifacts": {"type": "array", "items": _ARTIFACT_SCHEMA}, - "failure": _FAILURE_SCHEMA, - "paths": { - "type": "object", - "properties": { - "progress": {"type": "string"}, - "events": {"type": "string"}, - "calibration_progress": {"type": ["string", "null"]}, - }, - "required": ["progress", "events", "calibration_progress"], - "additionalProperties": False, - }, - }, - [*_RUN_REQUIRED, "sample", "delivery", "artifacts", "failure", "paths"], - ), - PROGRESS_SCHEMA: _object_schema( - PROGRESS_SCHEMA, - { - **_RUN_FIELDS, - "sample": _SAMPLE_SCHEMA, - "delivery": _DELIVERY_SCHEMA, - "message": {"type": ["string", "null"], "maxLength": 500}, - "details": {"type": "object"}, - "failure": _FAILURE_SCHEMA, - }, - [ - *_RUN_REQUIRED, - "sample", - "delivery", - "message", - "details", - "failure", - ], - ), - CALIBRATION_PROGRESS_SCHEMA: _object_schema( - CALIBRATION_PROGRESS_SCHEMA, - { - "run_id": _SAFE_ID_SCHEMA, - "candidate_id": _SAFE_ID_SCHEMA, - "updated_at": _TIMESTAMP_SCHEMA, - "events": { - "type": "array", - "items": { +def _schemas(version: int) -> dict[str, dict[str, Any]]: + """The four document schemas of one contract version.""" + + run_fields = {**_RUN_FIELDS, "status": {"enum": _RUN_STATUSES[version]}} + failure_schema = _FAILURE_SCHEMA if version == 2 else _FAILURE_SCHEMA_V3 + block = {} if version == 2 else {"block": _BLOCK_SCHEMA} + block_required = [] if version == 2 else ["block"] + return { + RUN_MANIFEST_SCHEMA: _object_schema( + RUN_MANIFEST_SCHEMA, + { + **run_fields, + "sample": _SAMPLE_SCHEMA, + "delivery": _DELIVERY_SCHEMA, + "artifacts": {"type": "array", "items": _ARTIFACT_SCHEMA}, + "failure": failure_schema, + **block, + "paths": { "type": "object", "properties": { - "timestamp": _TIMESTAMP_SCHEMA, - "epoch": {"type": ["integer", "null"], "minimum": 0}, - "epochs": {"type": ["integer", "null"], "minimum": 0}, - "phase": {"type": ["string", "null"]}, - "loss": {"type": ["number", "null"]}, - "iteration": {"type": ["integer", "null"], "minimum": 0}, - "budget_search": {"type": ["integer", "null"], "minimum": 0}, - "budget_iteration": {"type": ["integer", "null"], "minimum": 0}, - "budget_iters": {"type": ["integer", "null"], "minimum": 0}, - "l0_lambda": {"type": ["number", "null"]}, + "progress": {"type": "string"}, + "events": {"type": "string"}, + "calibration_progress": {"type": ["string", "null"]}, }, - "required": [ - "timestamp", - "epoch", - "epochs", - "phase", - "loss", - "iteration", - "budget_search", - "budget_iteration", - "budget_iters", - "l0_lambda", - ], + "required": ["progress", "events", "calibration_progress"], "additionalProperties": False, }, }, - }, - ["run_id", "candidate_id", "updated_at", "events"], - ), - EVENT_SCHEMA: _object_schema( - EVENT_SCHEMA, - { - "sequence": {"type": "integer", "minimum": 1}, - "timestamp": _TIMESTAMP_SCHEMA, - "event_type": {"enum": ["stage", "calibration"]}, - "run_id": _SAFE_ID_SCHEMA, - "stage_id": _SAFE_ID_SCHEMA, - "status": {"enum": ["started", "completed", "failed", "progress"]}, - "message": {"type": ["string", "null"], "maxLength": 500}, - "details": {"type": "object"}, - }, - [ - "sequence", - "timestamp", - "event_type", - "run_id", - "stage_id", - "status", - "message", - "details", - ], - ), + [ + *_RUN_REQUIRED, + "sample", + "delivery", + "artifacts", + "failure", + *block_required, + "paths", + ], + version, + ), + PROGRESS_SCHEMA: _object_schema( + PROGRESS_SCHEMA, + { + **run_fields, + "sample": _SAMPLE_SCHEMA, + "delivery": _DELIVERY_SCHEMA, + "message": {"type": ["string", "null"], "maxLength": 500}, + "details": {"type": "object"}, + "failure": failure_schema, + **block, + }, + [ + *_RUN_REQUIRED, + "sample", + "delivery", + "message", + "details", + "failure", + *block_required, + ], + version, + ), + CALIBRATION_PROGRESS_SCHEMA: _object_schema( + CALIBRATION_PROGRESS_SCHEMA, + { + "run_id": _SAFE_ID_SCHEMA, + "candidate_id": _SAFE_ID_SCHEMA, + "updated_at": _TIMESTAMP_SCHEMA, + "events": { + "type": "array", + "items": { + "type": "object", + "properties": { + "timestamp": _TIMESTAMP_SCHEMA, + "epoch": {"type": ["integer", "null"], "minimum": 0}, + "epochs": {"type": ["integer", "null"], "minimum": 0}, + "phase": {"type": ["string", "null"]}, + "loss": {"type": ["number", "null"]}, + "iteration": {"type": ["integer", "null"], "minimum": 0}, + "budget_search": { + "type": ["integer", "null"], + "minimum": 0, + }, + "budget_iteration": { + "type": ["integer", "null"], + "minimum": 0, + }, + "budget_iters": { + "type": ["integer", "null"], + "minimum": 0, + }, + "l0_lambda": {"type": ["number", "null"]}, + }, + "required": [ + "timestamp", + "epoch", + "epochs", + "phase", + "loss", + "iteration", + "budget_search", + "budget_iteration", + "budget_iters", + "l0_lambda", + ], + "additionalProperties": False, + }, + }, + }, + ["run_id", "candidate_id", "updated_at", "events"], + version, + ), + EVENT_SCHEMA: _object_schema( + EVENT_SCHEMA, + { + "sequence": {"type": "integer", "minimum": 1}, + "timestamp": _TIMESTAMP_SCHEMA, + "event_type": {"enum": ["stage", "calibration"]}, + "run_id": _SAFE_ID_SCHEMA, + "stage_id": _SAFE_ID_SCHEMA, + "status": {"enum": _EVENT_STATUSES[version]}, + "message": {"type": ["string", "null"], "maxLength": 500}, + "details": {"type": "object"}, + }, + [ + "sequence", + "timestamp", + "event_type", + "run_id", + "stage_id", + "status", + "message", + "details", + ], + version, + ), + } + + +SCHEMAS_BY_VERSION: dict[int, dict[str, dict[str, Any]]] = { + version: _schemas(version) for version in SUPPORTED_DOCUMENT_VERSIONS } +#: The schemas of the documents this writer emits. +SCHEMAS: dict[str, dict[str, Any]] = SCHEMAS_BY_VERSION[STAGING_DOCUMENT_VERSION] def validate_staging_delivery(payload: Mapping[str, Any]) -> dict[str, Any]: @@ -544,17 +627,19 @@ def disabled_staging_delivery(reason: str) -> dict[str, Any]: ) -def validate_v2_document(payload: Mapping[str, Any]) -> dict[str, Any]: - """Validate one independently identified version 2 document.""" +def validate_staging_document(payload: Mapping[str, Any]) -> dict[str, Any]: + """Validate one independently identified staging document (version 2 or 3).""" normalized = _jsonable(payload) name = normalized.get("schema_name") version = normalized.get("schema_version") - if name not in _SCHEMA_NAMES or version != STAGING_CONTRACT_VERSION: + if name not in _SCHEMA_NAMES or version not in SCHEMAS_BY_VERSION: raise StagingContractError( f"Unsupported staging schema identity: {name!r} version {version!r}." ) - validator = Draft202012Validator(SCHEMAS[name], format_checker=FormatChecker()) + validator = Draft202012Validator( + SCHEMAS_BY_VERSION[version][name], format_checker=FormatChecker() + ) errors = sorted( validator.iter_errors(normalized), key=lambda error: list(error.path) ) @@ -568,20 +653,31 @@ def validate_v2_document(payload: Mapping[str, Any]) -> dict[str, Any]: "non_release must be true exactly when release_id is absent." ) delivery = validate_staging_delivery(normalized["delivery"]) - if ( - normalized["status"] == "running" - and delivery["read_back"] != "not_requested" - ): + status = normalized["status"] + if status == "running" and delivery["read_back"] != "not_requested": raise StagingContractError( "Running staging runs cannot report completed remote read-back." ) - if normalized["status"] == "failed" and normalized["failure"] is None: + if status == "failed" and normalized["failure"] is None: raise StagingContractError("Failed runs require sanitized failure data.") - if normalized["status"] != "failed" and normalized["failure"] is not None: + if status != "failed" and normalized["failure"] is not None: raise StagingContractError("Non-failed runs cannot contain failure data.") + if version >= 3: + if status == "blocked" and normalized["block"] is None: + raise StagingContractError( + "Blocked runs require the gate phase and gates that refused them." + ) + if status != "blocked" and normalized["block"] is not None: + raise StagingContractError( + "Non-blocked runs cannot contain block data." + ) return normalized +#: The version 2 name, kept for callers that predate version 3. +validate_v2_document = validate_staging_document + + @dataclass(frozen=True) class StagingContentPolicy: """Validate every remote file before a transport operation.""" @@ -706,7 +802,11 @@ def _reject_record_collections(self, value: Any) -> None: class StagingRunBundleWriterV2: - """Record, validate, persist, and optionally upload a version 2 run bundle.""" + """Record, validate, persist, and optionally upload a staging run bundle. + + Writes version 3 documents (``STAGING_DOCUMENT_VERSION``); the delivery summary + keeps contract version 2. + """ def __init__( self, @@ -782,6 +882,7 @@ def __init__( self.details: dict[str, Any] = {} self.sample: dict[str, Any] | None = None self.failure: dict[str, Any] | None = None + self.block_details: dict[str, Any] | None = None self._artifacts: list[dict[str, Any]] = [] self._events: list[dict[str, Any]] = [] self._calibration_events: list[dict[str, Any]] = [] @@ -940,12 +1041,19 @@ def fail( error: BaseException, *, error_code: str = "BUILD_FAILED", + failure_class: str | None = None, local_diagnostic_reference: str | None = None, ) -> None: self._require_running("record a failure") error_code = error_code.strip().upper() if not re.fullmatch(r"[A-Z0-9_]+", error_code): raise StagingContractError("error_code must contain only A-Z, 0-9, and _.") + if failure_class is not None and not re.fullmatch( + r"[a-z][a-z0-9_]*", failure_class + ): + raise StagingContractError( + "failure_class must be a lowercase identifier (a-z, 0-9, _)." + ) error_type = type(error).__name__ if not re.fullmatch(r"[A-Za-z][A-Za-z0-9_]*", error_type): error_type = "BuildError" @@ -962,12 +1070,17 @@ def fail( "local_diagnostic_reference": _safe_local_reference( local_diagnostic_reference ), + "failure_class": failure_class, } self._append_event( stage_id="failed", status="failed", message=self.message, - details={"error_code": error_code, "error_type": error_type}, + details={ + "error_code": error_code, + "error_type": error_type, + "failure_class": failure_class, + }, timestamp=self.updated_at, ) self._persist_bundle() @@ -990,11 +1103,72 @@ def complete(self, *, message: str = "Staging run completed.") -> None: self._persist_bundle() self._terminal_upload() + def block( + self, + *, + phase: str, + blocking_gate_ids: list[str] | tuple[str, ...], + blocking_failure_count: int | None = None, + gate_statuses: Mapping[str, str] | None = None, + ) -> None: + """Close the run as ``blocked``: it reached a gate decision and the gates + refused its candidate. Distinct from ``failed``, where the build raised. + + ``blocking_gate_ids`` may be empty when the refusal named no gate; the + caller then gives ``blocking_failure_count``. Every check runs before + the run's state changes, so a refused block leaves the run ``running`` + and the caller can still close it ``failed``. + """ + + self._require_running("record a gate block") + phase = _safe_identifier(phase, label="block phase") + gate_ids = [str(gate_id) for gate_id in blocking_gate_ids] + count = ( + len(gate_ids) if blocking_failure_count is None else blocking_failure_count + ) + if ( + isinstance(count, bool) + or not isinstance(count, int) + or count < 1 + or any(not gate_id for gate_id in gate_ids) + ): + raise StagingContractError( + "A gate block needs at least one blocking failure, and every gate " + "id it names must be a non-empty string." + ) + block_details = { + "phase": phase, + "blocking_failure_count": int(count), + "blocking_gate_ids": gate_ids, + } + details: dict[str, Any] = dict(block_details) + if gate_statuses is not None: + details["gate_statuses"] = { + str(gate_id): str(status) for gate_id, status in gate_statuses.items() + } + details = _jsonable(details) + self._content_policy.validate_payload(details) + self.status = "blocked" + self.current_stage = "blocked" + self.updated_at = self._clock() + self.message = f"The gates refused the candidate at {phase}." + self.block_details = block_details + self.details = details + self._append_event( + stage_id="blocked", + status="blocked", + message=self.message, + details=details, + timestamp=self.updated_at, + ) + self._persist_bundle() + self._terminal_upload() + def verify_remote(self) -> None: if self.status == "running": raise StagingContractError( - "Authenticated remote read-back requires a completed or failed " - "staging run." + "Authenticated remote read-back requires a completed, blocked or " + "failed staging run." ) if self._transport is None: raise StagingReadBackError("Remote read-back requires remote staging mode.") @@ -1036,7 +1210,7 @@ def _require_running(self, action: str) -> None: ) def validate_local_bundle(self) -> dict[str, Any]: - return validate_v2_bundle( + return validate_staging_bundle( self.local_dir, self.run_id, content_policy=self._content_policy, @@ -1054,7 +1228,7 @@ def _append_event( ) -> None: event = { "schema_name": EVENT_SCHEMA, - "schema_version": STAGING_CONTRACT_VERSION, + "schema_version": STAGING_DOCUMENT_VERSION, "sequence": len(self._events) + 1, "timestamp": timestamp, "event_type": event_type, @@ -1065,7 +1239,7 @@ def _append_event( "details": _jsonable(details), } self._content_policy.validate_payload(event["details"]) - self._events.append(validate_v2_document(event)) + self._events.append(validate_staging_document(event)) def _run_fields(self) -> dict[str, Any]: return { @@ -1091,12 +1265,13 @@ def _manifest(self) -> dict[str, Any]: ) return { "schema_name": RUN_MANIFEST_SCHEMA, - "schema_version": STAGING_CONTRACT_VERSION, + "schema_version": STAGING_DOCUMENT_VERSION, **self._run_fields(), "sample": self.sample, "delivery": self.delivery_summary, "artifacts": list(self._artifacts), "failure": self.failure, + "block": self.block_details, "paths": { "progress": f"{self.repo_run_prefix}/progress.json", "events": f"{self.repo_run_prefix}/events.ndjson", @@ -1107,19 +1282,20 @@ def _manifest(self) -> dict[str, Any]: def _progress(self) -> dict[str, Any]: return { "schema_name": PROGRESS_SCHEMA, - "schema_version": STAGING_CONTRACT_VERSION, + "schema_version": STAGING_DOCUMENT_VERSION, **self._run_fields(), "sample": self.sample, "delivery": self.delivery_summary, "message": self.message, "details": self.details, "failure": self.failure, + "block": self.block_details, } def _calibration_progress(self) -> dict[str, Any]: return { "schema_name": CALIBRATION_PROGRESS_SCHEMA, - "schema_version": STAGING_CONTRACT_VERSION, + "schema_version": STAGING_DOCUMENT_VERSION, "run_id": self.run_id, "candidate_id": self.candidate_id, "updated_at": self.updated_at, @@ -1136,7 +1312,7 @@ def _persist_bundle(self) -> None: self._calibration_progress() ) for path, payload in documents.items(): - validate_v2_document(payload) + validate_staging_document(payload) path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(_json_bytes(payload)) (self.run_dir / "events.ndjson").write_bytes(_ndjson_bytes(self._events)) @@ -1264,13 +1440,13 @@ def _maybe_upload(self, *, force: bool = False) -> None: self._reconcile_remote_delivery_metadata() -def validate_v2_bundle( +def validate_staging_bundle( local_dir: Path | str, run_id: str, *, content_policy: StagingContentPolicy | None = None, ) -> dict[str, Any]: - """Validate a complete locally stored version 2 bundle.""" + """Validate a complete locally stored bundle (version 2 or 3, not mixed).""" run_id = _safe_identifier(run_id, label="run_id") prefix = DEFAULT_STAGING_PREFIX @@ -1283,7 +1459,7 @@ def validate_v2_bundle( } for label, (path, schema_name) in required.items(): try: - payload = validate_v2_document(json.loads(path.read_text())) + payload = validate_staging_document(json.loads(path.read_text())) except OSError as exc: raise StagingContractError( f"Missing required staging file: {path}." @@ -1294,7 +1470,7 @@ def validate_v2_bundle( event_path = run_dir / "events.ndjson" try: events = [ - validate_v2_document(json.loads(line)) + validate_staging_document(json.loads(line)) for line in event_path.read_text().splitlines() if line ] @@ -1324,7 +1500,16 @@ def validate_v2_bundle( raise StagingContractError( f"Run manifest {field} path does not identify the validated run." ) - for field in (*_RUN_REQUIRED, "sample", "delivery", "failure"): + version = manifest["schema_version"] + if any( + document["schema_version"] != version + for document in (documents["progress"], *events) + ): + raise StagingContractError("A staging run cannot mix contract versions.") + compared = (*_RUN_REQUIRED, "sample", "delivery", "failure") + if version >= 3: + compared = (*compared, "block") + for field in compared: if documents["run_manifest"][field] != documents["progress"][field]: raise StagingContractError(f"Bundle disagrees on {field}.") calibration_path = manifest["paths"]["calibration_progress"] @@ -1338,13 +1523,15 @@ def validate_v2_bundle( "Calibration progress path does not identify the validated run." ) try: - calibration = validate_v2_document( + calibration = validate_staging_document( json.loads((root / calibration_path).read_text()) ) except OSError as exc: raise StagingContractError( f"Missing calibration progress file: {root / calibration_path}." ) from exc + if calibration["schema_version"] != version: + raise StagingContractError("A staging run cannot mix contract versions.") if calibration["schema_name"] != CALIBRATION_PROGRESS_SCHEMA: raise StagingContractError( "Calibration progress has the wrong schema identity." @@ -1390,6 +1577,10 @@ def validate_v2_bundle( return documents +#: The version 2 name, kept for callers that predate version 3. +validate_v2_bundle = validate_staging_bundle + + def _optional_int(value: Any) -> int | None: if value is None: return None diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py index 761f49617..c9fb5aaf7 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py @@ -44,6 +44,7 @@ from microcosm.build.telemetry_protocol import ( ACTION_CLOSE, ACTION_EVENT, + BUILD_BLOCKED_MESSAGE, BUILD_COMPLETED_MESSAGE, BUILD_STARTED_MESSAGE, CALIBRATION_EVENT_KIND, @@ -57,10 +58,12 @@ LOCAL_PING_MESSAGE, MAX_TELEMETRY_MESSAGE_CHARS, SEQUENTIAL_STATUS_MAP, + STAGE_BLOCKED, STAGE_CALIBRATING, STAGE_COMPLETE, STAGE_CREATED, STAGE_FAILED, + STATUS_BLOCKED, STATUS_COMPLETED, STATUS_FAILED, STATUS_PROGRESS, @@ -370,6 +373,7 @@ def fail( *, failed_during: str | None = None, failure_class: str = "build_failure", + error_code: str | None = None, ) -> None: failed_stage = failed_during or self._transition_stage message = str(error)[:MAX_TELEMETRY_MESSAGE_CHARS] @@ -378,6 +382,8 @@ def fail( "failure_class": failure_class, "failed_during": failed_stage, } + if error_code is not None: + details["error_code"] = error_code self._close_transition_stage(status=STATUS_FAILED, message=message, **details) self.emit( event_type=EVENT_TYPE_RUN, @@ -388,6 +394,44 @@ def fail( ) self.close() + def block( + self, + *, + phase: str, + blocking_gate_ids: list[str] | tuple[str, ...], + blocking_failure_count: int | None = None, + gate_statuses: Mapping[str, str] | None = None, + ) -> None: + """End the run as ``blocked``: the gates refused its candidate. + + A dedicated run event rather than a stage transition, whose status map + would read an unknown ``blocked`` as progress. The open stage itself + finished; it closes as completed.""" + + gate_ids = [str(gate_id) for gate_id in blocking_gate_ids] + details: dict[str, Any] = { + "phase": phase, + "blocking_failure_count": ( + len(gate_ids) + if blocking_failure_count is None + else blocking_failure_count + ), + "blocking_gate_ids": gate_ids, + } + if gate_statuses is not None: + details["gate_statuses"] = { + str(gate_id): str(status) for gate_id, status in gate_statuses.items() + } + self._close_transition_stage() + self.emit( + event_type=EVENT_TYPE_RUN, + stage_id=STAGE_BLOCKED, + status=STATUS_BLOCKED, + message=BUILD_BLOCKED_MESSAGE.format(phase=phase), + details=details, + ) + self.close() + def complete(self) -> None: self._close_transition_stage() self.emit( diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/collector.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/collector.py index 795c61a17..0d7e4d294 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/collector.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/collector.py @@ -27,6 +27,7 @@ LOCAL_ONLY_MISSING_CREDENTIAL, LOCAL_ONLY_REJECTED_COLLECTOR_AUTHORIZATION, LOCAL_ONLY_REJECTED_CREDENTIAL, + LOCAL_ONLY_REJECTED_EVENTS, LOCAL_ONLY_REJECTED_REGISTRATION, LOOPBACK_HOSTS, MAX_HTTP_RESPONSE_BYTES, @@ -35,6 +36,7 @@ NO_CREDENTIAL_MESSAGE, PRODUCTION_COLLECTOR_URL, REJECTED_CREDENTIAL_MESSAGE, + REJECTED_EVENTS_MESSAGE, RUN_EVENTS_PATH_TEMPLATE, RUN_REGISTRATION_PATH, TOKEN_EXCHANGE_PATH, @@ -123,6 +125,18 @@ def _huggingface_token() -> str | None: ) +#: Client errors worth retrying: a timeout and a rate limit pass with time. +#: Every other 4xx is the collector's settled answer to the same request, so +#: retrying it only wedges the queue (a payload shape it does not accept, say). +_RETRYABLE_CLIENT_ERRORS = frozenset( + {HTTPStatus.REQUEST_TIMEOUT, HTTPStatus.TOO_MANY_REQUESTS} +) + + +def _permanent_client_error(status: int) -> bool: + return 400 <= int(status) < 500 and status not in _RETRYABLE_CLIENT_ERRORS + + def _registration_key(registration: Mapping[str, Any]) -> str: return f"{registration['run_id']}:{registration['producer_id']}" @@ -187,6 +201,12 @@ def flush_once(self) -> bool: registration, LOCAL_ONLY_REJECTED_COLLECTOR_AUTHORIZATION, ) + elif _permanent_client_error(status): + self._make_local_only( + registration, + LOCAL_ONLY_REJECTED_EVENTS, + message=REJECTED_EVENTS_MESSAGE.format(status=int(status)), + ) else: self._defer_retry() return made_progress @@ -244,7 +264,9 @@ def _register(self, registration: Mapping[str, Any], token: str) -> bool: return True if status == HTTPStatus.UNAUTHORIZED: self._session_token = None - elif status in {HTTPStatus.FORBIDDEN, HTTPStatus.CONFLICT}: + elif status in {HTTPStatus.FORBIDDEN, HTTPStatus.CONFLICT} or ( + _permanent_client_error(status) + ): self._make_local_only(registration, LOCAL_ONLY_REJECTED_REGISTRATION) else: self._defer_retry() @@ -254,20 +276,22 @@ def _make_local_only( self, registration: Mapping[str, Any], reason: str, + *, + message: str = REJECTED_CREDENTIAL_MESSAGE, ) -> None: run_id = str(registration["run_id"]) producer_id = str(registration["producer_id"]) registration_key = _registration_key(registration) self.spool.make_local_only(run_id, producer_id, reason) if reason != LOCAL_ONLY_MISSING_CREDENTIAL: - self._warn_denied(registration_key) + self._warn_denied(registration_key, message) def _defer_retry(self) -> None: self._next_attempt_at = time.monotonic() + self._retry_seconds self._retry_seconds = min(MAX_RETRY_SECONDS, self._retry_seconds * 2) - def _warn_denied(self, registration_key: str) -> None: + def _warn_denied(self, registration_key: str, message: str) -> None: if registration_key in self._warned_denied: return - print(REJECTED_CREDENTIAL_MESSAGE, file=sys.stderr, flush=True) + print(message, file=sys.stderr, flush=True) self._warned_denied.add(registration_key) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py index e4a55b7d7..83163fe53 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py @@ -25,6 +25,7 @@ LOCAL_ONLY_REJECTED_CREDENTIAL: Final = "huggingface_credential_rejected" LOCAL_ONLY_REJECTED_REGISTRATION: Final = "run_registration_rejected" LOCAL_ONLY_REJECTED_COLLECTOR_AUTHORIZATION: Final = "collector_authorization_rejected" +LOCAL_ONLY_REJECTED_EVENTS: Final = "collector_rejected_events" LOCAL_ONLY_PRE_ELIGIBILITY: Final = "created_before_upload_eligibility" NO_CREDENTIAL_MESSAGE: Final = ( @@ -36,6 +37,11 @@ "credential was not accepted as a PolicyEngine organization member. The " "dataset build will continue." ) +REJECTED_EVENTS_MESSAGE: Final = ( + "Microcosm telemetry is local-only for this run: the collector rejected its " + "events (HTTP {status}), so they will not be retried. The dataset build will " + "continue." +) COLLECTOR_URL_HTTPS_ERROR: Final = "collector URL must be an HTTPS origin" COLLECTOR_URL_ORIGIN_ERROR: Final = ( "collector URL must be an origin without credentials or path data" diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py index 42fbe231d..0bb1d9c2a 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py @@ -39,6 +39,7 @@ LOCAL_MESSAGE_DELIMITER, LOCAL_SOCKET_READ_BYTES, MAX_LOCAL_MESSAGE_BYTES, + STAGE_BLOCKED, STAGE_COMPLETE, STAGE_CREATED, STAGE_FAILED, @@ -167,6 +168,7 @@ def _handle(self, message: Mapping[str, Any]) -> None: raise ValueError(EVENT_OBJECT_ERROR) stage_id = event.get("stage_id") if isinstance(stage_id, str) and stage_id not in { + STAGE_BLOCKED, STAGE_COMPLETE, STAGE_FAILED, }: diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_protocol.py b/packages/microcosm-build/src/microcosm/build/telemetry_protocol.py index 1bc3f6839..cb27eb4e5 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_protocol.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_protocol.py @@ -9,7 +9,7 @@ type TelemetryEventType = Literal[ "run", "stage", "progress", "calibration", "heartbeat" ] -type TelemetryStatus = Literal["started", "progress", "completed", "failed"] +type TelemetryStatus = Literal["started", "progress", "completed", "failed", "blocked"] TELEMETRY_SCHEMA_VERSION: Final = 1 @@ -27,15 +27,19 @@ STATUS_PROGRESS: Final = "progress" STATUS_COMPLETED: Final = "completed" STATUS_FAILED: Final = "failed" +#: The run reached a gate decision and the gates refused its candidate. +STATUS_BLOCKED: Final = "blocked" STAGE_CREATED: Final = "created" STAGE_CALIBRATING: Final = "calibrating" STAGE_COMPLETE: Final = "complete" STAGE_FAILED: Final = "failed" +STAGE_BLOCKED: Final = "blocked" CALIBRATION_EVENT_KIND: Final = "calibration_epoch" BUILD_STARTED_MESSAGE: Final = "Microcosm build started." BUILD_COMPLETED_MESSAGE: Final = "Microcosm build completed." +BUILD_BLOCKED_MESSAGE: Final = "The gates refused the candidate at {phase}." UNEXPECTED_PROCESS_EXIT_MESSAGE: Final = ( "The build process exited without reporting completion." ) diff --git a/packages/microcosm-build/src/microcosm/build/termination.py b/packages/microcosm-build/src/microcosm/build/termination.py new file mode 100644 index 000000000..79125b594 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/termination.py @@ -0,0 +1,69 @@ +"""Turn SIGTERM into an exception the build's own failure path records. + +A supervisor, a budget stop or ``kill`` ends a process with SIGTERM, which by +default kills it without running any ``finally`` or ``except``: the staging run +stays ``running`` and no Logbook row is written. :func:`raise_on_sigterm` +raises :class:`BuildTerminatedError` instead, so the run closes as a failed run +with ``TERMINATED`` and the attempt is recorded (discarded, like an operator's +Ctrl-C). + +``BuildTerminatedError`` subclasses :class:`KeyboardInterrupt`, not +:class:`Exception`: the existing interrupt arms record it, and graph kernels' +``except Exception`` handlers cannot swallow it. Python reports the +signal-style exit status only for an exact ``KeyboardInterrupt``, so a command +re-raises it as ``SystemExit(BuildTerminatedError.exit_code)`` (143) after its +close-out, and supervisors still see a termination rather than a crash. + +SIGKILL and an out-of-memory kill cannot be caught; the hosted emitter's +heartbeat and its ``unexpected_process_exit`` event cover those. +""" + +from __future__ import annotations + +import contextlib +import signal +import threading +from collections.abc import Iterator + +__all__ = ["BuildTerminatedError", "raise_on_sigterm"] + + +class BuildTerminatedError(KeyboardInterrupt): + """SIGTERM ended the build (a supervisor, a budget stop, ``kill``).""" + + exit_code = 128 + int(signal.SIGTERM) + + def __str__(self) -> str: + return "The build was terminated by SIGTERM." + + +@contextlib.contextmanager +def raise_on_sigterm() -> Iterator[None]: + """Raise :class:`BuildTerminatedError` on the first SIGTERM inside the block. + + Off the main thread (where Python cannot install handlers) it does nothing. + The handler fires once and restores the default, so a second SIGTERM kills + the process at once: a supervisor's escalation still works, even while the + graph store settles after the first. The previous handler is restored when + the block exits. + """ + + if threading.current_thread() is not threading.main_thread(): + yield + return + + def handle(signum, frame): # noqa: ARG001 - the signal handler signature + signal.signal(signal.SIGTERM, signal.SIG_DFL) + raise BuildTerminatedError() + + try: + previous = signal.signal(signal.SIGTERM, handle) + except ValueError: # pragma: no cover - not the main interpreter thread + yield + return + try: + yield + finally: + signal.signal( + signal.SIGTERM, previous if previous is not None else signal.SIG_DFL + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index 37044ffaf..573a78c7e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -317,8 +317,15 @@ def load_bound_spine_checkpoint( frame: Frame, *, gate_report_path: Path | None = None, + input_sha256: str | None = None, ) -> dict[str, object]: - """Authenticate a canonical graph checkpoint, without historical bypasses.""" + """Authenticate a canonical graph checkpoint, without historical bypasses. + + ``input_sha256`` is the measured digest of the H5 being bound. A sidecar + that records its spine's file digest (``output.sha256``) must name that + same file; older sidecars carry no such key and are bound by content + identity alone. + """ from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity path = Path(path) @@ -331,6 +338,17 @@ def load_bound_spine_checkpoint( if not isinstance(sidecar, dict): raise ValueError(f"input H5 build sidecar must be a JSON object: {path}") _assert_spine_sidecar_binds_frame(sidecar, frame) + recorded = sidecar.get("output") + if ( + input_sha256 is not None + and isinstance(recorded, Mapping) + and recorded.get("sha256") + and recorded["sha256"] != input_sha256 + ): + raise ValueError( + "Spine checkpoint sidecar records another H5 file: its output.sha256 " + f"{recorded['sha256']} is not the input's {input_sha256}." + ) identity = sidecar.get("uk_frame_content_identity") if not isinstance(identity, str) or not identity: raise ValueError("Unbound spine checkpoint: no uk_frame_content_identity.") @@ -884,20 +902,24 @@ def finalize_uk_scoped_gate_report( posture: str, scope_exclusions: Mapping[str, str], aggregate_admin_measurement: object, + resign: bool = True, ) -> None: """Graft the scoped-report trio onto a battery payload and re-sign it. Every scoped UK producer (the calibration seam, the release-cut - certification) declares its posture, the rationale for each gate it - does not run, and its admin-anchor measurement receipt, then signs the - augmented bytes. One implementation, shared, so the parts the - certification composes over cannot drift apart in shape. + certification, the dense line's local battery) declares its posture, + the rationale for each gate it does not run, and its admin-anchor + measurement receipt, then signs the augmented bytes. One + implementation, shared, so the parts the certification composes over + cannot drift apart in shape. ``resign=False`` grafts the trio onto a + report that stays unsigned (its attestation keeps ``signing_error``). """ payload["posture"] = posture payload["scope_exclusions"] = dict(scope_exclusions) payload["aggregate_admin_measurement"] = aggregate_admin_measurement - resign_uk_gate_report(payload) + if resign: + resign_uk_gate_report(payload) def resign_uk_gate_report(payload: dict[str, object]) -> None: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py index 4deb97411..c39f63901 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py @@ -41,7 +41,7 @@ import tempfile import time import uuid -from collections.abc import Callable, Mapping +from collections.abc import Callable, Collection, Mapping from dataclasses import asdict, dataclass, replace from datetime import UTC, date, datetime from pathlib import Path @@ -49,6 +49,15 @@ import numpy as np from microcosm.build.artifact_files import file_artifact, materialize_bytes +from microcosm.build.run_outcome import ( + BuildRefusedError, + Classified, + GateBlock, + RunOutcome, + classify_failure, + classify_return, +) +from microcosm.build.termination import BuildTerminatedError, raise_on_sigterm from microcosm.graph import ( ArtifactInput, ContentStore, @@ -124,6 +133,7 @@ national_result_from_manifest, national_run_config, register_uk_national_kernels, + replay_uk_dense_gate_battery, replay_uk_national_gate_battery, uk_national_graph, ) @@ -179,6 +189,7 @@ from .rowwise_staging import ( STAGED_DATASET_PHASES, STAGING_UPLOAD_INTERVAL_SECONDS, + block_staging_run_bundle, create_staging_run_bundle, emit_calibration_progress, fail_staging_run_bundle, @@ -214,14 +225,24 @@ "uk_rowwise_posture", ] +#: The graph's full terminal gate document (every declared UK gate), as the +#: evidence materialization names it. The package binds its bytes; the signed +#: local battery report takes the posture's ``*.local_gates.json`` name. +FULL_GATE_REPORT_FILENAME = "uk.full.gates.calibrated.gate_report.json" + #: CLI labels for the three UK atomic-area support systems, in SYSTEMS order. _SUPPORT_ARGUMENTS = dict( zip(ATOMIC_SUPPORT_SYSTEMS, ("ew", "scotland", "ni"), strict=True) ) -def _run_graph_with_progress(compiled, **kwargs): - """Run a graph and emit one lightweight update per completed node.""" +def _run_graph_with_progress(compiled, *, execution=None, **kwargs): + """Run a graph and emit one lightweight update per completed node. + + ``execution`` (node id to cache hit) collects how this attempt reached each + node. The first run to reach a node decides: a later run in the same + attempt reports a hit for a node this attempt itself just computed. + """ completed = 0 previous = time.monotonic() @@ -239,12 +260,107 @@ def observe(node_id, _population) -> None: ) previous = now - return run_graph( + manifest = run_graph( compiled, _population_observer=observe, _population_observer_detach=False, **kwargs, ) + if execution is not None: + for node_id, receipt in manifest.nodes.items(): + execution.setdefault(str(node_id), bool(receipt.hit)) + return manifest + + +def _attempt_request(bindings: Mapping, state: AttemptState | None, telemetry) -> dict: + """The attempt's request evidence: its bindings and the ids that name it. + + The Logbook build id and the staging run id tie the attempt directory to + its row and its dashboard run, so a later attempt on the same graph store + can name the attempts before it. + """ + return { + **bindings, + "attempt": { + "build_id": None if state is None else state.build_id, + "run_id": getattr(telemetry, "run_id", None), + }, + } + + +def _execution_counts(execution: Mapping[str, bool]) -> dict[str, int]: + reused = sum(1 for hit in execution.values() if hit) + return { + "nodes_total": len(execution), + "nodes_reused": reused, + "nodes_computed": len(execution) - reused, + } + + +#: How many earlier attempts of the same request the lineage names; the counts +#: cover every attempt on the store. +_EARLIER_ATTEMPTS_LIMIT = 20 + + +def _graph_execution( + args: argparse.Namespace, execution: Mapping[str, bool], *, bindings: Mapping +) -> dict: + """How this attempt ran its graph: the store, its counts, the attempts before. + + A resumed build reuses the stored results of earlier attempts on the same + store; recording the store, the attempt directory, how many nodes were + reused rather than computed, and the earlier attempts that ran the same + request (their directories and the Logbook and staging ids their request + evidence carries, most recent first, at most ``_EARLIER_ATTEMPTS_LIMIT``) + links the attempt to the work it built on. Attempts of other requests on + the store are counted, not listed: a long-lived store holds many. The + block sits outside the run parameters, so the candidate identity is + unchanged. + """ + attempt = Path(args.attempt_evidence) + request = json.loads(canonical_json(bindings)) + siblings = ( + [path for path in attempt.parent.iterdir() if path.is_dir() and path != attempt] + if attempt.parent.is_dir() + else [] + ) + same_request: list[dict[str, object]] = [] + for directory in sorted( + siblings, + key=lambda path: (path.stat().st_mtime_ns, path.name), + reverse=True, + ): + try: + recorded = json.loads((directory / "request.json").read_text()) + except (OSError, ValueError): + continue + if not isinstance(recorded, Mapping): + continue + if {key: value for key, value in recorded.items() if key != "attempt"} != ( + request + ): + continue + entry: dict[str, object] = {"directory": str(directory)} + identity = recorded.get("attempt") + if isinstance(identity, Mapping): + entry["build_id"] = identity.get("build_id") + entry["run_id"] = identity.get("run_id") + same_request.append(entry) + return { + "graph_store": str(attempt.parent.parent), + "attempt_directory": str(attempt), + **_execution_counts(execution), + "earlier_attempts_on_store": len(siblings), + "earlier_attempts_same_request": len(same_request), + "earlier_attempts": same_request[:_EARLIER_ATTEMPTS_LIMIT], + } + + +def _stage_graph_execution(telemetry, record: Mapping) -> None: + """Emit the attempt's reuse counts before its staging run closes.""" + execution = record.get("graph_execution") + if execution: + stage(telemetry, "graph_execution", "completed", **_execution_counts(execution)) def _target_geographies(value: str) -> tuple[str, ...] | None: @@ -690,7 +806,10 @@ def prepare_full_build( ".spine_gates.json" ) sidecar = load_bound_spine_checkpoint( - sidecar_path, frame, gate_report_path=gates_path + sidecar_path, + frame, + gate_report_path=gates_path, + input_sha256=pins["dataset"]["sha256"], ) spine = bound_spine_graph(frame) endpoint = "uk.full.spine_checkpoint" @@ -1200,8 +1319,10 @@ def _close_attempt( ) -> None: """Stage the published bundle, close the telemetry, spool the Logbook row.""" state: AttemptState = attempt["state"] + _stage_graph_execution(telemetry, record) manifest = record.get("manifest") blocked = bool(record.get("blocking_failures")) + classified = classify_return(status, record.get("gate_block")) if manifest is not None: append_phase(state, "published") args._gate_report = {"gates": record.get("gate_rows", {})} @@ -1214,7 +1335,7 @@ def _close_attempt( ) append_phase(state, STAGED_DATASET_PHASES[staged_dataset["status"]]) try: - finalize_staging_run_bundle(args, telemetry) + _close_staging(args, telemetry, classified, record) finally: manifest["staging_delivery"] = staging_delivery(telemetry) manifest["staged_dataset"] = staged_dataset @@ -1228,14 +1349,14 @@ def _close_attempt( output / f"{stem}.h5", repository_hint=REPOSITORY ) else: - finalize_staging_run_bundle(args, telemetry) + _close_staging(args, telemetry, classified, record) spool_path = record_candidate_attempt( state=state, started_at=attempt["started_at"], started_ts=attempt["started_ts"], seed=args.seed, code_pin=str(attempt["code_pin"]), - disposition="failed" if (blocked or status != 0) else "iterating", + disposition=classified.disposition, predecessor=attempt["predecessor"], spool_dir=output / "logbook-spool", rung=UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], @@ -1249,21 +1370,100 @@ def _close_attempt( f"written, artifact unreleasable: {record['blocking_failures'][:5]}", file=sys.stderr, ) + elif classified.block is not None: + print( + f"Gate battery refused the candidate at {classified.block.phase}: " + f"{list(classified.block.blocking_gate_ids)[:5]}", + file=sys.stderr, + ) def _apply_graph_gate_verdicts( - state: AttemptState, gate_rows: dict, report_path: Path + state: AttemptState, + gate_rows: dict, + report_path: Path, + *, + outcome_index: Mapping[str, int] | None = None, + scoped_report_path: Path | None = None, + scoped_gate_ids: Collection[str] = (), ) -> None: + """Record each gate's verdict with a receipt that resolves in its report. + + A battery report keys its gates (``#/gates/``); a graph gate document + lists outcomes (``#/report/outcomes/``), so its caller passes the + outcome positions. Gates a signed scoped report also carries (the dense + line's local battery) point at that report instead. + """ receipt = local_artifact_reference(report_path, repository_hint=REPOSITORY) + scoped_receipt = ( + None + if scoped_report_path is None + else local_artifact_reference(scoped_report_path, repository_hint=REPOSITORY) + ) + + def reference(gate_id: str) -> str: + if scoped_receipt is not None and gate_id in scoped_gate_ids: + return f"{scoped_receipt}#/gates/{gate_id}" + if outcome_index is not None and gate_id in outcome_index: + return f"{receipt}#/report/outcomes/{outcome_index[gate_id]}" + return f"{receipt}#/gates/{gate_id}" + state.gate_verdicts = { str(gate_id): { "verdict": str(payload["status"]), - "receipt": f"{receipt}#/gates/{gate_id}", + "receipt": reference(str(gate_id)), } for gate_id, payload in gate_rows.items() } +#: Why a filtered dense build ships no local gate report. +LOCAL_GATE_REPORT_ABSENCE = ( + "The target filter selected no local targets, so the local fit claim the " + "signed local gate report attests does not apply; no report is written." +) + + +def _local_gate_report_state(report: Mapping | None) -> str: + """How the build left its local gate report: signed, unsigned, or absent.""" + if report is None: + return "not_written" + attestation = report.get("attestation") or {} + return "unsigned" if "signing_error" in attestation else "signed" + + +def _graph_gate_rows(document: Mapping) -> tuple[dict, dict[str, int]]: + """A graph gate document's outcomes as rows by gate id, with their positions.""" + outcomes = document["report"]["outcomes"] + rows = { + str(outcome["id"]): {k: v for k, v in outcome.items() if k != "id"} + for outcome in outcomes + } + return rows, {str(outcome["id"]): index for index, outcome in enumerate(outcomes)} + + +def _close_staging( + args: argparse.Namespace, telemetry, classified: Classified, record: dict +) -> None: + """Close both telemetry destinations the way the attempt ended.""" + if classified.outcome is RunOutcome.BLOCKED: + block_staging_run_bundle( + args, + telemetry, + classified.block, + gate_statuses=record.get("gate_statuses"), + ) + elif classified.outcome is RunOutcome.COMPLETED: + finalize_staging_run_bundle(args, telemetry) + else: + fail_staging_run_bundle( + telemetry, + BuildRefusedError( + "The build returned a non-zero status without a recorded gate block." + ), + ) + + def _materialize_size_checkpoint( manifest, store, *, args: argparse.Namespace, pins: dict, source_year: int ) -> dict | None: @@ -1352,6 +1552,7 @@ def _execute_full_build( posture = posture_of(args) state: AttemptState | None = None if attempt is None else attempt["state"] record = {} if record is None else record + execution = record.setdefault("graph_execution", {}) stem, gate_report_name = _release_stem(posture) published_root = Path(getattr(args, "published_out", args.out)) args.out.mkdir(parents=True, exist_ok=True) @@ -1367,7 +1568,8 @@ def _execute_full_build( canonical_json(full.operation_inventory()), args.out / "operations.json" ) materialize_bytes( - canonical_json(prepared.bindings), args.attempt_evidence / "request.json" + canonical_json(_attempt_request(prepared.bindings, state, telemetry)), + args.attempt_evidence / "request.json", ) # Preserve source evidence before downstream transforms can raise. Each is # an ancestor-closed checkpoint of this graph, with the same node keys/RNG. @@ -1386,6 +1588,7 @@ def _execute_full_build( store=store, kernels=kernels, resume=resume, + execution=execution, ) _persist_checkpoint( checkpoint, store, args, endpoint, graph=_through(graph, endpoint) @@ -1400,6 +1603,7 @@ def _execute_full_build( store=store, kernels=kernels, resume=resume, + execution=execution, ) _persist_checkpoint(preflight, store, args, "preflight", graph=preflight_graph) _materialize_evidence(preflight, store, args.out) @@ -1416,10 +1620,33 @@ def _execute_full_build( ) if state is not None: append_phase(state, "targets_bound") - _, admission = decode_full_gate_report( - _payload(preflight, store, "uk.full.gates.preflight", "gate_report") + preflight_bytes = _payload( + preflight, store, "uk.full.gates.preflight", "gate_report" ) + _, admission = decode_full_gate_report(preflight_bytes) if not admission["artifact_permitted"]: + # Refused before solving: a gate block at phase ``preflight``, recorded + # the way the terminal battery's is, so no record reads it as a pass. + preflight_rows, preflight_index = _graph_gate_rows(json.loads(preflight_bytes)) + blocking = list(admission["enforced_blocking"]) + record["gate_rows"] = preflight_rows + record["gate_statuses"] = gate_statuses({"gates": preflight_rows}) + record["gate_block"] = GateBlock.of("preflight", blocking) + if state is not None: + _apply_graph_gate_verdicts( + state, + preflight_rows, + published_root / "uk.full.gates.preflight.gate_report.json", + outcome_index=preflight_index, + ) + append_phase(state, "candidate_blocked_at_preflight") + stage( + telemetry, + "preflight_gates", + "completed", + gate_statuses=record["gate_statuses"], + blocking_failure_count=len(blocking), + ) return 1 stage( telemetry, @@ -1436,6 +1663,7 @@ def _execute_full_build( store=store, kernels=kernels, resume="require" if args.resume == "require" else "auto", + execution=execution, ) _persist_checkpoint( manifest, @@ -1452,9 +1680,10 @@ def _execute_full_build( gate_report_bytes = _payload( manifest, store, "uk.full.gates.calibrated", "gate_report" ) + full_gate_report_path = args.out / FULL_GATE_REPORT_FILENAME gate_report_path = args.out / gate_report_name terminal_files["full_gates"] = { - **materialize_bytes(gate_report_bytes, gate_report_path), + **materialize_bytes(gate_report_bytes, full_gate_report_path), "graph_artifact_key": manifest.nodes[ "uk.full.gates.calibrated" ].opaque_artifacts["gate_report"], @@ -1488,14 +1717,36 @@ def _execute_full_build( append_phase(state, "size_selection_checkpointed") _, enforcement = decode_full_gate_report(gate_report_bytes) gate_document = json.loads(gate_report_bytes) - gate_rows = { - str(outcome["id"]): {k: v for k, v in outcome.items() if k != "id"} - for outcome in gate_document["report"]["outcomes"] - } + # The signed local battery report, written before any refusal so a + # blocked candidate leaves it too; the graph's enforcement still decides + # the exit status. A filtered build that drops the local fit claim writes + # none. + local_report = replay_uk_dense_gate_battery( + gate_document, + report_path=gate_report_path, + release_id=None if state is None else state.build_id, + release_candidate=bool(args.release_candidate), + posture=posture, + ) + gate_rows, gate_index = _graph_gate_rows(gate_document) record["gate_rows"] = gate_rows record["blocking_failures"] = list(enforcement["enforced_blocking"]) + record["gate_statuses"] = gate_statuses({"gates": gate_rows}) + if enforcement["enforced_blocking"]: + record["gate_block"] = GateBlock.of( + "terminal", enforcement["enforced_blocking"] + ) if state is not None: - _apply_graph_gate_verdicts(state, gate_rows, published_root / gate_report_name) + _apply_graph_gate_verdicts( + state, + gate_rows, + published_root / FULL_GATE_REPORT_FILENAME, + outcome_index=gate_index, + scoped_report_path=None + if local_report is None + else published_root / gate_report_name, + scoped_gate_ids=posture.gate_scope, + ) append_phase( state, "candidate_blocked" @@ -1506,9 +1757,10 @@ def _execute_full_build( telemetry, "gate_battery", "completed", - gate_statuses=gate_statuses({"gates": gate_rows}), + gate_statuses=record["gate_statuses"], blocking_failure_count=len(enforcement["enforced_blocking"]), diagnostic_failure_count=len(enforcement["diagnostic_failures"]), + local_gate_report=_local_gate_report_state(local_report), ) if not enforcement["artifact_permitted"]: return 1 @@ -1519,6 +1771,7 @@ def _execute_full_build( store=store, kernels=kernels, resume="require" if args.resume == "require" else "auto", + execution=execution, ) _persist_checkpoint(manifest, store, args, "export", graph=graph) descriptor = json.loads( @@ -1567,7 +1820,7 @@ def _execute_full_build( ), ), evidence_files={ - "full_gates": gate_report_name, + "full_gates": FULL_GATE_REPORT_FILENAME, "diagnostics": terminal_files["calibration_diagnostics"]["filename"], "holdout": terminal_files["holdout"]["filename"], "target_diagnostics": terminal_files["target_diagnostics"]["filename"], @@ -1576,7 +1829,7 @@ def _execute_full_build( }, ) evidence_sources = { - "exported_evidence_full_gates": gate_report_path, + "exported_evidence_full_gates": full_gate_report_path, "exported_evidence_diagnostics": args.out / terminal_files["calibration_diagnostics"]["filename"], "exported_evidence_holdout": args.out / terminal_files["holdout"]["filename"], @@ -1605,6 +1858,7 @@ def _execute_full_build( store=store, kernels=kernels, resume="auto", + execution=execution, ) final.save(args.out / "build.graph.json") _persist_checkpoint(final, store, args, "final", graph=graph) @@ -1643,7 +1897,10 @@ def output_record(path: Path) -> dict: "calibration_diagnostics": output_record( args.out / terminal_files["calibration_diagnostics"]["filename"] ), - "local_gate_report": output_record(gate_report_path), + "local_gate_report": None + if local_report is None + else output_record(gate_report_path), + "full_gate_report": output_record(full_gate_report_path), "solve_diagnostics": output_record( args.out / terminal_files["target_diagnostics"]["filename"] ), @@ -1676,6 +1933,12 @@ def output_record(path: Path) -> dict: code={"git_commit": git_commit(), "git_dirty": git_dirty()}, runtime=runtime_provenance(), created_at=datetime.now(UTC).isoformat(), + local_gate_report_absence=None + if local_report is not None + else LOCAL_GATE_REPORT_ABSENCE, + ) + rowwise_manifest["execution"] = _graph_execution( + args, execution, bindings=prepared.bindings ) materialize_bytes( json_text(rowwise_manifest).encode(), args.out / MANIFEST_FILENAME @@ -1702,7 +1965,11 @@ def output_record(path: Path) -> dict: telemetry, "output_bundle", "completed", - output_bytes={key: int(entry["bytes"]) for key, entry in outputs.items()}, + output_bytes={ + key: int(entry["bytes"]) + for key, entry in outputs.items() + if entry is not None + }, ) return ( 0 if package["readback_passed"] and not enforcement["enforced_blocking"] else 1 @@ -1775,7 +2042,12 @@ def prepare_national_build( frame, _ = load_uk_national_frame(input_h5) sidecar_path = args.input_sidecar or input_h5.with_suffix(".build.json") gates_path = args.input_spine_gates or input_h5.with_suffix(".spine_gates.json") - load_bound_spine_checkpoint(sidecar_path, frame, gate_report_path=gates_path) + load_bound_spine_checkpoint( + sidecar_path, + frame, + gate_report_path=gates_path, + input_sha256=pins["dataset"]["sha256"], + ) spine = bound_spine_graph(frame) _rules_engine() # the uk extra must be installed; the identity is provenance engine_identity = hashlib.sha256( @@ -1971,6 +2243,7 @@ def _execute_national_build( posture = posture_of(args) state: AttemptState | None = None if attempt is None else attempt["state"] record = {} if record is None else record + execution = record.setdefault("graph_execution", {}) vintage = args._frs_vintage paths = output_paths(args.out, posture=posture, vintage=vintage) published_root = Path(getattr(args, "published_out", args.out)) @@ -1985,7 +2258,8 @@ def _execute_national_build( canonical_json(national.operation_inventory()), args.out / "operations.json" ) materialize_bytes( - canonical_json(prepared.bindings), args.attempt_evidence / "request.json" + canonical_json(_attempt_request(prepared.bindings, state, telemetry)), + args.attempt_evidence / "request.json", ) build_id = "dry-run" if state is None else state.build_id resume = args.resume @@ -1997,6 +2271,7 @@ def _execute_national_build( store=store, kernels=kernels, resume=resume, + execution=execution, ) _persist_checkpoint( checkpoint, @@ -2024,6 +2299,7 @@ def _execute_national_build( store=store, kernels=kernels, resume=resume, + execution=execution, ) _persist_checkpoint( targets, store, args, "targets", graph=_through(graph, NATIONAL_TARGETS_NODE) @@ -2080,6 +2356,7 @@ def _execute_national_build( store=store, kernels=kernels, resume=resume, + execution=execution, ) _persist_checkpoint( manifest, @@ -2189,6 +2466,17 @@ def _execute_national_build( written = json.loads(gate_report_path.read_text(encoding="utf-8")) gate_rows = written.get("gates", {}) record["gate_report"] = written + record["gate_statuses"] = gate_statuses({"gates": gate_rows}) + record["gate_block"] = GateBlock.of( + blocked.phase, + blocked.blocking_gate_ids + or [ + gate_id + for gate_id, row in gate_rows.items() + if row.get("status") not in {"passed", "not_applicable"} + ], + blocking_failure_count=len(blocked.failures), + ) if state is not None: _apply_graph_gate_verdicts( state, gate_rows, published_root / gate_report_path.name @@ -2200,6 +2488,7 @@ def _execute_national_build( "release_check_evaluation", "completed", check_count=len(gate_rows), + gate_statuses=record["gate_statuses"], blocking_failure_count=len(blocked.failures), ) print( @@ -2241,6 +2530,7 @@ def _execute_national_build( store=store, kernels=kernels, resume="auto", + execution=execution, ) _persist_checkpoint(final, store, args, "final", graph=continued) _save_graph_presentation(continued, args, scope="national") @@ -2323,6 +2613,9 @@ def _execute_national_build( reported_paths=published, graph={"artifacts": graph_keys, "readback": readback}, ) + manifest_payload["execution"] = _graph_execution( + args, execution, bindings=prepared.bindings + ) materialize_bytes(json_text(manifest_payload).encode(), paths["manifest"]) record["manifest"] = manifest_payload record["build_record"] = build_record @@ -2371,8 +2664,9 @@ def _close_national_attempt( posture = posture_of(args) state: AttemptState = attempt["state"] + _stage_graph_execution(telemetry, record) manifest = record.get("manifest") - blocked = bool(record.get("blocking_failures")) + classified = classify_return(status, record.get("gate_block")) paths = output_paths(output, posture=posture, vintage=args._frs_vintage) if manifest is not None: append_phase(state, "published") @@ -2395,7 +2689,7 @@ def _close_national_attempt( out_dir=output, ) try: - finalize_staging_run_bundle(args, telemetry) + _close_staging(args, telemetry, classified, record) finally: delivery = staging_delivery(telemetry) build_record = json.loads(paths["build_record"].read_text()) @@ -2429,14 +2723,14 @@ def _close_national_attempt( }, ) else: - finalize_staging_run_bundle(args, telemetry) + _close_staging(args, telemetry, classified, record) spool_path = record_candidate_attempt( state=state, started_at=attempt["started_at"], started_ts=attempt["started_ts"], seed=args.seed, code_pin=str(attempt["code_pin"]), - disposition="failed" if (blocked or status != 0) else "iterating", + disposition=classified.disposition, predecessor=attempt["predecessor"], spool_dir=output / "logbook-spool", rung="f100", @@ -2463,17 +2757,36 @@ def _run_with_telemetry( try: status = operation() except BaseException as error: + # The attempt's own close-out has normally closed the emitter with its + # classification already; an error raised before the attempt took over + # (its preflight digest, say) is classified the same way here. if emitter.available: - emitter.fail(error) + classified = classify_failure(error) + if classified.block is not None: + emitter.block( + phase=classified.block.phase, + blocking_gate_ids=list(classified.block.blocking_gate_ids), + blocking_failure_count=classified.block.blocking_failure_count, + gate_statuses=classified.block.gate_statuses, + ) + else: + emitter.fail( + error, + failure_class=classified.failure_class or "build_failure", + error_code=classified.error_code, + ) raise else: if emitter.available: if status == 0: emitter.complete() else: + # The same vocabulary as the attempt's own close-outs. + refused = classify_return(status, None) emitter.fail( RuntimeError(f"UK build returned exit status {status}."), - failure_class="build_failure", + failure_class=refused.failure_class, + error_code=refused.error_code, ) return status finally: @@ -2553,20 +2866,25 @@ def _national_attempt( except KeyboardInterrupt as interrupt: # An operator interrupt is a discarded attempt, not a failed one; the # seam recorded the row so and re-raised (every terminal disposition - # records a row), and the staging run closes as failed. + # records a row), and the staging run closes as failed (INTERRUPTED). _record_failure( args, interrupt, state=state, attempt=attempt, pipeline=posture.pipeline, - disposition="discarded", + disposition=classify_failure(interrupt).disposition, ) fail_staging_run_bundle(telemetry, interrupt) raise except Exception as error: _record_failure( - args, error, state=state, attempt=attempt, pipeline=posture.pipeline + args, + error, + state=state, + attempt=attempt, + pipeline=posture.pipeline, + disposition=classify_failure(error).disposition, ) fail_staging_run_bundle(telemetry, error) print(f"UK national build failed: {error}", file=sys.stderr) @@ -2642,6 +2960,15 @@ def _dry_run(args: argparse.Namespace) -> int: def main(argv: list[str] | None = None) -> int: + """Run one UK build; a SIGTERM is recorded like Ctrl-C, then exits 143.""" + with raise_on_sigterm(): + try: + return _main(argv) + except BuildTerminatedError as terminated: + raise SystemExit(terminated.exit_code) from terminated + + +def _main(argv: list[str] | None = None) -> int: args = parse_args(argv) validate_cli_args(args) posture = posture_of(args) @@ -2726,7 +3053,7 @@ def _dense_attempt( attempt=attempt, pipeline=posture.pipeline, prepared=prepared, - disposition="discarded", + disposition=classify_failure(interrupt).disposition, ) fail_staging_run_bundle(telemetry, interrupt) raise @@ -2738,6 +3065,7 @@ def _dense_attempt( attempt=attempt, pipeline=posture.pipeline, prepared=prepared, + disposition=classify_failure(error).disposition, ) fail_staging_run_bundle(telemetry, error) print(f"UK full build failed: {error}", file=sys.stderr) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py index 772048c1a..3081a6036 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py @@ -71,6 +71,8 @@ def __init__( blocking: Sequence[gate_battery.GateOutcome], ) -> None: self.report = report + self.phase = report.phase + self.blocking_gate_ids = tuple(outcome.entry.id for outcome in blocking) super().__init__( f"Stored {report.phase} spine gates block downstream execution: " + ", ".join(outcome.entry.id for outcome in blocking) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py index 6365f88b8..c19cd4a8a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py @@ -94,6 +94,7 @@ from .battery_bindings import UK_GATE_REGISTRY from .calibration_run import ( UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS, + uk_local_gate_scope_exclusions, uk_scoped_gate_manifest, ) from .cgt_projection import UK_CGT_PROJECTION_ARTIFACT_KEY @@ -121,7 +122,11 @@ ) from .national_doctrine import UKNationalSolveDoctrine, uk_national_target_loss_weights from .national_frame import load_uk_national_frame, uk_release_export_frame -from .rowwise_posture import UK_ROWWISE_NATIONAL_POSTURE, UKRowwisePosture +from .rowwise_posture import ( + UK_ROWWISE_DENSE_POSTURE, + UK_ROWWISE_NATIONAL_POSTURE, + UKRowwisePosture, +) NATIONAL_TARGET_TYPE = ArtifactType("microcosm.uk.national-target-registry", 1) NATIONAL_GATE_REPORT_TYPE = ArtifactType("microcosm.uk.national-gate-report", 1) @@ -1173,6 +1178,106 @@ def replay_uk_national_gate_battery( return payload +#: The release id of a dense gate report no Logbook attempt bound (a direct +#: ``execute_full_build`` call). It is never a build id, so the dense +#: contract's release-id binding cannot accept it, and the report stays +#: unsigned. +UK_DENSE_UNBOUND_RELEASE_ID = "unbound-uk-dense-attempt" + + +def replay_uk_dense_gate_battery( + gate_document: Mapping[str, Any], + *, + report_path: Path, + release_id: str | None, + release_candidate: bool, + posture: UKRowwisePosture = UK_ROWWISE_DENSE_POSTURE, +) -> dict[str, object] | None: + """Write the local gate report the dense release contract verifies. + + The graph evaluated every UK gate once (``uk.full.gates.calibrated``); + the dense line ships the six-gate local battery, signed and bound to its + Logbook attempt. The stored terminal outcomes are projected onto the + local scope in the scoped manifest's order — nothing is re-evaluated: + both manifests hold the same frozen declarations, and the projection + checks it — then replayed through the battery's write boundary under + ``MARKS_ARTIFACT`` (the graph's own enforcement decides the build's exit + status, so the replay never raises a block), and the scoped-report trio + is grafted as the legacy local battery grafted it. + + The report is signed only when an attempt bound the build (``release_id``) + and the UK signing key is present; otherwise it is written unsigned, its + attestation carrying ``signing_error``, and is never shippable. + + A target filter that selects no local targets drops the local fit claim: + the graph excluded the local fit entries, and recording them as + ``not_applicable`` would read as shippable, so no report is written and + ``None`` is returned. + """ + + from microcosm.build.gate_battery import ( + BlockingMode, + GateBatteryRun, + GatePhaseReport, + ) + + from .full_gates import uk_full_gate_scope_receipt + from .graph_terminal import decode_full_gate_report + + report, _ = decode_full_gate_report(gate_document) + if report.phase != "terminal": + raise ValueError("The dense gate report replays the terminal phase only.") + if bool(gate_document["release_candidate"]) != bool(release_candidate): + raise ValueError("The graph evaluated the gates under another release posture.") + scope = uk_full_gate_scope_receipt(gate_document["selection_receipt"]) + if not scope["local_fit_claim"]: + return None + gates = uk_scoped_gate_manifest( + tuple(posture.gate_scope), + phases=("terminal",), + policy_suffix=posture.gate_policy_suffix, + ) + by_id = {outcome.entry.id: outcome for outcome in report.outcomes} + missing = [entry.id for entry in gates.gates if entry.id not in by_id] + if missing: + raise ValueError( + f"The graph's terminal gate report has no outcome for local gate(s) " + f"{missing}." + ) + projected = tuple(by_id[entry.id] for entry in gates.gates) + if tuple(outcome.entry for outcome in projected) != gates.gates: + raise ValueError( + "The graph evaluated a local gate under a different declaration." + ) + battery = GateBatteryRun( + gates, + release_id=release_id or UK_DENSE_UNBOUND_RELEASE_ID, + report_path=report_path, + release_candidate=release_candidate, + registry=UK_GATE_REGISTRY, + ) + battery.record_phase(GatePhaseReport("terminal", projected)) + battery.enforce("terminal", mode=BlockingMode.MARKS_ARTIFACT) + payload = battery.report_payload() + attestation = payload["attestation"] + if release_id is None and "signing_error" not in attestation: + attestation["signature"] = None + attestation["signing_key_sha256"] = None + attestation["signing_error"] = ( + "No Logbook attempt bound this build; the report is not signed." + ) + payload["shippable"] = False + calibration_run.finalize_uk_scoped_gate_report( + payload, + posture="local_candidate", + scope_exclusions=uk_local_gate_scope_exclusions(), + aggregate_admin_measurement=None, + resign="signing_error" not in attestation, + ) + calibration_run._write_json(report_path, payload) + return payload + + def national_run_config( *, posture: UKRowwisePosture, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py index dcead4a3b..08f5da494 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py @@ -1328,13 +1328,14 @@ def rowwise_candidate_manifest_from_graph( pins: Mapping[str, Mapping[str, object]], terminal_files: Mapping[str, Mapping[str, object]], frame: Frame, - outputs: Mapping[str, Mapping[str, object]], + outputs: Mapping[str, Mapping[str, object] | None], source_year: int, inputs: Mapping[str, Mapping[str, object]], ladder_provenance: Mapping[str, object], code: Mapping[str, object], runtime: Mapping[str, str], created_at: str, + local_gate_report_absence: str | None = None, ) -> dict: """Project the schema-4 rowwise candidate manifest from stored artifacts. @@ -1568,7 +1569,17 @@ def rowwise_candidate_manifest_from_graph( "fraction": float(args.sample_fraction), "unreachable_check": "completed", }, - "outputs": {key: dict(value) for key, value in outputs.items()}, + "outputs": { + key: None if value is None else dict(value) + for key, value in outputs.items() + }, + # A filtered build that drops the local fit claim writes no local + # gate report (``outputs.local_gate_report`` is null); this says why. + **( + {} + if local_gate_report_absence is None + else {"local_gate_report_absence": local_gate_report_absence} + ), "geography": { "constituencies_assigned": int( support.loc[ diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py index b57cbd489..baa49c415 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py @@ -28,6 +28,7 @@ import numpy as np +from microcosm.build.gate_battery import _signing_key from microcosm.build.logbook import canonical_json_bytes from microcosm.build.logbook_adoption import ( AttemptState, @@ -566,11 +567,26 @@ def validate_cli_args(args: argparse.Namespace) -> None: if _geography_assignment(args) == "legacy": # The release line is the identity-keyed atomic assignment. refused.append("--geography-assignment legacy") + target_levels = getattr(args, "target_geographies", None) + if target_levels is not None and not set(target_levels) & { + "constituency", + "la", + }: + # No local targets, no local fit claim: the signed local gate + # report a release candidate ships would have nothing to attest. + refused.append("--target-geographies without constituency or la") if refused: raise ValueError( "--release-candidate refuses non-release settings: " + ", ".join(refused) ) + try: + _signing_key("uk") + except RuntimeError as error: + raise ValueError( + "--release-candidate signs its local gate report and needs the " + f"UK gate signing key: {error}" + ) from error if args.n_clones is not None and args.n_clones <= 0: raise ValueError("--n-clones must be positive.") if args.seed < 0: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py index 5fd2e8be9..5833fa391 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py @@ -22,6 +22,11 @@ from pathlib import Path from typing import Any +from microcosm.build.run_outcome import ( + UNRECORDED_GATE_BLOCK, + GateBlock, + classify_failure, +) from microcosm.build.staging_dataset import ( StagedDatasetBundle, disabled_staged_dataset, @@ -54,6 +59,7 @@ "STAGING_MAX_EPOCH_ROWS", "STAGING_UPLOAD_INTERVAL_SECONDS", "add_staging_artifact", + "block_staging_run_bundle", "create_staging_run_bundle", "emit_calibration_progress", "fail_staging_run_bundle", @@ -415,20 +421,143 @@ def gate_statuses(gate_report: Mapping[str, Any]) -> dict[str, str]: def fail_staging_run_bundle( staging_bundle: StagingRunBundleWriterV2 | None, error: BaseException ) -> None: + """Close both telemetry destinations for a build that raised. + + The error is classified once (:func:`classify_failure`): a gate refusal + anywhere in its cause chain closes the run as ``blocked``; anything else is a + ``failed`` run with its error code and failure class. + """ + + classified = classify_failure(error) + if classified.block is not None: + _close_blocked( + staging_bundle, classified.block, gate_statuses=None, read_back=False + ) + return if _ACTIVE_EMITTER is not None and _ACTIVE_EMITTER.available: - _ACTIVE_EMITTER.fail(error) + _ACTIVE_EMITTER.fail( + error, + failure_class=classified.failure_class or "error", + error_code=classified.error_code, + ) if staging_bundle is None: return if staging_bundle.status != "running": return try: - staging_bundle.fail(error) + staging_bundle.fail( + error, + error_code=classified.error_code or "BUILD_FAILED", + failure_class=classified.failure_class, + ) staging_bundle.validate_local_bundle() except Exception: pass +def block_staging_run_bundle( + args: argparse.Namespace, + staging_bundle: StagingRunBundleWriterV2 | None, + block: GateBlock, + *, + gate_statuses: Mapping[str, str] | None = None, +) -> None: + """Close both telemetry destinations for a build its gates refused.""" + + _close_blocked( + staging_bundle, + block, + gate_statuses=gate_statuses, + read_back=bool(getattr(args, "staging_read_back", False)), + ) + + +def close_run_blocked( + staging_bundle: StagingRunBundleWriterV2 | None, + emitter: LocalTelemetryEmitter | None, + block: GateBlock, + *, + gate_statuses: Mapping[str, str] | None = None, + read_back: bool = False, +) -> None: + """Close the staging run and the hosted emitter as ``blocked``, the same way. + + The gate statuses come from the caller when it holds the report, else from + the block itself (a raised refusal whose report could be read). If the + staging contract refuses the block's details, the run must still reach a + terminal state: both destinations then close ``failed`` with + :data:`~microcosm.build.run_outcome.UNRECORDED_GATE_BLOCK`, so they cannot + disagree and no run is left ``running``. + """ + + statuses = gate_statuses if gate_statuses is not None else block.gate_statuses + refusal: StagingContractError | None = None + if staging_bundle is not None and staging_bundle.status == "running": + try: + staging_bundle.block( + phase=block.phase, + blocking_gate_ids=list(block.blocking_gate_ids), + blocking_failure_count=block.blocking_failure_count, + gate_statuses=statuses, + ) + except StagingContractError as error: + refusal = error + _warn_telemetry( + "could not record the gate block; closing the staging run as failed", + error, + ) + try: + staging_bundle.fail( + error, + error_code=UNRECORDED_GATE_BLOCK.error_code, + failure_class=UNRECORDED_GATE_BLOCK.failure_class, + ) + except StagingContractError as failure: + _warn_telemetry("could not close the staging run", failure) + else: + try: + if read_back: + staging_bundle.verify_remote() + finally: + try: + staging_bundle.validate_local_bundle() + except StagingContractError as error: + _warn_telemetry("the local staging bundle does not validate", error) + if emitter is None or not emitter.available: + return + if refusal is None: + emitter.block( + phase=block.phase, + blocking_gate_ids=list(block.blocking_gate_ids), + blocking_failure_count=block.blocking_failure_count, + gate_statuses=statuses, + ) + else: + emitter.fail( + refusal, + failure_class=UNRECORDED_GATE_BLOCK.failure_class, + error_code=UNRECORDED_GATE_BLOCK.error_code, + ) + + +def _close_blocked( + staging_bundle: StagingRunBundleWriterV2 | None, + block: GateBlock, + *, + gate_statuses: Mapping[str, str] | None, + read_back: bool, +) -> None: + close_run_blocked( + staging_bundle, + _ACTIVE_EMITTER, + block, + gate_statuses=gate_statuses, + read_back=read_back, + ) + + _fail_staging_run_bundle = fail_staging_run_bundle +_block_staging_run_bundle = block_staging_run_bundle def finalize_staging_run_bundle( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py index 49a78cfe3..b5afe5934 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py @@ -15,6 +15,7 @@ from importlib import metadata from pathlib import Path +from microcosm.build.artifact_files import file_artifact from microcosm.build.country_spec import ( GatesManifest, load_country_spec, @@ -51,6 +52,7 @@ StageObserver, ) from microcosm.build.plan import StageRecord +from microcosm.build.run_outcome import classify_failure from microcosm.build.staging_cli import ( add_staging_arguments, validate_staging_arguments, @@ -63,6 +65,7 @@ LocalTelemetryEmitter, start_local_telemetry_emitter_service, ) +from microcosm.build.termination import BuildTerminatedError, raise_on_sigterm from microcosm.build.uk_runtime.age_tail import UKAgeTailStageTransform from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY from microcosm.build.uk_runtime.calibration_run import ( @@ -147,6 +150,7 @@ from microcosm.build.uk_runtime.regional_uprating import ( UKRegionalPropertyUpratingStageTransform, ) +from microcosm.build.uk_runtime.rowwise_staging import close_run_blocked from microcosm.build.uk_runtime.salary_sacrifice import UKSalarySacrificeStageTransform from microcosm.build.uk_runtime.spi_band_donors import ( UKSPIIncomeBandDonorStageTransform, @@ -909,10 +913,55 @@ def _structural_columns(frame) -> frozenset[str]: return frozenset(columns) +def _write_sha256_file(path: Path, sha256: str) -> Path: + """Write ``.sha256`` in the ``sha256sum`` format, atomically.""" + target = path.with_name(path.name + ".sha256") + temporary = target.with_name(target.name + ".tmp") + temporary.write_text(f"{sha256} {path.name}\n", encoding="utf-8") + temporary.replace(target) + return target + + def _new_build_id(timestamp: datetime) -> str: return f"uk-frs-spine-{timestamp.strftime('%Y%m%dT%H%M%SZ')}" +def _close_failed_telemetry( + staging_bundle: StagingRunBundleWriterV2 | None, + emitter: LocalTelemetryEmitter | None, + error: BaseException, + *, + rung_abort: bool = False, +) -> None: + """Close both telemetry destinations for a spine build that raised. + + A spine gate refusal closes the run as ``blocked`` at its gate phase; a rung + abort and every other error close it ``failed`` with a classified code. + """ + classified = classify_failure(error) + if classified.block is not None and not rung_abort: + close_run_blocked(staging_bundle, emitter, classified.block) + return + error_code, failure_class = ( + ("RUNG_ABORTED", "aborted") + if rung_abort + else ( + classified.error_code or "BUILD_FAILED", + classified.failure_class or "error", + ) + ) + if staging_bundle is not None and staging_bundle.status == "running": + try: + staging_bundle.fail( + error, error_code=error_code, failure_class=failure_class + ) + staging_bundle.validate_local_bundle() + except Exception: + pass + if emitter is not None and emitter.available: + emitter.fail(error, failure_class=failure_class, error_code=error_code) + + def _record_attempt( *, state: AttemptState, @@ -1773,6 +1822,15 @@ def prepare_uk_spine_execution( def main(argv: list[str] | None = None) -> int: + """Build the spine; a SIGTERM is recorded like Ctrl-C, then exits 143.""" + with raise_on_sigterm(): + try: + return _main(argv) + except BuildTerminatedError as terminated: + raise SystemExit(terminated.exit_code) from terminated + + +def _main(argv: list[str] | None = None) -> int: args = _parse_args(argv) rung = UK_SAMPLE_RUNG_TOKENS[args.sample_fraction] started_at = time.perf_counter() @@ -2002,12 +2060,19 @@ def main(argv: list[str] | None = None) -> int: output = write_uk_national_frame(frame, args.spine_h5) if args.smoke: _mark_non_release_h5(output, build_id=state.build_id) + # The output's file digest, measured once its last byte is written + # (smoke marking included). The sidecar, the stage event, a + # ``
.sha256`` file and the Logbook pipeline verdict carry it, so a + # downstream build pins exactly this file. + output_file = file_artifact(output) + _write_sha256_file(output, str(output_file["sha256"])) _record_stage( staging_bundle, emitter, "spine_h5_creation", "completed", - size_bytes=output.stat().st_size, + size_bytes=int(output_file["size_bytes"]), + sha256=output_file["sha256"], ) append_phase(state, "spine_written") if args.checkpoint_dir is not None: @@ -2087,6 +2152,7 @@ def _checkpoint_link(path: Path) -> str: "path": _checkpoint_link(graph_evidence_path), "sha256": hashlib.sha256(graph_evidence_path.read_bytes()).hexdigest(), } + sidecar["output"] = dict(output_file) stage_evidence = stored_evidence["stage_evidence"] if stage_evidence: sidecar["stage_evidence"] = stage_evidence @@ -2142,6 +2208,7 @@ def _checkpoint_link(path: Path) -> str: "receipt": local_artifact_reference( sidecar_path, repository_hint=_REPOSITORY ), + "artifact_sha256": output_file["sha256"], } } if spine_gate_path.is_file(): @@ -2166,6 +2233,35 @@ def _checkpoint_link(path: Path) -> str: ) print(f"Wrote FRS spine H5: {output}", file=sys.stderr) print(f"Wrote Logbook row: {spool_path}", file=sys.stderr) + except KeyboardInterrupt as interrupt: + # An operator's Ctrl-C or a SIGTERM (BuildTerminatedError) discards the + # attempt: the telemetry closes failed (INTERRUPTED or TERMINATED), the + # row is recorded, and the interrupt propagates. + _close_failed_telemetry(staging_bundle, emitter, interrupt) + try: + receipt_path = write_error_receipt( + error_receipt_path(args.spine_h5.parent, build_id=state.build_id), + state=state, + pipeline=_PIPELINE, + error=interrupt, + ) + apply_error_verdict( + state, + local_artifact_reference(receipt_path, repository_hint=_REPOSITORY), + ) + _record_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + code_pin=code_pin, + disposition=classify_failure(interrupt).disposition, + predecessor=predecessor, + rung=rung, + spool_dir=spool_dir, + ) + except Exception: + pass + raise except Exception as error: if spine_battery is not None: # A run the assembled gate blocked never returned a manifest for @@ -2177,17 +2273,11 @@ def _checkpoint_link(path: Path) -> str: materialize_blocked_spine_gate_report(error, battery=spine_battery) except GateBatteryBlockedError as blocked: error = blocked - if staging_bundle is not None and staging_bundle.status == "running": - try: - staging_bundle.fail(error) - staging_bundle.validate_local_bundle() - except Exception: - pass - if emitter.available: - emitter.fail(error) - if _is_sampled(args) and _exception_chain_contains( + rung_abort = _is_sampled(args) and _exception_chain_contains( error, _RUNG_NAMED_EDGE_SIGNATURE - ): + ) + _close_failed_telemetry(staging_bundle, emitter, error, rung_abort=rung_abort) + if rung_abort: rung_abort_path = args.spine_h5.with_suffix(".rung_abort.json") receipt = _rung_abort_receipt(args, error=error) atomic_write_json(rung_abort_path, receipt) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_run_outcome.py b/packages/microcosm-build/tests/engine_free/shared/test_run_outcome.py new file mode 100644 index 000000000..18d329b70 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_run_outcome.py @@ -0,0 +1,226 @@ +"""How a build's end is classified once for staging, telemetry and the Logbook.""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from microcosm.build.gate_battery import GateBatteryBlockedError +from microcosm.build.run_outcome import ( + UNRECORDED_GATE_BLOCK, + BuildRefusedError, + GateBlock, + RunOutcome, + classify_failure, + classify_return, + logbook_disposition, +) +from microcosm.graph.errors import NodeRejectedError + + +class SpineGateBlockedError(ValueError): + """A stand-in matched by name, as the country-agnostic classifier does.""" + + def __init__( + self, phase: str, blocking_gate_ids: tuple[str, ...], report=None + ) -> None: + self.phase = phase + self.blocking_gate_ids = blocking_gate_ids + self.report = report + super().__init__(f"Stored {phase} spine gates block downstream execution") + + +class BuildTerminatedError(KeyboardInterrupt): + """A stand-in for :mod:`microcosm.build.termination`'s error, matched by name.""" + + +def _wrapped(error: BaseException, wrapper: BaseException) -> BaseException: + try: + try: + raise error + except BaseException as inner: + raise wrapper from inner + except BaseException as outer: + return outer + + +def test_a_gate_battery_block_is_blocked_with_its_phase_and_gate_ids(): + error = GateBatteryBlockedError( + "terminal", + ["[gate_a] missed", "[gate_b] missed"], + Path("gates.json"), + blocking_gate_ids=("gate_a", "gate_b"), + ) + classified = classify_failure(error) + assert classified.outcome is RunOutcome.BLOCKED + assert classified.block == GateBlock("terminal", ("gate_a", "gate_b"), 2) + assert (classified.error_code, classified.failure_class) == (None, None) + assert classified.disposition == "failed" + + +def test_gate_ids_are_read_from_the_failure_lines_when_the_raiser_named_none(): + error = GateBatteryBlockedError("assembled", ["[gate_c] missed"], Path("g.json")) + assert classify_failure(error).block == GateBlock("assembled", ("gate_c",), 1) + + +def test_a_spine_block_wrapped_by_the_graph_executor_is_still_blocked(): + error = _wrapped( + SpineGateBlockedError("transferred", ("spine_gate",)), + NodeRejectedError("uk.spine.donors rejected"), + ) + classified = classify_failure(error) + assert classified.outcome is RunOutcome.BLOCKED + assert classified.block == GateBlock("transferred", ("spine_gate",), 1) + + +@pytest.mark.parametrize( + ("error", "outcome", "code", "failure_class", "disposition"), + [ + ( + KeyboardInterrupt(), + RunOutcome.INTERRUPTED, + "INTERRUPTED", + "interrupted", + "discarded", + ), + ( + _wrapped(KeyboardInterrupt(), RuntimeError("cleanup after the interrupt")), + RunOutcome.INTERRUPTED, + "INTERRUPTED", + "interrupted", + "discarded", + ), + ( + BuildTerminatedError(), + RunOutcome.TERMINATED, + "TERMINATED", + "terminated", + "discarded", + ), + (MemoryError(), RunOutcome.FAILED, "OUT_OF_MEMORY", "out_of_memory", "failed"), + ( + _wrapped(MemoryError(), RuntimeError("solve failed")), + RunOutcome.FAILED, + "OUT_OF_MEMORY", + "out_of_memory", + "failed", + ), + ( + NodeRejectedError("bad receipt"), + RunOutcome.FAILED, + "GRAPH_NODE_FAILED", + "error", + "failed", + ), + ( + BuildRefusedError("refused"), + RunOutcome.FAILED, + "BUILD_REFUSED", + "refused", + "failed", + ), + ( + ValueError("anything else"), + RunOutcome.FAILED, + "BUILD_FAILED", + "error", + "failed", + ), + ], +) +def test_raised_errors_get_distinct_codes_classes_and_dispositions( + error, outcome, code, failure_class, disposition +): + classified = classify_failure(error) + assert classified.outcome is outcome + assert (classified.error_code, classified.failure_class) == (code, failure_class) + assert classified.block is None + assert classified.disposition == disposition + + +def test_a_return_is_completed_only_when_it_passed_without_a_block(): + block = GateBlock.of("preflight", ["gate_a"]) + assert classify_return(0, None).outcome is RunOutcome.COMPLETED + assert classify_return(0, None).disposition == "iterating" + assert classify_return(1, block).outcome is RunOutcome.BLOCKED + assert classify_return(1, block).block == block + refused = classify_return(1, None) + assert (refused.outcome, refused.error_code, refused.failure_class) == ( + RunOutcome.FAILED, + "BUILD_REFUSED", + "refused", + ) + + +def test_every_outcome_has_a_logbook_disposition(): + assert {outcome: logbook_disposition(outcome) for outcome in RunOutcome} == { + RunOutcome.COMPLETED: "iterating", + RunOutcome.BLOCKED: "failed", + RunOutcome.FAILED: "failed", + RunOutcome.INTERRUPTED: "discarded", + RunOutcome.TERMINATED: "discarded", + RunOutcome.ABORTED: "discarded", + } + + +def test_a_gate_block_counts_at_least_one_failure(): + assert ( + GateBlock.of("terminal", [], blocking_failure_count=0).blocking_failure_count + == 1 + ) + + +def test_a_block_carries_the_gate_statuses_its_report_recorded(tmp_path): + """A raised refusal brings every gate's status along when its report can be + read: the battery's written report, or a spine refusal's in-memory one.""" + report = tmp_path / "gates.json" + report.write_text( + json.dumps( + {"gates": {"gate_a": {"status": "failed"}, "gate_b": {"status": "passed"}}} + ) + ) + error = GateBatteryBlockedError( + "terminal", ["[gate_a] missed"], report, blocking_gate_ids=("gate_a",) + ) + block = classify_failure(error).block + assert block.blocking_gate_ids == ("gate_a",) + assert block.gate_statuses == {"gate_a": "failed", "gate_b": "passed"} + outcomes = ( + SimpleNamespace( + entry=SimpleNamespace(id="spine_gate"), + status=SimpleNamespace(value="failed"), + ), + SimpleNamespace( + entry=SimpleNamespace(id="other"), status=SimpleNamespace(value="passed") + ), + ) + spine = SpineGateBlockedError( + "assembled", + ("spine_gate",), + report=SimpleNamespace(phase="assembled", outcomes=outcomes), + ) + assert classify_failure(spine).block.gate_statuses == { + "spine_gate": "failed", + "other": "passed", + } + + +def test_a_block_that_names_no_gate_keeps_its_count_and_no_placeholder(): + error = GateBatteryBlockedError( + "assembled", ["two failures", "with no gate id"], Path("missing.json") + ) + block = classify_failure(error).block + assert block == GateBlock("assembled", (), 2) + assert block.gate_statuses is None + + +def test_an_unrecorded_gate_block_is_a_failed_run_with_its_own_class(): + assert UNRECORDED_GATE_BLOCK.outcome is RunOutcome.FAILED + assert (UNRECORDED_GATE_BLOCK.error_code, UNRECORDED_GATE_BLOCK.failure_class) == ( + "GATE_BLOCK_UNRECORDED", + "unrecorded_gate_block", + ) + assert UNRECORDED_GATE_BLOCK.disposition == "failed" diff --git a/packages/microcosm-build/tests/engine_free/shared/test_staging_v2.py b/packages/microcosm-build/tests/engine_free/shared/test_staging_v2.py index 357c08acc..3e437416b 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_staging_v2.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_staging_v2.py @@ -24,11 +24,14 @@ EVENT_SCHEMA, PROGRESS_SCHEMA, RUN_MANIFEST_SCHEMA, + STAGING_CONTRACT_VERSION, + STAGING_DOCUMENT_VERSION, StagingContentError, StagingContractError, StagingReadBackError, StagingRunBundleWriterV2, disabled_staging_delivery, + validate_staging_bundle, validate_staging_delivery, validate_v2_bundle, validate_v2_document, @@ -226,7 +229,9 @@ def test_failure_uses_sanitized_contract_fields(tmp_path): "error_type": "RuntimeError", "message": "The build failed during input_verification.", "local_diagnostic_reference": "diagnostics/local-error.txt", + "failure_class": None, } + assert progress["block"] is None assert "licensed" not in serialized assert "secret-value" not in serialized assert "traceback" not in serialized.lower() @@ -293,7 +298,7 @@ def test_unknown_schema_version_is_incompatible(): validate_v2_document( { "schema_name": RUN_MANIFEST_SCHEMA, - "schema_version": 3, + "schema_version": 999, } ) @@ -402,7 +407,7 @@ def test_remote_read_back_requires_a_final_run_status(tmp_path): upload_interval_seconds=0, ) - with pytest.raises(StagingContractError, match="completed or failed"): + with pytest.raises(StagingContractError, match="completed, blocked or failed"): telemetry.verify_remote() assert telemetry.delivery_summary["read_back"] == "not_requested" @@ -618,7 +623,7 @@ def test_canonical_version_2_fixture_cases_validate(): validate_v2_document(cases["unknown_version"]) -@pytest.mark.parametrize("version", ["v1", "v2"]) +@pytest.mark.parametrize("version", ["v1", "v2", "v3"]) def test_contract_fixture_checksums_are_pinned(version): fixture_dir = FIXTURE_ROOT / version for line in (fixture_dir / "SHA256SUMS").read_text().splitlines(): @@ -628,7 +633,164 @@ def test_contract_fixture_checksums_are_pinned(version): ).hexdigest() == (expected) -def test_canonical_version_2_fixtures_are_reproducible(): +def test_canonical_version_3_fixture_cases_validate(): + v3 = FIXTURE_ROOT / "v3" + completed = validate_staging_bundle(v3 / "completed-spine", "uk-spine-v3-fixture") + calibration = validate_staging_bundle( + v3 / "calibration", "uk-calibration-v3-fixture" + ) + failed = validate_staging_bundle(v3 / "failed", "uk-failed-v3-fixture") + blocked = validate_staging_bundle(v3 / "blocked", "uk-blocked-v3-fixture") + + assert completed["progress"]["schema_version"] == STAGING_DOCUMENT_VERSION == 3 + assert completed["progress"]["delivery"]["contract_version"] == 2 + assert completed["progress"]["block"] is None + assert calibration["calibration_progress"]["schema_version"] == 3 + assert failed["progress"]["failure"]["failure_class"] == "error" + assert failed["events"][-1]["details"]["failure_class"] == "error" + assert blocked["progress"]["status"] == "blocked" + assert blocked["progress"]["failure"] is None + assert blocked["run_manifest"]["block"] == { + "phase": "terminal", + "blocking_failure_count": 1, + "blocking_gate_ids": ["uk_target_fit"], + } + terminal = blocked["events"][-1] + assert (terminal["stage_id"], terminal["status"]) == ("blocked", "blocked") + assert terminal["details"]["gate_statuses"]["uk_target_fit"] == "failed" + + +def test_delivery_contract_stays_version_2_while_documents_move_to_3(): + """Publishers and release assemblers pin the delivery summary's version.""" + + assert STAGING_CONTRACT_VERSION == 2 + assert disabled_staging_delivery("--no-staging")["contract_version"] == 2 + + +def test_blocked_run_closes_with_its_gate_phase(tmp_path): + telemetry = _recorder(tmp_path) + telemetry.stage("gate_battery", event_status="completed") + telemetry.block( + phase="preflight", + blocking_gate_ids=["uk_target_surface", "uk_support"], + gate_statuses={"uk_target_surface": "failed", "uk_support": "failed"}, + ) + + bundle = telemetry.validate_local_bundle() + assert bundle["progress"]["status"] == "blocked" + assert bundle["progress"]["block"]["phase"] == "preflight" + assert bundle["progress"]["block"]["blocking_failure_count"] == 2 + with pytest.raises(StagingContractError, match="status is 'blocked'"): + telemetry.complete() + + +@pytest.mark.parametrize( + "kwargs", + [ + {"phase": "terminal", "blocking_gate_ids": []}, + {"phase": "terminal", "blocking_gate_ids": [""]}, + {"phase": "../x", "blocking_gate_ids": ["uk_target_fit"]}, + {"phase": "terminal", "blocking_gate_ids": ["a"], "blocking_failure_count": 0}, + { + "phase": "terminal", + "blocking_gate_ids": ["token"], + "gate_statuses": {"token": "failed"}, + }, + ], +) +def test_block_refuses_an_empty_or_unsafe_block(tmp_path, kwargs): + telemetry = _recorder(tmp_path) + with pytest.raises(StagingContractError): + telemetry.block(**kwargs) + # Refused before any state changed: the run is still running and can + # still close another way. + assert telemetry.status == "running" + + +def test_a_refused_block_leaves_the_run_able_to_close_failed(tmp_path): + telemetry = _recorder(tmp_path) + telemetry.stage("gate_battery", event_status="completed") + with pytest.raises(StagingContractError): + telemetry.block( + phase="terminal", + blocking_gate_ids=["token"], + gate_statuses={"token": "failed"}, + ) + telemetry.fail( + RuntimeError("the block could not be recorded"), + error_code="GATE_BLOCK_UNRECORDED", + failure_class="unrecorded_gate_block", + ) + progress = telemetry.validate_local_bundle()["progress"] + assert progress["status"] == "failed" and progress["block"] is None + assert progress["failure"]["error_code"] == "GATE_BLOCK_UNRECORDED" + + +def test_a_block_may_name_no_gate_when_it_counts_its_failures(tmp_path): + telemetry = _recorder(tmp_path) + telemetry.block(phase="assembled", blocking_gate_ids=[], blocking_failure_count=3) + block = telemetry.validate_local_bundle()["progress"]["block"] + assert block == { + "phase": "assembled", + "blocking_failure_count": 3, + "blocking_gate_ids": [], + } + + +def test_failure_class_is_recorded_and_checked(tmp_path): + telemetry = _recorder(tmp_path) + with pytest.raises(StagingContractError, match="failure_class"): + telemetry.fail(RuntimeError("x"), failure_class="Not A Class") + telemetry.fail( + KeyboardInterrupt(), error_code="INTERRUPTED", failure_class="interrupted" + ) + failure = telemetry.validate_local_bundle()["progress"]["failure"] + assert (failure["error_code"], failure["failure_class"]) == ( + "INTERRUPTED", + "interrupted", + ) + + +def test_version_3_documents_keep_failure_and_block_apart(tmp_path): + telemetry = _recorder(tmp_path) + telemetry.block(phase="terminal", blocking_gate_ids=["uk_target_fit"]) + manifest = telemetry.validate_local_bundle()["run_manifest"] + + with pytest.raises(StagingContractError, match="Blocked runs require"): + validate_v2_document({**manifest, "block": None}) + failed = { + **manifest, + "status": "failed", + "failure": { + "error_code": "BUILD_FAILED", + "error_type": "RuntimeError", + "message": "The build failed during gate_battery.", + "local_diagnostic_reference": None, + "failure_class": None, + }, + } + with pytest.raises(StagingContractError, match="Non-blocked runs"): + validate_v2_document(failed) + with pytest.raises(StagingContractError, match="failure"): + validate_v2_document( + {**failed, "block": None, "failure": {**failed["failure"], "extra": 1}} + ) + + +def test_a_run_cannot_mix_contract_versions(tmp_path): + telemetry = _recorder(tmp_path) + telemetry.complete() + progress_path = telemetry.run_dir / "progress.json" + progress = json.loads(progress_path.read_text()) + progress["schema_version"] = 2 + progress.pop("block") + progress_path.write_text(json.dumps(progress)) + + with pytest.raises(StagingContractError, match="mix contract versions"): + telemetry.validate_local_bundle() + + +def test_canonical_version_3_fixtures_are_reproducible(): subprocess.run( [ sys.executable, diff --git a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py index 2c1f6f8f8..cee28f434 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py @@ -427,6 +427,45 @@ def reject(url, payload, token, *, timeout=5.0): assert capsys.readouterr().err.count("local-only for this run") == 1 +@pytest.mark.parametrize( + ("status", "retried"), [(422, False), (404, False), (429, True), (408, True)] +) +def test_a_settled_collector_rejection_goes_local_only_instead_of_retrying( + tmp_path, monkeypatch, capsys, status, retried +) -> None: + """A 4xx the collector will repeat (an event shape it does not accept, say) + must not wedge the queue: the run goes local-only with a reason and one + warning. A timeout or a rate limit still retries.""" + spool = EventSpool(tmp_path / "events.sqlite3") + registration = _registration() + spool.register(registration) + spool.append(registration, _event()) + monkeypatch.setattr(collector_module, "_huggingface_token", lambda: "hf-member") + + def fake_post(url, payload, token, *, timeout=5.0): + if url.endswith("/v1/auth/huggingface/exchange"): + return 200, {"access_token": "collector-token", "expires_in": 3600} + if url.endswith("/v1/runs"): + return 201, {"registered": True} + return status, {"detail": "rejected"} + + monkeypatch.setattr(collector_module, "_http_post", fake_post) + delivery = CollectorDelivery( + spool, development_collector_url="http://127.0.0.1:8080" + ) + assert not delivery.flush_once() + assert spool.has_pending() + if retried: + assert spool.pending_runs() == [registration] + assert "local-only" not in capsys.readouterr().err + return + assert spool.pending_runs() == [] + assert not delivery.flush_once() + err = capsys.readouterr().err + assert err.count("local-only for this run") == 1 + assert f"HTTP {status}" in err + + def test_identity_provider_outage_keeps_events_eligible_for_retry( tmp_path, monkeypatch ) -> None: diff --git a/packages/microcosm-build/tests/engine_free/shared/test_termination.py b/packages/microcosm-build/tests/engine_free/shared/test_termination.py new file mode 100644 index 000000000..52db96b69 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_termination.py @@ -0,0 +1,55 @@ +"""SIGTERM becomes an exception the build's own failure path records.""" + +from __future__ import annotations + +import os +import signal +import threading + +import pytest + +from microcosm.build.run_outcome import RunOutcome, classify_failure +from microcosm.build.termination import BuildTerminatedError, raise_on_sigterm + + +def test_sigterm_inside_the_block_raises_and_restores_the_previous_handler(): + previous = signal.getsignal(signal.SIGTERM) + with pytest.raises(BuildTerminatedError) as raised: + with raise_on_sigterm(): + os.kill(os.getpid(), signal.SIGTERM) + assert signal.getsignal(signal.SIGTERM) == previous + assert raised.value.exit_code == 143 + assert isinstance(raised.value, KeyboardInterrupt) + assert classify_failure(raised.value).outcome is RunOutcome.TERMINATED + + +def test_the_handler_fires_once_then_leaves_the_default(): + # ``os.kill`` runs the handler before it returns when the target is this + # process, so the error surfaces from the call itself. + with raise_on_sigterm(): + with pytest.raises(BuildTerminatedError): + os.kill(os.getpid(), signal.SIGTERM) + # Still inside the block, after the first signal: a second one would + # now kill the process (the supervisor's escalation). + assert signal.getsignal(signal.SIGTERM) == signal.SIG_DFL + + +def test_a_block_without_a_signal_leaves_the_handler_as_it_was(): + previous = signal.getsignal(signal.SIGTERM) + with raise_on_sigterm(): + assert signal.getsignal(signal.SIGTERM) not in (previous, signal.SIG_DFL) + assert signal.getsignal(signal.SIGTERM) == previous + + +def test_off_the_main_thread_it_installs_nothing(): + previous = signal.getsignal(signal.SIGTERM) + seen: list[object] = [] + + def worker(): + with raise_on_sigterm(): + seen.append(signal.getsignal(signal.SIGTERM)) + + thread = threading.Thread(target=worker) + thread.start() + thread.join() + assert seen == [previous] diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_calibration_run.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_calibration_run.py index cf8fc9f9d..fc9cf36b9 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_calibration_run.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_calibration_run.py @@ -72,6 +72,20 @@ def test_strict_checkpoint_accepts_explicit_declared_gate_path(tmp_path): assert provenance["spine_gate_report"]["path"] == str(moved) +def test_strict_checkpoint_binds_the_recorded_output_digest(tmp_path): + """A sidecar that records its H5's digest binds only that file; a sidecar + without the key (written before spines recorded it) still binds.""" + frame = _frame() + path, _, sidecar = _bound_checkpoint(tmp_path, frame) + calibration_run.load_bound_spine_checkpoint(path, frame, input_sha256="a" * 64) + sidecar["output"] = {"filename": "spine.h5", "sha256": "a" * 64, "size_bytes": 1} + path.write_text(json.dumps(sidecar)) + calibration_run.load_bound_spine_checkpoint(path, frame, input_sha256="a" * 64) + calibration_run.load_bound_spine_checkpoint(path, frame) + with pytest.raises(ValueError, match="records another H5 file"): + calibration_run.load_bound_spine_checkpoint(path, frame, input_sha256="b" * 64) + + def test_gate_scope_classifies_every_uk_gate(): all_ids = {entry.id for entry in load_country_spec("uk").gates.gates} assert ( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_dense_release_assembler.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_dense_release_assembler.py index b6af98901..95bfa6228 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_dense_release_assembler.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_dense_release_assembler.py @@ -865,17 +865,6 @@ def test_assembler_refuses_a_national_role_manifest( ) -def _resigned_report(release_id: str) -> dict: - report = _signed_report() - report["release_id"] = release_id - report["attestation"]["release_id"] = release_id - report["attestation"]["signature"] = None - report["attestation"]["signature"] = hmac.new( - KEY_BYTES, _canonical_json_bytes(report), hashlib.sha256 - ).hexdigest() - return report - - def test_assembler_stages_a_graph_built_dense_candidate( tmp_path: Path, monkeypatch, capsys ) -> None: @@ -883,9 +872,9 @@ def test_assembler_stages_a_graph_built_dense_candidate( ``main`` runs the synthetic graph with local-only staging, so the manifest carries the staging receipt, the staged-dataset sidecars and the Logbook - row the assembler hash-joins. The signed local battery report and the - incumbent-surface companions are supplied as the certification and - scoring steps supply them. + row the assembler hash-joins. The build signs its own local battery report, + bound to the Logbook attempt; only the incumbent-surface companions are + supplied, as the scoring step supplies them. """ pytest.importorskip("tables") @@ -915,10 +904,9 @@ def test_assembler_stages_a_graph_built_dense_candidate( assert manifest["staging_delivery"]["mode"] == "local_only" assert manifest["staged_dataset"]["status"] == "skipped" report_path = candidate / f"{STEM}.local_gates.json" - report_path.write_text(json.dumps(_resigned_report(row.build_id))) - manifest["outputs"]["local_gate_report"]["sha256"] = _sha(report_path) - manifest["outputs"]["local_gate_report"]["bytes"] = report_path.stat().st_size - manifest_path.write_text(json.dumps(manifest)) + report = json.loads(report_path.read_text()) + assert report["release_id"] == row.build_id and report["shippable"] is True + assert manifest["outputs"]["local_gate_report"]["sha256"] == _sha(report_path) diagnostics_path = Path(manifest["outputs"]["calibration_diagnostics"]["path"]) (candidate / "score_vs_incumbent.json").write_text( json.dumps( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py index 54fdf171a..b7669e815 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py @@ -3,6 +3,8 @@ import hashlib import importlib.util import json +import os +import signal import sys from datetime import UTC, datetime from importlib import metadata @@ -1987,8 +1989,8 @@ def __init__(self, run_id: str) -> None: def close(self) -> None: events.append(("close",)) - def fail(self, error: BaseException) -> None: - events.append(("failed", str(error))) + def fail(self, error: BaseException, **classification) -> None: + events.append(("failed", str(error), classification)) def start_emitter(**run): events.append(("started", run["run_id"])) @@ -2016,6 +2018,7 @@ def start_emitter(**run): assert events[0][0] == "started" assert events[1][0] == "failed" assert "--spi-tab must be an existing file" in events[1][1] + assert events[1][2] == {"failure_class": "error", "error_code": "BUILD_FAILED"} def test_driver_refuses_misnamed_spi_tab(tmp_path: Path) -> None: @@ -2387,6 +2390,22 @@ def test_driver_marks_full_fixture_smoke_outputs_non_release( assert file.attrs["populace_smoke_build_id"].startswith("uk-frs-spine-") rows = load_spool_rows(tmp_path / "logbook-spool") assert rows[0].rung == "f100" + # The written file's digest (after smoke marking) is recorded everywhere + # a downstream build or an operator would look for it. + digest = hashlib.sha256(output.read_bytes()).hexdigest() + assert sidecar["output"] == { + "filename": output.name, + "sha256": digest, + "size_bytes": output.stat().st_size, + } + assert (tmp_path / "smoke.h5.sha256").read_text() == f"{digest} smoke.h5\n" + created = [ + event + for event in bundle["events"] + if (event["stage_id"], event["status"]) == ("spine_h5_creation", "completed") + ] + assert created[0]["details"]["sha256"] == digest + assert rows[0].gate_verdicts["pipeline"]["artifact_sha256"] == digest def test_driver_records_sanitized_failed_staging_lifecycle( @@ -2440,6 +2459,75 @@ def _fail_graph(*args, **kwargs): assert str(tmp_path) not in serialized +@pytest.mark.parametrize( + ("stop", "exit_type", "error_code", "failure_class"), + [ + ("sigint", KeyboardInterrupt, "INTERRUPTED", "interrupted"), + ("sigterm", SystemExit, "TERMINATED", "terminated"), + ], +) +def test_driver_records_an_operator_stop_as_a_discarded_attempt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + stop: str, + exit_type: type[BaseException], + error_code: str, + failure_class: str, +) -> None: + """Ctrl-C and SIGTERM close the staging run failed with their own class, + record a discarded row, and leave the process with the matching exit.""" + raw_dir = tmp_path / "raw" + stage = _write_fixture(raw_dir) + output = tmp_path / "stopped.h5" + staging_dir = tmp_path / "staging" + tool = _load_tool() + monkeypatch.setattr( + tool, "load_country_spec", lambda country: _synthetic_spec(stage) + ) + monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) + _stub_policy_readers(monkeypatch) + spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) + + def _stop_graph(*args, **kwargs): + if stop == "sigterm": + os.kill(os.getpid(), signal.SIGTERM) + raise KeyboardInterrupt + + monkeypatch.setattr(tool, "run_graph", _stop_graph) + previous_handler = signal.getsignal(signal.SIGTERM) + + with pytest.raises(exit_type) as raised: + tool.main( + [ + "--frs-raw-dir", + str(raw_dir), + "--spine-h5", + str(output), + "--spi-tab", + str(spi_tab), + "--hmrc-ods", + str(hmrc_ods), + "--staging-local-only", + "--staging-dir", + str(staging_dir), + "--staging-run-id", + "stopped-staging-test", + ] + ) + + if exit_type is SystemExit: + assert raised.value.code == 143 + assert signal.getsignal(signal.SIGTERM) == previous_handler + manifest = validate_v2_bundle(staging_dir, "stopped-staging-test")["run_manifest"] + assert manifest["status"] == "failed" + assert ( + manifest["failure"]["error_code"], + manifest["failure"]["failure_class"], + ) == (error_code, failure_class) + rows = load_spool_rows(tmp_path / "logbook-spool") + assert [row.disposition for row in rows] == ["discarded"] + + def test_driver_materializes_a_blocked_assembled_gate_report_before_failing( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py index 9a38a932a..6aea249db 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py @@ -1,14 +1,18 @@ """The canonical CLI restores declared files and preserves failure/scope semantics.""" # ruff: noqa: F403, F405 +import base64 +import os +import signal from types import SimpleNamespace import test_support.microcosm_build.uk_full_build_cli as support # noqa: E402 from test_support.microcosm_build.uk_full_build_cli import * -def test_geography_assignment_arguments_are_closed(tmp_path): +def test_geography_assignment_arguments_are_closed(tmp_path, monkeypatch): """Atomic is the default; the cross-flag rules live in the validator.""" + monkeypatch.setenv(SIGNING_KEY_ENV, TEST_SIGNING_KEY) assert arguments(tmp_path).geography_assignment == "atomic" cli.validate_cli_args(arguments(tmp_path)) legacy = arguments(tmp_path, "--geography-assignment", "legacy", supports=()) @@ -370,6 +374,16 @@ def test_cli_cold_and_required_replay_recreate_dataset_and_sidecars( assert [phase["phase"] for phase in overlay["phases"]][-2:] == ["export", "final"] assert not list(out.glob("*.orrery.json")) assert (out / f"{STEM}.targets.csv").read_text().startswith("name,target_name") + cold = json.loads((out / cli.MANIFEST_FILENAME).read_text())["execution"] + assert cold["nodes_total"] > 0 and cold["nodes_reused"] == 0 + assert cold["nodes_computed"] == cold["nodes_total"] + assert cold["earlier_attempts"] == [] + # An attempt of another request on the same store is counted, not listed. + attempts = Path(cold["attempt_directory"]).parent + (attempts / "unrelated").mkdir() + (attempts / "unrelated" / "request.json").write_text( + json.dumps({"schema": "other-request", "attempt": {"build_id": "x"}}) + ) for path in out.iterdir(): if path.is_file(): path.unlink() @@ -385,6 +399,16 @@ def forbidden(*args): assert cli.execute_full_build(prepared(tmp_path), args) == 0 actual = json.loads((out / "build.json").read_text()) assert actual["content_sha256"] == expected["content_sha256"] + # The replay records that it reused the cold attempt's work and names it. + replay = json.loads((out / cli.MANIFEST_FILENAME).read_text())["execution"] + assert replay["nodes_reused"] > 0 + assert replay["nodes_total"] == replay["nodes_reused"] + replay["nodes_computed"] + assert [Path(item["directory"]) for item in replay["earlier_attempts"]] == [ + Path(cold["attempt_directory"]) + ] + assert replay["earlier_attempts_on_store"] == 2 + assert replay["earlier_attempts_same_request"] == 1 + assert replay["graph_store"] == cold["graph_store"] # The output names come from the dense posture and the FRS vintage. assert (out / f"{STEM}.h5").is_file() assert (out / f"{STEM}.holdout.json").is_file() @@ -392,10 +416,133 @@ def forbidden(*args): assert not (out / "microcosm_uk_2025.h5").exists() +def test_the_dense_gate_replay_signs_a_bound_attempt_the_contract_accepts( + tmp_path, monkeypatch +): + """The six local outcomes of the graph's terminal document become a signed + battery report bound to the attempt, which the dense contract verifies.""" + from microcosm.build.uk_runtime.graph_national import replay_uk_dense_gate_battery + from microcosm.data.contract import _check_uk_dense_gate_report + + monkeypatch.setenv(SIGNING_KEY_ENV, TEST_SIGNING_KEY) + document = json.loads(gate_payload("terminal", release_candidate=True)) + path = tmp_path / f"{STEM}.local_gates.json" + report = replay_uk_dense_gate_battery( + document, + report_path=path, + release_id="uk-local-candidate-f100-s0-test", + release_candidate=True, + ) + assert json.loads(path.read_text()) == report + assert set(report["gates"]) == set(UK_LOCAL_GATE_SCOPE) + assert report["posture"] == "local_candidate" + assert report["shippable"] is True and "signing_error" not in report["attestation"] + failures: list[str] = [] + _check_uk_dense_gate_report( + report, failures=failures, attempt_id="uk-local-candidate-f100-s0-test" + ) + assert failures == [] + # A dev-posture document cannot be replayed as a release candidate. + with pytest.raises(ValueError, match="another release posture"): + replay_uk_dense_gate_battery( + json.loads(gate_payload("terminal")), + report_path=path, + release_id="uk-local-candidate-f100-s0-test", + release_candidate=True, + ) + + +def test_the_dense_gate_replay_stays_unsigned_without_a_key_or_an_attempt( + tmp_path, monkeypatch +): + from microcosm.build.uk_runtime.graph_national import ( + UK_DENSE_UNBOUND_RELEASE_ID, + replay_uk_dense_gate_battery, + ) + from microcosm.data.contract import _check_uk_dense_gate_report + + document = json.loads(gate_payload("terminal", release_candidate=True)) + monkeypatch.setenv(SIGNING_KEY_ENV, TEST_SIGNING_KEY) + unbound = replay_uk_dense_gate_battery( + document, + report_path=tmp_path / "unbound.local_gates.json", + release_id=None, + release_candidate=True, + ) + assert unbound["release_id"] == UK_DENSE_UNBOUND_RELEASE_ID + assert unbound["attestation"]["signature"] is None + assert "No Logbook attempt" in unbound["attestation"]["signing_error"] + assert unbound["shippable"] is False + monkeypatch.delenv(SIGNING_KEY_ENV) + keyless = replay_uk_dense_gate_battery( + document, + report_path=tmp_path / "keyless.local_gates.json", + release_id="uk-local-candidate-f100-s0-test", + release_candidate=True, + ) + assert keyless["attestation"]["signature"] is None + assert SIGNING_KEY_ENV in keyless["attestation"]["signing_error"] + assert keyless["shippable"] is False and keyless["posture"] == "local_candidate" + monkeypatch.setenv(SIGNING_KEY_ENV, TEST_SIGNING_KEY) + failures: list[str] = [] + _check_uk_dense_gate_report(keyless, failures=failures) + assert any("signing error" in line for line in failures) + + +def test_a_filtered_build_writes_no_local_gate_report(tmp_path, monkeypatch): + """No local targets, no local fit claim: the replay writes nothing rather + than filling the excluded local gates in as not applicable.""" + from microcosm.build.uk_runtime.graph_national import replay_uk_dense_gate_battery + + national_only = { + **support.SELECTION, + "selector": {"geography_levels": ["country"], "explicit": False}, + "included": [{"name": "count", "period": 2025, "geography_level": "country"}], + } + monkeypatch.setattr(support, "SELECTION", national_only) + path = tmp_path / f"{STEM}.local_gates.json" + assert ( + replay_uk_dense_gate_battery( + json.loads(gate_payload("terminal")), + report_path=path, + release_id="uk-local-candidate-f100-s0-test", + release_candidate=False, + ) + is None + ) + assert not path.exists() + + +def test_a_release_candidate_needs_the_signing_key_and_local_targets( + tmp_path, monkeypatch +): + monkeypatch.delenv(SIGNING_KEY_ENV, raising=False) + release = arguments(tmp_path, "--release-candidate") + with pytest.raises(ValueError, match="needs the UK gate signing key"): + cli.validate_cli_args(release) + monkeypatch.setenv(SIGNING_KEY_ENV, base64.b64encode(b"\x05" * 16).decode()) + with pytest.raises(ValueError, match="exactly 32 bytes"): + cli.validate_cli_args(release) + monkeypatch.setenv(SIGNING_KEY_ENV, TEST_SIGNING_KEY) + cli.validate_cli_args(release) + with pytest.raises(ValueError, match="--target-geographies without"): + cli.validate_cli_args( + arguments( + tmp_path, + "--release-candidate", + "--target-geographies", + "country,region", + ) + ) + cli.validate_cli_args( + arguments(tmp_path, "--release-candidate", "--target-geographies", "la") + ) + + def test_dense_run_projects_the_rowwise_candidate_manifest(tmp_path): pytest.importorskip("tables") args = arguments(tmp_path, "--release-candidate") - assert cli.execute_full_build(prepared(tmp_path), args) == 0 + assert cli.execute_full_build(prepared(tmp_path, release_candidate=True), args) == 0 out = args.out manifest = json.loads((out / "rowwise_candidate_manifest.json").read_text()) assert manifest["schema_version"] == 4 @@ -576,8 +723,8 @@ class FakeEmitter: def transition_stage(self, stage_id, **details): events.append(("stage", stage_id, details)) - def fail(self, error): - events.append(("failed", None, str(error))) + def fail(self, error, **classification): + events.append(("failed", None, str(error), classification)) def close(self): events.append(("close",)) @@ -617,6 +764,8 @@ def refuse_preflight(requested): "close", ] assert events[-2][1] is None + # Raised before the attempt took over, the error is classified all the same. + assert events[-2][3] == {"failure_class": "error", "error_code": "BUILD_FAILED"} def test_hosted_stage_reporting_survives_staging_bundle_refusal(monkeypatch, capsys): @@ -704,10 +853,19 @@ def test_main_runs_the_logbook_envelope_around_a_dense_build(tmp_path, monkeypat assert row.disposition == "iterating" assert row.artifact_location.endswith(f"{STEM}.h5") assert "published" in row.phases_reached + # The six local gates resolve in the signed local report; every other + # gate in the graph's full document, by its outcome position. assert row.gate_verdicts and all( - item["verdict"] == "passed" and ".local_gates.json#/gates/" in item["receipt"] - for item in row.gate_verdicts.values() + item["verdict"] == "passed" for item in row.gate_verdicts.values() ) + for gate_id, item in row.gate_verdicts.items(): + if gate_id in UK_LOCAL_GATE_SCOPE: + assert item["receipt"].endswith(f".local_gates.json#/gates/{gate_id}") + else: + assert ( + f"{cli.FULL_GATE_REPORT_FILENAME}#/report/outcomes/" + in (item["receipt"]) + ) manifest = json.loads((args.out / "rowwise_candidate_manifest.json").read_text()) assert manifest["staging_delivery"]["enabled"] is False assert manifest["staged_dataset"]["status"] == "skipped" @@ -881,6 +1039,7 @@ def test_main_stages_the_bundle_locally_with_staging_local_only( "calibration", "gate_battery", "output_bundle", + "graph_execution", "dataset_staging", "complete", ] @@ -1387,10 +1546,13 @@ def test_blocked_gates_partition_failures_by_criticality(tmp_path, monkeypatch, report = json.loads( Path(manifest["outputs"]["local_gate_report"]["path"]).read_text() ) - outcomes = {outcome["id"]: outcome for outcome in report["report"]["outcomes"]} + # The local battery report carries the graph's outcomes for its six + # gates, blocked ones included, and is never shippable. + assert set(report["gates"]) == set(UK_LOCAL_GATE_SCOPE) for gate_id in ("uk_local_area_support", "uk_local_weight_ratio"): - assert outcomes[gate_id]["criticality"] == "release_blocking" - assert outcomes[gate_id]["status"] == "failed" + assert report["gates"][gate_id]["criticality"] == "release_blocking" + assert report["gates"][gate_id]["status"] == "failed" + assert report["shippable"] is False assert spool_rows(out)[0].disposition == "failed" @@ -1578,8 +1740,9 @@ def test_dense_validation_result_matches_emitted_run_status( for event in fake_telemetry_emitters[-1].events if event["event_type"] == "run" ] + # A candidate the gates refused is a blocked run, not a failed one. assert [event["status"] for event in events] == [ - "failed" if failed else "completed" + "blocked" if failed else "completed" ] @@ -1622,6 +1785,10 @@ def execute(*args, **kwargs): ] if outcome == "error": assert events[0]["message"] == str(error) + if outcome == 7: + # A non-zero return with no recorded block: the close-outs' vocabulary. + assert events[0]["details"]["error_code"] == "BUILD_REFUSED" + assert events[0]["details"]["failure_class"] == "refused" assert not emitter.available @@ -1660,3 +1827,275 @@ def test_staging_bundle_finalization_does_not_complete_hosted_run( rowwise_staging.finalize_staging_run_bundle(args, None) assert emitter.events == [] assert emitter.available + + +def test_an_attempt_names_itself_and_reports_its_graph_reuse(tmp_path, monkeypatch): + """The attempt's request evidence carries its Logbook and staging ids, and + the staging run gets the attempt's reuse counts before it closes.""" + pytest.importorskip("tables") + staging_dir = tmp_path / "staging-bundle" + status, out = run_dense_main( + tmp_path, + monkeypatch, + "--staging-dir", + str(staging_dir), + staging="--staging-local-only", + ) + assert status == 0 + row = spool_rows(out)[0] + execution = json.loads((out / cli.MANIFEST_FILENAME).read_text())["execution"] + request = json.loads( + (Path(execution["attempt_directory"]) / "request.json").read_text() + ) + bundle = _only_staging_run(staging_dir) + assert request["attempt"] == { + "build_id": row.build_id, + "run_id": bundle["run_manifest"]["run_id"], + } + events = [ + event + for event in bundle["events"] + if (event["stage_id"], event["status"]) == ("graph_execution", "completed") + ] + assert len(events) == 1 + assert events[0]["details"] == { + key: execution[key] for key in ("nodes_total", "nodes_reused", "nodes_computed") + } + + +def test_a_block_the_staging_contract_refuses_closes_both_destinations_failed( + tmp_path, +): + """A gate block whose details the staging content policy rejects (a gate id + that reads as a sensitive key) must not leave the run ``running``: the + staging run and the hosted emitter both close ``failed`` with the + unrecorded-block class, and a recordable block still closes both ``blocked`` + with the gate statuses.""" + from microcosm.build.run_outcome import UNRECORDED_GATE_BLOCK, GateBlock + from microcosm.build.staging_v2 import StagingRunBundleWriterV2 + from microcosm.build.telemetry_emitter import TelemetryRun + from microcosm.build.uk_runtime import rowwise_staging + from test_support.microcosm_build.telemetry import FakeTelemetryEmitter + + def bundle(name): + return StagingRunBundleWriterV2( + run_id=name, + country_code="GB", + operation_id="uk_full_build", + pipeline_id="uk_local_candidate", + pipeline_version="2026.10", + candidate_id=name, + local_dir=tmp_path / name, + release_id=None, + run_kind="smoke", + delivery_mode="local_only", + repo_id=None, + ) + + def emitter(name): + return FakeTelemetryEmitter( + TelemetryRun( + run_id=name, + country_code="GB", + pipeline="uk_local_candidate", + candidate_id=name, + producer_id="producer-a", + ) + ) + + def run_events(fake): + return [event for event in fake.events if event["event_type"] == "run"] + + refused_bundle, refused_emitter = bundle("refused"), emitter("refused") + rowwise_staging.close_run_blocked( + refused_bundle, + refused_emitter, + GateBlock.of("terminal", ["token"], gate_statuses={"token": "failed"}), + ) + failure = refused_bundle.validate_local_bundle()["progress"]["failure"] + assert (failure["error_code"], failure["failure_class"]) == ( + UNRECORDED_GATE_BLOCK.error_code, + UNRECORDED_GATE_BLOCK.failure_class, + ) + events = run_events(refused_emitter) + assert [event["status"] for event in events] == ["failed"] + assert events[0]["details"]["error_code"] == UNRECORDED_GATE_BLOCK.error_code + + blocked_bundle, blocked_emitter = bundle("blocked"), emitter("blocked") + rowwise_staging.close_run_blocked( + blocked_bundle, + blocked_emitter, + GateBlock.of( + "terminal", + ["uk_local_target_fit"], + gate_statuses={"uk_local_target_fit": "failed"}, + ), + ) + assert blocked_bundle.validate_local_bundle()["progress"]["status"] == "blocked" + events = run_events(blocked_emitter) + assert [event["status"] for event in events] == ["blocked"] + assert events[0]["details"]["gate_statuses"] == {"uk_local_target_fit": "failed"} + + +def _only_staging_run(staging_dir: Path): + from microcosm.build.staging_v2 import validate_staging_bundle + + runs = sorted(path.name for path in (staging_dir / "runs").iterdir()) + assert len(runs) == 1 + return validate_staging_bundle(staging_dir, runs[0]) + + +def test_a_gate_block_closes_the_staging_run_as_blocked(tmp_path, monkeypatch): + """The terminal battery's refusal is a ``blocked`` run, not a completed one, + and the Logbook row records the same end as ``failed``.""" + pytest.importorskip("tables") + staging_dir = tmp_path / "staging-bundle" + status, out = run_dense_main( + tmp_path, + monkeypatch, + "--staging-dir", + str(staging_dir), + staging="--staging-local-only", + failed=(("uk_local_area_support", "ESS 42.3 < 50"),), + ) + assert status == 1 + bundle = _only_staging_run(staging_dir) + assert bundle["progress"]["status"] == "blocked" + assert bundle["progress"]["failure"] is None + assert bundle["progress"]["block"] == { + "phase": "terminal", + "blocking_failure_count": 1, + "blocking_gate_ids": ["uk_local_area_support"], + } + terminal = bundle["events"][-1] + assert (terminal["stage_id"], terminal["status"]) == ("blocked", "blocked") + assert terminal["details"]["gate_statuses"]["uk_local_area_support"] == "failed" + assert spool_rows(out)[0].disposition == "failed" + + +def test_a_preflight_refusal_is_blocked_at_preflight_not_passed(tmp_path, monkeypatch): + """Refused before solving, the run closes ``blocked`` at phase ``preflight`` + with the refusing gate, and its Logbook receipts resolve in the preflight + gate document.""" + pytest.importorskip("tables") + gate = "uk_target_surface_local_default_2025" + staging_dir = tmp_path / "staging-bundle" + status, out = run_dense_main( + tmp_path, + monkeypatch, + "--staging-dir", + str(staging_dir), + staging="--staging-local-only", + failed=gate, + ) + assert status == 1 + bundle = _only_staging_run(staging_dir) + assert bundle["progress"]["status"] == "blocked" + assert bundle["progress"]["block"]["phase"] == "preflight" + assert bundle["progress"]["block"]["blocking_gate_ids"] == [gate] + stages = [event["stage_id"] for event in bundle["events"]] + assert "preflight_gates" in stages and "calibration" not in stages + row = spool_rows(out)[0] + assert row.disposition == "failed" + assert "candidate_blocked_at_preflight" in row.phases_reached + report_path = out / "uk.full.gates.preflight.gate_report.json" + assert report_path.is_file() + receipt = row.gate_verdicts[gate]["receipt"] + assert receipt.startswith(local_ref(report_path)) + index = int(receipt.rsplit("/", 1)[1]) + outcome = json.loads(report_path.read_text())["report"]["outcomes"][index] + assert outcome["id"] == gate and row.gate_verdicts[gate]["verdict"] == "failed" + + +def test_a_raised_error_is_classified_in_the_staging_run(tmp_path, monkeypatch): + """A build that raises closes ``failed`` with a mapped code and class.""" + pytest.importorskip("tables") + staging_dir = tmp_path / "staging-bundle" + original_run = cli.run_graph + + def run_out_of_memory(compiled, **kwargs): + if "uk.full.gates.calibrated" in {node.id for node in compiled.graph.nodes}: + raise MemoryError("pool too large") + return original_run(compiled, **kwargs) + + monkeypatch.setattr(cli, "run_graph", run_out_of_memory) + status, out = run_dense_main( + tmp_path, + monkeypatch, + "--staging-dir", + str(staging_dir), + staging="--staging-local-only", + ) + assert status == 1 + failure = _only_staging_run(staging_dir)["progress"]["failure"] + assert (failure["error_code"], failure["failure_class"]) == ( + "OUT_OF_MEMORY", + "out_of_memory", + ) + assert spool_rows(out)[0].disposition == "failed" + + +def test_a_sigterm_closes_the_dense_run_as_terminated_and_exits_143( + tmp_path, monkeypatch +): + """A supervisor's SIGTERM is recorded like Ctrl-C (a discarded row, a + failed staging run) with its own class, and the command exits 143.""" + pytest.importorskip("tables") + staging_dir = tmp_path / "staging-bundle" + original_run = cli.run_graph + + def terminated(compiled, **kwargs): + if "uk.full.gates.calibrated" in {node.id for node in compiled.graph.nodes}: + os.kill(os.getpid(), signal.SIGTERM) + return original_run(compiled, **kwargs) + + monkeypatch.setattr(cli, "run_graph", terminated) + previous_handler = signal.getsignal(signal.SIGTERM) + with pytest.raises(SystemExit) as raised: + run_dense_main( + tmp_path, + monkeypatch, + "--staging-dir", + str(staging_dir), + staging="--staging-local-only", + ) + assert raised.value.code == 143 + assert signal.getsignal(signal.SIGTERM) == previous_handler + out = tmp_path / "out" + failure = _only_staging_run(staging_dir)["progress"]["failure"] + assert (failure["error_code"], failure["failure_class"]) == ( + "TERMINATED", + "terminated", + ) + assert json.loads((out / "failure.json").read_text())["error_type"] == ( + "BuildTerminatedError" + ) + assert spool_rows(out)[0].disposition == "discarded" + + +def test_a_sigterm_during_the_national_build_records_a_discarded_row( + tmp_path, monkeypatch +): + """The national line routes a SIGTERM through its interrupt arm too.""" + argv = _national_argv(tmp_path) + monkeypatch.setattr(cli, "preflight_staged_dataset", lambda args: None) + monkeypatch.setattr( + cli, + "prepare_national_build", + lambda args, *, telemetry=None, attempt=None: "prepared", + ) + + def terminated(prepared, args, *, telemetry=None, attempt=None): + os.kill(os.getpid(), signal.SIGTERM) + raise AssertionError("SIGTERM did not stop the build") + + monkeypatch.setattr(cli, "execute_national_build", terminated) + with pytest.raises(SystemExit) as raised: + cli.main(argv) + assert raised.value.code == 143 + out = tmp_path / "out" + assert json.loads((out / "failure.json").read_text())["error_type"] == ( + "BuildTerminatedError" + ) + rows = load_spool_rows(out / "logbook-spool") + assert [row.disposition for row in rows] == ["discarded"] diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_local_release_preflight.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_local_release_preflight.py index 7e36f7795..3a5247b2d 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_local_release_preflight.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_local_release_preflight.py @@ -428,32 +428,14 @@ def test_preflight_does_not_infer_tenure_success_from_ladder_uprating(tmp_path): assert not any("A15" in failure for failure in failures) -def _resign(report: dict, *, release_id: str) -> dict: - """The fixture report re-signed for another attempt id.""" - - import hashlib - import hmac - - from microcosm.build.gate_battery import _canonical_json_bytes - - signed = json.loads(json.dumps(report)) - signed["release_id"] = release_id - signed["attestation"]["release_id"] = release_id - signed["attestation"]["signature"] = None - signed["attestation"]["signature"] = hmac.new( - base64.b64decode(KEY), _canonical_json_bytes(signed), hashlib.sha256 - ).hexdigest() - return signed - - def test_graph_built_manifest_passes_the_candidate_preflight( tmp_path: Path, monkeypatch ) -> None: """The graph driver's projected manifest is the schema the pre-flight reads. - The graph's own gate report is the full-build battery document, not the - signed local battery report; the signed report is supplied here as the - certification step will, under the graph's ``*.local_gates.json`` name. + The build signs its own local battery report under the posture's + ``*.local_gates.json`` name, bound to its Logbook attempt; nothing is + signed by hand. """ pytest.importorskip("tables") @@ -466,9 +448,8 @@ def test_graph_built_manifest_passes_the_candidate_preflight( out = graph_dense_bundle(tmp_path, monkeypatch, "--release-candidate") module = _load() build_id = load_spool_rows(out / "logbook-spool")[0].build_id - (out / f"{STEM}.local_gates.json").write_text( - json.dumps(_resign(_signed_report(), release_id=build_id)) - ) + report = json.loads((out / f"{STEM}.local_gates.json").read_text()) + assert report["release_id"] == build_id and report["shippable"] is True manifest = json.loads((out / "rowwise_candidate_manifest.json").read_text()) assert manifest["release_role"] == "dense" assert manifest["parameters"]["release_candidate"] is True diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py index 74c430385..654a25b69 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py @@ -256,6 +256,15 @@ def test_uk_national_role_builds_through_the_graph_and_stages_locally( assert json.loads(capsys.readouterr().out) == manifest run_id = manifest["build_id"] assert run_id.startswith("uk-frs-calibration-attempt-") + # The attempt records how it ran its graph and names itself in its request + # evidence, so a later attempt on the same store can link back to it. + execution = manifest["execution"] + assert execution["nodes_total"] > 0 and execution["nodes_reused"] == 0 + assert execution["earlier_attempts"] == [] + request = json.loads( + (Path(execution["attempt_directory"]) / "request.json").read_text() + ) + assert request["attempt"]["build_id"] == run_id # The seam's output names, from the posture. dataset = out / "microcosm_uk_2024_25.h5" gates = out / "microcosm_uk_2024_25.terminal_gates.json" @@ -771,6 +780,38 @@ def test_uk_national_role_blocks_on_a_failed_seam_gate(monkeypatch, tmp_path): assert rows[0].gate_verdicts["uk_aggregate_admin"]["verdict"] == "failed" +def test_uk_national_role_closes_a_seam_block_as_a_blocked_run(monkeypatch, tmp_path): + """The seam battery's refusal closes the staging run ``blocked`` at the + terminal phase, naming the refusing gate, while the row stays ``failed``.""" + pytest.importorskip("tables") + builder = candidate._load_builder_module() + input_h5, _registry, _artifact, _pin = _national_inputs(monkeypatch, tmp_path) + monkeypatch.setattr( + calibration_run, + "uk_aggregate_admin_totals", + lambda frame, manifest: ( + {name: value * 5.0 for name, value in seam._admin_anchor_values().items()}, + [], + ), + ) + out = tmp_path / "blocked" + + assert ( + builder.main(_argv(input_h5, out, "--staging-local-only", "--epochs", "5")) == 1 + ) + + runs = sorted(path.name for path in (out / "staging" / "runs").iterdir()) + bundle = validate_v2_bundle(out / "staging", runs[0]) + assert bundle["progress"]["status"] == "blocked" + assert bundle["progress"]["block"]["phase"] == "terminal" + assert "uk_aggregate_admin" in bundle["progress"]["block"]["blocking_gate_ids"] + checks = [ + e for e in bundle["events"] if e["stage_id"] == "release_check_evaluation" + ] + assert checks[-1]["details"]["gate_statuses"]["uk_aggregate_admin"] == "failed" + assert load_spool_rows(out / "logbook-spool")[0].disposition == "failed" + + # --- Behaviours re-anchored from the retired seam command's tests (B3) --- diff --git a/packages/microcosm-build/tests/fixtures/staging/v3/SHA256SUMS b/packages/microcosm-build/tests/fixtures/staging/v3/SHA256SUMS new file mode 100644 index 000000000..a4caf0981 --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/staging/v3/SHA256SUMS @@ -0,0 +1,14 @@ +35e62f17c830fe9651f8c5546430e6bb14e6ddfd0f15a10494540ec5b1e83164 blocked/runs/uk-blocked-v3-fixture/events.ndjson +f815083c6bdbb35c899975563f08d81995cb5930be062ec9987c3098d7d78781 blocked/runs/uk-blocked-v3-fixture/progress.json +d3f8a58d1e6c352c3aacee5f42b95b0d95f481c44aae9b45be9f9efbab16c70a blocked/runs/uk-blocked-v3-fixture/run_manifest.json +dc419d0f8ab3babae8371ac020bf0c8348eb2df80492583182103735fcce4d39 calibration/runs/uk-calibration-v3-fixture/calibration_progress.json +df00f9100e5d645224126ab0214d75a744cf65cbf55c0339baed6073e4c49dac calibration/runs/uk-calibration-v3-fixture/events.ndjson +cd8e51d2cb918c3e1f413b64d4ad65f0449369f2898698371d0f47614668edbb calibration/runs/uk-calibration-v3-fixture/progress.json +29deee29b720e702caaf27fa88e7868a37cb26aabf9ef1eab52192a0ff319113 calibration/runs/uk-calibration-v3-fixture/run_manifest.json +cec0bf866e479da8f7309028f98e24c308f411d9866fe5e50f7d0cd99b52511a completed-spine/runs/uk-spine-v3-fixture/events.ndjson +d37e63aa3d430050d335e986f3b569335a18ecc8409ef725e676453f6f51a95b completed-spine/runs/uk-spine-v3-fixture/progress.json +31db516ef7409838c5b66d192523578f612d5e165142093551e4e8de5e03e31c completed-spine/runs/uk-spine-v3-fixture/run_manifest.json +491fb1569ae3c39d234d57ddec20dda3f8871518f626b703abfe2d5e330eff32 contract-cases.json +ad6303cee988ab245793bf7201b58760861f11c5117a7f75fed073df1b5cc361 failed/runs/uk-failed-v3-fixture/events.ndjson +eff3de6e27cbd9a0591be951d4316559a838fff8f1d98a1032e6cc6758bc04b4 failed/runs/uk-failed-v3-fixture/progress.json +d6a7a044bff0f777982d80d78c062ad89422876aefa0736365375ba09ceb7edd failed/runs/uk-failed-v3-fixture/run_manifest.json diff --git a/packages/microcosm-build/tests/fixtures/staging/v3/blocked/runs/uk-blocked-v3-fixture/events.ndjson b/packages/microcosm-build/tests/fixtures/staging/v3/blocked/runs/uk-blocked-v3-fixture/events.ndjson new file mode 100644 index 000000000..4de9633b4 --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/staging/v3/blocked/runs/uk-blocked-v3-fixture/events.ndjson @@ -0,0 +1,4 @@ +{"details":{},"event_type":"stage","message":"Staging run created.","run_id":"uk-blocked-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":1,"stage_id":"created","status":"started","timestamp":"2026-01-05T00:00:00+00:00"} +{"details":{"target_count":4},"event_type":"stage","message":null,"run_id":"uk-blocked-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":2,"stage_id":"target_compilation","status":"completed","timestamp":"2026-01-05T00:00:01+00:00"} +{"details":{"blocking_failure_count":1,"diagnostic_failure_count":0,"gate_statuses":{"uk_target_fit":"failed","uk_weight_ess":"passed"}},"event_type":"stage","message":null,"run_id":"uk-blocked-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":3,"stage_id":"gate_battery","status":"completed","timestamp":"2026-01-05T00:00:02+00:00"} +{"details":{"blocking_failure_count":1,"blocking_gate_ids":["uk_target_fit"],"gate_statuses":{"uk_target_fit":"failed","uk_weight_ess":"passed"},"phase":"terminal"},"event_type":"stage","message":"The gates refused the candidate at terminal.","run_id":"uk-blocked-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":4,"stage_id":"blocked","status":"blocked","timestamp":"2026-01-05T00:00:03+00:00"} diff --git a/packages/microcosm-build/tests/fixtures/staging/v3/blocked/runs/uk-blocked-v3-fixture/progress.json b/packages/microcosm-build/tests/fixtures/staging/v3/blocked/runs/uk-blocked-v3-fixture/progress.json new file mode 100644 index 000000000..108e65248 --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/staging/v3/blocked/runs/uk-blocked-v3-fixture/progress.json @@ -0,0 +1,54 @@ +{ + "block": { + "blocking_failure_count": 1, + "blocking_gate_ids": [ + "uk_target_fit" + ], + "phase": "terminal" + }, + "candidate_id": "uk-blocked-v3-fixture", + "country_code": "GB", + "current_stage": "blocked", + "delivery": { + "configured_repository": null, + "contract_version": 2, + "enabled": true, + "last_error_code": null, + "mode": "local_only", + "opt_out_reason": null, + "read_back": "not_requested", + "run_id": "uk-blocked-v3-fixture", + "upload_attempts": 0, + "upload_successes": 0 + }, + "details": { + "blocking_failure_count": 1, + "blocking_gate_ids": [ + "uk_target_fit" + ], + "gate_statuses": { + "uk_target_fit": "failed", + "uk_weight_ess": "passed" + }, + "phase": "terminal" + }, + "failure": null, + "message": "The gates refused the candidate at terminal.", + "non_release": true, + "operation_id": "uk_rowwise_candidate", + "pipeline": { + "id": "uk_local_candidate", + "version": "2026.10" + }, + "release_id": null, + "run_id": "uk-blocked-v3-fixture", + "run_kind": "calibration", + "sample": { + "mode": "full" + }, + "schema_name": "microcosm.staging.progress", + "schema_version": 3, + "started_at": "2026-01-05T00:00:00+00:00", + "status": "blocked", + "updated_at": "2026-01-05T00:00:03+00:00" +} diff --git a/packages/microcosm-build/tests/fixtures/staging/v3/blocked/runs/uk-blocked-v3-fixture/run_manifest.json b/packages/microcosm-build/tests/fixtures/staging/v3/blocked/runs/uk-blocked-v3-fixture/run_manifest.json new file mode 100644 index 000000000..b00455782 --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/staging/v3/blocked/runs/uk-blocked-v3-fixture/run_manifest.json @@ -0,0 +1,48 @@ +{ + "artifacts": [], + "block": { + "blocking_failure_count": 1, + "blocking_gate_ids": [ + "uk_target_fit" + ], + "phase": "terminal" + }, + "candidate_id": "uk-blocked-v3-fixture", + "country_code": "GB", + "current_stage": "blocked", + "delivery": { + "configured_repository": null, + "contract_version": 2, + "enabled": true, + "last_error_code": null, + "mode": "local_only", + "opt_out_reason": null, + "read_back": "not_requested", + "run_id": "uk-blocked-v3-fixture", + "upload_attempts": 0, + "upload_successes": 0 + }, + "failure": null, + "non_release": true, + "operation_id": "uk_rowwise_candidate", + "paths": { + "calibration_progress": null, + "events": "runs/uk-blocked-v3-fixture/events.ndjson", + "progress": "runs/uk-blocked-v3-fixture/progress.json" + }, + "pipeline": { + "id": "uk_local_candidate", + "version": "2026.10" + }, + "release_id": null, + "run_id": "uk-blocked-v3-fixture", + "run_kind": "calibration", + "sample": { + "mode": "full" + }, + "schema_name": "microcosm.staging.run-manifest", + "schema_version": 3, + "started_at": "2026-01-05T00:00:00+00:00", + "status": "blocked", + "updated_at": "2026-01-05T00:00:03+00:00" +} diff --git a/packages/microcosm-build/tests/fixtures/staging/v3/calibration/runs/uk-calibration-v3-fixture/calibration_progress.json b/packages/microcosm-build/tests/fixtures/staging/v3/calibration/runs/uk-calibration-v3-fixture/calibration_progress.json new file mode 100644 index 000000000..6c6fd33b0 --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/staging/v3/calibration/runs/uk-calibration-v3-fixture/calibration_progress.json @@ -0,0 +1,21 @@ +{ + "candidate_id": "uk-calibration-candidate", + "events": [ + { + "budget_iteration": null, + "budget_iters": null, + "budget_search": null, + "epoch": 1, + "epochs": 3, + "iteration": 4, + "l0_lambda": null, + "loss": 2.5, + "phase": "solve", + "timestamp": "2026-01-03T00:00:02+00:00" + } + ], + "run_id": "uk-calibration-v3-fixture", + "schema_name": "microcosm.staging.calibration-progress", + "schema_version": 3, + "updated_at": "2026-01-03T00:00:04+00:00" +} diff --git a/packages/microcosm-build/tests/fixtures/staging/v3/calibration/runs/uk-calibration-v3-fixture/events.ndjson b/packages/microcosm-build/tests/fixtures/staging/v3/calibration/runs/uk-calibration-v3-fixture/events.ndjson new file mode 100644 index 000000000..a6e168609 --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/staging/v3/calibration/runs/uk-calibration-v3-fixture/events.ndjson @@ -0,0 +1,5 @@ +{"details":{},"event_type":"stage","message":"Staging run created.","run_id":"uk-calibration-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":1,"stage_id":"created","status":"started","timestamp":"2026-01-03T00:00:00+00:00"} +{"details":{"target_count":2},"event_type":"stage","message":null,"run_id":"uk-calibration-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":2,"stage_id":"target_compilation","status":"completed","timestamp":"2026-01-03T00:00:01+00:00"} +{"details":{"budget_iteration":null,"budget_iters":null,"budget_search":null,"epoch":1,"epochs":3,"iteration":4,"l0_lambda":null,"loss":2.5,"phase":"solve"},"event_type":"calibration","message":null,"run_id":"uk-calibration-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":3,"stage_id":"calibrating","status":"progress","timestamp":"2026-01-03T00:00:02+00:00"} +{"details":{"check_count":4},"event_type":"stage","message":null,"run_id":"uk-calibration-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":4,"stage_id":"diagnostics","status":"completed","timestamp":"2026-01-03T00:00:03+00:00"} +{"details":{},"event_type":"stage","message":"Staging run completed.","run_id":"uk-calibration-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":5,"stage_id":"complete","status":"completed","timestamp":"2026-01-03T00:00:04+00:00"} diff --git a/packages/microcosm-build/tests/fixtures/staging/v3/calibration/runs/uk-calibration-v3-fixture/progress.json b/packages/microcosm-build/tests/fixtures/staging/v3/calibration/runs/uk-calibration-v3-fixture/progress.json new file mode 100644 index 000000000..6fc30a214 --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/staging/v3/calibration/runs/uk-calibration-v3-fixture/progress.json @@ -0,0 +1,38 @@ +{ + "block": null, + "candidate_id": "uk-calibration-candidate", + "country_code": "GB", + "current_stage": "complete", + "delivery": { + "configured_repository": null, + "contract_version": 2, + "enabled": true, + "last_error_code": null, + "mode": "local_only", + "opt_out_reason": null, + "read_back": "not_requested", + "run_id": "uk-calibration-v3-fixture", + "upload_attempts": 0, + "upload_successes": 0 + }, + "details": {}, + "failure": null, + "message": "Staging run completed.", + "non_release": false, + "operation_id": "uk_national_calibration", + "pipeline": { + "id": "uk_national_calibration", + "version": "2026.09" + }, + "release_id": "populace-uk-2023-v3-fixture", + "run_id": "uk-calibration-v3-fixture", + "run_kind": "calibration", + "sample": { + "mode": "full" + }, + "schema_name": "microcosm.staging.progress", + "schema_version": 3, + "started_at": "2026-01-03T00:00:00+00:00", + "status": "completed", + "updated_at": "2026-01-03T00:00:04+00:00" +} diff --git a/packages/microcosm-build/tests/fixtures/staging/v3/calibration/runs/uk-calibration-v3-fixture/run_manifest.json b/packages/microcosm-build/tests/fixtures/staging/v3/calibration/runs/uk-calibration-v3-fixture/run_manifest.json new file mode 100644 index 000000000..9f76195f6 --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/staging/v3/calibration/runs/uk-calibration-v3-fixture/run_manifest.json @@ -0,0 +1,42 @@ +{ + "artifacts": [], + "block": null, + "candidate_id": "uk-calibration-candidate", + "country_code": "GB", + "current_stage": "complete", + "delivery": { + "configured_repository": null, + "contract_version": 2, + "enabled": true, + "last_error_code": null, + "mode": "local_only", + "opt_out_reason": null, + "read_back": "not_requested", + "run_id": "uk-calibration-v3-fixture", + "upload_attempts": 0, + "upload_successes": 0 + }, + "failure": null, + "non_release": false, + "operation_id": "uk_national_calibration", + "paths": { + "calibration_progress": "runs/uk-calibration-v3-fixture/calibration_progress.json", + "events": "runs/uk-calibration-v3-fixture/events.ndjson", + "progress": "runs/uk-calibration-v3-fixture/progress.json" + }, + "pipeline": { + "id": "uk_national_calibration", + "version": "2026.09" + }, + "release_id": "populace-uk-2023-v3-fixture", + "run_id": "uk-calibration-v3-fixture", + "run_kind": "calibration", + "sample": { + "mode": "full" + }, + "schema_name": "microcosm.staging.run-manifest", + "schema_version": 3, + "started_at": "2026-01-03T00:00:00+00:00", + "status": "completed", + "updated_at": "2026-01-03T00:00:04+00:00" +} diff --git a/packages/microcosm-build/tests/fixtures/staging/v3/completed-spine/runs/uk-spine-v3-fixture/events.ndjson b/packages/microcosm-build/tests/fixtures/staging/v3/completed-spine/runs/uk-spine-v3-fixture/events.ndjson new file mode 100644 index 000000000..13336defd --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/staging/v3/completed-spine/runs/uk-spine-v3-fixture/events.ndjson @@ -0,0 +1,8 @@ +{"details":{},"event_type":"stage","message":"Staging run created.","run_id":"uk-spine-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":1,"stage_id":"created","status":"started","timestamp":"2026-01-02T00:00:00+00:00"} +{"details":{"input_count":3},"event_type":"stage","message":null,"run_id":"uk-spine-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":2,"stage_id":"input_verification","status":"completed","timestamp":"2026-01-02T00:00:01+00:00"} +{"details":{"household_rows":9},"event_type":"stage","message":null,"run_id":"uk-spine-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":3,"stage_id":"sampling","status":"completed","timestamp":"2026-01-02T00:00:02+00:00"} +{"details":{"produced_columns":12},"event_type":"stage","message":null,"run_id":"uk-spine-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":4,"stage_id":"construction","status":"completed","timestamp":"2026-01-02T00:00:03+00:00"} +{"details":{"household_rows":9},"event_type":"stage","message":null,"run_id":"uk-spine-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":5,"stage_id":"validation","status":"completed","timestamp":"2026-01-02T00:00:04+00:00"} +{"details":{"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"event_type":"stage","message":null,"run_id":"uk-spine-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":6,"stage_id":"spine_h5_creation","status":"completed","timestamp":"2026-01-02T00:00:05+00:00"} +{"details":{"sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},"event_type":"stage","message":null,"run_id":"uk-spine-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":7,"stage_id":"sidecar_creation","status":"completed","timestamp":"2026-01-02T00:00:06+00:00"} +{"details":{},"event_type":"stage","message":"Staging run completed.","run_id":"uk-spine-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":8,"stage_id":"complete","status":"completed","timestamp":"2026-01-02T00:00:07+00:00"} diff --git a/packages/microcosm-build/tests/fixtures/staging/v3/completed-spine/runs/uk-spine-v3-fixture/progress.json b/packages/microcosm-build/tests/fixtures/staging/v3/completed-spine/runs/uk-spine-v3-fixture/progress.json new file mode 100644 index 000000000..073aea4cd --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/staging/v3/completed-spine/runs/uk-spine-v3-fixture/progress.json @@ -0,0 +1,38 @@ +{ + "block": null, + "candidate_id": "uk-spine-v3-fixture", + "country_code": "GB", + "current_stage": "complete", + "delivery": { + "configured_repository": null, + "contract_version": 2, + "enabled": true, + "last_error_code": null, + "mode": "local_only", + "opt_out_reason": null, + "read_back": "not_requested", + "run_id": "uk-spine-v3-fixture", + "upload_attempts": 0, + "upload_successes": 0 + }, + "details": {}, + "failure": null, + "message": "Staging run completed.", + "non_release": true, + "operation_id": "uk_frs_spine", + "pipeline": { + "id": "uk_national_spine", + "version": "2026.09" + }, + "release_id": null, + "run_id": "uk-spine-v3-fixture", + "run_kind": "smoke", + "sample": { + "mode": "full" + }, + "schema_name": "microcosm.staging.progress", + "schema_version": 3, + "started_at": "2026-01-02T00:00:00+00:00", + "status": "completed", + "updated_at": "2026-01-02T00:00:07+00:00" +} diff --git a/packages/microcosm-build/tests/fixtures/staging/v3/completed-spine/runs/uk-spine-v3-fixture/run_manifest.json b/packages/microcosm-build/tests/fixtures/staging/v3/completed-spine/runs/uk-spine-v3-fixture/run_manifest.json new file mode 100644 index 000000000..e630819b8 --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/staging/v3/completed-spine/runs/uk-spine-v3-fixture/run_manifest.json @@ -0,0 +1,42 @@ +{ + "artifacts": [], + "block": null, + "candidate_id": "uk-spine-v3-fixture", + "country_code": "GB", + "current_stage": "complete", + "delivery": { + "configured_repository": null, + "contract_version": 2, + "enabled": true, + "last_error_code": null, + "mode": "local_only", + "opt_out_reason": null, + "read_back": "not_requested", + "run_id": "uk-spine-v3-fixture", + "upload_attempts": 0, + "upload_successes": 0 + }, + "failure": null, + "non_release": true, + "operation_id": "uk_frs_spine", + "paths": { + "calibration_progress": null, + "events": "runs/uk-spine-v3-fixture/events.ndjson", + "progress": "runs/uk-spine-v3-fixture/progress.json" + }, + "pipeline": { + "id": "uk_national_spine", + "version": "2026.09" + }, + "release_id": null, + "run_id": "uk-spine-v3-fixture", + "run_kind": "smoke", + "sample": { + "mode": "full" + }, + "schema_name": "microcosm.staging.run-manifest", + "schema_version": 3, + "started_at": "2026-01-02T00:00:00+00:00", + "status": "completed", + "updated_at": "2026-01-02T00:00:07+00:00" +} diff --git a/packages/microcosm-build/tests/fixtures/staging/v3/contract-cases.json b/packages/microcosm-build/tests/fixtures/staging/v3/contract-cases.json new file mode 100644 index 000000000..af85dc8b5 --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/staging/v3/contract-cases.json @@ -0,0 +1,44 @@ +{ + "deliberate_opt_out": { + "configured_repository": null, + "contract_version": 2, + "enabled": false, + "last_error_code": null, + "mode": "disabled", + "opt_out_reason": "--no-staging", + "read_back": "not_requested", + "run_id": null, + "upload_attempts": 0, + "upload_successes": 0 + }, + "delivery_failure": { + "configured_repository": "policyengine/populace-uk-staging", + "contract_version": 2, + "enabled": true, + "last_error_code": "READ_BACK_FAILED", + "mode": "local_and_remote", + "opt_out_reason": null, + "read_back": "failed", + "run_id": "uk-delivery-failure", + "upload_attempts": 3, + "upload_successes": 0 + }, + "delivery_success": { + "configured_repository": "policyengine/populace-uk-staging", + "contract_version": 2, + "enabled": true, + "last_error_code": null, + "mode": "local_and_remote", + "opt_out_reason": null, + "read_back": "passed", + "run_id": "uk-delivery-success", + "upload_attempts": 6, + "upload_successes": 6 + }, + "schema_name": "microcosm.staging.fixture-cases", + "schema_version": 3, + "unknown_version": { + "schema_name": "microcosm.staging.run-manifest", + "schema_version": 999 + } +} diff --git a/packages/microcosm-build/tests/fixtures/staging/v3/failed/runs/uk-failed-v3-fixture/events.ndjson b/packages/microcosm-build/tests/fixtures/staging/v3/failed/runs/uk-failed-v3-fixture/events.ndjson new file mode 100644 index 000000000..a17c2837e --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/staging/v3/failed/runs/uk-failed-v3-fixture/events.ndjson @@ -0,0 +1,3 @@ +{"details":{},"event_type":"stage","message":"Staging run created.","run_id":"uk-failed-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":1,"stage_id":"created","status":"started","timestamp":"2026-01-04T00:00:00+00:00"} +{"details":{},"event_type":"stage","message":null,"run_id":"uk-failed-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":2,"stage_id":"input_verification","status":"started","timestamp":"2026-01-04T00:00:01+00:00"} +{"details":{"error_code":"BUILD_FAILED","error_type":"RuntimeError","failure_class":"error"},"event_type":"stage","message":"The build failed during input_verification.","run_id":"uk-failed-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":3,"stage_id":"failed","status":"failed","timestamp":"2026-01-04T00:00:02+00:00"} diff --git a/packages/microcosm-build/tests/fixtures/staging/v3/failed/runs/uk-failed-v3-fixture/progress.json b/packages/microcosm-build/tests/fixtures/staging/v3/failed/runs/uk-failed-v3-fixture/progress.json new file mode 100644 index 000000000..a33052ed8 --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/staging/v3/failed/runs/uk-failed-v3-fixture/progress.json @@ -0,0 +1,42 @@ +{ + "block": null, + "candidate_id": "uk-failed-v3-fixture", + "country_code": "GB", + "current_stage": "failed", + "delivery": { + "configured_repository": null, + "contract_version": 2, + "enabled": true, + "last_error_code": null, + "mode": "local_only", + "opt_out_reason": null, + "read_back": "not_requested", + "run_id": "uk-failed-v3-fixture", + "upload_attempts": 0, + "upload_successes": 0 + }, + "details": {}, + "failure": { + "error_code": "BUILD_FAILED", + "error_type": "RuntimeError", + "failure_class": "error", + "local_diagnostic_reference": "diagnostics/operator-error.txt", + "message": "The build failed during input_verification." + }, + "message": "The build failed during input_verification.", + "non_release": true, + "operation_id": "uk_frs_spine", + "pipeline": { + "id": "uk_national_spine", + "version": "2026.09" + }, + "release_id": null, + "run_id": "uk-failed-v3-fixture", + "run_kind": "smoke", + "sample": null, + "schema_name": "microcosm.staging.progress", + "schema_version": 3, + "started_at": "2026-01-04T00:00:00+00:00", + "status": "failed", + "updated_at": "2026-01-04T00:00:02+00:00" +} diff --git a/packages/microcosm-build/tests/fixtures/staging/v3/failed/runs/uk-failed-v3-fixture/run_manifest.json b/packages/microcosm-build/tests/fixtures/staging/v3/failed/runs/uk-failed-v3-fixture/run_manifest.json new file mode 100644 index 000000000..285d33f81 --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/staging/v3/failed/runs/uk-failed-v3-fixture/run_manifest.json @@ -0,0 +1,46 @@ +{ + "artifacts": [], + "block": null, + "candidate_id": "uk-failed-v3-fixture", + "country_code": "GB", + "current_stage": "failed", + "delivery": { + "configured_repository": null, + "contract_version": 2, + "enabled": true, + "last_error_code": null, + "mode": "local_only", + "opt_out_reason": null, + "read_back": "not_requested", + "run_id": "uk-failed-v3-fixture", + "upload_attempts": 0, + "upload_successes": 0 + }, + "failure": { + "error_code": "BUILD_FAILED", + "error_type": "RuntimeError", + "failure_class": "error", + "local_diagnostic_reference": "diagnostics/operator-error.txt", + "message": "The build failed during input_verification." + }, + "non_release": true, + "operation_id": "uk_frs_spine", + "paths": { + "calibration_progress": null, + "events": "runs/uk-failed-v3-fixture/events.ndjson", + "progress": "runs/uk-failed-v3-fixture/progress.json" + }, + "pipeline": { + "id": "uk_national_spine", + "version": "2026.09" + }, + "release_id": null, + "run_id": "uk-failed-v3-fixture", + "run_kind": "smoke", + "sample": null, + "schema_name": "microcosm.staging.run-manifest", + "schema_version": 3, + "started_at": "2026-01-04T00:00:00+00:00", + "status": "failed", + "updated_at": "2026-01-04T00:00:02+00:00" +} diff --git a/test_support/microcosm_build/uk_full_build_cli.py b/test_support/microcosm_build/uk_full_build_cli.py index 98465aed5..4e9ba2293 100644 --- a/test_support/microcosm_build/uk_full_build_cli.py +++ b/test_support/microcosm_build/uk_full_build_cli.py @@ -5,6 +5,7 @@ # ruff: noqa: F401 +import base64 import hashlib import importlib.util import json @@ -20,11 +21,13 @@ GatePhaseReport, GateStatus, gate_phase_report_payload, + gate_signing_key_env, ) from microcosm.build.gates import GateResult from microcosm.build.logbook import LOGBOOK_ROW_FIELDS, load_spool_rows from microcosm.build.logbook_adoption import local_artifact_reference from microcosm.build.uk_runtime import full_build_cli as cli +from microcosm.build.uk_runtime.calibration_run import UK_LOCAL_GATE_SCOPE from microcosm.build.uk_runtime.full_certification import FULL_CERTIFICATION_TYPE from microcosm.build.uk_runtime.full_gates import ( classify_full_gate_outcomes, @@ -115,6 +118,11 @@ def patch_support_register(monkeypatch, pins=None) -> None: STEM = "microcosm_uk_2024_25_local" +#: The UK gate signing key's variable and a synthetic 32-byte key: a +#: release-candidate request is refused without one, and the dense build signs +#: its local gate report with it. +SIGNING_KEY_ENV = gate_signing_key_env("uk") +TEST_SIGNING_KEY = base64.b64encode(b"\x05" * 32).decode() SUPPORT_ARGUMENTS = ( "--atomic-support-ew", "supports/ew.npz", @@ -272,7 +280,7 @@ def failure_lines(failed) -> dict[str, str]: return {str(gate_id): str(line) for gate_id, line in failed} -def gate_payload(phase, failed=None): +def gate_payload(phase, failed=None, release_candidate=False): gates = uk_full_gate_manifest(SELECTION) lines = failure_lines(failed) report = GatePhaseReport( @@ -298,10 +306,10 @@ def gate_payload(phase, failed=None): "kind": "uk_full_gate_report", "selection_receipt": SELECTION, "sample_fraction": 1.0, - "release_candidate": False, + "release_candidate": bool(release_candidate), "report": gate_phase_report_payload(report, gates=gates), "enforcement": classify_full_gate_outcomes( - report, sample_fraction=1.0, release_candidate=False + report, sample_fraction=1.0, release_candidate=bool(release_candidate) ), } ) @@ -551,7 +559,13 @@ def implementation_hash(self): def run(self, context): phase = context.params["phase"] - artifacts = {"gate_report": gate_payload(phase, context.params.get("failed"))} + artifacts = { + "gate_report": gate_payload( + phase, + context.params.get("failed"), + release_candidate=bool(context.params.get("release_candidate")), + ) + } if phase == "terminal": artifacts.update( calibration_diagnostics=diagnostics_payload(), @@ -681,7 +695,11 @@ def run_dense_main( monkeypatch.delenv("POPULACE_LOGBOOK_PREV_ROW_DIGEST", raising=False) patch_certification(monkeypatch) args = arguments(tmp_path, *extra, staging=staging) - build = prepared(tmp_path, failed) if build is None else build + build = ( + prepared(tmp_path, failed, release_candidate="--release-candidate" in extra) + if build is None + else build + ) def prepare(args, *, telemetry=None, attempt=None): if on_prepare is not None: @@ -705,7 +723,12 @@ def graph_dense_bundle(tmp_path, monkeypatch, *extra, staging="--no-staging") -> return out -def prepared(tmp_path, failed=None): +def prepared(tmp_path, failed=None, *, release_candidate=False): + """The synthetic prepared build; its gate nodes evaluate in the request's + posture (``release_candidate``), as the real gate nodes do.""" + # Only a release-candidate request adds the parameter, so every other + # synthetic build keeps its node keys. + posture = {"release_candidate": True} if release_candidate else {} frame = _frame() fixture = tmp_path / "fixture.txt" fixture.write_text("constant source") @@ -740,14 +763,14 @@ def prepared(tmp_path, failed=None): "uk.full.gates.preflight", Evidence.ref, population=root.id, - params={"phase": "preflight", "failed": failed}, + params={"phase": "preflight", "failed": failed, **posture}, artifact_outputs=(ArtifactOutput("gate_report", FULL_GATE_REPORT_TYPE),), ), Node( "uk.full.gates.calibrated", Evidence.ref, population=root.id, - params={"phase": "terminal", "failed": failed}, + params={"phase": "terminal", "failed": failed, **posture}, artifact_outputs=( ArtifactOutput("gate_report", FULL_GATE_REPORT_TYPE), ArtifactOutput("calibration_diagnostics", FULL_DIAGNOSTICS_TYPE), diff --git a/tools/generate_staging_contract_fixtures.py b/tools/generate_staging_contract_fixtures.py index 71563c8cd..ae6c29246 100644 --- a/tools/generate_staging_contract_fixtures.py +++ b/tools/generate_staging_contract_fixtures.py @@ -1,4 +1,9 @@ -"""Generate or verify the canonical staging telemetry version 2 fixtures.""" +"""Generate or verify the canonical staging contract fixtures (version 3). + +The version 2 fixtures under ``tests/fixtures/staging/v2`` are frozen: the writer now +emits version 3, so they are validated by their ``SHA256SUMS`` and the reader tests and +are no longer regenerated here. +""" from __future__ import annotations @@ -13,7 +18,7 @@ ROOT = Path(__file__).resolve().parents[1] DEFAULT_OUTPUT = ( - ROOT / "packages" / "microcosm-build" / "tests" / "fixtures" / "staging" / "v2" + ROOT / "packages" / "microcosm-build" / "tests" / "fixtures" / "staging" / "v3" ) @@ -30,12 +35,12 @@ def __call__(self) -> str: def _completed_spine(root: Path) -> None: recorder = StagingRunBundleWriterV2( - run_id="uk-spine-v2-fixture", + run_id="uk-spine-v3-fixture", country_code="GB", operation_id="uk_frs_spine", pipeline_id="uk_national_spine", pipeline_version="2026.09", - candidate_id="uk-spine-v2-fixture", + candidate_id="uk-spine-v3-fixture", local_dir=root / "completed-spine", run_kind="smoke", delivery_mode="local_only", @@ -55,13 +60,13 @@ def _completed_spine(root: Path) -> None: def _calibration(root: Path) -> None: recorder = StagingRunBundleWriterV2( - run_id="uk-calibration-v2-fixture", + run_id="uk-calibration-v3-fixture", country_code="GB", operation_id="uk_national_calibration", pipeline_id="uk_national_calibration", pipeline_version="2026.09", candidate_id="uk-calibration-candidate", - release_id="populace-uk-2023-v2-fixture", + release_id="populace-uk-2023-v3-fixture", local_dir=root / "calibration", run_kind="calibration", delivery_mode="local_only", @@ -87,12 +92,12 @@ def _calibration(root: Path) -> None: def _failed(root: Path) -> None: recorder = StagingRunBundleWriterV2( - run_id="uk-failed-v2-fixture", + run_id="uk-failed-v3-fixture", country_code="GB", operation_id="uk_frs_spine", pipeline_id="uk_national_spine", pipeline_version="2026.09", - candidate_id="uk-failed-v2-fixture", + candidate_id="uk-failed-v3-fixture", local_dir=root / "failed", run_kind="smoke", delivery_mode="local_only", @@ -102,15 +107,48 @@ def _failed(root: Path) -> None: recorder.stage("input_verification") recorder.fail( RuntimeError("/restricted/frs/adult.tab token=fixture-secret"), + error_code="BUILD_FAILED", + failure_class="error", local_diagnostic_reference="diagnostics/operator-error.txt", ) recorder.validate_local_bundle() +def _blocked(root: Path) -> None: + recorder = StagingRunBundleWriterV2( + run_id="uk-blocked-v3-fixture", + country_code="GB", + operation_id="uk_rowwise_candidate", + pipeline_id="uk_local_candidate", + pipeline_version="2026.10", + candidate_id="uk-blocked-v3-fixture", + local_dir=root / "blocked", + run_kind="calibration", + delivery_mode="local_only", + repo_id=None, + clock=FixtureClock(5), + ) + recorder.set_sample({"mode": "full"}) + recorder.stage("target_compilation", event_status="completed", target_count=4) + recorder.stage( + "gate_battery", + event_status="completed", + gate_statuses={"uk_target_fit": "failed", "uk_weight_ess": "passed"}, + blocking_failure_count=1, + diagnostic_failure_count=0, + ) + recorder.block( + phase="terminal", + blocking_gate_ids=["uk_target_fit"], + gate_statuses={"uk_target_fit": "failed", "uk_weight_ess": "passed"}, + ) + recorder.validate_local_bundle() + + def _contract_cases(root: Path) -> None: cases = { "schema_name": "microcosm.staging.fixture-cases", - "schema_version": 2, + "schema_version": 3, "delivery_success": { "contract_version": 2, "enabled": True, @@ -162,6 +200,7 @@ def _generate(root: Path) -> None: _completed_spine(root) _calibration(root) _failed(root) + _blocked(root) _contract_cases(root) files = sorted(path for path in root.rglob("*") if path.is_file()) checksums = [ @@ -186,11 +225,11 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) with tempfile.TemporaryDirectory(prefix="microcosm-staging-fixtures-") as temporary: - generated = Path(temporary) / "v2" + generated = Path(temporary) / "v3" _generate(generated) if args.check: if _tree(generated) != _tree(args.output): - raise SystemExit("staging version 2 fixtures are not reproducible") + raise SystemExit("staging version 3 fixtures are not reproducible") return 0 if args.output.exists(): shutil.rmtree(args.output) diff --git a/tools/preflight_uk_local_release_candidate.py b/tools/preflight_uk_local_release_candidate.py index 44edb9b3c..a5edef24d 100644 --- a/tools/preflight_uk_local_release_candidate.py +++ b/tools/preflight_uk_local_release_candidate.py @@ -62,9 +62,10 @@ def check_env( else: try: raw = base64.b64decode(key.strip(), validate=True) - if len(raw) < 32: + if len(raw) != 32: failures.append( - f"{SIGNING_KEY_ENV} decodes to {len(raw)} bytes; need ≥ 32." + f"{SIGNING_KEY_ENV} decodes to {len(raw)} bytes; the gate " + "battery signs only with exactly 32." ) except Exception: failures.append(f"{SIGNING_KEY_ENV} is not valid base64.")