From e9838914f602a6298fc1c79c479e0975771c9f25 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 16:03:59 -0400 Subject: [PATCH 1/7] Bind the numerical environment into each replay variant's identity A rebuilt environment for the same three wheels resolved other pydantic, pydantic-core and filelock versions and left out pypdf, and a raw RECORD digest also bound the installer (uv_cache.json, console-script shebangs that name the venv). Now: - identity.requirements_sha256 binds the exact freeze; verify_installed checks every pinned distribution at its version (extras allowed); - record_sha256 excludes installer bookkeeping, bytecode and generated scripts outside site-packages, so pip and uv installs of one wheel agree; - a freeze with local-wheel or commit installs is normalised to exact pins; - a variant id takes an identity-digest suffix only when another identity already holds its version slug; - compute_uk_event --variants-file selects from another variants document (a qualification run adds its candidate there). Schemas: requested_by.workflow_run_id is optional (a catch-up run started outside Actions has none); a failed receipt may have no execution block (a step failed before the candidate's identity was known). Both relax v1. Co-Authored-By: Claude Opus 5.5 --- data/uk/replay_variants.json | 18 ++-- docs/uk_replay/QUALIFICATION_INTERFACE.md | 6 +- docs/uk_replay/RELEASE_REPLAY.md | 9 +- .../schemas/qualification_request.v1.json | 2 +- docs/uk_replay/schemas/replay_receipt.v1.json | 28 ++++++- pipeline/compute_uk_event.py | 8 +- pipeline/uk_replay_variants.py | 83 ++++++++++++++++++- tests/test_uk_replay_variants.py | 28 +++++++ 8 files changed, 159 insertions(+), 23 deletions(-) diff --git a/data/uk/replay_variants.json b/data/uk/replay_variants.json index ab0a90a..60743ba 100644 --- a/data/uk/replay_variants.json +++ b/data/uk/replay_variants.json @@ -53,11 +53,11 @@ "packaged_bundle_manifest_sha256": "255dbb893f455d078a6bcece87399f0a22478f56bc4c6e526b758c4fd5b4db24", "packages": { "policyengine": { - "record_sha256": "5d5de97a40dbc525bc41a6347fc793ba772bd78f9e0630f2f1d382bc92575e68", + "record_sha256": "8d5eec548a5ce206a8fe59ef018542c701e26c0f927351f92339a7a57591b990", "version": "5.0.2" }, "policyengine-core": { - "record_sha256": "dd6570fe988159d89e45df66ccc43355f70abddb8a3a58d246ef38ce711e5687", + "record_sha256": "103128a1c2a3397268b65e33b83edddfa8e406a4eb64fb6f4ff2310db421c770", "version": "3.27.1" }, "policyengine-uk": { @@ -83,9 +83,10 @@ "model_package": "policyengine-uk", "model_version": "2.89.2", "policyengine_version": "5.0.2" - } + }, + "requirements_sha256": "450e64b173b66333027ac36e056e19928d23df1cbe2690e462bab09b31ddff9d" }, - "identity_sha256": "5c642304a4fff8f30074899b2bb530197920703add1b92304a4c3c3a48c5d176", + "identity_sha256": "3f1a33f31ac70a8f0beac2373b6408c8be3366128ec0d0e3c1b716de3880003c", "layout": "legacy_events", "note": "The bundle the track 1 replays were computed against. Its 1.3GB populace dataset needs Modal's 32GiB containers for full events.", "requirements": { @@ -145,12 +146,12 @@ "packaged_bundle_manifest_sha256": "8633ad814c9c85e53c7c513f69c4c7b1500a03cd2f6ef133d6e2ea8346b2db54", "packages": { "policyengine": { - "record_sha256": "072606d8f554123484caff5a1e0f12c3203b96d03f5990ebc4d80d6491441fd0", + "record_sha256": "f5990ac205b959621c56585c16ed963a5d1b07b1403adaa7242e0131fbf61f40", "version": "6.2.6", "wheel_sha256": "9ac20a2599cc1f4b50c128c2245042b105c40592e07fc25d39c99a529c9f410b" }, "policyengine-core": { - "record_sha256": "7a4e8a82c3467681971f5160428296b06e65357b8a70801f2cd631ea0943394a", + "record_sha256": "53a92f2e67982a16faeba61b8b3d145d3ab9a50d298e92c512d61166ffe5d5d4", "version": "3.33.0", "wheel_sha256": "2e32d02a8910ab683a2b7600987904b351d058fc2bd5910a8ea6641438bcd789" }, @@ -178,9 +179,10 @@ "model_package": "policyengine-uk", "model_version": "2.125.1", "policyengine_version": "6.2.6" - } + }, + "requirements_sha256": "5c333faef091e6780b2c178ac7584249ac7db0159841921727b913505577bcdb" }, - "identity_sha256": "f241374dea3ca00aeba8849437481da34e6c6c9ddb0bb9b75d3e72bbbe22fd69", + "identity_sha256": "3f62a208a9bf522cccb391fe65c1411124abaf4488ab760c522b0568f5b417aa", "layout": "replay_namespace", "note": "The newest certified pair on 11 October 2026; data built with policyengine-uk 2.122.2 and certified for 2.125.1.", "requirements": { diff --git a/docs/uk_replay/QUALIFICATION_INTERFACE.md b/docs/uk_replay/QUALIFICATION_INTERFACE.md index d616748..15b712a 100644 --- a/docs/uk_replay/QUALIFICATION_INTERFACE.md +++ b/docs/uk_replay/QUALIFICATION_INTERFACE.md @@ -102,8 +102,10 @@ Two flows use the same receiver: * `request_id` is not a field: it is the SHA-256 of the request's JCS bytes. Both sides test against the golden fixture. -* `requested_by` accepts additional provenance fields (for example - `run_attempt`); every other object is closed. +* `requested_by` names the requesting repository and commit; + `workflow_run_id` is optional (a scorecard catch-up run started outside + Actions has none), and additional provenance fields such as `run_attempt` + are accepted. Every other object is closed. * The wrapper is identified by `version` plus either a PyPI wheel (`{"pypi": {"wheel_sha256": …}}`) or, for an unreleased candidate whose regenerated manifest is not on PyPI yet, a public git commit. Scorecard diff --git a/docs/uk_replay/RELEASE_REPLAY.md b/docs/uk_replay/RELEASE_REPLAY.md index 769cdcb..3a4cc0c 100644 --- a/docs/uk_replay/RELEASE_REPLAY.md +++ b/docs/uk_replay/RELEASE_REPLAY.md @@ -81,14 +81,15 @@ A variant record: | Field | Meaning | |---|---| -| `variant_id` | `pe_uk_`; must equal the slug recomputed from `identity`. | +| `variant_id` | `pe_uk_`, recomputed from `identity`; when another identity already holds that slug (same versions, other wheels or numerical environment) the id adds `--`. | | `identity` | Everything that determines execution. Its canonical-JSON SHA-256 is `identity_sha256`. | -| `identity.packages` | For `policyengine`, `policyengine-uk`, `policyengine-core`: exact `version`, and `record_sha256`: the SHA-256 of the distribution's own `RECORD` entries, sorted, excluding installer bookkeeping (`INSTALLER`, `REQUESTED`, `RECORD`, `direct_url.json`) and bytecode. Every listed file is checked on disk against its recorded hash when the identity is read, so the digest binds the wheel's installed contents, and pip and uv installs of one wheel agree. Optional `wheel_sha256` when the wheel itself was hashed. | +| `identity.packages` | For `policyengine`, `policyengine-uk`, `policyengine-core`: exact `version`, and `record_sha256`: the SHA-256 of the distribution's own `RECORD` entries, sorted, excluding installer bookkeeping (`INSTALLER`, `REQUESTED`, `RECORD`, `direct_url.json`, `uv_cache.json`), bytecode, and the console-script wrappers the installer generates outside site-packages (their shebang names the environment's interpreter). Every listed file is checked on disk against its recorded hash when the identity is read, so the digest binds the wheel's installed contents, and pip and uv installs of one wheel agree. Optional `wheel_sha256` when the wheel itself was hashed. | | `identity.packaged_bundle_manifest_sha256` | SHA-256 of `policyengine/data/bundle/manifest.json` as installed. | | `identity.release_bundle` | The exact `pe.uk.uk_latest.release_bundle` dict expected at runtime under **packaged (offline) certification**. | | `identity.dataset` | `repo_id`, `repo_type`, `revision`, `path`, `sha256`, `size_bytes`, `uri`. | | `identity.materialization` | `hf_cache_mirror` (policyengine 5.x: the existing HF-cache plus `data/uk/policyengine-local` mirror) or `policyengine_data_dir` (6.x: a verified copy at `/data/`). | | `identity.python` | Interpreter minor version, e.g. `3.12`. | +| `identity.requirements_sha256` | The SHA-256 of the requirements freeze below: the numerical environment (numpy, pandas, h5py and the rest) is part of the identity, because it can move numbers. | | `requirements` | `{path, sha256}`: the exact freeze installed for this variant. Legacy: `docs/uk_replay/requirements.txt`. Others: `docs/uk_replay/requirements/.txt`. | | `data_years` | Calendar years the population supports: from the dataset's first `time_period` through the engine's extension end, as observed in `simulation.dataset.years`. | | `data_years_evidence` | The observed `time_period`, `dataset.years` and the engine file/line that sets the extension end. | @@ -115,7 +116,9 @@ failure: and `compatible_model_packages` agree with the record. 2. **Installed environment.** Installed versions of the three packages equal the record; every installed file matches its `RECORD` hash and the - normalised `RECORD` hashes to `record_sha256`; the + normalised `RECORD` hashes to `record_sha256`; every distribution pinned in + the requirements freeze is installed at its pinned version (extra + distributions, such as the receipt tools, are allowed); the installed packaged manifest hashes to `packaged_bundle_manifest_sha256`; the interpreter minor version matches. 3. **Certification path.** Before importing the engine the runner removes diff --git a/docs/uk_replay/schemas/qualification_request.v1.json b/docs/uk_replay/schemas/qualification_request.v1.json index 735b66c..16921af 100644 --- a/docs/uk_replay/schemas/qualification_request.v1.json +++ b/docs/uk_replay/schemas/qualification_request.v1.json @@ -105,7 +105,7 @@ "requested_by": { "type": "object", "additionalProperties": true, - "required": ["repository", "workflow_run_id", "commit"], + "required": ["repository", "commit"], "properties": { "repository": {"type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"}, "workflow_run_id": {"type": "integer", "minimum": 1}, diff --git a/docs/uk_replay/schemas/replay_receipt.v1.json b/docs/uk_replay/schemas/replay_receipt.v1.json index c5d8d9a..f127f28 100644 --- a/docs/uk_replay/schemas/replay_receipt.v1.json +++ b/docs/uk_replay/schemas/replay_receipt.v1.json @@ -429,10 +429,7 @@ "if": { "properties": { "status": { - "enum": [ - "complete", - "failed" - ] + "const": "complete" } } }, @@ -475,6 +472,29 @@ } } } + }, + { + "if": { + "properties": { + "status": { + "const": "failed" + } + } + }, + "then": { + "properties": { + "rejection": { + "type": "null" + }, + "integrity": { + "properties": { + "status": { + "const": "failed" + } + } + } + } + } } ], "$defs": { diff --git a/pipeline/compute_uk_event.py b/pipeline/compute_uk_event.py index aab3fca..e8e4b03 100644 --- a/pipeline/compute_uk_event.py +++ b/pipeline/compute_uk_event.py @@ -599,6 +599,12 @@ def main(argv: list[str] | None = None) -> int: "--variant", help="certified replay variant (data/uk/replay_variants.json); default: the file's default", ) + ap.add_argument( + "--variants-file", + type=Path, + default=variants.VARIANTS_PATH, + help="variant records to select from (a qualification run adds its candidate)", + ) ap.add_argument( "--dataset-file", type=Path, @@ -619,7 +625,7 @@ def main(argv: list[str] | None = None) -> int: help="spawn at most two year workers; each holds one live managed simulation", ) args = ap.parse_args(argv) - variant = variants.get_variant(args.variant) + variant = variants.get_variant(args.variant, path=args.variants_file.resolve()) legacy = variants.is_legacy(variant) variant_fields = variant_identity_fields(variant) default_registry, default_output = event_paths(args.event) diff --git a/pipeline/uk_replay_variants.py b/pipeline/uk_replay_variants.py index a2f1ee4..2513961 100644 --- a/pipeline/uk_replay_variants.py +++ b/pipeline/uk_replay_variants.py @@ -50,8 +50,16 @@ VERSION = re.compile(r"^[0-9]+(?:\.[0-9]+)*[a-z0-9.+-]*$") PIN = re.compile(r"([A-Za-z0-9][A-Za-z0-9_.-]*)==([A-Za-z0-9_.+!-]+)") # Installer bookkeeping differs between pip and uv for the same wheel; only -# the distribution's own files identify what was installed. -INSTALLER_METADATA = ("INSTALLER", "REQUESTED", "RECORD", "direct_url.json") +# the distribution's own files identify what was installed. Paths outside +# site-packages ("../../../bin/...") are console-script wrappers the +# installer generates, with the environment's interpreter in their shebang. +INSTALLER_METADATA = ( + "INSTALLER", + "REQUESTED", + "RECORD", + "direct_url.json", + "uv_cache.json", +) class VariantError(ValueError): @@ -91,6 +99,44 @@ def variant_slug(identity: dict) -> str: ) +def variant_id_for(identity: dict, taken: dict[str, str] | None = None) -> str: + """The version slug, suffixed with the identity digest when another + identity already holds that slug (same versions, other wheels or + numerical environment).""" + slug = variant_slug(identity) + digest = identity_sha256(identity) + holder = (taken or {}).get(slug) + return slug if holder in (None, digest) else f"{slug}--{digest[:10]}" + + +def normalized_freeze(text: str, versions: dict[str, str]) -> str: + """A freeze as exact pins. Engine packages installed from a verified local + wheel or a commit appear as ``name @ ``; they become ``name==version`` + (the wheel and installed-file digests bind them) with the source kept as a + comment. Any other non-pinned line refuses.""" + lines = [] + for line in text.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + name, at, source = stripped.partition(" @ ") + key = re.sub(r"[-_.]+", "-", name).lower() + if at and key in versions: + lines.append( + f"# {key} installed from {source.split('/')[-1] if source.startswith('file:') else source}" + ) + lines.append(f"{key}=={versions[key]}") + elif at: + raise VariantError(f"freeze line is not an exact pin: {stripped}") + else: + lines.append(stripped) + pins = requirements_pins("\n".join(lines)) + missing = sorted(set(versions) - set(pins)) + if missing: + raise VariantError(f"freeze lacks {missing}") + return "\n".join(lines) + "\n" + + def hf_uri(dataset: dict) -> str: return f"hf://{dataset['repo_id']}/{dataset['path']}@{dataset['revision']}" @@ -189,7 +235,11 @@ def need(condition: bool, message: str) -> None: uk = packages["policyengine-uk"]["version"] wrapper = packages["policyengine"]["version"] need( - record.get("variant_id") == variant_slug(identity), + record.get("variant_id") + in ( + variant_slug(identity), + f"{variant_slug(identity)}--{identity_sha256(identity)[:10]}", + ), "variant_id differs from its identity", ) need( @@ -280,6 +330,10 @@ def need(condition: bool, message: str) -> None: f"result_root must be {expected_root}", ) requirements = record.get("requirements") or {} + need( + identity.get("requirements_sha256") == requirements.get("sha256"), + "the identity binds another requirements file", + ) requirements_path = root / str(requirements.get("path", "")) if not requirements.get("path") or not requirements_path.is_file(): problems.append("requirements file is missing") @@ -342,6 +396,7 @@ def _record_rows(distribution) -> list[tuple[str, str, str]]: and name in INSTALLER_METADATA or "__pycache__/" in path or path.endswith(".pyc") + or path.startswith("../") ): continue rows.append((path, digest, size)) @@ -394,9 +449,25 @@ def installed_identity(*, verify_files: bool = True) -> dict: "packages": packages, "packaged_bundle_manifest_sha256": sha256_file(manifest), "python": f"{sys.version_info.major}.{sys.version_info.minor}", + "distributions": { + re.sub(r"[-_.]+", "-", d.metadata["Name"]).lower(): d.version + for d in importlib.metadata.distributions() + }, } +def environment_mismatches(pins: dict[str, str], installed: dict) -> list[str]: + """Every pinned distribution installed at its pinned version. Extra + distributions (the receipt tools) are allowed; a missing or different + numerical dependency is not.""" + have = installed.get("distributions", {}) + return [ + f"{name}: pinned {version}, installed {have.get(name)}" + for name, version in sorted(pins.items()) + if have.get(name) != version + ] + + def identity_mismatches(record: dict, installed: dict) -> list[str]: identity = record["identity"] out = [] @@ -424,9 +495,13 @@ def identity_mismatches(record: dict, installed: dict) -> list[str]: return out -def verify_installed(record: dict, *, installed: dict | None = None) -> dict: +def verify_installed( + record: dict, *, installed: dict | None = None, root: Path = ROOT +) -> dict: installed = installed if installed is not None else installed_identity() mismatches = identity_mismatches(record, installed) + pins = requirements_pins((root / record["requirements"]["path"]).read_text()) + mismatches += environment_mismatches(pins, installed) if mismatches: raise VariantError( f"{record['variant_id']}: installed environment is not the certified one: " diff --git a/tests/test_uk_replay_variants.py b/tests/test_uk_replay_variants.py index 7637a64..0e0ba24 100644 --- a/tests/test_uk_replay_variants.py +++ b/tests/test_uk_replay_variants.py @@ -178,9 +178,37 @@ def installed_for(r): }, "packaged_bundle_manifest_sha256": identity["packaged_bundle_manifest_sha256"], "python": identity["python"], + "distributions": rv.requirements_pins( + (rv.ROOT / r["requirements"]["path"]).read_text() + ), } +def test_every_pinned_distribution_is_part_of_the_installed_check(): + r = record() + exact = installed_for(r) + for name in ("numpy", "pandas", "h5py"): + changed = copy.deepcopy(exact) + changed["distributions"][name] = "0.0.0" + with pytest.raises(rv.VariantError, match=name): + rv.verify_installed(r, installed=changed) + missing = copy.deepcopy(exact) + del missing["distributions"][name] + with pytest.raises(rv.VariantError, match=name): + rv.verify_installed(r, installed=missing) + extra = copy.deepcopy(exact) + extra["distributions"]["rfc8785"] = "0.1.4" + rv.verify_installed(r, installed=extra) + + +def test_the_identity_binds_the_requirements_freeze(): + r = record() + r["identity"]["requirements_sha256"] = "5" * 64 + reseal(r) + with pytest.raises(rv.VariantError, match="another requirements file"): + rv.validate_record(r) + + def test_installed_environment_must_match_every_field(): r = record() exact = installed_for(r) From 1559150c21038ecb16596e06b8d318efaf866acc Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 16:04:37 -0400 Subject: [PATCH 2/7] Receive qualification requests, queue them durably and replay in CI .github/workflows/uk-replay.yml receives W4's uk-replay-qualification-request dispatches (and manual requests). The enqueue job writes each request to the replay-state branch first (pipeline/uk_replay_queue.py), so GitHub cancelling an older pending run never loses one; the drain job runs one request at a time, two simulation workers at most, and a schedule every three hours drains what is left. A request still running seven hours after its last update is retried. pipeline/uk_replay_qualify.py run-request takes one request end to end: schema check; the dataset from Hugging Face, checked by size and digest; the model and core wheels downloaded and checked against the request, then installed with the wrapper and, for a certified pair, that pair's exact freeze (a new candidate resolves within the newest certified freeze); a probe of the installed identity, the release bundle under packaged certification, the certification and the data years; the corpus; the diff against the newest other certified variant; and a sealed receipt (NAMESPACE_MANIFEST.json, REPLAY_RECEIPT.json). Every outcome, including a rejection or a crash, produces a receipt. The terminal STATUS.json names the receipt digest and the commit that added it. ci.yml now re-stages and re-compares every committed namespace, re-renders every committed diff (--check) and verifies every seal; and a replay engine job per certified variant installs its exact freeze, verifies the installed environment, runs the construction guards and regenerates the committed premises and diagnostics, and with the HUGGING_FACE_TOKEN secret replays one real measure-year and compares bytes. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/uk_replay_drain.sh | 65 ++ .github/workflows/ci.yml | 109 +++ .github/workflows/uk-replay.yml | 127 +++ docs/uk_replay/RECIPE.md | 7 + docs/uk_replay/RELEASE_REPLAY.md | 68 +- pipeline/uk_replay_qualify.py | 1169 ++++++++++++++++++++++++++++ pipeline/uk_replay_queue.py | 179 +++++ tests/test_uk_replay_qualify.py | 340 ++++++++ 8 files changed, 2063 insertions(+), 1 deletion(-) create mode 100755 .github/scripts/uk_replay_drain.sh create mode 100644 .github/workflows/uk-replay.yml create mode 100644 pipeline/uk_replay_qualify.py create mode 100644 pipeline/uk_replay_queue.py create mode 100644 tests/test_uk_replay_qualify.py diff --git a/.github/scripts/uk_replay_drain.sh b/.github/scripts/uk_replay_drain.sh new file mode 100755 index 0000000..18ec51a --- /dev/null +++ b/.github/scripts/uk_replay_drain.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Drain the UK replay qualification queue (see .github/workflows/uk-replay.yml). +# +# For each pending request, oldest first: mark it running, run it end to end +# (pipeline/uk_replay_qualify.py run-request), commit the receipt and results +# to the state branch, then write the terminal STATUS.json naming the +# receipt's digest and that commit. Stops when the queue is empty or the time +# budget is spent; the next scheduled run continues. +set -euo pipefail + +STATE=state +CONTROL=(uv run --with rfc8785==0.1.4 --with jsonschema --with huggingface_hub python) +BUDGET_SECONDS=$((300 * 60)) +started=$(date +%s) + +now() { date -u +%Y-%m-%dT%H:%M:%SZ; } + +push_state() { + git -C "$STATE" add -A + if git -C "$STATE" diff --cached --quiet; then + return 0 + fi + git -C "$STATE" commit -qm "$1" + for attempt in 1 2 3 4 5; do + if git -C "$STATE" push -q origin "HEAD:$STATE_BRANCH"; then + return 0 + fi + git -C "$STATE" pull -q --rebase origin "$STATE_BRANCH" + done + echo "could not push the state branch after five attempts" >&2 + return 1 +} + +while :; do + git -C "$STATE" pull -q --rebase origin "$STATE_BRANCH" + request_id=$("${CONTROL[@]}" -m pipeline.uk_replay_queue pending --state "$STATE" --now "$(now)" | head -n 1) + if [ -z "$request_id" ]; then + echo "queue empty" + break + fi + if [ $(( $(date +%s) - started )) -gt "$BUDGET_SECONDS" ]; then + echo "time budget spent; the next scheduled run continues" + break + fi + echo "::group::request $request_id" + "${CONTROL[@]}" -m pipeline.uk_replay_queue set-status --state "$STATE" \ + --request-id "$request_id" --status running --run-url "$RUN_URL" --now "$(now)" + push_state "Run $request_id" + + work="$RUNNER_TEMP/requests/$request_id" + outcome=$("${CONTROL[@]}" -m pipeline.uk_replay_qualify run-request \ + --request "$STATE/requests/$request_id/REQUEST.json" \ + --state-dir "$STATE" --work "$work" --runner "$REPLAY_RUNNER" | tail -n 1) + status=${outcome%% *} + receipt_sha256=${outcome##* } + push_state "Receipt for $request_id: $status" + receipt_commit=$(git -C "$STATE" rev-parse HEAD) + + "${CONTROL[@]}" -m pipeline.uk_replay_queue set-status --state "$STATE" \ + --request-id "$request_id" --status "$status" --run-url "$RUN_URL" --now "$(now)" \ + --receipt-sha256 "$receipt_sha256" --receipt-commit "$receipt_commit" + push_state "Status for $request_id: $status" + rm -rf "$work" + echo "::endgroup::" +done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd5132e..2c35025 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,41 @@ jobs: - run: uv run --with pytest --with hypothesis --with pypdf --with pyyaml --with numpy --with psutil --with rfc8785==0.1.4 --with jsonschema pytest tests/ -q env: PYTHONPATH: . + # REPLAY NAMESPACES: every committed event replay (the legacy + # results/uk/events and each results/uk/replays/) is re-staged + # and re-compared from its committed artifacts, every committed release + # diff is re-rendered, and every sealed namespace must match its + # receipt byte for byte. Engine-free; the no-drift gate below fails on + # any regenerated byte that differs. + - name: Replay namespaces regenerate and seals hold + env: + PYTHONPATH: . + run: | + set -euo pipefail + run() { uv run --with rfc8785==0.1.4 --with jsonschema python "$@"; } + for root in results/uk/events results/uk/replays/*/; do + [ -d "$root" ] || continue + case "$root" in */diffs/) continue ;; esac + found=0 + for manifest in "$root"/*/RUN_MANIFEST.json; do + [ -f "$manifest" ] || continue + found=1 + dir=$(dirname "$manifest"); event=$(basename "$dir") + run pipeline/stage_uk_event.py --event "$event" --artifact-dir "$dir" > /dev/null + run pipeline/compare_uk_event.py --event "$event" --staged "$dir/STAGED.jsonl" --output-dir "$dir" > /dev/null + done + if [ "$found" = 1 ]; then + run pipeline/compare_uk_event.py --summary --events-root "$root" > /dev/null + echo "regenerated $root" + fi + done + for diff in results/uk/replays/diffs/*/; do + [ -f "$diff/DIFF.json" ] || continue + base=$(run -c "import json,sys; print(json.load(open(sys.argv[1]))['base']['root'])" "$diff/DIFF.json") + head=$(run -c "import json,sys; print(json.load(open(sys.argv[1]))['head']['root'])" "$diff/DIFF.json") + run -m pipeline.diff_uk_event_runs --base "$base" --head "$head" --output-dir "$diff" --check + done + run -m pipeline.uk_replay_qualify verify-sealed - run: git diff --exit-code - name: Upload built database uses: actions/upload-artifact@v4 @@ -52,6 +87,80 @@ jobs: path: data/scorecard.db retention-days: 30 + replay-engine: + # The event replay against each certified variant's own engine: the + # exact environment from the variant's requirements freeze, the + # construction guards, and the committed premise and diagnostic files + # regenerated byte for byte. With the HUGGING_FACE_TOKEN secret it also + # replays one real measure-year on the variant's certified dataset and + # compares the artifact with the committed bytes (replay determinism); + # without the secret (forks) that step announces it did not run. + name: replay engine · ${{ matrix.variant }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + variant: + - pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z + - pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0 + env: + PYTHONPATH: . + VARIANT: ${{ matrix.variant }} + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + - name: Install the variant's exact environment + run: | + set -euo pipefail + field() { python3 -c "import json,sys; d=json.load(open('data/uk/replay_variants.json')); v=[v for v in d['variants'] if v['variant_id']==sys.argv[1]][0]; print(eval(sys.argv[2], {'v': v}))" "$VARIANT" "$1"; } + echo "REQUIREMENTS=$(field "v['requirements']['path']")" >> "$GITHUB_ENV" + echo "ENGINE=$(field "v['identity']['packages']['policyengine-uk']['version']")" >> "$GITHUB_ENV" + echo "RESULT_ROOT=$(field "v['result_root']")" >> "$GITHUB_ENV" + echo "LEGACY=$(field "v['layout'] == 'legacy_events'")" >> "$GITHUB_ENV" + - run: | + uv venv --python 3.12 .venv-variant + uv pip install --python .venv-variant/bin/python -r "$REQUIREMENTS" + uv pip install --python .venv-variant/bin/python rfc8785==0.1.4 jsonschema + - name: The installed environment is exactly the variant's + run: .venv-variant/bin/python -c "from pipeline import uk_replay_variants as rv; rv.verify_installed(rv.get_variant('$VARIANT')); print('installed environment matches $VARIANT')" + - name: Construction guards on the variant's engine + run: | + set -euo pipefail + for event in autumn_budget_2024 autumn_statement_2023 spring_budget_2024 spring_statement_2025 spring_budget_2023; do + .venv-variant/bin/python -m pipeline.compute_uk_event --event "$event" --variant "$VARIANT" --guards > /dev/null + echo "guards hold: $event" + done + - name: Construction premises regenerate + run: .venv-variant/bin/python -m pipeline.audit_uk_event_premises --all --expect-engine "$ENGINE" --output "$RESULT_ROOT/CONSTRUCTION_PREMISES.json" --check + - name: Re-verified diagnostics regenerate + if: env.LEGACY == 'False' + run: | + set -euo pipefail + .venv-variant/bin/python pipeline/diagnose_uk_event_models.py --reverify results/uk/events/MODEL_DIAGNOSTICS.json --expect-engine "$ENGINE" --output "$RUNNER_TEMP/MODEL_DIAGNOSTICS.json" + cmp "$RUNNER_TEMP/MODEL_DIAGNOSTICS.json" "$RESULT_ROOT/MODEL_DIAGNOSTICS.json" + - name: One real measure-year replays byte for byte + if: env.LEGACY == 'False' + env: + HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + run: | + set -euo pipefail + if [ -z "${HUGGING_FACE_TOKEN:-}" ]; then + echo "::notice::HUGGING_FACE_TOKEN not available (fork PR or unset secret): the real replay determinism check did NOT run" + exit 0 + fi + uv run --with huggingface_hub python - <<'PY' + import json, os + from huggingface_hub import hf_hub_download + d = json.load(open("data/uk/replay_variants.json")) + v = [v for v in d["variants"] if v["variant_id"] == os.environ["VARIANT"]][0]["identity"]["dataset"] + print(hf_hub_download(repo_id=v["repo_id"], repo_type=v["repo_type"], filename=v["path"], revision=v["revision"], token=os.environ["HUGGING_FACE_TOKEN"])) + PY + key=autumn_budget_2024__private_school_vat_20pct + .venv-variant/bin/python -m pipeline.compute_uk_event --event autumn_budget_2024 --variant "$VARIANT" \ + --measures "$key" --years 2026 --workers 1 --output-dir "$RUNNER_TEMP/determinism" + cmp "$RUNNER_TEMP/determinism/${key}_2026.json" "$RESULT_ROOT/autumn_budget_2024/${key}_2026.json" + echo "replayed $key 2026 byte-identical to the committed artifact" + app: name: typecheck · build runs-on: ubuntu-latest diff --git a/.github/workflows/uk-replay.yml b/.github/workflows/uk-replay.yml new file mode 100644 index 0000000..10b77d9 --- /dev/null +++ b/.github/workflows/uk-replay.yml @@ -0,0 +1,127 @@ +name: UK replay qualification + +# WHY THIS EXISTS +# +# Release qualification in policyengine.py (W4) asks this repository to rerun +# the OBR fiscal-event replay corpus on a candidate model/data pair before it +# is promoted, and to return an artifact-bound receipt. The interface is +# docs/uk_replay/QUALIFICATION_INTERFACE.md; the run is +# pipeline/uk_replay_qualify.py. +# +# GitHub keeps at most one pending run per concurrency group and cancels the +# older pending ones, so a request is first written to the persistent queue +# on the `replay-state` branch (enqueue job, no concurrency group). The drain +# job then runs queued requests one at a time. A scheduled run drains anything +# a cancelled run left behind. A request whose run died mid-way is retried +# after the stale bound in pipeline/uk_replay_queue.py. +# +# Simulations: one drain job at a time, at most two workers inside it, so +# never more than two simulations at once from this workflow. +# +# Secrets: HUGGING_FACE_TOKEN reads the certified datasets. Without it a +# request is rejected with data_access_failed rather than skipped. + +on: + repository_dispatch: + types: [uk-replay-qualification-request] + workflow_dispatch: + inputs: + request: + description: "A qualification request (JSON). Empty: drain the queue only." + required: false + default: "" + schedule: + - cron: "41 */3 * * *" + +permissions: + contents: write + +env: + STATE_BRANCH: replay-state + +jobs: + enqueue: + name: record the request + if: github.event_name == 'repository_dispatch' || (github.event_name == 'workflow_dispatch' && github.event.inputs.request != '') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + - name: Check out the state branch + run: | + set -euo pipefail + if git ls-remote --exit-code --heads origin "$STATE_BRANCH" >/dev/null; then + git fetch -q origin "$STATE_BRANCH" + git worktree add -q state "origin/$STATE_BRANCH" + git -C state switch -q -C "$STATE_BRANCH" + else + git worktree add -q --orphan -b "$STATE_BRANCH" state + fi + git -C state config user.name "github-actions[bot]" + git -C state config user.email "41898282+github-actions[bot]@users.noreply.github.com" + - name: Write the request + env: + DISPATCH_REQUEST: ${{ toJSON(github.event.client_payload.request) }} + MANUAL_REQUEST: ${{ github.event.inputs.request }} + run: | + set -euo pipefail + if [ "${{ github.event_name }}" = "repository_dispatch" ]; then + printf '%s' "$DISPATCH_REQUEST" > "$RUNNER_TEMP/request.json" + else + printf '%s' "$MANUAL_REQUEST" > "$RUNNER_TEMP/request.json" + fi + - name: Enqueue and push + env: + PYTHONPATH: . + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + now=$(date -u +%Y-%m-%dT%H:%M:%SZ) + uv run --with rfc8785==0.1.4 --with jsonschema python -m pipeline.uk_replay_queue \ + enqueue --state state --request "$RUNNER_TEMP/request.json" --run-url "$RUN_URL" --now "$now" \ + | tee "$RUNNER_TEMP/enqueued.txt" + git -C state add -A + if git -C state diff --cached --quiet; then + echo "already recorded: $(cat "$RUNNER_TEMP/enqueued.txt")" + exit 0 + fi + git -C state commit -qm "Queue $(cut -d' ' -f1 "$RUNNER_TEMP/enqueued.txt")" + for attempt in 1 2 3 4 5; do + git -C state push -q origin "HEAD:$STATE_BRANCH" && exit 0 + git -C state pull -q --rebase origin "$STATE_BRANCH" + done + exit 1 + + drain: + name: run queued requests + needs: [enqueue] + if: always() && (needs.enqueue.result == 'success' || needs.enqueue.result == 'skipped') + runs-on: ubuntu-latest + timeout-minutes: 340 + concurrency: + group: uk-replay-drain + cancel-in-progress: false + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + - name: Check out the state branch + run: | + set -euo pipefail + if ! git ls-remote --exit-code --heads origin "$STATE_BRANCH" >/dev/null; then + echo "no state branch yet; nothing queued" + echo "EMPTY=1" >> "$GITHUB_ENV" + exit 0 + fi + git fetch -q origin "$STATE_BRANCH" + git worktree add -q state "origin/$STATE_BRANCH" + git -C state switch -q -C "$STATE_BRANCH" + git -C state config user.name "github-actions[bot]" + git -C state config user.email "41898282+github-actions[bot]@users.noreply.github.com" + - name: Drain the queue + if: env.EMPTY != '1' + env: + PYTHONPATH: . + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + REPLAY_RUNNER: github-actions ${{ runner.os }} ${{ runner.arch }} + run: bash .github/scripts/uk_replay_drain.sh diff --git a/docs/uk_replay/RECIPE.md b/docs/uk_replay/RECIPE.md index a274129..675e268 100644 --- a/docs/uk_replay/RECIPE.md +++ b/docs/uk_replay/RECIPE.md @@ -422,6 +422,13 @@ PYTHONPATH=. .venv-candidate/bin/python -m pipeline.compute_uk_event --event aut PYTHONPATH=. .venv-candidate/bin/python pipeline/stage_uk_event.py --event autumn_budget_2024 --artifact-dir results/uk/replays/pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0/autumn_budget_2024 ``` +On the shared Mac, check the host reservation before a local run: +`python3 ~/reviews/fleet-ops/tools/reservation.py allowed w6-replay` exits 1 +while a reserved run (the 29 October–2 November triple-lock rebuild) holds +heavy local work back. Qualification runs from policyengine.py go through +the receiver workflow on GitHub runners instead; see +[RELEASE_REPLAY.md](RELEASE_REPLAY.md). + The dataset must already be local: the runner finds the certified bytes by SHA-256 in the Hugging Face cache, or takes `--dataset-file `, and copies them into `.venv-replay-checks/runtime//data/`. Results diff --git a/docs/uk_replay/RELEASE_REPLAY.md b/docs/uk_replay/RELEASE_REPLAY.md index 3a4cc0c..d0d3031 100644 --- a/docs/uk_replay/RELEASE_REPLAY.md +++ b/docs/uk_replay/RELEASE_REPLAY.md @@ -261,7 +261,73 @@ issues; this pipeline files none. ## Qualification requests and receipts See [QUALIFICATION_INTERFACE.md](QUALIFICATION_INTERFACE.md) for the request -W4 sends before promotion, the receipt returned, and the persistent queue. +W4 sends before promotion and the receipt returned. + +### The receiver + +`.github/workflows/uk-replay.yml` receives `repository_dispatch` events of +type `uk-replay-qualification-request` (and manual requests). Its `enqueue` +job writes the request to the persistent queue on the `replay-state` branch +before anything else (`pipeline/uk_replay_queue.py`), so GitHub cancelling +an older pending run never loses one. Its `drain` job, one at a time +(concurrency group `uk-replay-drain`), runs `.github/scripts/uk_replay_drain.sh`. +That script takes the oldest pending request, marks it running, and runs +`pipeline/uk_replay_qualify.py run-request`: + +1. Validate the request (`schema_invalid`). +2. Download the requested dataset with `HUGGING_FACE_TOKEN` into the Hugging + Face cache and check its size and SHA-256 (`data_access_failed`, + `artifact_hash_mismatch`). +3. Build the candidate environment: download the model and core wheels + without dependencies, check their SHA-256 against the request, then + install them, the wrapper (its PyPI wheel or its public commit) and the + receipt tools. Re-qualifying a certified pair installs that pair's exact + requirements freeze; a new candidate resolves within the newest certified + freeze, falling back to a free resolution only if its own pins conflict. +4. Probe inside that environment: the installed identity, the release + bundle under packaged certification (`release_bundle_mismatch`), the + certification naming this model and this dataset (`incompatible_pair`), + and the simulation's data years (`no_supported_years`). If the + environment reproduces a committed variant exactly, the run uses that + record; otherwise it builds a candidate record whose requirements are the + environment's normalised freeze. +5. Run the corpus: every event's compute, staging and comparison, the + summary, the construction premises, the re-verified diagnostics. +6. Diff against `compare_to` (default: the newest other certified variant + whose results are committed). +7. Seal the namespace (`NAMESPACE_MANIFEST.json`, then + `REPLAY_RECEIPT.json`) and copy it, the diff and the requirements to + `replay-state:requests//results/`. + +The receipt is committed first; the terminal `STATUS.json` follows in a +later commit naming the receipt's digest and that commit. A step that fails +before the candidate's identity is known yields a `failed` receipt with no +execution block; an unexpected exception also yields a `failed` receipt, so a +request never loops. A request still `running` seven hours after its last +update (a cancelled or crashed job) is retried. A scheduled run every three +hours drains whatever is left. + +At most one drain job runs, with at most two simulation workers, so this +workflow never runs more than two simulations at once. These runs use GitHub +runners; manual replays on the shared Mac also respect the host reservation +(`python3 ~/reviews/fleet-ops/tools/reservation.py allowed w6-replay`), which +holds heavy local work back from 29 October to 2 November. + +### Replay in CI + +`ci.yml` runs two kinds of replay check on every pull request and push: + +* Engine-free, in the main job: every committed namespace (legacy and each + variant) is re-staged and re-compared from its artifacts, every committed + release diff is re-rendered with `--check`, and every sealed namespace must + match its receipt byte for byte; the no-drift gate fails on any change. +* `replay engine`, once per certified variant: installs the variant's exact + freeze, checks the installed environment against the record, runs every + event's construction guards, regenerates the committed construction + premises (and, for non-legacy variants, the re-verified diagnostics) byte + for byte, and, with the `HUGGING_FACE_TOKEN` secret, replays one real + measure-year on the certified dataset and compares the artifact with the + committed bytes. Without the secret that step says it did not run. ## Invariants diff --git a/pipeline/uk_replay_qualify.py b/pipeline/uk_replay_qualify.py new file mode 100644 index 0000000..7326515 --- /dev/null +++ b/pipeline/uk_replay_qualify.py @@ -0,0 +1,1169 @@ +"""Qualify a candidate model/data pair by replaying the event corpus on it. + +A qualification request (docs/uk_replay/QUALIFICATION_INTERFACE.md) names a +candidate exactly. This module turns it into a candidate replay variant, or a +rejection with a code; runs the supported corpus on it; diffs the run against +a base variant; and seals the namespace with an artifact-bound receipt. + +Steps, each a subcommand so the receiver workflow can run them in the right +environment: + +* ``plan``: validate the request and print the install commands for its + exact wheels (control environment; needs rfc8785 and jsonschema). +* ``probe``: inside the candidate environment, after installation, verify + the installed identity against the request, materialise the dataset and + read the release bundle, certification and data years. +* ``corpus``: inside the candidate environment, run compute, staging and + comparison for every event, then premises, diagnostics and the diff. +* ``receipt``: build, validate and seal the receipt; write + NAMESPACE_MANIFEST.json and REPLAY_RECEIPT.json. + +Nothing here decides whether a candidate is good: the receipt reports +integrity and descriptive diff flags; the promotion policy is release +qualification's. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parent.parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from pipeline import uk_replay_variants as variants # noqa: E402 + +EVENTS = ( + "autumn_budget_2024", + "autumn_statement_2023", + "spring_budget_2024", + "spring_statement_2025", + "spring_budget_2023", +) +CORPUS_VERSION = "uk-fiscal-event-replay/1" +REJECTION_CODES = ( + "schema_invalid", + "artifact_hash_mismatch", + "release_bundle_mismatch", + "incompatible_pair", + "data_access_failed", + "no_supported_years", +) +MANIFEST_NAME = "NAMESPACE_MANIFEST.json" +POLICYENGINE_GIT = "https://github.com/PolicyEngine/policyengine.py" +# Run in the candidate environment beside the engine, never resolved freely. +RUNTIME_TOOLS = ("pytest", "hypothesis", "rfc8785==0.1.4", "jsonschema") + + +class Rejected(Exception): + """The request cannot be qualified; the receipt carries the code.""" + + def __init__(self, code: str, detail: str): + if code not in REJECTION_CODES: + raise ValueError(f"unknown rejection code {code}") + super().__init__(f"{code}: {detail}") + self.code = code + self.detail = detail + + +def canonical_bytes(value: Any) -> bytes: + return ( + json.dumps(value, indent=1, sort_keys=True, allow_nan=False) + "\n" + ).encode() + + +def sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +# --- the request ------------------------------------------------------------------ + + +def request_id(request: dict) -> str: + """Validate against the v1 schema; schema failures are rejections.""" + from pipeline import uk_replay_interface as interface + + try: + return interface.validate_request(request) + except Exception as exc: # noqa: BLE001 - jsonschema's errors, digest errors + raise Rejected("schema_invalid", str(exc).splitlines()[0]) from exc + + +def download_plan(request: dict, venv: Path, wheels: Path) -> list[list[str]]: + """Create the candidate environment and download the pinned wheels, + without dependencies, for ``verify_wheels`` to check.""" + candidate = request["candidate"] + python = str(venv / "bin" / "python") + pinned = [ + f"policyengine-uk=={candidate['policyengine_uk']['version']}", + f"policyengine-core=={candidate['policyengine_core']['version']}", + ] + if "pypi" in candidate["policyengine"]["source"]: + pinned.append(f"policyengine=={candidate['policyengine']['version']}") + return [ + ["uv", "venv", "--python", "3.12", str(venv)], + ["uv", "pip", "install", "--python", python, "pip"], + [ + python, + "-m", + "pip", + "download", + "--no-deps", + "--only-binary", + ":all:", + "-d", + str(wheels), + *pinned, + ], + ] + + +def requested_versions(request: dict) -> dict[str, str]: + candidate = request["candidate"] + return { + "policyengine": candidate["policyengine"]["version"], + "policyengine-uk": candidate["policyengine_uk"]["version"], + "policyengine-core": candidate["policyengine_core"]["version"], + } + + +def matching_certified(request: dict, document: dict) -> dict | None: + """A committed certified variant of exactly this pair, if any: same + versions, packaged manifest and dataset, and the same wheel digests where + both record them.""" + candidate = request["candidate"] + wheels = { + "policyengine-uk": candidate["policyengine_uk"]["wheel_sha256"], + "policyengine-core": candidate["policyengine_core"]["wheel_sha256"], + } + if "pypi" in candidate["policyengine"]["source"]: + wheels["policyengine"] = candidate["policyengine"]["source"]["pypi"][ + "wheel_sha256" + ] + for record in document["variants"]: + identity = record["identity"] + versions = {n: p["version"] for n, p in identity["packages"].items()} + if ( + record["status"] == "certified" + and versions == requested_versions(request) + and identity["packaged_bundle_manifest_sha256"] + == candidate["policyengine"]["packaged_bundle_manifest_sha256"] + and identity["dataset"]["sha256"] == candidate["data"]["sha256"] + and all( + identity["packages"][name].get("wheel_sha256", digest) == digest + for name, digest in wheels.items() + ) + ): + return record + return None + + +def latest_certified(document: dict) -> dict: + """The newest certified variant: the file is append-only, newest last.""" + return [r for r in document["variants"] if r["status"] == "certified"][-1] + + +def environment_lines(record: dict, *, root: Path = ROOT) -> list[str]: + """A variant's exact pins without the three engine packages.""" + pins = variants.requirements_pins( + (root / record["requirements"]["path"]).read_text() + ) + return [ + f"{name}=={version}" + for name, version in sorted(pins.items()) + if name not in variants.PACKAGES + ] + + +def install_command( + request: dict, + venv: Path, + wheels: Path, + *, + environment: Path | None = None, + exact: bool = False, +) -> list[str]: + """Install the verified local wheels, the wrapper from its public commit + when it is not a PyPI wheel, and the receipt tools. Re-qualifying a + certified pair installs that pair's exact numerical environment (``-r``); + a new candidate resolves within the newest certified one (``-c``). The + installed RECORD digests then bind the installed contents.""" + source = request["candidate"]["policyengine"]["source"] + command = [ + "uv", + "pip", + "install", + "--python", + str(venv / "bin" / "python"), + *sorted(str(path) for path in wheels.glob("*.whl")), + ] + if "git" in source: + command.append( + f"policyengine @ git+{POLICYENGINE_GIT}@{source['git']['commit']}" + ) + if environment is not None: + command += ["-r" if exact else "-c", str(environment)] + return command + list(RUNTIME_TOOLS) + + +def prepare( + request: dict, venv: Path, wheels: Path, *, root: Path = ROOT +) -> dict[str, str]: + """Build the candidate environment from exactly the requested wheels.""" + wheels.mkdir(parents=True, exist_ok=True) + for command in download_plan(request, venv, wheels): + run(command) + digests = verify_wheels(request, wheels) + document = variants.load_document(root / "data" / "uk" / "replay_variants.json") + match = matching_certified(request, document) + environment = wheels.parent / f"{wheels.name}-environment.txt" + environment.write_text( + "\n".join(environment_lines(match or latest_certified(document), root=root)) + + "\n" + ) + try: + run( + install_command( + request, venv, wheels, environment=environment, exact=bool(match) + ) + ) + except subprocess.CalledProcessError: + if match is not None: + raise + # The candidate's own requirements conflict with the newest certified + # environment: resolve freely; the recorded freeze shows what moved. + run(install_command(request, venv, wheels)) + return digests + + +def verify_wheels(request: dict, wheels: Path) -> dict[str, str]: + """Each downloaded wheel's SHA-256 against the request.""" + candidate = request["candidate"] + expected = { + "policyengine_uk": candidate["policyengine_uk"], + "policyengine_core": candidate["policyengine_core"], + } + if "pypi" in candidate["policyengine"]["source"]: + expected["policyengine"] = { + "version": candidate["policyengine"]["version"], + "wheel_sha256": candidate["policyengine"]["source"]["pypi"]["wheel_sha256"], + } + found = {} + for path in sorted(wheels.glob("*.whl")): + name, version = path.name.split("-")[:2] + found[name.lower()] = (version, variants.sha256_file(path)) + out = {} + for name, want in expected.items(): + version, digest = found.get(name, (None, None)) + if version != want["version"] or digest != want["wheel_sha256"]: + raise Rejected( + "artifact_hash_mismatch", + f"{name} wheel {version} sha256 {digest} is not the requested " + f"{want['version']} {want['wheel_sha256']}", + ) + out[name] = digest + return out + + +# --- probing the installed candidate --------------------------------------------------- + + +def probe( + request: dict, runtime_root: Path, *, dataset_file: Path | None = None +) -> dict: + """Run inside the candidate environment. Engine-dependent.""" + import importlib.metadata + + candidate = request["candidate"] + installed = variants.installed_identity() + want = { + "policyengine": candidate["policyengine"]["version"], + "policyengine-uk": candidate["policyengine_uk"]["version"], + "policyengine-core": candidate["policyengine_core"]["version"], + } + have = {name: row["version"] for name, row in installed["packages"].items()} + if have != want: + raise Rejected("artifact_hash_mismatch", f"installed {have}, requested {want}") + if ( + installed["packaged_bundle_manifest_sha256"] + != candidate["policyengine"]["packaged_bundle_manifest_sha256"] + ): + raise Rejected( + "artifact_hash_mismatch", + "installed packaged bundle manifest differs from the request", + ) + variants.isolate_certification_environment() + data = candidate["data"] + dataset_identity = { + key: data[key] + for key in ("repo_id", "repo_type", "revision", "path", "sha256", "size_bytes") + } + record_stub = { + "variant_id": "probe", + "identity": { + "dataset": dataset_identity, + "materialization": "policyengine_data_dir", + }, + } + try: + dataset = variants.materialize_dataset( + record_stub, runtime_root, source=dataset_file + ) + except variants.VariantError as exc: + raise Rejected("data_access_failed", str(exc)) from exc + os.chdir(runtime_root) + import policyengine as pe + + bundle = dict(pe.uk.uk_latest.release_bundle) + if bundle != candidate["expected_release_bundle"]: + differences = sorted( + key + for key in set(bundle) | set(candidate["expected_release_bundle"]) + if bundle.get(key) != candidate["expected_release_bundle"].get(key) + ) + raise Rejected("release_bundle_mismatch", f"fields differ: {differences}") + certification = pe.uk.uk_latest.data_certification + certified_for = getattr(certification, "certified_for_model_version", None) + if certified_for != want["policyengine-uk"]: + raise Rejected( + "incompatible_pair", + f"the packaged certification names policyengine-uk {certified_for}", + ) + if ( + bundle.get("certified_data_artifact_sha256") != data["sha256"] + or bundle.get("certified_data_build_id") != data["build_id"] + or bundle.get("default_dataset_uri") != variants.hf_uri(dataset_identity) + ): + raise Rejected( + "incompatible_pair", "the dataset is not the bundle's certified artifact" + ) + simulation = pe.uk.managed_microsimulation() + years = sorted(int(year) for year in simulation.dataset.years) + del simulation + return { + "installed": installed, + "release_bundle": bundle, + "certification": { + "source": ( + f"policyengine.py {want['policyengine']} packaged bundle manifest " + f"(qualification request; {json.dumps(candidate['policyengine']['source'], sort_keys=True)})" + ), + "compatibility_basis": bundle.get("compatibility_basis"), + "certified_for_model_version": certified_for, + "data_build_id": bundle.get("certified_data_build_id"), + }, + "dataset": dataset_identity, + "dataset_path": str(dataset), + "data_years": years, + "python": platform.python_version(), + "platform": platform.platform(), + "freeze": subprocess.run( + ["uv", "pip", "freeze", "--python", sys.executable], + check=True, + capture_output=True, + text=True, + ).stdout, + "versions": {name: importlib.metadata.version(name) for name in want}, + } + + +def candidate_record( + request: dict, probed: dict, *, root: Path = ROOT, wheel_sha256: dict | None = None +) -> dict: + """The variant this request runs as. + + A committed variant whose identity the installed environment reproduces + exactly (packages, manifest, bundle, dataset and every pinned + distribution) is returned unchanged. Otherwise a candidate record is + built, with its normalised freeze written as its requirements.""" + installed = probed["installed"] + document = variants.load_document(root / "data" / "uk" / "replay_variants.json") + + def engine(packages: dict) -> dict: + return { + name: {"version": row["version"], "record_sha256": row["record_sha256"]} + for name, row in packages.items() + } + + for record in document["variants"]: + identity = record["identity"] + pins = variants.requirements_pins( + (root / record["requirements"]["path"]).read_text() + ) + if ( + engine(identity["packages"]) == engine(installed["packages"]) + and identity["release_bundle"] == probed["release_bundle"] + and identity["dataset"] == probed["dataset"] + and not variants.identity_mismatches(record, installed) + and not variants.environment_mismatches(pins, installed) + ): + return record + packages = {} + for name in variants.PACKAGES: + packages[name] = dict(installed["packages"][name]) + key = name.replace("-", "_") + if wheel_sha256 and key in wheel_sha256: + packages[name]["wheel_sha256"] = wheel_sha256[key] + freeze = variants.normalized_freeze( + probed["freeze"], {n: p["version"] for n, p in installed["packages"].items()} + ) + identity = { + "packages": packages, + "packaged_bundle_manifest_sha256": installed["packaged_bundle_manifest_sha256"], + "release_bundle": probed["release_bundle"], + "dataset": probed["dataset"], + "materialization": "policyengine_data_dir", + "python": installed["python"], + "requirements_sha256": sha256_bytes(freeze.encode()), + } + taken = { + variants.variant_slug(r["identity"]): r["identity_sha256"] + for r in document["variants"] + } + variant_id = variants.variant_id_for(identity, taken) + requirements = root / "docs" / "uk_replay" / "requirements" / f"{variant_id}.txt" + if requirements.exists() and requirements.read_text() != freeze: + raise RuntimeError(f"{requirements} already holds another environment") + requirements.parent.mkdir(parents=True, exist_ok=True) + requirements.write_text(freeze) + years = probed["data_years"] + record = { + "variant_id": variant_id, + "identity_sha256": variants.identity_sha256(identity), + "identity": identity, + "status": "candidate" if request["kind"] == "candidate" else "certified", + "layout": "replay_namespace", + "result_root": f"{variants.REPLAY_RESULT_ROOT}/{variant_id}", + "requirements": { + "path": str(requirements.relative_to(root)), + "sha256": variants.sha256_file(requirements), + }, + "data_years": list(range(years[0], years[-1] + 1)), + "data_years_evidence": { + "simulation_dataset_years": years, + "source": "Measured by pipeline/uk_replay_qualify.py probe in the candidate environment.", + }, + "certification": probed["certification"], + "executor": ["local"], + "qualification_request_sha256": request_id(request), + } + variants.validate_record(record, root=root) + registry_years = set() + for event in EVENTS: + registry = json.loads( + (root / f"data/uk/events/{event}_measures.json").read_bytes() + ) + registry_years |= set(registry["calendar_years"]) + if not variants.supported_years(record, registry_years)[0]: + raise Rejected("no_supported_years", f"data years {years} meet no event year") + return record + + +# --- the corpus ------------------------------------------------------------------------ + + +def run(command: list[str], *, cwd: Path = ROOT) -> None: + print("+", " ".join(command), flush=True) + subprocess.run( + command, check=True, cwd=cwd, env={**os.environ, "PYTHONPATH": str(ROOT)} + ) + + +def run_corpus( + record: dict, + variants_file: Path, + *, + workers: int = 2, + dataset_file: Path | None = None, + events: tuple[str, ...] = EVENTS, +) -> list[dict]: + """Compute, stage and compare every event; returns integrity failures.""" + python = sys.executable + namespace = variants.namespace_root(record) + failures = [] + for event in events: + output = namespace / event + try: + command = [ + python, + "-m", + "pipeline.compute_uk_event", + "--event", + event, + "--variant", + record["variant_id"], + "--variants-file", + str(variants_file), + "--workers", + str(workers), + "--output-dir", + str(output), + ] + if dataset_file is not None: + command += ["--dataset-file", str(dataset_file)] + if (output / "RUN_MANIFEST.json").exists() or any( + output.glob("RUN_PROGRESS_*.json") + ): + command.append("--resume") + run(command) + run( + [ + python, + "pipeline/stage_uk_event.py", + "--event", + event, + "--artifact-dir", + str(output), + ] + ) + run( + [ + python, + "pipeline/compare_uk_event.py", + "--event", + event, + "--staged", + str(output / "STAGED.jsonl"), + "--output-dir", + str(output), + ] + ) + except subprocess.CalledProcessError as exc: + failures.append( + {"code": "event_run_failed", "detail": str(exc), "event": event} + ) + run( + [ + python, + "pipeline/compare_uk_event.py", + "--summary", + "--events-root", + str(namespace), + ] + ) + engine = record["identity"]["packages"]["policyengine-uk"]["version"] + run( + [ + python, + "-m", + "pipeline.audit_uk_event_premises", + "--all", + "--expect-engine", + engine, + "--output", + str(namespace / "CONSTRUCTION_PREMISES.json"), + ] + ) + run( + [ + python, + "pipeline/diagnose_uk_event_models.py", + "--reverify", + str(ROOT / "results/uk/events/MODEL_DIAGNOSTICS.json"), + "--expect-engine", + engine, + "--output", + str(namespace / "MODEL_DIAGNOSTICS.json"), + ] + ) + support = {} + for event in events: + manifest_path = namespace / event / "RUN_MANIFEST.json" + if manifest_path.exists(): + manifest = json.loads(manifest_path.read_bytes()) + support[event] = { + "supported_years": manifest["supported_years"], + "unsupported_years": manifest["unsupported_years"], + } + (namespace / "SUPPORT.json").write_bytes(canonical_bytes(support)) + return failures + + +# --- receipts and sealing ------------------------------------------------------------ + + +def namespace_files(namespace: Path) -> dict[str, str]: + return { + str(path.relative_to(namespace)): variants.sha256_file(path) + for path in sorted(namespace.rglob("*")) + if path.is_file() and path.name not in (MANIFEST_NAME, variants.RECEIPT_NAME) + } + + +def event_results(record: dict, events=EVENTS) -> tuple[dict, list[dict]]: + namespace = variants.namespace_root(record) + out, failures = {}, [] + for event in events: + directory = namespace / event + try: + manifest_bytes = (directory / "RUN_MANIFEST.json").read_bytes() + staging_bytes = (directory / "STAGING_MANIFEST.json").read_bytes() + provenance_bytes = (directory / "COMPARISON_PROVENANCE.json").read_bytes() + except FileNotFoundError as exc: + failures.append( + {"code": "event_outputs_missing", "detail": str(exc), "event": event} + ) + continue + manifest, staging = json.loads(manifest_bytes), json.loads(staging_bytes) + out[event] = { + "registry_sha256": manifest["registry_sha256"], + "supported_years": manifest["supported_years"], + "unsupported_years": manifest["unsupported_years"], + "run_manifest_sha256": sha256_bytes(manifest_bytes), + "staging_manifest_sha256": sha256_bytes(staging_bytes), + "comparison_provenance_sha256": sha256_bytes(provenance_bytes), + "artifacts": len(manifest["artifacts"]), + "full_supported_grid_complete": bool( + staging.get("full_supported_grid_complete") + ), + "inert_measure_years": staging.get("inert_measure_years", []), + } + return out, failures + + +def diff_summary(diff_dir: Path) -> dict: + payload = (diff_dir / "DIFF.json").read_bytes() + diff = json.loads(payload) + return { + "base_variant_id": diff["base"]["identity"]["variant_id"], + "diff_sha256": sha256_bytes(payload), + "flag_counts": diff["flag_counts"], + "flagged_rows": [ + { + "event": row["event"], + "measure_key": row["measure_key"], + "year": row["year"], + "level": row["level"], + "key": row["key"], + "flags": row["flags"], + "base_gbp": row["base_gbp"], + "head_gbp": row["head_gbp"], + } + for row in diff["rows"] + if row["flags"] + ], + } + + +def build_receipt( + request: dict, + *, + record: dict | None, + rejection: Rejected | None = None, + failures: list[dict] | None = None, + diff_dir: Path | None = None, + results_path: str, + ref: str = "replay-state", + runner: str = "local", +) -> dict: + from pipeline import uk_replay_interface as interface + + try: + rid = interface.request_id(request) + except Exception: # noqa: BLE001 - an unparseable request still gets a receipt + rid = interface.jcs_sha256(request) + failures = list(failures or []) + receipt = { + "schema": interface.RECEIPT_SCHEMA_ID, + "request_id": rid, + "status": "rejected" if rejection else "complete", + "rejection": {"code": rejection.code, "detail": rejection.detail} + if rejection + else None, + "variant_id": record["variant_id"] if record else None, + "variant_identity_sha256": record["identity_sha256"] if record else None, + "execution": None, + "corpus": None, + "integrity": {"status": "not_run", "failures": []}, + "diff": None, + "results": { + "repository": "PolicyEngine/policyengine-scorecard", + "ref": ref, + "path": results_path, + }, + } + if record is None and rejection is None: + # A step failed before the candidate's identity was established. + receipt["status"] = "failed" + receipt["integrity"] = {"status": "failed", "failures": failures} + if record is not None and rejection is None: + identity = record["identity"] + receipt["execution"] = { + "packages": identity["packages"], + "packaged_bundle_manifest_sha256": identity[ + "packaged_bundle_manifest_sha256" + ], + "release_bundle": identity["release_bundle"], + "dataset_sha256": identity["dataset"]["sha256"], + "requirements_sha256": record["requirements"]["sha256"], + "python": platform.python_version(), + "platform": platform.platform(), + "runner": runner, + } + events, missing = event_results(record) + failures += missing + receipt["corpus"] = {"corpus_version": CORPUS_VERSION, "events": events} + complete = events and all( + e["full_supported_grid_complete"] for e in events.values() + ) + receipt["integrity"] = { + "status": "failed" + if failures + else "complete" + if complete + else "incomplete", + "failures": failures, + } + if failures: + receipt["status"] = "failed" + if diff_dir is not None and (diff_dir / "DIFF.json").exists(): + receipt["diff"] = diff_summary(diff_dir) + return receipt + + +def seal(namespace: Path, receipt: dict, *, diff_dir: Path | None = None) -> dict: + """Bind every namespace file (and the diff) and write the sealed receipt.""" + from pipeline import uk_replay_interface as interface + + manifest = { + "schema_version": 1, + "files": namespace_files(namespace), + "diff_files": { + str(path.relative_to(diff_dir)): variants.sha256_file(path) + for path in sorted(diff_dir.rglob("*")) + if path.is_file() + } + if diff_dir is not None and diff_dir.is_dir() + else {}, + } + payload = canonical_bytes(manifest) + (namespace / MANIFEST_NAME).write_bytes(payload) + receipt = dict(receipt) + receipt["results"] = { + **receipt["results"], + "manifest_sha256": sha256_bytes(payload), + } + sealed = interface.seal_receipt(receipt) + interface.validate(sealed, interface.RECEIPT_SCHEMA_ID) + (namespace / variants.RECEIPT_NAME).write_bytes(canonical_bytes(sealed)) + return sealed + + +def verify_sealed(namespace: Path, *, diff_dir: Path | None = None) -> list[str]: + """Problems with a sealed namespace; empty when every byte matches.""" + from pipeline import uk_replay_interface as interface + + problems = [] + receipt_path = namespace / variants.RECEIPT_NAME + manifest_path = namespace / MANIFEST_NAME + if not receipt_path.is_file() or not manifest_path.is_file(): + return [f"{namespace}: not sealed"] + receipt = json.loads(receipt_path.read_bytes()) + try: + interface.validate(receipt, interface.RECEIPT_SCHEMA_ID) + except Exception as exc: # noqa: BLE001 + problems.append(f"{receipt_path}: {exc}") + manifest_bytes = manifest_path.read_bytes() + if receipt.get("results", {}).get("manifest_sha256") != sha256_bytes( + manifest_bytes + ): + problems.append(f"{manifest_path}: digest differs from the receipt") + manifest = json.loads(manifest_bytes) + actual = namespace_files(namespace) + if actual != manifest["files"]: + changed = sorted( + set(actual.items()) ^ set(manifest["files"].items()), + key=lambda item: item[0], + ) + problems.append( + f"{namespace}: files differ from the seal: {[c[0] for c in changed][:10]}" + ) + if diff_dir is not None and manifest.get("diff_files"): + actual_diff = { + str(path.relative_to(diff_dir)): variants.sha256_file(path) + for path in sorted(diff_dir.rglob("*")) + if path.is_file() + } + if actual_diff != manifest["diff_files"]: + problems.append(f"{diff_dir}: files differ from the seal") + return problems + + +# --- one request, end to end --------------------------------------------------------- + +PROBE_REJECTED = 3 + + +def fetch_dataset(request: dict) -> Path: + """Download the requested artifact into the Hugging Face cache with the + receiver's token; probe later finds it there by digest.""" + from huggingface_hub import hf_hub_download + + data = request["candidate"]["data"] + token = os.environ.get("HUGGING_FACE_TOKEN") or os.environ.get("HF_TOKEN") + try: + path = Path( + hf_hub_download( + repo_id=data["repo_id"], + repo_type=data["repo_type"], + filename=data["path"], + revision=data["revision"], + token=token or None, + ) + ) + except Exception as exc: # noqa: BLE001 - authentication, absence, network + raise Rejected( + "data_access_failed", f"{type(exc).__name__}: {exc}".splitlines()[0] + ) from exc + if ( + path.stat().st_size != data["size_bytes"] + or variants.sha256_file(path) != data["sha256"] + ): + raise Rejected( + "artifact_hash_mismatch", f"{data['path']} bytes differ from the request" + ) + return path + + +def base_namespace(request: dict, record: dict, document: dict) -> Path | None: + """The namespace to diff against: compare_to, or the newest certified + variant other than this one whose results are present.""" + wanted = request.get("compare_to", "latest_certified") + candidates = ( + [r for r in document["variants"] if r["variant_id"] == wanted] + if wanted != "latest_certified" + else [ + r + for r in reversed(document["variants"]) + if r["status"] == "certified" and r["variant_id"] != record["variant_id"] + ] + ) + for candidate in candidates: + root = variants.namespace_root(candidate) + if any(root.glob("*/COMPARISON.json")): + return root + return None + + +def run_request(request: dict, state_dir: Path, work: Path, *, runner: str) -> dict: + """Qualify one request and write its receipt and results to the state + checkout. Every outcome, including a rejection, produces a receipt.""" + import shutil + + from pipeline import uk_replay_interface as interface + + try: + rid = request_id(request) + except Rejected: + rid = interface.jcs_sha256(request) + destination = state_dir / "requests" / rid + results_path = f"requests/{rid}/results/" + work.mkdir(parents=True, exist_ok=True) + venv, wheels = work / "venv", work / "wheels" + variants_file = work / "variants.json" + record = rejection = diff_dir = None + failures: list[dict] = [] + try: + request_id(request) + fetch_dataset(request) + digests = prepare(request, venv, wheels) + (work / "wheels.json").write_bytes(canonical_bytes(digests)) + python = str(venv / "bin" / "python") + (work / "request.json").write_bytes(canonical_bytes(request)) + probe_run = subprocess.run( + [ + python, + "-m", + "pipeline.uk_replay_qualify", + "probe", + "--request", + str(work / "request.json"), + "--runtime-root", + str(work / "runtime"), + "--wheel-digests", + str(work / "wheels.json"), + "--variants-file", + str(variants_file), + "--rejection-file", + str(work / "rejection.json"), + ], + cwd=ROOT, + env={**os.environ, "PYTHONPATH": str(ROOT)}, + capture_output=True, + text=True, + ) + if probe_run.returncode == PROBE_REJECTED: + refused = json.loads((work / "rejection.json").read_bytes()) + raise Rejected(refused["code"], refused["detail"]) + if probe_run.returncode != 0: + raise subprocess.CalledProcessError( + probe_run.returncode, "probe", probe_run.stdout, probe_run.stderr + ) + variant_id = probe_run.stdout.strip().splitlines()[-1] + record = variants.get_variant(variant_id, path=variants_file) + run( + [ + python, + "-m", + "pipeline.uk_replay_qualify", + "corpus", + "--variant", + variant_id, + "--variants-file", + str(variants_file), + "--failures", + str(work / "failures.json"), + ] + ) + failures = json.loads((work / "failures.json").read_bytes()) + document = variants.load_document(variants_file) + base = base_namespace(request, record, document) + if base is not None: + base_id = ( + json.loads(next(base.glob("*/RUN_MANIFEST.json")).read_bytes()).get( + "variant_id" + ) + or variants.get_variant(path=variants_file)["variant_id"] + ) + diff_dir = ( + ROOT + / variants.REPLAY_RESULT_ROOT + / "diffs" + / f"{base_id}__{variant_id}" + ) + run( + [ + python, + "-m", + "pipeline.diff_uk_event_runs", + "--base", + str(base), + "--head", + str(variants.namespace_root(record)), + "--output-dir", + str(diff_dir), + ] + ) + except Rejected as exc: + rejection = exc + except subprocess.CalledProcessError as exc: + failures.append( + { + "code": "qualification_step_failed", + "detail": f"{exc.cmd}: exit {exc.returncode}", + } + ) + except Exception as exc: # noqa: BLE001 - a crash must still produce a receipt + failures.append( + { + "code": "unexpected_error", + "detail": f"{type(exc).__name__}: {exc}".splitlines()[0][:500], + } + ) + receipt = build_receipt( + request, + record=None if rejection else record, + rejection=rejection, + failures=failures, + diff_dir=diff_dir, + results_path=results_path, + runner=runner, + ) + if record is not None and rejection is None: + namespace = variants.namespace_root(record) + sealed = seal(namespace, receipt, diff_dir=diff_dir) + target = destination / "results" + shutil.copytree(namespace, target / record["result_root"], dirs_exist_ok=True) + if diff_dir is not None and diff_dir.is_dir(): + shutil.copytree( + diff_dir, + target / diff_dir.relative_to(ROOT), + dirs_exist_ok=True, + ) + requirements = ROOT / record["requirements"]["path"] + (target / record["requirements"]["path"]).parent.mkdir( + parents=True, exist_ok=True + ) + shutil.copy2(requirements, target / record["requirements"]["path"]) + (target / "VARIANT.json").write_bytes(canonical_bytes(record)) + else: + sealed = interface.seal_receipt(receipt) + interface.validate(sealed, interface.RECEIPT_SCHEMA_ID) + destination.mkdir(parents=True, exist_ok=True) + (destination / variants.RECEIPT_NAME).write_bytes(canonical_bytes(sealed)) + return sealed + + +# --- the command line ------------------------------------------------------------------ + + +def _load(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + commands = parser.add_subparsers(dest="command", required=True) + validate = commands.add_parser("validate") + validate.add_argument("--request", type=Path, required=True) + prep = commands.add_parser("prepare") + prep.add_argument("--request", type=Path, required=True) + prep.add_argument("--venv", type=Path, required=True) + prep.add_argument("--wheels", type=Path, required=True) + prep.add_argument("--output", type=Path, required=True) + probe_parser = commands.add_parser("probe") + probe_parser.add_argument("--request", type=Path, required=True) + probe_parser.add_argument("--runtime-root", type=Path, required=True) + probe_parser.add_argument("--dataset-file", type=Path) + probe_parser.add_argument("--wheel-digests", type=Path) + probe_parser.add_argument("--variants-file", type=Path, required=True) + probe_parser.add_argument("--rejection-file", type=Path) + request_parser = commands.add_parser("run-request") + request_parser.add_argument("--request", type=Path, required=True) + request_parser.add_argument("--state-dir", type=Path, required=True) + request_parser.add_argument("--work", type=Path, required=True) + request_parser.add_argument( + "--runner", default=os.environ.get("REPLAY_RUNNER", "local") + ) + corpus = commands.add_parser("corpus") + corpus.add_argument("--variant", required=True) + corpus.add_argument("--variants-file", type=Path, required=True) + corpus.add_argument("--workers", type=int, choices=(1, 2), default=2) + corpus.add_argument("--dataset-file", type=Path) + corpus.add_argument("--failures", type=Path, required=True) + receipt = commands.add_parser("receipt") + receipt.add_argument("--request", type=Path, required=True) + receipt.add_argument("--variant") + receipt.add_argument("--variants-file", type=Path) + receipt.add_argument("--rejection") + receipt.add_argument("--failures", type=Path) + receipt.add_argument("--diff-dir", type=Path) + receipt.add_argument("--results-path", required=True) + receipt.add_argument("--ref", default="replay-state") + receipt.add_argument("--runner", default=os.environ.get("REPLAY_RUNNER", "local")) + receipt.add_argument("--output", type=Path) + check = commands.add_parser("verify-sealed") + check.add_argument("namespaces", nargs="*", type=Path) + args = parser.parse_args(argv) + + if args.command == "validate": + print(request_id(_load(args.request))) + return 0 + if args.command == "prepare": + digests = prepare( + _load(args.request), args.venv.resolve(), args.wheels.resolve() + ) + args.output.write_bytes(canonical_bytes(digests)) + return 0 + if args.command == "probe": + request = _load(args.request) + digests = _load(args.wheel_digests) if args.wheel_digests else None + try: + probed = probe( + request, args.runtime_root.resolve(), dataset_file=args.dataset_file + ) + os.chdir(ROOT) + record = candidate_record(request, probed, wheel_sha256=digests) + except Rejected as exc: + if args.rejection_file is None: + raise + args.rejection_file.write_bytes( + canonical_bytes({"code": exc.code, "detail": exc.detail}) + ) + return PROBE_REJECTED + document = variants.load_document() + document["variants"] = [ + v for v in document["variants"] if v["variant_id"] != record["variant_id"] + ] + [record] + args.variants_file.write_bytes(canonical_bytes(document)) + print(record["variant_id"]) + return 0 + if args.command == "run-request": + receipt = run_request( + _load(args.request), args.state_dir, args.work.resolve(), runner=args.runner + ) + print(f"{receipt['status']} {receipt['receipt_sha256']}") + return 0 + if args.command == "corpus": + record = variants.get_variant(args.variant, path=args.variants_file.resolve()) + failures = run_corpus( + record, + args.variants_file.resolve(), + workers=args.workers, + dataset_file=args.dataset_file, + ) + args.failures.write_bytes(canonical_bytes(failures)) + return 0 + if args.command == "receipt": + request = _load(args.request) + record = ( + variants.get_variant(args.variant, path=args.variants_file.resolve()) + if args.variant + else None + ) + rejection = None + if args.rejection: + code, _, detail = args.rejection.partition(":") + rejection = Rejected(code.strip(), detail.strip() or code.strip()) + failures = ( + _load(args.failures) if args.failures and args.failures.exists() else [] + ) + built = build_receipt( + request, + record=record, + rejection=rejection, + failures=failures, + diff_dir=args.diff_dir, + results_path=args.results_path, + ref=args.ref, + runner=args.runner, + ) + if record is not None and rejection is None: + sealed = seal( + variants.namespace_root(record), built, diff_dir=args.diff_dir + ) + else: + from pipeline import uk_replay_interface as interface + + sealed = interface.seal_receipt(built) + interface.validate(sealed, interface.RECEIPT_SCHEMA_ID) + if args.output: + args.output.write_bytes(canonical_bytes(sealed)) + print(sealed["receipt_sha256"]) + return 0 + if args.command == "verify-sealed": + namespaces = args.namespaces or sorted( + path.parent + for path in (ROOT / variants.REPLAY_RESULT_ROOT).glob( + f"*/{variants.RECEIPT_NAME}" + ) + ) + problems = [] + for namespace in namespaces: + receipt = json.loads((namespace / variants.RECEIPT_NAME).read_bytes()) + diff = None + if receipt.get("diff"): + diff = ( + ROOT + / variants.REPLAY_RESULT_ROOT + / "diffs" + / f"{receipt['diff']['base_variant_id']}__{receipt['variant_id']}" + ) + problems += verify_sealed(namespace, diff_dir=diff) + for problem in problems: + print(problem) + print(f"{len(namespaces)} sealed namespace(s) checked") + return 1 if problems else 0 + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/uk_replay_queue.py b/pipeline/uk_replay_queue.py new file mode 100644 index 0000000..ea8f466 --- /dev/null +++ b/pipeline/uk_replay_queue.py @@ -0,0 +1,179 @@ +"""The persistent qualification queue on the ``replay-state`` branch. + +GitHub keeps at most one pending run per concurrency group and cancels older +pending runs, so a request must be written down before anything else can +lose it. Each request lives at ``requests//`` in a checkout of +the state branch: ``REQUEST.json`` (the request as received), ``STATUS.json`` +(schema policyengine.uk-replay-status/v1) and, once terminal, +``REPLAY_RECEIPT.json`` and ``results/``. Enqueueing is idempotent: a +request already present keeps its status. The workflow commits and pushes +after each call; this module only edits files. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from datetime import datetime, timedelta +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from pipeline import uk_replay_interface as interface # noqa: E402 + +TERMINAL = ("complete", "rejected", "failed") +STATUSES = ("queued", "running", *TERMINAL) +# A running request whose last update is older than this has lost its run +# (a cancelled or crashed job never writes a terminal status). +STALE_RUNNING = timedelta(hours=7) + + +def canonical_bytes(value) -> bytes: + return ( + json.dumps(value, indent=1, sort_keys=True, ensure_ascii=False) + "\n" + ).encode() + + +def request_dir(state: Path, request_id: str) -> Path: + if len(request_id) != 64 or any(c not in "0123456789abcdef" for c in request_id): + raise ValueError("request_id must be 64 lowercase hex characters") + return state / "requests" / request_id + + +def read_status(state: Path, request_id: str) -> dict | None: + path = request_dir(state, request_id) / "STATUS.json" + return json.loads(path.read_bytes()) if path.exists() else None + + +def write_status(state: Path, status: dict) -> None: + interface.validate(status, interface.STATUS_SCHEMA_ID) + path = request_dir(state, status["request_id"]) / "STATUS.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(canonical_bytes(status)) + + +def enqueue(state: Path, request: dict, *, run_url: str, now: str) -> tuple[str, str]: + """Record a request; returns (request_id, its status). Idempotent. + + A request that fails the schema is still recorded, by the digest of its + canonical bytes, so the receipt that rejects it has an address.""" + try: + request_id = interface.validate_request(request) + except Exception: # noqa: BLE001 - recorded, rejected by the drain step + request_id = interface.jcs_sha256(request) + existing = read_status(state, request_id) + if existing is not None: + return request_id, existing["status"] + directory = request_dir(state, request_id) + directory.mkdir(parents=True, exist_ok=True) + (directory / "REQUEST.json").write_bytes(canonical_bytes(request)) + write_status( + state, + { + "schema": interface.STATUS_SCHEMA_ID, + "request_id": request_id, + "status": "queued", + "history": [{"status": "queued", "at": now, "run_url": run_url}], + }, + ) + return request_id, "queued" + + +def set_status( + state: Path, + request_id: str, + status: str, + *, + run_url: str, + now: str, + receipt_sha256: str | None = None, + receipt_commit: str | None = None, +) -> dict: + current = read_status(state, request_id) + if current is None: + raise ValueError(f"{request_id} is not queued") + if current["status"] in TERMINAL: + raise ValueError( + f"{request_id} is already {current['status']}; terminal states are final" + ) + if status not in STATUSES: + raise ValueError(f"unknown status {status}") + updated = { + **current, + "status": status, + "history": [ + *current["history"], + {"status": status, "at": now, "run_url": run_url}, + ], + } + if status in TERMINAL: + if not (receipt_sha256 and receipt_commit): + raise ValueError("a terminal status names its receipt digest and commit") + updated.update(receipt_sha256=receipt_sha256, receipt_commit=receipt_commit) + write_status(state, updated) + return updated + + +def _at(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def pending(state: Path, *, now: str) -> list[str]: + """Requests to run, oldest first: queued, or running past the stale bound.""" + out = [] + for path in sorted((state / "requests").glob("*/STATUS.json")): + status = json.loads(path.read_bytes()) + last = status["history"][-1]["at"] + if status["status"] == "queued" or ( + status["status"] == "running" and _at(now) - _at(last) > STALE_RUNNING + ): + out.append((status["history"][0]["at"], status["request_id"])) + return [request_id for _, request_id in sorted(out)] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + commands = parser.add_subparsers(dest="command", required=True) + add = commands.add_parser("enqueue") + add.add_argument("--state", type=Path, required=True) + add.add_argument("--request", type=Path, required=True) + add.add_argument("--run-url", required=True) + add.add_argument("--now", required=True) + nxt = commands.add_parser("pending") + nxt.add_argument("--state", type=Path, required=True) + nxt.add_argument("--now", required=True) + mark = commands.add_parser("set-status") + mark.add_argument("--state", type=Path, required=True) + mark.add_argument("--request-id", required=True) + mark.add_argument("--status", choices=STATUSES, required=True) + mark.add_argument("--run-url", required=True) + mark.add_argument("--now", required=True) + mark.add_argument("--receipt-sha256") + mark.add_argument("--receipt-commit") + args = parser.parse_args(argv) + if args.command == "enqueue": + request = json.loads(args.request.read_text(encoding="utf-8")) + request_id, status = enqueue( + args.state, request, run_url=args.run_url, now=args.now + ) + print(f"{request_id} {status}") + elif args.command == "pending": + print("\n".join(pending(args.state, now=args.now))) + else: + set_status( + args.state, + args.request_id, + args.status, + run_url=args.run_url, + now=args.now, + receipt_sha256=args.receipt_sha256, + receipt_commit=args.receipt_commit, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_uk_replay_qualify.py b/tests/test_uk_replay_qualify.py new file mode 100644 index 0000000..1a9516c --- /dev/null +++ b/tests/test_uk_replay_qualify.py @@ -0,0 +1,340 @@ +"""Qualification requests, the persistent queue, receipts and seals. + +Engine-free: environments, simulations and downloads are not run; the +pieces that decide identity, rejection and sealing are exercised directly. +""" + +import copy +import hashlib +import json +from pathlib import Path + +import jsonschema +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from pipeline import uk_replay_interface as interface +from pipeline import uk_replay_qualify as qualify +from pipeline import uk_replay_queue as queue +from pipeline import uk_replay_variants as rv + +FIXTURES = Path(__file__).parent / "fixtures" / "uk_replay" +CURRENT = "pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0" +RUN = "https://github.com/PolicyEngine/policyengine-scorecard/actions/runs/1" + + +def golden_request(): + return json.loads((FIXTURES / "qualification_request_golden.json").read_text()) + + +# --- the queue -------------------------------------------------------------------- + + +def test_enqueue_is_idempotent_and_records_the_request(tmp_path): + request = golden_request() + rid, status = queue.enqueue( + tmp_path, request, run_url=RUN, now="2026-10-11T12:00:00Z" + ) + assert status == "queued" and rid == interface.request_id(request) + stored = json.loads((tmp_path / "requests" / rid / "REQUEST.json").read_text()) + assert stored == request + again = queue.enqueue(tmp_path, request, run_url=RUN, now="2026-10-11T13:00:00Z") + assert again == (rid, "queued") + status_doc = queue.read_status(tmp_path, rid) + assert len(status_doc["history"]) == 1 + + +def test_an_invalid_request_is_still_recorded_for_its_rejection(tmp_path): + request = {"schema": "nonsense"} + rid, status = queue.enqueue( + tmp_path, request, run_url=RUN, now="2026-10-11T12:00:00Z" + ) + assert rid == interface.jcs_sha256(request) and status == "queued" + + +def test_status_transitions_and_terminal_states(tmp_path): + rid, _ = queue.enqueue( + tmp_path, golden_request(), run_url=RUN, now="2026-10-11T12:00:00Z" + ) + queue.set_status(tmp_path, rid, "running", run_url=RUN, now="2026-10-11T12:01:00Z") + with pytest.raises(ValueError, match="terminal status names its receipt"): + queue.set_status( + tmp_path, rid, "complete", run_url=RUN, now="2026-10-11T12:30:00Z" + ) + done = queue.set_status( + tmp_path, + rid, + "complete", + run_url=RUN, + now="2026-10-11T12:30:00Z", + receipt_sha256="a" * 64, + receipt_commit="b" * 40, + ) + assert [h["status"] for h in done["history"]] == ["queued", "running", "complete"] + interface.validate(done, interface.STATUS_SCHEMA_ID) + with pytest.raises(ValueError, match="terminal states are final"): + queue.set_status( + tmp_path, rid, "running", run_url=RUN, now="2026-10-11T13:00:00Z" + ) + + +def test_pending_is_oldest_first_and_recovers_lost_runs(tmp_path): + first = golden_request() + second = copy.deepcopy(first) + second["attempt"] = 2 + third = copy.deepcopy(first) + third["attempt"] = 3 + a, _ = queue.enqueue(tmp_path, second, run_url=RUN, now="2026-10-11T10:00:00Z") + b, _ = queue.enqueue(tmp_path, first, run_url=RUN, now="2026-10-11T09:00:00Z") + c, _ = queue.enqueue(tmp_path, third, run_url=RUN, now="2026-10-11T11:00:00Z") + queue.set_status(tmp_path, c, "running", run_url=RUN, now="2026-10-11T11:05:00Z") + assert queue.pending(tmp_path, now="2026-10-11T12:00:00Z") == [b, a] + # A running request whose job vanished is retried after the stale bound. + assert queue.pending(tmp_path, now="2026-10-11T18:06:00Z") == [b, a, c] + + +def test_request_ids_must_be_digests(tmp_path): + with pytest.raises(ValueError): + queue.request_dir(tmp_path, "../escape") + + +# --- requests to environments ----------------------------------------------------- + + +def test_golden_candidate_matches_the_committed_certified_pair(): + document = rv.load_document() + match = qualify.matching_certified(golden_request(), document) + assert match is not None and match["variant_id"] == CURRENT + other = golden_request() + other["candidate"]["policyengine_uk"]["wheel_sha256"] = "0" * 64 + assert qualify.matching_certified(other, document) is None + assert qualify.latest_certified(document)["variant_id"] == CURRENT + + +def test_install_uses_exact_pins_for_a_certified_pair_and_constraints_otherwise( + tmp_path, +): + wheels = tmp_path / "wheels" + wheels.mkdir() + (wheels / "policyengine_uk-2.125.1-py3-none-any.whl").write_bytes(b"x") + request = golden_request() + env = tmp_path / "env.txt" + exact = qualify.install_command( + request, tmp_path / "venv", wheels, environment=env, exact=True + ) + assert exact[exact.index(str(env)) - 1] == "-r" + loose = qualify.install_command( + request, tmp_path / "venv", wheels, environment=env, exact=False + ) + assert loose[loose.index(str(env)) - 1] == "-c" + assert any( + part.startswith("policyengine @ git+") + and part.endswith( + request["candidate"]["policyengine"]["source"]["git"]["commit"] + ) + for part in exact + ) + lines = qualify.environment_lines(rv.get_variant(CURRENT)) + assert lines and not any(line.split("==")[0] in rv.PACKAGES for line in lines) + + +def test_downloaded_wheels_must_match_the_request(tmp_path): + request = golden_request() + wheels = tmp_path / "wheels" + wheels.mkdir() + (wheels / "policyengine_uk-2.125.1-py3-none-any.whl").write_bytes(b"not the wheel") + (wheels / "policyengine_core-3.33.0-py3-none-any.whl").write_bytes(b"core") + with pytest.raises(qualify.Rejected) as raised: + qualify.verify_wheels(request, wheels) + assert raised.value.code == "artifact_hash_mismatch" + request["candidate"]["policyengine_uk"]["wheel_sha256"] = hashlib.sha256( + b"not the wheel" + ).hexdigest() + request["candidate"]["policyengine_core"]["wheel_sha256"] = hashlib.sha256( + b"core" + ).hexdigest() + digests = qualify.verify_wheels(request, wheels) + assert set(digests) == {"policyengine_uk", "policyengine_core"} + + +def test_schema_failures_are_rejections(): + with pytest.raises(qualify.Rejected) as raised: + qualify.request_id({"schema": "x"}) + assert raised.value.code == "schema_invalid" + + +def test_freeze_normalisation(): + versions = { + "policyengine": "6.2.6", + "policyengine-uk": "2.125.1", + "policyengine-core": "3.33.0", + } + freeze = "\n".join( + [ + "numpy==2.5.4", + "policyengine @ git+https://github.com/PolicyEngine/policyengine.py@" + + "a" * 40, + "policyengine-core @ file:///tmp/wheels/policyengine_core-3.33.0-py3-none-any.whl", + "policyengine-uk==2.125.1", + ] + ) + text = rv.normalized_freeze(freeze, versions) + assert rv.requirements_pins(text) == { + "numpy": "2.5.4", + "policyengine": "6.2.6", + "policyengine-core": "3.33.0", + "policyengine-uk": "2.125.1", + } + assert ( + "# policyengine-core installed from policyengine_core-3.33.0-py3-none-any.whl" + in text + ) + with pytest.raises(rv.VariantError, match="not an exact pin"): + rv.normalized_freeze(freeze + "\nother @ file:///x.whl", versions) + with pytest.raises(rv.VariantError, match="lacks"): + rv.normalized_freeze("numpy==2.5.4", versions) + + +def test_variant_ids_take_a_digest_suffix_only_on_a_collision(): + identity = rv.get_variant(CURRENT)["identity"] + digest = rv.identity_sha256(identity) + slug = rv.variant_slug(identity) + assert rv.variant_id_for(identity, {}) == slug + assert rv.variant_id_for(identity, {slug: digest}) == slug + assert rv.variant_id_for(identity, {slug: "f" * 64}) == f"{slug}--{digest[:10]}" + + +def test_every_pinned_distribution_must_be_installed_at_its_version(): + pins = {"numpy": "2.5.4", "pandas": "3.0.6"} + assert ( + rv.environment_mismatches( + pins, {"distributions": {"numpy": "2.5.4", "pandas": "3.0.6", "extra": "1"}} + ) + == [] + ) + assert rv.environment_mismatches( + pins, {"distributions": {"numpy": "2.5.5", "pandas": "3.0.6"}} + ) + assert rv.environment_mismatches(pins, {"distributions": {"numpy": "2.5.4"}}) + + +# --- receipts and seals ------------------------------------------------------------- + + +def test_rejected_and_preidentity_failed_receipts_validate(): + request = golden_request() + rejected = interface.seal_receipt( + qualify.build_receipt( + request, + record=None, + rejection=qualify.Rejected("data_access_failed", "401"), + results_path="requests/x/results/", + ) + ) + interface.validate(rejected, interface.RECEIPT_SCHEMA_ID) + assert rejected["status"] == "rejected" and rejected["corpus"] is None + failed = interface.seal_receipt( + qualify.build_receipt( + request, + record=None, + failures=[ + {"code": "qualification_step_failed", "detail": "pip download: exit 1"} + ], + results_path="requests/x/results/", + ) + ) + interface.validate(failed, interface.RECEIPT_SCHEMA_ID) + assert failed["status"] == "failed" and failed["integrity"]["status"] == "failed" + assert failed["execution"] is None + + +def test_a_complete_receipt_needs_its_execution(): + receipt = interface.seal_receipt( + { + **json.loads((FIXTURES / "replay_receipt_golden.json").read_text()), + "execution": None, + } + ) + with pytest.raises(jsonschema.ValidationError): + interface.validate(receipt, interface.RECEIPT_SCHEMA_ID) + + +def sealed_namespace(tmp_path): + namespace = tmp_path / "ns" + (namespace / "event").mkdir(parents=True) + (namespace / "event" / "RUN_MANIFEST.json").write_text("{}") + (namespace / "SUMMARY.md").write_text("# summary\n") + diff = tmp_path / "diff" + (diff / "packets").mkdir(parents=True) + (diff / "DIFF.json").write_text("{}") + golden = json.loads((FIXTURES / "replay_receipt_golden.json").read_text()) + receipt = {k: v for k, v in golden.items() if k != "receipt_sha256"} + receipt["results"] = {k: v for k, v in receipt["results"].items() if k != "commit"} + sealed = qualify.seal(namespace, receipt, diff_dir=diff) + return namespace, diff, sealed + + +def test_seal_binds_every_file_and_detects_tampering(tmp_path): + namespace, diff, sealed = sealed_namespace(tmp_path) + assert qualify.verify_sealed(namespace, diff_dir=diff) == [] + manifest = json.loads((namespace / qualify.MANIFEST_NAME).read_text()) + assert set(manifest["files"]) == {"event/RUN_MANIFEST.json", "SUMMARY.md"} + assert ( + sealed["results"]["manifest_sha256"] + == hashlib.sha256((namespace / qualify.MANIFEST_NAME).read_bytes()).hexdigest() + ) + (namespace / "SUMMARY.md").write_text("# edited\n") + assert qualify.verify_sealed(namespace, diff_dir=diff) + + +def test_seal_detects_added_files_and_diff_edits(tmp_path): + namespace, diff, _ = sealed_namespace(tmp_path) + (namespace / "event" / "extra.json").write_text("{}") + assert any( + "files differ" in p for p in qualify.verify_sealed(namespace, diff_dir=diff) + ) + (namespace / "event" / "extra.json").unlink() + (diff / "DIFF.json").write_text("{ }") + assert any("diff" in p for p in qualify.verify_sealed(namespace, diff_dir=diff)) + + +def test_sealed_namespaces_refuse_writes(tmp_path): + namespace, _, _ = sealed_namespace(tmp_path) + with pytest.raises(rv.VariantError, match="sealed"): + rv.assert_not_sealed(namespace / "event", root=tmp_path) + + +@settings(max_examples=50, deadline=None) +@given(st.lists(st.sampled_from(["queued", "running"]), min_size=1, max_size=5)) +def test_pending_never_returns_terminal_requests(tmp_path_factory, statuses): + state = tmp_path_factory.mktemp("state") + ids = [] + for attempt, status in enumerate(statuses, start=1): + request = golden_request() + request["attempt"] = attempt + rid, _ = queue.enqueue( + state, request, run_url=RUN, now=f"2026-10-11T0{attempt}:00:00Z" + ) + if status == "running": + queue.set_status( + state, rid, "running", run_url=RUN, now=f"2026-10-11T0{attempt}:10:00Z" + ) + ids.append(rid) + finished = ids[0] + if queue.read_status(state, finished)["status"] == "queued": + queue.set_status( + state, finished, "running", run_url=RUN, now="2026-10-11T09:00:00Z" + ) + queue.set_status( + state, + finished, + "failed", + run_url=RUN, + now="2026-10-11T09:30:00Z", + receipt_sha256="c" * 64, + receipt_commit="d" * 40, + ) + later = queue.pending(state, now="2026-10-12T09:00:00Z") + assert finished not in later + assert set(later) == set(ids[1:]) From 6942df75acbd2b7e0e7b4eb0e6d84348a9ec5a97 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 15 Oct 2026 05:46:14 -0400 Subject: [PATCH 3/7] Receiver: answer the review of 6af674d Built by a GPT-6.1 Sol lane and committed here because the lane's sandbox could not write git metadata. - Adapts to #160: candidate runs pass --allow-uncertified and the work variants file; build_receipt writes the rejected and complete shapes the tightened schema requires on every path. - A failure after the candidate's identity is known always yields a schema-valid failed receipt (corpus null when nothing ran; a crash while building or sealing the receipt gives a minimal failed receipt); a request is tried at most three times, then gets a terminal failed status; the drain handles a non-zero run-request exit and reads the outcome from a result file, so run logs reach Actions. - The corpus runs in a fresh detached checkout of the same commit with that variant's committed namespace and diff removed, never resuming from or resealing committed results; the diff's base is read from the runner checkout; earlier attempts' files are removed before a copy; the runner checkout is left unchanged. - Least privilege: the dataset is fetched in its own job, the job that installs and runs candidate code has a read-only token, no persisted checkout credentials and no Hugging Face token, and a separate job records STATUS and the receipt from the run's uploaded artifact. - verify-sealed derives the namespaces that must be sealed from replay_variants.json; CI regenerates in a disposable checkout and fails on untracked files. - The replay-engine CI job says when it replayed nothing and what each leg covers. - Docs: the schedule drains recorded requests only (discovery arrives with #165); REQUEST.json is re-serialised and its request_id is the JCS digest of the received request. - Concurrent enqueue of one request_id recovers with backoff; the install mode (certified constraints or free resolution) is sealed in INSTALL.json; a wheel missing from PyPI is an artifact_hash_mismatch rejection. Co-Authored-By: GPT-6.1 Sol (Subfleet lane) Co-Authored-By: Claude Opus 5.5 --- .github/scripts/uk_replay_drain.sh | 75 +-- .github/scripts/uk_replay_enqueue.sh | 29 ++ .github/scripts/uk_replay_prepare.sh | 40 ++ .github/scripts/uk_replay_record.sh | 52 ++ .github/scripts/uk_replay_regenerate.sh | 61 +++ .github/scripts/uk_replay_state.sh | 42 ++ .github/workflows/ci.yml | 59 +-- .github/workflows/uk-replay.yml | 186 ++++--- docs/uk_replay/QUALIFICATION_INTERFACE.md | 80 ++- docs/uk_replay/RELEASE_REPLAY.md | 132 +++-- pipeline/uk_replay_qualify.py | 466 +++++++++++++---- pipeline/uk_replay_queue.py | 74 ++- tests/test_uk_replay_qualify.py | 592 ++++++++++++++++++++++ tests/test_uk_replay_queue.py | 545 ++++++++++++++++++++ 14 files changed, 2086 insertions(+), 347 deletions(-) create mode 100644 .github/scripts/uk_replay_enqueue.sh create mode 100644 .github/scripts/uk_replay_prepare.sh create mode 100644 .github/scripts/uk_replay_record.sh create mode 100644 .github/scripts/uk_replay_regenerate.sh create mode 100644 .github/scripts/uk_replay_state.sh create mode 100644 tests/test_uk_replay_queue.py diff --git a/.github/scripts/uk_replay_drain.sh b/.github/scripts/uk_replay_drain.sh index 18ec51a..60890ea 100755 --- a/.github/scripts/uk_replay_drain.sh +++ b/.github/scripts/uk_replay_drain.sh @@ -1,65 +1,18 @@ #!/usr/bin/env bash -# Drain the UK replay qualification queue (see .github/workflows/uk-replay.yml). -# -# For each pending request, oldest first: mark it running, run it end to end -# (pipeline/uk_replay_qualify.py run-request), commit the receipt and results -# to the state branch, then write the terminal STATUS.json naming the -# receipt's digest and that commit. Stops when the queue is empty or the time -# budget is spent; the next scheduled run continues. +# Candidate code runs only here, in the read-only job without the dataset token. set -euo pipefail - -STATE=state CONTROL=(uv run --with rfc8785==0.1.4 --with jsonschema --with huggingface_hub python) -BUDGET_SECONDS=$((300 * 60)) -started=$(date +%s) - -now() { date -u +%Y-%m-%dT%H:%M:%SZ; } - -push_state() { - git -C "$STATE" add -A - if git -C "$STATE" diff --cached --quiet; then - return 0 - fi - git -C "$STATE" commit -qm "$1" - for attempt in 1 2 3 4 5; do - if git -C "$STATE" push -q origin "HEAD:$STATE_BRANCH"; then - return 0 - fi - git -C "$STATE" pull -q --rebase origin "$STATE_BRANCH" - done - echo "could not push the state branch after five attempts" >&2 - return 1 -} - -while :; do - git -C "$STATE" pull -q --rebase origin "$STATE_BRANCH" - request_id=$("${CONTROL[@]}" -m pipeline.uk_replay_queue pending --state "$STATE" --now "$(now)" | head -n 1) - if [ -z "$request_id" ]; then - echo "queue empty" - break - fi - if [ $(( $(date +%s) - started )) -gt "$BUDGET_SECONDS" ]; then - echo "time budget spent; the next scheduled run continues" - break - fi - echo "::group::request $request_id" - "${CONTROL[@]}" -m pipeline.uk_replay_queue set-status --state "$STATE" \ - --request-id "$request_id" --status running --run-url "$RUN_URL" --now "$(now)" - push_state "Run $request_id" - - work="$RUNNER_TEMP/requests/$request_id" - outcome=$("${CONTROL[@]}" -m pipeline.uk_replay_qualify run-request \ - --request "$STATE/requests/$request_id/REQUEST.json" \ - --state-dir "$STATE" --work "$work" --runner "$REPLAY_RUNNER" | tail -n 1) - status=${outcome%% *} - receipt_sha256=${outcome##* } - push_state "Receipt for $request_id: $status" - receipt_commit=$(git -C "$STATE" rev-parse HEAD) +mkdir -p output/state - "${CONTROL[@]}" -m pipeline.uk_replay_queue set-status --state "$STATE" \ - --request-id "$request_id" --status "$status" --run-url "$RUN_URL" --now "$(now)" \ - --receipt-sha256 "$receipt_sha256" --receipt-commit "$receipt_commit" - push_state "Status for $request_id: $status" - rm -rf "$work" - echo "::endgroup::" -done +if "${CONTROL[@]}" -m pipeline.uk_replay_qualify run-request \ + --request input/REQUEST.json --state-dir output/state \ + --work "$RUNNER_TEMP/requests/$REQUEST_ID" --runner "$REPLAY_RUNNER" \ + --dataset-file input/dataset.h5 --dataset-result input/fetch-result.json \ + --result-file output/outcome.json; then + run_exit=0 +else + run_exit=$? + echo "::error::run-request exited $run_exit; the recorder will preserve its receipt or write a failed receipt" +fi +printf '%s\n' "$run_exit" > output/run-exit.txt +# Do not hide pipeline stdout, and always allow the result artifact to upload. diff --git a/.github/scripts/uk_replay_enqueue.sh b/.github/scripts/uk_replay_enqueue.sh new file mode 100644 index 0000000..c8d1e89 --- /dev/null +++ b/.github/scripts/uk_replay_enqueue.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Concurrent enqueues reapply the idempotent operation after a rebase conflict. +set -euo pipefail +# shellcheck source=.github/scripts/uk_replay_state.sh +source .github/scripts/uk_replay_state.sh +checkout_state + +for attempt in 1 2 3 4 5; do + "${CONTROL[@]}" -m pipeline.uk_replay_queue enqueue --state "$STATE" \ + --request "$REQUEST_FILE" --run-url "$RUN_URL" --now "$(now)" \ + | tee "$RUNNER_TEMP/enqueued.txt" + git -C "$STATE" add -A + if ! git -C "$STATE" diff --cached --quiet; then + git -C "$STATE" commit -qm "Queue $(cut -d' ' -f1 "$RUNNER_TEMP/enqueued.txt")" + fi + if git -C "$STATE" push -q origin "HEAD:$STATE_BRANCH"; then + exit 0 + fi + if ! git -C "$STATE" pull -q --rebase origin "$STATE_BRANCH"; then + git -C "$STATE" rebase --abort + git -C "$STATE" fetch -q origin "$STATE_BRANCH" + git -C "$STATE" reset --hard "origin/$STATE_BRANCH" + git -C "$STATE" clean -fd + echo "enqueue conflicted; reapplying the request to the current state branch" + fi + sleep "$attempt" +done +echo "could not enqueue and push the request after five attempts" >&2 +exit 1 diff --git a/.github/scripts/uk_replay_prepare.sh b/.github/scripts/uk_replay_prepare.sh new file mode 100644 index 0000000..8885532 --- /dev/null +++ b/.github/scripts/uk_replay_prepare.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Select one request. Exhausted stale attempts receive a committed failure first. +set -euo pipefail +# shellcheck source=.github/scripts/uk_replay_state.sh +source .github/scripts/uk_replay_state.sh +if ! git ls-remote --exit-code --heads origin "$STATE_BRANCH" >/dev/null; then + echo "queue empty: no state branch" + echo "request_id=" >> "$GITHUB_OUTPUT" + exit 0 +fi +checkout_state +mkdir -p input +while :; do + "${CONTROL[@]}" -m pipeline.uk_replay_queue pending --state "$STATE" --now "$(now)" > "$RUNNER_TEMP/pending.txt" + IFS= read -r request_id < "$RUNNER_TEMP/pending.txt" || true + if [ -z "$request_id" ]; then + echo "queue empty" + echo "request_id=" >> "$GITHUB_OUTPUT" + exit 0 + fi + attempts=$("${CONTROL[@]}" -m pipeline.uk_replay_queue attempts --state "$STATE" --request-id "$request_id") + if [ "$attempts" -ge 3 ]; then + receipt_sha256=$("${CONTROL[@]}" -m pipeline.uk_replay_queue failure-receipt \ + --state "$STATE" --request-id "$request_id" --code attempt_limit_exceeded \ + --detail "The request exhausted three running attempts without a terminal receipt") + push_state "Receipt for $request_id: attempts exhausted" + receipt_commit=$(git -C "$STATE" rev-parse HEAD) + "${CONTROL[@]}" -m pipeline.uk_replay_queue set-status --state "$STATE" \ + --request-id "$request_id" --status failed --run-url "$RUN_URL" --now "$(now)" \ + --receipt-sha256 "$receipt_sha256" --receipt-commit "$receipt_commit" + push_state "Status for $request_id: attempts exhausted" + continue + fi + "${CONTROL[@]}" -m pipeline.uk_replay_queue set-status --state "$STATE" \ + --request-id "$request_id" --status running --run-url "$RUN_URL" --now "$(now)" + push_state "Run $request_id" + cp "$STATE/requests/$request_id/REQUEST.json" input/REQUEST.json + echo "request_id=$request_id" >> "$GITHUB_OUTPUT" + break +done diff --git a/.github/scripts/uk_replay_record.sh b/.github/scripts/uk_replay_record.sh new file mode 100644 index 0000000..69b8cd5 --- /dev/null +++ b/.github/scripts/uk_replay_record.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Validate the uploaded outcome, commit its receipt, then commit terminal status. +set -euo pipefail +# shellcheck source=.github/scripts/uk_replay_state.sh +source .github/scripts/uk_replay_state.sh +checkout_state +"${CONTROL[@]}" - "$STATE" "$REQUEST_ID" <<'PY' +import json +import shutil +import sys +from pathlib import Path +from pipeline import uk_replay_interface as interface +from pipeline import uk_replay_queue as queue + +state, request_id = Path(sys.argv[1]), sys.argv[2] +destination = queue.request_dir(state, request_id) +try: + outcome = json.loads(Path("output/outcome.json").read_bytes()) + source = Path("output/state/requests") / request_id + receipt = json.loads((source / "REPLAY_RECEIPT.json").read_bytes()) + interface.validate(receipt, interface.RECEIPT_SCHEMA_ID) + if receipt["request_id"] != request_id or outcome != { + "status": receipt["status"], "receipt_sha256": receipt["receipt_sha256"] + }: + raise ValueError("the uploaded outcome does not name this request's receipt") + results = destination / "results" + if results.exists(): + shutil.rmtree(results) + if (source / "results").exists(): + shutil.copytree(source / "results", results) + shutil.copyfile(source / "REPLAY_RECEIPT.json", destination / "REPLAY_RECEIPT.json") +except Exception as error: + receipt = queue.write_failure_receipt( + state, request_id, code="run_artifact_failed", + detail=f"{type(error).__name__}: {str(error).splitlines()[0]}", + ) +Path("record-outcome.json").write_text(json.dumps({ + "status": receipt["status"], "receipt_sha256": receipt["receipt_sha256"] +})) +PY +status=$("${CONTROL[@]}" -c 'import json; print(json.load(open("record-outcome.json"))["status"])') +receipt_sha256=$("${CONTROL[@]}" -c 'import json; print(json.load(open("record-outcome.json"))["receipt_sha256"])') +push_state "Receipt for $REQUEST_ID: $status" +receipt_commit=$(git -C "$STATE" rev-parse HEAD) +"${CONTROL[@]}" -m pipeline.uk_replay_queue set-status --state "$STATE" \ + --request-id "$REQUEST_ID" --status "$status" --run-url "$RUN_URL" --now "$(now)" \ + --receipt-sha256 "$receipt_sha256" --receipt-commit "$receipt_commit" +push_state "Status for $REQUEST_ID: $status" +"${CONTROL[@]}" -m pipeline.uk_replay_queue pending --state "$STATE" --now "$(now)" > "$RUNNER_TEMP/pending.txt" +if [ -s "$RUNNER_TEMP/pending.txt" ]; then + gh workflow run uk-replay.yml --repo "$GITHUB_REPOSITORY" --ref "$DRAIN_REF" +fi diff --git a/.github/scripts/uk_replay_regenerate.sh b/.github/scripts/uk_replay_regenerate.sh new file mode 100644 index 0000000..746403d --- /dev/null +++ b/.github/scripts/uk_replay_regenerate.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Regenerate committed replay bytes in a disposable checkout, preserving seals. +set -euo pipefail +run() { uv run --with rfc8785==0.1.4 --with jsonschema python "$@"; } + +# Check the original receipts before making the disposable namespaces writable. +run -m pipeline.uk_replay_qualify verify-sealed +git worktree add -q --detach "$RUNNER_TEMP/replay-regeneration" HEAD +cd "$RUNNER_TEMP/replay-regeneration" + +seals=() +for root in results/uk/replays/*/; do + [ -d "$root" ] || continue + for name in REPLAY_RECEIPT.json NAMESPACE_MANIFEST.json; do + path="$root$name" + if [ -f "$path" ]; then + seals+=("$path") + rm "$path" + fi + done +done +restore_seals() { + for path in "${seals[@]}"; do + git show "HEAD:$path" > "$path" + done +} +trap restore_seals EXIT + +for root in results/uk/events results/uk/replays/*/; do + [ -d "$root" ] || continue + case "$root" in */diffs/) continue ;; esac + found=0 + for manifest in "$root"/*/RUN_MANIFEST.json; do + [ -f "$manifest" ] || continue + found=1 + dir=$(dirname "$manifest") + event=$(basename "$dir") + run pipeline/stage_uk_event.py --event "$event" --artifact-dir "$dir" > /dev/null + run pipeline/compare_uk_event.py --event "$event" --staged "$dir/STAGED.jsonl" --output-dir "$dir" > /dev/null + done + if [ "$found" = 1 ]; then + run pipeline/compare_uk_event.py --summary --events-root "$root" > /dev/null + echo "regenerated $root" + fi +done +for diff in results/uk/replays/diffs/*/; do + [ -f "$diff/DIFF.json" ] || continue + base=$(run -c "import json,sys; print(json.load(open(sys.argv[1]))['base']['root'])" "$diff/DIFF.json") + head=$(run -c "import json,sys; print(json.load(open(sys.argv[1]))['head']['root'])" "$diff/DIFF.json") + run -m pipeline.diff_uk_event_runs --base "$base" --head "$head" --output-dir "$diff" --check +done + +# Restore only the two seal files; keep regenerated bytes visible to the gate. +restore_seals +trap - EXIT +changes=$(git status --porcelain) +if [ -n "$changes" ]; then + printf '%s\n' "$changes" + echo "replay regeneration changed committed bytes or created untracked files" >&2 + exit 1 +fi diff --git a/.github/scripts/uk_replay_state.sh b/.github/scripts/uk_replay_state.sh new file mode 100644 index 0000000..28b4a6d --- /dev/null +++ b/.github/scripts/uk_replay_state.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Shared state-branch operations. Source from the control scripts only. +set -euo pipefail + +STATE=${STATE:-state} +# Used by the scripts that source this file. +# shellcheck disable=SC2034 +CONTROL=(uv run --with rfc8785==0.1.4 --with jsonschema python) + +now() { date -u +%Y-%m-%dT%H:%M:%SZ; } + +checkout_state() { + if git ls-remote --exit-code --heads origin "$STATE_BRANCH" >/dev/null; then + git fetch -q origin "$STATE_BRANCH" + git worktree add -q "$STATE" "origin/$STATE_BRANCH" + git -C "$STATE" switch -q -C "$STATE_BRANCH" + else + git worktree add -q --orphan -b "$STATE_BRANCH" "$STATE" + fi + git -C "$STATE" config user.name "github-actions[bot]" + git -C "$STATE" config user.email "41898282+github-actions[bot]@users.noreply.github.com" +} + +push_state() { + git -C "$STATE" add -A + if ! git -C "$STATE" diff --cached --quiet; then + git -C "$STATE" commit -qm "$1" + fi + for attempt in 1 2 3 4 5; do + if git -C "$STATE" push -q origin "HEAD:$STATE_BRANCH"; then + return 0 + fi + if ! git -C "$STATE" pull -q --rebase origin "$STATE_BRANCH"; then + git -C "$STATE" rebase --abort + echo "state rebase conflicted; refusing to publish conflicting results" >&2 + return 1 + fi + sleep "$attempt" + done + echo "could not push the state branch after five attempts" >&2 + return 1 +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c35025..0b8b213 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,12 +22,12 @@ jobs: # verified deterministic — two builds must agree on the logical # content hash before the suite runs against it. - run: | - uv run python -m scorecard_db.build_db data/scorecard.db > build1.json - uv run python -m scorecard_db.build_db /tmp/scorecard-det.db > build2.json + uv run python -m scorecard_db.build_db data/scorecard.db > "$RUNNER_TEMP/build1.json" + uv run python -m scorecard_db.build_db /tmp/scorecard-det.db > "$RUNNER_TEMP/build2.json" uv run python - <<'EOF' - import json - a = json.load(open("build1.json"))["content_hash"] - b = json.load(open("build2.json"))["content_hash"] + import json, os + a = json.load(open(os.path.join(os.environ["RUNNER_TEMP"], "build1.json")))["content_hash"] + b = json.load(open(os.path.join(os.environ["RUNNER_TEMP"], "build2.json")))["content_hash"] assert a == b, f"build is nondeterministic: {a} != {b}" print("deterministic:", a) EOF @@ -48,38 +48,14 @@ jobs: # results/uk/events and each results/uk/replays/) is re-staged # and re-compared from its committed artifacts, every committed release # diff is re-rendered, and every sealed namespace must match its - # receipt byte for byte. Engine-free; the no-drift gate below fails on - # any regenerated byte that differs. + # receipt byte for byte. Verify the original seals, regenerate in a + # fresh checkout with its seal files temporarily removed, restore only + # those files, and require an unchanged tree, including untracked files. - name: Replay namespaces regenerate and seals hold env: PYTHONPATH: . - run: | - set -euo pipefail - run() { uv run --with rfc8785==0.1.4 --with jsonschema python "$@"; } - for root in results/uk/events results/uk/replays/*/; do - [ -d "$root" ] || continue - case "$root" in */diffs/) continue ;; esac - found=0 - for manifest in "$root"/*/RUN_MANIFEST.json; do - [ -f "$manifest" ] || continue - found=1 - dir=$(dirname "$manifest"); event=$(basename "$dir") - run pipeline/stage_uk_event.py --event "$event" --artifact-dir "$dir" > /dev/null - run pipeline/compare_uk_event.py --event "$event" --staged "$dir/STAGED.jsonl" --output-dir "$dir" > /dev/null - done - if [ "$found" = 1 ]; then - run pipeline/compare_uk_event.py --summary --events-root "$root" > /dev/null - echo "regenerated $root" - fi - done - for diff in results/uk/replays/diffs/*/; do - [ -f "$diff/DIFF.json" ] || continue - base=$(run -c "import json,sys; print(json.load(open(sys.argv[1]))['base']['root'])" "$diff/DIFF.json") - head=$(run -c "import json,sys; print(json.load(open(sys.argv[1]))['head']['root'])" "$diff/DIFF.json") - run -m pipeline.diff_uk_event_runs --base "$base" --head "$head" --output-dir "$diff" --check - done - run -m pipeline.uk_replay_qualify verify-sealed - - run: git diff --exit-code + run: bash .github/scripts/uk_replay_regenerate.sh + - run: test -z "$(git status --porcelain)" - name: Upload built database uses: actions/upload-artifact@v4 with: @@ -91,10 +67,11 @@ jobs: # The event replay against each certified variant's own engine: the # exact environment from the variant's requirements freeze, the # construction guards, and the committed premise and diagnostic files - # regenerated byte for byte. With the HUGGING_FACE_TOKEN secret it also - # replays one real measure-year on the variant's certified dataset and - # compares the artifact with the committed bytes (replay determinism); - # without the secret (forks) that step announces it did not run. + # regenerated byte for byte. The legacy variant skips diagnostics and + # determinism. For other variants, a real measure-year replays only when + # HUGGING_FACE_TOKEN and a committed sealed namespace are both present. + # Otherwise the notice says that no real replay ran; the secret awaits + # administrator setup and is also unavailable to fork pull requests. name: replay engine · ${{ matrix.variant }} runs-on: ubuntu-latest strategy: @@ -145,7 +122,11 @@ jobs: run: | set -euo pipefail if [ -z "${HUGGING_FACE_TOKEN:-}" ]; then - echo "::notice::HUGGING_FACE_TOKEN not available (fork PR or unset secret): the real replay determinism check did NOT run" + echo "::notice::HUGGING_FACE_TOKEN unavailable (fork PR or secret awaiting administrator setup): nothing was replayed by the real replay determinism step" + exit 0 + fi + if [ ! -f "$RESULT_ROOT/REPLAY_RECEIPT.json" ] || [ ! -f "$RESULT_ROOT/NAMESPACE_MANIFEST.json" ]; then + echo "::notice::The variant has no committed sealed namespace: nothing was replayed by the real replay determinism step" exit 0 fi uv run --with huggingface_hub python - <<'PY' diff --git a/.github/workflows/uk-replay.yml b/.github/workflows/uk-replay.yml index 10b77d9..a31eb72 100644 --- a/.github/workflows/uk-replay.yml +++ b/.github/workflows/uk-replay.yml @@ -1,25 +1,17 @@ name: UK replay qualification -# WHY THIS EXISTS -# -# Release qualification in policyengine.py (W4) asks this repository to rerun -# the OBR fiscal-event replay corpus on a candidate model/data pair before it -# is promoted, and to return an artifact-bound receipt. The interface is -# docs/uk_replay/QUALIFICATION_INTERFACE.md; the run is -# pipeline/uk_replay_qualify.py. -# -# GitHub keeps at most one pending run per concurrency group and cancels the -# older pending ones, so a request is first written to the persistent queue -# on the `replay-state` branch (enqueue job, no concurrency group). The drain -# job then runs queued requests one at a time. A scheduled run drains anything -# a cancelled run left behind. A request whose run died mid-way is retried -# after the stale bound in pipeline/uk_replay_queue.py. -# -# Simulations: one drain job at a time, at most two workers inside it, so -# never more than two simulations at once from this workflow. -# -# Secrets: HUGGING_FACE_TOKEN reads the certified datasets. Without it a -# request is rejected with data_access_failed rather than skipped. +# Incoming request runs only enqueue, before dispatching an empty drain run. +# Their concurrency groups are unique. Empty manual runs and scheduled drains +# share a workflow concurrency group, held across preparation, run and recording. +# This preserves the persistent queue even when a pending drain is cancelled. +# One request runs per drain, with at most two simulation workers; the recorder +# dispatches another drain while requests remain eligible. Stale runs retry +# after seven hours, with three running attempts maximum. +# Dataset access happens before the candidate job, in a separate job. The +# candidate job has contents: read, no checkout credentials and no dataset token. +# Receipts are pushed before their terminal status names the pushed commit. +# GitHub documents GITHUB_TOKEN-triggered workflow_dispatch runs at: +# https://docs.github.com/en/actions/concepts/security/github_token on: repository_dispatch: @@ -34,94 +26,136 @@ on: - cron: "41 */3 * * *" permissions: - contents: write + contents: read + +concurrency: + group: ${{ (github.event_name == 'repository_dispatch' || (github.event_name == 'workflow_dispatch' && github.event.inputs.request != '')) && format('uk-replay-enqueue-{0}', github.run_id) || 'uk-replay-drain' }} + cancel-in-progress: false env: STATE_BRANCH: replay-state + PYTHONPATH: . jobs: enqueue: name: record the request if: github.event_name == 'repository_dispatch' || (github.event_name == 'workflow_dispatch' && github.event.inputs.request != '') runs-on: ubuntu-latest + permissions: + contents: write + actions: write steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 - - name: Check out the state branch - run: | - set -euo pipefail - if git ls-remote --exit-code --heads origin "$STATE_BRANCH" >/dev/null; then - git fetch -q origin "$STATE_BRANCH" - git worktree add -q state "origin/$STATE_BRANCH" - git -C state switch -q -C "$STATE_BRANCH" - else - git worktree add -q --orphan -b "$STATE_BRANCH" state - fi - git -C state config user.name "github-actions[bot]" - git -C state config user.email "41898282+github-actions[bot]@users.noreply.github.com" - - name: Write the request + - name: Write the received request env: + EVENT_NAME: ${{ github.event_name }} DISPATCH_REQUEST: ${{ toJSON(github.event.client_payload.request) }} MANUAL_REQUEST: ${{ github.event.inputs.request }} run: | set -euo pipefail - if [ "${{ github.event_name }}" = "repository_dispatch" ]; then + if [ "$EVENT_NAME" = repository_dispatch ]; then printf '%s' "$DISPATCH_REQUEST" > "$RUNNER_TEMP/request.json" else printf '%s' "$MANUAL_REQUEST" > "$RUNNER_TEMP/request.json" fi - - name: Enqueue and push + - name: Enqueue with conflict recovery and start a drain env: - PYTHONPATH: . RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_TOKEN: ${{ github.token }} + DRAIN_REF: ${{ github.event.repository.default_branch }} run: | set -euo pipefail - now=$(date -u +%Y-%m-%dT%H:%M:%SZ) - uv run --with rfc8785==0.1.4 --with jsonschema python -m pipeline.uk_replay_queue \ - enqueue --state state --request "$RUNNER_TEMP/request.json" --run-url "$RUN_URL" --now "$now" \ - | tee "$RUNNER_TEMP/enqueued.txt" - git -C state add -A - if git -C state diff --cached --quiet; then - echo "already recorded: $(cat "$RUNNER_TEMP/enqueued.txt")" - exit 0 - fi - git -C state commit -qm "Queue $(cut -d' ' -f1 "$RUNNER_TEMP/enqueued.txt")" - for attempt in 1 2 3 4 5; do - git -C state push -q origin "HEAD:$STATE_BRANCH" && exit 0 - git -C state pull -q --rebase origin "$STATE_BRANCH" - done - exit 1 + export REQUEST_FILE="$RUNNER_TEMP/request.json" + bash .github/scripts/uk_replay_enqueue.sh + gh workflow run uk-replay.yml --repo "$GITHUB_REPOSITORY" --ref "$DRAIN_REF" - drain: - name: run queued requests - needs: [enqueue] - if: always() && (needs.enqueue.result == 'success' || needs.enqueue.result == 'skipped') + prepare: + name: select a request and fetch its dataset + if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.request == '') runs-on: ubuntu-latest - timeout-minutes: 340 - concurrency: - group: uk-replay-drain - cancel-in-progress: false + permissions: + contents: write + outputs: + request_id: ${{ steps.select.outputs.request_id }} steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 - - name: Check out the state branch - run: | - set -euo pipefail - if ! git ls-remote --exit-code --heads origin "$STATE_BRANCH" >/dev/null; then - echo "no state branch yet; nothing queued" - echo "EMPTY=1" >> "$GITHUB_ENV" - exit 0 - fi - git fetch -q origin "$STATE_BRANCH" - git worktree add -q state "origin/$STATE_BRANCH" - git -C state switch -q -C "$STATE_BRANCH" - git -C state config user.name "github-actions[bot]" - git -C state config user.email "41898282+github-actions[bot]@users.noreply.github.com" - - name: Drain the queue - if: env.EMPTY != '1' + - name: Mark the oldest eligible request running + id: select env: - PYTHONPATH: . RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: bash .github/scripts/uk_replay_prepare.sh + - name: Fetch and verify the dataset without candidate code + if: steps.select.outputs.request_id != '' + env: HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + run: | + uv run --with rfc8785==0.1.4 --with jsonschema --with huggingface_hub python \ + -m pipeline.uk_replay_qualify fetch-dataset --request input/REQUEST.json \ + --output input/dataset.h5 --result-file input/fetch-result.json + - name: Hand the verified dataset and request to the candidate job + if: steps.select.outputs.request_id != '' + uses: actions/upload-artifact@v4 + with: + name: replay-input + path: input/ + if-no-files-found: error + retention-days: 1 + + run: + name: run one candidate with read permission + needs: prepare + if: needs.prepare.outputs.request_id != '' + runs-on: ubuntu-latest + timeout-minutes: 340 + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@v5 + - uses: actions/download-artifact@v4 + with: + name: replay-input + path: input/ + - name: Run the candidate and preserve logs and outcome + env: + REQUEST_ID: ${{ needs.prepare.outputs.request_id }} REPLAY_RUNNER: github-actions ${{ runner.os }} ${{ runner.arch }} run: bash .github/scripts/uk_replay_drain.sh + - name: Upload the receipt and results for the recorder + if: always() + uses: actions/upload-artifact@v4 + with: + name: replay-output + path: output/ + if-no-files-found: warn + retention-days: 7 + + record: + name: commit the receipt then terminal status + needs: [prepare, run] + if: always() && needs.prepare.outputs.request_id != '' + runs-on: ubuntu-latest + permissions: + contents: write + actions: write + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + - name: Download the run artifact + id: download + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: replay-output + path: output/ + - name: Record the outcome and continue the queue + env: + REQUEST_ID: ${{ needs.prepare.outputs.request_id }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_TOKEN: ${{ github.token }} + DRAIN_REF: ${{ github.event.repository.default_branch }} + run: bash .github/scripts/uk_replay_record.sh diff --git a/docs/uk_replay/QUALIFICATION_INTERFACE.md b/docs/uk_replay/QUALIFICATION_INTERFACE.md index d99fac7..3b24c21 100644 --- a/docs/uk_replay/QUALIFICATION_INTERFACE.md +++ b/docs/uk_replay/QUALIFICATION_INTERFACE.md @@ -32,8 +32,8 @@ Two flows use the same receiver: * **Post-promotion consumer dispatch.** After a policyengine.py release, W4's NotifyConsumers sends the same event with `kind: "released_bundle"`, the policyengine PyPI wheel SHA-256 and the packaged manifest SHA-256. The - same receiver handles it, so scorecard ends up with a certified variant - and a sealed namespace for every release; there is no separate handler. + same receiver handles it and records the variant and sealed namespace for + each successfully replayed dispatched release; there is no separate handler. ## Transport @@ -42,26 +42,46 @@ Two flows use the same receiver: `client_payload: {"request": }`. W4 alone edits the policyengine.py dispatcher and its GitHub App token scope; scorecard needs `contents: read` on policyengine.py (public) and nothing else from W4. -* GitHub keeps at most one pending run per concurrency group and cancels - older pending runs, so the receiver's first step writes the request to the - persistent queue on the `replay-state` branch before anything else. A - scheduled catch-up (every 3 hours) drains the queue and discovers - policyengine.py releases that were never dispatched. +* The receiver writes incoming requests to the persistent queue on the + `replay-state` branch, then dispatches a drain run. Incoming runs have + distinct concurrency groups; the prepare/run/record drain sequence shares + `uk-replay-drain`. GitHub's default concurrency queue can replace an older + pending run, so requests remain in the persistent queue until handled. + ([GitHub concurrency documentation](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency)) + A scheduled catch-up every 3 hours drains recorded requests. Discovery of + policyengine.py releases that were never dispatched is separate work in + PR #165. * State for request `` lives at `replay-state:requests//`: - `REQUEST.json` (the bytes received), `STATUS.json` + `REQUEST.json` (the received request re-serialised as sorted, indented JSON; + `request_id` remains the JCS digest of the received request), `STATUS.json` (`queued` → `running` → `complete` | `rejected` | `failed`, with a history of timestamps and Actions run URLs), and on completion `REPLAY_RECEIPT.json`. The receipt is committed first; a terminal `STATUS.json` is written in a later commit and names `receipt_sha256` and `receipt_commit`, so one contents read binds the status to the receipt bytes. +* Dataset fetching and candidate execution are separate jobs. The control + job fetches and verifies the file with the dataset token; the run job has + `contents: read`, no persisted checkout credential and no Hugging Face + token. It receives the file through an artifact and supplies + `--dataset-file` to `run-request`. The qualifier writes a structured result + file with `status` and `receipt_sha256`, while logs remain visible in + Actions. A separate recording job has `contents: write` and `actions: write`, + validates the uploaded receipt, records it before terminal status and + dispatches another drain run while requests remain pending. + `workflow_dispatch` can trigger a run using `GITHUB_TOKEN`. + ([GitHub workflow-trigger documentation](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/trigger-a-workflow)) * W4 polls `GET /repos/PolicyEngine/policyengine-scorecard/contents/requests//STATUS.json?ref=replay-state` (the repository is public). A receipt is addressed by the request's own digest, so W4 needs no callback and no scorecard token. * Duplicate requests (same `request_id`) are idempotent: an existing terminal status is returned, never rerun. A failed request can be retried only by a new request (a different `attempt`). +* A running request becomes stale after 7 hours. The receiver permits at + most three `running` entries in its history; the next stale attempt records + a minimal `failed` receipt with `attempt_limit_exceeded`, followed by a + terminal status naming that receipt and its pushed commit. ## Request @@ -100,11 +120,12 @@ Two flows use the same receiver: } ``` -* `request_id` is not a field: it is the SHA-256 of the request's JCS - bytes. Both sides test against the golden fixture. +* `request_id` is not a field: it is the SHA-256 of the received request's + JCS bytes, independent of the formatting used to store `REQUEST.json`. + Both sides test against the golden fixture. * `requested_by` names the requesting repository and commit; - `workflow_run_id` is optional (a scorecard catch-up run started outside - Actions has none), and additional provenance fields such as `run_attempt` + `workflow_run_id` is optional (a manual request made outside Actions has + none), and additional provenance fields such as `run_attempt` are accepted. Every other object is closed. * The wrapper is identified by `version` plus either a PyPI wheel (`{"pypi": {"wheel_sha256": …}}`) or, for an unreleased candidate whose @@ -122,13 +143,27 @@ The candidate becomes a `candidate` variant record (RELEASE_REPLAY.md). It is | Code | Condition | |---|---| | `schema_invalid` | The request fails the v1 schema. | -| `artifact_hash_mismatch` | A downloaded wheel, the installed packaged manifest or the dataset bytes differ from the request. | +| `artifact_hash_mismatch` | A requested wheel is missing from PyPI, or a downloaded wheel, the installed packaged manifest or the dataset bytes differ from the request. | | `release_bundle_mismatch` | `pe.uk.uk_latest.release_bundle` under packaged certification differs from `expected_release_bundle`. | | `incompatible_pair` | The packaged certification does not name the candidate model version, or the dataset is not the bundle's certified artifact. | | `data_access_failed` | The dataset could not be fetched with the receiver's credential. | | `no_supported_years` | No event year falls in the dataset's supported years. | A rejected request has no numerical results; its receipt says why. +`variant_id`, `variant_identity_sha256`, `execution`, `corpus` and `diff` +are null, and `integrity.status` is `not_run`, even if a record was known +before the rejection. + +Every corpus replay runs in a fresh detached checkout of the same commit +under its work directory. The candidate's copied namespace and diff are +removed before running, no committed results are resumed, and the runner +checkout supplies the read-only diff base. With `latest_certified`, an exact +certified pair is compared with its own committed namespace, which also +checks determinism. Sealing happens in the fresh checkout, and the request's +previous `results/` directory is replaced before copying the sealed output. +`INSTALL.json` records whether installation used exact certified requirements, +certified constraints or a fallback to free resolution; the receipt binds +that file through the namespace manifest. ## Receipt @@ -179,13 +214,21 @@ A rejected request has no numerical results; its receipt says why. "results": { "repository": "PolicyEngine/policyengine-scorecard", "ref": "replay-state", - "commit": "<40-hex>", - "path": "requests//results/" + "path": "requests//results/", + "manifest_sha256": "<64-hex SHA-256 of NAMESPACE_MANIFEST.json>" }, "receipt_sha256": "<64-hex of the canonical receipt without this field>" } ``` +A complete receipt requires `variant_identity_sha256` and +`results.manifest_sha256`. A failed receipt has `corpus: null` when no event +produced outputs. If receipt building, sealing or result copying crashes, +the receiver writes a minimal schema-valid failed receipt with +`receipt_step_failed` and the error's first line. Missing or invalid uploaded +run output is recorded as `run_artifact_failed`. The terminal status still +names the receipt digest and the commit that was pushed before it. + ### Division of judgement How flags arise, so a gate built on them sees the intended events: @@ -197,9 +240,10 @@ How flags arise, so a gate built on them sees the intended events: check changed its result. A change in the current-law value of a parameter the construction touches is recorded as context in the row's details, not as a flag, because it is what a model release is expected to change. -* For a candidate, the base is the latest certified variant, normally on the - same data build; the first certified-variant run diffs against the legacy - populace variant, where the data family itself changes. +* For a candidate, the default base is the newest certified variant with + committed results; an exact certified pair uses its own committed run. + The first certified-variant run diffs against the legacy populace variant, + where the data family itself changes. * `integrity.status` is `complete`, `incomplete` or `failed`. `failed` means a crash, a non-finite aggregate, an identity mismatch after the run started, diff --git a/docs/uk_replay/RELEASE_REPLAY.md b/docs/uk_replay/RELEASE_REPLAY.md index 90a77e3..25a185e 100644 --- a/docs/uk_replay/RELEASE_REPLAY.md +++ b/docs/uk_replay/RELEASE_REPLAY.md @@ -183,7 +183,9 @@ FY2023–24 counterparts; that loss is shown, never hidden. * Every other variant writes to `results/uk/replays//`: `/` (the same files as a legacy event directory), `SUMMARY.md`, `SUPPORT.json`, `CONSTRUCTION_PREMISES.json`, `MODEL_DIAGNOSTICS.json`, - `REQUEST.json` (the qualification request the run answers), and finally + `REQUEST.json` (the re-serialised qualification request the run answers), + `INSTALL.json` (whether installation used the certified requirements, + certified constraints or free resolution), and finally `NAMESPACE_MANIFEST.json` and `REPLAY_RECEIPT.json`. * `data/uk/replay_variants.json` is the index: each record's `result_root` names its namespace (legacy: `results/uk/events`) and its `status`. A @@ -193,10 +195,21 @@ FY2023–24 counterparts; that loss is shown, never hidden. `NAMESPACE_MANIFEST.json` lists the SHA-256 of every file under it (and of its release diff), the receipt binds that manifest's digest, the runner refuses to write there (compute, staging, comparison and the summary all - check), and CI fails if any sealed byte changes. A rerun of - a sealed variant writes to an explicit separate output directory, and its - bytes against the sealed run are a determinism check (CI replays one real - measure-year this way). + check), and CI fails if any sealed byte changes. `verify-sealed` derives + the expected namespaces from the variants index: every record with + `layout: "replay_namespace"` whose `result_root` exists must have both + sealing files. Deleting a receipt or manifest therefore fails the check. +* Each corpus qualification creates a detached checkout of the runner's `HEAD` + under `/checkout`. It deletes the candidate's namespace and selected + diff directory inside that checkout, computes every event there without + `--resume`, seals there, then copies the namespace and diff to the request's + `results/` directory. The previous attempt's `results/` directory is removed + before copying. Every pipeline subprocess uses that fresh checkout as its + working directory and module root, keeping artifact paths repository-relative. + The runner's committed namespaces are only read, including the diff's base. + Re-qualifying an exactly certified pair compares the fresh run against its + own committed run when `compare_to` is `latest_certified`; this doubles as + a determinism check. * Release differences live in `results/uk/replays/diffs/__/`. ## Version-scoped diagnoses @@ -295,24 +308,37 @@ W4 sends before promotion and the receipt returned. ### The receiver `.github/workflows/uk-replay.yml` receives `repository_dispatch` events of -type `uk-replay-qualification-request` (and manual requests). Its `enqueue` -job writes the request to the persistent queue on the `replay-state` branch -before anything else (`pipeline/uk_replay_queue.py`), so GitHub cancelling -an older pending run never loses one. Its `drain` job, one at a time -(concurrency group `uk-replay-drain`), runs `.github/scripts/uk_replay_drain.sh`. -That script takes the oldest pending request, marks it running, and runs -`pipeline/uk_replay_qualify.py run-request`: +type `uk-replay-qualification-request` (and manual requests). An incoming +request uses an `enqueue` job to write it to the persistent queue on the +`replay-state` branch (`pipeline/uk_replay_queue.py`), then dispatches a drain +run. Incoming runs have distinct concurrency groups; drain runs share +`uk-replay-drain`, so only one prepare/run/record sequence executes at a time. +GitHub's default concurrency queue can replace an older pending run; the +request remains in `replay-state` for a later drain. +([GitHub concurrency documentation](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency)) + +The `prepare` job selects the oldest eligible request and marks it running. +It fetches the dataset with `fetch-dataset` in a control step holding +`HUGGING_FACE_TOKEN`, checks its size and SHA-256, and uploads the dataset, +request and fetch outcome as an artifact. A fetch rejection is passed to the +run without installing candidate code. The separate `run` job has +`contents: read`, checks out with `persist-credentials: false`, receives no +Hugging Face token, and runs `.github/scripts/uk_replay_drain.sh` for that one +request. `pipeline/uk_replay_qualify.py run-request` then: 1. Validate the request (`schema_invalid`). -2. Download the requested dataset with `HUGGING_FACE_TOKEN` into the Hugging - Face cache and check its size and SHA-256 (`data_access_failed`, - `artifact_hash_mismatch`). +2. Check the dataset file supplied by the control job against the request + (`data_access_failed`, `artifact_hash_mismatch`). 3. Build the candidate environment: download the model and core wheels without dependencies, check their SHA-256 against the request, then install them, the wrapper (its PyPI wheel or its public commit) and the - receipt tools. Re-qualifying a certified pair installs that pair's exact - requirements freeze; a new candidate resolves within the newest certified - freeze, falling back to a free resolution only if its own pins conflict. + receipt tools. A requested wheel missing from PyPI is an + `artifact_hash_mismatch` rejection. Re-qualifying a certified pair installs + that pair's exact requirements freeze; a new candidate first uses the + newest certified freeze as constraints and falls back to free resolution + if that installation fails. `INSTALL.json` records `exact_requirements`, + `certified_constraints` or `free_resolution`, the reference variant and + the environment file's digest; the namespace seal binds this provenance. 4. Probe inside that environment: the installed identity, the release bundle under packaged certification (`release_bundle_mismatch`), the certification naming this model and this dataset (`incompatible_pair`), @@ -320,23 +346,43 @@ That script takes the oldest pending request, marks it running, and runs environment reproduces a committed variant exactly, the run uses that record; otherwise it builds a candidate record whose requirements are the environment's normalised freeze. -5. Run the corpus: every event's compute, staging and comparison, the - summary, the construction premises, the re-verified diagnostics. -6. Diff against `compare_to` (default: the newest other certified variant - whose results are committed). +5. Run the corpus in the fresh checkout: every event's compute (allowing the + candidate record), staging (using the work variants document), comparison, + the summary, the construction premises, the re-verified diagnostics. +6. Diff against the runner checkout's committed `compare_to` namespace + (default: the newest certified variant with committed results, preferring + the matching certified variant itself for an exact re-qualification). 7. Seal the namespace (`NAMESPACE_MANIFEST.json`, then `REPLAY_RECEIPT.json`) and copy it, the diff and the requirements to `replay-state:requests//results/`. -The receipt is committed first; the terminal `STATUS.json` follows in a -later commit naming the receipt's digest and that commit. A step that fails -before the candidate's identity is known yields a `failed` receipt with no -execution block; an unexpected exception also yields a `failed` receipt, so a -request never loops. A request still `running` seven hours after its last -update (a cancelled or crashed job) is retried. A scheduled run every three -hours drains whatever is left. - -At most one drain job runs, with at most two simulation workers, so this +The qualifier writes a result file containing `status` and `receipt_sha256`; +the drain handles a non-zero qualifier exit and leaves run logs visible in +Actions. The separate `record` job, with `contents: write` and `actions: write`, +reads that result file, validates the uploaded output and commits the receipt +first. The terminal `STATUS.json` follows in a later commit naming the receipt's +digest and the pushed receipt commit. Missing or invalid run output yields a +minimal `failed` receipt. + +A step that fails before the candidate's identity is known yields a `failed` +receipt with no execution block. If no event outputs exist, `corpus` is null. +An exception while building, sealing or copying the receipt's results yields +a minimal `failed` receipt with `receipt_step_failed` and the error's first +line. A request still `running` seven hours after its last update (a cancelled +or crashed job) is eligible for retry; at most three `running` history entries +are allowed. The next stale attempt records a minimal `failed` receipt with +`attempt_limit_exceeded` and terminal status instead of running again. + +The recording job dispatches another drain run while pending requests remain; +`workflow_dispatch` can create a run using `GITHUB_TOKEN`. +([GitHub workflow-trigger documentation](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/trigger-a-workflow)) +A scheduled run every three hours also drains the queue. This schedule only +processes recorded requests; undispatched release discovery is separate work +in PR #165. Concurrent enqueues retry with backoff and, on a rebase conflict, +abort, check out the latest state branch and enqueue the request again. +Exhausting retries reports an error. + +At most one drain run executes, with at most two simulation workers, so this workflow never runs more than two simulations at once. These runs use GitHub runners; manual replays on the shared Mac also respect the host reservation (`python3 ~/reviews/fleet-ops/tools/reservation.py allowed w6-replay`), which @@ -346,17 +392,23 @@ holds heavy local work back from 29 October to 2 November. `ci.yml` runs two kinds of replay check on every pull request and push: -* Engine-free, in the main job: every committed namespace (legacy and each - variant) is re-staged and re-compared from its artifacts, every committed - release diff is re-rendered with `--check`, and every sealed namespace must - match its receipt byte for byte; the no-drift gate fails on any change. -* `replay engine`, once per certified variant: installs the variant's exact +* Engine-free, in the main job: every committed namespace with event run + manifests (legacy and each variant) is re-staged and re-compared from its + artifacts, every committed release diff is re-rendered with `--check`, and + every present namespace indexed as `replay_namespace` must have a receipt + and manifest matching its bytes. The no-drift gate uses + `git status --porcelain`, so newly generated untracked files fail too. +* `replay engine`, for the two certified variants in its CI matrix: installs + each variant's exact freeze, checks the installed environment against the record, runs every event's construction guards, regenerates the committed construction - premises (and, for non-legacy variants, the re-verified diagnostics) byte - for byte, and, with the `HUGGING_FACE_TOKEN` secret, replays one real - measure-year on the certified dataset and compares the artifact with the - committed bytes. Without the secret that step says it did not run. + premises byte for byte. Non-legacy variants also regenerate the re-verified + diagnostics and, with `HUGGING_FACE_TOKEN` and a committed sealed namespace, + replay one real measure-year on the certified dataset and compare its + artifact with the committed bytes. The legacy leg skips diagnostics and + this determinism step. Without the secret (pending an administrator), or + without a committed sealed namespace, the step emits a notice stating that + nothing was replayed. ## Invariants diff --git a/pipeline/uk_replay_qualify.py b/pipeline/uk_replay_qualify.py index 7326515..49dca0a 100644 --- a/pipeline/uk_replay_qualify.py +++ b/pipeline/uk_replay_qualify.py @@ -8,8 +8,9 @@ Steps, each a subcommand so the receiver workflow can run them in the right environment: -* ``plan``: validate the request and print the install commands for its - exact wheels (control environment; needs rfc8785 and jsonschema). +* ``fetch-dataset``: validate and fetch the requested data in the control + environment, before candidate code runs without dataset credentials. +* ``prepare``: install and verify the candidate's requested wheels. * ``probe``: inside the candidate environment, after installation, verify the installed identity against the request, materialise the dataset and read the release bundle, certification and data years. @@ -17,6 +18,8 @@ comparison for every event, then premises, diagnostics and the diff. * ``receipt``: build, validate and seal the receipt; write NAMESPACE_MANIFEST.json and REPLAY_RECEIPT.json. +* ``run-request``: run these steps in a fresh checkout of this commit and + copy the sealed results to the request's state directory. Nothing here decides whether a candidate is good: the receipt reports integrity and descriptive diff flags; the promotion policy is release @@ -30,6 +33,7 @@ import json import os import platform +import shutil import subprocess import sys from pathlib import Path @@ -219,8 +223,32 @@ def prepare( ) -> dict[str, str]: """Build the candidate environment from exactly the requested wheels.""" wheels.mkdir(parents=True, exist_ok=True) - for command in download_plan(request, venv, wheels): - run(command) + commands = download_plan(request, venv, wheels) + for command in commands[:-1]: + run(command, cwd=root) + command = commands[-1] + print("+", " ".join(command), flush=True) + downloaded = subprocess.run( + command, + cwd=root, + env=candidate_environment(root), + capture_output=True, + text=True, + ) + print(downloaded.stdout, end="", flush=True) + print(downloaded.stderr, end="", file=sys.stderr, flush=True) + if downloaded.returncode: + detail = downloaded.stdout + downloaded.stderr + if ( + "No matching distribution found" in detail + or "Could not find a version that satisfies" in detail + ): + raise Rejected( + "artifact_hash_mismatch", "a requested wheel is missing from PyPI" + ) + raise subprocess.CalledProcessError( + downloaded.returncode, command, downloaded.stdout, downloaded.stderr + ) digests = verify_wheels(request, wheels) document = variants.load_document(root / "data" / "uk" / "replay_variants.json") match = matching_certified(request, document) @@ -233,14 +261,28 @@ def prepare( run( install_command( request, venv, wheels, environment=environment, exact=bool(match) - ) + ), + cwd=root, ) + resolution = "exact_requirements" if match else "certified_constraints" except subprocess.CalledProcessError: if match is not None: raise # The candidate's own requirements conflict with the newest certified # environment: resolve freely; the recorded freeze shows what moved. - run(install_command(request, venv, wheels)) + run(install_command(request, venv, wheels), cwd=root) + resolution = "free_resolution" + (wheels.parent / "INSTALL.json").write_bytes( + canonical_bytes( + { + "resolution": resolution, + "certified_variant_id": (match or latest_certified(document))[ + "variant_id" + ], + "environment_sha256": variants.sha256_file(environment), + } + ) + ) return digests @@ -470,11 +512,17 @@ def engine(packages: dict) -> dict: # --- the corpus ------------------------------------------------------------------------ +def candidate_environment(root: Path) -> dict[str, str]: + """Candidate code never inherits dataset credentials from the control process.""" + environment = {**os.environ, "PYTHONPATH": str(root)} + for name in ("HUGGING_FACE_TOKEN", "HF_TOKEN", "HUGGINGFACE_HUB_TOKEN"): + environment.pop(name, None) + return environment + + def run(command: list[str], *, cwd: Path = ROOT) -> None: print("+", " ".join(command), flush=True) - subprocess.run( - command, check=True, cwd=cwd, env={**os.environ, "PYTHONPATH": str(ROOT)} - ) + subprocess.run(command, check=True, cwd=cwd, env=candidate_environment(cwd)) def run_corpus( @@ -484,10 +532,14 @@ def run_corpus( workers: int = 2, dataset_file: Path | None = None, events: tuple[str, ...] = EVENTS, + root: Path = ROOT, + diagnostics_file: Path | None = None, ) -> list[dict]: """Compute, stage and compare every event; returns integrity failures.""" python = sys.executable - namespace = variants.namespace_root(record) + namespace = variants.namespace_root(record, root=root) + variants.assert_not_sealed(namespace, root=root) + namespace.mkdir(parents=True, exist_ok=True) failures = [] for event in events: output = namespace / event @@ -500,6 +552,7 @@ def run_corpus( event, "--variant", record["variant_id"], + "--allow-uncertified", "--variants-file", str(variants_file), "--workers", @@ -509,11 +562,7 @@ def run_corpus( ] if dataset_file is not None: command += ["--dataset-file", str(dataset_file)] - if (output / "RUN_MANIFEST.json").exists() or any( - output.glob("RUN_PROGRESS_*.json") - ): - command.append("--resume") - run(command) + run(command, cwd=root) run( [ python, @@ -522,7 +571,10 @@ def run_corpus( event, "--artifact-dir", str(output), - ] + "--variants-file", + str(variants_file), + ], + cwd=root, ) run( [ @@ -534,7 +586,8 @@ def run_corpus( str(output / "STAGED.jsonl"), "--output-dir", str(output), - ] + ], + cwd=root, ) except subprocess.CalledProcessError as exc: failures.append( @@ -547,7 +600,8 @@ def run_corpus( "--summary", "--events-root", str(namespace), - ] + ], + cwd=root, ) engine = record["identity"]["packages"]["policyengine-uk"]["version"] run( @@ -560,19 +614,21 @@ def run_corpus( engine, "--output", str(namespace / "CONSTRUCTION_PREMISES.json"), - ] + ], + cwd=root, ) run( [ python, "pipeline/diagnose_uk_event_models.py", "--reverify", - str(ROOT / "results/uk/events/MODEL_DIAGNOSTICS.json"), + str(diagnostics_file or root / "results/uk/events/MODEL_DIAGNOSTICS.json"), "--expect-engine", engine, "--output", str(namespace / "MODEL_DIAGNOSTICS.json"), - ] + ], + cwd=root, ) support = {} for event in events: @@ -591,15 +647,18 @@ def run_corpus( def namespace_files(namespace: Path) -> dict[str, str]: + excluded = {namespace / MANIFEST_NAME, namespace / variants.RECEIPT_NAME} return { str(path.relative_to(namespace)): variants.sha256_file(path) for path in sorted(namespace.rglob("*")) - if path.is_file() and path.name not in (MANIFEST_NAME, variants.RECEIPT_NAME) + if path.is_file() and path not in excluded } -def event_results(record: dict, events=EVENTS) -> tuple[dict, list[dict]]: - namespace = variants.namespace_root(record) +def event_results( + record: dict, events=EVENTS, *, root: Path = ROOT +) -> tuple[dict, list[dict]]: + namespace = variants.namespace_root(record, root=root) out, failures = {}, [] for event in events: directory = namespace / event @@ -663,6 +722,7 @@ def build_receipt( results_path: str, ref: str = "replay-state", runner: str = "local", + root: Path = ROOT, ) -> dict: from pipeline import uk_replay_interface as interface @@ -671,6 +731,8 @@ def build_receipt( except Exception: # noqa: BLE001 - an unparseable request still gets a receipt rid = interface.jcs_sha256(request) failures = list(failures or []) + if rejection is not None: + record = None receipt = { "schema": interface.RECEIPT_SCHEMA_ID, "request_id": rid, @@ -708,9 +770,11 @@ def build_receipt( "platform": platform.platform(), "runner": runner, } - events, missing = event_results(record) + events, missing = event_results(record, root=root) failures += missing - receipt["corpus"] = {"corpus_version": CORPUS_VERSION, "events": events} + receipt["corpus"] = ( + {"corpus_version": CORPUS_VERSION, "events": events} if events else None + ) complete = events and all( e["full_supported_grid_complete"] for e in events.values() ) @@ -726,14 +790,18 @@ def build_receipt( receipt["status"] = "failed" if diff_dir is not None and (diff_dir / "DIFF.json").exists(): receipt["diff"] = diff_summary(diff_dir) + receipt["results"]["manifest_sha256"] = sha256_bytes( + canonical_bytes( + namespace_manifest( + variants.namespace_root(record, root=root), diff_dir=diff_dir + ) + ) + ) return receipt -def seal(namespace: Path, receipt: dict, *, diff_dir: Path | None = None) -> dict: - """Bind every namespace file (and the diff) and write the sealed receipt.""" - from pipeline import uk_replay_interface as interface - - manifest = { +def namespace_manifest(namespace: Path, *, diff_dir: Path | None = None) -> dict: + return { "schema_version": 1, "files": namespace_files(namespace), "diff_files": { @@ -744,7 +812,14 @@ def seal(namespace: Path, receipt: dict, *, diff_dir: Path | None = None) -> dic if diff_dir is not None and diff_dir.is_dir() else {}, } - payload = canonical_bytes(manifest) + + +def seal(namespace: Path, receipt: dict, *, diff_dir: Path | None = None) -> dict: + """Bind every namespace file (and the diff) and write the sealed receipt.""" + from pipeline import uk_replay_interface as interface + + variants.assert_not_sealed(namespace) + payload = canonical_bytes(namespace_manifest(namespace, diff_dir=diff_dir)) (namespace / MANIFEST_NAME).write_bytes(payload) receipt = dict(receipt) receipt["results"] = { @@ -833,31 +908,81 @@ def fetch_dataset(request: dict) -> Path: return path -def base_namespace(request: dict, record: dict, document: dict) -> Path | None: - """The namespace to diff against: compare_to, or the newest certified - variant other than this one whose results are present.""" +def base_namespace( + request: dict, record: dict, document: dict, *, root: Path = ROOT +) -> Path | None: + """Read the requested certified base from the unchanged runner checkout. + + An exactly certified pair compares against its own committed run, which + also checks determinism when the pair is qualified again. + """ wanted = request.get("compare_to", "latest_certified") candidates = ( [r for r in document["variants"] if r["variant_id"] == wanted] if wanted != "latest_certified" - else [ - r - for r in reversed(document["variants"]) - if r["status"] == "certified" and r["variant_id"] != record["variant_id"] - ] + else [r for r in reversed(document["variants"]) if r["status"] == "certified"] ) + if wanted == "latest_certified": + candidates.sort(key=lambda r: r["variant_id"] != record["variant_id"]) for candidate in candidates: - root = variants.namespace_root(candidate) - if any(root.glob("*/COMPARISON.json")): - return root + namespace = variants.namespace_root(candidate, root=root) + if any(namespace.glob("*/COMPARISON.json")): + return namespace return None -def run_request(request: dict, state_dir: Path, work: Path, *, runner: str) -> dict: - """Qualify one request and write its receipt and results to the state - checkout. Every outcome, including a rejection, produces a receipt.""" - import shutil +def fresh_checkout(work: Path, *, root: Path = ROOT) -> Path: + """An isolated checkout of the same commit; only its results may change.""" + checkout = work / "checkout" + if checkout.exists(): + run(["git", "worktree", "remove", "--force", str(checkout)], cwd=root) + run(["git", "worktree", "add", "--detach", str(checkout), "HEAD"], cwd=root) + return checkout + +def minimal_failed_receipt( + rid: str, *, record: dict | None, results_path: str, detail: str +) -> dict: + """A recovery receipt independent of event parsing, building and sealing.""" + from pipeline import uk_replay_interface as interface + + receipt = interface.seal_receipt( + { + "schema": interface.RECEIPT_SCHEMA_ID, + "request_id": rid, + "status": "failed", + "rejection": None, + "variant_id": record["variant_id"] if record else None, + "variant_identity_sha256": record["identity_sha256"] if record else None, + "execution": None, + "corpus": None, + "integrity": { + "status": "failed", + "failures": [{"code": "receipt_step_failed", "detail": detail}], + }, + "diff": None, + "results": { + "repository": "PolicyEngine/policyengine-scorecard", + "ref": "replay-state", + "path": results_path, + }, + } + ) + interface.validate(receipt, interface.RECEIPT_SCHEMA_ID) + return receipt + + +def run_request( + request: dict, + state_dir: Path, + work: Path, + *, + runner: str, + dataset_file: Path | None = None, + dataset_result: Path | None = None, + root: Path = ROOT, +) -> dict: + """Qualify in a fresh checkout and emit a receipt for every ordinary failure.""" from pipeline import uk_replay_interface as interface try: @@ -865,16 +990,37 @@ def run_request(request: dict, state_dir: Path, work: Path, *, runner: str) -> d except Rejected: rid = interface.jcs_sha256(request) destination = state_dir / "requests" / rid + target = destination / "results" results_path = f"requests/{rid}/results/" work.mkdir(parents=True, exist_ok=True) venv, wheels = work / "venv", work / "wheels" variants_file = work / "variants.json" - record = rejection = diff_dir = None + checkout = record = rejection = diff_dir = None failures: list[dict] = [] try: request_id(request) - fetch_dataset(request) - digests = prepare(request, venv, wheels) + if dataset_result is not None: + fetched = _load(dataset_result) + if fetched.get("rejection"): + raise Rejected(**fetched["rejection"]) + if dataset_file is None: + raise Rejected( + "data_access_failed", + "run-request requires a previously fetched dataset file", + ) + data = request["candidate"]["data"] + if not dataset_file.is_file(): + raise Rejected("data_access_failed", "the fetched dataset file is missing") + if ( + dataset_file.stat().st_size != data["size_bytes"] + or variants.sha256_file(dataset_file) != data["sha256"] + ): + raise Rejected( + "artifact_hash_mismatch", + "the fetched dataset bytes differ from the request", + ) + checkout = fresh_checkout(work, root=root) + digests = prepare(request, venv, wheels, root=checkout) (work / "wheels.json").write_bytes(canonical_bytes(digests)) python = str(venv / "bin" / "python") (work / "request.json").write_bytes(canonical_bytes(request)) @@ -888,6 +1034,8 @@ def run_request(request: dict, state_dir: Path, work: Path, *, runner: str) -> d str(work / "request.json"), "--runtime-root", str(work / "runtime"), + "--dataset-file", + str(dataset_file), "--wheel-digests", str(work / "wheels.json"), "--variants-file", @@ -895,20 +1043,48 @@ def run_request(request: dict, state_dir: Path, work: Path, *, runner: str) -> d "--rejection-file", str(work / "rejection.json"), ], - cwd=ROOT, - env={**os.environ, "PYTHONPATH": str(ROOT)}, + cwd=checkout, + env=candidate_environment(checkout), capture_output=True, text=True, ) + print(probe_run.stdout, end="", flush=True) + print(probe_run.stderr, end="", file=sys.stderr, flush=True) if probe_run.returncode == PROBE_REJECTED: - refused = json.loads((work / "rejection.json").read_bytes()) + refused = _load(work / "rejection.json") raise Rejected(refused["code"], refused["detail"]) - if probe_run.returncode != 0: + if probe_run.returncode: raise subprocess.CalledProcessError( probe_run.returncode, "probe", probe_run.stdout, probe_run.stderr ) variant_id = probe_run.stdout.strip().splitlines()[-1] - record = variants.get_variant(variant_id, path=variants_file) + record = variants.get_variant( + variant_id, path=variants_file, root=checkout, allow_uncertified=True + ) + document = variants.load_document(root / "data/uk/replay_variants.json") + base = base_namespace(request, record, document, root=root) + namespace = variants.namespace_root(record, root=checkout) + # Keep the diagnosis reference available even if the candidate is legacy. + diagnostics = work / "diagnostics.json" + shutil.copy2(root / "results/uk/events/MODEL_DIAGNOSTICS.json", diagnostics) + if namespace.exists(): + shutil.rmtree(namespace) + namespace.mkdir(parents=True) + (namespace / "REQUEST.json").write_bytes(canonical_bytes(request)) + shutil.copy2(work / "INSTALL.json", namespace / "INSTALL.json") + if base is not None: + base_id = ( + _load(next(base.glob("*/RUN_MANIFEST.json"))).get("variant_id") + or document["default_variant"] + ) + diff_dir = ( + checkout + / variants.REPLAY_RESULT_ROOT + / "diffs" + / f"{base_id}__{variant_id}" + ) + if diff_dir.exists(): + shutil.rmtree(diff_dir) run( [ python, @@ -919,38 +1095,33 @@ def run_request(request: dict, state_dir: Path, work: Path, *, runner: str) -> d variant_id, "--variants-file", str(variants_file), + "--dataset-file", + str(dataset_file), + "--diagnostics-file", + str(diagnostics), "--failures", str(work / "failures.json"), - ] + ], + cwd=checkout, ) - failures = json.loads((work / "failures.json").read_bytes()) - document = variants.load_document(variants_file) - base = base_namespace(request, record, document) + failures = _load(work / "failures.json") if base is not None: - base_id = ( - json.loads(next(base.glob("*/RUN_MANIFEST.json")).read_bytes()).get( - "variant_id" - ) - or variants.get_variant(path=variants_file)["variant_id"] - ) - diff_dir = ( - ROOT - / variants.REPLAY_RESULT_ROOT - / "diffs" - / f"{base_id}__{variant_id}" - ) run( [ python, "-m", - "pipeline.diff_uk_event_runs", + "pipeline.uk_replay_qualify", + "diff", "--base", str(base), + "--base-root", + str(root), "--head", - str(variants.namespace_root(record)), + str(namespace), "--output-dir", str(diff_dir), - ] + ], + cwd=checkout, ) except Rejected as exc: rejection = exc @@ -961,43 +1132,52 @@ def run_request(request: dict, state_dir: Path, work: Path, *, runner: str) -> d "detail": f"{exc.cmd}: exit {exc.returncode}", } ) - except Exception as exc: # noqa: BLE001 - a crash must still produce a receipt + except Exception as exc: # noqa: BLE001 - unexpected failures still produce a receipt failures.append( { "code": "unexpected_error", "detail": f"{type(exc).__name__}: {exc}".splitlines()[0][:500], } ) - receipt = build_receipt( - request, - record=None if rejection else record, - rejection=rejection, - failures=failures, - diff_dir=diff_dir, - results_path=results_path, - runner=runner, - ) - if record is not None and rejection is None: - namespace = variants.namespace_root(record) - sealed = seal(namespace, receipt, diff_dir=diff_dir) - target = destination / "results" - shutil.copytree(namespace, target / record["result_root"], dirs_exist_ok=True) - if diff_dir is not None and diff_dir.is_dir(): - shutil.copytree( - diff_dir, - target / diff_dir.relative_to(ROOT), - dirs_exist_ok=True, + destination.mkdir(parents=True, exist_ok=True) + if target.exists(): + shutil.rmtree(target) + try: + receipt = build_receipt( + request, + record=record, + rejection=rejection, + failures=failures, + diff_dir=diff_dir, + results_path=results_path, + runner=runner, + root=checkout or root, + ) + if record is not None and rejection is None: + namespace = variants.namespace_root(record, root=checkout) + namespace.mkdir(parents=True, exist_ok=True) + sealed = seal(namespace, receipt, diff_dir=diff_dir) + shutil.copytree(namespace, target / record["result_root"]) + if diff_dir is not None and diff_dir.is_dir(): + shutil.copytree(diff_dir, target / diff_dir.relative_to(checkout)) + requirements = checkout / record["requirements"]["path"] + (target / record["requirements"]["path"]).parent.mkdir( + parents=True, exist_ok=True ) - requirements = ROOT / record["requirements"]["path"] - (target / record["requirements"]["path"]).parent.mkdir( - parents=True, exist_ok=True + shutil.copy2(requirements, target / record["requirements"]["path"]) + (target / "VARIANT.json").write_bytes(canonical_bytes(record)) + else: + sealed = interface.seal_receipt(receipt) + interface.validate(sealed, interface.RECEIPT_SCHEMA_ID) + except Exception as exc: # noqa: BLE001 - parsing, sealing and copying are all recoverable + if target.exists(): + shutil.rmtree(target) + sealed = minimal_failed_receipt( + rid, + record=record, + results_path=results_path, + detail=f"{type(exc).__name__}: {exc}".splitlines()[0][:500], ) - shutil.copy2(requirements, target / record["requirements"]["path"]) - (target / "VARIANT.json").write_bytes(canonical_bytes(record)) - else: - sealed = interface.seal_receipt(receipt) - interface.validate(sealed, interface.RECEIPT_SCHEMA_ID) - destination.mkdir(parents=True, exist_ok=True) (destination / variants.RECEIPT_NAME).write_bytes(canonical_bytes(sealed)) return sealed @@ -1014,6 +1194,10 @@ def main(argv: list[str] | None = None) -> int: commands = parser.add_subparsers(dest="command", required=True) validate = commands.add_parser("validate") validate.add_argument("--request", type=Path, required=True) + fetch = commands.add_parser("fetch-dataset") + fetch.add_argument("--request", type=Path, required=True) + fetch.add_argument("--output", type=Path, required=True) + fetch.add_argument("--result-file", type=Path, required=True) prep = commands.add_parser("prepare") prep.add_argument("--request", type=Path, required=True) prep.add_argument("--venv", type=Path, required=True) @@ -1030,6 +1214,9 @@ def main(argv: list[str] | None = None) -> int: request_parser.add_argument("--request", type=Path, required=True) request_parser.add_argument("--state-dir", type=Path, required=True) request_parser.add_argument("--work", type=Path, required=True) + request_parser.add_argument("--dataset-file", type=Path) + request_parser.add_argument("--dataset-result", type=Path) + request_parser.add_argument("--result-file", type=Path, required=True) request_parser.add_argument( "--runner", default=os.environ.get("REPLAY_RUNNER", "local") ) @@ -1038,7 +1225,13 @@ def main(argv: list[str] | None = None) -> int: corpus.add_argument("--variants-file", type=Path, required=True) corpus.add_argument("--workers", type=int, choices=(1, 2), default=2) corpus.add_argument("--dataset-file", type=Path) + corpus.add_argument("--diagnostics-file", type=Path) corpus.add_argument("--failures", type=Path, required=True) + diff_parser = commands.add_parser("diff") + diff_parser.add_argument("--base", type=Path, required=True) + diff_parser.add_argument("--base-root", type=Path, required=True) + diff_parser.add_argument("--head", type=Path, required=True) + diff_parser.add_argument("--output-dir", type=Path, required=True) receipt = commands.add_parser("receipt") receipt.add_argument("--request", type=Path, required=True) receipt.add_argument("--variant") @@ -1057,6 +1250,22 @@ def main(argv: list[str] | None = None) -> int: if args.command == "validate": print(request_id(_load(args.request))) return 0 + if args.command == "fetch-dataset": + try: + request = _load(args.request) + request_id(request) + downloaded = fetch_dataset(request) + args.output.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(downloaded, args.output) + result = {"dataset_file": str(args.output.resolve()), "rejection": None} + except Rejected as exc: + result = { + "dataset_file": None, + "rejection": {"code": exc.code, "detail": exc.detail}, + } + args.result_file.parent.mkdir(parents=True, exist_ok=True) + args.result_file.write_bytes(canonical_bytes(result)) + return 0 if args.command == "prepare": digests = prepare( _load(args.request), args.venv.resolve(), args.wheels.resolve() @@ -1088,24 +1297,55 @@ def main(argv: list[str] | None = None) -> int: return 0 if args.command == "run-request": receipt = run_request( - _load(args.request), args.state_dir, args.work.resolve(), runner=args.runner + _load(args.request), + args.state_dir.resolve(), + args.work.resolve(), + runner=args.runner, + dataset_file=args.dataset_file.resolve() if args.dataset_file else None, + dataset_result=args.dataset_result, + ) + args.result_file.parent.mkdir(parents=True, exist_ok=True) + args.result_file.write_bytes( + canonical_bytes( + { + "status": receipt["status"], + "receipt_sha256": receipt["receipt_sha256"], + } + ) ) print(f"{receipt['status']} {receipt['receipt_sha256']}") return 0 if args.command == "corpus": - record = variants.get_variant(args.variant, path=args.variants_file.resolve()) + record = variants.get_variant( + args.variant, path=args.variants_file.resolve(), allow_uncertified=True + ) failures = run_corpus( record, args.variants_file.resolve(), workers=args.workers, dataset_file=args.dataset_file, + diagnostics_file=args.diagnostics_file, ) args.failures.write_bytes(canonical_bytes(failures)) return 0 + if args.command == "diff": + from pipeline import diff_uk_event_runs as diff + + base = diff.load_side(args.base, artifact_root=args.base_root) + head = diff.load_side(args.head, artifact_root=ROOT) + # Loading verified each side against its own checkout. The published + # diff names repository-relative roots rather than temporary runners. + base["root"] = ROOT / args.base.resolve().relative_to(args.base_root.resolve()) + files = diff.render(base, head) + diff.write(files, args.output_dir) + print(f"wrote {len(files)} diff files to {args.output_dir}") + return 0 if args.command == "receipt": request = _load(args.request) record = ( - variants.get_variant(args.variant, path=args.variants_file.resolve()) + variants.get_variant( + args.variant, path=args.variants_file.resolve(), allow_uncertified=True + ) if args.variant else None ) @@ -1141,13 +1381,19 @@ def main(argv: list[str] | None = None) -> int: return 0 if args.command == "verify-sealed": namespaces = args.namespaces or sorted( - path.parent - for path in (ROOT / variants.REPLAY_RESULT_ROOT).glob( - f"*/{variants.RECEIPT_NAME}" - ) + variants.namespace_root(record) + for record in variants.load_document()["variants"] + if record["layout"] == "replay_namespace" + and variants.namespace_root(record).exists() ) problems = [] for namespace in namespaces: + if ( + not (namespace / variants.RECEIPT_NAME).is_file() + or not (namespace / MANIFEST_NAME).is_file() + ): + problems += verify_sealed(namespace) + continue receipt = json.loads((namespace / variants.RECEIPT_NAME).read_bytes()) diff = None if receipt.get("diff"): diff --git a/pipeline/uk_replay_queue.py b/pipeline/uk_replay_queue.py index ea8f466..8e9b606 100644 --- a/pipeline/uk_replay_queue.py +++ b/pipeline/uk_replay_queue.py @@ -3,7 +3,7 @@ GitHub keeps at most one pending run per concurrency group and cancels older pending runs, so a request must be written down before anything else can lose it. Each request lives at ``requests//`` in a checkout of -the state branch: ``REQUEST.json`` (the request as received), ``STATUS.json`` +the state branch: ``REQUEST.json`` (the re-serialised received request), ``STATUS.json`` (schema policyengine.uk-replay-status/v1) and, once terminal, ``REPLAY_RECEIPT.json`` and ``results/``. Enqueueing is idempotent: a request already present keeps its status. The workflow commits and pushes @@ -14,6 +14,7 @@ import argparse import json +import shutil import sys from datetime import datetime, timedelta from pathlib import Path @@ -29,6 +30,47 @@ # A running request whose last update is older than this has lost its run # (a cancelled or crashed job never writes a terminal status). STALE_RUNNING = timedelta(hours=7) +MAX_ATTEMPTS = 3 + + +def running_attempts(status: dict) -> int: + return sum(entry["status"] == "running" for entry in status["history"]) + + +def write_failure_receipt( + state: Path, request_id: str, *, code: str, detail: str +) -> dict: + """Write a minimal failed receipt; the recorder commits it before status.""" + directory = request_dir(state, request_id) + receipt = interface.seal_receipt( + { + "schema": interface.RECEIPT_SCHEMA_ID, + "request_id": request_id, + "status": "failed", + "rejection": None, + "variant_id": None, + "variant_identity_sha256": None, + "execution": None, + "corpus": None, + "integrity": { + "status": "failed", + "failures": [{"code": code, "detail": detail.splitlines()[0]}], + }, + "diff": None, + "results": { + "repository": "PolicyEngine/policyengine-scorecard", + "ref": "replay-state", + "path": f"requests/{request_id}/results", + }, + } + ) + interface.validate(receipt, interface.RECEIPT_SCHEMA_ID) + directory.mkdir(parents=True, exist_ok=True) + results = directory / "results" + if results.exists(): + shutil.rmtree(results) + (directory / "REPLAY_RECEIPT.json").write_bytes(canonical_bytes(receipt)) + return receipt def canonical_bytes(value) -> bytes: @@ -101,6 +143,8 @@ def set_status( ) if status not in STATUSES: raise ValueError(f"unknown status {status}") + if status == "running" and running_attempts(current) >= MAX_ATTEMPTS: + raise ValueError(f"{request_id} exhausted its {MAX_ATTEMPTS} running attempts") updated = { **current, "status": status, @@ -122,7 +166,11 @@ def _at(value: str) -> datetime: def pending(state: Path, *, now: str) -> list[str]: - """Requests to run, oldest first: queued, or running past the stale bound.""" + """Requests to handle, oldest first: queued, or running past the stale bound. + + Exhausted requests stay eligible so the control job can commit their + failure receipt, then terminal status. They cannot enter running again. + """ out = [] for path in sorted((state / "requests").glob("*/STATUS.json")): status = json.loads(path.read_bytes()) @@ -153,6 +201,14 @@ def main(argv: list[str] | None = None) -> int: mark.add_argument("--now", required=True) mark.add_argument("--receipt-sha256") mark.add_argument("--receipt-commit") + attempts = commands.add_parser("attempts") + attempts.add_argument("--state", type=Path, required=True) + attempts.add_argument("--request-id", required=True) + failure = commands.add_parser("failure-receipt") + failure.add_argument("--state", type=Path, required=True) + failure.add_argument("--request-id", required=True) + failure.add_argument("--code", required=True) + failure.add_argument("--detail", required=True) args = parser.parse_args(argv) if args.command == "enqueue": request = json.loads(args.request.read_text(encoding="utf-8")) @@ -161,7 +217,19 @@ def main(argv: list[str] | None = None) -> int: ) print(f"{request_id} {status}") elif args.command == "pending": - print("\n".join(pending(args.state, now=args.now))) + requests = pending(args.state, now=args.now) + if requests: + print("\n".join(requests)) + elif args.command == "attempts": + status = read_status(args.state, args.request_id) + if status is None: + raise ValueError(f"{args.request_id} is not queued") + print(running_attempts(status)) + elif args.command == "failure-receipt": + receipt = write_failure_receipt( + args.state, args.request_id, code=args.code, detail=args.detail + ) + print(receipt["receipt_sha256"]) else: set_status( args.state, diff --git a/tests/test_uk_replay_qualify.py b/tests/test_uk_replay_qualify.py index 1a9516c..22a0a88 100644 --- a/tests/test_uk_replay_qualify.py +++ b/tests/test_uk_replay_qualify.py @@ -338,3 +338,595 @@ def test_pending_never_returns_terminal_requests(tmp_path_factory, statuses): later = queue.pending(state, now="2026-10-12T09:00:00Z") assert finished not in later assert set(later) == set(ids[1:]) + + +def test_rejection_discards_a_supplied_record_and_cli_accepts_candidate(tmp_path): + request = golden_request() + record = copy.deepcopy(rv.get_variant(CURRENT)) + record["status"] = "candidate" + document = rv.load_document() + document["variants"][-1] = record + variants_file = tmp_path / "variants.json" + variants_file.write_bytes(qualify.canonical_bytes(document)) + request_file = tmp_path / "request.json" + request_file.write_bytes(qualify.canonical_bytes(request)) + output = tmp_path / "receipt.json" + assert ( + qualify.main( + [ + "receipt", + "--request", + str(request_file), + "--variant", + CURRENT, + "--variants-file", + str(variants_file), + "--rejection", + "incompatible_pair: refused", + "--results-path", + "requests/x/results/", + "--output", + str(output), + ] + ) + == 0 + ) + receipt = json.loads(output.read_bytes()) + interface.validate(receipt, interface.RECEIPT_SCHEMA_ID) + assert receipt["variant_id"] is None + assert receipt["variant_identity_sha256"] is None + assert receipt["execution"] is None + assert receipt["integrity"] == {"status": "not_run", "failures": []} + + +def test_identity_without_event_outputs_has_a_valid_failed_receipt(tmp_path): + record = rv.get_variant(CURRENT) + built = qualify.build_receipt( + golden_request(), + record=record, + root=tmp_path, + failures=[{"code": "event_run_failed", "detail": "preflight failed"}], + results_path="requests/x/results/", + ) + assert built["corpus"] is None + assert built["execution"] is not None + assert built["variant_identity_sha256"] == record["identity_sha256"] + namespace = rv.namespace_root(record, root=tmp_path) + namespace.mkdir(parents=True) + receipt = qualify.seal(namespace, built) + interface.validate(receipt, interface.RECEIPT_SCHEMA_ID) + assert receipt["status"] == "failed" + + +def test_corpus_passes_candidate_and_work_variants_without_resume( + tmp_path, monkeypatch +): + record = rv.get_variant(CURRENT) + event = qualify.EVENTS[0] + directory = rv.namespace_root(record, root=tmp_path) / event + directory.mkdir(parents=True) + (directory / "RUN_MANIFEST.json").write_text( + json.dumps({"supported_years": [2024], "unsupported_years": []}) + ) + calls = [] + monkeypatch.setattr( + qualify, "run", lambda command, *, cwd: calls.append((command, cwd)) + ) + work_variants = tmp_path / "variants.json" + assert ( + qualify.run_corpus(record, work_variants, events=(event,), root=tmp_path) == [] + ) + compute = calls[0][0] + assert "--allow-uncertified" in compute + assert "--resume" not in compute + staging = calls[1][0] + assert staging[staging.index("--variants-file") + 1] == str(work_variants) + assert all(cwd == tmp_path for _, cwd in calls) + + +def local_runner(tmp_path, variant_id=CURRENT): + import shutil + import subprocess + + root = tmp_path / "runner" + root.mkdir() + record = rv.get_variant(variant_id) + for relative in ( + "data/uk/replay_variants.json", + "data/uk/certified_bundle.json", + record["requirements"]["path"], + "results/uk/events/MODEL_DIAGNOSTICS.json", + ): + target = root / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(qualify.ROOT / relative, target) + namespace = rv.namespace_root(record, root=root) + (namespace / qualify.EVENTS[0]).mkdir(parents=True) + (namespace / qualify.EVENTS[0] / "RUN_MANIFEST.json").write_text( + json.dumps({"variant_id": variant_id}) + ) + (namespace / qualify.EVENTS[0] / "COMPARISON.json").write_text( + "committed comparison" + ) + (namespace / "committed-only.txt").write_text( + "old results must be deleted in the fresh checkout" + ) + (namespace / rv.RECEIPT_NAME).write_text("committed receipt") + diff = root / rv.REPLAY_RESULT_ROOT / "diffs" / f"{variant_id}__{variant_id}" + diff.mkdir(parents=True) + (diff / "committed-only.txt").write_text("old diff") + for command in ( + ["git", "init", "-q"], + ["git", "config", "user.email", "test@example.com"], + ["git", "config", "user.name", "Test"], + ["git", "add", "."], + ["git", "commit", "-qm", "fixture"], + ): + subprocess.run(command, cwd=root, check=True) + dataset = tmp_path / "dataset.h5" + dataset.write_bytes(b"dataset fixture") + request = golden_request() + request["candidate"]["data"]["sha256"] = rv.sha256_file(dataset) + request["candidate"]["data"]["size_bytes"] = dataset.stat().st_size + return root, record, dataset, request + + +def stub_candidate(monkeypatch, root, record, *, no_outputs=False): + import subprocess + + real_run = subprocess.run + calls = [] + + def prepare(request, venv, wheels, *, root): + (wheels.parent / "INSTALL.json").write_bytes( + qualify.canonical_bytes({"resolution": "certified_constraints"}) + ) + return {} + + def process(command, **kwargs): + if "probe" not in command: + return real_run(command, **kwargs) + calls.append((command, kwargs["cwd"], kwargs["env"])) + document = rv.load_document(root / "data/uk/replay_variants.json") + Path(command[command.index("--variants-file") + 1]).write_bytes( + qualify.canonical_bytes(document) + ) + return subprocess.CompletedProcess(command, 0, record["variant_id"] + "\n", "") + + def run(command, *, cwd): + if command[0] == "git": + return real_run(command, cwd=cwd, check=True) + calls.append((command, cwd, qualify.candidate_environment(cwd))) + namespace = rv.namespace_root(record, root=cwd) + if "corpus" in command: + assert not (namespace / "committed-only.txt").exists() + assert not (namespace / rv.RECEIPT_NAME).exists() + if not no_outputs: + for event in qualify.EVENTS: + directory = namespace / event + directory.mkdir(parents=True) + (directory / "RUN_MANIFEST.json").write_bytes( + qualify.canonical_bytes( + { + "registry_sha256": "a" * 64, + "supported_years": [2024], + "unsupported_years": [], + "artifacts": {}, + } + ) + ) + (directory / "STAGING_MANIFEST.json").write_text( + '{"full_supported_grid_complete":true}' + ) + (directory / "COMPARISON_PROVENANCE.json").write_text("{}") + Path(command[command.index("--failures") + 1]).write_text("[]") + elif "diff" in command: + base = Path(command[command.index("--base") + 1]) + assert base == rv.namespace_root(record, root=root) + assert (base / rv.RECEIPT_NAME).read_text() == "committed receipt" + assert command[command.index("--base-root") + 1] == str(root) + diff = Path(command[command.index("--output-dir") + 1]) + assert not (diff / "committed-only.txt").exists() + diff.mkdir(parents=True) + (diff / "DIFF.json").write_bytes( + qualify.canonical_bytes( + { + "base": {"identity": {"variant_id": record["variant_id"]}}, + "flag_counts": { + flag: 0 + for flag in ( + "numerical", + "structural", + "coverage", + "mapping", + "support", + "inertness", + "construction", + ) + }, + "rows": [], + } + ) + ) + else: + raise AssertionError(command) + + monkeypatch.setattr(qualify, "prepare", prepare) + monkeypatch.setattr(qualify.subprocess, "run", process) + monkeypatch.setattr(qualify, "run", run) + return calls, real_run + + +def test_requalification_uses_fresh_results_and_leaves_runner_unchanged( + tmp_path, monkeypatch +): + root, record, dataset, request = local_runner(tmp_path) + for name in ("HUGGING_FACE_TOKEN", "HF_TOKEN", "HUGGINGFACE_HUB_TOKEN"): + monkeypatch.setenv(name, "secret") + calls, real_run = stub_candidate(monkeypatch, root, record) + state = tmp_path / "state" + rid = qualify.request_id(request) + target = state / "requests" / rid / "results" + target.mkdir(parents=True) + (target / "previous-attempt.txt").write_text("stale") + work = tmp_path / "work" + receipt = qualify.run_request( + request, state, work, runner="test", dataset_file=dataset, root=root + ) + interface.validate(receipt, interface.RECEIPT_SCHEMA_ID) + assert receipt["status"] == "complete" + assert receipt["diff"]["base_variant_id"] == CURRENT + assert not (target / "previous-attempt.txt").exists() + copied = target / record["result_root"] + diff = target / rv.REPLAY_RESULT_ROOT / "diffs" / f"{CURRENT}__{CURRENT}" + assert qualify.verify_sealed(copied, diff_dir=diff) == [] + assert ( + "INSTALL.json" + in json.loads((copied / qualify.MANIFEST_NAME).read_bytes())["files"] + ) + assert all(cwd == work / "checkout" for _, cwd, _ in calls) + assert all( + not any( + name in env + for name in ("HUGGING_FACE_TOKEN", "HF_TOKEN", "HUGGINGFACE_HUB_TOKEN") + ) + for _, _, env in calls + ) + assert ( + real_run( + ["git", "status", "--porcelain"], + cwd=root, + capture_output=True, + text=True, + check=True, + ).stdout + == "" + ) + assert ( + rv.namespace_root(record, root=root) / rv.RECEIPT_NAME + ).read_text() == "committed receipt" + + +@pytest.mark.parametrize( + "crash_step", ["build_receipt", "seal", "copytree", "event_results"] +) +def test_receipt_step_crashes_produce_minimal_valid_failed_receipts( + tmp_path, monkeypatch, crash_step +): + root, record, dataset, request = local_runner(tmp_path) + stub_candidate(monkeypatch, root, record) + + def crash(*args, **kwargs): + raise RuntimeError("first line\nprivate traceback details") + + if crash_step == "copytree": + monkeypatch.setattr(qualify.shutil, "copytree", crash) + else: + monkeypatch.setattr(qualify, crash_step, crash) + state = tmp_path / "state" + receipt = qualify.run_request( + request, + state, + tmp_path / "work", + runner="test", + dataset_file=dataset, + root=root, + ) + interface.validate(receipt, interface.RECEIPT_SCHEMA_ID) + assert receipt["status"] == "failed" + assert receipt["variant_id"] == CURRENT + assert receipt["corpus"] is None + assert receipt["integrity"]["failures"] == [ + {"code": "receipt_step_failed", "detail": "RuntimeError: first line"} + ] + assert not (state / "requests" / qualify.request_id(request) / "results").exists() + + +def test_run_request_with_identity_and_no_outputs_still_seals_failure( + tmp_path, monkeypatch +): + root, record, dataset, request = local_runner(tmp_path) + stub_candidate(monkeypatch, root, record, no_outputs=True) + receipt = qualify.run_request( + request, + tmp_path / "state", + tmp_path / "work", + runner="test", + dataset_file=dataset, + root=root, + ) + interface.validate(receipt, interface.RECEIPT_SCHEMA_ID) + assert receipt["status"] == "failed" and receipt["corpus"] is None + assert receipt["execution"] is not None + assert any( + f["code"] == "event_outputs_missing" for f in receipt["integrity"]["failures"] + ) + + +@pytest.mark.parametrize( + "missing", ["REPLAY_RECEIPT.json", "NAMESPACE_MANIFEST.json", "both"] +) +def test_verify_sealed_checks_expected_namespace_with_missing_seal( + tmp_path, monkeypatch, missing, capsys +): + namespace, _, _ = sealed_namespace(tmp_path) + if missing in ("REPLAY_RECEIPT.json", "both"): + (namespace / rv.RECEIPT_NAME).unlink() + if missing in ("NAMESPACE_MANIFEST.json", "both"): + (namespace / qualify.MANIFEST_NAME).unlink() + monkeypatch.setattr( + rv, + "load_document", + lambda: { + "variants": [{"layout": "replay_namespace", "result_root": str(namespace)}] + }, + ) + assert qualify.main(["verify-sealed"]) == 1 + assert "not sealed" in capsys.readouterr().out + + +@pytest.mark.parametrize("code", [None, "artifact_hash_mismatch", "data_access_failed"]) +def test_dataset_fetch_cli_hands_file_or_rejection_to_run_request( + tmp_path, monkeypatch, code +): + request_path = tmp_path / "request.json" + request_path.write_bytes(qualify.canonical_bytes(golden_request())) + source = tmp_path / "source.h5" + source.write_bytes(b"verified dataset") + + def fetch(request): + if code: + raise qualify.Rejected(code, "fetch detail") + return source + + monkeypatch.setattr(qualify, "fetch_dataset", fetch) + result = tmp_path / "fetch-result.json" + output = tmp_path / "output.h5" + assert ( + qualify.main( + [ + "fetch-dataset", + "--request", + str(request_path), + "--output", + str(output), + "--result-file", + str(result), + ] + ) + == 0 + ) + fetched = json.loads(result.read_bytes()) + if code: + assert fetched == { + "dataset_file": None, + "rejection": {"code": code, "detail": "fetch detail"}, + } + receipt = qualify.run_request( + golden_request(), + tmp_path / "state", + tmp_path / "work", + runner="test", + dataset_file=output, + dataset_result=result, + ) + interface.validate(receipt, interface.RECEIPT_SCHEMA_ID) + assert receipt["rejection"]["code"] == code + else: + assert output.read_bytes() == source.read_bytes() + assert fetched == {"dataset_file": str(output), "rejection": None} + + +def test_run_request_cli_writes_outcome_file(tmp_path, monkeypatch): + request = tmp_path / "request.json" + request.write_bytes(qualify.canonical_bytes(golden_request())) + receipt = qualify.minimal_failed_receipt( + qualify.request_id(golden_request()), + record=None, + results_path="requests/x/results/", + detail="crash", + ) + monkeypatch.setattr(qualify, "run_request", lambda *args, **kwargs: receipt) + outcome = tmp_path / "outcome.json" + assert ( + qualify.main( + [ + "run-request", + "--request", + str(request), + "--state-dir", + str(tmp_path / "state"), + "--work", + str(tmp_path / "work"), + "--result-file", + str(outcome), + ] + ) + == 0 + ) + assert json.loads(outcome.read_bytes()) == { + "status": "failed", + "receipt_sha256": receipt["receipt_sha256"], + } + + +@pytest.mark.parametrize( + "resolution", ["exact_requirements", "certified_constraints", "free_resolution"] +) +def test_prepare_records_resolution_mode(tmp_path, monkeypatch, resolution): + import subprocess + + request = golden_request() + if resolution != "exact_requirements": + request["candidate"]["policyengine_uk"]["wheel_sha256"] = "0" * 64 + attempts = [] + + def run(command, *, cwd): + if "-c" in command and resolution == "free_resolution": + raise subprocess.CalledProcessError(1, command) + attempts.append(command) + + monkeypatch.setattr(qualify, "run", run) + monkeypatch.setattr( + qualify.subprocess, + "run", + lambda command, **kwargs: subprocess.CompletedProcess(command, 0, "", ""), + ) + monkeypatch.setattr(qualify, "verify_wheels", lambda *args: {}) + qualify.prepare(request, tmp_path / "venv", tmp_path / "wheels") + installation = json.loads((tmp_path / "INSTALL.json").read_bytes()) + assert installation["resolution"] == resolution + assert installation["certified_variant_id"] == CURRENT + assert len(installation["environment_sha256"]) == 64 + assert attempts + + +def test_missing_pypi_wheel_is_artifact_hash_rejection(tmp_path, monkeypatch): + import subprocess + + monkeypatch.setattr(qualify, "run", lambda *args, **kwargs: None) + monkeypatch.setattr( + qualify.subprocess, + "run", + lambda command, **kwargs: subprocess.CompletedProcess( + command, + 1, + "", + "ERROR: No matching distribution found for policyengine-uk==0.0.1", + ), + ) + with pytest.raises(qualify.Rejected) as raised: + qualify.prepare(golden_request(), tmp_path / "venv", tmp_path / "wheels") + assert raised.value.code == "artifact_hash_mismatch" + + +def test_nested_receipt_named_files_are_bound_and_sealed_results_cannot_be_resealed( + tmp_path, +): + namespace, diff, sealed = sealed_namespace(tmp_path) + original_manifest = (namespace / qualify.MANIFEST_NAME).read_bytes() + original_receipt = (namespace / rv.RECEIPT_NAME).read_bytes() + with pytest.raises(rv.VariantError, match="sealed"): + qualify.seal(namespace, sealed, diff_dir=diff) + assert (namespace / qualify.MANIFEST_NAME).read_bytes() == original_manifest + assert (namespace / rv.RECEIPT_NAME).read_bytes() == original_receipt + for name in (rv.RECEIPT_NAME, qualify.MANIFEST_NAME): + nested = namespace / "event" / name + nested.write_text("added file") + assert any( + "files differ" in problem + for problem in qualify.verify_sealed(namespace, diff_dir=diff) + ) + nested.unlink() + + +def test_corpus_refuses_a_sealed_namespace_before_any_subprocess(tmp_path, monkeypatch): + record = copy.deepcopy(rv.get_variant(CURRENT)) + namespace, _, _ = sealed_namespace(tmp_path) + record["result_root"] = str(namespace) + monkeypatch.setattr( + qualify, + "run", + lambda *args, **kwargs: pytest.fail("must not start a subprocess"), + ) + with pytest.raises(rv.VariantError, match="sealed"): + qualify.run_corpus(record, tmp_path / "variants.json", root=tmp_path) + + +def test_legacy_requalification_preserves_original_diagnostics(tmp_path, monkeypatch): + legacy_id = rv.load_document()["default_variant"] + root, record, dataset, request = local_runner(tmp_path, variant_id=legacy_id) + _, real_run = stub_candidate(monkeypatch, root, record) + receipt = qualify.run_request( + request, + tmp_path / "state", + tmp_path / "work", + runner="test", + dataset_file=dataset, + root=root, + ) + interface.validate(receipt, interface.RECEIPT_SCHEMA_ID) + assert receipt["status"] == "complete" + assert receipt["variant_id"] == legacy_id + diagnostics = root / "results/uk/events/MODEL_DIAGNOSTICS.json" + assert (tmp_path / "work/diagnostics.json").read_bytes() == diagnostics.read_bytes() + assert ( + real_run( + ["git", "status", "--porcelain"], + cwd=root, + capture_output=True, + text=True, + check=True, + ).stdout + == "" + ) + + +def test_diff_reads_base_and_head_artifacts_from_their_own_checkouts( + tmp_path, monkeypatch +): + from test_uk_event_diff import make_namespace + from pipeline import compute_uk_event as compute + from pipeline import diff_uk_event_runs as diff + + base_root = tmp_path / "runner" + head_root = tmp_path / "checkout" + monkeypatch.setattr(compute, "ROOT", base_root) + base = make_namespace(base_root, "pair") + monkeypatch.setattr(compute, "ROOT", head_root) + head = make_namespace( + head_root, + "pair", + effects={ + 2024: {"income_tax": 200e6, "universal_credit": -60e6}, + 2025: {"income_tax": 220e6, "universal_credit": -66e6}, + }, + ) + monkeypatch.setattr(qualify, "ROOT", head_root) + monkeypatch.setattr(diff, "ROOT", head_root) + output = head_root / "diff" + command = [ + "diff", + "--base", + str(base["root"]), + "--base-root", + str(base_root), + "--head", + str(head["root"]), + "--output-dir", + str(output), + ] + assert qualify.main(command) == 0 + result = json.loads((output / "DIFF.json").read_bytes()) + assert result["flag_counts"]["numerical"] > 0 + row = next( + row + for row in result["rows"] + if row["level"] == "head" and row["key"] == "income_tax" and row["year"] == 2024 + ) + assert row["base_gbp"] == -100e6 and row["head_gbp"] == -200e6 + assert result["base"]["root"] == result["head"]["root"] == "ns/pair" + artifact = next((base["root"] / "test_event").glob("test_event__*.json")) + artifact.write_text("tampered committed artifact") + with pytest.raises(diff.DiffError): + qualify.main(command) diff --git a/tests/test_uk_replay_queue.py b/tests/test_uk_replay_queue.py new file mode 100644 index 0000000..56b5c2e --- /dev/null +++ b/tests/test_uk_replay_queue.py @@ -0,0 +1,545 @@ +"""Queue bounds and control-job behavior against local git repositories.""" + +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +from pipeline import uk_replay_interface as interface +from pipeline import uk_replay_queue as queue + +ROOT = Path(__file__).resolve().parent.parent +RUN = "https://github.com/PolicyEngine/policyengine-scorecard/actions/runs/1" + + +def git(directory, *args): + return subprocess.run( + ["git", "-C", str(directory), *args], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def local_state(tmp_path): + state = tmp_path / "state" + state.mkdir() + git(state, "init", "-q") + git(state, "config", "user.name", "Test") + git(state, "config", "user.email", "test@example.org") + return state + + +def test_fourth_stale_attempt_is_terminal_with_a_committed_failed_receipt(tmp_path): + state = local_state(tmp_path) + request_id, _ = queue.enqueue( + state, {"invalid": "request"}, run_url=RUN, now="2026-10-01T00:00:00Z" + ) + for day in (1, 2, 3): + queue.set_status( + state, + request_id, + "running", + run_url=RUN, + now=f"2026-10-0{day}T01:00:00Z", + ) + assert queue.pending(state, now="2026-10-04T09:00:00Z") == [request_id] + with pytest.raises(ValueError, match="exhausted its 3 running attempts"): + queue.set_status( + state, + request_id, + "running", + run_url=RUN, + now="2026-10-04T09:00:00Z", + ) + stale_results = state / "requests" / request_id / "results" + stale_results.mkdir() + (stale_results / "old-attempt.json").write_text("{}") + receipt = queue.write_failure_receipt( + state, + request_id, + code="attempt_limit_exceeded", + detail="The request exhausted three running attempts", + ) + git(state, "add", ".") + git(state, "commit", "-qm", "Failure receipt") + receipt_commit = git(state, "rev-parse", "HEAD") + status = queue.set_status( + state, + request_id, + "failed", + run_url=RUN, + now="2026-10-04T09:00:00Z", + receipt_sha256=receipt["receipt_sha256"], + receipt_commit=receipt_commit, + ) + assert not stale_results.exists() + interface.validate(receipt, interface.RECEIPT_SCHEMA_ID) + interface.validate(status, interface.STATUS_SCHEMA_ID) + assert queue.running_attempts(status) == 3 + assert queue.pending(state, now="2026-10-05T09:00:00Z") == [] + committed = json.loads( + git( + state, "show", f"{receipt_commit}:requests/{request_id}/REPLAY_RECEIPT.json" + ) + ) + assert committed == receipt + + +@pytest.mark.parametrize("exhaust_push_retries", [False, True]) +def test_enqueue_reapplies_after_a_real_concurrent_git_conflict( + tmp_path, exhaust_push_retries +): + """The first push loses to another enqueue of exactly the same request.""" + seed = tmp_path / "seed" + seed.mkdir() + git(seed, "init", "-q", "-b", "main") + git(seed, "config", "user.name", "Test") + git(seed, "config", "user.email", "test@example.org") + for relative in ( + "pipeline/uk_replay_queue.py", + "pipeline/uk_replay_interface.py", + ".github/scripts/uk_replay_state.sh", + ".github/scripts/uk_replay_enqueue.sh", + ): + destination = seed / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ROOT / relative, destination) + shutil.copytree(ROOT / "docs/uk_replay/schemas", seed / "docs/uk_replay/schemas") + git(seed, "add", ".") + git(seed, "commit", "-qm", "Code") + git(seed, "switch", "--orphan", "replay-state") + git(seed, "commit", "--allow-empty", "-qm", "Queue") + remote = tmp_path / "remote.git" + subprocess.run(["git", "clone", "-q", "--bare", str(seed), str(remote)], check=True) + runner, competitor = tmp_path / "runner", tmp_path / "competitor" + for path in (runner, competitor): + subprocess.run(["git", "clone", "-q", str(remote), str(path)], check=True) + git(path, "config", "user.name", "Test") + git(path, "config", "user.email", "test@example.org") + git(runner, "switch", "-q", "main") + request = {"invalid": "same received request"} + request_file = tmp_path / "request.json" + request_file.write_text(json.dumps(request)) + request_id, _ = queue.enqueue( + competitor, request, run_url=RUN, now="2026-10-10T00:00:00Z" + ) + git(competitor, "add", ".") + git(competitor, "commit", "-qm", "Concurrent queue") + competing_commit = git(competitor, "rev-parse", "HEAD") + # A wrapper injects the competing push at the first push from the script. + # All fetches, rebases, conflicts and subsequent pushes are real local git. + tools = tmp_path / "bin" + tools.mkdir() + real_git = shutil.which("git") + wrapper = tools / "git" + wrapper.write_text( + f"#!{sys.executable}\n" + "import os, subprocess, sys\n" + "from pathlib import Path\n" + "args = sys.argv[1:]\n" + "marker = Path(os.environ['PUSH_MARKER'])\n" + "if 'push' in args and not marker.exists():\n" + " marker.touch()\n" + f" subprocess.run([{real_git!r}, '-C', os.environ['COMPETITOR'], 'push', '-q', 'origin', 'HEAD:replay-state'], check=True)\n" + "if 'push' in args and os.environ['REJECT_PUSHES'] == '1':\n" + " sys.exit(1)\n" + f"os.execv({real_git!r}, [{real_git!r}, *args])\n" + ) + wrapper.chmod(0o755) + uv = tools / "uv" + uv.write_text( + f"#!{sys.executable}\nimport os, sys\n" + "args = sys.argv[1:]\n" + f"os.execv({sys.executable!r}, [{sys.executable!r}, *args[args.index('python') + 1:]])\n" + ) + uv.chmod(0o755) + sleep = tools / "sleep" + sleep.write_text('#!/usr/bin/env bash\nprintf "%s\\n" "$1" >> "$BACKOFF_LOG"\n') + sleep.chmod(0o755) + environment = { + **os.environ, + "PATH": f"{tools}:{os.environ['PATH']}", + "PYTHONPATH": str(runner), + "STATE_BRANCH": "replay-state", + "REQUEST_FILE": str(request_file), + "RUNNER_TEMP": str(tmp_path), + "RUN_URL": RUN, + "COMPETITOR": str(competitor), + "PUSH_MARKER": str(tmp_path / "pushed"), + "BACKOFF_LOG": str(tmp_path / "backoff"), + "REJECT_PUSHES": "1" if exhaust_push_retries else "0", + } + result = subprocess.run( + ["bash", ".github/scripts/uk_replay_enqueue.sh"], + cwd=runner, + env=environment, + capture_output=True, + text=True, + check=False, + ) + if exhaust_push_retries: + assert result.returncode == 1 + assert ( + "could not enqueue and push the request after five attempts" + in result.stderr + ) + assert (tmp_path / "backoff").read_text() == "1\n2\n3\n4\n5\n" + else: + assert result.returncode == 0 + assert (tmp_path / "backoff").read_text() == "1\n" + assert "enqueue conflicted; reapplying" in result.stdout + assert git(remote, "rev-parse", "replay-state") == competing_commit + status = json.loads( + git(remote, "show", f"replay-state:requests/{request_id}/STATUS.json") + ) + interface.validate(status, interface.STATUS_SCHEMA_ID) + assert len(status["history"]) == 1 + assert status["history"][0]["at"] == "2026-10-10T00:00:00Z" + assert git(runner / "state", "status", "--porcelain") == "" + + +def test_candidate_job_has_read_permission_and_no_dataset_token_or_credentials(): + workflow = yaml.safe_load((ROOT / ".github/workflows/uk-replay.yml").read_text()) + candidate = workflow["jobs"]["run"] + assert candidate["permissions"] == {"contents": "read"} + checkout = next( + step for step in candidate["steps"] if step.get("uses") == "actions/checkout@v4" + ) + assert checkout["with"]["persist-credentials"] is False + assert "HUGGING_FACE_TOKEN" not in json.dumps(candidate) + assert workflow["jobs"]["record"]["permissions"] == { + "contents": "write", + "actions": "write", + } + assert ( + "format('uk-replay-enqueue-{0}', github.run_id)" + in workflow["concurrency"]["group"] + ) + assert workflow["concurrency"]["cancel-in-progress"] is False + for job in workflow["jobs"].values(): + for step in job["steps"]: + assert "${{" not in step.get("run", "") + + +def test_ci_checks_untracked_regeneration_and_announces_skipped_real_replay(): + workflow = yaml.safe_load((ROOT / ".github/workflows/ci.yml").read_text()) + steps = workflow["jobs"]["python"]["steps"] + build = next(step for step in steps if "build1.json" in step.get("run", "")) + assert ' > "$RUNNER_TEMP/build1.json"' in build["run"] + assert ' > "$RUNNER_TEMP/build2.json"' in build["run"] + regeneration_index = next( + i + for i, step in enumerate(steps) + if step.get("name") == "Replay namespaces regenerate and seals hold" + ) + assert "git status --porcelain" in steps[regeneration_index + 1]["run"] + replay = next( + step + for step in workflow["jobs"]["replay-engine"]["steps"] + if step.get("name") == "One real measure-year replays byte for byte" + ) + assert '"$RESULT_ROOT/REPLAY_RECEIPT.json"' in replay["run"] + assert '"$RESULT_ROOT/NAMESPACE_MANIFEST.json"' in replay["run"] + assert replay["run"].count("nothing was replayed") == 2 + + +def test_nonzero_candidate_exit_keeps_logs_and_uploadable_outcome(tmp_path): + scripts = tmp_path / ".github/scripts" + scripts.mkdir(parents=True) + shutil.copyfile( + ROOT / ".github/scripts/uk_replay_drain.sh", scripts / "uk_replay_drain.sh" + ) + tools = tmp_path / "bin" + tools.mkdir() + uv = tools / "uv" + uv.write_text( + f"#!{sys.executable}\n" + "import json, sys\n" + "from pathlib import Path\n" + "args = sys.argv[1:]\n" + "print('corpus log before a crash', flush=True)\n" + "Path(args[args.index('--result-file') + 1]).write_text(json.dumps({'status': 'failed', 'receipt_sha256': 'a' * 64}))\n" + "sys.exit(17)\n" + ) + uv.chmod(0o755) + result = subprocess.run( + ["bash", ".github/scripts/uk_replay_drain.sh"], + cwd=tmp_path, + env={ + **os.environ, + "PATH": f"{tools}:{os.environ['PATH']}", + "REQUEST_ID": "b" * 64, + "RUNNER_TEMP": str(tmp_path), + "REPLAY_RUNNER": "test runner", + }, + capture_output=True, + text=True, + check=True, + ) + assert "corpus log before a crash" in result.stdout + assert "run-request exited 17" in result.stdout + assert (tmp_path / "output/run-exit.txt").read_text() == "17\n" + assert json.loads((tmp_path / "output/outcome.json").read_text()) == { + "status": "failed", + "receipt_sha256": "a" * 64, + } + + +def control_runner(tmp_path, seed): + """A code checkout and a queue remote, both temporary and entirely local.""" + git(seed, "branch", "-M", "replay-state") + remote = tmp_path / "remote.git" + subprocess.run(["git", "clone", "-q", "--bare", str(seed), str(remote)], check=True) + runner = tmp_path / "runner" + subprocess.run(["git", "clone", "-q", str(remote), str(runner)], check=True) + git(runner, "switch", "-q", "--detach") + for relative in ("pipeline/uk_replay_queue.py", "pipeline/uk_replay_interface.py"): + destination = runner / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ROOT / relative, destination) + shutil.copytree(ROOT / "docs/uk_replay/schemas", runner / "docs/uk_replay/schemas") + scripts = runner / ".github/scripts" + scripts.mkdir(parents=True) + for path in (ROOT / ".github/scripts").glob("uk_replay_*.sh"): + shutil.copyfile(path, scripts / path.name) + tools = tmp_path / "bin" + tools.mkdir() + uv = tools / "uv" + uv.write_text( + f"#!{sys.executable}\nimport os, sys\nargs = sys.argv[1:]\n" + f"os.execv({sys.executable!r}, [{sys.executable!r}, *args[args.index('python') + 1:]])\n" + ) + uv.chmod(0o755) + gh = tools / "gh" + gh.write_text( + "#!/usr/bin/env bash\necho 'unexpected dispatch in an empty local queue' >&2\nexit 93\n" + ) + gh.chmod(0o755) + environment = { + **os.environ, + "PATH": f"{tools}:{os.environ['PATH']}", + "PYTHONPATH": str(runner), + "STATE_BRANCH": "replay-state", + "RUNNER_TEMP": str(tmp_path), + "RUN_URL": RUN, + "GITHUB_OUTPUT": str(tmp_path / "job-output.txt"), + } + return runner, remote, environment + + +def test_prepare_terminalizes_the_fourth_stale_attempt_before_selecting_again(tmp_path): + seed = local_state(tmp_path) + request_id, _ = queue.enqueue( + seed, {"invalid": "request"}, run_url=RUN, now="2020-01-01T00:00:00Z" + ) + for day in (1, 2, 3): + queue.set_status( + seed, request_id, "running", run_url=RUN, now=f"2020-01-0{day}T01:00:00Z" + ) + git(seed, "add", ".") + git(seed, "commit", "-qm", "Three stale attempts") + runner, remote, environment = control_runner(tmp_path, seed) + subprocess.run( + ["bash", ".github/scripts/uk_replay_prepare.sh"], + cwd=runner, + env=environment, + check=True, + capture_output=True, + text=True, + ) + status = json.loads( + git(remote, "show", f"replay-state:requests/{request_id}/STATUS.json") + ) + interface.validate(status, interface.STATUS_SCHEMA_ID) + assert status["status"] == "failed" + assert queue.running_attempts(status) == 3 + receipt_commit = status["receipt_commit"] + receipt = json.loads( + git( + remote, + "show", + f"{receipt_commit}:requests/{request_id}/REPLAY_RECEIPT.json", + ) + ) + interface.validate(receipt, interface.RECEIPT_SCHEMA_ID) + assert receipt["receipt_sha256"] == status["receipt_sha256"] + assert receipt["integrity"]["failures"][0]["code"] == "attempt_limit_exceeded" + prior_status = json.loads( + git(remote, "show", f"{receipt_commit}:requests/{request_id}/STATUS.json") + ) + assert prior_status["status"] == "running" + assert (tmp_path / "job-output.txt").read_text() == "request_id=\n" + + +@pytest.mark.parametrize("invalid_outcome", [False, True]) +def test_recorder_handles_missing_or_invalid_run_artifact_and_clears_stale_results( + tmp_path, invalid_outcome +): + seed = local_state(tmp_path) + request_id, _ = queue.enqueue( + seed, {"invalid": "request"}, run_url=RUN, now="2020-01-01T00:00:00Z" + ) + queue.set_status( + seed, request_id, "running", run_url=RUN, now="2020-01-01T01:00:00Z" + ) + results = seed / "requests" / request_id / "results" + results.mkdir() + (results / "earlier-attempt.json").write_text("{}") + git(seed, "add", ".") + git(seed, "commit", "-qm", "Running request with stale results") + runner, remote, environment = control_runner(tmp_path, seed) + if invalid_outcome: + output = runner / "output" + output.mkdir() + (output / "outcome.json").write_text( + '{"status":"complete","receipt_sha256":"bad"}' + ) + subprocess.run( + ["bash", ".github/scripts/uk_replay_record.sh"], + cwd=runner, + env={**environment, "REQUEST_ID": request_id}, + check=True, + capture_output=True, + text=True, + ) + status = json.loads( + git(remote, "show", f"replay-state:requests/{request_id}/STATUS.json") + ) + interface.validate(status, interface.STATUS_SCHEMA_ID) + assert status["status"] == "failed" + receipt = json.loads( + git( + remote, + "show", + f"{status['receipt_commit']}:requests/{request_id}/REPLAY_RECEIPT.json", + ) + ) + interface.validate(receipt, interface.RECEIPT_SCHEMA_ID) + assert receipt["receipt_sha256"] == status["receipt_sha256"] + assert receipt["integrity"]["failures"][0]["code"] == "run_artifact_failed" + assert ( + git( + remote, + "ls-tree", + "-r", + "--name-only", + "replay-state", + f"requests/{request_id}/results", + ) + == "" + ) + prior_status = json.loads( + git( + remote, + "show", + f"{status['receipt_commit']}:requests/{request_id}/STATUS.json", + ) + ) + assert prior_status["status"] == "running" + + +def test_empty_pending_cli_has_no_output_for_redispatch_decision(tmp_path, capsys): + assert ( + queue.main( + ["pending", "--state", str(tmp_path), "--now", "2026-10-11T12:00:00Z"] + ) + == 0 + ) + assert capsys.readouterr().out == "" + + +@pytest.mark.parametrize("regeneration", ["identical", "drift", "untracked"]) +def test_ci_regeneration_preserves_sealed_source_and_detects_every_changed_byte( + tmp_path, regeneration +): + source = tmp_path / "source" + source.mkdir() + git(source, "init", "-q", "-b", "main") + git(source, "config", "user.name", "Test") + git(source, "config", "user.email", "test@example.org") + namespace = Path("results/uk/replays/candidate") + original = { + namespace / "REPLAY_RECEIPT.json": b'{"receipt":"original"}\n', + namespace / "NAMESPACE_MANIFEST.json": b'{"manifest":"original"}\n', + namespace / "event/RUN_MANIFEST.json": b"{}\n", + namespace / "event/STAGED.jsonl": b'{"value":"original"}\n', + namespace / "SUMMARY.md": b"original summary\n", + } + for relative, contents in original.items(): + path = source / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(contents) + script = source / ".github/scripts/uk_replay_regenerate.sh" + script.parent.mkdir(parents=True) + shutil.copyfile(ROOT / ".github/scripts/uk_replay_regenerate.sh", script) + git(source, "add", ".") + git(source, "commit", "-qm", "Sealed namespace") + tools = tmp_path / "bin" + tools.mkdir() + uv = tools / "uv" + uv.write_text( + f"#!{sys.executable}\n" + "import os, sys\nfrom pathlib import Path\n" + "args = sys.argv[1:]\n" + "namespace = Path('results/uk/replays/candidate')\n" + "if 'verify-sealed' in args:\n" + " assert Path.cwd() == Path(os.environ['SOURCE'])\n" + " assert (namespace / 'REPLAY_RECEIPT.json').exists()\n" + " assert (namespace / 'NAMESPACE_MANIFEST.json').exists()\n" + "else:\n" + " assert Path.cwd() == Path(os.environ['RUNNER_TEMP']) / 'replay-regeneration'\n" + " assert os.environ['PYTHONPATH'] == '.'\n" + " assert not (namespace / 'REPLAY_RECEIPT.json').exists()\n" + " assert not (namespace / 'NAMESPACE_MANIFEST.json').exists()\n" + " if 'pipeline/stage_uk_event.py' in args:\n" + " artifact = args[args.index('--artifact-dir') + 1]\n" + " assert artifact.startswith('results/uk/replays/candidate/')\n" + " assert not Path(artifact).is_absolute()\n" + " if os.environ['REGENERATION'] == 'drift':\n" + " (namespace / 'event/STAGED.jsonl').write_text('drift\\n')\n" + " if os.environ['REGENERATION'] == 'untracked':\n" + " (namespace / 'event/NEW.json').write_text('{}\\n')\n" + ) + uv.chmod(0o755) + result = subprocess.run( + ["bash", ".github/scripts/uk_replay_regenerate.sh"], + cwd=source, + env={ + **os.environ, + "PATH": f"{tools}:{os.environ['PATH']}", + "PYTHONPATH": ".", + "RUNNER_TEMP": str(tmp_path), + "SOURCE": str(source), + "REGENERATION": regeneration, + }, + capture_output=True, + text=True, + ) + assert result.returncode == (0 if regeneration == "identical" else 1), result.stderr + if regeneration != "identical": + assert ( + "replay regeneration changed committed bytes or created untracked files" + in result.stderr + ) + assert ( + "STAGED.jsonl" in result.stdout + if regeneration == "drift" + else "NEW.json" in result.stdout + ) + for relative, contents in original.items(): + assert (source / relative).read_bytes() == contents + assert git(source, "status", "--porcelain") == "" + fresh = tmp_path / "replay-regeneration" + for name in ("REPLAY_RECEIPT.json", "NAMESPACE_MANIFEST.json"): + assert (fresh / namespace / name).read_bytes() == original[namespace / name] + assert ( + git(fresh, "status", "--porcelain") == "" + if regeneration == "identical" + else git(fresh, "status", "--porcelain") != "" + ) From 9a919e141c79bd5465cab141c409cedf61b852ff Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 15 Oct 2026 05:51:42 -0400 Subject: [PATCH 4/7] Receiver: scoped data token, dataset stays on the runner - The dataset token is the org secret PE_UK_PRIVATE_HF_READ_TOKEN (read-only, policyengine-uk-data-private), used in one step of the run job that fetches and verifies the dataset before any candidate code, and only in workflow_dispatch and repository_dispatch runs. CI's real-replay leg uses it on pushes only; pull requests get a notice. - Scorecard is public (d1383): prepare hands over only REQUEST.json, the dataset never becomes an artifact or cache, and the run's output is refused (and not uploaded) if it holds a dataset-shaped file, a file over 25 MB or a symlink. - The qualification diff verifies the base against the runner's committed variants document and the candidate against the work document holding its record (--variants-file). - Tests: the guard's refusals and its wiring before upload; only the request and guarded output cross jobs; the token's steps and triggers; an inert construction makes the receipt incomplete and names the pair; each diff side is looked up in its own checkout's document. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/uk_replay_output_guard.sh | 28 ++++++ .github/workflows/ci.yml | 22 +++-- .github/workflows/uk-replay.yml | 55 ++++++++---- docs/uk_replay/QUALIFICATION_INTERFACE.md | 15 ++-- docs/uk_replay/RELEASE_REPLAY.md | 45 ++++++---- pipeline/uk_replay_qualify.py | 17 +++- tests/test_uk_replay_qualify.py | 66 +++++++++++++- tests/test_uk_replay_queue.py | 105 +++++++++++++++++++++- 8 files changed, 305 insertions(+), 48 deletions(-) create mode 100755 .github/scripts/uk_replay_output_guard.sh diff --git a/.github/scripts/uk_replay_output_guard.sh b/.github/scripts/uk_replay_output_guard.sh new file mode 100755 index 0000000..7c71fdb --- /dev/null +++ b/.github/scripts/uk_replay_output_guard.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Scorecard is public: the replay's upload may hold only the receipt, status +# and sealed aggregate results. Refuse any dataset file, any file large +# enough to be one (the largest aggregate artifact is far below the limit), +# and any symlink (the upload would follow it out of the directory). +set -euo pipefail +root="${1:?usage: uk_replay_output_guard.sh }" +limit_bytes=$((25 * 1024 * 1024)) +[ -d "$root" ] || exit 0 +status=0 +while IFS= read -r -d '' path; do + case "$path" in + *.h5 | *.hdf5 | *.h5.* | *.parquet | *.feather | *.pkl | *.pickle) + echo "::error::refusing to upload a dataset-shaped file: $path" + status=1 + ;; + esac + size=$(wc -c < "$path") + if [ "$size" -gt "$limit_bytes" ]; then + echo "::error::refusing to upload a ${size}-byte file: $path" + status=1 + fi +done < <(find "$root" -type f -print0) +while IFS= read -r -d '' path; do + echo "::error::refusing to upload a symlink: $path" + status=1 +done < <(find "$root" -type l -print0) +exit "$status" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b8b213..267cb22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,10 +68,11 @@ jobs: # exact environment from the variant's requirements freeze, the # construction guards, and the committed premise and diagnostic files # regenerated byte for byte. The legacy variant skips diagnostics and - # determinism. For other variants, a real measure-year replays only when - # HUGGING_FACE_TOKEN and a committed sealed namespace are both present. - # Otherwise the notice says that no real replay ran; the secret awaits - # administrator setup and is also unavailable to fork pull requests. + # determinism. For other variants, a real measure-year replays on pushes + # (the dataset token, the org secret PE_UK_PRIVATE_HF_READ_TOKEN, is never + # used for pull requests) when the variant has a committed sealed + # namespace; otherwise a notice says that no real replay ran. The dataset + # stays on the runner: it is never uploaded or cached. name: replay engine · ${{ matrix.variant }} runs-on: ubuntu-latest strategy: @@ -115,14 +116,17 @@ jobs: set -euo pipefail .venv-variant/bin/python pipeline/diagnose_uk_event_models.py --reverify results/uk/events/MODEL_DIAGNOSTICS.json --expect-engine "$ENGINE" --output "$RUNNER_TEMP/MODEL_DIAGNOSTICS.json" cmp "$RUNNER_TEMP/MODEL_DIAGNOSTICS.json" "$RESULT_ROOT/MODEL_DIAGNOSTICS.json" + - name: No real replay on pull requests + if: env.LEGACY == 'False' && github.event_name != 'push' + run: echo "::notice::The dataset token is used on pushes only; nothing was replayed by the real replay determinism step" - name: One real measure-year replays byte for byte - if: env.LEGACY == 'False' + if: env.LEGACY == 'False' && github.event_name == 'push' env: - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} run: | set -euo pipefail if [ -z "${HUGGING_FACE_TOKEN:-}" ]; then - echo "::notice::HUGGING_FACE_TOKEN unavailable (fork PR or secret awaiting administrator setup): nothing was replayed by the real replay determinism step" + echo "::notice::PE_UK_PRIVATE_HF_READ_TOKEN unavailable: nothing was replayed by the real replay determinism step" exit 0 fi if [ ! -f "$RESULT_ROOT/REPLAY_RECEIPT.json" ] || [ ! -f "$RESULT_ROOT/NAMESPACE_MANIFEST.json" ]; then @@ -138,8 +142,8 @@ jobs: PY key=autumn_budget_2024__private_school_vat_20pct .venv-variant/bin/python -m pipeline.compute_uk_event --event autumn_budget_2024 --variant "$VARIANT" \ - --measures "$key" --years 2026 --workers 1 --output-dir "$RUNNER_TEMP/determinism" - cmp "$RUNNER_TEMP/determinism/${key}_2026.json" "$RESULT_ROOT/autumn_budget_2024/${key}_2026.json" + --measures "$key" --years 2026 --workers 1 --output-dir .venv-replay-checks/determinism + cmp ".venv-replay-checks/determinism/${key}_2026.json" "$RESULT_ROOT/autumn_budget_2024/${key}_2026.json" echo "replayed $key 2026 byte-identical to the committed artifact" app: diff --git a/.github/workflows/uk-replay.yml b/.github/workflows/uk-replay.yml index a31eb72..421954c 100644 --- a/.github/workflows/uk-replay.yml +++ b/.github/workflows/uk-replay.yml @@ -7,8 +7,14 @@ name: UK replay qualification # One request runs per drain, with at most two simulation workers; the recorder # dispatches another drain while requests remain eligible. Stale runs retry # after seven hours, with three running attempts maximum. -# Dataset access happens before the candidate job, in a separate job. The -# candidate job has contents: read, no checkout credentials and no dataset token. +# The dataset token (the org secret PE_UK_PRIVATE_HF_READ_TOKEN, read-only) +# is used only in workflow_dispatch and repository_dispatch runs: the schedule +# only dispatches a drain. In the run job it is in one step's environment, +# which fetches and verifies the dataset before any candidate code is +# installed; candidate steps never see it, and that job has contents: read and +# no checkout credentials. Scorecard is public: the dataset and anything +# household-level never leave the runner (no artifact, no cache); only the +# request and the sealed aggregate results cross jobs, checked before upload. # Receipts are pushed before their terminal status names the pushed commit. # GitHub documents GITHUB_TOKEN-triggered workflow_dispatch runs at: # https://docs.github.com/en/actions/concepts/security/github_token @@ -37,6 +43,19 @@ env: PYTHONPATH: . jobs: + tick: + name: dispatch a drain on schedule + if: github.event_name == 'schedule' + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Start a drain run (workflow_dispatch, no request) + env: + GH_TOKEN: ${{ github.token }} + DRAIN_REF: ${{ github.event.repository.default_branch }} + run: gh workflow run uk-replay.yml --repo "$GITHUB_REPOSITORY" --ref "$DRAIN_REF" + enqueue: name: record the request if: github.event_name == 'repository_dispatch' || (github.event_name == 'workflow_dispatch' && github.event.inputs.request != '') @@ -71,8 +90,8 @@ jobs: gh workflow run uk-replay.yml --repo "$GITHUB_REPOSITORY" --ref "$DRAIN_REF" prepare: - name: select a request and fetch its dataset - if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.request == '') + name: select a request + if: github.event_name == 'workflow_dispatch' && github.event.inputs.request == '' runs-on: ubuntu-latest permissions: contents: write @@ -86,20 +105,12 @@ jobs: env: RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: bash .github/scripts/uk_replay_prepare.sh - - name: Fetch and verify the dataset without candidate code - if: steps.select.outputs.request_id != '' - env: - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} - run: | - uv run --with rfc8785==0.1.4 --with jsonschema --with huggingface_hub python \ - -m pipeline.uk_replay_qualify fetch-dataset --request input/REQUEST.json \ - --output input/dataset.h5 --result-file input/fetch-result.json - - name: Hand the verified dataset and request to the candidate job + - name: Hand the request (never the dataset) to the run job if: steps.select.outputs.request_id != '' uses: actions/upload-artifact@v4 with: name: replay-input - path: input/ + path: input/REQUEST.json if-no-files-found: error retention-days: 1 @@ -120,13 +131,27 @@ jobs: with: name: replay-input path: input/ + - name: Fetch and verify the dataset before any candidate code + if: contains(fromJSON('["workflow_dispatch", "repository_dispatch"]'), github.event_name) + env: + HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} + run: | + uv run --with rfc8785==0.1.4 --with jsonschema --with huggingface_hub python \ + -m pipeline.uk_replay_qualify fetch-dataset --request input/REQUEST.json \ + --output input/dataset.h5 --result-file input/fetch-result.json - name: Run the candidate and preserve logs and outcome env: REQUEST_ID: ${{ needs.prepare.outputs.request_id }} REPLAY_RUNNER: github-actions ${{ runner.os }} ${{ runner.arch }} run: bash .github/scripts/uk_replay_drain.sh - - name: Upload the receipt and results for the recorder + - name: Nothing household-level leaves the runner + id: guard if: always() + run: bash .github/scripts/uk_replay_output_guard.sh output + # A refused output is never uploaded; the recorder then finds no + # artifact and records the attempt as failed. + - name: Upload the receipt and results for the recorder + if: always() && steps.guard.outcome == 'success' uses: actions/upload-artifact@v4 with: name: replay-output diff --git a/docs/uk_replay/QUALIFICATION_INTERFACE.md b/docs/uk_replay/QUALIFICATION_INTERFACE.md index 7305e57..9ba211d 100644 --- a/docs/uk_replay/QUALIFICATION_INTERFACE.md +++ b/docs/uk_replay/QUALIFICATION_INTERFACE.md @@ -61,11 +61,16 @@ Two flows use the same receiver: `STATUS.json` is written in a later commit and names `receipt_sha256` and `receipt_commit`, so one contents read binds the status to the receipt bytes. -* Dataset fetching and candidate execution are separate jobs. The control - job fetches and verifies the file with the dataset token; the run job has - `contents: read`, no persisted checkout credential and no Hugging Face - token. It receives the file through an artifact and supplies - `--dataset-file` to `run-request`. The qualifier writes a structured result +* Dataset fetching and candidate execution are separate steps of the run + job, which has `contents: read` and no persisted checkout credential. The + fetch step runs first, before any candidate code is installed, and is the + only step holding the read-only dataset token (the organisation secret + `PE_UK_PRIVATE_HF_READ_TOKEN`, only in `workflow_dispatch` and + `repository_dispatch` runs). It verifies the file, which then goes to + `run-request` as `--dataset-file`. Scorecard is public: the dataset never + leaves the runner (no artifact, no cache), the control job hands over only + the request, and the run's output is checked for dataset-shaped files, + large files and symlinks before it is uploaded. The qualifier writes a structured result file with `status` and `receipt_sha256`, while logs remain visible in Actions. A separate recording job has `contents: write` and `actions: write`, validates the uploaded receipt, records it before terminal status and diff --git a/docs/uk_replay/RELEASE_REPLAY.md b/docs/uk_replay/RELEASE_REPLAY.md index d0d933e..4579c68 100644 --- a/docs/uk_replay/RELEASE_REPLAY.md +++ b/docs/uk_replay/RELEASE_REPLAY.md @@ -385,14 +385,26 @@ GitHub's default concurrency queue can replace an older pending run; the request remains in `replay-state` for a later drain. ([GitHub concurrency documentation](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency)) -The `prepare` job selects the oldest eligible request and marks it running. -It fetches the dataset with `fetch-dataset` in a control step holding -`HUGGING_FACE_TOKEN`, checks its size and SHA-256, and uploads the dataset, -request and fetch outcome as an artifact. A fetch rejection is passed to the -run without installing candidate code. The separate `run` job has -`contents: read`, checks out with `persist-credentials: false`, receives no -Hugging Face token, and runs `.github/scripts/uk_replay_drain.sh` for that one -request. `pipeline/uk_replay_qualify.py run-request` then: +The `prepare` job selects the oldest eligible request, marks it running and +hands only `REQUEST.json` to the run job. The separate `run` job has +`contents: read` and checks out with `persist-credentials: false`. Its first +step after that fetches the dataset with `fetch-dataset`, checks its size and +SHA-256, and records the fetch outcome, before any candidate code is +installed. That one step holds the data token: the organisation secret +`PE_UK_PRIVATE_HF_READ_TOKEN` (read-only, policyengine-uk-data-private only), +passed as `HUGGING_FACE_TOKEN`, and only in `workflow_dispatch` and +`repository_dispatch` runs; the schedule only dispatches a drain, and the +workflow has no pull-request trigger. A fetch rejection is passed to the run +without installing candidate code. The job then runs +`.github/scripts/uk_replay_drain.sh` for that one request. + +Scorecard is public, so the dataset and anything household-level stay on the +runner: nothing is cached, and the only artifacts are the request and the +run's `output/` directory. Before that upload, +`.github/scripts/uk_replay_output_guard.sh` refuses dataset-shaped files +(`.h5`, `.hdf5`, `.parquet`, `.feather`, pickles), any file over 25 MB and +any symlink; a refused output is not uploaded, and the recorder records the +attempt as failed. `pipeline/uk_replay_qualify.py run-request` then: 1. Validate the request (`schema_invalid`). 2. Check the dataset file supplied by the control job against the request @@ -419,7 +431,9 @@ request. `pipeline/uk_replay_qualify.py run-request` then: the summary, the construction premises, the re-verified diagnostics. 6. Diff against the runner checkout's committed `compare_to` namespace (default: the newest certified variant with committed results, preferring - the matching certified variant itself for an exact re-qualification). + the matching certified variant itself for an exact re-qualification). The + base is verified against the runner's committed variants document and the + candidate against the work document that adds its record. 7. Seal the namespace (`NAMESPACE_MANIFEST.json`, then `REPLAY_RECEIPT.json`) and copy it, the diff and the requirements to `replay-state:requests//results/`. @@ -471,12 +485,13 @@ holds heavy local work back from 29 October to 2 November. freeze, checks the installed environment against the record, runs every event's construction guards, regenerates the committed construction premises byte for byte. Non-legacy variants also regenerate the re-verified - diagnostics and, with `HUGGING_FACE_TOKEN` and a committed sealed namespace, - replay one real measure-year on the certified dataset and compare its - artifact with the committed bytes. The legacy leg skips diagnostics and - this determinism step. Without the secret (pending an administrator), or - without a committed sealed namespace, the step emits a notice stating that - nothing was replayed. + diagnostics and, on pushes to main with the `PE_UK_PRIVATE_HF_READ_TOKEN` + secret and a committed sealed namespace, replay one real measure-year on + the certified dataset and compare its artifact with the committed bytes. + Pull requests never receive the token: their run emits a notice that + nothing was replayed. The legacy leg skips diagnostics and this + determinism step. Without the secret, or without a committed sealed + namespace, the step emits a notice stating that nothing was replayed. ## Invariants diff --git a/pipeline/uk_replay_qualify.py b/pipeline/uk_replay_qualify.py index 49dca0a..198654e 100644 --- a/pipeline/uk_replay_qualify.py +++ b/pipeline/uk_replay_qualify.py @@ -1118,6 +1118,8 @@ def run_request( str(root), "--head", str(namespace), + "--variants-file", + str(variants_file), "--output-dir", str(diff_dir), ], @@ -1231,6 +1233,8 @@ def main(argv: list[str] | None = None) -> int: diff_parser.add_argument("--base", type=Path, required=True) diff_parser.add_argument("--base-root", type=Path, required=True) diff_parser.add_argument("--head", type=Path, required=True) + # The head's record is the candidate's, held only in the work document. + diff_parser.add_argument("--variants-file", type=Path, required=True) diff_parser.add_argument("--output-dir", type=Path, required=True) receipt = commands.add_parser("receipt") receipt.add_argument("--request", type=Path, required=True) @@ -1331,8 +1335,17 @@ def main(argv: list[str] | None = None) -> int: if args.command == "diff": from pipeline import diff_uk_event_runs as diff - base = diff.load_side(args.base, artifact_root=args.base_root) - head = diff.load_side(args.head, artifact_root=ROOT) + # Each side is verified against its own checkout: the base against the + # runner's committed variants document, the head against the work + # document that adds the candidate's record. + base = diff.load_side( + args.base, + artifact_root=args.base_root, + variants_path=args.base_root / "data" / "uk" / "replay_variants.json", + ) + head = diff.load_side( + args.head, artifact_root=ROOT, variants_path=args.variants_file.resolve() + ) # Loading verified each side against its own checkout. The published # diff names repository-relative roots rather than temporary runners. base["root"] = ROOT / args.base.resolve().relative_to(args.base_root.resolve()) diff --git a/tests/test_uk_replay_qualify.py b/tests/test_uk_replay_qualify.py index 22a0a88..ed6d9f6 100644 --- a/tests/test_uk_replay_qualify.py +++ b/tests/test_uk_replay_qualify.py @@ -340,6 +340,51 @@ def test_pending_never_returns_terminal_requests(tmp_path_factory, statuses): assert set(later) == set(ids[1:]) +@pytest.mark.parametrize("inert", [False, True]) +def test_an_inert_construction_makes_the_receipt_incomplete_and_names_the_pair( + tmp_path, monkeypatch, inert +): + # Coordinator, 2026-10-11: identical worlds in an in-force year on the + # candidate's engine must not let a receipt claim a complete corpus. + import functools + import shutil + + from test_uk_event_diff import EVENT, KEY, make_namespace, synthetic_record + + from pipeline import compute_uk_event as compute + from pipeline import stage_uk_event as stage + + monkeypatch.setattr(compute, "ROOT", tmp_path) + monkeypatch.setattr(stage, "manifest_variant", synthetic_record) + make_namespace( + tmp_path, "candidate", variant=CURRENT, identical=(2025,) if inert else () + ) + record = rv.get_variant(CURRENT) + shutil.copytree( + tmp_path / "ns" / "candidate", rv.namespace_root(record, root=tmp_path) + ) + monkeypatch.setattr( + qualify, + "event_results", + functools.partial(qualify.event_results, events=[EVENT]), + ) + receipt = qualify.build_receipt( + golden_request(), + record=record, + root=tmp_path, + results_path="requests/x/results/", + ) + event = receipt["corpus"]["events"][EVENT] + assert receipt["integrity"]["failures"] == [] + if inert: + assert receipt["integrity"]["status"] == "incomplete" + assert event["full_supported_grid_complete"] is False + assert event["inert_measure_years"] == [{"measure_key": KEY, "year": 2025}] + else: + assert receipt["integrity"]["status"] == "complete" + assert event["inert_measure_years"] == [] + + def test_rejection_discards_a_supplied_record_and_cli_accepts_candidate(tmp_path): request = golden_request() record = copy.deepcopy(rv.get_variant(CURRENT)) @@ -885,12 +930,24 @@ def test_legacy_requalification_preserves_original_diagnostics(tmp_path, monkeyp def test_diff_reads_base_and_head_artifacts_from_their_own_checkouts( tmp_path, monkeypatch ): - from test_uk_event_diff import make_namespace + from test_uk_event_diff import make_namespace, synthetic_record from pipeline import compute_uk_event as compute from pipeline import diff_uk_event_runs as diff + from pipeline import stage_uk_event as stage base_root = tmp_path / "runner" head_root = tmp_path / "checkout" + work_variants = tmp_path / "work" / "variants.json" + # The synthetic sides name synthetic datasets, so the variant lookup is + # stood in (as in test_uk_event_diff); what is checked here is that each + # side is looked up in its own checkout's document. + looked_up = [] + + def record(manifest, registry, *, variants_path): + looked_up.append(Path(variants_path)) + return synthetic_record(manifest, registry, variants_path=variants_path) + + monkeypatch.setattr(stage, "manifest_variant", record) monkeypatch.setattr(compute, "ROOT", base_root) base = make_namespace(base_root, "pair") monkeypatch.setattr(compute, "ROOT", head_root) @@ -913,10 +970,17 @@ def test_diff_reads_base_and_head_artifacts_from_their_own_checkouts( str(base_root), "--head", str(head["root"]), + "--variants-file", + str(work_variants), "--output-dir", str(output), ] + looked_up.clear() assert qualify.main(command) == 0 + assert set(looked_up) == { + base_root / "data" / "uk" / "replay_variants.json", + work_variants.resolve(), + } result = json.loads((output / "DIFF.json").read_bytes()) assert result["flag_counts"]["numerical"] > 0 row = next( diff --git a/tests/test_uk_replay_queue.py b/tests/test_uk_replay_queue.py index 56b5c2e..1c4cb7c 100644 --- a/tests/test_uk_replay_queue.py +++ b/tests/test_uk_replay_queue.py @@ -212,7 +212,27 @@ def test_candidate_job_has_read_permission_and_no_dataset_token_or_credentials() step for step in candidate["steps"] if step.get("uses") == "actions/checkout@v4" ) assert checkout["with"]["persist-credentials"] is False - assert "HUGGING_FACE_TOKEN" not in json.dumps(candidate) + # The read-only data token reaches only the dataset fetch, which runs + # before any candidate code and only on dispatches (never on a schedule + # tick or a pull request), and only from the scoped org secret. + with_token = [step for step in candidate["steps"] if "secrets." in json.dumps(step)] + assert [step["name"] for step in with_token] == [ + "Fetch and verify the dataset before any candidate code" + ] + (fetch,) = with_token + assert fetch["env"] == { + "HUGGING_FACE_TOKEN": "${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}" + } + assert fetch["if"] == ( + 'contains(fromJSON(\'["workflow_dispatch", "repository_dispatch"]\'), ' + "github.event_name)" + ) + steps = [step.get("name") for step in candidate["steps"]] + assert steps.index(fetch["name"]) < steps.index( + "Run the candidate and preserve logs and outcome" + ) + rest = {k: v for k, v in candidate.items() if k != "steps"} + assert "secrets." not in json.dumps(rest) assert workflow["jobs"]["record"]["permissions"] == { "contents": "write", "actions": "write", @@ -543,3 +563,86 @@ def test_ci_regeneration_preserves_sealed_source_and_detects_every_changed_byte( if regeneration == "identical" else git(fresh, "status", "--porcelain") != "" ) + + +GUARD = ROOT / ".github/scripts/uk_replay_output_guard.sh" + + +def guard(directory: Path) -> subprocess.CompletedProcess: + return subprocess.run( + ["bash", str(GUARD), str(directory)], capture_output=True, text=True + ) + + +def test_output_guard_refuses_datasets_large_files_and_symlinks(tmp_path): + # Scorecard is public (d1383): only the receipt, status and sealed + # aggregate results may be uploaded from the run job. + output = tmp_path / "output" + results = output / "results" / "autumn_budget_2024" + results.mkdir(parents=True) + (results / "COMPARISON.json").write_text("{}") + (output / "REPLAY_RECEIPT.json").write_text("{}") + assert guard(output).returncode == 0 + assert guard(tmp_path / "absent").returncode == 0 + for name in ("dataset.h5", "frs.hdf5", "x.h5.partial", "people.parquet"): + bad = results / name + bad.write_bytes(b"\0") + refused = guard(output) + assert refused.returncode == 1 and name in refused.stdout + bad.unlink() + big = results / "artifact.json" + with big.open("wb") as handle: + handle.truncate(25 * 1024 * 1024 + 1) + assert guard(output).returncode == 1 + big.unlink() + outside = tmp_path / "input.json" + outside.write_text("{}") + (results / "link.json").symlink_to(outside) + refused = guard(output) + assert refused.returncode == 1 and "symlink" in refused.stdout + + +def test_nothing_crosses_jobs_but_the_request_and_guarded_output(): + workflow = yaml.safe_load((ROOT / ".github/workflows/uk-replay.yml").read_text()) + uploads = [ + (name, step) + for name, job in workflow["jobs"].items() + for step in job["steps"] + if step.get("uses", "").startswith("actions/upload-artifact") + ] + assert [(name, step["with"]["path"]) for name, step in uploads] == [ + ("prepare", "input/REQUEST.json"), + ("run", "output/"), + ] + steps = workflow["jobs"]["run"]["steps"] + names = [step.get("name") for step in steps] + guard_step = steps[names.index("Nothing household-level leaves the runner")] + upload = steps[names.index("Upload the receipt and results for the recorder")] + assert names.index(guard_step["name"]) < names.index(upload["name"]) + assert "uk_replay_output_guard.sh output" in guard_step["run"] + assert upload["if"] == "always() && steps.guard.outcome == 'success'" + assert guard_step["id"] == "guard" and guard_step["if"] == "always()" + for path in (".github/workflows/uk-replay.yml", ".github/workflows/ci.yml"): + document = yaml.safe_load((ROOT / path).read_text()) + for job in document["jobs"].values(): + for step in job["steps"]: + assert not step.get("uses", "").startswith("actions/cache"), path + + +def test_the_data_token_is_used_only_outside_pull_requests(): + replay = yaml.safe_load((ROOT / ".github/workflows/uk-replay.yml").read_text()) + # PyYAML reads the bare key `on` as True. + assert set(replay[True]) == {"repository_dispatch", "workflow_dispatch", "schedule"} + ci = yaml.safe_load((ROOT / ".github/workflows/ci.yml").read_text()) + for document in (replay, ci): + for job in document["jobs"].values(): + for step in job["steps"]: + text = json.dumps(step) + if "HF_READ_TOKEN" in text or "HUGGING_FACE" in text: + assert "secrets.HUGGING_FACE_TOKEN" not in text + if "secrets.PE_UK_PRIVATE_HF_READ_TOKEN" in text: + assert "pull_request" not in step["if"] + assert ( + step["if"].endswith("github.event_name == 'push'") + or "workflow_dispatch" in step["if"] + ) From 289661055a5de6626a0bd62769bee36e97aee229 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 15 Oct 2026 07:21:25 -0400 Subject: [PATCH 5/7] Receiver: answer the review of 176e974 - CI: a variant namespace must be sealed once it holds replay results (an event's RUN_MANIFEST.json). The 6.2.6 namespace on main holds only premises, diagnostics and an empty summary, so verify-sealed no longer fails before #164's sealed results land. Tests cover both cases and the committed tree. - No cache channel: setup-uv v5 caches by default; every use in uk-replay.yml and in CI's replay-engine job sets enable-cache: false. - Candidate code runs as an unprivileged account (.github/scripts/uk_replay_sandbox.sh): no sudo (the job fails if it has any), its own copy of the repository and inputs and its own uv, an empty starting environment, no access to the runner user's processes (secrets, runtime token), workspace or command files. Its processes are killed and a copy of output/ comes back before the guard. - The output guard is an allowlist in standard-library Python: regular .json/.jsonl/.csv/.md/.txt files that parse as their format, 10 MB each, 100 MB in all. The recorder applies it again before committing and drops a refused output (recorded as failed). - CI replay-engine: contents: read, no persisted checkout credential, the token only in a fetch step before the variant's engine runs; the token steps pin huggingface_hub==2.2.0 (the certified freezes' pin); the drain no longer installs huggingface_hub. - Tests: exact conditions for the only two token steps, no secret at workflow or job level, the run job's step order, the sandbox's invariants, the guard's refusals (renamed binaries, broken JSON, non-UTF-8, NUL, links, FIFOs, caps), the recorder's success path replacing an earlier attempt's results, and the inert receipt sealed and validated. - Docs state what this protects and what it does not: a deliberately malicious candidate can encode data in parsable aggregates and forge its own receipt. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/uk_replay_drain.sh | 6 +- .github/scripts/uk_replay_output_guard.py | 93 ++++++++ .github/scripts/uk_replay_output_guard.sh | 28 --- .github/scripts/uk_replay_sandbox.sh | 73 ++++++ .github/workflows/ci.yml | 19 +- .github/workflows/uk-replay.yml | 42 +++- docs/uk_replay/QUALIFICATION_INTERFACE.md | 14 +- docs/uk_replay/RELEASE_REPLAY.md | 55 ++++- pipeline/uk_replay_qualify.py | 5 +- tests/test_uk_replay_qualify.py | 32 +++ tests/test_uk_replay_queue.py | 268 +++++++++++++++++++--- 11 files changed, 545 insertions(+), 90 deletions(-) create mode 100644 .github/scripts/uk_replay_output_guard.py delete mode 100755 .github/scripts/uk_replay_output_guard.sh create mode 100644 .github/scripts/uk_replay_sandbox.sh diff --git a/.github/scripts/uk_replay_drain.sh b/.github/scripts/uk_replay_drain.sh index 60890ea..2a0ca9d 100755 --- a/.github/scripts/uk_replay_drain.sh +++ b/.github/scripts/uk_replay_drain.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash -# Candidate code runs only here, in the read-only job without the dataset token. +# Candidate code runs only here: as the unprivileged candidate account, in its +# private copy of the repository (uk_replay_sandbox.sh run), after the dataset +# was fetched and verified. Nothing here holds a token. set -euo pipefail -CONTROL=(uv run --with rfc8785==0.1.4 --with jsonschema --with huggingface_hub python) +CONTROL=(uv run --with rfc8785==0.1.4 --with jsonschema python) mkdir -p output/state if "${CONTROL[@]}" -m pipeline.uk_replay_qualify run-request \ diff --git a/.github/scripts/uk_replay_output_guard.py b/.github/scripts/uk_replay_output_guard.py new file mode 100644 index 0000000..1b729f8 --- /dev/null +++ b/.github/scripts/uk_replay_output_guard.py @@ -0,0 +1,93 @@ +"""Refuse a replay output directory unless every file is an expected aggregate. + +Scorecard is public, so the run job's upload, and the record job's commit to +the public replay-state branch, may hold only the receipt, status and the +sealed aggregate results. This is an allowlist: every entry must be a regular +file (no symlink, device or other special file) with an allowed extension, +whose bytes parse as that format, under a per-file and a total size cap. The +committed sealed namespace has 30 MB over 316 files, the largest 3.3 MB. + +It stops accidental publication (an honest run that writes a dataset, +household-level table or binary). Content that parses can still carry +anything a deliberately malicious candidate encodes in it; RELEASE_REPLAY.md +states that limit. + +Standard library only: it runs with the runner's system Python, before and +independently of anything the candidate installed. + + python3 -I .github/scripts/uk_replay_output_guard.py output +""" + +from __future__ import annotations + +import json +import stat +import sys +from pathlib import Path + +ALLOWED_SUFFIXES = (".json", ".jsonl", ".csv", ".md", ".txt") +MAX_FILE_BYTES = 10 * 1024 * 1024 +MAX_TOTAL_BYTES = 100 * 1024 * 1024 + + +def _parses(path: Path, data: bytes) -> str | None: + """None when the bytes are what the extension says; else the reason.""" + try: + text = data.decode("utf-8") + except UnicodeDecodeError: + return "not UTF-8 text" + if "\x00" in text: + return "contains a NUL byte" + try: + if path.suffix == ".json": + json.loads(text) + elif path.suffix == ".jsonl": + for line in text.splitlines(): + if line.strip(): + json.loads(line) + except ValueError as error: + return f"does not parse as {path.suffix[1:]} ({error})" + return None + + +def problems(root: Path) -> list[str]: + """Every reason to refuse ``root``; empty when it may leave the runner.""" + if not root.exists() and not root.is_symlink(): + return [] + found, total = [], 0 + entries = [root, *sorted(root.rglob("*"))] if root.is_dir() else [root] + for path in entries: + mode = path.lstat().st_mode + if stat.S_ISDIR(mode): + continue + if not stat.S_ISREG(mode): + found.append(f"{path}: not a regular file (symlink or special file)") + continue + if path.suffix not in ALLOWED_SUFFIXES: + found.append(f"{path}: extension not allowed ({path.suffix or 'none'})") + continue + size = path.stat().st_size + total += size + if size > MAX_FILE_BYTES: + found.append(f"{path}: {size} bytes, over {MAX_FILE_BYTES}") + continue + reason = _parses(path, path.read_bytes()) + if reason: + found.append(f"{path}: {reason}") + if total > MAX_TOTAL_BYTES: + found.append(f"{root}: {total} bytes in all, over {MAX_TOTAL_BYTES}") + return found + + +def main(argv: list[str]) -> int: + if len(argv) != 1: + print("usage: uk_replay_output_guard.py ", file=sys.stderr) + return 2 + refused = problems(Path(argv[0])) + for reason in refused: + print(f"::error::refusing to publish {reason}") + return 1 if refused else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.github/scripts/uk_replay_output_guard.sh b/.github/scripts/uk_replay_output_guard.sh deleted file mode 100755 index 7c71fdb..0000000 --- a/.github/scripts/uk_replay_output_guard.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# Scorecard is public: the replay's upload may hold only the receipt, status -# and sealed aggregate results. Refuse any dataset file, any file large -# enough to be one (the largest aggregate artifact is far below the limit), -# and any symlink (the upload would follow it out of the directory). -set -euo pipefail -root="${1:?usage: uk_replay_output_guard.sh }" -limit_bytes=$((25 * 1024 * 1024)) -[ -d "$root" ] || exit 0 -status=0 -while IFS= read -r -d '' path; do - case "$path" in - *.h5 | *.hdf5 | *.h5.* | *.parquet | *.feather | *.pkl | *.pickle) - echo "::error::refusing to upload a dataset-shaped file: $path" - status=1 - ;; - esac - size=$(wc -c < "$path") - if [ "$size" -gt "$limit_bytes" ]; then - echo "::error::refusing to upload a ${size}-byte file: $path" - status=1 - fi -done < <(find "$root" -type f -print0) -while IFS= read -r -d '' path; do - echo "::error::refusing to upload a symlink: $path" - status=1 -done < <(find "$root" -type l -print0) -exit "$status" diff --git a/.github/scripts/uk_replay_sandbox.sh b/.github/scripts/uk_replay_sandbox.sh new file mode 100644 index 0000000..34eb1b3 --- /dev/null +++ b/.github/scripts/uk_replay_sandbox.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Candidate code runs as an unprivileged account, never as the runner user. +# +# The runner user has passwordless sudo, owns this job's workspace and its +# command files ($GITHUB_ENV, $GITHUB_PATH), and owns the runner processes +# that hold the job's secrets and the Actions runtime token. The candidate +# account has none of these: no sudoers entry, a private copy of the +# repository and the verified request inputs, an environment built from +# nothing (env -i), and no read access to another user's process memory or +# environment. Nothing it starts survives `collect`, and only a copy of its +# output/ comes back, to be checked by the runner's own guard. +# +# uk_replay_sandbox.sh setup # after the dataset fetch, before any candidate code +# uk_replay_sandbox.sh run # the drain, as the candidate account +# uk_replay_sandbox.sh collect # kill its processes, copy output/ back +set -euo pipefail +CANDIDATE=replay-candidate +CANDIDATE_HOME=/home/$CANDIDATE +COPY=$CANDIDATE_HOME/scorecard + +case "${1:?usage: uk_replay_sandbox.sh setup|run|collect}" in + setup) + sudo useradd --create-home --shell /bin/bash "$CANDIDATE" + if ! sudo -l -U "$CANDIDATE" | grep -q "not allowed to run sudo"; then + echo "::error::$CANDIDATE can use sudo" + exit 1 + fi + sudo chmod 700 "$CANDIDATE_HOME" + # Its own uv, whatever the permissions of the runner's tool cache. + sudo install -d -o "$CANDIDATE" -g "$CANDIDATE" "$CANDIDATE_HOME/bin" + sudo install -o "$CANDIDATE" -g "$CANDIDATE" -m 0755 "$(command -v uv)" "$CANDIDATE_HOME/bin/uv" + # The repository with its history (the corpus runs in a worktree of + # HEAD) and input/: the request, the verified dataset, the fetch result. + sudo cp -a "$GITHUB_WORKSPACE/." "$COPY" + sudo rm -rf "$COPY/output" + sudo chown -R "$CANDIDATE:$CANDIDATE" "$COPY" + ;; + run) + status=0 + sudo -n -u "$CANDIDATE" env -i \ + HOME="$CANDIDATE_HOME" USER="$CANDIDATE" LANG=C.UTF-8 \ + PATH="$CANDIDATE_HOME/bin:/usr/local/bin:/usr/bin:/bin" \ + PYTHONPATH=. RUNNER_TEMP="$CANDIDATE_HOME/tmp" \ + REQUEST_ID="${REQUEST_ID:?}" REPLAY_RUNNER="${REPLAY_RUNNER:?}" \ + bash -c 'mkdir -p "$RUNNER_TEMP" && cd "$HOME/scorecard" && exec bash .github/scripts/uk_replay_drain.sh' \ + || status=$? + # The drain records its own exit; the recorder handles a missing outcome. + echo "candidate drain exited $status" + ;; + collect) + # Nothing the candidate started outlives this step. + sudo pkill -KILL -u "$CANDIDATE" || true + for _ in $(seq 30); do + pgrep -u "$CANDIDATE" > /dev/null || break + sleep 1 + done + if pgrep -u "$CANDIDATE" > /dev/null; then + echo "::error::$CANDIDATE processes survive" + exit 1 + fi + rm -rf output + if sudo test -d "$COPY/output"; then + # -R copies links as links and special files as special files, which + # the guard then refuses; --no-preserve drops the candidate's modes. + sudo cp -R --no-preserve=all "$COPY/output" output + sudo chown -R "$(id -u):$(id -g)" output + fi + ;; + *) + echo "usage: uk_replay_sandbox.sh setup|run|collect" >&2 + exit 2 + ;; +esac diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21c35fc..61dc42c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,6 +75,8 @@ jobs: # stays on the runner: it is never uploaded or cached. name: replay engine · ${{ matrix.variant }} runs-on: ubuntu-latest + permissions: + contents: read strategy: fail-fast: false matrix: @@ -86,7 +88,11 @@ jobs: VARIANT: ${{ matrix.variant }} steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: astral-sh/setup-uv@v5 + with: + enable-cache: false - name: Install the variant's exact environment run: | set -euo pipefail @@ -119,7 +125,11 @@ jobs: - name: No real replay on pull requests if: env.LEGACY == 'False' && github.event_name != 'push' run: echo "::notice::The dataset token is used on pushes only; nothing was replayed by the real replay determinism step" - - name: One real measure-year replays byte for byte + # The token is in this step only: it fetches the certified dataset into + # the Hugging Face cache before the variant's engine runs, in the next + # step, without it. + - name: Fetch the certified dataset for the real replay + id: fetch if: env.LEGACY == 'False' && github.event_name == 'push' env: HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} @@ -133,13 +143,18 @@ jobs: echo "::notice::The variant has no committed sealed namespace: nothing was replayed by the real replay determinism step" exit 0 fi - uv run --with huggingface_hub python - <<'PY' + uv run --with huggingface_hub==2.2.0 python - <<'PY' import json, os from huggingface_hub import hf_hub_download d = json.load(open("data/uk/replay_variants.json")) v = [v for v in d["variants"] if v["variant_id"] == os.environ["VARIANT"]][0]["identity"]["dataset"] print(hf_hub_download(repo_id=v["repo_id"], repo_type=v["repo_type"], filename=v["path"], revision=v["revision"], token=os.environ["HUGGING_FACE_TOKEN"])) PY + echo "fetched=true" >> "$GITHUB_OUTPUT" + - name: One real measure-year replays byte for byte + if: steps.fetch.outputs.fetched == 'true' + run: | + set -euo pipefail key=autumn_budget_2024__private_school_vat_20pct .venv-variant/bin/python -m pipeline.compute_uk_event --event autumn_budget_2024 --variant "$VARIANT" \ --measures "$key" --years 2026 --workers 1 --output-dir .venv-replay-checks/determinism diff --git a/.github/workflows/uk-replay.yml b/.github/workflows/uk-replay.yml index 421954c..9594fbc 100644 --- a/.github/workflows/uk-replay.yml +++ b/.github/workflows/uk-replay.yml @@ -11,10 +11,15 @@ name: UK replay qualification # is used only in workflow_dispatch and repository_dispatch runs: the schedule # only dispatches a drain. In the run job it is in one step's environment, # which fetches and verifies the dataset before any candidate code is -# installed; candidate steps never see it, and that job has contents: read and -# no checkout credentials. Scorecard is public: the dataset and anything -# household-level never leave the runner (no artifact, no cache); only the -# request and the sealed aggregate results cross jobs, checked before upload. +# installed. That job has contents: read and no checkout credentials, and +# candidate code runs as an unprivileged account with no sudo, no access to +# the workspace, the runner's command files or the processes holding the +# job's secrets, and an environment built from nothing +# (.github/scripts/uk_replay_sandbox.sh). Scorecard is public: the dataset and +# anything household-level never leave the runner. setup-uv's cache is off +# everywhere here, so no job saves or restores a cache; the only artifacts are +# the request and the run's output, which an allowlist guard checks before the +# upload and again before the recorder commits anything. # Receipts are pushed before their terminal status names the pushed commit. # GitHub documents GITHUB_TOKEN-triggered workflow_dispatch runs at: # https://docs.github.com/en/actions/concepts/security/github_token @@ -66,6 +71,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 + with: + enable-cache: false - name: Write the received request env: EVENT_NAME: ${{ github.event_name }} @@ -100,6 +107,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 + with: + enable-cache: false - name: Mark the oldest eligible request running id: select env: @@ -127,6 +136,8 @@ jobs: with: persist-credentials: false - uses: astral-sh/setup-uv@v5 + with: + enable-cache: false - uses: actions/download-artifact@v4 with: name: replay-input @@ -136,18 +147,23 @@ jobs: env: HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} run: | - uv run --with rfc8785==0.1.4 --with jsonschema --with huggingface_hub python \ + uv run --with rfc8785==0.1.4 --with jsonschema --with huggingface_hub==2.2.0 python \ -m pipeline.uk_replay_qualify fetch-dataset --request input/REQUEST.json \ --output input/dataset.h5 --result-file input/fetch-result.json + - name: Hand a private copy to the unprivileged candidate account + run: bash .github/scripts/uk_replay_sandbox.sh setup - name: Run the candidate and preserve logs and outcome env: REQUEST_ID: ${{ needs.prepare.outputs.request_id }} REPLAY_RUNNER: github-actions ${{ runner.os }} ${{ runner.arch }} - run: bash .github/scripts/uk_replay_drain.sh + run: bash .github/scripts/uk_replay_sandbox.sh run + - name: Stop every candidate process and take its output back + if: always() + run: bash .github/scripts/uk_replay_sandbox.sh collect - name: Nothing household-level leaves the runner id: guard if: always() - run: bash .github/scripts/uk_replay_output_guard.sh output + run: python3 -I .github/scripts/uk_replay_output_guard.py output # A refused output is never uploaded; the recorder then finds no # artifact and records the attempt as failed. - name: Upload the receipt and results for the recorder @@ -170,6 +186,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 + with: + enable-cache: false - name: Download the run artifact id: download continue-on-error: true @@ -177,6 +195,16 @@ jobs: with: name: replay-output path: output/ + # The run job's guard ran beside candidate code; check again here, + # where nothing of the candidate's has run, before anything reaches the + # public branch. A refused output is dropped: the recorder then records + # a failed receipt. + - name: Nothing household-level reaches the public branch + run: | + if ! python3 -I .github/scripts/uk_replay_output_guard.py output; then + echo "::error::the run's output was refused; recording a failed receipt" + rm -rf output + fi - name: Record the outcome and continue the queue env: REQUEST_ID: ${{ needs.prepare.outputs.request_id }} diff --git a/docs/uk_replay/QUALIFICATION_INTERFACE.md b/docs/uk_replay/QUALIFICATION_INTERFACE.md index 9ba211d..348ac59 100644 --- a/docs/uk_replay/QUALIFICATION_INTERFACE.md +++ b/docs/uk_replay/QUALIFICATION_INTERFACE.md @@ -67,10 +67,16 @@ Two flows use the same receiver: only step holding the read-only dataset token (the organisation secret `PE_UK_PRIVATE_HF_READ_TOKEN`, only in `workflow_dispatch` and `repository_dispatch` runs). It verifies the file, which then goes to - `run-request` as `--dataset-file`. Scorecard is public: the dataset never - leaves the runner (no artifact, no cache), the control job hands over only - the request, and the run's output is checked for dataset-shaped files, - large files and symlinks before it is uploaded. The qualifier writes a structured result + `run-request` as `--dataset-file`. Candidate code runs as an unprivileged + account with no sudo, its own copy of the repository and inputs, and an + empty starting environment. Scorecard is public: the dataset never leaves + the runner, no job uses a cache, the control job hands over only the + request, and the run's output must pass an allowlist (parsable `.json`, + `.jsonl`, `.csv`, `.md` and `.txt` files under size caps) before it is + uploaded and again before it is committed. RELEASE_REPLAY.md states what + this protects and what it does not; in particular, a receipt is produced + beside the candidate, so requests are for PolicyEngine's own releases + only. The qualifier writes a structured result file with `status` and `receipt_sha256`, while logs remain visible in Actions. A separate recording job has `contents: write` and `actions: write`, validates the uploaded receipt, records it before terminal status and diff --git a/docs/uk_replay/RELEASE_REPLAY.md b/docs/uk_replay/RELEASE_REPLAY.md index 2b3eb2d..6d76f9b 100644 --- a/docs/uk_replay/RELEASE_REPLAY.md +++ b/docs/uk_replay/RELEASE_REPLAY.md @@ -409,16 +409,52 @@ installed. That one step holds the data token: the organisation secret passed as `HUGGING_FACE_TOKEN`, and only in `workflow_dispatch` and `repository_dispatch` runs; the schedule only dispatches a drain, and the workflow has no pull-request trigger. A fetch rejection is passed to the run -without installing candidate code. The job then runs -`.github/scripts/uk_replay_drain.sh` for that one request. +without installing candidate code. + +Candidate code then runs as an unprivileged account +(`.github/scripts/uk_replay_sandbox.sh`), never as the runner user. The +runner user has passwordless sudo, owns the workspace and the runner's +command files (`$GITHUB_ENV`, `$GITHUB_PATH`), and owns the runner processes +that keep the job's secrets in memory for the whole job and pass the Actions +runtime token to later steps. The candidate account has no sudoers entry +(checked: the job fails if it can use sudo), works in its own copy of the +repository and of `input/` (the request, the verified dataset and the fetch +outcome), starts from an empty environment (`env -i`), and cannot read +another user's process memory or environment. It runs +`.github/scripts/uk_replay_drain.sh` for that one request. Afterwards every +process it started is killed and a copy of its `output/` comes back to the +runner user; links are copied as links and special files as special files, +for the guard to refuse. Scorecard is public, so the dataset and anything household-level stay on the -runner: nothing is cached, and the only artifacts are the request and the -run's `output/` directory. Before that upload, -`.github/scripts/uk_replay_output_guard.sh` refuses dataset-shaped files -(`.h5`, `.hdf5`, `.parquet`, `.feather`, pickles), any file over 25 MB and -any symlink; a refused output is not uploaded, and the recorder records the -attempt as failed. `pipeline/uk_replay_qualify.py run-request` then: +runner. No job saves or restores a cache (setup-uv's default cache is off on +every use, and nothing uses `actions/cache`), so no cache links a candidate +to the jobs that can write. The only artifacts are the request and the run's +`output/` directory. `.github/scripts/uk_replay_output_guard.py` (standard +library, run with the runner's own Python) is an allowlist: every entry must +be a regular file ending `.json`, `.jsonl`, `.csv`, `.md` or `.txt` whose +bytes parse as that format (UTF-8 text, no NUL bytes), at most 10 MB each and +100 MB in all; the committed sealed namespace for 6.2.6 is 30 MB, its largest +file 3.3 MB. The run job uploads only if the guard passes. The `record` job, +where no candidate code has run, applies the guard again before committing +anything to the public `replay-state` branch, and drops a refused output, so +the attempt is recorded as `failed`. + +What this protects, and what it does not. A request names its candidate by +version and wheel SHA-256, and only a caller with write access to this +repository can send one (`repository_dispatch` and `workflow_dispatch` both +require it); requests are meant for PolicyEngine's own releases, from W4. A +new candidate's dependencies resolve within the newest certified freeze +where they can. The candidate and its dependencies are trusted with the +dataset: they compute on it, and the guard stops accidental publication +(an honest run that writes a dataset copy, a household-level table or a +binary). Code in the candidate's dependency closure cannot reach the data +token, the runtime token, the write jobs' credentials or a cache, and cannot +alter the guard or what the recorder checks. A deliberately malicious +candidate could still encode data in aggregates that parse, and could forge a +schema-valid receipt for its own request, since the receipt and seal are +produced beside it; neither is in scope, and both would require a malicious +PolicyEngine release. `pipeline/uk_replay_qualify.py run-request` then: 1. Validate the request (`schema_invalid`). 2. Check the dataset file supplied by the control job against the request @@ -502,6 +538,9 @@ holds heavy local work back from 29 October to 2 November. diagnostics and, on pushes to main with the `PE_UK_PRIVATE_HF_READ_TOKEN` secret and a committed sealed namespace, replay one real measure-year on the certified dataset and compare its artifact with the committed bytes. + The token is in one step that fetches the dataset into the Hugging Face + cache; the variant's engine runs in the next step, without it, in a job + with `contents: read`, no persisted checkout credential and no cache. Pull requests never receive the token: their run emits a notice that nothing was replayed. The legacy leg skips diagnostics and this determinism step. Without the secret, or without a committed sealed diff --git a/pipeline/uk_replay_qualify.py b/pipeline/uk_replay_qualify.py index 198654e..4752043 100644 --- a/pipeline/uk_replay_qualify.py +++ b/pipeline/uk_replay_qualify.py @@ -1393,11 +1393,14 @@ def main(argv: list[str] | None = None) -> int: print(sealed["receipt_sha256"]) return 0 if args.command == "verify-sealed": + # A variant namespace holding replay results (any event's + # RUN_MANIFEST.json) must be sealed. Premises, diagnostics and an + # empty summary committed before the replay are not results. namespaces = args.namespaces or sorted( variants.namespace_root(record) for record in variants.load_document()["variants"] if record["layout"] == "replay_namespace" - and variants.namespace_root(record).exists() + and any(variants.namespace_root(record).glob("*/RUN_MANIFEST.json")) ) problems = [] for namespace in namespaces: diff --git a/tests/test_uk_replay_qualify.py b/tests/test_uk_replay_qualify.py index ed6d9f6..d055980 100644 --- a/tests/test_uk_replay_qualify.py +++ b/tests/test_uk_replay_qualify.py @@ -383,6 +383,10 @@ def test_an_inert_construction_makes_the_receipt_incomplete_and_names_the_pair( else: assert receipt["integrity"]["status"] == "complete" assert event["inert_measure_years"] == [] + # The receipt W4 reads: sealed and valid, the run complete either way. + sealed = qualify.seal(rv.namespace_root(record, root=tmp_path), receipt) + interface.validate(sealed, interface.RECEIPT_SCHEMA_ID) + assert sealed["status"] == "complete" def test_rejection_discards_a_supplied_record_and_cli_accepts_candidate(tmp_path): @@ -730,6 +734,34 @@ def test_verify_sealed_checks_expected_namespace_with_missing_seal( assert "not sealed" in capsys.readouterr().out +def test_a_namespace_must_be_sealed_once_it_holds_replay_results( + tmp_path, monkeypatch, capsys +): + # Review of 176e974: premises, diagnostics and an empty summary committed + # before a variant's replay are not results; one event's run manifest is. + namespace = tmp_path / "ns" + namespace.mkdir() + (namespace / "CONSTRUCTION_PREMISES.json").write_text("{}") + (namespace / "SUMMARY.md").write_text("# summary\n") + monkeypatch.setattr( + rv, + "load_document", + lambda: { + "variants": [{"layout": "replay_namespace", "result_root": str(namespace)}] + }, + ) + assert qualify.main(["verify-sealed"]) == 0 + (namespace / "an_event").mkdir() + (namespace / "an_event" / "RUN_MANIFEST.json").write_text("{}") + assert qualify.main(["verify-sealed"]) == 1 + assert "not sealed" in capsys.readouterr().out + + +def test_the_committed_namespaces_pass_the_seal_check(): + # CI's "Replay namespaces regenerate and seals hold" step, on this tree. + assert qualify.main(["verify-sealed"]) == 0 + + @pytest.mark.parametrize("code", [None, "artifact_hash_mismatch", "data_access_failed"]) def test_dataset_fetch_cli_hands_file_or_rejection_to_run_request( tmp_path, monkeypatch, code diff --git a/tests/test_uk_replay_queue.py b/tests/test_uk_replay_queue.py index 1c4cb7c..dc151c4 100644 --- a/tests/test_uk_replay_queue.py +++ b/tests/test_uk_replay_queue.py @@ -227,10 +227,30 @@ def test_candidate_job_has_read_permission_and_no_dataset_token_or_credentials() 'contains(fromJSON(\'["workflow_dispatch", "repository_dispatch"]\'), ' "github.event_name)" ) + # Then a private copy for the unprivileged account, the run as that + # account, every candidate process killed and the output taken back, the + # guard, and the upload only when the guard passed. steps = [step.get("name") for step in candidate["steps"]] - assert steps.index(fetch["name"]) < steps.index( - "Run the candidate and preserve logs and outcome" - ) + order = [ + fetch["name"], + "Hand a private copy to the unprivileged candidate account", + "Run the candidate and preserve logs and outcome", + "Stop every candidate process and take its output back", + "Nothing household-level leaves the runner", + "Upload the receipt and results for the recorder", + ] + assert [steps.index(name) for name in order] == sorted( + steps.index(name) for name in order + ) + by_name = {step.get("name"): step for step in candidate["steps"]} + assert by_name[order[1]]["run"] == "bash .github/scripts/uk_replay_sandbox.sh setup" + assert by_name[order[2]]["run"] == "bash .github/scripts/uk_replay_sandbox.sh run" + assert by_name[order[3]] == { + "name": order[3], + "if": "always()", + "run": "bash .github/scripts/uk_replay_sandbox.sh collect", + } + assert "huggingface_hub==2.2.0" in fetch["run"] rest = {k: v for k, v in candidate.items() if k != "steps"} assert "secrets." not in json.dumps(rest) assert workflow["jobs"]["record"]["permissions"] == { @@ -259,14 +279,14 @@ def test_ci_checks_untracked_regeneration_and_announces_skipped_real_replay(): if step.get("name") == "Replay namespaces regenerate and seals hold" ) assert "git status --porcelain" in steps[regeneration_index + 1]["run"] - replay = next( + fetch = next( step for step in workflow["jobs"]["replay-engine"]["steps"] - if step.get("name") == "One real measure-year replays byte for byte" + if step.get("name") == "Fetch the certified dataset for the real replay" ) - assert '"$RESULT_ROOT/REPLAY_RECEIPT.json"' in replay["run"] - assert '"$RESULT_ROOT/NAMESPACE_MANIFEST.json"' in replay["run"] - assert replay["run"].count("nothing was replayed") == 2 + assert '"$RESULT_ROOT/REPLAY_RECEIPT.json"' in fetch["run"] + assert '"$RESULT_ROOT/NAMESPACE_MANIFEST.json"' in fetch["run"] + assert fetch["run"].count("nothing was replayed") == 2 def test_nonzero_candidate_exit_keeps_logs_and_uploadable_outcome(tmp_path): @@ -464,6 +484,57 @@ def test_recorder_handles_missing_or_invalid_run_artifact_and_clears_stale_resul assert prior_status["status"] == "running" +def test_recorder_replaces_an_earlier_attempts_results_with_this_runs(tmp_path): + # Review of 176e974: the success path, not only the failure receipt, + # must leave no file of an earlier attempt beside this run's results. + seed = local_state(tmp_path) + request_id, _ = queue.enqueue( + seed, {"invalid": "request"}, run_url=RUN, now="2020-01-01T00:00:00Z" + ) + queue.set_status( + seed, request_id, "running", run_url=RUN, now="2020-01-01T01:00:00Z" + ) + stale = seed / "requests" / request_id / "results" + stale.mkdir() + (stale / "earlier-attempt.json").write_text("{}") + git(seed, "add", ".") + git(seed, "commit", "-qm", "Running request with stale results") + runner, remote, environment = control_runner(tmp_path, seed) + scratch = tmp_path / "scratch" + receipt = queue.write_failure_receipt( + scratch, request_id, code="event_run_failed", detail="this run's outcome" + ) + uploaded = runner / "output" / "state" / "requests" / request_id + (uploaded / "results").mkdir(parents=True) + (uploaded / "results" / "this-run.json").write_text("{}") + (uploaded / "REPLAY_RECEIPT.json").write_text(json.dumps(receipt)) + (runner / "output" / "outcome.json").write_text( + json.dumps( + {"status": receipt["status"], "receipt_sha256": receipt["receipt_sha256"]} + ) + ) + subprocess.run( + ["bash", ".github/scripts/uk_replay_record.sh"], + cwd=runner, + env={**environment, "REQUEST_ID": request_id}, + check=True, + capture_output=True, + text=True, + ) + status = json.loads( + git(remote, "show", f"replay-state:requests/{request_id}/STATUS.json") + ) + assert status["receipt_sha256"] == receipt["receipt_sha256"] + assert git( + remote, + "ls-tree", + "-r", + "--name-only", + "replay-state", + f"requests/{request_id}/results", + ).splitlines() == [f"requests/{request_id}/results/this-run.json"] + + def test_empty_pending_cli_has_no_output_for_redispatch_decision(tmp_path, capsys): assert ( queue.main( @@ -565,41 +636,97 @@ def test_ci_regeneration_preserves_sealed_source_and_detects_every_changed_byte( ) -GUARD = ROOT / ".github/scripts/uk_replay_output_guard.sh" +GUARD = ROOT / ".github/scripts/uk_replay_output_guard.py" def guard(directory: Path) -> subprocess.CompletedProcess: + # As the workflow runs it: isolated, standard library only. return subprocess.run( - ["bash", str(GUARD), str(directory)], capture_output=True, text=True + [sys.executable, "-I", str(GUARD), str(directory)], + capture_output=True, + text=True, ) -def test_output_guard_refuses_datasets_large_files_and_symlinks(tmp_path): - # Scorecard is public (d1383): only the receipt, status and sealed - # aggregate results may be uploaded from the run job. +def guarded_output(tmp_path): output = tmp_path / "output" - results = output / "results" / "autumn_budget_2024" + results = output / "state" / "requests" / ("a" * 64) / "results" / "ab2024" results.mkdir(parents=True) - (results / "COMPARISON.json").write_text("{}") - (output / "REPLAY_RECEIPT.json").write_text("{}") - assert guard(output).returncode == 0 + (results / "COMPARISON.json").write_text('{"rows": []}') + (results / "STAGED.jsonl").write_text('{"row": 1}\n\n{"row": 2}\n') + (results / "COMPARISON.csv").write_text("a,b\n1,2\n") + (results / "SUMMARY.md").write_text("# Summary\n") + (output / "run-exit.txt").write_text("0\n") + return output, results + + +def test_output_guard_allows_only_parsable_aggregates(tmp_path): + # Scorecard is public (d1383): an allowlist, not a denylist. Every file + # must be a regular file with an allowed extension whose bytes parse as + # that format, under per-file and total caps. + output, results = guarded_output(tmp_path) + assert guard(output).returncode == 0, guard(output).stdout assert guard(tmp_path / "absent").returncode == 0 - for name in ("dataset.h5", "frs.hdf5", "x.h5.partial", "people.parquet"): + refused = { + "dataset.h5": b"\x89HDF", + "people.parquet": b"PAR1", + "frame.feather": b"ARROW1", + "frame.pkl": b"\x80\x04", + "no_extension": b"{}", + "split.json": b"\x1f\x8b\x08\x00", # gzip bytes renamed + "broken.json": b'{"rows": [', + "broken.jsonl": b'{"row": 1}\nnot json\n', + "binary.csv": b"a,b\n\xff\xfe\n", + "nul.md": b"# x\x00\n", + } + for name, data in refused.items(): bad = results / name - bad.write_bytes(b"\0") - refused = guard(output) - assert refused.returncode == 1 and name in refused.stdout + bad.write_bytes(data) + result = guard(output) + assert result.returncode == 1 and name in result.stdout, name bad.unlink() big = results / "artifact.json" - with big.open("wb") as handle: - handle.truncate(25 * 1024 * 1024 + 1) + big.write_text('"' + "x" * (10 * 1024 * 1024) + '"') assert guard(output).returncode == 1 big.unlink() outside = tmp_path / "input.json" outside.write_text("{}") (results / "link.json").symlink_to(outside) - refused = guard(output) - assert refused.returncode == 1 and "symlink" in refused.stdout + result = guard(output) + assert result.returncode == 1 and "not a regular file" in result.stdout + (results / "link.json").unlink() + os.mkfifo(results / "pipe.json") + assert guard(output).returncode == 1 + (results / "pipe.json").unlink() + assert guard(output).returncode == 0 + + +def test_output_guard_caps_the_total(tmp_path): + output, results = guarded_output(tmp_path) + for index in range(12): # 108 MiB in all, each under 10 MiB + (results / f"part_{index}.json").write_text('"' + "x" * (9 * 1024 * 1024) + '"') + result = guard(output) + assert result.returncode == 1 and "in all" in result.stdout + + +def test_candidate_code_runs_as_an_unprivileged_account_with_nothing_inherited(): + script = (ROOT / ".github/scripts/uk_replay_sandbox.sh").read_text() + # The account has no sudo; a check fails the job if it ever does. + assert 'sudo useradd --create-home --shell /bin/bash "$CANDIDATE"' in script + assert "not allowed to run sudo" in script + # Its environment is built from nothing: no token, no runtime token, no + # GITHUB_ENV or GITHUB_PATH, no uv cache directory from setup-uv. + assert 'sudo -n -u "$CANDIDATE" env -i' in script + assert "--preserve-env" not in script and "sudo -E" not in script + assert "exec bash .github/scripts/uk_replay_drain.sh" in script + # Nothing it starts outlives the run, and its output comes back as a + # copy that keeps links as links, for the guard to refuse. + assert 'sudo pkill -KILL -u "$CANDIDATE"' in script + assert 'sudo cp -R --no-preserve=all "$COPY/output" output' in script + result = subprocess.run( + ["bash", "-n", str(ROOT / ".github/scripts/uk_replay_sandbox.sh")] + ) + assert result.returncode == 0 def test_nothing_crosses_jobs_but_the_request_and_guarded_output(): @@ -619,14 +746,46 @@ def test_nothing_crosses_jobs_but_the_request_and_guarded_output(): guard_step = steps[names.index("Nothing household-level leaves the runner")] upload = steps[names.index("Upload the receipt and results for the recorder")] assert names.index(guard_step["name"]) < names.index(upload["name"]) - assert "uk_replay_output_guard.sh output" in guard_step["run"] + assert guard_step["run"] == ( + "python3 -I .github/scripts/uk_replay_output_guard.py output" + ) assert upload["if"] == "always() && steps.guard.outcome == 'success'" assert guard_step["id"] == "guard" and guard_step["if"] == "always()" - for path in (".github/workflows/uk-replay.yml", ".github/workflows/ci.yml"): - document = yaml.safe_load((ROOT / path).read_text()) + # The recorder checks again, where no candidate code has run, and drops + # a refused output before anything reaches the public branch. + record = workflow["jobs"]["record"]["steps"] + record_names = [step.get("name") for step in record] + recheck = record[ + record_names.index("Nothing household-level reaches the public branch") + ] + assert ( + "python3 -I .github/scripts/uk_replay_output_guard.py output" in recheck["run"] + ) + assert "rm -rf output" in recheck["run"] + assert record_names.index(recheck["name"]) < record_names.index( + "Record the outcome and continue the queue" + ) + # No cache crosses jobs: setup-uv caches by default (v5: enable-cache + # auto), so every use here turns it off, as does the CI job that runs a + # variant's engine; nothing uses actions/cache. + uses = [ + step + for job in workflow["jobs"].values() + for step in job["steps"] + if step.get("uses", "").startswith("astral-sh/setup-uv") + ] + assert len(uses) == 4 + ci = yaml.safe_load((ROOT / ".github/workflows/ci.yml").read_text()) + uses += [ + step + for step in ci["jobs"]["replay-engine"]["steps"] + if step.get("uses", "").startswith("astral-sh/setup-uv") + ] + assert all(step.get("with", {}).get("enable-cache") is False for step in uses) + for document in (workflow, ci): for job in document["jobs"].values(): for step in job["steps"]: - assert not step.get("uses", "").startswith("actions/cache"), path + assert not step.get("uses", "").startswith("actions/cache") def test_the_data_token_is_used_only_outside_pull_requests(): @@ -634,15 +793,48 @@ def test_the_data_token_is_used_only_outside_pull_requests(): # PyYAML reads the bare key `on` as True. assert set(replay[True]) == {"repository_dispatch", "workflow_dispatch", "schedule"} ci = yaml.safe_load((ROOT / ".github/workflows/ci.yml").read_text()) + token = "secrets.PE_UK_PRIVATE_HF_READ_TOKEN" + # Exactly two steps hold it, each with exactly this condition. + holders = { + (step["name"], step["if"]) + for document in (replay, ci) + for job in document["jobs"].values() + for step in job["steps"] + if token in json.dumps(step) + } + assert holders == { + ( + "Fetch and verify the dataset before any candidate code", + 'contains(fromJSON(\'["workflow_dispatch", "repository_dispatch"]\'), ' + "github.event_name)", + ), + ( + "Fetch the certified dataset for the real replay", + "env.LEGACY == 'False' && github.event_name == 'push'", + ), + } for document in (replay, ci): + # Never at workflow or job level, where every step would see it. + assert "secrets." not in json.dumps(document.get("env", {})) for job in document["jobs"].values(): + assert "secrets." not in json.dumps(job.get("env", {})) for step in job["steps"]: - text = json.dumps(step) - if "HF_READ_TOKEN" in text or "HUGGING_FACE" in text: - assert "secrets.HUGGING_FACE_TOKEN" not in text - if "secrets.PE_UK_PRIVATE_HF_READ_TOKEN" in text: - assert "pull_request" not in step["if"] - assert ( - step["if"].endswith("github.event_name == 'push'") - or "workflow_dispatch" in step["if"] - ) + assert "secrets.HUGGING_FACE_TOKEN" not in json.dumps(step) + # The CI job that runs a variant's engine reads only, keeps no checkout + # credential, and runs the engine in a later step than the token. + engine = ci["jobs"]["replay-engine"] + assert engine["permissions"] == {"contents": "read"} + checkout = next( + s for s in engine["steps"] if s.get("uses") == "actions/checkout@v4" + ) + assert checkout["with"]["persist-credentials"] is False + names = [step.get("name") for step in engine["steps"]] + fetch = engine["steps"][ + names.index("Fetch the certified dataset for the real replay") + ] + replay_step = engine["steps"][ + names.index("One real measure-year replays byte for byte") + ] + assert names.index(fetch["name"]) < names.index(replay_step["name"]) + assert replay_step["if"] == "steps.fetch.outputs.fetched == 'true'" + assert "env" not in replay_step and "huggingface_hub==2.2.0" in fetch["run"] From c6a95cde15c62befa0a28816a9f1d3537ccae381 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 15 Oct 2026 07:45:53 -0400 Subject: [PATCH 6/7] Receiver: answer the review of 2896610 - CI was red: the regeneration script's uv run lacked numpy and pyyaml, which staging reaches at import time (compute_uk_event -> compute_uk_ab2025). A test now walks the import-time closure of every module the script runs and checks the script installs each package. - Sandbox: setup checks, as the account itself, that it cannot use sudo, has no supplementary group, and cannot write the workspace, the runner's command files or the docker socket; the account is denied cron and at; collect streams output out as the account (tar) instead of copying as root inside its home. The docs say "no write access", not "no access". - A new secret-free CI job (receiver sandbox) runs setup, run and collect on the runner image with a stub drain that leaves a process behind, and checks nothing survives and the guarded output comes back. - Guard: tables only as COMPARISON.csv and STAGED.jsonl; the docstring and docs no longer claim it stops a household table under those names. - The recorder's recheck is a script with a behavioural test (a refused output is dropped, a good one kept). - verify-sealed also checks a namespace whose receipt or manifest remains, so deleting its event directories no longer passes. - Docs name the other routes to malicious candidate code (a fork commit through the wrapper's git source, a third-party dependency under free resolution) and the CI replay-engine job's weaker isolation. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/uk_replay_output_guard.py | 19 ++- .github/scripts/uk_replay_recheck.sh | 9 ++ .github/scripts/uk_replay_regenerate.sh | 3 +- .github/scripts/uk_replay_sandbox.sh | 55 +++++-- .github/workflows/ci.yml | 45 ++++++ .github/workflows/uk-replay.yml | 15 +- docs/uk_replay/RELEASE_REPLAY.md | 52 ++++-- pipeline/uk_replay_qualify.py | 14 +- tests/test_uk_replay_qualify.py | 19 +++ tests/test_uk_replay_queue.py | 187 ++++++++++++++++++++-- 10 files changed, 351 insertions(+), 67 deletions(-) create mode 100644 .github/scripts/uk_replay_recheck.sh diff --git a/.github/scripts/uk_replay_output_guard.py b/.github/scripts/uk_replay_output_guard.py index 1b729f8..888073d 100644 --- a/.github/scripts/uk_replay_output_guard.py +++ b/.github/scripts/uk_replay_output_guard.py @@ -4,13 +4,15 @@ the public replay-state branch, may hold only the receipt, status and the sealed aggregate results. This is an allowlist: every entry must be a regular file (no symlink, device or other special file) with an allowed extension, -whose bytes parse as that format, under a per-file and a total size cap. The -committed sealed namespace has 30 MB over 316 files, the largest 3.3 MB. +whose bytes parse as that format, under a per-file and a total size cap. +Tables are allowed only under the names the pipeline writes them: +COMPARISON.csv and STAGED.jsonl. The committed sealed namespace has 30 MB +over 316 files, the largest 3.3 MB. -It stops accidental publication (an honest run that writes a dataset, -household-level table or binary). Content that parses can still carry -anything a deliberately malicious candidate encodes in it; RELEASE_REPLAY.md -states that limit. +It stops an honest run from publishing a dataset copy, a binary, or a table +under any other name. It does not read what a parsable file says: a +household-level table written as COMPARISON.csv, or anything a deliberately +malicious candidate encodes, would pass; RELEASE_REPLAY.md states that limit. Standard library only: it runs with the runner's system Python, before and independently of anything the candidate installed. @@ -26,6 +28,8 @@ from pathlib import Path ALLOWED_SUFFIXES = (".json", ".jsonl", ".csv", ".md", ".txt") +# Tables only under the names the pipeline writes them. +TABLE_NAMES = {".csv": {"COMPARISON.csv"}, ".jsonl": {"STAGED.jsonl"}} MAX_FILE_BYTES = 10 * 1024 * 1024 MAX_TOTAL_BYTES = 100 * 1024 * 1024 @@ -66,6 +70,9 @@ def problems(root: Path) -> list[str]: if path.suffix not in ALLOWED_SUFFIXES: found.append(f"{path}: extension not allowed ({path.suffix or 'none'})") continue + if path.suffix in TABLE_NAMES and path.name not in TABLE_NAMES[path.suffix]: + found.append(f"{path}: a table under an unexpected name") + continue size = path.stat().st_size total += size if size > MAX_FILE_BYTES: diff --git a/.github/scripts/uk_replay_recheck.sh b/.github/scripts/uk_replay_recheck.sh new file mode 100644 index 0000000..080fd76 --- /dev/null +++ b/.github/scripts/uk_replay_recheck.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# The record job's second guard, where no candidate code has run: a refused +# output is dropped, so the recorder writes a failed receipt and nothing of +# it reaches the public replay-state branch. +set -euo pipefail +if ! python3 -I .github/scripts/uk_replay_output_guard.py output; then + echo "::error::the run's output was refused; recording a failed receipt" + rm -rf output +fi diff --git a/.github/scripts/uk_replay_regenerate.sh b/.github/scripts/uk_replay_regenerate.sh index 746403d..f533885 100644 --- a/.github/scripts/uk_replay_regenerate.sh +++ b/.github/scripts/uk_replay_regenerate.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # Regenerate committed replay bytes in a disposable checkout, preserving seals. set -euo pipefail -run() { uv run --with rfc8785==0.1.4 --with jsonschema python "$@"; } +# Every third-party module the scripts below import (tests check the list). +run() { uv run --with rfc8785==0.1.4 --with jsonschema --with numpy --with pyyaml python "$@"; } # Check the original receipts before making the disposable namespaces writable. run -m pipeline.uk_replay_qualify verify-sealed diff --git a/.github/scripts/uk_replay_sandbox.sh b/.github/scripts/uk_replay_sandbox.sh index 34eb1b3..1f38e58 100644 --- a/.github/scripts/uk_replay_sandbox.sh +++ b/.github/scripts/uk_replay_sandbox.sh @@ -4,11 +4,14 @@ # The runner user has passwordless sudo, owns this job's workspace and its # command files ($GITHUB_ENV, $GITHUB_PATH), and owns the runner processes # that hold the job's secrets and the Actions runtime token. The candidate -# account has none of these: no sudoers entry, a private copy of the -# repository and the verified request inputs, an environment built from -# nothing (env -i), and no read access to another user's process memory or -# environment. Nothing it starts survives `collect`, and only a copy of its -# output/ comes back, to be checked by the runner's own guard. +# account has no sudoers entry, no supplementary groups (so no docker +# socket), no cron or at jobs, a private copy of the repository and the +# verified request inputs, and an environment built from nothing (env -i). +# It cannot write the runner user's files or read another user's process +# memory or environment; `setup` checks the first of these as the account +# itself and fails the job otherwise. Its processes are killed in `collect`, +# and only a copy of its output/, streamed out by the account itself, comes +# back, to be checked by the runner's own guard. # # uk_replay_sandbox.sh setup # after the dataset fetch, before any candidate code # uk_replay_sandbox.sh run # the drain, as the candidate account @@ -18,13 +21,17 @@ CANDIDATE=replay-candidate CANDIDATE_HOME=/home/$CANDIDATE COPY=$CANDIDATE_HOME/scorecard +as_candidate() { + sudo -n -u "$CANDIDATE" env -i HOME="$CANDIDATE_HOME" PATH=/usr/bin:/bin "$@" +} + case "${1:?usage: uk_replay_sandbox.sh setup|run|collect}" in setup) sudo useradd --create-home --shell /bin/bash "$CANDIDATE" - if ! sudo -l -U "$CANDIDATE" | grep -q "not allowed to run sudo"; then - echo "::error::$CANDIDATE can use sudo" - exit 1 - fi + # No scheduled jobs that could outlive `collect`. + for deny in /etc/cron.deny /etc/at.deny; do + echo "$CANDIDATE" | sudo tee -a "$deny" > /dev/null + done sudo chmod 700 "$CANDIDATE_HOME" # Its own uv, whatever the permissions of the runner's tool cache. sudo install -d -o "$CANDIDATE" -g "$CANDIDATE" "$CANDIDATE_HOME/bin" @@ -34,6 +41,23 @@ case "${1:?usage: uk_replay_sandbox.sh setup|run|collect}" in sudo cp -a "$GITHUB_WORKSPACE/." "$COPY" sudo rm -rf "$COPY/output" sudo chown -R "$CANDIDATE:$CANDIDATE" "$COPY" + # Check the account as itself: no sudo, no group beyond its own, and no + # write access to the workspace, the runner's command files or docker. + if as_candidate sudo -n true 2> /dev/null; then + echo "::error::$CANDIDATE can use sudo" + exit 1 + fi + if [ "$(as_candidate id -G)" != "$(as_candidate id -g)" ]; then + echo "::error::$CANDIDATE has supplementary groups: $(as_candidate id -Gn)" + exit 1 + fi + for target in "$GITHUB_WORKSPACE" "$(dirname "$GITHUB_ENV")" "$GITHUB_ENV" /var/run/docker.sock; do + if [ -e "$target" ] && as_candidate test -w "$target"; then + echo "::error::$CANDIDATE can write $target" + exit 1 + fi + done + echo "$CANDIDATE: no sudo, no supplementary groups, no write access to the runner's files" ;; run) status=0 @@ -59,11 +83,14 @@ case "${1:?usage: uk_replay_sandbox.sh setup|run|collect}" in exit 1 fi rm -rf output - if sudo test -d "$COPY/output"; then - # -R copies links as links and special files as special files, which - # the guard then refuses; --no-preserve drops the candidate's modes. - sudo cp -R --no-preserve=all "$COPY/output" output - sudo chown -R "$(id -u):$(id -g)" output + # The account streams its own output/ out (tar keeps links as links and + # special files as special files, which the guard refuses); nothing runs + # as root inside its home. The runner user unpacks it as itself. + if as_candidate test -d "$COPY/output"; then + mkdir output + as_candidate tar -C "$COPY/output" -cf - . | tar -C output --no-same-owner --no-same-permissions -xf - + # Kill anything the stream itself might have started. + sudo pkill -KILL -u "$CANDIDATE" || true fi ;; *) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61dc42c..ad0b074 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,6 +161,51 @@ jobs: cmp ".venv-replay-checks/determinism/${key}_2026.json" "$RESULT_ROOT/autumn_budget_2024/${key}_2026.json" echo "replayed $key 2026 byte-identical to the committed artifact" + receiver-sandbox: + # The receiver's unprivileged candidate account + # (.github/scripts/uk_replay_sandbox.sh) on a real runner image, with a + # stub drain that leaves a process behind. No secrets, so it runs on pull + # requests too: setup checks the account has no sudo, no supplementary + # groups and no write access to the runner's files; collect must kill + # everything and bring a guarded output back. + name: receiver sandbox + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: false + - name: Set up the candidate account and check it + run: bash .github/scripts/uk_replay_sandbox.sh setup + - name: Replace the copy's drain with a stub that leaves a process behind + run: | + sudo -u replay-candidate tee /home/replay-candidate/scorecard/.github/scripts/uk_replay_drain.sh > /dev/null <<'SH' + set -euo pipefail + mkdir -p output/state + python3 -c 'import json; print(json.dumps({"status": "failed", "receipt_sha256": "a" * 64}))' > output/outcome.json + echo 0 > output/run-exit.txt + nohup sleep 600 > /dev/null 2>&1 & + SH + - name: Run the stub as the candidate + env: + REQUEST_ID: stub + REPLAY_RUNNER: ci + run: bash .github/scripts/uk_replay_sandbox.sh run + - name: Nothing survives and the guarded output comes back + run: | + set -euo pipefail + bash .github/scripts/uk_replay_sandbox.sh collect + if pgrep -u replay-candidate; then + echo "::error::candidate processes survive collect" + exit 1 + fi + test "$(stat -c %U output/outcome.json)" = "$(id -un)" + python3 -I .github/scripts/uk_replay_output_guard.py output + app: name: typecheck · build runs-on: ubuntu-latest diff --git a/.github/workflows/uk-replay.yml b/.github/workflows/uk-replay.yml index 9594fbc..908e4a0 100644 --- a/.github/workflows/uk-replay.yml +++ b/.github/workflows/uk-replay.yml @@ -12,10 +12,11 @@ name: UK replay qualification # only dispatches a drain. In the run job it is in one step's environment, # which fetches and verifies the dataset before any candidate code is # installed. That job has contents: read and no checkout credentials, and -# candidate code runs as an unprivileged account with no sudo, no access to -# the workspace, the runner's command files or the processes holding the -# job's secrets, and an environment built from nothing -# (.github/scripts/uk_replay_sandbox.sh). Scorecard is public: the dataset and +# candidate code runs as an unprivileged account with no sudo, no groups, no +# write access to the workspace or the runner's command files, no access to +# the processes holding the job's secrets, and an environment built from +# nothing (.github/scripts/uk_replay_sandbox.sh; CI's receiver-sandbox job +# checks it on the runner image). Scorecard is public: the dataset and # anything household-level never leave the runner. setup-uv's cache is off # everywhere here, so no job saves or restores a cache; the only artifacts are # the request and the run's output, which an allowlist guard checks before the @@ -200,11 +201,7 @@ jobs: # public branch. A refused output is dropped: the recorder then records # a failed receipt. - name: Nothing household-level reaches the public branch - run: | - if ! python3 -I .github/scripts/uk_replay_output_guard.py output; then - echo "::error::the run's output was refused; recording a failed receipt" - rm -rf output - fi + run: bash .github/scripts/uk_replay_recheck.sh - name: Record the outcome and continue the queue env: REQUEST_ID: ${{ needs.prepare.outputs.request_id }} diff --git a/docs/uk_replay/RELEASE_REPLAY.md b/docs/uk_replay/RELEASE_REPLAY.md index 6d76f9b..9b3482c 100644 --- a/docs/uk_replay/RELEASE_REPLAY.md +++ b/docs/uk_replay/RELEASE_REPLAY.md @@ -416,15 +416,22 @@ Candidate code then runs as an unprivileged account runner user has passwordless sudo, owns the workspace and the runner's command files (`$GITHUB_ENV`, `$GITHUB_PATH`), and owns the runner processes that keep the job's secrets in memory for the whole job and pass the Actions -runtime token to later steps. The candidate account has no sudoers entry -(checked: the job fails if it can use sudo), works in its own copy of the -repository and of `input/` (the request, the verified dataset and the fetch -outcome), starts from an empty environment (`env -i`), and cannot read -another user's process memory or environment. It runs -`.github/scripts/uk_replay_drain.sh` for that one request. Afterwards every -process it started is killed and a copy of its `output/` comes back to the -runner user; links are copied as links and special files as special files, -for the guard to refuse. +runtime token to later steps. The candidate account has no sudoers entry, no +supplementary group (so no docker socket) and no cron or at jobs. `setup` +checks, as the account itself, that it cannot use sudo, has no group beyond +its own, and cannot write the workspace, the runner's command files or the +docker socket, and fails the job otherwise. The account works in its own +copy of the repository and of `input/` (the request, the verified dataset +and the fetch outcome), starts from an empty environment (`env -i`), and +cannot read another user's process memory or environment. It may be able to +read the runner user's files if the runner image leaves them readable; none +holds a secret. It runs `.github/scripts/uk_replay_drain.sh` for that one +request. Afterwards every process it started is killed and the account +streams its `output/` out with `tar`, which the runner user unpacks as +itself; links stay links and special files stay special files, for the guard +to refuse. CI's `receiver sandbox` job runs `setup`, `run` and `collect` on +the runner image on every pull request, with a stub drain that leaves a +process behind, and no secrets. Scorecard is public, so the dataset and anything household-level stay on the runner. No job saves or restores a cache (setup-uv's default cache is off on @@ -434,9 +441,11 @@ to the jobs that can write. The only artifacts are the request and the run's library, run with the runner's own Python) is an allowlist: every entry must be a regular file ending `.json`, `.jsonl`, `.csv`, `.md` or `.txt` whose bytes parse as that format (UTF-8 text, no NUL bytes), at most 10 MB each and -100 MB in all; the committed sealed namespace for 6.2.6 is 30 MB, its largest -file 3.3 MB. The run job uploads only if the guard passes. The `record` job, -where no candidate code has run, applies the guard again before committing +100 MB in all; tables are allowed only as `COMPARISON.csv` and +`STAGED.jsonl`, the names the pipeline writes. The committed sealed namespace +for 6.2.6 is 30 MB, its largest file 3.3 MB. The run job uploads only if the +guard passes. The `record` job, where no candidate code has run, applies the +guard again (`.github/scripts/uk_replay_recheck.sh`) before committing anything to the public `replay-state` branch, and drops a refused output, so the attempt is recorded as `failed`. @@ -446,15 +455,24 @@ repository can send one (`repository_dispatch` and `workflow_dispatch` both require it); requests are meant for PolicyEngine's own releases, from W4. A new candidate's dependencies resolve within the newest certified freeze where they can. The candidate and its dependencies are trusted with the -dataset: they compute on it, and the guard stops accidental publication -(an honest run that writes a dataset copy, a household-level table or a -binary). Code in the candidate's dependency closure cannot reach the data +dataset: they compute on it, and the guard stops an honest run from +publishing a dataset copy, a binary or a table under any other name. The +guard does not read what a parsable file says: a household-level table +written as `COMPARISON.csv` would pass. Code in the candidate's dependency closure cannot reach the data token, the runtime token, the write jobs' credentials or a cache, and cannot alter the guard or what the recorder checks. A deliberately malicious candidate could still encode data in aggregates that parse, and could forge a schema-valid receipt for its own request, since the receipt and seal are -produced beside it; neither is in scope, and both would require a malicious -PolicyEngine release. `pipeline/uk_replay_qualify.py run-request` then: +produced beside it. Neither is in scope. Each needs malicious code in the +candidate's environment: a malicious PolicyEngine release, a commit reached +through the wrapper's `PolicyEngine/policyengine.py` git source (which can +name a fork's commit), or a malicious third-party dependency pulled in when +installation falls back to free resolution. + +CI's `replay engine` job is weaker: the variant's engine runs as the runner +user, so code in a certified freeze could reach the token held by the fetch +step's runner process. That job runs only on pushes to main, with committed, +certified freezes. `pipeline/uk_replay_qualify.py run-request` then: 1. Validate the request (`schema_invalid`). 2. Check the dataset file supplied by the control job against the request diff --git a/pipeline/uk_replay_qualify.py b/pipeline/uk_replay_qualify.py index 4752043..57387a1 100644 --- a/pipeline/uk_replay_qualify.py +++ b/pipeline/uk_replay_qualify.py @@ -1394,13 +1394,21 @@ def main(argv: list[str] | None = None) -> int: return 0 if args.command == "verify-sealed": # A variant namespace holding replay results (any event's - # RUN_MANIFEST.json) must be sealed. Premises, diagnostics and an - # empty summary committed before the replay are not results. + # RUN_MANIFEST.json), or that was sealed once (its receipt or + # manifest is still there), must be sealed. Premises, diagnostics + # and an empty summary committed before the replay are not results. + def must_be_sealed(namespace: Path) -> bool: + return ( + any(namespace.glob("*/RUN_MANIFEST.json")) + or (namespace / variants.RECEIPT_NAME).exists() + or (namespace / MANIFEST_NAME).exists() + ) + namespaces = args.namespaces or sorted( variants.namespace_root(record) for record in variants.load_document()["variants"] if record["layout"] == "replay_namespace" - and any(variants.namespace_root(record).glob("*/RUN_MANIFEST.json")) + and must_be_sealed(variants.namespace_root(record)) ) problems = [] for namespace in namespaces: diff --git a/tests/test_uk_replay_qualify.py b/tests/test_uk_replay_qualify.py index d055980..3a67cb1 100644 --- a/tests/test_uk_replay_qualify.py +++ b/tests/test_uk_replay_qualify.py @@ -7,6 +7,7 @@ import copy import hashlib import json +import shutil from pathlib import Path import jsonschema @@ -757,6 +758,24 @@ def test_a_namespace_must_be_sealed_once_it_holds_replay_results( assert "not sealed" in capsys.readouterr().out +def test_a_once_sealed_namespace_stays_checked_after_its_events_go( + tmp_path, monkeypatch, capsys +): + # Review of 2896610: deleting every event directory but keeping the + # receipt must not pass. + namespace, _, _ = sealed_namespace(tmp_path) + shutil.rmtree(namespace / "event") + monkeypatch.setattr( + rv, + "load_document", + lambda: { + "variants": [{"layout": "replay_namespace", "result_root": str(namespace)}] + }, + ) + assert qualify.main(["verify-sealed"]) == 1 + capsys.readouterr() + + def test_the_committed_namespaces_pass_the_seal_check(): # CI's "Replay namespaces regenerate and seals hold" step, on this tree. assert qualify.main(["verify-sealed"]) == 0 diff --git a/tests/test_uk_replay_queue.py b/tests/test_uk_replay_queue.py index dc151c4..7ca1efe 100644 --- a/tests/test_uk_replay_queue.py +++ b/tests/test_uk_replay_queue.py @@ -545,6 +545,100 @@ def test_empty_pending_cli_has_no_output_for_redispatch_decision(tmp_path, capsy assert capsys.readouterr().out == "" +# Import names whose PyPI distribution has another name. +DISTRIBUTIONS = {"yaml": "pyyaml"} + + +def third_party_imports(entry: str) -> set[str]: + """Top-level non-standard-library modules imported at import time (not + inside functions) by the repository modules reachable from ``entry``.""" + import ast + + def import_time(body): + for node in body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + if isinstance(node, (ast.Import, ast.ImportFrom)): + yield node + for field in ("body", "orelse", "finalbody", "handlers"): + yield from import_time(getattr(node, field, []) or []) + + def local(name: str) -> str | None: + """The repository file a module name refers to, if any.""" + parts = name.split(".") + if parts[0] == "pipeline": + parts = parts[1:] + for candidate in ( + "/".join(["pipeline", *parts]) + ".py", + "/".join(parts) + ".py", + "/".join([*parts, "__init__.py"]), + ): + if parts and (ROOT / candidate).is_file(): + return candidate + return None + + seen, pending, found = set(), [entry], set() + while pending: + relative = pending.pop() + if relative in seen: + continue + seen.add(relative) + tree = ast.parse((ROOT / relative).read_text()) + for node in import_time(tree.body): + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif node.level: + continue + else: + names = [node.module] + [ + f"{node.module}.{alias.name}" for alias in node.names + ] + for name in names: + path = local(name) + if path is not None: + pending.append(path) + elif ( + name.split(".")[0] not in sys.stdlib_module_names + and not local(name.split(".")[0]) + and name.split(".")[0] not in {"pipeline", "scorecard_db"} + ): + found.add(name.split(".")[0]) + return found + + +def test_ci_regeneration_installs_every_package_its_scripts_import(): + # Review of 2896610: the regeneration step failed in CI on numpy, which + # stage_uk_event reaches through compute_uk_event and compute_uk_ab2025. + script = (ROOT / ".github/scripts/uk_replay_regenerate.sh").read_text() + run = next(line for line in script.splitlines() if line.startswith("run() {")) + installed = { + word.split("==")[0] + for word, before in zip(run.split()[1:], run.split()) + if before == "--with" + } + entries = [ + "pipeline/uk_replay_qualify.py", + "pipeline/stage_uk_event.py", + "pipeline/compare_uk_event.py", + "pipeline/diff_uk_event_runs.py", + ] + # Engines are imported only inside functions the regeneration never + # calls (simulations); they are not regeneration dependencies. + engines = { + "policyengine", + "policyengine_uk", + "policyengine_core", + "huggingface_hub", + "modal", + "h5py", + "pandas", + "tables", + } + needed = set().union(*(third_party_imports(entry) for entry in entries)) + needed = {DISTRIBUTIONS.get(name, name) for name in needed} - engines + assert needed <= installed, sorted(needed - installed) + + @pytest.mark.parametrize("regeneration", ["identical", "drift", "untracked"]) def test_ci_regeneration_preserves_sealed_source_and_detects_every_changed_byte( tmp_path, regeneration @@ -678,6 +772,9 @@ def test_output_guard_allows_only_parsable_aggregates(tmp_path): "broken.jsonl": b'{"row": 1}\nnot json\n', "binary.csv": b"a,b\n\xff\xfe\n", "nul.md": b"# x\x00\n", + # Tables only under the names the pipeline writes them. + "households.csv": b"id,income\n1,2\n", + "people.jsonl": b'{"id": 1}\n', } for name, data in refused.items(): bad = results / name @@ -711,22 +808,81 @@ def test_output_guard_caps_the_total(tmp_path): def test_candidate_code_runs_as_an_unprivileged_account_with_nothing_inherited(): script = (ROOT / ".github/scripts/uk_replay_sandbox.sh").read_text() - # The account has no sudo; a check fails the job if it ever does. - assert 'sudo useradd --create-home --shell /bin/bash "$CANDIDATE"' in script - assert "not allowed to run sudo" in script + code = "\n".join( + line for line in script.splitlines() if not line.lstrip().startswith("#") + ) + # A fresh account, denied cron and at, never added to a group. + assert 'sudo useradd --create-home --shell /bin/bash "$CANDIDATE"' in code + assert "/etc/cron.deny /etc/at.deny" in code + assert "usermod" not in code and " -G " not in code and "gpasswd" not in code + # setup checks the account as itself and fails the job otherwise. + assert "if as_candidate sudo -n true" in code + assert '"$(as_candidate id -G)" != "$(as_candidate id -g)"' in code + assert 'as_candidate test -w "$target"' in code + for target in ('"$GITHUB_WORKSPACE"', '"$GITHUB_ENV"', "/var/run/docker.sock"): + assert target in code # Its environment is built from nothing: no token, no runtime token, no # GITHUB_ENV or GITHUB_PATH, no uv cache directory from setup-uv. - assert 'sudo -n -u "$CANDIDATE" env -i' in script - assert "--preserve-env" not in script and "sudo -E" not in script - assert "exec bash .github/scripts/uk_replay_drain.sh" in script - # Nothing it starts outlives the run, and its output comes back as a - # copy that keeps links as links, for the guard to refuse. - assert 'sudo pkill -KILL -u "$CANDIDATE"' in script - assert 'sudo cp -R --no-preserve=all "$COPY/output" output' in script - result = subprocess.run( - ["bash", "-n", str(ROOT / ".github/scripts/uk_replay_sandbox.sh")] + assert 'sudo -n -u "$CANDIDATE" env -i' in code + assert "--preserve-env" not in code and "sudo -E" not in code + assert "exec bash .github/scripts/uk_replay_drain.sh" in code + # Nothing it starts outlives the run, and its output comes back streamed + # out by the account itself, links kept as links for the guard. + assert 'sudo pkill -KILL -u "$CANDIDATE"' in code + assert 'as_candidate tar -C "$COPY/output" -cf - .' in code + assert "sudo cp -R" not in code + assert ( + subprocess.run( + ["bash", "-n", str(ROOT / ".github/scripts/uk_replay_sandbox.sh")] + ).returncode + == 0 ) - assert result.returncode == 0 + + +def test_the_sandbox_is_exercised_on_a_real_runner_image_without_secrets(): + # The script needs sudo and useradd, so its behaviour is checked in CI on + # the runner image itself, with a stub drain that leaves a process. + job = yaml.safe_load((ROOT / ".github/workflows/ci.yml").read_text())["jobs"][ + "receiver-sandbox" + ] + assert job["permissions"] == {"contents": "read"} + assert "secrets." not in json.dumps(job) + runs = [step.get("run", "") for step in job["steps"]] + joined = "\n".join(runs) + for command in ("sandbox.sh setup", "sandbox.sh run", "sandbox.sh collect"): + assert command in joined + assert "nohup sleep 600" in joined + assert "if pgrep -u replay-candidate" in joined + assert "python3 -I .github/scripts/uk_replay_output_guard.py output" in joined + + +def test_the_recorder_drops_a_refused_output_and_keeps_a_good_one(tmp_path): + scripts = tmp_path / ".github/scripts" + scripts.mkdir(parents=True) + for name in ("uk_replay_recheck.sh", "uk_replay_output_guard.py"): + shutil.copyfile(ROOT / ".github/scripts" / name, scripts / name) + tools = tmp_path / "bin" + tools.mkdir() + (tools / "python3").symlink_to(sys.executable) + env = {**os.environ, "PATH": f"{tools}:{os.environ['PATH']}"} + + def recheck(): + return subprocess.run( + ["bash", ".github/scripts/uk_replay_recheck.sh"], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + + output = tmp_path / "output" + output.mkdir() + (output / "outcome.json").write_text("{}") + assert recheck().returncode == 0 and (output / "outcome.json").exists() + (output / "dataset.h5").write_bytes(b"\x89HDF") + result = recheck() + assert result.returncode == 0 and not output.exists() + assert "recording a failed receipt" in result.stdout def test_nothing_crosses_jobs_but_the_request_and_guarded_output(): @@ -758,10 +914,7 @@ def test_nothing_crosses_jobs_but_the_request_and_guarded_output(): recheck = record[ record_names.index("Nothing household-level reaches the public branch") ] - assert ( - "python3 -I .github/scripts/uk_replay_output_guard.py output" in recheck["run"] - ) - assert "rm -rf output" in recheck["run"] + assert recheck["run"] == "bash .github/scripts/uk_replay_recheck.sh" assert record_names.index(recheck["name"]) < record_names.index( "Record the outcome and continue the queue" ) From be30312c8720a0504e1d42e82c8e85d6b045fcf9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 15 Oct 2026 07:55:02 -0400 Subject: [PATCH 7/7] Receiver: run candidate code in a systemd sandbox (review of c6a95cd) The hosted image leaves /opt world-writable, and /opt/pipx_bin and the setup-uv tool directory are on the runner's PATH, so a separate account alone could plant a program there that a later step running as the runner user (collect, the guard, checkout's post step) would execute, regaining the runner's sudo and with it the token. The candidate now runs as a transient systemd service (systemd-run --uid) with ProtectSystem=strict (read-only except its home), InaccessiblePaths=/home/runner, PrivateTmp, PrivateDevices, NoNewPrivileges, RestrictSUIDSGID and ProtectProc=invisible. setup checks from inside that sandbox that sudo fails, that the account has no group beyond its own, and that nothing is writable among every directory on the runner's PATH and its parents, the tool cache, /opt, /usr, /etc, the workspace, the command files and the docker socket. CI's receiver-sandbox job runs these checks on the real image. Also: the upload needs collect to succeed as well as the guard; the guard's docstring and the docs say exactly which tables it restricts by name. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/uk_replay_output_guard.py | 9 +- .github/scripts/uk_replay_sandbox.sh | 110 ++++++++++++++++------ .github/workflows/uk-replay.yml | 13 +-- docs/uk_replay/RELEASE_REPLAY.md | 52 +++++----- tests/test_uk_replay_queue.py | 51 +++++++--- 5 files changed, 160 insertions(+), 75 deletions(-) diff --git a/.github/scripts/uk_replay_output_guard.py b/.github/scripts/uk_replay_output_guard.py index 888073d..57ef5a3 100644 --- a/.github/scripts/uk_replay_output_guard.py +++ b/.github/scripts/uk_replay_output_guard.py @@ -9,10 +9,11 @@ COMPARISON.csv and STAGED.jsonl. The committed sealed namespace has 30 MB over 316 files, the largest 3.3 MB. -It stops an honest run from publishing a dataset copy, a binary, or a table -under any other name. It does not read what a parsable file says: a -household-level table written as COMPARISON.csv, or anything a deliberately -malicious candidate encodes, would pass; RELEASE_REPLAY.md states that limit. +It stops an honest run from publishing a dataset copy, a binary, or a .csv +or .jsonl table under any other name. It does not read what a parsable file +says: a household-level table written as COMPARISON.csv, or as JSON or +Markdown, or anything a deliberately malicious candidate encodes, would pass; +RELEASE_REPLAY.md states that limit. Standard library only: it runs with the runner's system Python, before and independently of anything the candidate installed. diff --git a/.github/scripts/uk_replay_sandbox.sh b/.github/scripts/uk_replay_sandbox.sh index 1f38e58..5510ef1 100644 --- a/.github/scripts/uk_replay_sandbox.sh +++ b/.github/scripts/uk_replay_sandbox.sh @@ -1,26 +1,61 @@ #!/usr/bin/env bash -# Candidate code runs as an unprivileged account, never as the runner user. +# Candidate code runs as an unprivileged account inside a systemd sandbox, +# never as the runner user. # # The runner user has passwordless sudo, owns this job's workspace and its # command files ($GITHUB_ENV, $GITHUB_PATH), and owns the runner processes -# that hold the job's secrets and the Actions runtime token. The candidate -# account has no sudoers entry, no supplementary groups (so no docker -# socket), no cron or at jobs, a private copy of the repository and the -# verified request inputs, and an environment built from nothing (env -i). -# It cannot write the runner user's files or read another user's process -# memory or environment; `setup` checks the first of these as the account -# itself and fails the job otherwise. Its processes are killed in `collect`, -# and only a copy of its output/, streamed out by the account itself, comes -# back, to be checked by the runner's own guard. +# that hold the job's secrets and the Actions runtime token. Later steps run +# as that user and search its PATH, which on the hosted image includes +# world-writable directories under /opt. So a separate account is not +# enough: the candidate runs as a transient systemd service +# (systemd-run --uid) in which the whole filesystem is read-only except the +# account's home (ProtectSystem=strict), the runner's home is inaccessible, +# /tmp is private, setuid programs such as sudo cannot gain privileges, and +# other users' processes are invisible. The account itself has no sudoers +# entry, no supplementary group (so no docker socket) and no cron or at +# jobs. `setup` checks all of this from inside the same sandbox and fails +# the job otherwise. When the service's main process ends, systemd kills +# everything left in it; `collect` then makes sure no process of the account +# survives, and only a copy of its output/, streamed out by the account, +# comes back to be checked by the runner's own guard. # # uk_replay_sandbox.sh setup # after the dataset fetch, before any candidate code -# uk_replay_sandbox.sh run # the drain, as the candidate account -# uk_replay_sandbox.sh collect # kill its processes, copy output/ back +# uk_replay_sandbox.sh run # the drain, in the sandbox +# uk_replay_sandbox.sh collect # make sure nothing survives, copy output/ back set -euo pipefail CANDIDATE=replay-candidate CANDIDATE_HOME=/home/$CANDIDATE COPY=$CANDIDATE_HOME/scorecard +# Run a command as the candidate in the sandbox, with an environment built +# from nothing but the arguments given before the command (VAR=value ...). +in_sandbox() { + local settings=() + while [ "$#" -gt 0 ] && [[ "$1" == *=* ]]; do + settings+=(--setenv "$1") + shift + done + sudo systemd-run --quiet --collect --pipe --wait \ + --uid="$CANDIDATE" --gid="$CANDIDATE" \ + --working-directory="$CANDIDATE_HOME" \ + --setenv HOME="$CANDIDATE_HOME" --setenv USER="$CANDIDATE" \ + --setenv LANG=C.UTF-8 \ + --setenv PATH="$CANDIDATE_HOME/bin:/usr/local/bin:/usr/bin:/bin" \ + "${settings[@]}" \ + -p ProtectSystem=strict \ + -p ReadWritePaths="$CANDIDATE_HOME" \ + -p InaccessiblePaths=-/home/runner \ + -p PrivateTmp=yes \ + -p PrivateDevices=yes \ + -p NoNewPrivileges=yes \ + -p RestrictSUIDSGID=yes \ + -p ProtectProc=invisible \ + -p ProtectKernelTunables=yes \ + -p ProtectControlGroups=yes \ + "$@" +} + +# The account, outside the sandbox, for streaming its output out. as_candidate() { sudo -n -u "$CANDIDATE" env -i HOME="$CANDIDATE_HOME" PATH=/usr/bin:/bin "$@" } @@ -41,38 +76,52 @@ case "${1:?usage: uk_replay_sandbox.sh setup|run|collect}" in sudo cp -a "$GITHUB_WORKSPACE/." "$COPY" sudo rm -rf "$COPY/output" sudo chown -R "$CANDIDATE:$CANDIDATE" "$COPY" - # Check the account as itself: no sudo, no group beyond its own, and no - # write access to the workspace, the runner's command files or docker. - if as_candidate sudo -n true 2> /dev/null; then - echo "::error::$CANDIDATE can use sudo" + # Check, from inside the sandbox: no sudo, no group beyond its own, and + # nothing writable that a later step as the runner user could read or + # execute: every directory on the runner's PATH and its parents, the + # tool cache, /opt, /usr, /etc, the workspace and command files, docker. + if in_sandbox sudo -n true > /dev/null 2>&1; then + echo "::error::$CANDIDATE can use sudo in the sandbox" exit 1 fi - if [ "$(as_candidate id -G)" != "$(as_candidate id -g)" ]; then - echo "::error::$CANDIDATE has supplementary groups: $(as_candidate id -Gn)" + groups=$(in_sandbox id -G) + if [ "$groups" != "$(in_sandbox id -g)" ]; then + echo "::error::$CANDIDATE has supplementary groups: $groups" exit 1 fi - for target in "$GITHUB_WORKSPACE" "$(dirname "$GITHUB_ENV")" "$GITHUB_ENV" /var/run/docker.sock; do - if [ -e "$target" ] && as_candidate test -w "$target"; then - echo "::error::$CANDIDATE can write $target" - exit 1 - fi + targets=("$GITHUB_WORKSPACE" "$(dirname "$GITHUB_ENV")" "$GITHUB_ENV" + /var/run/docker.sock /opt /usr /usr/share /usr/local /etc /home/runner + "${RUNNER_TOOL_CACHE:-/opt/hostedtoolcache}") + IFS=: read -r -a path_entries <<< "$PATH" + for entry in "${path_entries[@]}"; do + while [ -n "$entry" ] && [ "$entry" != / ]; do + targets+=("$entry") + entry=$(dirname "$entry") + done done - echo "$CANDIDATE: no sudo, no supplementary groups, no write access to the runner's files" + # shellcheck disable=SC2016 # expanded by the sandboxed shell + writable=$(in_sandbox /bin/sh -c \ + 'for t in "$@"; do if [ -e "$t" ] && [ -w "$t" ]; then echo "$t"; fi; done' \ + sh "${targets[@]}") + if [ -n "$writable" ]; then + echo "::error::$CANDIDATE can write, in the sandbox: $(echo "$writable" | sort -u | tr '\n' ' ')" + exit 1 + fi + echo "$CANDIDATE: no sudo, no supplementary groups, nothing writable outside its home (${#targets[@]} paths checked)" ;; run) status=0 - sudo -n -u "$CANDIDATE" env -i \ - HOME="$CANDIDATE_HOME" USER="$CANDIDATE" LANG=C.UTF-8 \ - PATH="$CANDIDATE_HOME/bin:/usr/local/bin:/usr/bin:/bin" \ - PYTHONPATH=. RUNNER_TEMP="$CANDIDATE_HOME/tmp" \ + # shellcheck disable=SC2016 # expanded by the sandboxed shell + in_sandbox PYTHONPATH=. RUNNER_TEMP="$CANDIDATE_HOME/tmp" \ REQUEST_ID="${REQUEST_ID:?}" REPLAY_RUNNER="${REPLAY_RUNNER:?}" \ - bash -c 'mkdir -p "$RUNNER_TEMP" && cd "$HOME/scorecard" && exec bash .github/scripts/uk_replay_drain.sh' \ + /bin/bash -c 'mkdir -p "$RUNNER_TEMP" && cd "$HOME/scorecard" && exec bash .github/scripts/uk_replay_drain.sh' \ || status=$? # The drain records its own exit; the recorder handles a missing outcome. echo "candidate drain exited $status" ;; collect) - # Nothing the candidate started outlives this step. + # systemd killed what was left in the service; make sure nothing of the + # account runs anywhere. sudo pkill -KILL -u "$CANDIDATE" || true for _ in $(seq 30); do pgrep -u "$CANDIDATE" > /dev/null || break @@ -89,7 +138,6 @@ case "${1:?usage: uk_replay_sandbox.sh setup|run|collect}" in if as_candidate test -d "$COPY/output"; then mkdir output as_candidate tar -C "$COPY/output" -cf - . | tar -C output --no-same-owner --no-same-permissions -xf - - # Kill anything the stream itself might have started. sudo pkill -KILL -u "$CANDIDATE" || true fi ;; diff --git a/.github/workflows/uk-replay.yml b/.github/workflows/uk-replay.yml index 908e4a0..f250e2d 100644 --- a/.github/workflows/uk-replay.yml +++ b/.github/workflows/uk-replay.yml @@ -12,11 +12,11 @@ name: UK replay qualification # only dispatches a drain. In the run job it is in one step's environment, # which fetches and verifies the dataset before any candidate code is # installed. That job has contents: read and no checkout credentials, and -# candidate code runs as an unprivileged account with no sudo, no groups, no -# write access to the workspace or the runner's command files, no access to -# the processes holding the job's secrets, and an environment built from -# nothing (.github/scripts/uk_replay_sandbox.sh; CI's receiver-sandbox job -# checks it on the runner image). Scorecard is public: the dataset and +# candidate code runs as an unprivileged account inside a systemd sandbox +# (.github/scripts/uk_replay_sandbox.sh): read-only filesystem except its own +# home, no sudo, no groups, the runner's home inaccessible, other processes +# invisible, and an environment built from nothing. CI's receiver-sandbox job +# checks it on the runner image. Scorecard is public: the dataset and # anything household-level never leave the runner. setup-uv's cache is off # everywhere here, so no job saves or restores a cache; the only artifacts are # the request and the run's output, which an allowlist guard checks before the @@ -159,6 +159,7 @@ jobs: REPLAY_RUNNER: github-actions ${{ runner.os }} ${{ runner.arch }} run: bash .github/scripts/uk_replay_sandbox.sh run - name: Stop every candidate process and take its output back + id: collect if: always() run: bash .github/scripts/uk_replay_sandbox.sh collect - name: Nothing household-level leaves the runner @@ -168,7 +169,7 @@ jobs: # A refused output is never uploaded; the recorder then finds no # artifact and records the attempt as failed. - name: Upload the receipt and results for the recorder - if: always() && steps.guard.outcome == 'success' + if: always() && steps.collect.outcome == 'success' && steps.guard.outcome == 'success' uses: actions/upload-artifact@v4 with: name: replay-output diff --git a/docs/uk_replay/RELEASE_REPLAY.md b/docs/uk_replay/RELEASE_REPLAY.md index 9b3482c..216f48c 100644 --- a/docs/uk_replay/RELEASE_REPLAY.md +++ b/docs/uk_replay/RELEASE_REPLAY.md @@ -411,27 +411,36 @@ passed as `HUGGING_FACE_TOKEN`, and only in `workflow_dispatch` and workflow has no pull-request trigger. A fetch rejection is passed to the run without installing candidate code. -Candidate code then runs as an unprivileged account -(`.github/scripts/uk_replay_sandbox.sh`), never as the runner user. The -runner user has passwordless sudo, owns the workspace and the runner's +Candidate code then runs as an unprivileged account inside a systemd +sandbox (`.github/scripts/uk_replay_sandbox.sh`), never as the runner user. +The runner user has passwordless sudo, owns the workspace and the runner's command files (`$GITHUB_ENV`, `$GITHUB_PATH`), and owns the runner processes that keep the job's secrets in memory for the whole job and pass the Actions -runtime token to later steps. The candidate account has no sudoers entry, no +runtime token to later steps. Later steps run as that user and search its +`PATH`, which on the hosted image includes world-writable directories under +`/opt`; a separate account alone could plant a program there for a later +step to run. So the candidate runs as a transient systemd service +(`systemd-run --uid`): the whole filesystem is read-only except the +account's home (`ProtectSystem=strict`), the runner's home is inaccessible, +`/tmp` is private, setuid programs such as sudo cannot gain privileges +(`NoNewPrivileges`), and other users' processes are invisible +(`ProtectProc=invisible`). The account has no sudoers entry, no supplementary group (so no docker socket) and no cron or at jobs. `setup` -checks, as the account itself, that it cannot use sudo, has no group beyond -its own, and cannot write the workspace, the runner's command files or the -docker socket, and fails the job otherwise. The account works in its own -copy of the repository and of `input/` (the request, the verified dataset -and the fetch outcome), starts from an empty environment (`env -i`), and -cannot read another user's process memory or environment. It may be able to -read the runner user's files if the runner image leaves them readable; none -holds a secret. It runs `.github/scripts/uk_replay_drain.sh` for that one -request. Afterwards every process it started is killed and the account -streams its `output/` out with `tar`, which the runner user unpacks as -itself; links stay links and special files stay special files, for the guard -to refuse. CI's `receiver sandbox` job runs `setup`, `run` and `collect` on -the runner image on every pull request, with a stub drain that leaves a -process behind, and no secrets. +checks from inside the sandbox that sudo fails, that the account has no +group beyond its own, and that nothing is writable among every directory on +the runner's `PATH` and its parents, the tool cache, `/opt`, `/usr`, `/etc`, +the workspace, the command files and the docker socket; otherwise the job +fails. The account works in its own copy of the repository and of `input/` +(the request, the verified dataset and the fetch outcome), with an +environment built from nothing. It runs `.github/scripts/uk_replay_drain.sh` +for that one request. When the service's main process ends, systemd kills +everything left in it; `collect` then checks that no process of the account +survives, and the account streams its `output/` out with `tar`, which the +runner user unpacks as itself; links stay links and special files stay +special files, for the guard to refuse. The upload needs both `collect` and +the guard to succeed. CI's `receiver sandbox` job runs `setup`, `run` and +`collect` on the runner image on every pull request, with a stub drain that +leaves a process behind, and no secrets. Scorecard is public, so the dataset and anything household-level stay on the runner. No job saves or restores a cache (setup-uv's default cache is off on @@ -456,9 +465,10 @@ require it); requests are meant for PolicyEngine's own releases, from W4. A new candidate's dependencies resolve within the newest certified freeze where they can. The candidate and its dependencies are trusted with the dataset: they compute on it, and the guard stops an honest run from -publishing a dataset copy, a binary or a table under any other name. The -guard does not read what a parsable file says: a household-level table -written as `COMPARISON.csv` would pass. Code in the candidate's dependency closure cannot reach the data +publishing a dataset copy, a binary, or a `.csv` or `.jsonl` table under +any other name. It does not read what a parsable file says: a +household-level table written as `COMPARISON.csv`, or as JSON or Markdown, +would pass. Code in the candidate's dependency closure cannot reach the data token, the runtime token, the write jobs' credentials or a cache, and cannot alter the guard or what the recorder checks. A deliberately malicious candidate could still encode data in aggregates that parse, and could forge a diff --git a/tests/test_uk_replay_queue.py b/tests/test_uk_replay_queue.py index 7ca1efe..e750781 100644 --- a/tests/test_uk_replay_queue.py +++ b/tests/test_uk_replay_queue.py @@ -247,6 +247,7 @@ def test_candidate_job_has_read_permission_and_no_dataset_token_or_credentials() assert by_name[order[2]]["run"] == "bash .github/scripts/uk_replay_sandbox.sh run" assert by_name[order[3]] == { "name": order[3], + "id": "collect", "if": "always()", "run": "bash .github/scripts/uk_replay_sandbox.sh collect", } @@ -806,7 +807,7 @@ def test_output_guard_caps_the_total(tmp_path): assert result.returncode == 1 and "in all" in result.stdout -def test_candidate_code_runs_as_an_unprivileged_account_with_nothing_inherited(): +def test_candidate_code_runs_in_a_systemd_sandbox_with_nothing_writable_outside_home(): script = (ROOT / ".github/scripts/uk_replay_sandbox.sh").read_text() code = "\n".join( line for line in script.splitlines() if not line.lstrip().startswith("#") @@ -815,17 +816,38 @@ def test_candidate_code_runs_as_an_unprivileged_account_with_nothing_inherited() assert 'sudo useradd --create-home --shell /bin/bash "$CANDIDATE"' in code assert "/etc/cron.deny /etc/at.deny" in code assert "usermod" not in code and " -G " not in code and "gpasswd" not in code - # setup checks the account as itself and fails the job otherwise. - assert "if as_candidate sudo -n true" in code - assert '"$(as_candidate id -G)" != "$(as_candidate id -g)"' in code - assert 'as_candidate test -w "$target"' in code - for target in ('"$GITHUB_WORKSPACE"', '"$GITHUB_ENV"', "/var/run/docker.sock"): - assert target in code - # Its environment is built from nothing: no token, no runtime token, no - # GITHUB_ENV or GITHUB_PATH, no uv cache directory from setup-uv. - assert 'sudo -n -u "$CANDIDATE" env -i' in code - assert "--preserve-env" not in code and "sudo -E" not in code - assert "exec bash .github/scripts/uk_replay_drain.sh" in code + # Review of c6a95cd: the hosted image leaves /opt world-writable and on + # the runner's PATH, so the candidate runs where only its home is + # writable, sudo cannot gain privileges and the runner's home is hidden. + for setting in ( + "sudo systemd-run --quiet --collect --pipe --wait", + '--uid="$CANDIDATE"', + "-p ProtectSystem=strict", + '-p ReadWritePaths="$CANDIDATE_HOME"', + "-p InaccessiblePaths=-/home/runner", + "-p PrivateTmp=yes", + "-p NoNewPrivileges=yes", + "-p RestrictSUIDSGID=yes", + "-p ProtectProc=invisible", + ): + assert setting in code, setting + run = code[code.index(" run)") : code.index(" collect)")] + assert "in_sandbox PYTHONPATH=. " in run + assert "exec bash .github/scripts/uk_replay_drain.sh" in run + assert "sudo -n -u" not in run and "--preserve-env" not in code + # setup checks from inside the sandbox, over the runner's whole PATH. + setup = code[code.index(" setup)") : code.index(" run)")] + assert "if in_sandbox sudo -n true" in setup + assert 'IFS=: read -r -a path_entries <<< "$PATH"' in setup + assert "writable=$(in_sandbox /bin/sh -c" in setup + for target in ( + "/opt", + "/usr", + "/etc", + '"$GITHUB_WORKSPACE"', + "/var/run/docker.sock", + ): + assert target in setup # Nothing it starts outlives the run, and its output comes back streamed # out by the account itself, links kept as links for the guard. assert 'sudo pkill -KILL -u "$CANDIDATE"' in code @@ -905,7 +927,10 @@ def test_nothing_crosses_jobs_but_the_request_and_guarded_output(): assert guard_step["run"] == ( "python3 -I .github/scripts/uk_replay_output_guard.py output" ) - assert upload["if"] == "always() && steps.guard.outcome == 'success'" + assert upload["if"] == ( + "always() && steps.collect.outcome == 'success' " + "&& steps.guard.outcome == 'success'" + ) assert guard_step["id"] == "guard" and guard_step["if"] == "always()" # The recorder checks again, where no candidate code has run, and drops # a refused output before anything reaches the public branch.