diff --git a/.github/scripts/cloud-run-simulation-entry-candidate.sh b/.github/scripts/cloud-run-simulation-entry-candidate.sh new file mode 100755 index 000000000..52cf435c4 --- /dev/null +++ b/.github/scripts/cloud-run-simulation-entry-candidate.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# Resolve the URL, revision and image digest of the no-traffic candidate +# revision that carries the given Cloud Run tag. +# +# Required env vars: ENTRYPOINT_SERVICE, TAG, PROJECT_ID, REGION, GITHUB_OUTPUT. +# Outputs: url, revision and image in GITHUB_OUTPUT. Fails when the tag has no +# URL or revision, or the revision's image is not pinned by digest. + +set -euo pipefail + +service_json="$(gcloud run services describe "${ENTRYPOINT_SERVICE}" \ + --project "${PROJECT_ID}" \ + --region "${REGION}" \ + --format=json)" +url="$(jq -er --arg tag "${TAG}" ' + .status.traffic[] + | select(.tag == $tag) + | .url + | select(type == "string" and length > 0) +' <<<"${service_json}")" +revision="$(jq -er --arg tag "${TAG}" ' + .status.traffic[] + | select(.tag == $tag) + | .revisionName + | select(type == "string" and length > 0) +' <<<"${service_json}")" +revision_json="$(gcloud run revisions describe "${revision}" \ + --project "${PROJECT_ID}" \ + --region "${REGION}" \ + --format=json)" +image="$(jq -er ' + (.status.imageDigest // .spec.containers[0].image) + | select( + type == "string" + and contains("@sha256:") + ) +' <<<"${revision_json}")" +{ + echo "url=${url}" + echo "revision=${revision}" + echo "image=${image}" +} >> "$GITHUB_OUTPUT" diff --git a/.github/scripts/cloud-run-simulation-entry-stable-state.sh b/.github/scripts/cloud-run-simulation-entry-stable-state.sh new file mode 100755 index 000000000..a225bdb69 --- /dev/null +++ b/.github/scripts/cloud-run-simulation-entry-stable-state.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Record the stable Cloud Run service URL and the one revision serving 100 +# percent of its traffic, before a candidate is deployed. +# +# Required env vars: ENTRYPOINT_SERVICE, PROJECT_ID, REGION, GITHUB_OUTPUT. +# Outputs: stable_url and revision in GITHUB_OUTPUT. Fails when the service has +# no URL or not exactly one revision at 100 percent. + +set -euo pipefail + +service_json="$(gcloud run services describe "${ENTRYPOINT_SERVICE}" \ + --project "${PROJECT_ID}" \ + --region "${REGION}" \ + --format=json)" +stable_url="$(jq -er '.status.url | select(type == "string" and length > 0)' <<<"${service_json}")" +revision="$(jq -er ' + [ + .status.traffic[]? + | select((.percent // 0) == 100) + | .revisionName + | select(type == "string" and length > 0) + ] + | if length == 1 then .[0] + else error("service must have exactly one revision at 100 percent") + end +' <<<"${service_json}")" +{ + echo "stable_url=${stable_url}" + echo "revision=${revision}" +} >> "$GITHUB_OUTPUT" diff --git a/.github/scripts/simulation-deployment-ready.sh b/.github/scripts/simulation-deployment-ready.sh new file mode 100755 index 000000000..31d1f30e8 --- /dev/null +++ b/.github/scripts/simulation-deployment-ready.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Require every deployment phase to have succeeded, and the promotion to have +# run exactly when it was requested, so a skipped phase cannot report a +# successful partial deployment. An existing-stack deployment also requires +# its routing publication and a journal finalization that completed it. +# Required env vars: PROMOTION_REQUESTED, EXISTING_STACK_REQUESTED, +# ENTRYPOINT_RESULT, ROUTING_RESULT, INTEGRATION_RESULT, +# AUTHENTICATED_TEST_RESULT, PROMOTION_RESULT, FINALIZE_RESULT (GitHub job +# results) and FINALIZE_OUTCOME (the finalize_routing outcome output). + +set -euo pipefail + +test "${ENTRYPOINT_RESULT}" = "success" +test "${INTEGRATION_RESULT}" = "success" +test "${AUTHENTICATED_TEST_RESULT}" = "success" +if [[ "${PROMOTION_REQUESTED}" = "true" ]]; then + test "${PROMOTION_RESULT}" = "success" +else + test "${PROMOTION_RESULT}" = "skipped" +fi +if [[ "${EXISTING_STACK_REQUESTED}" = "true" ]]; then + test "${ROUTING_RESULT}" = "success" + test "${FINALIZE_RESULT}" = "success" + test "${FINALIZE_OUTCOME}" = "completed" +fi diff --git a/.github/scripts/stage12-deployment-ready.sh b/.github/scripts/stage12-deployment-ready.sh new file mode 100755 index 000000000..68a7932eb --- /dev/null +++ b/.github/scripts/stage12-deployment-ready.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Require the Stage 12 v2 jobs to have run exactly when they were requested: +# all skipped when not requested, all successful when requested. +# Required env vars: REQUESTED, CONFIGURATION_RESULT, DEPLOYMENT_RESULT, +# PUBLICATION_RESULT (GitHub job results). + +set -euo pipefail + +if [[ "${REQUESTED}" != "true" ]]; then + test "${CONFIGURATION_RESULT}" = "skipped" + test "${DEPLOYMENT_RESULT}" = "skipped" + test "${PUBLICATION_RESULT}" = "skipped" + exit 0 +fi +test "${CONFIGURATION_RESULT}" = "success" +test "${DEPLOYMENT_RESULT}" = "success" +test "${PUBLICATION_RESULT}" = "success" diff --git a/.github/scripts/stage12-v2-application-name.sh b/.github/scripts/stage12-v2-application-name.sh new file mode 100755 index 000000000..ad637df6e --- /dev/null +++ b/.github/scripts/stage12-v2-application-name.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Select the versioned v2 Modal application name from the packaged +# PolicyEngine.py bundle. +# Run from projects/policyengine-simulation-executor. +# Outputs: name in GITHUB_OUTPUT. + +set -euo pipefail + +name="$(uv run python -c 'from policyengine_simulation_contract.stage12_manifest import v2_application_name; from policyengine_simulation_executor.stage12_bundle import load_stage12_bundle; print(v2_application_name(load_stage12_bundle().bundle.policyengine_version))')" +echo "name=${name}" >> "$GITHUB_OUTPUT" diff --git a/.github/scripts/stage12-validate-v2-workers.sh b/.github/scripts/stage12-validate-v2-workers.sh new file mode 100755 index 000000000..ad0ed6e7b --- /dev/null +++ b/.github/scripts/stage12-validate-v2-workers.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Validate the installed bundles, secrets, datasets and calculations of the +# deployed v2 workers, and keep the result for manifest publication. +# Usage: stage12-validate-v2-workers.sh +# Run from projects/policyengine-simulation-executor. +# Required env vars: RUNNER_TEMP, GITHUB_OUTPUT. +# Outputs: file (the validation result path) in GITHUB_OUTPUT. + +set -euo pipefail + +MODAL_ENV="${1:?Modal environment required}" + +output="${RUNNER_TEMP}/stage12-worker-validation.json" +uv run python -m src.modal.utils.validate_v2_worker_release \ + --environment "${MODAL_ENV}" \ + --output "${output}" +echo "file=${output}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/simulation-deploy.reusable.yml b/.github/workflows/simulation-deploy.reusable.yml index 33ab8e49a..11d47de71 100644 --- a/.github/workflows/simulation-deploy.reusable.yml +++ b/.github/workflows/simulation-deploy.reusable.yml @@ -137,11 +137,7 @@ jobs: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} - run: | - uv run modal secret create pe-uk-private-hf-read-token \ - "HUGGING_FACE_TOKEN=${PE_UK_PRIVATE_HF_READ_TOKEN}" \ - --env="${{ inputs.modal_environment }}" \ - --force + run: uv run modal secret create pe-uk-private-hf-read-token "HUGGING_FACE_TOKEN=${PE_UK_PRIVATE_HF_READ_TOKEN}" --env="${{ inputs.modal_environment }}" --force - name: Synchronize Modal runtime secrets if: ${{ inputs.deploy_existing_stack }} @@ -200,39 +196,19 @@ jobs: - name: Capture stable service state before deployment id: previous - run: | - service_json="$(gcloud run services describe "${{ inputs.entrypoint_service }}" \ - --project "${PROJECT_ID}" \ - --region "${REGION}" \ - --format=json)" - stable_url="$(jq -er '.status.url | select(type == "string" and length > 0)' <<<"${service_json}")" - revision="$(jq -er ' - [ - .status.traffic[]? - | select((.percent // 0) == 100) - | .revisionName - | select(type == "string" and length > 0) - ] - | if length == 1 then .[0] - else error("service must have exactly one revision at 100 percent") - end - ' <<<"${service_json}")" - { - echo "stable_url=${stable_url}" - echo "revision=${revision}" - } >> "$GITHUB_OUTPUT" + env: + ENTRYPOINT_SERVICE: ${{ inputs.entrypoint_service }} + run: .github/scripts/cloud-run-simulation-entry-stable-state.sh - name: Select immutable image id: image run: echo "uri=${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPOSITORY}/${IMAGE_NAME}:${GITHUB_SHA}" >> "$GITHUB_OUTPUT" - - name: Build and push entrypoint image - run: | - docker build \ - --file "${PROJECT_DIR}/Dockerfile" \ - --tag "${{ steps.image.outputs.uri }}" \ - . - docker push "${{ steps.image.outputs.uri }}" + - name: Build entrypoint image + run: docker build --file "${PROJECT_DIR}/Dockerfile" --tag "${{ steps.image.outputs.uri }}" . + + - name: Push entrypoint image + run: docker push "${{ steps.image.outputs.uri }}" - name: Select no-traffic revision tag id: metadata @@ -270,40 +246,9 @@ jobs: - name: Resolve exact candidate metadata id: candidate env: + ENTRYPOINT_SERVICE: ${{ inputs.entrypoint_service }} TAG: ${{ steps.metadata.outputs.tag }} - run: | - service_json="$(gcloud run services describe "${{ inputs.entrypoint_service }}" \ - --project "${PROJECT_ID}" \ - --region "${REGION}" \ - --format=json)" - url="$(jq -er --arg tag "${TAG}" ' - .status.traffic[] - | select(.tag == $tag) - | .url - | select(type == "string" and length > 0) - ' <<<"${service_json}")" - revision="$(jq -er --arg tag "${TAG}" ' - .status.traffic[] - | select(.tag == $tag) - | .revisionName - | select(type == "string" and length > 0) - ' <<<"${service_json}")" - revision_json="$(gcloud run revisions describe "${revision}" \ - --project "${PROJECT_ID}" \ - --region "${REGION}" \ - --format=json)" - image="$(jq -er ' - (.status.imageDigest // .spec.containers[0].image) - | select( - type == "string" - and contains("@sha256:") - ) - ' <<<"${revision_json}")" - { - echo "url=${url}" - echo "revision=${revision}" - echo "image=${image}" - } >> "$GITHUB_OUTPUT" + run: .github/scripts/cloud-run-simulation-entry-candidate.sh - name: Install entrypoint unit-test dependencies working-directory: ${{ env.PROJECT_DIR }} @@ -402,9 +347,7 @@ jobs: - name: Select versioned executor app id: app - run: | - version_safe="${POLICYENGINE_VERSION//./-}" - echo "name=policyengine-simulation-py${version_safe}" >> "$GITHUB_OUTPUT" + run: echo "name=policyengine-simulation-py${POLICYENGINE_VERSION//./-}" >> "$GITHUB_OUTPUT" - name: Deploy versioned Modal executor working-directory: projects/policyengine-simulation-executor @@ -647,19 +590,12 @@ jobs: - name: Validate the packaged PolicyEngine.py bundle working-directory: projects/policyengine-simulation-executor - run: | - uv run python -m src.modal.utils.extract_stage12_bundle \ - --expect "policyengine_version=${{ needs.prepare.outputs.policyengine_version }}" \ - --expect "countries.us.country_package_version=${{ needs.prepare.outputs.us_version }}" \ - --expect "countries.uk.country_package_version=${{ needs.prepare.outputs.uk_version }}" \ - --output stage12-bundle.json + run: uv run python -m src.modal.utils.extract_stage12_bundle --expect "policyengine_version=${{ needs.prepare.outputs.policyengine_version }}" --expect "countries.us.country_package_version=${{ needs.prepare.outputs.us_version }}" --expect "countries.uk.country_package_version=${{ needs.prepare.outputs.uk_version }}" --output stage12-bundle.json - name: Select the bundle-derived v2 application name id: application working-directory: projects/policyengine-simulation-executor - run: | - name="$(uv run python -c 'from policyengine_simulation_contract.stage12_manifest import v2_application_name; from policyengine_simulation_executor.stage12_bundle import load_stage12_bundle; print(v2_application_name(load_stage12_bundle().bundle.policyengine_version))')" - echo "name=${name}" >> "$GITHUB_OUTPUT" + run: ../../.github/scripts/stage12-v2-application-name.sh - name: Synchronize the separate Stage 12 runtime secret working-directory: projects/policyengine-simulation-executor @@ -698,12 +634,7 @@ jobs: env: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} - run: | - output="${RUNNER_TEMP}/stage12-worker-validation.json" - uv run python -m src.modal.utils.validate_v2_worker_release \ - --environment "${{ inputs.modal_environment }}" \ - --output "${output}" - echo "file=${output}" >> "$GITHUB_OUTPUT" + run: ../../.github/scripts/stage12-validate-v2-workers.sh "${{ inputs.modal_environment }}" - name: Retain worker validation for manifest publication uses: actions/upload-artifact@v7 @@ -742,11 +673,7 @@ jobs: env: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} - run: | - uv run python -m src.modal.utils.update_v2_version_manifest \ - --environment "${{ inputs.modal_environment }}" \ - --manifest-name simulation-api-v2-version-manifest \ - --validation-file "${RUNNER_TEMP}/stage12-worker-validation/stage12-worker-validation.json" + run: uv run python -m src.modal.utils.update_v2_version_manifest --environment "${{ inputs.modal_environment }}" --manifest-name simulation-api-v2-version-manifest --validation-file "${RUNNER_TEMP}/stage12-worker-validation/stage12-worker-validation.json" stage12_v2_ready: name: Require requested Stage 12 v2 deployment to succeed @@ -755,22 +682,15 @@ jobs: [configure_stage12_v2, deploy_stage12_v2, publish_stage12_v2_manifest] runs-on: ubuntu-latest steps: + - uses: actions/checkout@v6 + - name: Check Stage 12 deployment result env: REQUESTED: ${{ inputs.deploy_stage12_v2 }} CONFIGURATION_RESULT: ${{ needs.configure_stage12_v2.result }} DEPLOYMENT_RESULT: ${{ needs.deploy_stage12_v2.result }} PUBLICATION_RESULT: ${{ needs.publish_stage12_v2_manifest.result }} - run: | - if [[ "${REQUESTED}" != "true" ]]; then - test "${CONFIGURATION_RESULT}" = "skipped" - test "${DEPLOYMENT_RESULT}" = "skipped" - test "${PUBLICATION_RESULT}" = "skipped" - exit 0 - fi - test "${CONFIGURATION_RESULT}" = "success" - test "${DEPLOYMENT_RESULT}" = "success" - test "${PUBLICATION_RESULT}" = "success" + run: .github/scripts/stage12-deployment-ready.sh integration: name: Verify deployed three-service stack @@ -1088,6 +1008,8 @@ jobs: shell: bash environment: ${{ inputs.release_environment }} steps: + - uses: actions/checkout@v6 + - name: Check deployment result env: PROMOTION_REQUESTED: ${{ inputs.promote_entrypoint }} @@ -1099,20 +1021,7 @@ jobs: PROMOTION_RESULT: ${{ needs.promote_entrypoint.result }} FINALIZE_RESULT: ${{ needs.finalize_routing.result }} FINALIZE_OUTCOME: ${{ needs.finalize_routing.outputs.outcome }} - run: | - test "${ENTRYPOINT_RESULT}" = "success" - test "${INTEGRATION_RESULT}" = "success" - test "${AUTHENTICATED_TEST_RESULT}" = "success" - if [[ "${PROMOTION_REQUESTED}" = "true" ]]; then - test "${PROMOTION_RESULT}" = "success" - else - test "${PROMOTION_RESULT}" = "skipped" - fi - if [[ "${EXISTING_STACK_REQUESTED}" = "true" ]]; then - test "${ROUTING_RESULT}" = "success" - test "${FINALIZE_RESULT}" = "success" - test "${FINALIZE_OUTCOME}" = "completed" - fi + run: .github/scripts/simulation-deployment-ready.sh cleanup_entrypoint_revisions: name: Remove superseded Cloud Run revisions diff --git a/.github/workflows/simulation-deploy.yml b/.github/workflows/simulation-deploy.yml index f5691c9f1..ab3d6d6f4 100644 --- a/.github/workflows/simulation-deploy.yml +++ b/.github/workflows/simulation-deploy.yml @@ -82,9 +82,4 @@ jobs: - uses: actions/checkout@v6 - name: Report promotion status - run: | - .github/scripts/simulation-deployment-summary.sh \ - "${{ needs.beta.result }}" \ - "${{ needs.beta.outputs.simulation_api_url }}" \ - "${{ needs.prod.result }}" \ - "${{ needs.prod.outputs.stable_url }}" + run: .github/scripts/simulation-deployment-summary.sh "${{ needs.beta.result }}" "${{ needs.beta.outputs.simulation_api_url }}" "${{ needs.prod.result }}" "${{ needs.prod.outputs.stable_url }}" diff --git a/changelog_entry.yaml b/changelog_entry.yaml index ae00357e7..a362bd8df 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -4,6 +4,7 @@ - Made CLAUDE.md and the Copilot instructions thin pointers to AGENTS.md, required workflow logic longer than one line to live in tested and shellchecked scripts, and registered the tracking-issue rule - Added a rules registry (rules/registry.toml) with a standard-library tool that validates it, checks every rule is enforced where it claims, and renders docs/generated/rules.md and the Rules block in AGENTS.md changed: + - Moved the multi-line `run` blocks of the simulation deployment workflows into single-line steps and tested scripts without changing their behaviour or outputs, leaving only the release-sentinel and routing-rollback blocks in the one-line `run` rule's baseline - Moved multi-line `run` blocks in the PR, image smoke and client publishing workflows into single-line steps and tested scripts, tested the dependency-update scripts, fixed the shellcheck finding in modal-precompute.sh, and enforced the CI-script testing and shellcheck rules with empty baselines - Added CODEOWNERS for the rule machinery, pinned CI's ruff to 0.15.22 with an explicit rule selection, and documented that agents never bypass branch rules - Ran the Modal image smoke for Stage 12 persistence changes and added policyengine-fastapi to the PR test matrix diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 7a62b21b0..9882a1c96 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -63,7 +63,7 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf ### Ratcheted (existing violations still tolerated) -- `sim.ci.multiline-run-is-script`: 25 remaining in `rules/baselines/ci-multiline-run.txt` +- `sim.ci.multiline-run-is-script`: 13 remaining in `rules/baselines/ci-multiline-run.txt` ### Exceptions and review dates diff --git a/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py b/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py new file mode 100644 index 000000000..2a3c502b6 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py @@ -0,0 +1,542 @@ +"""Tests for the scripts the simulation deployment workflows run. + +Each test runs a script with fake gcloud or uv commands first on PATH (jq is +real) and asserts on the exact calls, $GITHUB_OUTPUT and exit status, so the +production steps keep behaving as the inline shell they replaced. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import pytest + +from workflow_script_support import ( + WORKFLOWS_DIR, + FakeCommands, + run_script, +) + +REUSABLE_DEPLOY_WORKFLOW = WORKFLOWS_DIR / "simulation-deploy.reusable.yml" + + +@pytest.fixture +def fakes(tmp_path: Path) -> FakeCommands: + return FakeCommands.create(tmp_path) + + +def _step(workflow: str, name: str) -> str: + """The text of one workflow step, from its ``- name:`` to the next step.""" + match = re.search( + rf"^(?P +)- name: {re.escape(name)}\n(?P.*?)(?=^(?P=indent)- |^ \S|\Z)", + workflow, + flags=re.MULTILINE | re.DOTALL, + ) + assert match is not None, name + return match.group("body") + + +def _output(path: Path) -> str: + return path.read_text(encoding="utf-8") if path.exists() else "" + + +def test_every_job_checks_out_the_repository_before_running_its_scripts(): + import yaml + + jobs = yaml.safe_load(REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8"))["jobs"] + for job_name, job in jobs.items(): + checked_out = False + for step in job.get("steps", []): + if str(step.get("uses", "")).startswith("actions/checkout@"): + checked_out = True + if ".github/scripts/" in step.get("run", ""): + assert checked_out, f"{job_name}: {step.get('name')}" + + +# --- deploy_entrypoint: Cloud Run state ----------------------------------- + +STABLE_STATE = "cloud-run-simulation-entry-stable-state.sh" +CANDIDATE = "cloud-run-simulation-entry-candidate.sh" +SERVICE = "policyengine-simulation-entry-staging" +PROJECT = "policyengine-api" +REGION = "us-central1" +STABLE_URL = "https://policyengine-simulation-entry-staging-abc-uc.a.run.app" +CANDIDATE_URL = "https://s-42---policyengine-simulation-entry-staging-abc-uc.a.run.app" +DIGEST_IMAGE = ( + "us-central1-docker.pkg.dev/policyengine-api/policyengine-simulation-entry/" + "policyengine-simulation-entry@sha256:" + "a" * 64 +) + + +FAKE_GCLOUD = """\ +case "$1 $2 $3" in + "run services describe") printf '%s' "$FAKE_SERVICE_JSON" ;; + "run revisions describe") printf '%s' "$FAKE_REVISION_JSON" ;; + *) exit 2 ;; +esac +exit "${FAKE_GCLOUD_EXIT:-0}" +""" + + +def _service(traffic: list[dict], url: str | None = STABLE_URL) -> str: + status: dict = {"traffic": traffic} + if url is not None: + status["url"] = url + return json.dumps({"status": status}) + + +def _run_cloud_run_script( + fakes: FakeCommands, + tmp_path: Path, + script: str, + service_json: str, + revision_json: str = "{}", + **env: str, +): + output = tmp_path / "github-output" + fakes.add("gcloud", FAKE_GCLOUD) + result = run_script( + script, + env=fakes.env( + **{ + "ENTRYPOINT_SERVICE": SERVICE, + "PROJECT_ID": PROJECT, + "REGION": REGION, + "TAG": "s-42", + "GITHUB_OUTPUT": str(output), + "FAKE_SERVICE_JSON": service_json, + "FAKE_REVISION_JSON": revision_json, + **env, + } + ), + ) + return result, _output(output) + + +def _describe_service() -> list[str]: + return [ + "gcloud", + "run", + "services", + "describe", + SERVICE, + "--project", + PROJECT, + "--region", + REGION, + "--format=json", + ] + + +def test_deploy_workflow_runs_the_cloud_run_state_scripts(): + workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + + previous = _step(workflow, "Capture stable service state before deployment") + assert "id: previous" in previous + assert "ENTRYPOINT_SERVICE: ${{ inputs.entrypoint_service }}" in previous + assert f"run: .github/scripts/{STABLE_STATE}\n" in previous + + candidate = _step(workflow, "Resolve exact candidate metadata") + assert "id: candidate" in candidate + assert "ENTRYPOINT_SERVICE: ${{ inputs.entrypoint_service }}" in candidate + assert "TAG: ${{ steps.metadata.outputs.tag }}" in candidate + assert f"run: .github/scripts/{CANDIDATE}\n" in candidate + + # The job still reads every output name the scripts write. + for output in ( + "image: ${{ steps.candidate.outputs.image }}", + "revision: ${{ steps.candidate.outputs.revision }}", + "candidate_url: ${{ steps.candidate.outputs.url }}", + "stable_url: ${{ steps.previous.outputs.stable_url }}", + "previous_revision: ${{ steps.previous.outputs.revision }}", + ): + assert output in workflow + + +def test_stable_state_records_the_url_and_the_fully_serving_revision(fakes, tmp_path): + service_json = _service( + [ + {"revisionName": "entry-00001-old", "percent": 100}, + {"revisionName": "entry-00002-new", "tag": "s-41"}, + ] + ) + + result, output = _run_cloud_run_script(fakes, tmp_path, STABLE_STATE, service_json) + + assert result.returncode == 0, result.stderr + assert output == f"stable_url={STABLE_URL}\nrevision=entry-00001-old\n" + assert fakes.calls() == [_describe_service()] + + +@pytest.mark.parametrize( + "traffic", + [ + [ + {"revisionName": "entry-00001-one", "percent": 50}, + {"revisionName": "entry-00002-two", "percent": 50}, + ], + [ + {"revisionName": "entry-00001-one", "percent": 100}, + {"revisionName": "entry-00002-two", "percent": 100}, + ], + [], + ], + ids=["split", "two-at-100", "no-traffic"], +) +def test_stable_state_requires_exactly_one_revision_at_100_percent( + fakes, tmp_path, traffic +): + result, output = _run_cloud_run_script( + fakes, tmp_path, STABLE_STATE, _service(traffic) + ) + + assert result.returncode != 0 + assert "service must have exactly one revision at 100 percent" in result.stderr + assert output == "" + + +def test_stable_state_requires_a_service_url(fakes, tmp_path): + service_json = _service( + [{"revisionName": "entry-00001-old", "percent": 100}], url=None + ) + + result, output = _run_cloud_run_script(fakes, tmp_path, STABLE_STATE, service_json) + + assert result.returncode != 0 + assert output == "" + + +def test_stable_state_fails_when_gcloud_fails(fakes, tmp_path): + result, output = _run_cloud_run_script( + fakes, tmp_path, STABLE_STATE, "{}", FAKE_GCLOUD_EXIT="1" + ) + + assert result.returncode != 0 + assert output == "" + + +CANDIDATE_TRAFFIC = [ + {"revisionName": "entry-00001-old", "percent": 100}, + {"revisionName": "entry-00002-new", "tag": "s-42", "url": CANDIDATE_URL}, +] + + +@pytest.mark.parametrize( + "revision_json", + [ + {"status": {"imageDigest": DIGEST_IMAGE}}, + {"spec": {"containers": [{"image": DIGEST_IMAGE}]}}, + ], + ids=["status-digest", "spec-image"], +) +def test_candidate_resolves_the_tagged_revision_and_its_digest( + fakes, tmp_path, revision_json +): + result, output = _run_cloud_run_script( + fakes, + tmp_path, + CANDIDATE, + _service(CANDIDATE_TRAFFIC), + json.dumps(revision_json), + ) + + assert result.returncode == 0, result.stderr + assert output == ( + f"url={CANDIDATE_URL}\nrevision=entry-00002-new\nimage={DIGEST_IMAGE}\n" + ) + assert fakes.calls() == [ + _describe_service(), + [ + "gcloud", + "run", + "revisions", + "describe", + "entry-00002-new", + "--project", + PROJECT, + "--region", + REGION, + "--format=json", + ], + ] + + +def test_candidate_rejects_an_image_not_pinned_by_digest(fakes, tmp_path): + revision_json = { + "spec": {"containers": [{"image": "example/policyengine-simulation-entry:a"}]} + } + + result, output = _run_cloud_run_script( + fakes, + tmp_path, + CANDIDATE, + _service(CANDIDATE_TRAFFIC), + json.dumps(revision_json), + ) + + assert result.returncode != 0 + assert output == "" + + +def test_candidate_fails_when_the_tag_is_not_deployed(fakes, tmp_path): + result, output = _run_cloud_run_script( + fakes, tmp_path, CANDIDATE, _service(CANDIDATE_TRAFFIC), TAG="s-43" + ) + + assert result.returncode != 0 + assert output == "" + assert fakes.calls() == [_describe_service()] + + +# --- deploy_stage12_v2 and stage12_v2_ready ------------------------------- + +V2_APPLICATION_NAME = "stage12-v2-application-name.sh" +VALIDATE_V2_WORKERS = "stage12-validate-v2-workers.sh" +STAGE12_READY = "stage12-deployment-ready.sh" +V2_APPLICATION_CODE = ( + "from policyengine_simulation_contract.stage12_manifest import " + "v2_application_name; from policyengine_simulation_executor.stage12_bundle " + "import load_stage12_bundle; print(v2_application_name(" + "load_stage12_bundle().bundle.policyengine_version))" +) + + +def test_deploy_workflow_runs_the_stage12_worker_scripts(): + workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + + application = _step(workflow, "Select the bundle-derived v2 application name") + assert "id: application" in application + assert "working-directory: projects/policyengine-simulation-executor" in ( + application + ) + assert f"run: ../../.github/scripts/{V2_APPLICATION_NAME}\n" in application + + validation = _step( + workflow, "Validate installed bundles, secrets, datasets, and calculations" + ) + assert "id: validation" in validation + assert "working-directory: projects/policyengine-simulation-executor" in ( + validation + ) + assert ( + f'run: ../../.github/scripts/{VALIDATE_V2_WORKERS} "${{{{ inputs.modal_environment }}}}"' + in validation + ) + + assert "application_name: ${{ steps.application.outputs.name }}" in workflow + assert "validation_file: ${{ steps.validation.outputs.file }}" in workflow + assert "path: ${{ steps.validation.outputs.file }}" in workflow + + +def test_v2_application_name_comes_from_the_packaged_bundle(fakes, tmp_path): + output = tmp_path / "github-output" + fakes.add("uv", "echo policyengine-simulation-v2-py4-1-0\n") + + result = run_script(V2_APPLICATION_NAME, env=fakes.env(GITHUB_OUTPUT=str(output))) + + assert result.returncode == 0, result.stderr + assert _output(output) == "name=policyengine-simulation-v2-py4-1-0\n" + assert fakes.calls() == [["uv", "run", "python", "-c", V2_APPLICATION_CODE]] + + +def test_v2_application_name_fails_without_writing_when_the_bundle_fails( + fakes, tmp_path +): + output = tmp_path / "github-output" + fakes.add("uv", "exit 1\n") + + result = run_script(V2_APPLICATION_NAME, env=fakes.env(GITHUB_OUTPUT=str(output))) + + assert result.returncode == 1 + assert _output(output) == "" + + +def _validate_v2_workers(fakes: FakeCommands, tmp_path: Path, *args: str): + output = tmp_path / "github-output" + runner_temp = tmp_path / "runner-temp" + runner_temp.mkdir() + result = run_script( + VALIDATE_V2_WORKERS, + *args, + env=fakes.env(GITHUB_OUTPUT=str(output), RUNNER_TEMP=str(runner_temp)), + ) + return result, _output(output), runner_temp / "stage12-worker-validation.json" + + +def test_validate_v2_workers_records_the_validation_file(fakes, tmp_path): + fakes.add("uv") + + result, output, validation_file = _validate_v2_workers(fakes, tmp_path, "main") + + assert result.returncode == 0, result.stderr + assert output == f"file={validation_file}\n" + assert fakes.calls() == [ + [ + "uv", + "run", + "python", + "-m", + "src.modal.utils.validate_v2_worker_release", + "--environment", + "main", + "--output", + str(validation_file), + ] + ] + + +def test_validate_v2_workers_fails_without_writing_when_validation_fails( + fakes, tmp_path +): + fakes.add("uv", "exit 1\n") + + result, output, _ = _validate_v2_workers(fakes, tmp_path, "main") + + assert result.returncode == 1 + assert output == "" + + +def test_validate_v2_workers_requires_the_modal_environment(fakes, tmp_path): + fakes.add("uv") + + result, output, _ = _validate_v2_workers(fakes, tmp_path) + + assert result.returncode != 0 + assert "Modal environment required" in result.stderr + assert output == "" + assert fakes.calls() == [] + + +def test_deploy_workflow_checks_stage12_results_with_the_tested_script(): + workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + step = _step(workflow, "Check Stage 12 deployment result") + + for declaration in ( + "REQUESTED: ${{ inputs.deploy_stage12_v2 }}", + "CONFIGURATION_RESULT: ${{ needs.configure_stage12_v2.result }}", + "DEPLOYMENT_RESULT: ${{ needs.deploy_stage12_v2.result }}", + "PUBLICATION_RESULT: ${{ needs.publish_stage12_v2_manifest.result }}", + f"run: .github/scripts/{STAGE12_READY}\n", + ): + assert declaration in step + + +@pytest.mark.parametrize( + ("requested", "configuration", "deployment", "publication", "passes"), + [ + ("true", "success", "success", "success", True), + ("true", "success", "success", "failure", False), + ("true", "success", "skipped", "skipped", False), + ("true", "skipped", "skipped", "skipped", False), + ("false", "skipped", "skipped", "skipped", True), + ("false", "success", "skipped", "skipped", False), + ("false", "skipped", "skipped", "failure", False), + ], +) +def test_stage12_ready_requires_all_or_none_of_the_v2_jobs( + fakes, requested, configuration, deployment, publication, passes +): + result = run_script( + STAGE12_READY, + env=fakes.env( + REQUESTED=requested, + CONFIGURATION_RESULT=configuration, + DEPLOYMENT_RESULT=deployment, + PUBLICATION_RESULT=publication, + ), + ) + + assert result.returncode == (0 if passes else 1) + + +# --- deployment_ready ------------------------------------------------------- + +DEPLOYMENT_READY = "simulation-deployment-ready.sh" + + +def test_deploy_workflow_checks_deployment_results_with_the_tested_script(): + workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + step = _step(workflow, "Check deployment result") + + for declaration in ( + "PROMOTION_REQUESTED: ${{ inputs.promote_entrypoint }}", + "ENTRYPOINT_RESULT: ${{ needs.deploy_entrypoint.result }}", + "INTEGRATION_RESULT: ${{ needs.integration.result }}", + "AUTHENTICATED_TEST_RESULT: ${{ needs.authenticated_test.result }}", + "PROMOTION_RESULT: ${{ needs.promote_entrypoint.result }}", + "EXISTING_STACK_REQUESTED: ${{ inputs.deploy_existing_stack }}", + "ROUTING_RESULT: ${{ needs.update_routing.result }}", + "FINALIZE_RESULT: ${{ needs.finalize_routing.result }}", + "FINALIZE_OUTCOME: ${{ needs.finalize_routing.outputs.outcome }}", + f"run: .github/scripts/{DEPLOYMENT_READY}\n", + ): + assert declaration in step + + +@pytest.mark.parametrize( + ("promotion_requested", "results", "passes"), + [ + ("true", ("success", "success", "success", "success"), True), + ("false", ("success", "success", "success", "skipped"), True), + ("true", ("success", "success", "success", "skipped"), False), + ("false", ("success", "success", "success", "success"), False), + ("true", ("success", "success", "skipped", "success"), False), + ("false", ("success", "skipped", "success", "skipped"), False), + ("false", ("failure", "success", "success", "skipped"), False), + ], +) +def test_deployment_ready_rejects_unexpected_skipped_phases( + fakes, promotion_requested, results, passes +): + entrypoint, integration, authenticated_test, promotion = results + + result = run_script( + DEPLOYMENT_READY, + env=fakes.env( + PROMOTION_REQUESTED=promotion_requested, + ENTRYPOINT_RESULT=entrypoint, + INTEGRATION_RESULT=integration, + AUTHENTICATED_TEST_RESULT=authenticated_test, + PROMOTION_RESULT=promotion, + EXISTING_STACK_REQUESTED="false", + ROUTING_RESULT="skipped", + FINALIZE_RESULT="skipped", + FINALIZE_OUTCOME="", + ), + ) + + assert result.returncode == (0 if passes else 1) + + +@pytest.mark.parametrize( + ("routing", "finalize", "outcome", "passes"), + [ + ("success", "success", "completed", True), + ("success", "success", "restored", False), + ("success", "failure", "completed", False), + ("success", "skipped", "", False), + ("skipped", "success", "completed", False), + ("failure", "success", "completed", False), + ], +) +def test_deployment_ready_requires_a_completed_routing_publication( + fakes, routing, finalize, outcome, passes +): + result = run_script( + DEPLOYMENT_READY, + env=fakes.env( + PROMOTION_REQUESTED="true", + ENTRYPOINT_RESULT="success", + INTEGRATION_RESULT="success", + AUTHENTICATED_TEST_RESULT="success", + PROMOTION_RESULT="success", + EXISTING_STACK_REQUESTED="true", + ROUTING_RESULT=routing, + FINALIZE_RESULT=finalize, + FINALIZE_OUTCOME=outcome, + ), + ) + + assert result.returncode == (0 if passes else 1) diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py index 639b3f95f..e2351c18e 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py @@ -27,6 +27,9 @@ CLOUD_RUN_DEPLOY_SCRIPT = ( REPOSITORY_ROOT / ".github" / "scripts" / "deploy-cloud-run-simulation-entry.sh" ) +DEPLOYMENT_READY_SCRIPT = ( + REPOSITORY_ROOT / ".github" / "scripts" / "simulation-deployment-ready.sh" +) DEPLOY_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "simulation-deploy.yml" REUSABLE_DEPLOY_WORKFLOW = ( REPOSITORY_ROOT / ".github" / "workflows" / "simulation-deploy.reusable.yml" @@ -674,6 +677,15 @@ def test_reusable_deployment_rejects_unexpected_skipped_phases(): "promote_entrypoint", ): assert dependency in completion_body + assert "run: .github/scripts/simulation-deployment-ready.sh" in completion_body + for result in ( + "ENTRYPOINT_RESULT: ${{ needs.deploy_entrypoint.result }}", + "INTEGRATION_RESULT: ${{ needs.integration.result }}", + "AUTHENTICATED_TEST_RESULT: ${{ needs.authenticated_test.result }}", + "PROMOTION_RESULT: ${{ needs.promote_entrypoint.result }}", + ): + assert result in completion_body + readiness_check = DEPLOYMENT_READY_SCRIPT.read_text(encoding="utf-8") for expected_result in ( 'test "${ENTRYPOINT_RESULT}" = "success"', 'test "${INTEGRATION_RESULT}" = "success"', @@ -681,7 +693,7 @@ def test_reusable_deployment_rejects_unexpected_skipped_phases(): 'test "${PROMOTION_RESULT}" = "success"', 'test "${PROMOTION_RESULT}" = "skipped"', ): - assert expected_result in completion_body + assert expected_result in readiness_check def test_complete_integration_suite_is_configured_for_beta_only(): @@ -1467,6 +1479,7 @@ def test_failed_release_gate_cannot_report_a_successful_release(tmp_path, failed capture_output=True, text=True, env=env, + cwd=REPOSITORY_ROOT, ) assert result.returncode != 0 assert not marker.exists() @@ -1500,6 +1513,7 @@ def test_release_success_requires_completed_finalization_for_published_routing( check=False, capture_output=True, text=True, + cwd=REPOSITORY_ROOT, env={ "PATH": os.environ["PATH"], "PROMOTION_REQUESTED": "false", diff --git a/rules/baselines/ci-multiline-run.txt b/rules/baselines/ci-multiline-run.txt index dae7b3755..24dbf8f72 100644 --- a/rules/baselines/ci-multiline-run.txt +++ b/rules/baselines/ci-multiline-run.txt @@ -1,23 +1,11 @@ # Tolerated existing violations. This list may only shrink. -.github/workflows/simulation-deploy.reusable.yml :: deploy_entrypoint :: Build and push entrypoint image -.github/workflows/simulation-deploy.reusable.yml :: deploy_entrypoint :: Capture stable service state before deployment -.github/workflows/simulation-deploy.reusable.yml :: deploy_entrypoint :: Resolve exact candidate metadata -.github/workflows/simulation-deploy.reusable.yml :: deploy_executor :: Select versioned executor app -.github/workflows/simulation-deploy.reusable.yml :: deploy_stage12_v2 :: Select the bundle-derived v2 application name -.github/workflows/simulation-deploy.reusable.yml :: deploy_stage12_v2 :: Validate installed bundles, secrets, datasets, and calculations -.github/workflows/simulation-deploy.reusable.yml :: deploy_stage12_v2 :: Validate the packaged PolicyEngine.py bundle -.github/workflows/simulation-deploy.reusable.yml :: deployment_ready :: Check deployment result .github/workflows/simulation-deploy.reusable.yml :: finalize_routing :: Finalize the routing publication .github/workflows/simulation-deploy.reusable.yml :: integration :: Check the deployed stack reproduces the sentinel numbers -.github/workflows/simulation-deploy.reusable.yml :: prepare :: Synchronize the Modal UK private-data credential .github/workflows/simulation-deploy.reusable.yml :: promote_entrypoint :: Verify the stable service reproduces the sentinel numbers -.github/workflows/simulation-deploy.reusable.yml :: publish_stage12_v2_manifest :: Publish only the separate v2 manifest .github/workflows/simulation-deploy.reusable.yml :: sentinel_expected :: Check the sentinel names this deployment's bundle -.github/workflows/simulation-deploy.reusable.yml :: stage12_v2_ready :: Check Stage 12 deployment result .github/workflows/simulation-deploy.reusable.yml :: update_routing :: Plan the routing change .github/workflows/simulation-deploy.reusable.yml :: update_routing :: Publish the planned routing .github/workflows/simulation-deploy.reusable.yml :: update_routing :: Record the unfinished routing publication -.github/workflows/simulation-deploy.yml :: summary :: Report promotion status .github/workflows/simulation-rollback.yml :: rollback :: Read the restored release identity .github/workflows/simulation-rollback.yml :: rollback :: Resolve the stable service .github/workflows/simulation-rollback.yml :: rollback :: Resolve the unfinished routing publication diff --git a/rules/registry.toml b/rules/registry.toml index 72b76b281..3cf4b55eb 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -760,7 +760,7 @@ job = "quality-guards" invocation = "run_quality_guards.py" [rule.baseline] file = "rules/baselines/ci-multiline-run.txt" -count = 25 +count = 13 [rule.proof] bad_fixture = "scripts/rules/tests/fixtures/ci_scripts/multiline_bad" good_fixture = "scripts/rules/tests/fixtures/ci_scripts/multiline_good"