Move the deploy pipeline's multi-line steps into tested scripts

.github/scripts/build_changelog.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# Build the changelog for the version in pyproject.toml with towncrier.
+# Run from the repository root, after .github/bump_version.py and uv lock.
+
+set -euo pipefail
+
+version="$(
+  python - <<'PY'
+import re
+
+match = re.search(r'version = "(.+?)"', open("pyproject.toml").read())
+if match is None:
+    raise SystemExit("no version found in pyproject.toml")
+print(match.group(1))
+PY
+)"
+
+towncrier build --yes --version "${version}"

.github/scripts/compute_cloud_run_metadata.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Write the image tag and revision tag for a Cloud Run deploy to GITHUB_OUTPUT.
+#
+# Usage: compute_cloud_run_metadata.sh <revision-tag-prefix>
+# Reads GITHUB_SHA and GITHUB_RUN_NUMBER, which GitHub Actions sets for every
+# job. Keep the prefix short: Cloud Run caps tag + service name at 46 combined
+# chars.
+
+set -euo pipefail
+
+prefix="${1:?revision tag prefix is required}"
+sha="${GITHUB_SHA:?GITHUB_SHA is required}"
+run_number="${GITHUB_RUN_NUMBER:?GITHUB_RUN_NUMBER is required}"
+output="${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}"
+
+{
+  echo "image_tag=${sha}"
+  echo "revision_tag=${prefix}-${run_number}-${sha::7}"
+} >> "${output}"

.github/scripts/restore_cloud_run_revision.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Roll Cloud Run traffic back to the previous revision, then confirm it is
+# healthy.
+#
+# Usage: restore_cloud_run_revision.sh <readiness-url>
+# set_cloud_run_revision.sh reads CLOUD_RUN_TARGET_REVISION,
+# CLOUD_RUN_EXPECTED_CURRENT_REVISION and CLOUD_RUN_SERVICE from the
+# environment. CLOUD_RUN_SCRIPTS_DIR overrides where the two scripts are found
+# (tests use it).
+
+set -euo pipefail
+
+readiness_url="${1:?readiness URL is required}"
+scripts_dir="${CLOUD_RUN_SCRIPTS_DIR:-.github/scripts}"
+
+bash "${scripts_dir}/set_cloud_run_revision.sh"
+bash "${scripts_dir}/health_check.sh" "${readiness_url}"

.github/scripts/run_release_tests.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+# Run the release test suite (make test) before the staging deploy.
+#
+# The deploy job's environment carries deployment settings (route
+# implementations, database policies, Cloud Run and v2 database targets) that
+# must not leak into the tests, so they are removed here. The local Redis
+# runtime-cache settings come from the step's env.
+
+set -euo pipefail
+
+exec env \
+  -u DEPLOYMENT_ENVIRONMENT \
+  -u ROUTE_IMPL_HEALTH \
+  -u ROUTE_IMPL_SPECIFICATION \
+  -u ROUTE_IMPL_METADATA \
+  -u ROUTE_IMPL_POLICY \
+  -u ROUTE_IMPL_HOUSEHOLD \
+  -u DB_READ_POLICY \
+  -u DB_WRITE_POLICY \
+  -u DB_READ_HOUSEHOLD \
+  -u DB_WRITE_HOUSEHOLD \
+  -u CLOUD_RUN_SERVICE \
+  -u CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT \
+  -u CLOUD_RUN_RUNTIME_CACHE_URL_SECRET \
+  -u CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET \
+  -u CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET \
+  -u CLOUD_RUN_MIN_INSTANCES \
+  -u CLOUD_RUN_SERVICE_MIN_INSTANCES \
+  -u CLOUD_RUN_MAX_INSTANCES \
+  -u V2_SUPABASE_PROJECT_REF \
+  -u V2_SUPABASE_ENVIRONMENT \
+  -u V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE \
+  -u PRODUCTION_V2_SUPABASE_PROJECT_REF \
+  -u PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE \
+  -u PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME \
+  make test

.github/workflows/push.yml

@@ -120,7 +120,7 @@ jobs:
       - name: Refresh uv lock
         run: uv lock
       - name: Build changelog
-        run: towncrier build --yes --version "$(python -c "import re; print(re.search(r'version = \"(.+?)\"', open('pyproject.toml').read()).group(1))")"
+        run: bash .github/scripts/build_changelog.sh
       - name: Preview changelog update
         run: ".github/get-changelog-diff.sh"
       - name: Update changelog
@@ -241,17 +241,16 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.12"
+      - name: Update package lists
+        run: sudo apt-get update
       - name: Install release test dependencies
-        run: sudo apt-get update && sudo apt-get install -y redis-server
+        run: sudo apt-get install -y redis-server
       - name: Start release test Redis
         run: sudo systemctl start redis-server
       - name: Install application dependencies
         run: make install
       - name: Run release tests
-        run: >-
-          env -u DEPLOYMENT_ENVIRONMENT -u ROUTE_IMPL_HEALTH -u ROUTE_IMPL_SPECIFICATION -u ROUTE_IMPL_METADATA -u ROUTE_IMPL_POLICY -u ROUTE_IMPL_HOUSEHOLD -u DB_READ_POLICY -u DB_WRITE_POLICY -u DB_READ_HOUSEHOLD -u DB_WRITE_HOUSEHOLD
-          -u CLOUD_RUN_SERVICE -u CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT -u CLOUD_RUN_RUNTIME_CACHE_URL_SECRET -u CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET -u CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET -u CLOUD_RUN_MIN_INSTANCES -u CLOUD_RUN_SERVICE_MIN_INSTANCES -u CLOUD_RUN_MAX_INSTANCES
-          -u V2_SUPABASE_PROJECT_REF -u V2_SUPABASE_ENVIRONMENT -u V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE -u PRODUCTION_V2_SUPABASE_PROJECT_REF -u PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE -u PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME make test
+        run: bash .github/scripts/run_release_tests.sh
         env:
           RUNTIME_CACHE_MODE: local
           RUNTIME_CACHE_URL: redis://127.0.0.1:6379/0
@@ -288,10 +287,7 @@ jobs:
         run: bash .github/scripts/capture_cloud_run_service_state.sh >> "$GITHUB_OUTPUT"
       - name: Compute Cloud Run staging metadata
         id: cloud_run
-        run: |
-          echo "image_tag=${GITHUB_SHA}" >> "$GITHUB_OUTPUT"
-          # Short tag: Cloud Run caps tag + service name at 46 combined chars.
-          echo "revision_tag=stg-${GITHUB_RUN_NUMBER}-${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
+        run: bash .github/scripts/compute_cloud_run_metadata.sh stg
       - name: Build and push Cloud Run image
         run: bash .github/scripts/build_cloud_run_image.sh
         env:
@@ -412,9 +408,7 @@ jobs:
         run: bash .github/scripts/health_check.sh "${{ needs.deploy-cloud-run-staging.outputs.stable_url }}/readiness-check"
       - name: Restore previous Cloud Run staging revision
         if: ${{ steps.promote.outcome == 'failure' || steps.stable_health.outcome == 'failure' }}
-        run: |
-          bash .github/scripts/set_cloud_run_revision.sh
-          bash .github/scripts/health_check.sh "${{ needs.deploy-cloud-run-staging.outputs.stable_url }}/readiness-check"
+        run: bash .github/scripts/restore_cloud_run_revision.sh "${{ needs.deploy-cloud-run-staging.outputs.stable_url }}/readiness-check"
         env:
           CLOUD_RUN_TARGET_REVISION: ${{ needs.deploy-cloud-run-staging.outputs.previous_revision }}
           CLOUD_RUN_EXPECTED_CURRENT_REVISION: ${{ needs.deploy-cloud-run-staging.outputs.revision }}
@@ -538,9 +532,7 @@ jobs:
         run: bash .github/scripts/capture_cloud_run_service_state.sh >> "$GITHUB_OUTPUT"
       - name: Compute Cloud Run candidate metadata
         id: cloud_run
-        run: |
-          echo "image_tag=${GITHUB_SHA}" >> "$GITHUB_OUTPUT"
-          echo "revision_tag=stage3-prod-${GITHUB_RUN_NUMBER}-${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
+        run: bash .github/scripts/compute_cloud_run_metadata.sh stage3-prod
       - name: Deploy tagged Cloud Run candidate
         run: bash .github/scripts/deploy_cloud_run_candidate.sh
         env:
@@ -591,9 +583,7 @@ jobs:
         run: bash .github/scripts/health_check.sh "${{ steps.previous.outputs.stable_url }}/readiness-check"
       - name: Restore previous Cloud Run production revision
         if: ${{ steps.promote.outcome == 'failure' || steps.stable_health.outcome == 'failure' }}
-        run: |
-          bash .github/scripts/set_cloud_run_revision.sh
-          bash .github/scripts/health_check.sh "${{ steps.previous.outputs.stable_url }}/readiness-check"
+        run: bash .github/scripts/restore_cloud_run_revision.sh "${{ steps.previous.outputs.stable_url }}/readiness-check"
         env:
           CLOUD_RUN_TARGET_REVISION: ${{ steps.previous.outputs.revision }}
           CLOUD_RUN_EXPECTED_CURRENT_REVISION: ${{ steps.candidate.outputs.revision }}

changelog.d/ci-scripts-deploy.changed.md

@@ -0,0 +1 @@
+Moved the multi-line steps in the deploy pipeline (push.yml) into tested scripts, so every workflow run: step is now a single line.

docs/generated/rules.md

@@ -8,7 +8,7 @@ Every rule that governs `policyengine-api` is listed here with how it is enforce
 
 | Tier | enforced | ratcheted | advisory | doc-only | Total |
 | --- | ---: | ---: | ---: | ---: | ---: |
-| mechanical | 39 | 1 | 2 | 12 | 54 |
+| mechanical | 40 | 0 | 2 | 12 | 54 |
 | metric | 0 | 0 | 0 | 0 | 0 |
 | judgment | 0 | 0 | 0 | 0 | 0 |
 
@@ -64,7 +64,7 @@ Every rule that governs `policyengine-api` is listed here with how it is enforce
 
 ### Ratcheted (existing violations still tolerated)
 
-- `api.ci.multiline-run-is-script`: 7 remaining in `rules/baselines/ci-multiline-run.txt`
+- None.
 
 ### Exceptions and review dates
 
@@ -325,13 +325,13 @@ flowchart LR
 
 | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof |
 | --- | --- | --- | --- | --- | --- | --- | --- |
-| `api.ci.multiline-run-is-script` | A GitHub Actions run: block holds one line of code and no chained commands (&&, \|\|, ;); anything more lives in a tested script under .github/scripts/, called from a single-line step. | mechanical | ratcheted | yes | quality guard `ci-scripts` | `pr.yml:quality-guards` | yes |
+| `api.ci.multiline-run-is-script` | A GitHub Actions run: block holds one line of code and no chained commands (&&, \|\|, ;); anything more lives in a tested script under .github/scripts/, called from a single-line step. | mechanical | enforced | yes | quality guard `ci-scripts` | `pr.yml:quality-guards` | yes |
 | `api.ci.shell-scripts-pass-shellcheck` | Every shell script under .github/scripts/ passes shellcheck. | mechanical | enforced | yes | quality guard `ci-scripts` | `pr.yml:quality-guards` | yes |
 | `api.ci.workflow-scripts-tested` | Every shell script a workflow calls is run by at least one test. | mechanical | enforced | yes | quality guard `ci-scripts` | `pr.yml:quality-guards` | yes |
 
 ```mermaid
 flowchart LR
-  r_api_ci_multiline_run_is_script("api.ci.multiline-run-is-script (ratcheted)")
+  r_api_ci_multiline_run_is_script("api.ci.multiline-run-is-script (enforced)")
   e_quality_guard_ci_scripts["quality guard ci-scripts"]
   j_CI_pr_yml_quality_guards["CI pr.yml:quality-guards"]
   r_api_ci_shell_scripts_pass_shellcheck("api.ci.shell-scripts-pass-shellcheck (enforced)")

rules/baselines/ci-multiline-run.txt

@@ -1,8 +1 @@
 # Tolerated existing violations. This list may only shrink.
-.github/workflows/push.yml :: deploy-cloud-run-candidate :: Compute Cloud Run candidate metadata
-.github/workflows/push.yml :: deploy-cloud-run-candidate :: Restore previous Cloud Run production revision
-.github/workflows/push.yml :: deploy-cloud-run-staging :: Compute Cloud Run staging metadata
-.github/workflows/push.yml :: deploy-cloud-run-staging :: Install release test dependencies
-.github/workflows/push.yml :: deploy-cloud-run-staging :: Run release tests
-.github/workflows/push.yml :: promote-cloud-run-staging :: Restore previous Cloud Run staging revision
-.github/workflows/push.yml :: versioning :: Build changelog

rules/registry.toml

@@ -760,7 +760,7 @@ statement = "A GitHub Actions run: block holds one line of code and no chained c
 rationale = "docs/engineering/skills/ci-workflows.md#one-line-of-code-per-run-block"
 scope = [".github/workflows/*.yml"]
 tier = "mechanical"
-status = "ratcheted"
+status = "enforced"
 autofix = false
 [rule.enforcer]
 kind = "quality-guard"
@@ -769,9 +769,6 @@ name = "ci-scripts"
 workflow = ".github/workflows/pr.yml"
 job = "quality-guards"
 invocation = "run_quality_guards.py"
-[rule.baseline]
-file = "rules/baselines/ci-multiline-run.txt"
-count = 7
 [rule.proof]
 bad_fixture = "scripts/rules/tests/fixtures/ci_scripts/multiline_bad"
 good_fixture = "scripts/rules/tests/fixtures/ci_scripts/multiline_good"

tests/unit/test_alembic_workflows.py

@@ -135,7 +135,12 @@ def test_versioning_refreshes_the_lock_after_bumping_the_project_version():
     assert versioning_job.index("python .github/bump_version.py") < (
         versioning_job.index("uv lock")
     )
-    assert versioning_job.index("uv lock") < versioning_job.index("towncrier build")
+    assert versioning_job.index("uv lock") < (
+        versioning_job.index("bash .github/scripts/build_changelog.sh")
+    )
+    assert "towncrier build" in (REPO / ".github/scripts/build_changelog.sh").read_text(
+        encoding="utf-8"
+    )
     assert changelog_target.index("python .github/bump_version.py") < (
         changelog_target.index("uv lock")
     )
@@ -252,13 +257,13 @@ def test_release_migration_fails_closed_before_tests_and_cloud_run_deploy():
     cloud_run_job = workflow[workflow.index("  deploy-cloud-run-staging:") :]
     cloud_run_job = re.split(r"\n(?=  \S)", cloud_run_job, maxsplit=1)[0]
     assert "migrate-v1-staging-cloud-sql" in cloud_run_job
-    assert "make test" in cloud_run_job
-    assert cloud_run_job.index("make test") < cloud_run_job.index(
-        'uses: "google-github-actions/auth@v2"'
-    )
-    assert cloud_run_job.index("make test") < cloud_run_job.index(
-        "Build and push Cloud Run image"
-    )
+    assert "bash .github/scripts/run_release_tests.sh" in cloud_run_job
+    assert cloud_run_job.index(
+        "bash .github/scripts/run_release_tests.sh"
+    ) < cloud_run_job.index('uses: "google-github-actions/auth@v2"')
+    assert cloud_run_job.index(
+        "bash .github/scripts/run_release_tests.sh"
+    ) < cloud_run_job.index("Build and push Cloud Run image")
 
 
 def test_cloud_sql_workflow_uses_oidc_and_separate_database_credentials():

tests/unit/test_app_engine_decommission.py

@@ -80,13 +80,13 @@ def test_cloud_run_is_the_complete_release_sequence() -> None:
     assert "  release-tests:" not in workflow
     assert "migrate-v1-staging-cloud-sql" in staging_seed
     assert "- seed-v2-staging-database" in staging_deploy
-    assert "make test" in staging_deploy
-    assert staging_deploy.index("make test") < staging_deploy.index(
-        'uses: "google-github-actions/auth@v2"'
-    )
-    assert staging_deploy.index("make test") < staging_deploy.index(
-        "Build and push Cloud Run image"
-    )
+    assert "bash .github/scripts/run_release_tests.sh" in staging_deploy
+    assert staging_deploy.index(
+        "bash .github/scripts/run_release_tests.sh"
+    ) < staging_deploy.index('uses: "google-github-actions/auth@v2"')
+    assert staging_deploy.index(
+        "bash .github/scripts/run_release_tests.sh"
+    ) < staging_deploy.index("Build and push Cloud Run image")
     assert staging_deploy.index("Wait for Cloud Run staging health") < (
         staging_deploy.index("Run staging integration tests")
     )

tests/unit/test_cloud_run_deploy_scripts.py

@@ -1865,13 +1865,13 @@ def test_push_workflow_runs_release_and_cloud_run_staging_tests_in_deploy_job():
     )
     live_test_script = (REPO / ".github/scripts/run_live_staging_tests.sh").read_text()
 
-    assert "make test" in cloud_run_deploy
-    assert cloud_run_deploy.index("make test") < cloud_run_deploy.index(
-        'uses: "google-github-actions/auth@v2"'
-    )
-    assert cloud_run_deploy.index("make test") < cloud_run_deploy.index(
-        "Build and push Cloud Run image"
-    )
+    assert "bash .github/scripts/run_release_tests.sh" in cloud_run_deploy
+    assert cloud_run_deploy.index(
+        "bash .github/scripts/run_release_tests.sh"
+    ) < cloud_run_deploy.index('uses: "google-github-actions/auth@v2"')
+    assert cloud_run_deploy.index(
+        "bash .github/scripts/run_release_tests.sh"
+    ) < cloud_run_deploy.index("Build and push Cloud Run image")
     assert cloud_run_test_command in cloud_run_deploy
     assert "API_BASE_URL: ${{ steps.candidate.outputs.url }}" in cloud_run_deploy
     assert "environment: staging" in cloud_run_deploy
@@ -1956,15 +1956,22 @@ def test_push_workflow_uses_local_redis_for_predeployment_test_suite():
     assert "RUNTIME_CACHE_URL: redis://127.0.0.1:6379/0" in test_step
     assert "RUNTIME_CACHE_ENVIRONMENT: test" in test_step
     assert "RUNTIME_CACHE_SERVICE: api" in test_step
-    assert "-u ROUTE_IMPL_HEALTH" in test_step
-    assert "-u ROUTE_IMPL_POLICY" in test_step
-    assert "-u ROUTE_IMPL_HOUSEHOLD" in test_step
-    assert "-u DB_READ_POLICY" in test_step
-    assert "-u DB_WRITE_POLICY" in test_step
-    assert "-u DB_READ_HOUSEHOLD" in test_step
-    assert "-u DB_WRITE_HOUSEHOLD" in test_step
-    assert "-u CLOUD_RUN_SERVICE" in test_step
-    assert "-u V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE" in test_step
+    assert "run: bash .github/scripts/run_release_tests.sh" in test_step
+    release_tests = (REPO / ".github/scripts/run_release_tests.sh").read_text(
+        encoding="utf-8"
+    )
+    for variable in (
+        "ROUTE_IMPL_HEALTH",
+        "ROUTE_IMPL_POLICY",
+        "ROUTE_IMPL_HOUSEHOLD",
+        "DB_READ_POLICY",
+        "DB_WRITE_POLICY",
+        "DB_READ_HOUSEHOLD",
+        "DB_WRITE_HOUSEHOLD",
+        "CLOUD_RUN_SERVICE",
+        "V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE",
+    ):
+        assert f"-u {variable} " in release_tests
 
 
 def test_push_workflow_staging_precedes_all_production_deployments():
@@ -1974,7 +1981,7 @@ def test_push_workflow_staging_precedes_all_production_deployments():
 
     assert "needs: seed-v2-production-database" in cloud_run_production
     assert "needs: deploy-cloud-run-candidate" in docker_publish
-    assert "stage3-prod-" in cloud_run_production
+    assert "compute_cloud_run_metadata.sh stage3-prod" in cloud_run_production
     assert "Build and push Cloud Run image" not in cloud_run_production