simulation-deploy.reusable.yml

name: Reusable simulation deployment

on:
  workflow_call:
    inputs:
      release_environment:
        required: true
        type: string
        description: "GitHub environment for Modal deployment secrets"
      deployment_environment:
        required: true
        type: string
        description: "Logical deployment environment: staging or production"
      modal_environment:
        required: true
        type: string
        description: "Modal environment name"
      entrypoint_service:
        required: true
        type: string
        description: "Cloud Run service name"
      entrypoint_revision_prefix:
        required: true
        type: string
        description: "Prefix for the no-traffic Cloud Run revision tag"
      entrypoint_min_instances:
        required: true
        type: number
      entrypoint_max_instances:
        required: true
        type: number
      promote_entrypoint:
        required: true
        type: boolean
        description: "Assign stable service traffic after all environment tests pass"
      run_full_integration:
        required: true
        type: boolean
        description: "Run the complete calculation suite in this environment"
      deploy_existing_stack:
        required: false
        default: true
        type: boolean
        description: "Deploy the existing Modal routing service, executor, and v1 manifest"
      force_latest:
        required: false
        default: false
        type: boolean
        description: "Force latest routing pointers for an intentional rollback"
      force_recompute:
        required: false
        default: false
        type: boolean
        description: "Recompute artifacts even when the store already has them"
      deploy_stage12_v2:
        required: false
        default: false
        type: boolean
        description: "Deploy and publish the additional Stage 12 v2 workers"
    outputs:
      simulation_api_url:
        description: "Tagged Cloud Run candidate URL qualified by this deployment"
        value: ${{ jobs.deploy_entrypoint.outputs.candidate_url }}
      stable_url:
        description: "Stable Cloud Run service URL"
        value: ${{ jobs.deploy_entrypoint.outputs.stable_url }}
      entrypoint_revision:
        description: "Exact Cloud Run revision qualified by this deployment"
        value: ${{ jobs.deploy_entrypoint.outputs.revision }}
      entrypoint_image:
        description: "Image reference resolved on the deployed Cloud Run revision"
        value: ${{ jobs.deploy_entrypoint.outputs.image }}
      policyengine_version:
        description: "Deployed policyengine.py package version"
        value: ${{ jobs.prepare.outputs.policyengine_version }}
      policyengine_core_version:
        description: "Deployed policyengine-core package version"
        value: ${{ jobs.prepare.outputs.policyengine_core_version }}
      us_version:
        description: "Deployed policyengine-us package version"
        value: ${{ jobs.prepare.outputs.us_version }}
      us_data_version:
        description: "Bundled US data release version"
        value: ${{ jobs.prepare.outputs.us_data_version }}
      uk_version:
        description: "Deployed policyengine-uk package version"
        value: ${{ jobs.prepare.outputs.uk_version }}
      uk_data_version:
        description: "Bundled UK data release version"
        value: ${{ jobs.prepare.outputs.uk_data_version }}

permissions:
  contents: read

jobs:
  prepare:
    name: Prepare deployment inputs
    runs-on: ubuntu-latest
    environment: ${{ inputs.release_environment }}
    outputs:
      policyengine_version: ${{ steps.versions.outputs.policyengine_version }}
      policyengine_core_version: ${{ steps.versions.outputs.policyengine_core_version }}
      spm_calculator_version: ${{ steps.versions.outputs.spm_calculator_version }}
      us_version: ${{ steps.versions.outputs.us_version }}
      us_data_version: ${{ steps.versions.outputs.us_data_version }}
      uk_version: ${{ steps.versions.outputs.uk_version }}
      uk_data_version: ${{ steps.versions.outputs.uk_data_version }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-python@v6
        with:
          python-version: "3.13"

      - uses: astral-sh/setup-uv@v8.1.0

      - name: Install deployment dependencies
        working-directory: projects/policyengine-simulation-executor
        run: uv sync --frozen

      - name: Extract package versions
        id: versions
        run: .github/scripts/modal-extract-versions.sh projects/policyengine-simulation-executor

      - name: Validate the GitHub UK private-data credential
        if: ${{ inputs.deploy_existing_stack || inputs.deploy_stage12_v2 }}
        working-directory: projects/policyengine-simulation-executor
        env:
          HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}
        run: uv run python -m src.modal.utils.validate_hf_access

      - name: Synchronize the Modal UK private-data credential
        if: ${{ inputs.deploy_existing_stack || inputs.deploy_stage12_v2 }}
        working-directory: projects/policyengine-simulation-executor
        env:
          MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
          MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
          PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}
        run: |
          uv run modal secret create pe-uk-private-hf-read-token \
            "HUGGING_FACE_TOKEN=${PE_UK_PRIVATE_HF_READ_TOKEN}" \
            --env="${{ inputs.modal_environment }}" \
            --force

      - name: Synchronize Modal runtime secrets
        if: ${{ inputs.deploy_existing_stack }}
        working-directory: projects/policyengine-simulation-executor
        env:
          MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
          MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
          GCP_CREDENTIALS_JSON: ${{ secrets.GCP_CREDENTIALS_JSON }}
          GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }}
          GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }}
          GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }}
          GATEWAY_AUTH_CLIENT_SECRET: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET }}
          GATEWAY_AUTH_REQUIRED: ${{ vars.GATEWAY_AUTH_REQUIRED }}
        run: ../../.github/scripts/modal-sync-secrets.sh "${{ inputs.modal_environment }}"

  deploy_entrypoint:
    name: Deploy and unit test Cloud Run entrypoint
    if: ${{ always() && needs.prepare.result == 'success' && ((inputs.deploy_stage12_v2 && needs.configure_stage12_v2.result == 'success') || (!inputs.deploy_stage12_v2 && needs.configure_stage12_v2.result == 'skipped')) }}
    needs: [prepare, configure_stage12_v2]
    runs-on: ubuntu-latest
    environment: ${{ inputs.release_environment }}
    permissions:
      contents: read
      id-token: write
    env:
      PROJECT_DIR: projects/policyengine-simulation-entry
      REGION: ${{ vars.SIMULATION_ENTRYPOINT_GCP_REGION || 'us-central1' }}
      PROJECT_ID: ${{ vars.SIMULATION_ENTRYPOINT_GCP_PROJECT_ID }}
      REPOSITORY: ${{ vars.SIMULATION_ENTRYPOINT_ARTIFACT_REPOSITORY || 'policyengine-simulation-entry' }}
      IMAGE_NAME: policyengine-simulation-entry
    outputs:
      image: ${{ steps.candidate.outputs.image }}
      revision_tag: ${{ steps.metadata.outputs.tag }}
      revision: ${{ steps.candidate.outputs.revision }}
      candidate_url: ${{ steps.candidate.outputs.url }}
      stable_url: ${{ steps.previous.outputs.stable_url }}
      previous_revision: ${{ steps.previous.outputs.revision }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-python@v6
        with:
          python-version: "3.13"

      - uses: astral-sh/setup-uv@v8.1.0

      - uses: google-github-actions/auth@v2
        with:
          workload_identity_provider: ${{ secrets.SIMULATION_ENTRYPOINT_GCP_WIF_PROVIDER }}
          service_account: ${{ secrets.SIMULATION_ENTRYPOINT_GCP_DEPLOY_SERVICE_ACCOUNT }}

      - uses: google-github-actions/setup-gcloud@v2

      - name: Configure Artifact Registry authentication
        run: gcloud auth configure-docker "${REGION}-docker.pkg.dev" --quiet

      - name: Capture stable service state before deployment
        id: previous
        run: |
          service_json="$(gcloud run services describe "${{ inputs.entrypoint_service }}" \
            --project "${PROJECT_ID}" \
            --region "${REGION}" \
            --format=json)"
          stable_url="$(jq -er '.status.url | select(type == "string" and length > 0)' <<<"${service_json}")"
          revision="$(jq -er '
            [
              .status.traffic[]?
              | select((.percent // 0) == 100)
              | .revisionName
              | select(type == "string" and length > 0)
            ]
            | if length == 1 then .[0]
              else error("service must have exactly one revision at 100 percent")
              end
          ' <<<"${service_json}")"
          {
            echo "stable_url=${stable_url}"
            echo "revision=${revision}"
          } >> "$GITHUB_OUTPUT"

      - name: Select immutable image
        id: image
        run: echo "uri=${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPOSITORY}/${IMAGE_NAME}:${GITHUB_SHA}" >> "$GITHUB_OUTPUT"

      - name: Build and push entrypoint image
        run: |
          docker build \
            --file "${PROJECT_DIR}/Dockerfile" \
            --tag "${{ steps.image.outputs.uri }}" \
            .
          docker push "${{ steps.image.outputs.uri }}"

      - name: Select no-traffic revision tag
        id: metadata
        run: echo "tag=${{ inputs.entrypoint_revision_prefix }}${GITHUB_RUN_NUMBER}" >> "$GITHUB_OUTPUT"

      - name: Deploy Cloud Run entrypoint candidate
        env:
          IMAGE: ${{ steps.image.outputs.uri }}
          TAG: ${{ steps.metadata.outputs.tag }}
          OBSERVABILITY_SERVICE_NAMESPACE: ${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }}
          OBSERVABILITY_TRACE_PROJECT_ID: ${{ vars.OBSERVABILITY_TRACE_PROJECT_ID }}
          OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OBSERVABILITY_OTLP_ENDPOINT }}
          POLICYENGINE_OTEL_GOOGLE_AUDIENCE: ${{ vars.OBSERVABILITY_OTLP_GOOGLE_AUDIENCE }}
          DEPLOY_STAGE12_V2: ${{ inputs.deploy_stage12_v2 }}
          APP_ENVIRONMENT: ${{ inputs.deployment_environment }}
          MODAL_ENVIRONMENT: ${{ inputs.modal_environment }}
          ENTRYPOINT_SERVICE: ${{ inputs.entrypoint_service }}
          ENTRYPOINT_RUNTIME_SERVICE_ACCOUNT: ${{ vars.SIMULATION_ENTRYPOINT_RUNTIME_SERVICE_ACCOUNT }}
          ENTRYPOINT_MIN_INSTANCES: ${{ inputs.entrypoint_min_instances }}
          ENTRYPOINT_MAX_INSTANCES: ${{ inputs.entrypoint_max_instances }}
          SIMULATION_ENTRYPOINT_AUTH_ISSUER_VALUE: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_ISSUER }}
          SIMULATION_ENTRYPOINT_AUTH_AUDIENCE_VALUE: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_AUDIENCE }}
          OLD_GATEWAY_URL_VALUE: ${{ secrets.OLD_GATEWAY_URL }}
          OLD_GATEWAY_AUTH_ISSUER_VALUE: ${{ secrets.OLD_GATEWAY_AUTH_ISSUER }}
          OLD_GATEWAY_AUTH_AUDIENCE_VALUE: ${{ secrets.OLD_GATEWAY_AUTH_AUDIENCE }}
          OLD_GATEWAY_AUTH_CLIENT_ID_VALUE: ${{ secrets.OLD_GATEWAY_AUTH_CLIENT_ID }}
          OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME: ${{ vars.OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME }}
          STAGE12_ENABLED_VALUE: ${{ vars.STAGE12_ENABLED }}
          STAGE12_ARTIFACT_BUCKET_VALUE: ${{ vars.STAGE12_ARTIFACT_BUCKET }}
          STAGE12_DATABASE_URL_SECRET_NAME: ${{ vars.STAGE12_DATABASE_URL_SECRET_NAME }}
          MODAL_TOKEN_ID_SECRET_NAME: ${{ vars.MODAL_TOKEN_ID_SECRET_NAME }}
          MODAL_TOKEN_SECRET_SECRET_NAME: ${{ vars.MODAL_TOKEN_SECRET_SECRET_NAME }}
        run: .github/scripts/deploy-cloud-run-simulation-entry.sh

      - name: Resolve exact candidate metadata
        id: candidate
        env:
          TAG: ${{ steps.metadata.outputs.tag }}
        run: |
          service_json="$(gcloud run services describe "${{ inputs.entrypoint_service }}" \
            --project "${PROJECT_ID}" \
            --region "${REGION}" \
            --format=json)"
          url="$(jq -er --arg tag "${TAG}" '
            .status.traffic[]
            | select(.tag == $tag)
            | .url
            | select(type == "string" and length > 0)
          ' <<<"${service_json}")"
          revision="$(jq -er --arg tag "${TAG}" '
            .status.traffic[]
            | select(.tag == $tag)
            | .revisionName
            | select(type == "string" and length > 0)
          ' <<<"${service_json}")"
          revision_json="$(gcloud run revisions describe "${revision}" \
            --project "${PROJECT_ID}" \
            --region "${REGION}" \
            --format=json)"
          image="$(jq -er '
            (.status.imageDigest // .spec.containers[0].image)
            | select(
                type == "string"
                and contains("@sha256:")
              )
          ' <<<"${revision_json}")"
          {
            echo "url=${url}"
            echo "revision=${revision}"
            echo "image=${image}"
          } >> "$GITHUB_OUTPUT"

      - name: Install entrypoint unit-test dependencies
        working-directory: ${{ env.PROJECT_DIR }}
        run: uv sync --extra test --frozen

      - name: Run entrypoint unit tests
        working-directory: ${{ env.PROJECT_DIR }}
        run: uv run pytest tests/ -v

  deploy_gateway:
    name: Deploy and unit test Modal gateway
    if: ${{ inputs.deploy_existing_stack }}
    needs: [prepare, deploy_executor]
    runs-on: ubuntu-latest
    environment: ${{ inputs.release_environment }}
    outputs:
      url: ${{ steps.url.outputs.simulation_api_url }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-python@v6
        with:
          python-version: "3.13"

      - uses: astral-sh/setup-uv@v8.1.0

      - name: Install gateway dependencies
        working-directory: projects/policyengine-simulation-gateway
        run: uv sync --extra test --frozen

      - name: Deploy stable Modal gateway
        working-directory: projects/policyengine-simulation-gateway
        env:
          MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
          MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
          OBSERVABILITY_SERVICE_NAMESPACE: ${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }}
          OBSERVABILITY_TRACE_PROJECT_ID: ${{ vars.OBSERVABILITY_TRACE_PROJECT_ID }}
          OBSERVABILITY_LOGGING_PROJECT_ID: ${{ vars.OBSERVABILITY_LOGGING_PROJECT_ID }}
          OBSERVABILITY_LOG_NAME: ${{ vars.OBSERVABILITY_LOG_NAME }}
          OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
          OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL: ${{ vars.OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL }}
          OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OBSERVABILITY_OTLP_ENDPOINT }}
          POLICYENGINE_OTEL_GOOGLE_AUDIENCE: ${{ vars.OBSERVABILITY_OTLP_GOOGLE_AUDIENCE }}
        run: uv run modal deploy --env="${{ inputs.modal_environment }}" src/policyengine_simulation_gateway/app.py

      - name: Run gateway unit tests
        working-directory: projects/policyengine-simulation-gateway
        run: uv run pytest tests/ -v

      - name: Resolve gateway URL
        id: url
        working-directory: projects/policyengine-simulation-executor
        env:
          MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
          MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
        run: ../../.github/scripts/modal-get-url.sh "${{ inputs.modal_environment }}"

      - name: Verify gateway health
        run: .github/scripts/modal-health-check.sh "${{ steps.url.outputs.simulation_api_url }}"

  deploy_executor:
    name: Deploy and unit test Modal executor
    if: ${{ inputs.deploy_existing_stack }}
    needs: prepare
    runs-on: ubuntu-latest
    environment: ${{ inputs.release_environment }}
    outputs:
      app_name: ${{ steps.app.outputs.name }}
      manifest_digest: ${{ steps.precompute.outputs.manifest_digest }}
    env:
      POLICYENGINE_VERSION: ${{ needs.prepare.outputs.policyengine_version }}
      POLICYENGINE_CORE_VERSION: ${{ needs.prepare.outputs.policyengine_core_version }}
      POLICYENGINE_US_VERSION: ${{ needs.prepare.outputs.us_version }}
      POLICYENGINE_UK_VERSION: ${{ needs.prepare.outputs.uk_version }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-python@v6
        with:
          python-version: "3.13"

      - uses: astral-sh/setup-uv@v8.1.0

      - name: Install executor dependencies
        working-directory: projects/policyengine-simulation-executor
        run: uv sync --extra test --frozen

      - name: Run artifact precompute
        id: precompute
        working-directory: projects/policyengine-simulation-executor
        env:
          MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
          MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
          POLICYENGINE_ARTIFACT_BUCKET: ${{ vars.POLICYENGINE_ARTIFACT_BUCKET }}
        run: ../../.github/scripts/modal-precompute.sh "${{ inputs.modal_environment }}" "${{ inputs.force_recompute }}"

      - name: Select versioned executor app
        id: app
        run: |
          version_safe="${POLICYENGINE_VERSION//./-}"
          echo "name=policyengine-simulation-py${version_safe}" >> "$GITHUB_OUTPUT"

      - name: Deploy versioned Modal executor
        working-directory: projects/policyengine-simulation-executor
        env:
          MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
          MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
          MODAL_APP_NAME: ${{ steps.app.outputs.name }}
          POLICYENGINE_MANIFEST_DIGEST: ${{ steps.precompute.outputs.manifest_digest }}
          POLICYENGINE_ARTIFACT_BUCKET: ${{ vars.POLICYENGINE_ARTIFACT_BUCKET }}
          GCP_CREDENTIALS_JSON: ${{ secrets.GCP_CREDENTIALS_JSON }}
          OBSERVABILITY_SERVICE_NAMESPACE: ${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }}
          OBSERVABILITY_TRACE_PROJECT_ID: ${{ vars.OBSERVABILITY_TRACE_PROJECT_ID }}
          OBSERVABILITY_LOGGING_PROJECT_ID: ${{ vars.OBSERVABILITY_LOGGING_PROJECT_ID }}
          OBSERVABILITY_LOG_NAME: ${{ vars.OBSERVABILITY_LOG_NAME }}
          OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
          OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL: ${{ vars.OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL }}
          OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OBSERVABILITY_OTLP_ENDPOINT }}
          POLICYENGINE_OTEL_GOOGLE_AUDIENCE: ${{ vars.OBSERVABILITY_OTLP_GOOGLE_AUDIENCE }}
        run: uv run modal deploy --env="${{ inputs.modal_environment }}" src/modal/app.py

      - name: Run executor unit tests
        working-directory: projects/policyengine-simulation-executor
        run: uv run pytest tests/ -v

  update_routing:
    name: Publish routing after all services pass
    if: ${{ inputs.deploy_existing_stack }}
    needs: [prepare, deploy_entrypoint, deploy_gateway, deploy_executor]
    runs-on: ubuntu-latest
    environment: ${{ inputs.release_environment }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-python@v6
        with:
          python-version: "3.13"

      - uses: astral-sh/setup-uv@v8.1.0

      - name: Install routing publisher dependencies
        working-directory: projects/policyengine-simulation-executor
        run: uv sync --frozen

      - name: Publish active routing state
        working-directory: projects/policyengine-simulation-executor
        env:
          MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
          MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
          APP_NAME: ${{ needs.deploy_executor.outputs.app_name }}
          POLICYENGINE_VERSION: ${{ needs.prepare.outputs.policyengine_version }}
          POLICYENGINE_US_VERSION: ${{ needs.prepare.outputs.us_version }}
          POLICYENGINE_UK_VERSION: ${{ needs.prepare.outputs.uk_version }}
          FORCE_LATEST: ${{ inputs.force_latest }}
        run: |
          args=(
            --app-name "${APP_NAME}"
            --policyengine-version "${POLICYENGINE_VERSION}"
            --us-version "${POLICYENGINE_US_VERSION}"
            --uk-version "${POLICYENGINE_UK_VERSION}"
            --environment "${{ inputs.modal_environment }}"
          )
          if [[ "${FORCE_LATEST}" == "true" ]]; then
            args+=(--force-latest)
          fi
          uv run python -m src.modal.utils.update_version_registry "${args[@]}"

  configure_stage12_v2:
    name: Validate pre-provisioned Stage 12 runtime access
    if: ${{ inputs.deploy_stage12_v2 }}
    needs: prepare
    runs-on: ubuntu-latest
    environment: ${{ inputs.release_environment }}
    permissions:
      contents: read
      id-token: write
    steps:
      - uses: actions/checkout@v6

      - uses: google-github-actions/auth@v2
        with:
          workload_identity_provider: ${{ secrets.SIMULATION_ENTRYPOINT_GCP_WIF_PROVIDER }}
          service_account: ${{ secrets.SIMULATION_ENTRYPOINT_GCP_DEPLOY_SERVICE_ACCOUNT }}

      - uses: google-github-actions/setup-gcloud@v2

      - uses: actions/setup-python@v6
        with:
          python-version: "3.13"

      - uses: astral-sh/setup-uv@v8.1.0

      - name: Install Stage 12 infrastructure validation dependencies
        working-directory: projects/policyengine-simulation-entry
        run: uv sync --frozen

      - name: Validate pre-provisioned Stage 12 deployment resources
        env:
          STAGE12_ENVIRONMENT: ${{ inputs.deployment_environment }}
          STAGE12_GCP_PROJECT_ID: ${{ vars.SIMULATION_ENTRYPOINT_GCP_PROJECT_ID }}
          STAGE12_ARTIFACT_BUCKET: ${{ vars.STAGE12_ARTIFACT_BUCKET }}
          STAGE12_ENTRYPOINT_SERVICE_ACCOUNT: ${{ vars.SIMULATION_ENTRYPOINT_RUNTIME_SERVICE_ACCOUNT }}
          STAGE12_DEPLOY_SERVICE_ACCOUNT: ${{ secrets.SIMULATION_ENTRYPOINT_GCP_DEPLOY_SERVICE_ACCOUNT }}
          STAGE12_MODAL_SERVICE_ACCOUNT: ${{ vars.STAGE12_MODAL_SERVICE_ACCOUNT }}
          STAGE12_MODAL_TOKEN_ID_SECRET_NAME: ${{ vars.MODAL_TOKEN_ID_SECRET_NAME }}
          STAGE12_MODAL_TOKEN_SECRET_SECRET_NAME: ${{ vars.MODAL_TOKEN_SECRET_SECRET_NAME }}
          STAGE12_GCP_CREDENTIALS_SECRET_NAME: ${{ vars.STAGE12_GCP_CREDENTIALS_SECRET_NAME }}
          STAGE12_DATABASE_URL_SECRET_NAME: ${{ vars.STAGE12_DATABASE_URL_SECRET_NAME }}
        run: bash .github/scripts/stage12-validate-infrastructure.sh

  deploy_stage12_v2:
    name: Deploy and validate separate Stage 12 v2 workers
    if: ${{ inputs.deploy_stage12_v2 }}
    needs: [prepare, configure_stage12_v2]
    runs-on: ubuntu-latest
    environment: ${{ inputs.release_environment }}
    permissions:
      contents: read
      id-token: write
    outputs:
      application_name: ${{ steps.application.outputs.name }}
      validation_file: ${{ steps.validation.outputs.file }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-python@v6
        with:
          python-version: "3.13"

      - uses: astral-sh/setup-uv@v8.1.0

      - uses: google-github-actions/auth@v2
        with:
          workload_identity_provider: ${{ secrets.SIMULATION_ENTRYPOINT_GCP_WIF_PROVIDER }}
          service_account: ${{ secrets.SIMULATION_ENTRYPOINT_GCP_DEPLOY_SERVICE_ACCOUNT }}

      - uses: google-github-actions/setup-gcloud@v2

      - name: Install Stage 12 deployment dependencies
        working-directory: projects/policyengine-simulation-executor
        run: uv sync --extra test --frozen

      - name: Validate the packaged PolicyEngine.py bundle
        working-directory: projects/policyengine-simulation-executor
        run: |
          uv run python -m src.modal.utils.extract_stage12_bundle \
            --expect "policyengine_version=${{ needs.prepare.outputs.policyengine_version }}" \
            --expect "countries.us.country_package_version=${{ needs.prepare.outputs.us_version }}" \
            --expect "countries.uk.country_package_version=${{ needs.prepare.outputs.uk_version }}" \
            --output stage12-bundle.json

      - name: Select the bundle-derived v2 application name
        id: application
        working-directory: projects/policyengine-simulation-executor
        run: |
          name="$(uv run python -c 'from policyengine_simulation_contract.stage12_manifest import v2_application_name; from policyengine_simulation_executor.stage12_bundle import load_stage12_bundle; print(v2_application_name(load_stage12_bundle().bundle.policyengine_version))')"
          echo "name=${name}" >> "$GITHUB_OUTPUT"

      - name: Synchronize the separate Stage 12 runtime secret
        working-directory: projects/policyengine-simulation-executor
        env:
          MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
          MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
          STAGE12_ARTIFACT_BUCKET: ${{ vars.STAGE12_ARTIFACT_BUCKET }}
          STAGE12_DATABASE_URL_SECRET_NAME: ${{ vars.STAGE12_DATABASE_URL_SECRET_NAME }}
          STAGE12_GCP_PROJECT_ID: ${{ vars.SIMULATION_ENTRYPOINT_GCP_PROJECT_ID }}
          STAGE12_GCP_CREDENTIALS_SECRET_NAME: ${{ vars.STAGE12_GCP_CREDENTIALS_SECRET_NAME }}
        run: ../../.github/scripts/stage12-sync-modal-secrets.sh "${{ inputs.modal_environment }}"

      - name: Deploy the additional versioned v2 Modal application
        working-directory: projects/policyengine-simulation-executor
        env:
          MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
          MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
          MODAL_APP_NAME: ${{ steps.application.outputs.name }}
          STAGE12_EXPECT_POLICYENGINE_VERSION: ${{ needs.prepare.outputs.policyengine_version }}
          STAGE12_EXPECT_US_VERSION: ${{ needs.prepare.outputs.us_version }}
          STAGE12_EXPECT_UK_VERSION: ${{ needs.prepare.outputs.uk_version }}
          STAGE12_ARTIFACT_BUCKET: ${{ vars.STAGE12_ARTIFACT_BUCKET }}
          OBSERVABILITY_SERVICE_NAMESPACE: ${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }}
          OBSERVABILITY_TRACE_PROJECT_ID: ${{ vars.OBSERVABILITY_TRACE_PROJECT_ID }}
          OBSERVABILITY_LOGGING_PROJECT_ID: ${{ vars.OBSERVABILITY_LOGGING_PROJECT_ID }}
          OBSERVABILITY_LOG_NAME: ${{ vars.OBSERVABILITY_LOG_NAME }}
          OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
          OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL: ${{ vars.OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL }}
          OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OBSERVABILITY_OTLP_ENDPOINT }}
          POLICYENGINE_OTEL_GOOGLE_AUDIENCE: ${{ vars.OBSERVABILITY_OTLP_GOOGLE_AUDIENCE }}
        run: uv run modal deploy --env="${{ inputs.modal_environment }}" src/modal/v2_app.py

      - name: Validate installed bundles, secrets, datasets, and calculations
        id: validation
        working-directory: projects/policyengine-simulation-executor
        env:
          MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
          MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
        run: |
          output="${RUNNER_TEMP}/stage12-worker-validation.json"
          uv run python -m src.modal.utils.validate_v2_worker_release \
            --environment "${{ inputs.modal_environment }}" \
            --output "${output}"
          echo "file=${output}" >> "$GITHUB_OUTPUT"

      - name: Retain worker validation for manifest publication
        uses: actions/upload-artifact@v7
        with:
          name: stage12-worker-validation-${{ inputs.release_environment }}
          path: ${{ steps.validation.outputs.file }}
          if-no-files-found: error

  publish_stage12_v2_manifest:
    name: Publish separate Stage 12 v2 version manifest
    if: ${{ inputs.deploy_stage12_v2 }}
    needs: [prepare, deploy_stage12_v2]
    runs-on: ubuntu-latest
    environment: ${{ inputs.release_environment }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-python@v6
        with:
          python-version: "3.13"

      - uses: astral-sh/setup-uv@v8.1.0

      - name: Install Stage 12 manifest publisher dependencies
        working-directory: projects/policyengine-simulation-executor
        run: uv sync --frozen

      - name: Download validated worker result
        uses: actions/download-artifact@v8
        with:
          name: stage12-worker-validation-${{ inputs.release_environment }}
          path: ${{ runner.temp }}/stage12-worker-validation

      - name: Publish only the separate v2 manifest
        working-directory: projects/policyengine-simulation-executor
        env:
          MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
          MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
        run: |
          uv run python -m src.modal.utils.update_v2_version_manifest \
            --environment "${{ inputs.modal_environment }}" \
            --manifest-name simulation-api-v2-version-manifest \
            --validation-file "${RUNNER_TEMP}/stage12-worker-validation/stage12-worker-validation.json"

  stage12_v2_ready:
    name: Require requested Stage 12 v2 deployment to succeed
    if: ${{ always() }}
    needs:
      [configure_stage12_v2, deploy_stage12_v2, publish_stage12_v2_manifest]
    runs-on: ubuntu-latest
    steps:
      - name: Check Stage 12 deployment result
        env:
          REQUESTED: ${{ inputs.deploy_stage12_v2 }}
          CONFIGURATION_RESULT: ${{ needs.configure_stage12_v2.result }}
          DEPLOYMENT_RESULT: ${{ needs.deploy_stage12_v2.result }}
          PUBLICATION_RESULT: ${{ needs.publish_stage12_v2_manifest.result }}
        run: |
          if [[ "${REQUESTED}" != "true" ]]; then
            test "${CONFIGURATION_RESULT}" = "skipped"
            test "${DEPLOYMENT_RESULT}" = "skipped"
            test "${PUBLICATION_RESULT}" = "skipped"
            exit 0
          fi
          test "${CONFIGURATION_RESULT}" = "success"
          test "${DEPLOYMENT_RESULT}" = "success"
          test "${PUBLICATION_RESULT}" = "success"

  integration:
    name: Verify deployed three-service stack
    if: ${{ always() && needs.prepare.result == 'success' && needs.deploy_entrypoint.result == 'success' && needs.stage12_v2_ready.result == 'success' && ((inputs.deploy_existing_stack && needs.deploy_gateway.result == 'success' && needs.deploy_executor.result == 'success' && needs.update_routing.result == 'success') || (!inputs.deploy_existing_stack && needs.deploy_gateway.result == 'skipped' && needs.deploy_executor.result == 'skipped' && needs.update_routing.result == 'skipped')) }}
    needs:
      [
        prepare,
        deploy_entrypoint,
        deploy_gateway,
        deploy_executor,
        update_routing,
        stage12_v2_ready,
      ]
    runs-on: ubuntu-latest
    environment: ${{ inputs.release_environment }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-python@v6
        with:
          python-version: "3.13"

      - uses: astral-sh/setup-uv@v8.1.0

      - name: Verify the complete deployed request path
        run: .github/scripts/cloud-run-simulation-entry-smoke.sh "${{ needs.deploy_entrypoint.outputs.candidate_url }}" "${{ needs.deploy_entrypoint.outputs.revision }}"

      - name: Generate API clients for complete integration tests
        if: ${{ inputs.run_full_integration }}
        run: ./scripts/generate-clients.sh

      - name: Run complete beta integration tests through all services
        if: ${{ inputs.run_full_integration }}
        env:
          SIMULATION_TEST_AUTH_REQUIRED: "1"
          SIMULATION_TEST_AUTH_ISSUER: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_ISSUER }}
          SIMULATION_TEST_AUTH_AUDIENCE: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_AUDIENCE }}
          SIMULATION_TEST_AUTH_CLIENT_ID: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID }}
          SIMULATION_TEST_AUTH_CLIENT_SECRET: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }}
        run: .github/scripts/simulation-run-integ-tests.sh "${{ inputs.release_environment }}" "${{ needs.deploy_entrypoint.outputs.candidate_url }}" "${{ needs.prepare.outputs.us_version }}" "${{ needs.prepare.outputs.uk_version }}"

      - name: Record deployment marker
        if: ${{ inputs.deploy_existing_stack }}
        working-directory: projects/policyengine-simulation-executor
        env:
          POLICYENGINE_ARTIFACT_BUCKET: ${{ vars.POLICYENGINE_ARTIFACT_BUCKET }}
          GCP_CREDENTIALS_JSON: ${{ secrets.GCP_CREDENTIALS_JSON }}
          POLICYENGINE_VERSION: ${{ needs.prepare.outputs.policyengine_version }}
          POLICYENGINE_US_VERSION: ${{ needs.prepare.outputs.us_version }}
          POLICYENGINE_UK_VERSION: ${{ needs.prepare.outputs.uk_version }}
          US_DATA_VERSION: ${{ needs.prepare.outputs.us_data_version }}
          UK_DATA_VERSION: ${{ needs.prepare.outputs.uk_data_version }}
        run: ../../.github/scripts/modal-record-deployment.sh "${{ inputs.release_environment }}" "${{ needs.deploy_executor.outputs.manifest_digest }}"

  authenticated_test:
    name: Run authenticated deployment tests
    # Stage 12-only deployments intentionally skip the existing v1 Modal jobs.
    # Use an explicit status function and direct result checks so those expected
    # upstream skips do not suppress authenticated candidate testing.
    if: >-
      ${{
        !cancelled() &&
        needs.deploy_entrypoint.result == 'success' &&
        needs.integration.result == 'success'
      }}
    needs: [deploy_entrypoint, integration]
    runs-on: ubuntu-latest
    environment: ${{ inputs.release_environment }}
    env:
      SIMULATION_ENTRYPOINT_TEST_BASE_URL: ${{ needs.deploy_entrypoint.outputs.candidate_url }}
      SIMULATION_ENTRYPOINT_TEST_AUTH_ISSUER: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_ISSUER }}
      SIMULATION_ENTRYPOINT_TEST_AUTH_AUDIENCE: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_AUDIENCE }}
      SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID }}
      SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-python@v6
        with:
          python-version: "3.13"

      - uses: astral-sh/setup-uv@v8.1.0

      - name: Install authenticated-test dependencies
        working-directory: projects/policyengine-simulation-entry
        run: uv sync --extra test --frozen

      - name: Run authenticated deployment tests
        working-directory: projects/policyengine-simulation-entry
        run: uv run pytest authenticated_tests/ -v

  promote_entrypoint:
    name: Promote and verify Cloud Run entrypoint
    # Require the exact candidate and its authenticated tests to succeed while
    # overriding propagation from intentionally skipped existing-stack jobs.
    if: >-
      ${{
        !cancelled() &&
        inputs.promote_entrypoint &&
        needs.deploy_entrypoint.result == 'success' &&
        needs.authenticated_test.result == 'success'
      }}
    needs: [deploy_entrypoint, authenticated_test]
    runs-on: ubuntu-latest
    environment: ${{ inputs.release_environment }}
    permissions:
      contents: read
      id-token: write
    env:
      REGION: ${{ vars.SIMULATION_ENTRYPOINT_GCP_REGION || 'us-central1' }}
      PROJECT_ID: ${{ vars.SIMULATION_ENTRYPOINT_GCP_PROJECT_ID }}
      SERVICE: ${{ inputs.entrypoint_service }}
      TARGET_REVISION: ${{ needs.deploy_entrypoint.outputs.revision }}
      PREVIOUS_REVISION: ${{ needs.deploy_entrypoint.outputs.previous_revision }}
      STABLE_URL: ${{ needs.deploy_entrypoint.outputs.stable_url }}
      CUSTOM_URL: ${{ vars.SIMULATION_ENTRYPOINT_CUSTOM_URL }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-python@v6
        with:
          python-version: "3.13"

      - uses: astral-sh/setup-uv@v8.1.0

      - name: Install stable-endpoint test dependencies
        working-directory: projects/policyengine-simulation-entry
        run: uv sync --extra test --frozen

      - uses: google-github-actions/auth@v2
        with:
          workload_identity_provider: ${{ secrets.SIMULATION_ENTRYPOINT_GCP_WIF_PROVIDER }}
          service_account: ${{ secrets.SIMULATION_ENTRYPOINT_GCP_DEPLOY_SERVICE_ACCOUNT }}

      - uses: google-github-actions/setup-gcloud@v2

      - name: Promote exact tested revision
        id: promote
        env:
          SIMULATION_ENTRYPOINT_GCP_PROJECT_ID: ${{ env.PROJECT_ID }}
          SIMULATION_ENTRYPOINT_GCP_REGION: ${{ env.REGION }}
          SIMULATION_ENTRYPOINT_SERVICE: ${{ env.SERVICE }}
          SIMULATION_ENTRYPOINT_TARGET_REVISION: ${{ env.TARGET_REVISION }}
          SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION: ${{ env.PREVIOUS_REVISION }}
        run: .github/scripts/set-cloud-run-simulation-entry-revision.sh

      - name: Verify stable service routing
        run: .github/scripts/cloud-run-simulation-entry-smoke.sh "${STABLE_URL}" "${TARGET_REVISION}" "${PREVIOUS_REVISION}"

      - name: Verify stable service authentication
        working-directory: projects/policyengine-simulation-entry
        env:
          SIMULATION_ENTRYPOINT_TEST_BASE_URL: ${{ env.STABLE_URL }}
          SIMULATION_ENTRYPOINT_TEST_AUTH_ISSUER: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_ISSUER }}
          SIMULATION_ENTRYPOINT_TEST_AUTH_AUDIENCE: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_AUDIENCE }}
          SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID }}
          SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }}
        run: uv run pytest authenticated_tests/ -v

      - name: Verify configured custom hostname
        if: ${{ env.CUSTOM_URL != '' }}
        run: .github/scripts/cloud-run-simulation-entry-smoke.sh "${CUSTOM_URL}" "${TARGET_REVISION}" "${PREVIOUS_REVISION}"

      - name: Verify configured custom-hostname authentication
        if: ${{ env.CUSTOM_URL != '' }}
        working-directory: projects/policyengine-simulation-entry
        env:
          SIMULATION_ENTRYPOINT_TEST_BASE_URL: ${{ env.CUSTOM_URL }}
          SIMULATION_ENTRYPOINT_TEST_AUTH_ISSUER: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_ISSUER }}
          SIMULATION_ENTRYPOINT_TEST_AUTH_AUDIENCE: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_AUDIENCE }}
          SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID }}
          SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }}
        run: uv run pytest authenticated_tests/ -v

      - name: Restore previous revision after failed verification
        if: ${{ failure() && steps.promote.outcome == 'success' }}
        env:
          SIMULATION_ENTRYPOINT_GCP_PROJECT_ID: ${{ env.PROJECT_ID }}
          SIMULATION_ENTRYPOINT_GCP_REGION: ${{ env.REGION }}
          SIMULATION_ENTRYPOINT_SERVICE: ${{ env.SERVICE }}
          SIMULATION_ENTRYPOINT_TARGET_REVISION: ${{ env.PREVIOUS_REVISION }}
          SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION: ${{ env.TARGET_REVISION }}
        run: .github/scripts/set-cloud-run-simulation-entry-revision.sh

  deployment_ready:
    name: Require requested deployment to complete
    # Run after failures and expected skips so a required skipped phase cannot
    # make the reusable workflow report a successful partial deployment.
    if: ${{ !cancelled() }}
    needs: [deploy_entrypoint, integration, authenticated_test, promote_entrypoint]
    runs-on: ubuntu-latest
    steps:
      - name: Check deployment result
        env:
          PROMOTION_REQUESTED: ${{ inputs.promote_entrypoint }}
          ENTRYPOINT_RESULT: ${{ needs.deploy_entrypoint.result }}
          INTEGRATION_RESULT: ${{ needs.integration.result }}
          AUTHENTICATED_TEST_RESULT: ${{ needs.authenticated_test.result }}
          PROMOTION_RESULT: ${{ needs.promote_entrypoint.result }}
        run: |
          test "${ENTRYPOINT_RESULT}" = "success"
          test "${INTEGRATION_RESULT}" = "success"
          test "${AUTHENTICATED_TEST_RESULT}" = "success"
          if [[ "${PROMOTION_REQUESTED}" = "true" ]]; then
            test "${PROMOTION_RESULT}" = "success"
          else
            test "${PROMOTION_RESULT}" = "skipped"
          fi

  cleanup_entrypoint_revisions:
    name: Remove superseded Cloud Run revisions
    if: ${{ !cancelled() && inputs.promote_entrypoint && needs.deployment_ready.result == 'success' }}
    needs: [deployment_ready, deploy_entrypoint]
    runs-on: ubuntu-latest
    environment: ${{ inputs.release_environment }}
    permissions:
      contents: read
      id-token: write
    env:
      SIMULATION_ENTRYPOINT_GCP_PROJECT_ID: ${{ vars.SIMULATION_ENTRYPOINT_GCP_PROJECT_ID }}
      SIMULATION_ENTRYPOINT_GCP_REGION: ${{ vars.SIMULATION_ENTRYPOINT_GCP_REGION || 'us-central1' }}
      SIMULATION_ENTRYPOINT_SERVICE: ${{ inputs.entrypoint_service }}
      SIMULATION_ENTRYPOINT_SUCCESSFUL_REVISION: ${{ needs.deploy_entrypoint.outputs.revision }}
    steps:
      - uses: actions/checkout@v6

      - uses: google-github-actions/auth@v2
        with:
          workload_identity_provider: ${{ secrets.SIMULATION_ENTRYPOINT_GCP_WIF_PROVIDER }}
          service_account: ${{ secrets.SIMULATION_ENTRYPOINT_GCP_DEPLOY_SERVICE_ACCOUNT }}

      - uses: google-github-actions/setup-gcloud@v2

      - name: Remove superseded Cloud Run revisions
        run: .github/scripts/cleanup-cloud-run-simulation-entry-revisions.sh