gitops-update.yml

name: "GitOps Update"

# Reusable workflow for updating GitOps repository with new image tags
# Supports multiple clusters and environments: dev (beta), stg (rc), prd (production), sandbox
# Handles single or multiple components with flexible YAML key mapping

on:
  workflow_call:
    inputs:
      gitops_repository:
        description: 'GitOps repository to update (org/repo format). Defaults to the GITOPS_REPOSITORY org-level variable when empty.'
        type: string
        default: ''
      app_name:
        description: 'Application name (defaults to repository name if not provided)'
        type: string
        required: false
      update_sandbox:
        description: 'Include sandbox environment on production tags. Set to false to skip sandbox even on a production release.'
        type: boolean
        default: false
      deploy_in_anacleto:
        description: 'Force-off override for Anacleto. Set to false to suppress deployment even if the manifest includes this app on Anacleto.'
        type: boolean
        default: true
      deployment_matrix_file:
        description: 'Path (within the shared-workflows checkout) to the deployment matrix manifest. Override only if you maintain a fork/alternative manifest.'
        type: string
        default: 'config/deployment-matrix.yml'
      deployment_matrix_ref:
        description: 'Git ref of LerianStudio/github-actions-shared-workflows to read the deployment matrix from. Defaults to main (always latest). Override only when testing a branch.'
        type: string
        default: 'main'
      artifact_pattern:
        description: 'Pattern to download artifacts (defaults to "gitops-tags-<repo-name>*" if not provided, matching both single-app and monorepo artifact names)'
        type: string
        required: false
      yaml_key_mappings:
        description: 'JSON object mapping artifact names to YAML keys (e.g., {"backend.tag": ".auth.image.tag", "frontend.tag": ".frontend.image.tag"}). Required for gitops_layout=helmfile, ignored for gitops_layout=kustomize.'
        type: string
        required: false
        default: ''
      commit_message_prefix:
        description: 'Prefix for commit message (defaults to repository name if not provided)'
        type: string
        required: false
      enable_argocd_sync:
        description: 'Enable ArgoCD sync after GitOps update'
        type: boolean
        default: true
      argocd_prune:
        description: 'Pass --prune to argocd app sync so orphaned resources are cleaned up automatically. Opt-in; safer left disabled in production.'
        type: boolean
        default: false
      use_dynamic_mapping:
        description: 'Use dynamic mapping for multiple components (like midaz)'
        type: boolean
        default: false
      yq_version:
        description: 'Version of yq to install'
        type: string
        default: 'v4.44.3'
      enable_docker_login:
        description: 'Enable Docker Hub login to avoid rate limits. Disabled by default since GitOps updates do not require Docker registry access.'
        type: boolean
        default: false
      configmap_updates:
        description: 'JSON object mapping artifact names to configmap keys (e.g., {"pix.tag": ".pix.configmap.VERSION"}). Helmfile layout only; ignored when gitops_layout=kustomize.'
        type: string
        required: false
      gitops_layout:
        description: 'GitOps layout strategy: "helmfile" (default, current behavior) or "kustomize".'
        type: string
        default: 'helmfile'
      kustomize_base_path:
        description: 'Required when gitops_layout=kustomize. Path within the gitops repo to the kustomization folder (e.g. environments/anacleto/kustomize/ungoliant-controller). Supports ${SERVER} and ${ENV} placeholders.'
        type: string
        default: ''
      kustomize_image_name:
        description: 'Required when gitops_layout=kustomize, unless every image is covered by kustomize_image_mappings. Image reference matched by `kustomize edit set image` (e.g. ghcr.io/lerianstudio/ungoliant-controller). Also the fallback for artifacts absent from kustomize_image_mappings.'
        type: string
        default: ''
      kustomize_image_mappings:
        description: 'JSON object mapping artifact names to kustomize image references (e.g. {"ungoliant-api":"ghcr.io/lerianstudio/ungoliant-api"}). Use when one release publishes more than one image into the same kustomization.yaml — each downloaded artifact is named after the app that built it. Artifacts not listed fall back to kustomize_image_name. Kustomize layout only.'
        type: string
        default: ''
      kustomize_environments:
        description: 'Optional space-separated env list overriding the default tag-based env loop when gitops_layout=kustomize. Leave empty for layouts without env split.'
        type: string
        default: ''
      kustomize_version:
        description: 'Version of kustomize CLI to install (used only when gitops_layout=kustomize).'
        type: string
        default: 'v5.4.3'
      argocd_app_name_template:
        description: 'Template for the ArgoCD application name. Supports placeholders {server}, {app}, {env}. Default {server}-{app}-{env} preserves current behavior. For kustomize layouts without env split, use e.g. {server}-{app}.'
        type: string
        default: '{server}-{app}-{env}'
      argocd_sync_timeout:
        description: 'Timeout in seconds for argocd app wait after sync. Increase for larger applications or clusters under load.'
        type: number
        default: 600
      force_env:
        description: 'Override tag-based environment detection with development|staging|production|sandbox. Set when running on a branch (no tag in GITHUB_REF), e.g. same-run gitops update after a stable release. Empty = detect from the tag.'
        type: string
        default: ''
      beta_environments:
        description: 'Space-separated environments updated by a beta release (develop branch). Default "dev".'
        type: string
        default: 'dev'
      rc_environments:
        description: 'Space-separated environments updated by an rc release (release-candidate branch). Default "stg".'
        type: string
        default: 'stg'
      stable_environments:
        description: 'Space-separated environments updated by a stable release (main branch). Default "prd" so a hotfix does not overwrite features still in dev/stg. Set to "dev stg prd" to refresh lower environments too. Sandbox is controlled separately by update_sandbox.'
        type: string
        default: 'prd'
    outputs:
      anacleto_sync_ok:
        description: 'true if every argocd_sync target with server=="anacleto" synced successfully (or none ran), false if any anacleto target failed. Ignores non-anacleto targets (e.g. benedita) entirely — use to gate steps that only care about the anacleto/chaos-fuzzing environments without being blocked by an unrelated server failure.'
        value: ${{ jobs.aggregate_sync_status.outputs.anacleto_sync_ok }}

jobs:
  update_gitops:
    runs-on: ${{ vars.GITOPS_RUNNERS || 'blacksmith-4vcpu-ubuntu-2404' }}
    outputs:
      app_name: ${{ steps.setup.outputs.app_name }}
      sync_matrix: ${{ steps.apply_tags.outputs.sync_matrix }}
      has_sync_targets: ${{ steps.apply_tags.outputs.has_sync_targets }}
    env:
      GITOPS_REPO: ${{ inputs.gitops_repository || vars.GITOPS_REPOSITORY }}
      IS_RC: ${{ contains(github.ref, '-rc.') }}
      IS_BETA: ${{ contains(github.ref, '-beta.') }}
      IS_SANDBOX: ${{ contains(github.ref, '-sandbox.') }}
      IS_PRODUCTION: ${{ !contains(github.ref, '-rc.') && !contains(github.ref, '-beta.') && !contains(github.ref, '-sandbox.') && startsWith(github.ref, 'refs/tags/') }}
    steps:
      - name: Log in to Docker Hub
        if: ${{ inputs.enable_docker_login }}
        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKERHUB_IMAGE_PULL_TOKEN }}

      # Trust model: this is a `workflow_call` reusable workflow, NOT triggered by
      # untrusted PRs. The gitops repository is resolved from inputs.gitops_repository
      # (caller-supplied) or the GITOPS_REPOSITORY org-level variable, and the
      # MANAGE_TOKEN is required for the subsequent commit/push step. The allowlist
      # step below enforces that the repository is a LerianStudio-owned `*-gitops`
      # repo so a compromised caller cannot redirect the privileged checkout to an
      # arbitrary target (CodeQL: `actions/untrusted-checkout`).
      - name: Validate gitops_repository against allowlist
        shell: bash
        env:
          GITOPS_REPOSITORY: ${{ env.GITOPS_REPO }}
        run: |
          set -euo pipefail
          if [[ -z "$GITOPS_REPOSITORY" ]]; then
            echo "::error::gitops_repository is empty. Set the input or the GITOPS_REPOSITORY org-level variable."
            exit 1
          fi
          if [[ ! "$GITOPS_REPOSITORY" =~ ^LerianStudio/[A-Za-z0-9._-]+-gitops$ ]]; then
            echo "::error::gitops_repository '$GITOPS_REPOSITORY' is not allowed. Must match ^LerianStudio/<name>-gitops$ (e.g. LerianStudio/lerian-internal-gitops)."
            exit 1
          fi
          echo "gitops_repository validated: $GITOPS_REPOSITORY"

      - name: Checkout GitOps Repository
        # codeql[actions/untrusted-checkout] -- env.GITOPS_REPO resolves to a fixed,
        # config-driven LerianStudio GitOps repo (never PR/fork-controlled); this
        # workflow is workflow_call-only, not triggered by pull_request_target.
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
        with:
          repository: ${{ env.GITOPS_REPO }}
          token: ${{ secrets.MANAGE_TOKEN }}
          path: gitops
          fetch-depth: 0

      # Read-only sparse checkout of a single manifest file from this same
      # (LerianStudio-owned) repository — never a caller-controlled repo, no
      # credentials persisted, and the file is only read as data (YAML config),
      # never executed (CodeQL: actions/untrusted-checkout).
      # codeql[actions/untrusted-checkout] -- repository is hardcoded to this
      # same LerianStudio repo; ref is a maintainer-controlled input, not
      # PR/fork-controlled (workflow_call-only, no pull_request_target trigger).
      - name: Checkout deployment matrix manifest
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
        with:
          repository: LerianStudio/github-actions-shared-workflows
          ref: ${{ inputs.deployment_matrix_ref }}
          path: shared-workflows
          sparse-checkout: |
            ${{ inputs.deployment_matrix_file }}
          sparse-checkout-cone-mode: false
          persist-credentials: false

      - name: Setup application name and paths
        id: setup
        shell: bash
        run: |
          # Extract app name from repository if not provided
          if [[ -z "${{ inputs.app_name }}" ]]; then
            APP_NAME="${GITHUB_REPOSITORY##*/}"
          else
            APP_NAME="${{ inputs.app_name }}"
          fi
          echo "app_name=$APP_NAME" >> "$GITHUB_OUTPUT"
          echo "Application name: $APP_NAME"

          # Generate commit message prefix if not provided
          if [[ -z "${{ inputs.commit_message_prefix }}" ]]; then
            COMMIT_PREFIX="$APP_NAME"
          else
            COMMIT_PREFIX="${{ inputs.commit_message_prefix }}"
          fi
          echo "commit_prefix=$COMMIT_PREFIX" >> "$GITHUB_OUTPUT"
          echo "Commit message prefix: $COMMIT_PREFIX"

          # Generate artifact pattern if not provided
          if [[ -z "${{ inputs.artifact_pattern }}" ]]; then
            ARTIFACT_PATTERN="gitops-tags-${APP_NAME}*"
          else
            ARTIFACT_PATTERN="${{ inputs.artifact_pattern }}"
          fi
          echo "artifact_pattern=$ARTIFACT_PATTERN" >> "$GITHUB_OUTPUT"
          echo "Artifact pattern: $ARTIFACT_PATTERN"

      - name: Install yq
        shell: bash
        env:
          YQ_VERSION: ${{ inputs.yq_version }}
        run: |
          set -euo pipefail
          if command -v yq &>/dev/null; then
            echo "yq already installed: $(yq --version)"
          else
            YQ_URL="https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64"
            mkdir -p ~/.local/bin
            curl -sSL "$YQ_URL" -o ~/.local/bin/yq
            chmod +x ~/.local/bin/yq
            echo "$HOME/.local/bin" >> "$GITHUB_PATH"
            export PATH="$HOME/.local/bin:$PATH"
          fi
          yq --version

      - name: Install kustomize
        if: ${{ inputs.gitops_layout == 'kustomize' }}
        shell: bash
        env:
          KUSTOMIZE_VERSION: ${{ inputs.kustomize_version }}
        run: |
          set -euo pipefail
          # The kustomize release tag is "kustomize/vX.Y.Z" — the "/" must be
          # URL-encoded as %2F, and the asset filename keeps the "v" prefix.
          # Stripping "v" or leaving the slash unencoded produces a 404 page,
          # which then fails extraction with "gzip: stdin: not in gzip format".
          VERSION="${KUSTOMIZE_VERSION#v}"
          VERSION="v${VERSION}"
          URL="https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${VERSION}/kustomize_${VERSION}_linux_amd64.tar.gz"
          curl -fsSL "$URL" -o /tmp/kustomize.tar.gz
          tar -xzf /tmp/kustomize.tar.gz -C /tmp
          mkdir -p ~/.local/bin
          install -m 755 /tmp/kustomize ~/.local/bin/kustomize
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
          export PATH="$HOME/.local/bin:$PATH"
          kustomize version

      - name: Validate kustomize inputs
        if: ${{ inputs.gitops_layout == 'kustomize' }}
        shell: bash
        env:
          KUSTOMIZE_BASE_PATH: ${{ inputs.kustomize_base_path }}
          KUSTOMIZE_IMAGE_NAME: ${{ inputs.kustomize_image_name }}
          KUSTOMIZE_IMAGE_MAPPINGS: ${{ inputs.kustomize_image_mappings }}
        run: |
          set -euo pipefail
          if [[ -z "$KUSTOMIZE_BASE_PATH" ]]; then
            echo "::error::gitops_layout=kustomize requires kustomize_base_path."
            exit 1
          fi
          # An empty or null mapping value would resolve to nothing and silently
          # fall back to kustomize_image_name, writing the extra artifact's tag
          # onto the primary image — the exact failure the mapping exists to
          # prevent. Reject it instead of resolving it.
          MAPPED_IMAGES=0
          if [[ -n "$KUSTOMIZE_IMAGE_MAPPINGS" ]]; then
            if ! jq -e 'type == "object" and all(.[]; type == "string" and length > 0)' >/dev/null 2>&1 <<< "$KUSTOMIZE_IMAGE_MAPPINGS"; then
              echo "::error::kustomize_image_mappings must be a JSON object whose every value is a non-empty image reference string."
              exit 1
            fi
            MAPPED_IMAGES=$(jq -r 'length' <<< "$KUSTOMIZE_IMAGE_MAPPINGS")
            echo "kustomize_image_mappings:"
            jq -r 'to_entries[] | "  \(.key) -> \(.value)"' <<< "$KUSTOMIZE_IMAGE_MAPPINGS"
          fi
          if [[ -z "$KUSTOMIZE_IMAGE_NAME" && "$MAPPED_IMAGES" -eq 0 ]]; then
            echo "::error::gitops_layout=kustomize requires kustomize_image_name or a non-empty kustomize_image_mappings."
            exit 1
          fi

      - name: Validate helmfile inputs
        if: ${{ inputs.gitops_layout == 'helmfile' }}
        shell: bash
        env:
          MAPPINGS: ${{ inputs.yaml_key_mappings }}
        run: |
          set -euo pipefail
          if [[ -z "$MAPPINGS" ]]; then
            echo "::error::gitops_layout=helmfile requires yaml_key_mappings."
            exit 1
          fi

      - name: Resolve target clusters from deployment matrix
        id: resolve_clusters
        shell: bash
        env:
          APP_NAME: ${{ steps.setup.outputs.app_name }}
          MANIFEST: shared-workflows/${{ inputs.deployment_matrix_file }}
          DEPLOY_ANACLETO: ${{ inputs.deploy_in_anacleto }}
        run: |
          set -euo pipefail

          if [[ ! -f "$MANIFEST" ]]; then
            echo "::error::Deployment matrix manifest not found: $MANIFEST"
            exit 1
          fi

          echo "Reading deployment matrix: $MANIFEST"
          echo "App name: $APP_NAME"

          # Resolve clusters whose `apps:` list contains $APP_NAME.
          # Convert manifest to JSON via yq, then query with jq (more portable than yq's
          # native env() — and jq is preinstalled on the runner anyway).
          #
          # NOTE: We use `index($app)` instead of `contains([$app])` because jq's
          # contains() does substring matching on strings — "plugin-br-bank-transfer"
          # would falsely match "plugin-br-bank-transfer-jd". `index()` is exact equality.
          # Output one cluster per line, sorted for determinism.
          # Do not swallow yq/jq errors with `|| true` — a corrupt manifest or
          # broken query should fail fast, not be confused with "app not registered".
          # Pipefail ensures intermediate failures surface; an empty RESOLVED from
          # a *successful* query is the legitimate "no matching clusters" case and
          # is handled below.
          RESOLVED=$(yq -o=json '.' "$MANIFEST" \
            | jq -r --arg app "$APP_NAME" \
              '.clusters | to_entries[] | select(.value.apps // [] | index($app)) | .key' \
            | sort -u)

          if [[ -z "$RESOLVED" ]]; then
            echo "::warning::App '$APP_NAME' is not registered in any cluster of the deployment matrix."
            echo "If this is a new app, add it to $MANIFEST in shared-workflows."
            echo "clusters=" >> "$GITHUB_OUTPUT"
            echo "has_clusters=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          echo "Manifest resolution for '$APP_NAME':"
          while IFS= read -r line; do
            echo "  - $line"
          done <<< "$RESOLVED"

          # Apply force-off overrides from inputs.
          # An input set to "false" removes that cluster from the resolved set,
          # even if the manifest includes the app on it.
          FILTERED=""
          while IFS= read -r cluster; do
            [[ -z "$cluster" ]] && continue
            case "$cluster" in
              anacleto)
                if [[ "$DEPLOY_ANACLETO" == "false" ]]; then
                  echo "  ✗ anacleto — suppressed by deploy_in_anacleto=false"
                  continue
                fi
                ;;
            esac
            FILTERED="${FILTERED:+$FILTERED }$cluster"
          done <<< "$RESOLVED"

          if [[ -z "$FILTERED" ]]; then
            echo "::warning::All clusters resolved from the manifest were suppressed by force-off inputs."
            echo "clusters=" >> "$GITHUB_OUTPUT"
            echo "has_clusters=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          echo ""
          echo "Final cluster set: $FILTERED"
          echo "clusters=$FILTERED" >> "$GITHUB_OUTPUT"
          echo "has_clusters=true" >> "$GITHUB_OUTPUT"

          # Emit a JSON map of per-cluster env_suffixes / suffix_excludes_envs for
          # downstream steps. Defaults preserve the pre-multi-variant behavior:
          # - env_suffixes defaults to [""] (one expansion, empty suffix)
          # - suffix_excludes_envs defaults to []
          # Clusters that did not declare these fields therefore produce the exact
          # same env list as before (e.g. dev/stg/prd/sandbox for a prod tag).
          # Build a JSON array of resolved cluster names for jq. Splitting on the
          # known single-space delimiter (FILTERED is built with " " separators
          # above), then feeding each element to jq -R explicitly avoids the need
          # to rely on word-splitting via `$FILTERED` (shellcheck SC2086).
          CLUSTERS_JSON=$(printf '%s' "$FILTERED" | tr ' ' '\n' | jq -R . | jq -s .)
          ENV_CONFIG=$(yq -o=json '.' "$MANIFEST" \
            | jq -c --argjson clusters "$CLUSTERS_JSON" '
                .clusters
                | with_entries(select(.key as $k | $clusters | index($k)))
                | with_entries(.value = {
                    env_suffixes: (.value.env_suffixes // [""]),
                    suffix_excludes_envs: (.value.suffix_excludes_envs // []),
                    env_contexts: (.value.env_contexts // [])
                  })
              ')
          echo "env_config=$ENV_CONFIG" >> "$GITHUB_OUTPUT"
          echo "Per-cluster env config: $ENV_CONFIG"

      - name: Git pull before update
        shell: bash
        run: |
          set -e
          cd gitops || exit 1
          git pull origin main

      - name: Cleanup old artifacts (self-hosted runner safety)
        shell: bash
        run: |
          # Remove any existing artifacts from previous runs (important for self-hosted runners)
          echo "Cleaning up old artifacts directory..."
          rm -rf .gitops-tags
          echo "Cleanup complete"

      - name: Download GitOps tag artifacts (pattern-based)
        id: download-pattern
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          pattern: ${{ steps.setup.outputs.artifact_pattern }}
          path: .gitops-tags
          merge-multiple: true
          # Only download from this workflow run (default behavior, but explicit for safety)
          run-id: ${{ github.run_id }}
        continue-on-error: true

      - name: Fallback to legacy artifact name
        if: steps.download-pattern.outcome == 'failure'
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: gitops-tags
          path: .gitops-tags
          run-id: ${{ github.run_id }}

      - name: Validate artifacts exist
        id: validate_artifacts
        shell: bash
        run: |
          echo "Checking for downloaded artifacts..."

          # Check if directory exists and has files
          if [[ ! -d ".gitops-tags" ]]; then
            echo "::error::No artifacts directory found. Build job may have been skipped or failed to produce artifacts."
            echo "artifacts_valid=false" >> "$GITHUB_OUTPUT"
            exit 1
          fi

          # Count actual artifact files (not directories)
          ARTIFACT_COUNT=$(find .gitops-tags -type f -name "*.tag" 2>/dev/null | wc -l || echo "0")

          if [[ "$ARTIFACT_COUNT" -eq 0 ]]; then
            echo "::error::No artifact files found in .gitops-tags/. Build job may have been skipped or failed."
            echo "Directory contents:"
            ls -laR .gitops-tags/ 2>/dev/null || echo "Directory is empty or doesn't exist"
            echo "artifacts_valid=false" >> "$GITHUB_OUTPUT"
            exit 1
          fi

          echo "Found $ARTIFACT_COUNT artifact file(s):"
          ls -la .gitops-tags/
          echo "artifacts_valid=true" >> "$GITHUB_OUTPUT"

      - name: Apply tags to values.yaml (multi-server)
        id: apply_tags
        shell: bash
        env:
          # Pass through env to avoid `${{ ... }}` interpolation directly into the
          # script body (CodeQL: actions/code-injection). IS_* are inherited from job env.
          APP_NAME: ${{ steps.setup.outputs.app_name }}
          HAS_CLUSTERS: ${{ steps.resolve_clusters.outputs.has_clusters }}
          RESOLVED_SERVERS: ${{ steps.resolve_clusters.outputs.clusters }}
          ENV_CONFIG: ${{ steps.resolve_clusters.outputs.env_config }}
          MAPPINGS: ${{ inputs.yaml_key_mappings }}
          CONFIGMAP_MAPPINGS: ${{ inputs.configmap_updates }}
          GITOPS_LAYOUT: ${{ inputs.gitops_layout }}
          KUSTOMIZE_BASE_PATH: ${{ inputs.kustomize_base_path }}
          KUSTOMIZE_IMAGE_NAME: ${{ inputs.kustomize_image_name }}
          KUSTOMIZE_IMAGE_MAPPINGS: ${{ inputs.kustomize_image_mappings }}
          KUSTOMIZE_ENVIRONMENTS: ${{ inputs.kustomize_environments }}
          MANIFEST_FILE: shared-workflows/${{ inputs.deployment_matrix_file }}
          UPDATE_SANDBOX: ${{ inputs.update_sandbox }}
          FORCE_ENV: ${{ inputs.force_env }}
          BETA_ENVIRONMENTS: ${{ inputs.beta_environments }}
          RC_ENVIRONMENTS: ${{ inputs.rc_environments }}
          STABLE_ENVIRONMENTS: ${{ inputs.stable_environments }}
        run: |
          set -euo pipefail

          # Determine environments to update. force_env overrides tag-based detection
          # (used on branch-push runs where GITHUB_REF has no tag); otherwise IS_* from job env.
          # Per-release-type env targets are configurable via inputs
          # (beta/rc/stable_environments). Defaults keep each release scoped to
          # its own environment (beta->dev, rc->stg, stable->prd) so a stable
          # hotfix does not overwrite features still living in dev/stg. Sandbox
          # is appended to stable/production only when update_sandbox=true.
          # RELEASE_TYPE is the normalized release kind (beta|rc|stable|sandbox),
          # used as the lookup key for the manifest's per-app app_extra_envs.
          # ENV_LABEL stays as-is — it is the human-facing commit message label.
          if [[ -n "$FORCE_ENV" ]]; then
            case "$FORCE_ENV" in
              production)
                ENVIRONMENTS="$STABLE_ENVIRONMENTS"
                if [[ "$UPDATE_SANDBOX" == "true" ]]; then
                  ENVIRONMENTS="$ENVIRONMENTS sandbox"
                fi
                ENV_LABEL="production"
                RELEASE_TYPE="stable"
                ;;
              staging)     ENVIRONMENTS="$RC_ENVIRONMENTS"; ENV_LABEL="rc/stg"; RELEASE_TYPE="rc" ;;
              development) ENVIRONMENTS="$BETA_ENVIRONMENTS"; ENV_LABEL="beta/dev"; RELEASE_TYPE="beta" ;;
              sandbox)     ENVIRONMENTS="sandbox"; ENV_LABEL="sandbox"; RELEASE_TYPE="sandbox" ;;
              *)
                echo "::error::Invalid force_env '$FORCE_ENV' (must be: development, staging, production, sandbox)"
                exit 1
                ;;
            esac
          elif [[ "$IS_BETA" == "true" ]]; then
            ENVIRONMENTS="$BETA_ENVIRONMENTS"
            ENV_LABEL="beta/dev"
            RELEASE_TYPE="beta"
          elif [[ "$IS_RC" == "true" ]]; then
            ENVIRONMENTS="$RC_ENVIRONMENTS"
            ENV_LABEL="rc/stg"
            RELEASE_TYPE="rc"
          elif [[ "$IS_PRODUCTION" == "true" ]]; then
            ENVIRONMENTS="$STABLE_ENVIRONMENTS"
            if [[ "$UPDATE_SANDBOX" == "true" ]]; then
              ENVIRONMENTS="$ENVIRONMENTS sandbox"
            fi
            ENV_LABEL="production"
            RELEASE_TYPE="stable"
          elif [[ "$IS_SANDBOX" == "true" ]]; then
            ENVIRONMENTS="sandbox"
            ENV_LABEL="sandbox"
            RELEASE_TYPE="sandbox"
          else
            echo "Unable to detect environment from tag: ${GITHUB_REF}"
            exit 1
          fi
          echo "env_label=$ENV_LABEL" >> "$GITHUB_OUTPUT"
          echo "Detected tag type: $ENV_LABEL (release_type: $RELEASE_TYPE)"
          echo "Default environments (helmfile/tag-based): $ENVIRONMENTS"

          # Kustomize layout overrides the env loop based on user input or path placeholders.
          if [[ "$GITOPS_LAYOUT" == "kustomize" ]]; then
            if [[ -n "$KUSTOMIZE_ENVIRONMENTS" ]]; then
              ENVIRONMENTS="$KUSTOMIZE_ENVIRONMENTS"
              echo "Kustomize: env loop overridden via kustomize_environments: $ENVIRONMENTS"
            else
              # Detect literal "${ENV}" placeholder in the path. Build the needle
              # by concatenating to avoid SC2016 (single-quoted "${ENV}" looks like
              # an unexpanded expression to shellcheck).
              ENV_NEEDLE='$''{ENV}'
              if [[ "$KUSTOMIZE_BASE_PATH" != *"$ENV_NEEDLE"* ]]; then
                ENVIRONMENTS="-"
                echo "Kustomize: no \${ENV} placeholder in path, running once per server (sentinel env '-')"
              else
                echo "Kustomize: \${ENV} placeholder present, using tag-based env list"
              fi
            fi
          fi

          # Validate app_extra_envs before touching any gitops file.
          #
          # This runs in the main shell on purpose: the per-server lookup below
          # happens inside a command substitution, where a failure would kill
          # only the subshell and be silently swallowed — leaving the release
          # green while the requested extra env was never updated, which is the
          # precise failure the feature exists to prevent. A bad shape is still
          # valid YAML, so `resolve_clusters` does not catch it either.
          #
          # This deliberately mirrors the full contract enforced by
          # src/lint/deployment-matrix, rather than trusting it: the lint only
          # ever sees manifests in this repo, and deployment_matrix_ref lets a
          # caller read the matrix from an unlinted ref. So the structure AND
          # the env names are checked here — a name with whitespace would split
          # into two envs in the space-separated env list, and one with ".." or
          # a leading "/" would push VALUES_FILE outside applications/.
          if [[ "$GITOPS_LAYOUT" != "kustomize" && -f "$MANIFEST_FILE" ]]; then
            if ! yq -o=json '.' "$MANIFEST_FILE" \
              | jq -e '
                  [ (.clusters // {})[] | .app_extra_envs // {} ]
                  | all(
                      type == "object"
                      and all(.[];
                        type == "object"
                        and all(.[];
                          type == "array"
                          and length > 0
                          and all(.[];
                            type == "string"
                            and test("^[A-Za-z0-9._/-]+$")
                            and (startswith("/") | not)
                            and (split("/") | all(. != "" and . != "." and . != ".."))
                          )
                        )
                      )
                    )' >/dev/null 2>&1; then
              echo "::error::Invalid app_extra_envs in ${MANIFEST_FILE}. Expected clusters.<server>.app_extra_envs.<app>.<release_type> to be a non-empty list of relative env names matching [A-Za-z0-9._/-]+ with no '.' or '..' path components."
              exit 1
            fi
          fi

          # Relationship checks for the app being released. Two ways an entry
          # can be silently ignored, both ending with a green release and the
          # requested env untouched:
          #   - an unknown release type ("beta_typo") never matches RELEASE_TYPE
          #   - an app_helmfile_env override replaces the tag-derived env for
          #     every iteration, so the extra env is visited but written to the
          #     override path instead
          #
          # Deliberately narrower than the structural check above, along two
          # axes, because failing a release over an entry that this run never
          # consults is collateral damage:
          #   - by app: the lookup is keyed by app name, so another app's bad
          #     entry is inert here
          #   - by cluster: restricted to RESOLVED_SERVERS, the post-filter
          #     cluster set, so an entry under a cluster that does not host this
          #     app, or one suppressed via deploy_in_<cluster>=false, is inert too
          # The structural check stays manifest-wide by contrast: a malformed
          # file is malformed regardless of who is releasing or where.
          if [[ "$GITOPS_LAYOUT" != "kustomize" && -f "$MANIFEST_FILE" && -n "$RESOLVED_SERVERS" ]]; then
            RESOLVED_SERVERS_JSON=$(printf '%s' "$RESOLVED_SERVERS" | tr ' ' '\n' | jq -R . | jq -s -c .)
            if ! yq -o=json '.' "$MANIFEST_FILE" \
              | jq -e --arg app "$APP_NAME" --argjson servers "$RESOLVED_SERVERS_JSON" '
                  [ (.clusters // {}) | to_entries[]
                    | select(.key as $k | $servers | index($k))
                    | .value
                    | select((.app_extra_envs // {}) | has($app))
                    | { types: (.app_extra_envs[$app] | keys),
                        override: ((.app_helmfile_env // {}) | has($app)) } ]
                  | all(
                      (.types | all(. == "beta" or . == "rc" or . == "stable" or . == "sandbox"))
                      and (.override | not)
                    )' >/dev/null 2>&1; then
              echo "::error::Invalid app_extra_envs for '${APP_NAME}' in ${MANIFEST_FILE}, on one of the target clusters (${RESOLVED_SERVERS}). Release types must be one of beta|rc|stable|sandbox, and the app must not also have an app_helmfile_env override on the same cluster (the override would suppress the extra envs)."
              exit 1
            fi
          fi

          # Servers come from the deployment matrix manifest, filtered by force-off inputs.
          # See step `resolve_clusters` for resolution logic.
          if [[ "$HAS_CLUSTERS" != "true" ]]; then
            echo "No clusters selected for deployment (manifest empty for this app, or all clusters suppressed)."
            echo "sync_matrix=[]" >> "$GITHUB_OUTPUT"
            echo "has_sync_targets=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          SERVERS="$RESOLVED_SERVERS"
          echo "Servers to deploy to: $SERVERS"

          # First, check which artifacts actually exist
          echo ""
          echo "Checking available artifacts..."
          AVAILABLE_ARTIFACTS=""
          for artifact_file in .gitops-tags/*; do
            if [[ -f "$artifact_file" ]]; then
              artifact_name=$(basename "$artifact_file")
              echo "  Found artifact: $artifact_name"
              AVAILABLE_ARTIFACTS="${AVAILABLE_ARTIFACTS}${artifact_name} "
            fi
          done

          if [[ -z "$AVAILABLE_ARTIFACTS" ]]; then
            echo "No artifacts found in .gitops-tags/"
            echo "sync_matrix=[]" >> "$GITHUB_OUTPUT"
            echo "has_sync_targets=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          # Track results
          UPDATED_FILES=""
          MISSING_FILES=""
          UPDATED_SERVERS_ENVS=""

          # Function to update a single YAML key using sed (preserves formatting)
          update_yaml_tag() {
            local file="$1"
            local yaml_path="$2"
            local new_tag="$3"

            # Extract the key name from yaml path (e.g., ".image.tag" -> "tag")
            local key_name="${yaml_path##*.}"

            # Use sed to replace only the tag value, preserving all formatting
            # This handles patterns like "tag: v1.0.0" or "tag: 'v1.0.0'" or 'tag: "v1.0.0"'
            if grep -q "^[[:space:]]*${key_name}:" "$file"; then
              # Replace the value after the key, preserving indentation and quotes
              sed -i -E "s/^([[:space:]]*${key_name}:[[:space:]]*)(['\"]?)([^'\"[:space:]]+)(['\"]?)$/\1\2${new_tag}\4/" "$file"
              return 0
            fi
            return 1
          }

          # Expand the tag-derived envs for a given server against the cluster's
          # env_suffixes and env_contexts,
          # except for envs listed in suffix_excludes_envs which stay bare.
          # When env_contexts is set, each expansion is prefixed: {context}/{env-variant}.
          # Falls back to a single empty-suffix expansion when env_config is empty
          # (e.g. kustomize layout with sentinel env "-", or clusters without the new fields).
          expand_suffixes_for_server() {
            local server="$1"
            local base_envs="$2"
            # Kustomize sentinel env "-" or empty config → return base_envs unchanged.
            if [[ -z "$ENV_CONFIG" || "$base_envs" == "-" ]]; then
              echo "$base_envs"
              return
            fi
            local suffixes excludes contexts
            suffixes=$(echo "$ENV_CONFIG" | jq -r --arg s "$server" '.[$s].env_suffixes // [""] | .[]' 2>/dev/null || echo "")
            excludes=$(echo "$ENV_CONFIG" | jq -r --arg s "$server" '.[$s].suffix_excludes_envs // [] | .[]' 2>/dev/null || echo "")
            contexts=$(echo "$ENV_CONFIG" | jq -r --arg s "$server" '.[$s].env_contexts // [] | .[]' 2>/dev/null || echo "")
            if [[ -z "$suffixes" && -z "$contexts" ]]; then
              echo "$base_envs"
              return
            fi
            local out=""
            for base_env in $base_envs; do
              # Expand base_env × suffixes (or keep bare if excluded).
              local expanded=""
              if echo "$excludes" | grep -qx "$base_env" 2>/dev/null; then
                expanded="$base_env"
              else
                while IFS= read -r suffix; do
                  expanded="${expanded:+$expanded }${base_env}${suffix}"
                done <<< "$suffixes"
              fi
              # Apply contexts: prefix each expanded variant with "{context}/".
              # When no contexts are declared, emit variants as-is.
              if [[ -n "$contexts" ]]; then
                while IFS= read -r ctx; do
                  for variant in $expanded; do
                    out="${out:+$out }${ctx}/${variant}"
                  done
                done <<< "$contexts"
              else
                out="${out:+$out }$expanded"
              fi
            done
            echo "$out"
          }

          # Per-app, per-release-type extra envs read from the manifest at
          # clusters.<server>.app_extra_envs.<app>.<release_type>. These are
          # literal env names that already carry whatever suffix/context the
          # cluster uses (e.g. "stg-mt") — they are appended verbatim and never
          # re-expanded. This is the exception mechanism for apps whose variant
          # lives in an env the tag-derived list cannot reach: benedita has no
          # dev-mt, so a beta release must target stg-mt directly.
          # Ignored for kustomize layouts, which drive their env loop through
          # kustomize_environments / the ${ENV} placeholder instead.
          extra_envs_for_server() {
            local server="$1"
            if [[ "$GITOPS_LAYOUT" == "kustomize" || ! -f "$MANIFEST_FILE" ]]; then
              return
            fi
            # A missing field is already the empty case (`// []`), so the only
            # way this query fails is a malformed app_extra_envs shape — and that
            # is rejected up-front by the validation step above, which is where
            # it has to happen: this function is always called from inside a
            # command substitution, so a `set -e` abort here would kill only the
            # subshell and be swallowed by the caller.
            yq -o=json '.' "$MANIFEST_FILE" \
              | jq -r --arg server "$server" --arg app "$APP_NAME" --arg rt "$RELEASE_TYPE" \
                '.clusters[$server].app_extra_envs[$app][$rt] // [] | .[]' \
              | tr '\n' ' '
          }

          # Final env list for a server: suffix/context expansion, then the
          # per-app extras unioned in. Expansion order is preserved and
          # duplicates are dropped, so an extra env that the tag already
          # produced (e.g. stg-mt on an rc release) is not processed twice.
          expand_envs_for_server() {
            local server="$1"
            local base_envs="$2"
            local expanded extras env_name out=""
            expanded=$(expand_suffixes_for_server "$server" "$base_envs")
            extras=$(extra_envs_for_server "$server")
            for env_name in $expanded $extras; do
              [[ " $out " == *" $env_name "* ]] && continue
              out="${out:+$out }$env_name"
            done
            echo "$out"
          }

          # Process each server and environment
          for SERVER in $SERVERS; do
            SERVER_ENVS=$(expand_envs_for_server "$SERVER" "$ENVIRONMENTS")
            if [[ "$SERVER_ENVS" != "$ENVIRONMENTS" ]]; then
              echo ""
              echo "Server $SERVER env expansion: '$ENVIRONMENTS' -> '$SERVER_ENVS'"
            fi
            for ENV in $SERVER_ENVS; do

              if [[ "$GITOPS_LAYOUT" == "kustomize" ]]; then
                # Resolve kustomize target dir (substitute ${SERVER} / ${ENV} placeholders).
                TARGET_DIR="$KUSTOMIZE_BASE_PATH"
                TARGET_DIR="${TARGET_DIR//\$\{SERVER\}/$SERVER}"
                TARGET_DIR="${TARGET_DIR//\$\{ENV\}/$ENV}"
                KUSTOMIZATION_FILE="gitops/${TARGET_DIR}/kustomization.yaml"

                echo ""
                echo "Processing: $SERVER/$ENV (kustomize)"
                echo "  Path: $KUSTOMIZATION_FILE"

                if [[ ! -f "$KUSTOMIZATION_FILE" ]]; then
                  echo "  WARNING: kustomization.yaml not found: $KUSTOMIZATION_FILE"
                  MISSING_FILES="${MISSING_FILES}${SERVER}/${ENV}:${KUSTOMIZATION_FILE}\n"
                  continue
                fi

                FILE_CHANGED=false
                FILE_BEFORE_HASH=$(sha256sum "$KUSTOMIZATION_FILE" | cut -d' ' -f1)

                # Apply each available artifact tag via `kustomize edit set image`.
                # Each artifact file is named after the app that built it
                # (build.yml writes gitops-tags/<app>.tag), so that basename is the
                # key used to resolve which image the tag belongs to:
                #   kustomize_image_mappings[<app>], falling back to
                #   kustomize_image_name when the app is not listed.
                # A release publishing more than one image into the same
                # kustomization.yaml maps every extra app explicitly; single-image
                # callers leave the mapping empty and every artifact resolves to
                # kustomize_image_name, exactly as before.
                for artifact_file in .gitops-tags/*; do
                  [[ -f "$artifact_file" ]] || continue
                  TAG="$(cut -d= -f2 < "$artifact_file" | tr -d '[:space:]')"
                  [[ -n "$TAG" ]] || continue

                  ARTIFACT_APP="$(basename "$artifact_file")"
                  ARTIFACT_APP="${ARTIFACT_APP%.tag}"

                  IMAGE_NAME=""
                  if [[ -n "$KUSTOMIZE_IMAGE_MAPPINGS" ]]; then
                    IMAGE_NAME=$(jq -r --arg app "$ARTIFACT_APP" '.[$app] // empty' <<< "$KUSTOMIZE_IMAGE_MAPPINGS")
                  fi
                  if [[ -z "$IMAGE_NAME" ]]; then
                    IMAGE_NAME="$KUSTOMIZE_IMAGE_NAME"
                  fi
                  if [[ -z "$IMAGE_NAME" ]]; then
                    echo "::warning::No kustomize image resolved for artifact '${ARTIFACT_APP}' — not in kustomize_image_mappings and kustomize_image_name is empty. Skipping."
                    continue
                  fi

                  echo "    ${ARTIFACT_APP}: kustomize edit set image ${IMAGE_NAME}=${IMAGE_NAME}:${TAG}"
                  ( cd "gitops/${TARGET_DIR}" && kustomize edit set image "${IMAGE_NAME}=${IMAGE_NAME}:${TAG}" )
                done

                FILE_AFTER_HASH=$(sha256sum "$KUSTOMIZATION_FILE" | cut -d' ' -f1)
                if [[ "$FILE_BEFORE_HASH" != "$FILE_AFTER_HASH" ]]; then
                  FILE_CHANGED=true
                  UPDATED_FILES="${UPDATED_FILES}${KUSTOMIZATION_FILE}\n"
                  UPDATED_SERVERS_ENVS="${UPDATED_SERVERS_ENVS}${SERVER}:${ENV}\n"
                  echo "  Updated kustomization.yaml"
                else
                  echo "  No changes needed for this kustomization.yaml"
                fi

                if [[ -n "$CONFIGMAP_MAPPINGS" ]]; then
                  echo "::warning::configmap_updates is ignored when gitops_layout=kustomize (out of scope for v1)."
                fi

                continue
              fi

              # Check for a per-app helmfile_env override in the deployment matrix.
              # Some apps (e.g. rosiehr, cs-platform, forge, lerian-map on Benedita) have
              # their values.yaml under cross/ instead of the tag-derived env (dev/stg/prd).
              # The override is declared as clusters.<server>.app_helmfile_env.<app>: <env>.
              HELMFILE_ENV="$ENV"
              if [[ -f "$MANIFEST_FILE" ]]; then
                HELMFILE_ENV_OVERRIDE=$(yq -o=json '.' "$MANIFEST_FILE" \
                  | jq -r --arg server "$SERVER" --arg app "$APP_NAME" \
                    '.clusters[$server].app_helmfile_env[$app] // empty' 2>/dev/null || echo "")
                if [[ -n "$HELMFILE_ENV_OVERRIDE" ]]; then
                  HELMFILE_ENV="$HELMFILE_ENV_OVERRIDE"
                fi
              fi

              VALUES_FILE="gitops/environments/${SERVER}/helmfile/applications/${HELMFILE_ENV}/${APP_NAME}/values.yaml"

              echo ""
              echo "Processing: $SERVER/$ENV"
              if [[ "$HELMFILE_ENV" != "$ENV" ]]; then
                echo "  helmfile_env override: $HELMFILE_ENV (tag-derived: $ENV)"
              fi
              echo "  Path: $VALUES_FILE"

              # Check if file exists
              if [[ ! -f "$VALUES_FILE" ]]; then
                echo "  WARNING: Values file not found for ${SERVER}/${ENV}: ${VALUES_FILE}"
                MISSING_FILES="${MISSING_FILES}${SERVER}/${ENV}:${VALUES_FILE}\n"
                continue
              fi

              # Track if any changes were made to this file
              FILE_CHANGED=false

              # Apply mappings from inputs - only if artifact exists
              # MAPPINGS is set in step env: (avoids code-injection from inputs)
              while IFS='|' read -r artifact_key yaml_key; do
                ARTIFACT_FILE=".gitops-tags/${artifact_key}"
                if [[ -f "$ARTIFACT_FILE" ]]; then
                  TAG="$(cut -d= -f2 < "$ARTIFACT_FILE" | tr -d '[:space:]')"
                  if [[ -n "$TAG" ]]; then
                    # Get current value to check if update is needed
                    CURRENT_TAG=$(yq e "$yaml_key" "$VALUES_FILE" 2>/dev/null || echo "")
                    if [[ "$CURRENT_TAG" != "$TAG" ]]; then
                      # Use yq with explicit output to preserve formatting
                      TAG="$TAG" yq e "$yaml_key = strenv(TAG)" "$VALUES_FILE" > "${VALUES_FILE}.tmp"
                      mv "${VALUES_FILE}.tmp" "$VALUES_FILE"
                      echo "    Updated $yaml_key: $CURRENT_TAG -> $TAG"
                      FILE_CHANGED=true
                    else
                      echo "    Skipped $yaml_key: already set to $TAG"
                    fi
                  fi
                fi
              done < <(echo "$MAPPINGS" | jq -r 'to_entries[] | "\(.key)|\(.value)"')

              # Apply configmap updates if configured - only if artifact exists
              # CONFIGMAP_MAPPINGS is set in step env: (avoids code-injection from inputs)
              if [[ -n "$CONFIGMAP_MAPPINGS" ]]; then
                while IFS='|' read -r artifact_key configmap_key; do
                  ARTIFACT_FILE=".gitops-tags/${artifact_key}"
                  if [[ -f "$ARTIFACT_FILE" ]]; then
                    TAG="$(cut -d= -f2 < "$ARTIFACT_FILE" | tr -d '[:space:]')"
                    if [[ -n "$TAG" ]]; then
                      CURRENT_TAG=$(yq e "$configmap_key" "$VALUES_FILE" 2>/dev/null || echo "")
                      if [[ "$CURRENT_TAG" != "$TAG" ]]; then
                        TAG="$TAG" yq e "$configmap_key = strenv(TAG)" "$VALUES_FILE" > "${VALUES_FILE}.tmp"
                        mv "${VALUES_FILE}.tmp" "$VALUES_FILE"
                        echo "    Updated $configmap_key: $CURRENT_TAG -> $TAG"
                        FILE_CHANGED=true
                      else
                        echo "    Skipped $configmap_key: already set to $TAG"
                      fi
                    fi
                  fi
                done < <(echo "$CONFIGMAP_MAPPINGS" | jq -r 'to_entries[] | "\(.key)|\(.value)"')
              fi

              # Only track as updated if changes were actually made.
              # When app_helmfile_env overrides the env (e.g. forge: cross), the
              # sync target's env must use the override (HELMFILE_ENV), not the
              # tag-derived env, so the ArgoCD app name resolves correctly
              # (e.g. benedita-forge-cross, not benedita-forge-dev-st).
              if [[ "$FILE_CHANGED" == "true" ]]; then
                UPDATED_FILES="${UPDATED_FILES}${VALUES_FILE}\n"
                UPDATED_SERVERS_ENVS="${UPDATED_SERVERS_ENVS}${SERVER}:${HELMFILE_ENV}\n"
              else
                echo "  No changes needed for this file"
              fi
            done
          done

          # Report results
          echo ""
          echo "========================================="
          echo "Summary:"
          echo "========================================="
          if [[ -n "$UPDATED_FILES" ]]; then
            echo "Updated files:"
            echo -e "$UPDATED_FILES"
          else
            echo "No files were updated"
          fi

          if [[ -n "$MISSING_FILES" ]]; then
            echo ""
            echo "Missing files (skipped):"
            echo -e "$MISSING_FILES"
          fi

          # Save updated servers/envs for ArgoCD sync - only include files that actually changed.
          # Deduplicate (server:env) pairs: cross apps are visited once per outer env
          # iteration (e.g. 7 times on benedita for a prod tag) but always write to
          # the same override path, so a single sync target is correct.
          if [[ -n "$UPDATED_SERVERS_ENVS" ]]; then
            echo -e "$UPDATED_SERVERS_ENVS" | grep -v '^$' | sort -u > /tmp/updated_servers_envs.txt

            # Output sync targets as JSON array for matrix job
            SYNC_MATRIX=$(while IFS=: read -r s e; do
              [[ -n "$s" && -n "$e" ]] && echo "{\"server\":\"$s\",\"env\":\"$e\"}"
            done < /tmp/updated_servers_envs.txt | jq -s -c '.')
            echo "sync_matrix=$SYNC_MATRIX" >> "$GITHUB_OUTPUT"
            echo "has_sync_targets=true" >> "$GITHUB_OUTPUT"
            echo "Sync matrix: $SYNC_MATRIX"
          else
            echo "sync_matrix=[]" >> "$GITHUB_OUTPUT"
            echo "has_sync_targets=false" >> "$GITHUB_OUTPUT"
          fi

          COUNT=0
          if [[ -n "$UPDATED_FILES" ]]; then
            COUNT=$(echo -e "$UPDATED_FILES" | grep -c -v '^$' || true)
          fi
          echo "updated_count=$COUNT" >> "$GITHUB_OUTPUT"

      - name: Show git diff
        shell: bash
        run: |
          cd gitops || exit 1
          echo "Changes to be committed:"
          git diff --stat
          echo ""
          echo "Detailed changes:"
          git diff

      - name: Import GPG key
        uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7
        with:
          gpg_private_key: ${{ secrets.LERIAN_CI_CD_USER_GPG_KEY }}
          passphrase: ${{ secrets.LERIAN_CI_CD_USER_GPG_KEY_PASSWORD }}
          git_committer_name: ${{ secrets.LERIAN_CI_CD_USER_NAME }}
          git_committer_email: ${{ secrets.LERIAN_CI_CD_USER_EMAIL }}
          git_config_global: true
          git_user_signingkey: true
          git_commit_gpgsign: true
          workdir: gitops

      - name: Commit & push (GitOps)
        run: |
          set -e
          cd gitops || exit 1

          # Get env label from apply_tags step
          ENV_LABEL="${{ steps.apply_tags.outputs.env_label }}"

          # Check if there are changes to commit
          if git diff --quiet; then
            echo "No changes to commit"
            exit 0
          fi

          git commit -am "ci(${{ steps.setup.outputs.commit_prefix }}): update image tags ($ENV_LABEL)" || echo "No changes to commit"

          # Retry push with rebase and exponential backoff to handle concurrent updates
          MAX_RETRIES=5
          for i in $(seq 1 $MAX_RETRIES); do
            if git push origin main; then
              echo "Push succeeded on attempt $i"
              break
            fi

            if [ "$i" -eq "$MAX_RETRIES" ]; then
              echo "ERROR: Failed to push after $MAX_RETRIES attempts"
              exit 1
            fi

            BACKOFF=$((2 ** i))
            echo "Push failed (attempt $i/$MAX_RETRIES), rebasing and retrying in ${BACKOFF}s..."
            sleep "$BACKOFF"
            git pull --rebase origin main
          done

  # ArgoCD Sync Job - runs in parallel for each server/env combination
  argocd_sync:
    name: ArgoCD Sync (${{ matrix.target.server }}/${{ matrix.target.env }})
    needs: [update_gitops]
    if: needs.update_gitops.outputs.has_sync_targets == 'true' && inputs.enable_argocd_sync
    runs-on: ${{ vars.GITOPS_RUNNERS || 'blacksmith-4vcpu-ubuntu-2404' }}
    strategy:
      fail-fast: false
      matrix:
        target: ${{ fromJson(needs.update_gitops.outputs.sync_matrix) }}
    steps:
      - name: Install ArgoCD CLI
        shell: bash
        run: |
          set -euo pipefail
          if command -v argocd &>/dev/null; then
            echo "argocd already installed: $(argocd version --client --short 2>/dev/null || argocd version --client)"
          else
            mkdir -p ~/.local/bin
            curl -sSL -o ~/.local/bin/argocd https://github.com/argoproj/argo-cd/releases/download/v3.0.6/argocd-linux-amd64
            chmod +x ~/.local/bin/argocd
            echo "$HOME/.local/bin" >> "$GITHUB_PATH"
            export PATH="$HOME/.local/bin:$PATH"
            argocd version --client
          fi

      - name: Resolve ArgoCD application name
        id: resolve_argocd_app
        shell: bash
        env:
          TEMPLATE: ${{ inputs.argocd_app_name_template }}
          SERVER: ${{ matrix.target.server }}
          APP: ${{ needs.update_gitops.outputs.app_name }}
          ENV_NAME: ${{ matrix.target.env }}
        run: |
          set -euo pipefail
          NAME="$TEMPLATE"
          NAME="${NAME//\{server\}/$SERVER}"
          NAME="${NAME//\{app\}/$APP}"
          NAME="${NAME//\{env\}/$ENV_NAME}"
          # Normalize context-prefixed envs: "chaos/dev-st" → "chaos-dev-st".
          NAME="${NAME//\//-}"
          # Collapse "--" sequences (sentinel env "-" produces "--" inside template).
          while [[ "$NAME" == *--* ]]; do NAME="${NAME//--/-}"; done
          # Strip trailing "-" left over when sentinel env "-" lands at the end.
          NAME="${NAME%-}"
          echo "name=$NAME" >> "$GITHUB_OUTPUT"
          echo "Resolved ArgoCD app name: $NAME"

      - name: Sync ArgoCD application
        id: sync
        shell: bash
        env:
          ARGOCD_URL: ${{ vars.ARGOCD_URL }}
          ARGOCD_TOKEN: ${{ secrets.ARGOCD_TOKEN }}
          APP_NAME: ${{ steps.resolve_argocd_app.outputs.name }}
          ARGOCD_PRUNE: ${{ inputs.argocd_prune }}
          ARGOCD_SYNC_TIMEOUT: ${{ inputs.argocd_sync_timeout }}
        run: |
          set -uo pipefail

          PRUNE_FLAG=""
          if [[ "$ARGOCD_PRUNE" == "true" ]]; then
            PRUNE_FLAG="--prune"
          fi

          echo "::group::argocd app get $APP_NAME"
          argocd_rc=0
          argocd app get "$APP_NAME" --server "$ARGOCD_URL" --auth-token "$ARGOCD_TOKEN" --grpc-web || argocd_rc=$?
          echo "::endgroup::"
          if [[ "$argocd_rc" -ne 0 ]]; then
            echo "::warning::Failed to get ArgoCD app '$APP_NAME' (exit $argocd_rc). Skipping sync — check whether the app exists, auth is valid, and the user has 'applications, get' permission."
            echo "sync_result=failure" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          echo "::group::argocd app sync $APP_NAME"
          for attempt in 1 2 3 4 5; do
            if argocd app sync "$APP_NAME" --server "$ARGOCD_URL" --auth-token "$ARGOCD_TOKEN" --grpc-web --async --timeout 180 $PRUNE_FLAG; then
              break
            fi
            if [[ "$attempt" == "5" ]]; then
              echo "::endgroup::"
              echo "::error::Sync failed for $APP_NAME after 5 attempts"
              exit 1
            fi
            echo "Sync attempt $attempt failed, retrying in 30s..."
            sleep 30
          done
          echo "::endgroup::"

          echo "::group::argocd app wait $APP_NAME"
          wait_delay=30
          for attempt in 1 2 3; do
            if argocd app wait "$APP_NAME" --server "$ARGOCD_URL" --auth-token "$ARGOCD_TOKEN" --grpc-web --timeout "$ARGOCD_SYNC_TIMEOUT"; then
              break
            fi
            if [[ "$attempt" == "3" ]]; then
              echo "::endgroup::"
              echo "::error::Timeout waiting for sync completion of $APP_NAME after 3 attempts"
              exit 1
            fi
            echo "Wait attempt $attempt failed, retrying in ${wait_delay}s..."
            sleep "$wait_delay"
            wait_delay=$((wait_delay * 2))
          done
          echo "::endgroup::"
          echo "sync_result=success" >> "$GITHUB_OUTPUT"

      # Per-leg sync result, uploaded so a downstream job can compute a
      # per-server status (e.g. "did every anacleto target sync ok") — a
      # matrix job's own needs.<job>.outputs only ever surfaces one arbitrary
      # leg's value to the caller, never all of them, so this can't be a
      # plain job output.
      - name: Record sync result
        if: always()
        id: record
        shell: bash
        env:
          SERVER: ${{ matrix.target.server }}
          ENV_NAME: ${{ matrix.target.env }}
          # steps.sync.outputs.sync_result is set explicitly by the sync step
          # itself for both its success and its "app not found, skip" exit-0
          # path — steps.sync.outcome alone can't tell those two apart (both
          # report "success" since the step exits 0 either way).
          OUTCOME: ${{ steps.sync.outputs.sync_result || steps.sync.outcome }}
        run: |
          set -euo pipefail
          mkdir -p /tmp/argocd-sync-result
          echo "$OUTCOME" > "/tmp/argocd-sync-result/result.txt"
          echo "$SERVER" > "/tmp/argocd-sync-result/server.txt"
          # Artifact names can't contain "/" (env can be "chaos/dev-mt").
          SAFE_ENV="${ENV_NAME//\//-}"
          echo "safe_env=$SAFE_ENV" >> "$GITHUB_OUTPUT"

      - name: Upload sync result artifact
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v5
        with:
          name: argocd-sync-result-${{ matrix.target.server }}-${{ steps.record.outputs.safe_env }}
          path: /tmp/argocd-sync-result/
          retention-days: 1

  # Aggregate per-target sync results (server=anacleto only) into a single
  # pass/fail output, so callers like ungoliant_release_diff can gate on
  # "anacleto synced ok" without being blocked by an unrelated server's
  # (e.g. benedita) sync failure.
  aggregate_sync_status:
    name: Aggregate ArgoCD Sync Status
    needs: [update_gitops, argocd_sync]
    if: always()
    runs-on: ${{ vars.GITOPS_RUNNERS || 'blacksmith-4vcpu-ubuntu-2404' }}
    permissions:
      contents: read
    outputs:
      anacleto_sync_ok: ${{ steps.aggregate.outputs.anacleto_sync_ok }}
    steps:
      - name: Download all sync result artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          pattern: argocd-sync-result-anacleto-*
          path: .sync-results
          merge-multiple: false
          if-no-artifact-found: ignore

      - name: Aggregate anacleto sync results
        id: aggregate
        shell: bash
        env:
          UPDATE_GITOPS_RESULT: ${{ needs.update_gitops.result }}
          SYNC_MATRIX: ${{ needs.update_gitops.outputs.sync_matrix }}
        run: |
          set -euo pipefail

          # update_gitops is the single config job that produces sync_matrix —
          # its own failure/cancellation is a real blocker regardless of
          # anacleto. needs.argocd_sync.result is deliberately NOT checked
          # here: it rolls up the WHOLE matrix (anacleto + any other server,
          # e.g. benedita), so a non-anacleto leg failing there must not flip
          # this output — that's the entire point of this job existing.
          if [[ "$UPDATE_GITOPS_RESULT" =~ ^(failure|cancelled)$ ]]; then
            echo "update_gitops=$UPDATE_GITOPS_RESULT — treating as not ok."
            echo "anacleto_sync_ok=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          EXPECTED=$(echo "$SYNC_MATRIX" | jq '[.[] | select(.server == "anacleto")] | length')

          if [ "$EXPECTED" -eq 0 ]; then
            echo "No anacleto sync targets were scheduled — nothing to aggregate, treating as ok."
            echo "anacleto_sync_ok=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          OK=true
          RESULT_COUNT=0

          process_result() {
            local result_file="$1" server_file="$2" label="$3"
            if [ ! -f "$result_file" ] || [ ! -f "$server_file" ]; then
              echo "  $label -> missing result.txt/server.txt, treating as not ok"
              OK=false
              return
            fi
            RESULT_COUNT=$((RESULT_COUNT + 1))
            local outcome server
            outcome=$(cat "$result_file")
            server=$(cat "$server_file")
            echo "  $label -> server=$server outcome=$outcome"
            if [ "$outcome" != "success" ]; then
              OK=false
            fi
          }

          # A single matching artifact is downloaded flat, straight into
          # .sync-results/ (no subdirectory) — actions/download-artifact only
          # creates one subdirectory per artifact when there's more than one.
          if [ -f .sync-results/result.txt ]; then
            process_result .sync-results/result.txt .sync-results/server.txt ".sync-results (single artifact)"
          fi

          for dir in .sync-results/*/; do
            [ -d "$dir" ] || continue
            process_result "${dir}result.txt" "${dir}server.txt" "$dir"
          done

          if [ "$RESULT_COUNT" -ne "$EXPECTED" ]; then
            echo "Expected $EXPECTED anacleto result(s) but found $RESULT_COUNT — treating as not ok."
            OK=false
          fi

          [ "$RESULT_COUNT" -gt 0 ] || OK=false
          echo "anacleto_sync_ok=$OK" >> "$GITHUB_OUTPUT"

  # Slack notification
  notify:
    name: Notify
    needs: [update_gitops, argocd_sync]
    if: always()
    uses: ./.github/workflows/slack-notify.yml
    with:
      status: ${{ needs.update_gitops.result == 'failure' && 'failure' || needs.argocd_sync.result == 'failure' && 'failure' || 'success' }}
      workflow_name: "GitOps Update"
      failed_jobs: ${{ needs.update_gitops.result == 'failure' && 'Update GitOps' || '' }}${{ needs.argocd_sync.result == 'failure' && 'ArgoCD Sync' || '' }}
    secrets:
      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}