Taskfile-e2e.yml

version: '3'

vars:
  CTX: '{{.CTX | default "k3d-gitops-reverser-test-e2e"}}'
  CLUSTER_NAME:
    sh: |
      ctx='{{.CTX}}'
      ctx="${ctx#k3d-}"
      ctx="${ctx#kind-}"
      printf '%s' "${ctx}"
  INSTALL_MODE: '{{.INSTALL_MODE | default "config-dir"}}'
  INSTALL_NAME: '{{.INSTALL_NAME | default "gitops-reverser"}}'
  NAMESPACE: '{{.NAMESPACE | default "gitops-reverser"}}'
  HELM_CHART_SOURCE: '{{.HELM_CHART_SOURCE | default "charts/gitops-reverser"}}'
  CONTROLLER_CONTAINER: '{{.CONTROLLER_CONTAINER | default "manager"}}'
  CONTROLLER_DEPLOY_SELECTOR: '{{.CONTROLLER_DEPLOY_SELECTOR | default "app.kubernetes.io/part-of=gitops-reverser"}}'
  CS: '.stamps/cluster/{{.CTX}}'
  IS: '.stamps/image'
  AUDIT_ASSET_SOURCE_DIR: 'test/e2e/cluster/audit'
  AUDIT_ASSET_DIR: '{{.CS}}/audit'
  AUDIT_POLICY_SOURCE: '{{.AUDIT_ASSET_SOURCE_DIR}}/policy.yaml'
  AUDIT_WEBHOOK_BOOTSTRAP_SOURCE: '{{.AUDIT_ASSET_SOURCE_DIR}}/webhook-config.yaml'
  AUDIT_POLICY_PATH: '{{.AUDIT_ASSET_DIR}}/policy.yaml'
  AUDIT_WEBHOOK_CONFIG_PATH: '{{.AUDIT_ASSET_DIR}}/webhook-config.yaml'
  KUBECTL: '{{.KUBECTL | default "kubectl"}}'
  K3D: '{{.K3D | default "k3d"}}'
  HELM: '{{.HELM | default "helm"}}'
  FLUX: '{{.FLUX | default "flux"}}'
  FLUX_OPERATOR: '{{.FLUX_OPERATOR | default "flux-operator"}}'
  KUSTOMIZE: '{{.KUSTOMIZE | default "kustomize"}}'
  CONTAINER_TOOL: '{{.CONTAINER_TOOL | default "docker"}}'
  TASK_BIN: '{{.TASK_BIN | default "task"}}'
  E2E_LOCAL_IMAGE: '{{.E2E_LOCAL_IMAGE | default "gitops-reverser:e2e-local"}}'
  PROJECT_IMAGE:
    sh: |
      if [ -n "${PROJECT_IMAGE:-}" ]; then
        printf '%s' "${PROJECT_IMAGE}"
      else
        printf '%s' '{{.E2E_LOCAL_IMAGE}}'
      fi
  PROJECT_IMAGE_PROVIDED:
    sh: |
      if [ -n "${PROJECT_IMAGE:-}" ] && [ "${PROJECT_IMAGE}" != '{{.E2E_LOCAL_IMAGE}}' ]; then
        printf 'true'
      fi
  PROJECT_IMAGE_REPO:
    sh: |
      ref="${PROJECT_IMAGE:-}"
      if [ -z "${ref}" ]; then
        ref='{{.E2E_LOCAL_IMAGE}}'
      fi
      repo="${ref%:*}"
      tag="${ref##*:}"
      if [ "${repo}" = "${tag}" ]; then
        printf '%s' "${ref}"
      else
        printf '%s' "${repo}"
      fi
  PROJECT_IMAGE_TAG:
    sh: |
      ref="${PROJECT_IMAGE:-}"
      if [ -z "${ref}" ]; then
        ref='{{.E2E_LOCAL_IMAGE}}'
      fi
      repo="${ref%:*}"
      tag="${ref##*:}"
      if [ "${repo}" = "${tag}" ]; then
        printf 'latest'
      else
        printf '%s' "${tag}"
      fi
  IMAGE_DELIVERY_MODE: '{{.IMAGE_DELIVERY_MODE | default "load"}}'
  GITEA_ADMIN_USER: '{{.GITEA_ADMIN_USER | default "giteaadmin"}}'
  GITEA_ADMIN_PASS: '{{.GITEA_ADMIN_PASS | default "giteapassword123"}}'
  GITEA_ORG_NAME: '{{.GITEA_ORG_NAME | default "testorg"}}'
  REPO_NAME: '{{.REPO_NAME | default ""}}'
  CHECKOUT_DIR: '{{.CHECKOUT_DIR | default ""}}'
  E2E_GIT_SECRET_HTTP: '{{.E2E_GIT_SECRET_HTTP | default ""}}'
  E2E_GIT_SECRET_SSH: '{{.E2E_GIT_SECRET_SSH | default ""}}'
  E2E_GIT_SECRET_INVALID: '{{.E2E_GIT_SECRET_INVALID | default ""}}'
  GITEA_PORT: '{{.GITEA_PORT | default "13000"}}'
  PROMETHEUS_PORT: '{{.PROMETHEUS_PORT | default "19090"}}'
  VALKEY_PORT: '{{.VALKEY_PORT | default "16379"}}'
  FLUX_WEB_PORT: '{{.FLUX_WEB_PORT | default "19080"}}'
  DEFAULT_AUDIT_REDIS_ADDR: '{{.DEFAULT_AUDIT_REDIS_ADDR | default "valkey:6379"}}'
  E2E_AUDIT_REDIS_ADDR: '{{.E2E_AUDIT_REDIS_ADDR | default "valkey.valkey-e2e.svc.cluster.local:6379"}}'
  E2E_VALKEY_PASSWORD: '{{.E2E_VALKEY_PASSWORD | default "e2e-valkey-password"}}'
  E2E_CONTROLLER_SERVICE_CLUSTER_IP: '{{.E2E_CONTROLLER_SERVICE_CLUSTER_IP | default "10.43.200.200"}}'
  FLUX_NAMESPACE: '{{.FLUX_NAMESPACE | default "flux-system"}}'
  FLUX_INSTANCE_NAME: '{{.FLUX_INSTANCE_NAME | default "flux"}}'
  FLUX_INSTANCE_FILE: '{{.FLUX_INSTANCE_FILE | default "test/e2e/setup/flux-operator/flux-instance.yaml"}}'
  FLUX_OPERATOR_DIR: '{{.FLUX_OPERATOR_DIR | default "test/e2e/setup/flux-operator"}}'
  FLUX_WAIT_TIMEOUT: '{{.FLUX_WAIT_TIMEOUT | default "500s"}}'
  FLUX_SERVICES_WAIT_TIMEOUT: '{{.FLUX_SERVICES_WAIT_TIMEOUT | default "120s"}}'
  # In-cluster Flux versions are pinned (no moving `:latest` / patch range) and
  # come straight from the FLUX_VERSION / FLUX_OPERATOR_VERSION env vars baked into
  # the CI + dev container by .devcontainer/Dockerfile — the single source of
  # truth, so a bump there is the only edit needed. e2e always runs inside that
  # container (local dev and every ci.yml e2e step), so these are always set. The
  # distribution version is consumed as ${FLUX_VERSION} directly when
  # _flux-installed renders the FluxInstance with envsubst.
  #
  # Operator manifests OCI artifact tag = the operator version with a 'v' prefix
  # (ghcr publishes v0.53.0, not 0.53.0).
  FLUX_OPERATOR_MANIFESTS_VERSION: 'v{{.FLUX_OPERATOR_VERSION}}'
  # Preflight (see _ghcr-preflight): the registry artifact `_flux-installed` pulls,
  # and how long a successful reachability check stays valid before we re-probe.
  # Probe the exact pinned artifact (not `:latest`) so the check matches the pull.
  GHCR_PREFLIGHT_REF: '{{.GHCR_PREFLIGHT_REF | default (printf "ghcr.io/controlplaneio-fluxcd/flux-operator-manifests:%s" .FLUX_OPERATOR_MANIFESTS_VERSION)}}'
  GHCR_PREFLIGHT_MAX_AGE_MIN: '{{.GHCR_PREFLIGHT_MAX_AGE_MIN | default "5"}}'
  E2E_GO_TEST_TIMEOUT: '{{.E2E_GO_TEST_TIMEOUT | default "15m"}}'
  ALLURE_RESULTS_DIR: '{{.ALLURE_RESULTS_DIR | default ".stamps/allure-results"}}'
  ALLURE_REPORT_DIR: '{{.ALLURE_REPORT_DIR | default ".stamps/allure-report"}}'
  ALLURE_CLI_DIR: '{{.ALLURE_CLI_DIR | default ".stamps/allure-cli"}}'
  ALLURE_VERSION: '{{.ALLURE_VERSION | default "2.42.0"}}'
  ALLURE_CONTAINER_IMAGE: '{{.ALLURE_CONTAINER_IMAGE | default "eclipse-temurin:17-jre"}}'
  ALLURE_PORT: '{{.ALLURE_PORT | default "19081"}}'
  # Number of parallel Ginkgo processes for the e2e suite. >1 enables Ginkgo
  # parallelism (Phase 2.5 of docs/finished/e2e-speedup-plan.md). Files that touch
  # shared cluster-scoped state are marked Serial (see docs/spec/e2e-serial-registry.md).
  # Default 4 locally: GitTarget isolation on rule changes is now per-target
  # (docs/spec/gittarget-isolation-on-rule-change.md), so higher parallelism no
  # longer trips cross-target snapshot coupling.
  E2E_GINKGO_PROCS: '{{.E2E_GINKGO_PROCS | default "4"}}'
  FLUX_SERVICES_DIR: '{{.FLUX_SERVICES_DIR | default "test/e2e/setup/flux"}}'
  # Argo CD — installed ONLY for the bi-directional corner
  # (docs/spec/e2e-bi-directional-corner.md). Unlike Flux, which is load-bearing
  # for every cluster (Gitea/Valkey/Prometheus arrive as Flux HelmReleases), Argo
  # CD is needed by two specs, so it hangs off _argocd-installed and nothing else.
  #
  # Installed from the official argo-helm chart (test/e2e/setup/argocd/values.yaml).
  # ARGOCD_CHART_VERSION is the single place to bump it — the chart carries the
  # Argo CD release (its appVersion) the specs pin behaviour against. Unlike
  # FLUX_VERSION it is not a devcontainer env var: nothing outside the cluster needs
  # an argocd binary — the specs drive Argo through kubectl alone — so pinning it
  # here avoids a container rebuild. test/e2e/setup/argocd/README.md lists what to
  # re-verify when bumping.
  ARGOCD_CHART_VERSION: '{{.ARGOCD_CHART_VERSION | default "10.11.1"}}'
  ARGOCD_HELM_REPO: '{{.ARGOCD_HELM_REPO | default "https://argoproj.github.io/argo-helm"}}'
  ARGOCD_NAMESPACE: '{{.ARGOCD_NAMESPACE | default "argocd"}}'
  ARGOCD_DIR: '{{.ARGOCD_DIR | default "test/e2e/setup/argocd"}}'
  ARGOCD_PORT: '{{.ARGOCD_PORT | default "18080"}}'
  ARGOCD_WAIT_TIMEOUT: '{{.ARGOCD_WAIT_TIMEOUT | default "300s"}}'
  # kcp — installed ONLY for the source-cluster corner (docs/finished/config-plane-split.md),
  # the same way Argo CD hangs off the bi-directional corner. kcp-operator arrives as a Flux
  # HelmRelease (test/e2e/setup/kcp/base), like every other e2e dependency; the source-cluster
  # specs mirror kcp *workspaces* (cheap logical clusters) as remote source clusters. Not a
  # dependency of prepare-e2e, so the other CI legs never pay for it.
  KCP_DIR: '{{.KCP_DIR | default "test/e2e/setup/kcp"}}'
  KCP_WAIT_TIMEOUT: '{{.KCP_WAIT_TIMEOUT | default "300s"}}'
  DEMO_MANIFESTS_DIR: 'test/e2e/setup/demo-only'
  DEMO_TUNNEL_CREDENTIALS: '{{.DEMO_MANIFESTS_DIR}}/cloudflared-public/tunnel-credentials.yaml'
  DEMO_PULL_SECRET: '{{.DEMO_MANIFESTS_DIR}}/vote/pull-secret.yaml'
  LOADTEST_USERS: '{{.LOADTEST_USERS | default "200"}}'
  LOADTEST_RAMP: '{{.LOADTEST_RAMP | default "20s"}}'
  LOADTEST_BASE_URL: '{{.LOADTEST_BASE_URL | default "https://demo.configbutler.ai"}}'
  LOADTEST_SESSION: '{{.LOADTEST_SESSION | default "kubecon-2026"}}'
  LOADTEST_NS: '{{.LOADTEST_NS | default "voter-production"}}'
tasks:
  prepare-e2e:
    desc: Prepare E2E prerequisites for Go tests
    # portforward-ensure runs *after* _prepare-e2e-ready, not in parallel
    # with it: _webhook-tls-ready inside the prepare graph restarts the
    # k3d server node, which would kill any kubectl port-forwards started
    # concurrently with it.
    cmds:
      - task: _prepare-e2e-ready
      - task: portforward-ensure

  reset-e2e-observability:
    desc: Discard Prometheus data and reset controller counters so a run is judged on its own evidence
    # No sources/generates/status on purpose: this must run on EVERY invocation. Putting it in the
    # stamp graph would let a cached prepare silently skip it, which is the failure mode it exists
    # to prevent -- an assertion about "this run" reading a previous run's samples.
    cmds:
      - |
        export CTX="{{.CTX}}"
        export NAMESPACE="{{.NAMESPACE}}"
        export CONTROLLER_DEPLOY_SELECTOR="{{.CONTROLLER_DEPLOY_SELECTOR}}"
        bash hack/e2e/reset-observability.sh
      # The reset replaces the Prometheus pod, which kills its port-forward.
      - task: portforward-ensure

  prepare-e2e-demo:
    desc: Prepare demo-only prerequisites for talk/demo e2e runs
    cmds:
      - task: prepare-e2e
      - task: _demo-ready

  portforward-ensure:
    desc: Ensure port-forwards are running
    deps:
      - _services-ready
    cmds:
      - |
        export E2E_KUBECONTEXT="{{.CTX}}"
        export GITEA_PORT="{{.GITEA_PORT}}"
        export PROMETHEUS_PORT="{{.PROMETHEUS_PORT}}"
        export VALKEY_PORT="{{.VALKEY_PORT}}"
        export FLUX_WEB_PORT="{{.FLUX_WEB_PORT}}"
        bash hack/e2e/setup-port-forwards.sh

  install:
    desc: Install using the configured INSTALL_MODE
    cmds:
      # Validate INSTALL_MODE before dispatching. The actual dispatch uses
      # in-process task: invocation so the full dependency graph stays in
      # one go-task run (the previous shell-subprocess form lost CTX and
      # other task vars when not also exported as env).
      - |
        case "{{.INSTALL_MODE}}" in
          helm|config-dir|plain-manifests-file) ;;
          *)
            echo "ERROR: INSTALL_MODE must be one of [config-dir helm plain-manifests-file]" >&2
            exit 2
            ;;
        esac
      - task: 'install-{{.INSTALL_MODE}}'

  install-helm:
    desc: Install via Helm
    deps:
      - _install-helm

  install-plain-manifests-file:
    desc: Install via dist/install.yaml
    deps:
      - _install-plain-manifests-file

  install-config-dir:
    desc: Install via config directory
    deps:
      - _install-config-dir

  test-e2e:
    desc: Run the standard e2e suite without CI-only workflow checks
    vars:
      # The whole suite at ~22s avg + ~6m cold prepare ≈ 25m, so it gets a 30m
      # timeout by default; a shorter budget times out mid-run and gives a
      # goroutine dump instead of a useful report.
      E2E_FULL_TIMEOUT: '{{.E2E_FULL_TIMEOUT | default "30m"}}'
    # Ginkgo parallelism (--procs) is orchestrated by the ginkgo CLI, not by
    # `go test`. Run it via `go run` so the CLI version matches the pinned
    # github.com/onsi/ginkgo/v2 module (a globally-installed ginkgo can differ
    # and refuse to run on a version mismatch).
    cmds:
      - |
        export CTX="{{.CTX}}"
        export INSTALL_MODE="{{.INSTALL_MODE}}"
        export NAMESPACE="{{.NAMESPACE}}"
        export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt"
        # E2E_LABEL_FILTER lets CI shard the suite across runners (e.g.
        # 'manager' vs '!manager && !image-refresh && !bi-directional'); it
        # defaults to the whole suite minus the two opt-in corners.
        # `bi-directional` is excluded because it needs an Argo CD install that
        # `task prepare-e2e` does not perform — run `task test-e2e-bi-directional`.
        # E2E_REPORT_NAME keeps each shard's Ginkgo JSON report distinct.
        go run github.com/onsi/ginkgo/v2/ginkgo \
          --procs={{.E2E_GINKGO_PROCS}} --timeout="{{.E2E_FULL_TIMEOUT}}" -v \
          --label-filter='{{.E2E_LABEL_FILTER | default "!image-refresh && !bi-directional && !source-cluster && !ado"}}' \
          --output-dir="{{.CS}}/{{.NAMESPACE}}" \
          --json-report=ginkgo-report-{{.E2E_REPORT_NAME | default "full"}}.json \
          ./test/e2e/

  e2e-coverage-collect:
    desc: >-
      Flush and collect e2e coverage from the deployed controller into
      e2e-cover.out (requires a -cover image; run the suite with E2E_COVERAGE=1).
    cmds:
      - |
        set -euo pipefail
        ctx="{{.CTX}}"
        ns="{{.NAMESPACE}}"
        covdir=".stamps/coverage"
        rm -rf "${covdir}"; mkdir -p "${covdir}"
        pod="$(kubectl --context "${ctx}" -n "${ns}" get pod \
          -l app.kubernetes.io/name=gitops-reverser -o jsonpath='{.items[0].metadata.name}')"
        [ -n "${pod}" ] || { echo "no controller pod in ${ns}" >&2; exit 1; }
        before="$(kubectl --context "${ctx}" -n "${ns}" get pod "${pod}" \
          -o jsonpath='{.status.containerStatuses[?(@.name=="manager")].restartCount}')"
        # The Go runtime writes integration coverage to GOCOVERDIR on exit, so
        # SIGTERM the manager (PID 1). The kubelet restarts it; the GOCOVERDIR
        # emptyDir survives the restart, so the counters remain readable.
        echo "🧹 Flushing coverage: SIGTERM manager (PID 1) in ${ns}/${pod}…"
        kubectl --context "${ctx}" -n "${ns}" exec "${pod}" -c manager -- /busybox/sh -c 'kill -TERM 1' || true
        for _ in $(seq 1 30); do
          now="$(kubectl --context "${ctx}" -n "${ns}" get pod "${pod}" \
            -o jsonpath='{.status.containerStatuses[?(@.name=="manager")].restartCount}' 2>/dev/null || echo "${before}")"
          [ "${now}" != "${before}" ] && break
          sleep 2
        done
        kubectl --context "${ctx}" -n "${ns}" wait --for=condition=Ready "pod/${pod}" --timeout=90s || true
        # Stream GOCOVERDIR out via the image's bundled busybox tar (absolute
        # path avoids PATH assumptions; kubectl exec without a TTY is binary-safe).
        kubectl --context "${ctx}" -n "${ns}" exec "${pod}" -c manager -- \
          /busybox/tar cf - -C /var/gocoverage . > "${covdir}/cov.tar"
        tar xf "${covdir}/cov.tar" -C "${covdir}" && rm -f "${covdir}/cov.tar"
        files="$(find "${covdir}" -name 'covcounters.*' 2>/dev/null | wc -l | tr -d ' ')"
        if [ "${files}" = "0" ]; then
          echo "No coverage counters collected. Was the image built with GOCOVER=1 (E2E_COVERAGE=1)?" >&2
          exit 1
        fi
        go tool covdata textfmt -i="${covdir}" -o=e2e-cover.out
        echo "✅ Wrote e2e-cover.out"
        go tool cover -func=e2e-cover.out | tail -1

  test-e2e-bi-directional:
    desc: Run the bi-directional corner (Flux + Argo CD). Installs Argo CD into the cluster.
    vars:
      # Three Ordered specs, each driving real GitOps controllers through several
      # reconcile round-trips. They own separate Gitea repos, separate namespaces,
      # and separate resource names (the two bi-directional ones also own separate
      # IceCreamOrder CRD groups), and none is Serial, so they run side by side.
      # The provenance spec drives no GitTarget and never patches argocd-secret, so
      # it shares the argocd namespace with the Argo CD spec without racing it.
      # The applier-echo scenarios (bi_directional_echoes_e2e_test.go) are two more long Ordered
      # specs, and the restart ones run Serial at the end: four procs keep the leg inside the CI
      # job's 45 minutes.
      BI_DIRECTIONAL_GINKGO_PROCS: '{{.BI_DIRECTIONAL_GINKGO_PROCS | default "4"}}'
      BI_DIRECTIONAL_TIMEOUT: '{{.BI_DIRECTIONAL_TIMEOUT | default "35m"}}'
    cmds:
      # Sequential, not deps: _prepare-e2e-ready restarts the k3d server node when
      # it injects the webhook TLS cert, which would break an Argo CD install
      # running concurrently. Same reason prepare-e2e-demo chains rather than fans.
      - task: prepare-e2e
      - task: _argocd-installed
      - |
        export CTX="{{.CTX}}"
        export INSTALL_MODE="{{.INSTALL_MODE}}"
        export NAMESPACE="{{.NAMESPACE}}"
        export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt"
        export E2E_ENABLE_BI_DIRECTIONAL=true
        export ARGOCD_NAMESPACE="{{.ARGOCD_NAMESPACE}}"
        go run github.com/onsi/ginkgo/v2/ginkgo \
          --procs={{.BI_DIRECTIONAL_GINKGO_PROCS}} --timeout="{{.BI_DIRECTIONAL_TIMEOUT}}" -v \
          --label-filter='bi-directional' \
          --output-dir="{{.CS}}/{{.NAMESPACE}}" \
          --json-report=ginkgo-report-{{.E2E_REPORT_NAME | default "bi-directional"}}.json \
          ./test/e2e/

  test-e2e-source-cluster:
    desc: >-
      Run the source-cluster corner: kcp workspaces as remote source clusters
      (KubeProvider.spec.secretRef). Installs kcp into the cluster.
    vars:
      # One process: the specs share the kcp front-proxy port-forward and the workspaces
      # they provision, and the suite is Ordered — parallel processes would contend on the
      # local forward port and the shared control plane.
      SOURCE_CLUSTER_GINKGO_PROCS: '{{.SOURCE_CLUSTER_GINKGO_PROCS | default "1"}}'
      SOURCE_CLUSTER_TIMEOUT: '{{.SOURCE_CLUSTER_TIMEOUT | default "25m"}}'
    cmds:
      # Sequential, not deps: _prepare-e2e-ready restarts the k3d server node when it injects
      # the webhook TLS cert, which would disrupt a kcp install running concurrently — the
      # same reason test-e2e-bi-directional chains rather than fans.
      - task: prepare-e2e
      - task: _kcp-ready
      - |
        export CTX="{{.CTX}}"
        export INSTALL_MODE="{{.INSTALL_MODE}}"
        export NAMESPACE="{{.NAMESPACE}}"
        export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt"
        export E2E_ENABLE_SOURCE_CLUSTER=true
        go run github.com/onsi/ginkgo/v2/ginkgo \
          --procs={{.SOURCE_CLUSTER_GINKGO_PROCS}} --timeout="{{.SOURCE_CLUSTER_TIMEOUT}}" -v \
          --label-filter='source-cluster' \
          --output-dir="{{.CS}}/{{.NAMESPACE}}" \
          --json-report=ginkgo-report-{{.E2E_REPORT_NAME | default "source-cluster"}}.json \
          ./test/e2e/

  argocd-ui:
    desc: Install Argo CD if needed, port-forward its UI, and print the admin password
    cmds:
      - task: _argocd-installed
      - |
        CTX="{{.CTX}}" ARGOCD_NAMESPACE="{{.ARGOCD_NAMESPACE}}" ARGOCD_PORT="{{.ARGOCD_PORT}}" \
          bash hack/e2e/argocd-port-forward.sh

  test-e2e-aggregated-api:
    desc: Run aggregated API server e2e scenarios
    cmds:
      - |
        export CTX="{{.CTX}}"
        export INSTALL_MODE="{{.INSTALL_MODE}}"
        export NAMESPACE="{{.NAMESPACE}}"
        export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt"
        go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=aggregated-api \
          -ginkgo.json-report="{{.CS}}/{{.NAMESPACE}}/ginkgo-report-aggregated-api.json"

  allure-e2e-results:
    desc: Convert local e2e Ginkgo JSON reports into Allure results
    aliases:
      - e2e-allure-results
    cmds:
      - |
        reports_file="$(mktemp)"
        trap 'rm -f "${reports_file}"' EXIT
        if [ -n "${ALLURE_GINKGO_REPORTS:-}" ]; then
          printf '%s\n' ${ALLURE_GINKGO_REPORTS} | sort > "${reports_file}"
        else
          reports_root="${ALLURE_GINKGO_REPORTS_DIR:-{{.CS}}/{{.NAMESPACE}}}"
          if [ -d "${reports_root}" ]; then
            if [ "${ALLURE_INCLUDE_ALL_GINKGO_REPORTS:-}" = "true" ]; then
              find "${reports_root}" -maxdepth 1 -name "ginkgo-report-*.json" -type f | sort > "${reports_file}"
            else
              find "${reports_root}" -maxdepth 1 -name "ginkgo-report-*.json" -type f -printf '%T@ %p\n' \
                | sort -rn \
                | sed -n '1s/^[^ ]* //p' > "${reports_file}"
            fi
          fi
        fi
        if [ ! -s "${reports_file}" ]; then
          echo "No Ginkgo JSON reports found for the current e2e context/namespace. Run an e2e task first." >&2
          exit 1
        fi

        rm -rf "{{.ALLURE_RESULTS_DIR}}"
        mkdir -p "{{.ALLURE_RESULTS_DIR}}"
        xargs go run ./test/e2e/tools/ginkgo-allure --output-dir "{{.ALLURE_RESULTS_DIR}}" < "${reports_file}"
        echo "Wrote Allure results to {{.ALLURE_RESULTS_DIR}}"

  allure-e2e-report:
    desc: Generate local Allure HTML from e2e Ginkgo reports
    aliases:
      - e2e-allure-report
    cmds:
      - task: allure-e2e-results
      - |
        if ! command -v docker >/dev/null 2>&1; then
          echo "Docker is required to render Allure without adding Java to the devcontainer." >&2
          exit 1
        fi

        workspace_host="${HOST_PROJECT_PATH:-$(pwd)}"
        workspace_container="$(pwd)"
        mkdir -p "{{.ALLURE_CLI_DIR}}"
        docker run --rm \
          --user "$(id -u):$(id -g)" \
          -e HOME=/tmp \
          -e ALLURE_CLI_DIR="{{.ALLURE_CLI_DIR}}" \
          -e ALLURE_REPORT_DIR="{{.ALLURE_REPORT_DIR}}" \
          -e ALLURE_RESULTS_DIR="{{.ALLURE_RESULTS_DIR}}" \
          -e ALLURE_VERSION="{{.ALLURE_VERSION}}" \
          -v "${workspace_host}:${workspace_container}" \
          -w "${workspace_container}" \
          "{{.ALLURE_CONTAINER_IMAGE}}" \
          sh -eu -c '
            allure_home="${ALLURE_CLI_DIR}/allure-${ALLURE_VERSION}"
            if [ ! -x "${allure_home}/bin/allure" ]; then
              rm -rf "${allure_home}"
              mkdir -p "${ALLURE_CLI_DIR}"
              curl -fsSL "https://github.com/allure-framework/allure2/releases/download/${ALLURE_VERSION}/allure-${ALLURE_VERSION}.tgz" \
                | tar -xz -C "${ALLURE_CLI_DIR}"
            fi
            "${allure_home}/bin/allure" generate "${ALLURE_RESULTS_DIR}" --clean -o "${ALLURE_REPORT_DIR}"
          '
        echo "Wrote Allure HTML report to {{.ALLURE_REPORT_DIR}}"

  allure-e2e-open:
    desc: Generate and serve the local e2e Allure report
    aliases:
      - e2e-allure-open
    cmds:
      - task: allure-e2e-report
      - python3 -m http.server "{{.ALLURE_PORT}}" --bind 0.0.0.0 --directory "{{.ALLURE_REPORT_DIR}}"

  test-image-refresh:
    desc: Validate the image-refresh dependency chain
    cmds:
      - |
        export CTX="{{.CTX}}"
        export INSTALL_MODE="{{.INSTALL_MODE}}"
        export NAMESPACE="{{.NAMESPACE}}"
        export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt"
        go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=image-refresh \
          -ginkgo.json-report="{{.CS}}/{{.NAMESPACE}}/ginkgo-report-image-refresh.json"

  test-e2e-ado:
    desc: 'Mirror into a real Azure DevOps repository (needs E2E_ADO_REPO_URL and E2E_ADO_PAT)'
    cmds:
      - |
        if [ -z "${E2E_ADO_REPO_URL:-}" ] || [ -z "${E2E_ADO_PAT:-}" ]; then
          echo "E2E_ADO_REPO_URL and E2E_ADO_PAT must be set; see docs/azure-devops-getting-started.md" >&2
          exit 1
        fi
        export CTX="{{.CTX}}"
        export INSTALL_MODE="{{.INSTALL_MODE}}"
        export NAMESPACE="{{.NAMESPACE}}"
        export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt"
        go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=ado \
          -ginkgo.json-report="{{.CS}}/{{.NAMESPACE}}/ginkgo-report-ado.json"

  test-e2e-quickstart-helm:
    desc: Run quickstart install validation with Helm install
    cmds:
      - |
        export CTX="{{.CTX}}"
        export INSTALL_MODE=helm
        export NAMESPACE="{{.NAMESPACE}}"
        export E2E_ENABLE_QUICKSTART_FRAMEWORK=true
        export E2E_QUICKSTART_MODE=helm
        export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt"
        go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework \
          -ginkgo.json-report="{{.CS}}/{{.NAMESPACE}}/ginkgo-report-quickstart-helm.json"

  test-e2e-quickstart-manifest:
    desc: Run quickstart install validation with manifest install
    cmds:
      - |
        export CTX="{{.CTX}}"
        export INSTALL_MODE=plain-manifests-file
        export NAMESPACE="{{.NAMESPACE}}"
        export E2E_ENABLE_QUICKSTART_FRAMEWORK=true
        export E2E_QUICKSTART_MODE=plain-manifests-file
        export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt"
        go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework \
          -ginkgo.json-report="{{.CS}}/{{.NAMESPACE}}/ginkgo-report-quickstart-manifest.json"

  playground-bootstrap:
    desc: Bootstrap a reusable repo-backed playground
    cmds:
      - |
        ctx="{{.CTX}}"
        ns="tilt-playground"
        repo="playground"
        provider="playground-provider"
        target="playground-target"
        watchrule="playground-watchrule"
        path="live-cluster"
        if [ -n "${CHECKOUT_DIR:-}" ]; then
          checkout="${CHECKOUT_DIR}"
        elif [ -n "${REPOS_DIR:-}" ]; then
          case "${REPOS_DIR}" in
            /*) checkout="${REPOS_DIR}/${repo}" ;;
            *) checkout="$(pwd)/${REPOS_DIR}/${repo}" ;;
          esac
        else
          checkout="$(pwd)/.stamps/repos/${repo}"
        fi

        secrets_manifest="$(pwd)/.stamps/e2e-repo-artifacts/${ns}/${repo}/secrets.yaml"
        ready_jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
        resource_ready() {
          kind="$1"
          name="$2"
          "{{.KUBECTL}}" --context "${ctx}" -n "${ns}" get "${kind}" "${name}" \
            -o "jsonpath=${ready_jsonpath}" 2>/dev/null | grep -qx 'True'
        }

        if [ -f "${secrets_manifest}" ] && [ -d "${checkout}/.git" ] \
          && "{{.KUBECTL}}" --context "${ctx}" get namespace "${ns}" >/dev/null 2>&1 \
          && resource_ready gitprovider "${provider}" \
          && resource_ready gittarget "${target}" \
          && resource_ready watchrule "${watchrule}"; then
          echo "Playground already ready in namespace ${ns}; skipping bootstrap."
          exit 0
        fi

        export CTX="${ctx}"
        export INSTALL_MODE="config-dir"
        export NAMESPACE="{{.NAMESPACE}}"
        export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt"
        go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=playground

  playground-cleanup:
    desc: Remove the reusable playground namespace, repo, and local artifacts
    cmds:
      - |
        "{{.TASK_BIN}}" portforward-ensure CTX="{{.CTX}}" >/dev/null
        go run ./test/e2e/tools/playground-cleanup \
          --context "{{.CTX}}" \
          --project-dir "$(pwd)"

  playground-status:
    desc: Show playground resources and recent repo commits
    cmds:
      - |
        ctx="{{.CTX}}"
        ns="tilt-playground"
        repo="playground"
        path="live-cluster"
        manifests_dir="$(pwd)/test/playground"
        if [ -n "${CHECKOUT_DIR:-}" ]; then
          checkout="${CHECKOUT_DIR}"
        elif [ -n "${REPOS_DIR:-}" ]; then
          case "${REPOS_DIR}" in
            /*) checkout="${REPOS_DIR}/${repo}" ;;
            *) checkout="$(pwd)/${REPOS_DIR}/${repo}" ;;
          esac
        else
          checkout="$(pwd)/.stamps/repos/${repo}"
        fi

        echo "Playground namespace: ${ns}"
        echo "Repo checkout: ${checkout}"
        echo "Watched path: ${path}"
        echo "Manifest dir: ${manifests_dir}"
        echo

        if "{{.KUBECTL}}" --context "${ctx}" get namespace "${ns}" >/dev/null 2>&1; then
          "{{.KUBECTL}}" --context "${ctx}" get namespace "${ns}"
          echo
          "{{.KUBECTL}}" --context "${ctx}" -n "${ns}" get gitprovider,gittarget,watchrule,configmap,secret
          echo
        else
          echo "Playground namespace not found. Run: task playground-bootstrap"
          echo
        fi

        if [ -d "${checkout}/.git" ]; then
          git -C "${checkout}" pull --ff-only >/dev/null 2>&1 || true
          git -C "${checkout}" --no-pager log --oneline -n 5
        else
          echo "Repo checkout not found yet. Run: task playground-bootstrap"
        fi

  playground-upsert-configmap:
    desc: Create or update a watched ConfigMap in the playground namespace
    cmds:
      - |
        ctx="{{.CTX}}"
        ns="tilt-playground"
        name="playground-configmap"
        stamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
        value="tilt-configmap-${stamp}"

        if ! "{{.KUBECTL}}" --context "${ctx}" get namespace "${ns}" >/dev/null 2>&1; then
          echo "Playground namespace not found. Run: task playground-bootstrap" >&2
          exit 1
        fi
        "{{.KUBECTL}}" --context "${ctx}" -n "${ns}" create configmap "${name}" \
          --from-literal=value="${value}" \
          --from-literal=updatedAt="${stamp}" \
          --from-literal=trigger=tilt \
          --dry-run=client -o yaml \
          | "{{.KUBECTL}}" --context "${ctx}" apply -f -

        echo "Applied ConfigMap ${ns}/${name} with value=${value}"

  playground-delete-configmap:
    desc: Delete the watched ConfigMap in the playground namespace
    cmds:
      - |
        ctx="{{.CTX}}"
        ns="tilt-playground"
        name="playground-configmap"
        if ! "{{.KUBECTL}}" --context "${ctx}" get namespace "${ns}" >/dev/null 2>&1; then
          echo "Playground namespace not found; nothing to delete."
          exit 0
        fi
        "{{.KUBECTL}}" --context "${ctx}" -n "${ns}" delete configmap "${name}" --ignore-not-found=true

  playground-upsert-secret:
    desc: Create or update a watched Secret in the playground namespace
    cmds:
      - |
        ctx="{{.CTX}}"
        ns="tilt-playground"
        name="playground-secret"
        stamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
        value="tilt-secret-${stamp}"

        if ! "{{.KUBECTL}}" --context "${ctx}" get namespace "${ns}" >/dev/null 2>&1; then
          echo "Playground namespace not found. Run: task playground-bootstrap" >&2
          exit 1
        fi
        "{{.KUBECTL}}" --context "${ctx}" -n "${ns}" create secret generic "${name}" \
          --from-literal=username=tilt \
          --from-literal=password="${value}" \
          --from-literal=updatedAt="${stamp}" \
          --dry-run=client -o yaml \
          | "{{.KUBECTL}}" --context "${ctx}" apply -f -

        echo "Applied Secret ${ns}/${name} with password=${value}"

  playground-delete-secret:
    desc: Delete the watched Secret in the playground namespace
    cmds:
      - |
        ctx="{{.CTX}}"
        ns="tilt-playground"
        name="playground-secret"
        if ! "{{.KUBECTL}}" --context "${ctx}" get namespace "${ns}" >/dev/null 2>&1; then
          echo "Playground namespace not found; nothing to delete."
          exit 0
        fi
        "{{.KUBECTL}}" --context "${ctx}" -n "${ns}" delete secret "${name}" --ignore-not-found=true

  # test-e2e-demo intentionally has no -ginkgo.json-report flag: this target
  # is a demo-setup runner, not a measurable test suite.
  test-e2e-demo:
    desc: Prepare a reusable demo repo and leave demo resources in place
    cmds:
      - task: prepare-e2e-demo
      - rm -rf ".stamps/repos/demo"
      - |
        export CTX="{{.CTX}}"
        export REPO_NAME=demo
        export INSTALL_MODE="{{.INSTALL_MODE}}"
        export NAMESPACE="{{.NAMESPACE}}"
        export E2E_ENABLE_DEMO=true
        export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt"
        export TESTNAMESPACE=vote
        go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=demo
      - |
        echo
        echo "Voter login codes:"
        for entry in \
          "demo.configbutler.ai:voter-production" \
          "demo-test.configbutler.ai:voter-test"; do
          host="${entry%%:*}"
          voter_ns="${entry##*:}"
          code=""

          for _ in $(seq 1 30); do
            code="$(
              CTX="{{.CTX}}" VOTER_NAMESPACE="${voter_ns}" \
                hack/e2e/voter-auth-logs.sh --code-only 2>/dev/null || true
            )"
            [ -n "${code}" ] && break
            sleep 2
          done

          if [ -n "${code}" ]; then
            printf '  %s (%s): %s\n' "${host}" "${voter_ns}" "${code}"
          else
            printf '  %s (%s): <not found>\n' "${host}" "${voter_ns}"
          fi
        done

  loadtest:
    desc: Simulate load against the live quiz demo
    cmds:
      - |
        code="{{.LOADTEST_CODE | default ""}}"
        if [ -z "${code}" ]; then
          code="$(
            CTX="{{.CTX}}" VOTER_NAMESPACE="{{.LOADTEST_NS}}" \
              hack/e2e/voter-auth-logs.sh --code-only 2>/dev/null || true
          )"
        fi

        [ -n "${code}" ] || {
          echo "ERROR: could not determine join code automatically." >&2
          echo "Pass it explicitly: task loadtest LOADTEST_CODE=xxxx" >&2
          exit 1
        }

        go run ./test/loadtest \
          --code "${code}" \
          --users "{{.LOADTEST_USERS}}" \
          --ramp-duration "{{.LOADTEST_RAMP}}" \
          --base-url "{{.LOADTEST_BASE_URL}}" \
          --session "{{.LOADTEST_SESSION}}" \
          --namespace "{{.LOADTEST_NS}}"  

  clean-port-forwards:
    desc: Stop all port-forwards
    cmds:
      - echo "Stopping port-forwards..."
      - pkill -f "kubectl.*port-forward.*{{.GITEA_PORT}}" 2>/dev/null || true
      - pkill -f "kubectl.*port-forward.*{{.PROMETHEUS_PORT}}" 2>/dev/null || true
      - pkill -f "kubectl.*port-forward.*{{.VALKEY_PORT}}" 2>/dev/null || true
      - pkill -f "kubectl.*port-forward.*{{.FLUX_WEB_PORT}}" 2>/dev/null || true
      - pkill -f "kubectl.*port-forward.*{{.ARGOCD_PORT}}" 2>/dev/null || true
      - echo "Port-forwards stopped"

  clean-cluster:
    desc: Tear down the E2E cluster used for tests and remove its stamps
    cmds:
      # Delete unconditionally rather than guarding on `k3d cluster list`. An
      # interrupted create can leave orphaned artifacts (e.g. a `-tools`
      # container and `-images` volume) that `k3d cluster list` does not show
      # but `k3d cluster create` still rejects as "a cluster with that name
      # already exists". `k3d cluster delete` cleans up that partial state and
      # exits 0 when there is genuinely nothing to delete, so it is safe to run
      # every time.
      - |
        echo "Deleting k3d cluster '{{.CLUSTER_NAME}}' (including any orphaned partial state)"
        {{.K3D}} cluster delete "{{.CLUSTER_NAME}}"
      - rm -rf "{{.CS}}"

  cleanup-stamp-repos:
    desc: Delete accumulated per-spec Git checkouts under .stamps/repos (reclaim disk)
    cmds:
      # Every e2e spec clones its Gitea repo into .stamps/repos/<name> and keeps it
      # on purpose: the checkout has a working remote + credentials, so you can push,
      # pull, or read the reversed Git history of a past run to investigate. Nothing
      # ever prunes them, so across many runs this grows without bound (multiple GB).
      # This task reclaims that space; the checkout helper re-creates .stamps/repos
      # (mkdir -p) on the next run, so deleting it wholesale is safe. The paired
      # generated GitProvider Secret manifests in .stamps/e2e-repo-artifacts belong
      # to those same repos, so they are cleared in the same sweep.
      - |
        for dir in .stamps/repos .stamps/e2e-repo-artifacts; do
          if [ -d "${dir}" ]; then
            count="$(find "${dir}" -maxdepth 1 -mindepth 1 | wc -l | tr -d ' ')"
            size="$(du -sh "${dir}" 2>/dev/null | cut -f1)"
            echo "Removing ${dir} (${count} entries, ${size})"
            rm -rf "${dir}"
          else
            echo "${dir} already clean"
          fi
        done
        echo "Done."

  _ghcr-preflight:
    # Fail fast — BEFORE we spend ~30s creating a k3d cluster — when ghcr.io cannot
    # serve the Flux operator manifests that _flux-installed pulls. Otherwise an
    # unreachable/denied registry only surfaces deep in the Ginkgo
    # SynchronizedBeforeSuite as an opaque "DENIED: denied" that aborts all specs
    # (0 run). check-ghcr-pull.sh tells two causes apart: (1) a ghcr.io outage /
    # throttling (denied for everyone), and (2) a STALE ghcr.io credential handed
    # out by the dev-container Docker credsStore (an expired GitHub token → 403 on
    # the authenticated pull even though anonymous would succeed). Fix for (2):
    # `docker logout ghcr.io`.
    #
    # Leaf (no deps): wiring it ahead of _cluster-ready does NOT re-create the
    # cluster — go-task evaluates each task's own up-to-date status independently,
    # so a warm _cluster-ready stays up to date while this still re-probes when its
    # stamp goes stale.
    status:
      # Up to date (skip the probe) only while a prior success is younger than the
      # max age. `find -mmin +N` prints the stamp once it is older than N minutes,
      # so a non-empty result flips the task back to out-of-date.
      - test -f "{{.CS}}/ghcr-preflight.ok"
      - test -z "$(find "{{.CS}}/ghcr-preflight.ok" -mmin "+{{.GHCR_PREFLIGHT_MAX_AGE_MIN}}" 2>/dev/null)"
    cmds:
      - mkdir -p "{{.CS}}"
      - GHCR_PREFLIGHT_REF="{{.GHCR_PREFLIGHT_REF}}" bash hack/e2e/check-ghcr-pull.sh
      - touch "{{.CS}}/ghcr-preflight.ok"

  _cluster-ready:
    method: timestamp
    # The stamps below only prove a PAST run succeeded; they say nothing about
    # the cluster right now. A cluster left wedged by a host suspend/resume or
    # OOM (clock-skewed node certs, agents NotReady) keeps its stamp, so every
    # task depending on this one reads "up to date" and the run sails straight
    # into the suite, which then hangs until the 30m ginkgo timeout. Probe the
    # live cluster instead, and abort with the cleanup instructions.
    #
    # Preconditions are evaluated before status, and unlike a status entry a
    # failing one stops the run instead of re-running start-cluster.sh — which
    # would only re-derive the same verdict, slowly, inside its retry loop.
    # `test -f .../ready || exit 0` keeps the very first run (nothing stamped,
    # no cluster yet) out of the way. 5s is a generous budget for one GET
    # against a healthy apiserver; if it cannot answer in that time the cluster
    # is not fit to run e2e against.
    preconditions:
      - sh: |
          test -f "{{.CS}}/ready" || exit 0
          kubectl --context "{{.CTX}}" --request-timeout=5s get nodes \
            -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null \
            | awk 'BEGIN{n=0;t=0} {t++; if ($0=="True") n++} END{exit !(t>0 && n==t)}'
        msg: |
          Cluster '{{.CLUSTER_NAME}}' is stamped as ready but did not report all nodes Ready within 5s.
          It is wedged (host suspend/resume, OOM, or a killed run) — a fresh cluster answers instantly.

          Cleanup and retry:
            task clean-cluster
    deps:
      # Probe ghcr.io reachability before building the cluster (see _ghcr-preflight).
      - _ghcr-preflight
    sources:
      - test/e2e/cluster/start-cluster.sh
      - '{{.AUDIT_POLICY_SOURCE}}'
      - '{{.AUDIT_WEBHOOK_BOOTSTRAP_SOURCE}}'
    generates:
      - '{{.CS}}/ready'
    status:
      - test -f "{{.CS}}/ready"
      - test -f "{{.AUDIT_POLICY_PATH}}"
      - test -f "{{.AUDIT_WEBHOOK_CONFIG_PATH}}"
    cmds:
      - mkdir -p "{{.CS}}" "{{.AUDIT_ASSET_DIR}}"
      - cp "{{.AUDIT_POLICY_SOURCE}}" "{{.AUDIT_POLICY_PATH}}"
      - cp "{{.AUDIT_WEBHOOK_BOOTSTRAP_SOURCE}}" "{{.AUDIT_WEBHOOK_CONFIG_PATH}}"
      - |
        export CLUSTER_NAME="{{.CLUSTER_NAME}}"
        export AUDIT_DIR_REL="{{.AUDIT_ASSET_DIR}}"
        bash test/e2e/cluster/start-cluster.sh
        kubectl --context "{{.CTX}}" --request-timeout=5s get ns >/dev/null
        touch "{{.CS}}/ready"

  _flux-installed:
    method: timestamp
    deps:
      - _cluster-ready
    sources:
      - '{{.CS}}/ready'
      - '{{.FLUX_OPERATOR_DIR}}/**/*.yaml'
    generates:
      - '{{.CS}}/flux.installed'
    status:
      - test -f "{{.CS}}/flux.installed"
      - test "{{.CS}}/ready" -ot "{{.CS}}/flux.installed"
      - test "{{.FLUX_INSTANCE_FILE}}" -ot "{{.CS}}/flux.installed"
      - test -z "$(find "{{.FLUX_OPERATOR_DIR}}" -type f -name '*.yaml' -newer "{{.CS}}/flux.installed" -print -quit)"
      - |
        kubectl --context "{{.CTX}}" \
          -n "{{.FLUX_NAMESPACE}}" \
          get fluxinstance "{{.FLUX_INSTANCE_NAME}}" >/dev/null
    cmds:
      - |
        # The FluxInstance pins its controller distribution to ${FLUX_VERSION}
        # (the flux version from the container env — see the vars block). Render
        # the operator dir through envsubst — only ${FLUX_VERSION} is substituted,
        # every other line is copied verbatim — into the cluster stamp dir
        # (cleaned with the cluster), then install/apply the rendered copies. The
        # raw file holds a placeholder and must not be applied directly.
        rendered="{{.CS}}/flux-operator"
        mkdir -p "$rendered"
        for f in "{{.FLUX_OPERATOR_DIR}}"/*.yaml; do
          envsubst '${FLUX_VERSION}' <"$f" >"$rendered/$(basename "$f")"
        done
        {{.FLUX_OPERATOR}} install \
          --kube-context "{{.CTX}}" \
          --namespace "{{.FLUX_NAMESPACE}}" \
          --timeout "{{.FLUX_WAIT_TIMEOUT}}" \
          --instance-distribution-artifact "oci://ghcr.io/controlplaneio-fluxcd/flux-operator-manifests:{{.FLUX_OPERATOR_MANIFESTS_VERSION}}" \
          --auto-update=false \
          -f "$rendered/$(basename "{{.FLUX_INSTANCE_FILE}}")"
        kubectl --context "{{.CTX}}" apply -f "$rendered"
        {{.FLUX_OPERATOR}} \
          --kube-context "{{.CTX}}" \
          -n "{{.FLUX_NAMESPACE}}" \
          wait instance "{{.FLUX_INSTANCE_NAME}}" \
          --timeout "{{.FLUX_WAIT_TIMEOUT}}"
        kubectl --context "{{.CTX}}" -n "{{.FLUX_NAMESPACE}}" wait \
          deployment \
          -l app.kubernetes.io/part-of=flux \
          --for=condition=Available \
          --timeout="{{.FLUX_WAIT_TIMEOUT}}"
        kubectl --context "{{.CTX}}" wait --for=condition=Established \
          crd/fluxinstances.fluxcd.controlplane.io \
          crd/gitrepositories.source.toolkit.fluxcd.io \
          crd/helmrepositories.source.toolkit.fluxcd.io \
          crd/helmreleases.helm.toolkit.fluxcd.io \
          crd/kustomizations.kustomize.toolkit.fluxcd.io \
          crd/ocirepositories.source.toolkit.fluxcd.io \
          crd/receivers.notification.toolkit.fluxcd.io \
          --timeout="{{.FLUX_WAIT_TIMEOUT}}"
        {{.FLUX_OPERATOR}} version --client > "{{.CS}}/flux.installed"
        {{.FLUX}} version --client >> "{{.CS}}/flux.installed"

  _flux-setup-ready:
    method: timestamp
    deps:
      - _flux-installed
    sources:
      - '{{.CS}}/flux.installed'
      - '{{.FLUX_SERVICES_DIR}}/**/*'
    generates:
      - '{{.CS}}/flux-setup.ready'
    status:
      - test -f "{{.CS}}/flux-setup.ready"
    cmds:
      - |
        kubectl --context "{{.CTX}}" apply -k "{{.FLUX_SERVICES_DIR}}"
        CTX="{{.CTX}}" FLUX_SERVICES_WAIT_TIMEOUT="{{.FLUX_SERVICES_WAIT_TIMEOUT}}" \
          bash hack/e2e/wait-flux-services.sh
        touch "{{.CS}}/flux-setup.ready"

  _services-ready:
    method: timestamp
    deps:
      - _flux-setup-ready
    sources:
      - '{{.CS}}/flux-setup.ready'
      - 'test/e2e/setup/manifests/**/*.yaml'
    generates:
      - '{{.CS}}/services.ready'
    status:
      - test -f "{{.CS}}/services.ready"
    cmds:
      - |
        kubectl --context "{{.CTX}}" wait --for=condition=Established \
          crd/prometheuses.monitoring.coreos.com \
          crd/servicemonitors.monitoring.coreos.com \
          --timeout=180s
        kubectl --context "{{.CTX}}" apply -k test/e2e/setup/manifests
        touch "{{.CS}}/services.ready"

  _argocd-installed:
    # Argo CD for the bi-directional corner. NOT a dependency of prepare-e2e —
    # only test-e2e-bi-directional and argocd-ui pull it in, so the other CI legs
    # never install it. See docs/spec/e2e-bi-directional-corner.md.
    #
    # _services-ready, because the Argo Application sources from Gitea, which
    # arrives as a Flux HelmRelease.
    method: timestamp
    deps:
      - _services-ready
    sources:
      - '{{.CS}}/services.ready'
      - '{{.ARGOCD_DIR}}/**/*.yaml'
    generates:
      - '{{.CS}}/argocd.installed'
    status:
      - test -f "{{.CS}}/argocd.installed"
      - test -z "$(find "{{.ARGOCD_DIR}}" -type f -name '*.yaml' -newer "{{.CS}}/argocd.installed" -print -quit)"
      # Re-install when the pinned chart version or the helm repo changes. Neither
      # is a file under ARGOCD_DIR, so the -newer check above cannot see a bump —
      # the stamp records both (written by the cmds below) and we match them here.
      # Without this, a warm cluster keeps running the old Argo CD after a bump, or
      # after switching to a different repo/mirror at the same version.
      - grep -qxF '{{.ARGOCD_CHART_VERSION}}' "{{.CS}}/argocd.installed"
      - grep -qxF '{{.ARGOCD_HELM_REPO}}' "{{.CS}}/argocd.installed"
      - |
        kubectl --context "{{.CTX}}" -n "{{.ARGOCD_NAMESPACE}}" get deployment argocd-server >/dev/null
    cmds:
      - |
        set -euo pipefail
        # helm, not `kubectl apply`, because the chart's redis-secret-init is a
        # pre-install hook that must finish before redis and the controllers start
        # (they read REDIS_PASSWORD from the secret it writes), and because helm sees
        # the cluster's monitoring.coreos.com API — the gate the chart puts on
        # rendering the ServiceMonitors.
        rendered="{{.CS}}/argocd"
        mkdir -p "$rendered"
        echo "⬇️  Fetching Argo CD chart {{.ARGOCD_CHART_VERSION}}…"
        for attempt in 1 2 3; do
          if helm pull argo-cd --repo "{{.ARGOCD_HELM_REPO}}" \
              --version "{{.ARGOCD_CHART_VERSION}}" --destination "$rendered"; then
            break
          fi
          [ "$attempt" -eq 3 ] && {
            echo "ERROR: could not fetch the argo-cd {{.ARGOCD_CHART_VERSION}} chart from {{.ARGOCD_HELM_REPO}}" >&2
            echo "       Argo CD is only needed by the bi-directional corner; check network/proxy." >&2
            exit 1
          }
          sleep 2
        done
        # One retry, with a diagnostic dump first. A CI run timed out with argocd-redis
        # at Available 0/1 and nothing in the log said why; the dump makes the next
        # occurrence diagnosable, and the retry is safe because upgrade --install is
        # idempotent and a slow image pull keeps its progress across attempts.
        for attempt in 1 2; do
          if helm upgrade --install argocd "$rendered/argo-cd-{{.ARGOCD_CHART_VERSION}}.tgz" \
              --kube-context "{{.CTX}}" \
              --namespace "{{.ARGOCD_NAMESPACE}}" --create-namespace \
              --values "{{.ARGOCD_DIR}}/values.yaml" \
              --wait --timeout "{{.ARGOCD_WAIT_TIMEOUT}}"; then
            break
          fi
          echo "⚠️  Argo CD install attempt $attempt did not become ready; state of {{.ARGOCD_NAMESPACE}}:" >&2
          kubectl --context "{{.CTX}}" -n "{{.ARGOCD_NAMESPACE}}" get pods -o wide >&2 || true
          kubectl --context "{{.CTX}}" -n "{{.ARGOCD_NAMESPACE}}" get events --sort-by=.lastTimestamp >&2 || true
          # Every pod, not only non-Running ones: a pod failing readiness or crash-looping
          # still reports phase Running.
          kubectl --context "{{.CTX}}" -n "{{.ARGOCD_NAMESPACE}}" describe pods >&2 || true
          [ "$attempt" -eq 2 ] && exit 1
        done
        # helm --wait covers workload readiness but not CRD establishment; the specs
        # create Applications immediately, so gate on it explicitly.
        kubectl --context "{{.CTX}}" wait --for=condition=Established \
          crd/applications.argoproj.io \
          crd/appprojects.argoproj.io \
          crd/applicationsets.argoproj.io \
          --timeout=120s
        # Record the chart version and helm repo so the status gate above can detect
        # a bump or a repo/mirror switch.
        printf '%s\n%s\n' "{{.ARGOCD_CHART_VERSION}}" "{{.ARGOCD_HELM_REPO}}" > "{{.CS}}/argocd.installed"

  _kcp-ready:
    # kcp for the source-cluster corner. NOT a dependency of prepare-e2e — only
    # test-e2e-source-cluster pulls it in, so the other CI legs never install kcp.
    # See docs/finished/config-plane-split.md. Modeled on _argocd-installed.
    #
    # _cluster-ready + _flux-setup-ready, because the kcp-operator HelmRelease needs Flux
    # and its dependsOn: cert-manager (both arrive via the Flux setup).
    method: timestamp
    deps:
      - _flux-setup-ready
    sources:
      - '{{.CS}}/flux-setup.ready'
      - '{{.KCP_DIR}}/**/*'
      - 'hack/e2e/setup-kcp.sh'
    generates:
      - '{{.CS}}/kcp.ready'
    status:
      - test -f "{{.CS}}/kcp.ready"
      - test -z "$(find "{{.KCP_DIR}}" hack/e2e/setup-kcp.sh -type f -newer "{{.CS}}/kcp.ready" -print -quit)"
      - |
        kubectl --context "{{.CTX}}" -n kcp get secret kcp-admin-kubeconfig >/dev/null 2>&1
    cmds:
      - CTX="{{.CTX}}" KCP_DIR="{{.KCP_DIR}}" KCP_WAIT_TIMEOUT="{{.KCP_WAIT_TIMEOUT}}" bash hack/e2e/setup-kcp.sh
      - touch "{{.CS}}/kcp.ready"

  _aggregated-api-ready:
    method: timestamp
    deps:
      - _services-ready
    sources:
      - '{{.CS}}/services.ready'
      - 'test/e2e/setup/manifests/sample-apiserver/**/*.yaml'
    generates:
      - '{{.CS}}/aggregated-api.ready'
    status:
      - test -f "{{.CS}}/aggregated-api.ready"
    cmds:
      - |
        kubectl --context "{{.CTX}}" -n wardle rollout status deployment/wardle-server --timeout=180s
        kubectl --context "{{.CTX}}" wait apiservice/v1alpha1.wardle.example.com \
          --for=condition=Available --timeout=180s
        for i in $(seq 1 30); do
          if kubectl --context "{{.CTX}}" api-resources --api-group=wardle.example.com | grep -q flunders; then
            break
          fi
          if [ "${i}" -eq 30 ]; then
            echo "wardle APIService became Available, but discovery never surfaced flunders" >&2
            kubectl --context "{{.CTX}}" get apiservice v1alpha1.wardle.example.com -o yaml >&2
            kubectl --context "{{.CTX}}" api-resources --api-group=wardle.example.com >&2 || true
            exit 1
          fi
          sleep 2
        done
        touch "{{.CS}}/aggregated-api.ready"

  _controller-image-id:
    method: timestamp
    sources:
      - cmd/**/*.go
      - internal/**/*.go
      - api/**/*.go
      - go.mod
      - go.sum
      - Dockerfile
      - exclude: cmd/**/*_test.go
      - exclude: internal/**/*_test.go
      - exclude: api/**/*_test.go
      - exclude: cmd/**/zz_generated.deepcopy.go
      - exclude: internal/**/zz_generated.deepcopy.go
      - exclude: api/**/zz_generated.deepcopy.go
    generates:
      - '{{.IS}}/controller.id'
    status:
      - test -f "{{.IS}}/controller.id"
      # Rebuild when the coverage mode is toggled (E2E_COVERAGE on/off), since
      # the Go sources are unchanged but the binary must be (de)instrumented.
      - test "$(cat {{.IS}}/controller.cover 2>/dev/null)" = "${E2E_COVERAGE:-}"
    cmds:
      - mkdir -p "{{.IS}}"
      - |
        GIT_COMMIT="$(git rev-parse --short HEAD 2>/dev/null || echo unknown)"
        GIT_DIRTY=0
        [ -z "$(git status --porcelain 2>/dev/null)" ] || GIT_DIRTY=1
        BUILD_DATE="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
        VERSION="$(git describe --tags --always --dirty 2>/dev/null || echo dev)"
        # E2E_COVERAGE=1 builds a coverage-instrumented controller (see Dockerfile).
        COVER_ARG=""
        [ -n "${E2E_COVERAGE:-}" ] && COVER_ARG="--build-arg GOCOVER=1"
        {{.CONTAINER_TOOL}} build \
          --build-arg TARGETOS="$(go env GOOS)" \
          --build-arg TARGETARCH="$(go env GOARCH)" \
          --build-arg VERSION="${VERSION}" \
          --build-arg GIT_COMMIT="${GIT_COMMIT}" \
          --build-arg GIT_DIRTY="${GIT_DIRTY}" \
          --build-arg BUILD_DATE="${BUILD_DATE}" \
          ${COVER_ARG} \
          -t "{{.E2E_LOCAL_IMAGE}}" \
          .
        {{.CONTAINER_TOOL}} inspect --format='{{"{{"}}.Id{{"}}"}}' "{{.E2E_LOCAL_IMAGE}}" > "{{.IS}}/controller.id"
        printf '%s' "${E2E_COVERAGE:-}" > "{{.IS}}/controller.cover"

  _project-image-ready-local:
    method: timestamp
    deps:
      - _controller-image-id
    sources:
      - '{{.IS}}/controller.id'
    generates:
      - '{{.IS}}/project-image.ready'
    status:
      - test -f "{{.IS}}/project-image.ready"
      - test "$(<"{{.IS}}/project-image.ready")" = "{{.PROJECT_IMAGE}}"
    cmds:
      - mkdir -p "{{.IS}}"
      - printf '%s\n' "{{.PROJECT_IMAGE}}" > "{{.IS}}/project-image.ready"

  _project-image-ready-provided:
    status:
      - test -f "{{.IS}}/project-image.ready"
      - test "$(<"{{.IS}}/project-image.ready")" = "{{.PROJECT_IMAGE}}"
      - '{{.CONTAINER_TOOL}} image inspect "{{.PROJECT_IMAGE}}" >/dev/null 2>&1'
    cmds:
      - mkdir -p "{{.IS}}"
      - |
        if ! {{.CONTAINER_TOOL}} image inspect "{{.PROJECT_IMAGE}}" >/dev/null 2>&1; then
          echo "Pulling PROJECT_IMAGE={{.PROJECT_IMAGE}}"
          {{.CONTAINER_TOOL}} pull "{{.PROJECT_IMAGE}}"
        fi
        printf '%s\n' "{{.PROJECT_IMAGE}}" > "{{.IS}}/project-image.ready"

  _project-image-ready:
    # In-process dispatch so the leaf task runs inside the same go-task
    # graph as its caller — preserves CTX and lets it run in parallel with
    # the cluster/Flux chain via deps on _image-loaded.
    cmds:
      - task: '{{if .PROJECT_IMAGE_PROVIDED}}_project-image-ready-provided{{else}}_project-image-ready-local{{end}}'

  _image-loaded:
    method: timestamp
    deps:
      - _cluster-ready
      - _project-image-ready
    sources:
      - '{{.CS}}/ready'
      - '{{.IS}}/project-image.ready'
    generates:
      - '{{.CS}}/image.loaded'
    status:
      - test -f "{{.CS}}/image.loaded"
    cmds:
      - mkdir -p "{{.CS}}"
      - |
        CTX="{{.CTX}}" CLUSTER_NAME="{{.CLUSTER_NAME}}" PROJECT_IMAGE="{{.PROJECT_IMAGE}}" \
          PROJECT_IMAGE_PROVIDED="{{.PROJECT_IMAGE_PROVIDED}}" \
          IMAGE_DELIVERY_MODE="{{.IMAGE_DELIVERY_MODE}}" \
          CONTROLLER_ID_TASK="_controller-image-id" TASK_BIN="{{.TASK_BIN}}" \
          CONTAINER_TOOL="{{.CONTAINER_TOOL}}" K3D="{{.K3D}}" STAMP_FILE="{{.CS}}/image.loaded" \
          bash hack/e2e/load-image.sh

  _install-cleanup:
    # Wipe prior install state so a new install mode can take over cleanly.
    # Runs cleanup-installs.sh, which deletes any existing install.yaml-owned
    # resources, drops the install namespace, and rm -rf's every stamp under
    # .stamps/cluster/<ctx>/<ns>/ except git-* fixtures.
    #
    # This used to live inline at the top of every _install-<mode> task. As
    # its own DAG node it can be a parallel-safe prerequisite for both the
    # active _install-<mode> and for tasks like _sops-secret-yaml that write
    # into the same namespace dir.
    #
    # Status: if the install.yaml for the *current* INSTALL_MODE already
    # exists we're either warm or just re-running the same mode, and the
    # in-place apply/upgrade does not need a wipe. Cleanup is only required
    # when switching modes or on a brand-new CTX where the file is missing.
    method: timestamp
    deps:
      - _cluster-ready
    generates:
      - '{{.CS}}/install-cleanup.done'
    status:
      - test -f "{{.CS}}/{{.NAMESPACE}}/{{.INSTALL_MODE}}/install.yaml"
    cmds:
      - |
        CTX="{{.CTX}}" KUBECTL="{{.KUBECTL}}" bash hack/cleanup-installs.sh
        mkdir -p "{{.CS}}"
        touch "{{.CS}}/install-cleanup.done"

  _install-helm:
    deps:
      - _services-ready
      - helm-sync
      - _install-cleanup
    sources:
      - '{{.CS}}/services.ready'
      - charts/gitops-reverser/Chart.yaml
      - charts/gitops-reverser/values.yaml
      - charts/gitops-reverser/templates/**
      - charts/gitops-reverser/crds/*.yaml
      - charts/gitops-reverser/config/role.yaml
      - test/e2e/helm-values-suite-grants.yaml
    generates:
      - '{{.CS}}/{{.NAMESPACE}}/helm/install.yaml'
    cmds:
      - |
        mkdir -p "{{.CS}}/{{.NAMESPACE}}/helm"
        skip_crds_arg=""
        if kubectl --context "{{.CTX}}" get crd \
          gitproviders.configbutler.ai \
          gittargets.configbutler.ai \
          watchrules.configbutler.ai \
          clusterwatchrules.configbutler.ai >/dev/null 2>&1; then
          skip_crds_arg="--skip-crds"
        fi
        CTX="{{.CTX}}" NAMESPACE="{{.NAMESPACE}}" E2E_VALKEY_PASSWORD="{{.E2E_VALKEY_PASSWORD}}" \
          bash hack/e2e/ensure-valkey-auth.sh
        {{.HELM}} --kube-context "{{.CTX}}" upgrade --install "{{.INSTALL_NAME}}" "{{.HELM_CHART_SOURCE}}" \
          --namespace "{{.NAMESPACE}}" \
          --create-namespace \
          --set-string image.repository="{{.PROJECT_IMAGE_REPO}}" \
          --set-string image.tag="{{.PROJECT_IMAGE_TAG}}" \
          --set queue.redis.addr="{{.E2E_AUDIT_REDIS_ADDR}}" \
          --set queue.redis.auth.existingSecret=valkey-auth \
          --set queue.redis.auth.existingSecretKey=password \
          --set service.clusterIP="{{.E2E_CONTROLLER_SERVICE_CLUSTER_IP}}" \
          --values test/e2e/helm-values-suite-grants.yaml \
          ${skip_crds_arg}
        tmp_manifest="{{.CS}}/{{.NAMESPACE}}/helm/.install.yaml.tmp"
        {{.HELM}} --kube-context "{{.CTX}}" get manifest "{{.INSTALL_NAME}}" \
          --namespace "{{.NAMESPACE}}" > "${tmp_manifest}"
        if [ -f "{{.CS}}/{{.NAMESPACE}}/helm/install.yaml" ] && cmp -s "${tmp_manifest}" "{{.CS}}/{{.NAMESPACE}}/helm/install.yaml"; then
          rm -f "${tmp_manifest}"
        else
          mv "${tmp_manifest}" "{{.CS}}/{{.NAMESPACE}}/helm/install.yaml"
        fi

  # _crds-installed applies the project CRDs and waits for them to be Established BEFORE any
  # kustomize/flat-file install applies a custom resource of them. The config-dir bundle (and,
  # once regenerated, dist/install.yaml) ships the `local` KubeProvider CR in the
  # SAME `kubectl apply` as its CRD; kubectl builds its RESTMapper up front, so a CR whose CRD
  # is created in that same apply fails with `no matches for kind "KubeProvider"`. Applying
  # the CRDs first — and waiting for Established — closes that race. Re-runs whenever the
  # generated CRDs change (sources). The Helm chart ships CRDs in charts/gitops-reverser/crds/
  # (helm's own first phase), so _install-helm deliberately does NOT depend on this: the
  # helm/quickstart legs must keep exercising helm's own CRD install.
  _crds-installed:
    deps:
      - _cluster-ready
      - manifests
    sources:
      - config/crd/bases/*.yaml
    generates:
      - '{{.CS}}/crds.installed'
    cmds:
      - mkdir -p "{{.CS}}"
      - '{{.KUSTOMIZE}} build config/crd | {{.KUBECTL}} --context "{{.CTX}}" apply -f -'
      # Not a bare `kubectl wait`: on a CRD created by the apply above it can fail at once.
      - |
        CTX="{{.CTX}}" KUBECTL="{{.KUBECTL}}" bash hack/e2e/wait-crds-established.sh 60 \
          crd/kubeproviders.configbutler.ai crd/clusterwatchrules.configbutler.ai \
          crd/commitrequests.configbutler.ai crd/gitproviders.configbutler.ai \
          crd/gitbranches.configbutler.ai crd/gitreceivers.configbutler.ai \
          crd/gittargets.configbutler.ai crd/watchrules.configbutler.ai \
          crd/fluxclaims.configbutler.ai crd/argocdclaims.configbutler.ai
      - touch "{{.CS}}/crds.installed"

  _install-config-dir:
    deps:
      - _services-ready
      - manifests
      - _crds-installed
      - _install-cleanup
    sources:
      - '{{.CS}}/services.ready'
      - config/**/*
    generates:
      - '{{.CS}}/{{.NAMESPACE}}/config-dir/install.yaml'
    cmds:
      - |
        mkdir -p "{{.CS}}/{{.NAMESPACE}}/config-dir"
        CTX="{{.CTX}}" NAMESPACE="{{.NAMESPACE}}" PROJECT_IMAGE="{{.PROJECT_IMAGE}}" \
          E2E_VALKEY_PASSWORD="{{.E2E_VALKEY_PASSWORD}}" \
          KUSTOMIZE="{{.KUSTOMIZE}}" KUBECTL="{{.KUBECTL}}" \
          STAMP_FILE="{{.CS}}/{{.NAMESPACE}}/config-dir/install.yaml" \
          bash hack/e2e/install-config-dir.sh

  _install-plain-manifests-file:
    method: timestamp
    # Deliberately does NOT depend on _crds-installed: install-plain-manifests.sh applies
    # dist/crds.yaml itself (then dist/install.yaml), so this leg exercises the real two-step
    # user flow end to end. Pre-installing the CRDs from config/crd here would mask whether the
    # shipped dist/crds.yaml actually works.
    deps:
      - _services-ready
      - dist-install
      - _install-cleanup
    sources:
      - '{{.CS}}/services.ready'
      - dist/crds.yaml
      - dist/install.yaml
    generates:
      - '{{.CS}}/{{.NAMESPACE}}/plain-manifests-file/install.yaml'
    cmds:
      - |
        mkdir -p "{{.CS}}/{{.NAMESPACE}}/plain-manifests-file"
        CTX="{{.CTX}}" NAMESPACE="{{.NAMESPACE}}" E2E_VALKEY_PASSWORD="{{.E2E_VALKEY_PASSWORD}}" \
          bash hack/e2e/ensure-valkey-auth.sh
        CTX="{{.CTX}}" NAMESPACE="{{.NAMESPACE}}" PROJECT_IMAGE="{{.PROJECT_IMAGE}}" \
          DEFAULT_AUDIT_REDIS_ADDR="{{.DEFAULT_AUDIT_REDIS_ADDR}}" \
          E2E_AUDIT_REDIS_ADDR="{{.E2E_AUDIT_REDIS_ADDR}}" \
          E2E_CONTROLLER_SERVICE_CLUSTER_IP="{{.E2E_CONTROLLER_SERVICE_CLUSTER_IP}}" \
          KUSTOMIZE="{{.KUSTOMIZE}}" KUBECTL="{{.KUBECTL}}" \
          STAMP_FILE="{{.CS}}/{{.NAMESPACE}}/plain-manifests-file/install.yaml" \
          bash hack/e2e/install-plain-manifests.sh

  _controller-deployed:
    method: timestamp
    deps:
      - install
      - _image-loaded
    sources:
      - '{{.CS}}/{{.NAMESPACE}}/{{.INSTALL_MODE}}/install.yaml'
      - '{{.CS}}/image.loaded'
    generates:
      - '{{.CS}}/{{.NAMESPACE}}/controller.deployed'
    status:
      - test -f "{{.CS}}/{{.NAMESPACE}}/controller.deployed"
    cmds:
      - mkdir -p "{{.CS}}/{{.NAMESPACE}}"
      - |
        CTX="{{.CTX}}" NAMESPACE="{{.NAMESPACE}}" PROJECT_IMAGE="{{.PROJECT_IMAGE}}" \
          IMAGE_DELIVERY_MODE="{{.IMAGE_DELIVERY_MODE}}" \
          IMAGE_PULL_SECRET_NAME="${IMAGE_PULL_SECRET_NAME:-}" \
          IMAGE_PULL_SECRET_REGISTRY="${IMAGE_PULL_SECRET_REGISTRY:-}" \
          IMAGE_PULL_SECRET_USERNAME="${IMAGE_PULL_SECRET_USERNAME:-}" \
          IMAGE_PULL_SECRET_PASSWORD="${IMAGE_PULL_SECRET_PASSWORD:-}" \
          CONTROLLER_CONTAINER="{{.CONTROLLER_CONTAINER}}" \
          CONTROLLER_DEPLOY_SELECTOR="{{.CONTROLLER_DEPLOY_SELECTOR}}" \
          bash hack/e2e/deploy-controller.sh
        cat "{{.CS}}/image.loaded" > "{{.CS}}/{{.NAMESPACE}}/controller.deployed"

  _webhook-tls-ready:
    method: timestamp
    deps:
      - _controller-deployed
    sources:
      - '{{.CS}}/{{.NAMESPACE}}/controller.deployed'
    generates:
      - '{{.CS}}/{{.NAMESPACE}}/webhook-tls.ready'
    status:
      - test -f "{{.CS}}/{{.NAMESPACE}}/webhook-tls.ready"
    cmds:
      - |
        CTX="{{.CTX}}" CLUSTER_NAME="{{.CLUSTER_NAME}}" NAMESPACE="{{.NAMESPACE}}" \
          WEBHOOK_CONFIG="{{.AUDIT_WEBHOOK_CONFIG_PATH}}" \
          CONTROLLER_DEPLOY_SELECTOR="{{.CONTROLLER_DEPLOY_SELECTOR}}" \
          bash hack/e2e/inject-webhook-tls.sh
        touch "{{.CS}}/{{.NAMESPACE}}/webhook-tls.ready"

  _age-key:
    method: timestamp
    sources:
      - test/e2e/tools/gen-age-key/main.go
    generates:
      - '{{.CS}}/age-key.txt'
    status:
      - test -f "{{.CS}}/age-key.txt"
    cmds:
      - mkdir -p "{{.CS}}"
      - |
        go run ./test/e2e/tools/gen-age-key --key-file "{{.CS}}/age-key.txt"

  _sops-secret-yaml:
    method: timestamp
    # _install-cleanup runs first: it wipes everything under
    # .stamps/cluster/<ctx>/<ns>/ except git-* fixtures, including this
    # task's output. Without this dep, status: was evaluated in parallel
    # with the cleanup, found a stale file from a prior run, reported "up
    # to date", and then _sops-secret-applied tried to kubectl apply a
    # file the cleanup had just deleted.
    deps:
      - _age-key
      - _install-cleanup
    sources:
      - '{{.CS}}/age-key.txt'
      - test/e2e/tools/gen-age-key/main.go
    generates:
      - '{{.CS}}/{{.NAMESPACE}}/sops-secret.yaml'
    status:
      - test -f "{{.CS}}/{{.NAMESPACE}}/sops-secret.yaml"
    cmds:
      - mkdir -p "{{.CS}}/{{.NAMESPACE}}"
      - |
        tmp="{{.CS}}/{{.NAMESPACE}}/sops-secret.yaml.tmp"
        go run ./test/e2e/tools/gen-age-key \
          --key-file "{{.CS}}/age-key.txt" \
          --secret-file "${tmp}" \
          --namespace "{{.NAMESPACE}}" \
          --secret-name sops-age-key
        if [ -f "{{.CS}}/{{.NAMESPACE}}/sops-secret.yaml" ] && cmp -s "${tmp}" "{{.CS}}/{{.NAMESPACE}}/sops-secret.yaml"; then
          rm -f "${tmp}"
        else
          mv "${tmp}" "{{.CS}}/{{.NAMESPACE}}/sops-secret.yaml"
        fi

  _sops-secret-applied:
    method: timestamp
    deps:
      - _sops-secret-yaml
      - install
    sources:
      - '{{.CS}}/{{.NAMESPACE}}/sops-secret.yaml'
      - '{{.CS}}/{{.NAMESPACE}}/{{.INSTALL_MODE}}/install.yaml'
    generates:
      - '{{.CS}}/{{.NAMESPACE}}/sops-secret.applied'
    status:
      - test -f "{{.CS}}/{{.NAMESPACE}}/sops-secret.applied"
    cmds:
      - |
        kubectl --context "{{.CTX}}" apply -f "{{.CS}}/{{.NAMESPACE}}/sops-secret.yaml"
        touch "{{.CS}}/{{.NAMESPACE}}/sops-secret.applied"

  _prepare-e2e-ready:
    method: timestamp
    deps:
      - _controller-deployed
      - _webhook-tls-ready
      - _sops-secret-applied
      - _aggregated-api-ready
    sources:
      - '{{.CS}}/{{.NAMESPACE}}/{{.INSTALL_MODE}}/install.yaml'
      - '{{.CS}}/image.loaded'
      - '{{.CS}}/{{.NAMESPACE}}/controller.deployed'
      - '{{.CS}}/{{.NAMESPACE}}/webhook-tls.ready'
      - '{{.CS}}/{{.NAMESPACE}}/sops-secret.applied'
      - '{{.CS}}/aggregated-api.ready'
    generates:
      - '{{.CS}}/{{.NAMESPACE}}/prepare-e2e.ready'
    status:
      - test -f "{{.CS}}/{{.NAMESPACE}}/prepare-e2e.ready"
    cmds:
      - mkdir -p "{{.CS}}/{{.NAMESPACE}}"
      - touch "{{.CS}}/{{.NAMESPACE}}/prepare-e2e.ready"

  _demo-ready:
    method: timestamp
    sources:
      - '{{.CS}}/services.ready'
      - '{{.CS}}/{{.NAMESPACE}}/prepare-e2e.ready'
      - '{{.DEMO_MANIFESTS_DIR}}/**/*.yaml'
    generates:
      - '{{.CS}}/demo.ready'
    status:
      - test -f "{{.CS}}/demo.ready"
    cmds:
      - |
        [ -f "{{.DEMO_TUNNEL_CREDENTIALS}}" ] || {
          echo "ERROR: missing required demo tunnel credentials file '{{.DEMO_TUNNEL_CREDENTIALS}}'" >&2
          echo "Create it from '{{.DEMO_TUNNEL_CREDENTIALS}}.example' and set stringData.token before running this target." >&2
          exit 1
        }
        [ -f "{{.DEMO_PULL_SECRET}}" ] || {
          echo "ERROR: missing required demo image pull secret file '{{.DEMO_PULL_SECRET}}'" >&2
          echo "Create it from '{{.DEMO_PULL_SECRET}}.example' before running this target." >&2
          exit 1
        }
        kubectl --context "{{.CTX}}" apply -f "{{.DEMO_MANIFESTS_DIR}}/vote/ns.yaml"
        kubectl --context "{{.CTX}}" apply -f "{{.DEMO_MANIFESTS_DIR}}/voter-gitops/production/namespace.yaml"
        kubectl --context "{{.CTX}}" apply -f "{{.DEMO_MANIFESTS_DIR}}/voter-gitops/test/namespace.yaml"
        kubectl --context "{{.CTX}}" apply -f "{{.DEMO_MANIFESTS_DIR}}/cloudflared-public/ns.yaml"
        kubectl --context "{{.CTX}}" wait --for=jsonpath='{.status.phase}'=Active \
          namespace/vote namespace/voter-production namespace/voter-test namespace/cloudflared-public --timeout=120s
        kubectl --context "{{.CTX}}" apply -f "{{.DEMO_PULL_SECRET}}"
        dockerconfigjson="$(
          kubectl --context "{{.CTX}}" -n vote get secret zot-pull \
            -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d
        )"
        for voter_ns in voter-production voter-test; do
          kubectl --context "{{.CTX}}" -n "${voter_ns}" create secret generic zot-pull \
            --type=kubernetes.io/dockerconfigjson \
            --from-literal=.dockerconfigjson="${dockerconfigjson}" \
            --dry-run=client -o yaml | kubectl --context "{{.CTX}}" apply -f -
        done
        kubectl --context "{{.CTX}}" apply -f "{{.DEMO_MANIFESTS_DIR}}/voter-gitops/crds/coffeeconfigs.yaml"
        kubectl --context "{{.CTX}}" apply -f "{{.DEMO_MANIFESTS_DIR}}/vote/crds"
        CTX="{{.CTX}}" bash hack/e2e/wait-crds-established.sh 180 \
          crd/coffeeconfigs.examples.configbutler.ai \
          crd/quizsessions.examples.configbutler.ai \
          crd/quizsubmissions.examples.configbutler.ai
        # Traefik and kro live in the demo path (not the core bring-up): the
        # demo podinfo flows below need kro's ResourceGraphDefinition CRD and
        # traefik's IngressRoutes. Install them here and wait before applying
        # anything that depends on them.
        kubectl --context "{{.CTX}}" apply -k "{{.DEMO_MANIFESTS_DIR}}/flux"
        kubectl --context "{{.CTX}}" -n flux-system wait \
          helmrelease/traefik helmrelease/kro \
          --for=condition=Ready --timeout=300s
        kubectl --context "{{.CTX}}" apply -f "{{.DEMO_MANIFESTS_DIR}}/podinfo-kro-template.yaml"
        kubectl --context "{{.CTX}}" wait --for=condition=Ready \
          resourcegraphdefinition/podinfo-app --timeout=120s
        kubectl --context "{{.CTX}}" apply -k "{{.DEMO_MANIFESTS_DIR}}"
        touch "{{.CS}}/demo.ready"