fix(chart): preserve trust and persistent claims through Argo CD upgrades

.github/workflows/ci.yaml

@@ -1273,6 +1273,8 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
       - run: helm template gameplane charts/gameplane > /tmp/rendered.yaml
+      - name: deterministic offline native chart and upgrade renders
+        run: python3 charts/tests/check_argocd.py
       - name: gateway isolation and API-disabled chart renders
         run: python3 charts/tests/check_gateway.py
       - name: standalone panel isolation and persistence chart renders
@@ -2156,7 +2158,7 @@ jobs:
     needs: [changes, build-images]
     if: (needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-upgrade == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'e2e-upgrade')
     runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
-    timeout-minutes: 60 # Same, plus a chart upgrade across versions.
+    timeout-minutes: 85 # Published upgrade, RBAC transition and live Argo CD adoption/rotation.
     permissions:
       contents: read
     strategy:
@@ -2171,6 +2173,10 @@ jobs:
       GAMEPLANE_UPGRADE_FROM: 0.3.0
     steps: &e2e-upgrade-steps
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          # Argo fetches the exact tested commit from a read-only Git server.
+          # A shallow server can advertise a ref that a fresh fetch rejects.
+          fetch-depth: ${{ matrix.arch == 'amd64' && '0' || '1' }}
       - uses: ./.github/actions/go-cache
         with:
           key-suffix: e2e
@@ -2185,13 +2191,19 @@ jobs:
           arch: ${{ matrix.arch }}
       - name: bring up cluster on the PREVIOUS release
         run: ./deploy/kind/upgrade.sh up "$CLUSTER" e2e
+      - name: install Argo CD for the populated GitOps upgrade
+        if: matrix.arch == 'amd64'
+        run: bash deploy/kind/argocd.sh
       - name: run go e2e (upgrade)
+        env:
+          GAMEPLANE_E2E_ARGOCD: ${{ matrix.arch == 'amd64' && '1' || '' }}
+          UPGRADE_TEST_TIMEOUT: ${{ matrix.arch == 'amd64' && '55m' || '35m' }}
         run: |
           RUN_REGEX="$(./test/e2e/buckets.sh regex upgrade)"
           cd test/e2e && \
             GAMEPLANE_E2E_REUSE_CLUSTER=1 \
             GAMEPLANE_E2E_CLUSTER="$CLUSTER" \
-            go test -tags=e2e -timeout 35m -v -run "$RUN_REGEX" ./...
+            go test -tags=e2e -timeout "$UPGRADE_TEST_TIMEOUT" -v -run "$RUN_REGEX" ./...
       - name: dump cluster state on failure (e2e-upgrade)
         if: failure()
         uses: ./.github/actions/dump-cluster-state

changelog/unreleased/901-argocd-upgrades.md

@@ -0,0 +1,5 @@
+# Argo CD upgrades
+
+The native Argo CD profile supports deterministic offline rendering and externally managed
+trust material. Its upgrade checks exercise resource pruning and certificate rotation on a
+populated installation while preserving its claims. [PR #901](https://github.com/GameplanePanel/Gameplane/pull/901)

charts/gameplane/examples/argocd-values.yaml

@@ -0,0 +1,13 @@
+# Native Argo CD profile. Pre-create these immutable Secrets and set the
+# workload Application's source.helm.skipCrds=true (see docs/argocd.md).
+argoCD:
+  enabled: true
+crds:
+  autoApply:
+    enabled: false
+api:
+  agentMTLS:
+    caSecretRef:
+      name: gameplane-external-ca-v1
+    clientCertRef:
+      name: gameplane-external-client-v1

charts/gameplane/templates/_helpers.tpl

@@ -107,6 +107,10 @@ CA or client Secret it references. mtls.yaml does not render those Secrets, so
 without them rotating a custom Secret would not roll the API and operator.
 */}}
 {{- define "gameplane.mtlsChecksum" -}}
+{{- if (.Values.argoCD | default dict).enabled -}}
+{{- /* Immutable external Secret names are the rotation inputs available to an offline renderer. */ -}}
+{{- list .Values.api.agentMTLS.caSecretRef.name .Values.api.agentMTLS.clientCertRef.name | toJson | sha256sum -}}
+{{- else -}}
 {{- $parts := list (include (print .Template.BasePath "/mtls.yaml") .) -}}
 {{- $m := .Values.api.agentMTLS -}}
 {{- range $name := list ($m.caSecretRef.name | default "") ($m.clientCertRef.name | default "") -}}
@@ -117,6 +121,7 @@ without them rotating a custom Secret would not roll the API and operator.
 {{- end -}}
 {{- join "\n---\n" $parts | sha256sum -}}
 {{- end -}}
+{{- end -}}
 
 {{- /* Absent on releases predating optional gateway installs (--reuse-values). */}}
 {{- define "gameplane.apiEnabled" -}}

charts/gameplane/templates/api.yaml

@@ -185,6 +185,9 @@ metadata:
   labels: {{- include "gameplane.labels" . | nindent 4 }}
   annotations:
     helm.sh/resource-policy: keep
+    {{- if (.Values.argoCD | default dict).enabled }}
+    argocd.argoproj.io/sync-options: Prune=false,Delete=false
+    {{- end }}
 spec:
   accessModes: [ReadWriteOnce]
   resources:

charts/gameplane/templates/mtls.yaml

@@ -1,4 +1,5 @@
 {{- if or (eq (include "gameplane.operatorEnabled" .) "true") (and (eq (include "gameplane.apiEnabled" .) "true") (not .Values.api.standalone)) }}
+{{- if not (and (.Values.argoCD | default dict).enabled (ne (include "gameplane.liveCluster" .) "true")) }}
 {{- /*
   Agent mTLS material.
 
@@ -112,3 +113,4 @@ data:
 {{- end }}
 
 {{- end }}
+{{- end }}

charts/gameplane/templates/namespaces.yaml

@@ -19,5 +19,8 @@ metadata:
     # StatefulSet, PVC and per-server Secret Kubernetes garbage-collects
     # underneath it). Same pattern as the API PVC in api.yaml.
     helm.sh/resource-policy: keep
+    {{- if (.Values.argoCD | default dict).enabled }}
+    argocd.argoproj.io/sync-options: Prune=false,Delete=false
+    {{- end }}
 
 {{- end }}

charts/gameplane/templates/validate-argocd.yaml

@@ -0,0 +1,21 @@
+{{- if (.Values.argoCD | default dict).enabled }}
+{{- if or .Values.api.standalone (ne (include "gameplane.operatorEnabled" .) "true") }}
+{{- fail "argoCD.enabled is the native operator delivery profile" }}
+{{- end }}
+{{- if .Values.crds.autoApply.enabled }}
+{{- fail "argoCD.enabled requires crds.autoApply.enabled=false and one external CRD owner" }}
+{{- end }}
+{{- $mtls := .Values.api.agentMTLS }}
+{{- $ca := $mtls.caSecretRef.name | default "" }}
+{{- $client := $mtls.clientCertRef.name | default "" }}
+{{- $generated := list "" "gameplane-agent-ca" "gameplane-agent-client" }}
+{{- if or (has $ca $generated) (has $client $generated) (eq $ca $client) }}
+{{- fail "argoCD.enabled requires distinct external CA and client Secret names in api.agentMTLS" }}
+{{- end }}
+{{- range $kind := list "ValidatingAdmissionPolicy" "ValidatingAdmissionPolicyBinding" }}
+{{- $api := printf "admissionregistration.k8s.io/v1/%s" $kind }}
+{{- if not ($.Capabilities.APIVersions.Has $api) }}
+{{- fail (printf "argoCD.enabled requires target API capability %s for waker enforcement" $api) }}
+{{- end }}
+{{- end }}
+{{- end }}

charts/gameplane/values.yaml

@@ -11,6 +11,11 @@
 ##   serviceMonitors — Prometheus Operator integration
 ##   podSecurity     — cluster-wide hardening knobs
 
+# Offline rendering and resource retention for an Argo CD native installation.
+# See examples/argocd-values.yaml and docs/argocd.md.
+argoCD:
+  enabled: false
+
 # Native Kubernetes grants. The defaults retain the full native dashboard.
 # See docs/native-rbac.md for single-cluster and GitOps profiles.
 nativeRBAC:

charts/tests/check_argocd.py

@@ -0,0 +1,82 @@
+"""CI contract for Argo CD's offline native Helm profile."""
+import subprocess
+
+import yaml
+
+from check_gateway import CHART, one
+
+
+PROFILE = CHART / "examples" / "argocd-values.yaml"
+ADMISSION = "admissionregistration.k8s.io/v1/"
+POLICY = ADMISSION + "ValidatingAdmissionPolicy"
+BINDING = ADMISSION + "ValidatingAdmissionPolicyBinding"
+
+
+def render(*flags, capabilities=(POLICY, BINDING), failure=None):
+    cmd = ["helm", "template", "gameplane", str(CHART), "--namespace", "gameplane-system",
+           "--kube-version", "1.34.0", "--values", str(PROFILE)]
+    for capability in capabilities:
+        cmd.extend(["--api-versions", capability])
+    result = subprocess.run(cmd + list(flags), capture_output=True, check=False)
+    if failure:
+        assert result.returncode and failure.encode() in result.stderr, result.stderr.decode()
+    else:
+        result.check_returncode()
+    return result.stdout
+
+
+def objects(output):
+    return [obj for obj in yaml.safe_load_all(output) if obj]
+
+
+def check_trust(resources, ca, client):
+    checksums = []
+    for name in ("gameplane-api", "gameplane-operator"):
+        template = one(resources, "Deployment", name)["spec"]["template"]
+        volumes = {v["name"]: v for v in template["spec"]["volumes"]}
+        assert volumes["agent-ca"]["secret"]["secretName"] == ca
+        assert volumes["agent-client"]["secret"]["secretName"] == client
+        checksums.append(template["metadata"]["annotations"]["checksum/mtls"])
+    assert checksums[0] == checksums[1]
+    return checksums[0]
+
+
+def main():
+    output = render()
+    assert output == render() == render("--is-upgrade"), "render depends on time, randomness or release state"
+    resources = objects(output)
+    for obj in resources:
+        assert obj["kind"] != "CustomResourceDefinition", obj["metadata"]["name"]
+        assert "helm.sh/hook" not in obj["metadata"].get("annotations", {}), obj["metadata"]["name"]
+        assert not (obj["kind"] == "Secret" and {"ca.key", "tls.key"} & obj.get("data", {}).keys())
+    policy = one(resources, "ValidatingAdmissionPolicy", "gameplane-waker-wake-only")
+    binding = one(resources, "ValidatingAdmissionPolicyBinding", "gameplane-waker-wake-only")
+    assert policy["spec"]["failurePolicy"] == "Fail"
+    assert binding["spec"]["policyName"] == policy["metadata"]["name"]
+    assert "Deny" in binding["spec"]["validationActions"]
+    for kind, name in (("PersistentVolumeClaim", "gameplane-api-data"), ("Namespace", "gameplane-games")):
+        annotations = one(resources, kind, name)["metadata"]["annotations"]
+        assert set(annotations["argocd.argoproj.io/sync-options"].split(",")) == {"Prune=false", "Delete=false"}
+    current = check_trust(resources, "gameplane-external-ca-v1", "gameplane-external-client-v1")
+    rotated = objects(render("--set", "api.agentMTLS.caSecretRef.name=external-ca-v2",
+                             "--set", "api.agentMTLS.clientCertRef.name=external-client-v2"))
+    assert check_trust(rotated, "external-ca-v2", "external-client-v2") != current
+    for capabilities, missing in (((), POLICY), ((POLICY,), BINDING), ((BINDING,), POLICY)):
+        render(capabilities=capabilities, failure=missing)
+    for path, value, reason in (
+        ("api.agentMTLS.caSecretRef.name", "gameplane-agent-ca", "distinct external"),
+        ("api.agentMTLS.clientCertRef.name", "gameplane-agent-client", "distinct external"),
+        ("api.agentMTLS.clientCertRef.name", "gameplane-external-ca-v1", "distinct external"),
+        ("crds.autoApply.enabled", "true", "one external CRD owner"),
+    ):
+        render("--set", f"{path}={value}", failure=reason)
+    handoff = objects(render("--is-upgrade", "--set", "api.storage.existingClaim=gameplane-api-data"))
+    assert not any(obj["kind"] == "PersistentVolumeClaim" and obj["metadata"]["name"] == "gameplane-api-data" for obj in handoff)
+    api = one(handoff, "Deployment", "gameplane-api")
+    claims = [vol["persistentVolumeClaim"]["claimName"] for vol in api["spec"]["template"]["spec"]["volumes"]
+              if "persistentVolumeClaim" in vol]
+    assert "gameplane-api-data" in claims
+
+
+if __name__ == "__main__":
+    main()

deploy/kind/argocd.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Install the upstream core controller only on the disposable upgrade cluster.
+set -euo pipefail
+context="${CTX:?kind upgrade context required}"
+[[ "$context" == "kind-${CLUSTER:?kind upgrade cluster required}" ]]
+kubectl() { command kubectl --context "$context" "$@"; }
+bundle="$(mktemp)"
+trap 'rm -f "$bundle"' EXIT
+# argoproj/argo-cd v3.6.1
+curl --fail --location --silent --show-error --retry 4 --max-time 80 \
+    https://raw.githubusercontent.com/argoproj/argo-cd/94037c4d4e1af4bd0651d37a179fbd9e347e668c/manifests/core-install.yaml \
+    --output "$bundle"
+echo "0b38189d6e1fd76ef5c3db10355026c48a02020d82bf21a1a1b5ab16dffe4f3c  $bundle" | sha256sum --check
+kubectl create namespace argocd
+kubectl apply --server-side --namespace argocd --filename "$bundle"
+kubectl set env -n argocd deployment/argocd-repo-server --containers=argocd-repo-server ARGOCD_GIT_MODULES_ENABLED=false
+# Immediately after creation a CRD can report null conditions. Wait for
+# the API to publish Established=True on both resources before discovery.
+crds=(crd/applications.argoproj.io crd/appprojects.argoproj.io)
+deadline=$((SECONDS + 120))
+until kubectl get "${crds[@]}" --request-timeout=5s -o json |
+    jq -e '.items | length == 2 and all(.[]; any(.status.conditions[]?; .type == "Established" and .status == "True"))' >/dev/null; do
+    if ((SECONDS >= deadline)); then
+        kubectl get "${crds[@]}" -o json | jq '.items[] | {name: .metadata.name, conditions: .status.conditions}' >&2
+        exit 1
+    fi
+    sleep 1
+done
+kubectl rollout status -n argocd deployment/argocd-repo-server --timeout=4m
+kubectl rollout status -n argocd statefulset/argocd-application-controller --timeout=4m

docs/argocd.md

@@ -0,0 +1,50 @@
+# Argo CD delivery of a native Gameplane installation
+
+Use the chart’s Argo CD values for a combined API and operator installation. The profile uses two externally managed mTLS Secrets and requires discovery of the GA admission policy and binding APIs. It also gives the games Namespace and chart-managed SQLite claim explicit Argo CD retention annotations.
+
+## One owner for each resource
+
+Pin both Applications to the same Gameplane Git commit. Sync the CRD Application first and wait for its definitions to be Established before syncing the workload Application.
+
+| Application | Source | Settings |
+| --- | --- | --- |
+| CRDs | Plain directory `charts/gameplane/charts/gameplane-crds/crds` | Use `ServerSideApply=true` for the large schemas. Keep automated pruning off and sync this Application without prune. For Application deletion, preserve its resources by using non-cascading deletion. |
+| Workload | Helm chart `charts/gameplane` | Set `spec.source.helm.skipCrds: true` and include `examples/argocd-values.yaml` in `spec.source.helm.valueFiles`. The profile disables the CRD apply hook. Set the release name to `gameplane` and the destination namespace to `gameplane-system`. |
+
+Provision the release namespace, CA and client Secret before syncing. Keep ownership of the release namespace and external Secrets outside the workload Application. The example expects `gameplane-external-ca-v1` with `ca.crt` and `ca.key`, and a different Secret named `gameplane-external-client-v1` with `tls.crt` and `tls.key`. Use immutable, versioned Secret names. The operator signs per-game serving certificates with this CA, and the API and operator present the client certificate to the agents.
+
+Carry the installation's `nativeRBAC` settings, additional game namespaces, and credentials namespace into the workload Application. For a single-cluster panel where GitOps owns catalogs, configure the reduced Kubernetes permissions described in the [native RBAC guide](native-rbac.md).
+
+Argo CD ordinarily passes the destination cluster’s API capabilities to Helm. Verify that both `admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy` and `admissionregistration.k8s.io/v1/ValidatingAdmissionPolicyBinding` are served and that admission is enabled. For an offline comparison, pass the discovered capabilities and the destination Kubernetes version. This example assumes both kinds on Kubernetes 1.34.
+
+```sh
+for run in 1 2; do
+  helm template gameplane charts/gameplane -n gameplane-system \
+    -f charts/gameplane/examples/argocd-values.yaml --kube-version 1.34.0 \
+    --api-versions admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy \
+    --api-versions admissionregistration.k8s.io/v1/ValidatingAdmissionPolicyBinding \
+    > "gameplane-$run.yaml"
+done
+cmp gameplane-1.yaml gameplane-2.yaml
+```
+
+The chart fails the profile render if either kind is missing. Supplying a capability flag locally does not install that API on the destination. See the [Argo CD Helm source parameters](https://argo-cd.readthedocs.io/en/stable/user-guide/application-specification/) and [Helm hook mapping](https://argo-cd.readthedocs.io/en/stable/user-guide/helm/#helm-hooks).
+
+## Adopting a populated installation
+
+1. Back up the database and game data. Record the API and game claim UIDs, the current CA public certificate fingerprint, and each Gameplane CRD’s UID and `status.storedVersions`. Verify each stored version appears in the target CRD’s `spec.versions`. A removed stored version needs a storage-version migration first.
+2. Copy the current CA and client material into the two external Secrets through your secret manager. Compare the CA’s public certificate digest and test the client against a live agent. Keep the existing trust pair for the initial handoff. Disable the prior delivery owner so it cannot concurrently change these resources.
+3. Check both Deployments’ live update strategy before handoff. Offline rendering cannot detect an old server-defaulted `spec.strategy.rollingUpdate`. When the desired strategy is `Recreate`, migrate the live Deployment through the existing Helm upgrade path or a controlled merge patch that sets `type: Recreate` and `rollingUpdate: null` together.
+4. Sync with pruning disabled. Keep the SQLite claim name and size from the populated release. After the first sync, verify the live games Namespace and the live API PVC both carry `argocd.argoproj.io/sync-options: Prune=false,Delete=false`. If switching to `api.storage.existingClaim`, verify retention on the **old live claim before** the sync that removes its manifest. Point the API at that same populated PVC, or migrate and verify the data separately first.
+5. Review the workload prune preview. Compare the UIDs and CA fingerprint from step 1, log in with a pre-upgrade account, read a persisted game marker and make an authenticated call to the running game’s agent. On a wake-enabled test GameServer, impersonate its waker ServiceAccount for server-side dry-run patches. Its wake annotation should succeed and a spec mutation should be denied by the wake policy. Verify the policy and binding names and denial reason.
+6. Enable workload pruning after reviewing the retained objects. Argo CD may report retained resources as OutOfSync. `Prune=false` preserves an object removed from Git during sync. `Delete=false` preserves it on Application deletion. Helm’s `helm.sh/resource-policy: keep` maps to deletion protection in Argo CD. Deleting a containing namespace can still remove its claims. See [Argo CD sync options](https://argo-cd.readthedocs.io/en/stable/user-guide/sync-options/).
+
+## Rotating external certificates
+
+For a client renewal under the same CA, stage a new immutable client Secret and change its reference in Git. For a CA rollover, schedule a maintenance window, stage a new CA and matching client Secret, then change both references in one revision. The reference change rolls the API and operator. The operator reissues each GameServer’s agent TLS Secret against the active CA. Kubelet projection and the next TLS handshake determine when each agent starts using the new trust.
+
+Finish the rollout by checking the new CA fingerprint on each `<server>-agent-tls` Secret and making a fresh authenticated agent call for every running game. A stopped game needs that check on its next start. Retain the previous Secret pair until the checks finish. If rolling back the CA, restore both references together and repeat the per-agent checks while certificates are reissued.
+
+## CI qualification
+
+The amd64 upgrade job installs Argo CD Core v3.6.1 on a disposable kind cluster. Two Applications sync the PR commit from a read-only Git endpoint on the kind bridge. After adoption, the workload Application renews the client, switches to the previously populated database claim and prunes a tracked dashboard. A second sync replaces the CA and client together. At each stage the fixture verifies the claim and CRD identities, validates stored versions, authenticates with the original account and reads a saved world marker through the agent. Server-side dry runs check the waker's allowed annotation and denied spec mutation. [Populated upgrade fixture](../test/e2e/argocd_upgrade_e2e_test.go)