diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 87e0e37a7..5ba140252 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1273,6 +1273,8 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - run: helm template gameplane charts/gameplane > /tmp/rendered.yaml + - name: deterministic offline native chart and upgrade renders + run: python3 charts/tests/check_argocd.py - name: gateway isolation and API-disabled chart renders run: python3 charts/tests/check_gateway.py - name: standalone panel isolation and persistence chart renders @@ -2156,7 +2158,7 @@ jobs: needs: [changes, build-images] if: (needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-upgrade == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'e2e-upgrade') runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} - timeout-minutes: 60 # Same, plus a chart upgrade across versions. + timeout-minutes: 85 # Published upgrade, RBAC transition and live Argo CD adoption/rotation. permissions: contents: read strategy: @@ -2171,6 +2173,10 @@ jobs: GAMEPLANE_UPGRADE_FROM: 0.3.0 steps: &e2e-upgrade-steps - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # Argo fetches the exact tested commit from a read-only Git server. + # A shallow server can advertise a ref that a fresh fetch rejects. + fetch-depth: ${{ matrix.arch == 'amd64' && '0' || '1' }} - uses: ./.github/actions/go-cache with: key-suffix: e2e @@ -2185,13 +2191,19 @@ jobs: arch: ${{ matrix.arch }} - name: bring up cluster on the PREVIOUS release run: ./deploy/kind/upgrade.sh up "$CLUSTER" e2e + - name: install Argo CD for the populated GitOps upgrade + if: matrix.arch == 'amd64' + run: bash deploy/kind/argocd.sh - name: run go e2e (upgrade) + env: + GAMEPLANE_E2E_ARGOCD: ${{ matrix.arch == 'amd64' && '1' || '' }} + UPGRADE_TEST_TIMEOUT: ${{ matrix.arch == 'amd64' && '55m' || '35m' }} run: | RUN_REGEX="$(./test/e2e/buckets.sh regex upgrade)" cd test/e2e && \ GAMEPLANE_E2E_REUSE_CLUSTER=1 \ GAMEPLANE_E2E_CLUSTER="$CLUSTER" \ - go test -tags=e2e -timeout 35m -v -run "$RUN_REGEX" ./... + go test -tags=e2e -timeout "$UPGRADE_TEST_TIMEOUT" -v -run "$RUN_REGEX" ./... - name: dump cluster state on failure (e2e-upgrade) if: failure() uses: ./.github/actions/dump-cluster-state diff --git a/changelog/unreleased/901-argocd-upgrades.md b/changelog/unreleased/901-argocd-upgrades.md new file mode 100644 index 000000000..84838da18 --- /dev/null +++ b/changelog/unreleased/901-argocd-upgrades.md @@ -0,0 +1,5 @@ +# Argo CD upgrades + +The native Argo CD profile supports deterministic offline rendering and externally managed +trust material. Its upgrade checks exercise resource pruning and certificate rotation on a +populated installation while preserving its claims. [PR #901](https://github.com/GameplanePanel/Gameplane/pull/901) diff --git a/charts/gameplane/examples/argocd-values.yaml b/charts/gameplane/examples/argocd-values.yaml new file mode 100644 index 000000000..0dfe686c7 --- /dev/null +++ b/charts/gameplane/examples/argocd-values.yaml @@ -0,0 +1,13 @@ +# Native Argo CD profile. Pre-create these immutable Secrets and set the +# workload Application's source.helm.skipCrds=true (see docs/argocd.md). +argoCD: + enabled: true +crds: + autoApply: + enabled: false +api: + agentMTLS: + caSecretRef: + name: gameplane-external-ca-v1 + clientCertRef: + name: gameplane-external-client-v1 diff --git a/charts/gameplane/templates/_helpers.tpl b/charts/gameplane/templates/_helpers.tpl index 483c16ec0..b502042b4 100644 --- a/charts/gameplane/templates/_helpers.tpl +++ b/charts/gameplane/templates/_helpers.tpl @@ -107,6 +107,10 @@ CA or client Secret it references. mtls.yaml does not render those Secrets, so without them rotating a custom Secret would not roll the API and operator. */}} {{- define "gameplane.mtlsChecksum" -}} +{{- if (.Values.argoCD | default dict).enabled -}} +{{- /* Immutable external Secret names are the rotation inputs available to an offline renderer. */ -}} +{{- list .Values.api.agentMTLS.caSecretRef.name .Values.api.agentMTLS.clientCertRef.name | toJson | sha256sum -}} +{{- else -}} {{- $parts := list (include (print .Template.BasePath "/mtls.yaml") .) -}} {{- $m := .Values.api.agentMTLS -}} {{- range $name := list ($m.caSecretRef.name | default "") ($m.clientCertRef.name | default "") -}} @@ -117,6 +121,7 @@ without them rotating a custom Secret would not roll the API and operator. {{- end -}} {{- join "\n---\n" $parts | sha256sum -}} {{- end -}} +{{- end -}} {{- /* Absent on releases predating optional gateway installs (--reuse-values). */}} {{- define "gameplane.apiEnabled" -}} diff --git a/charts/gameplane/templates/api.yaml b/charts/gameplane/templates/api.yaml index 31c9ab8fa..87d2b6334 100644 --- a/charts/gameplane/templates/api.yaml +++ b/charts/gameplane/templates/api.yaml @@ -185,6 +185,9 @@ metadata: labels: {{- include "gameplane.labels" . | nindent 4 }} annotations: helm.sh/resource-policy: keep + {{- if (.Values.argoCD | default dict).enabled }} + argocd.argoproj.io/sync-options: Prune=false,Delete=false + {{- end }} spec: accessModes: [ReadWriteOnce] resources: diff --git a/charts/gameplane/templates/mtls.yaml b/charts/gameplane/templates/mtls.yaml index 3eda0bbcd..27fe933c3 100644 --- a/charts/gameplane/templates/mtls.yaml +++ b/charts/gameplane/templates/mtls.yaml @@ -1,4 +1,5 @@ {{- if or (eq (include "gameplane.operatorEnabled" .) "true") (and (eq (include "gameplane.apiEnabled" .) "true") (not .Values.api.standalone)) }} +{{- if not (and (.Values.argoCD | default dict).enabled (ne (include "gameplane.liveCluster" .) "true")) }} {{- /* Agent mTLS material. @@ -112,3 +113,4 @@ data: {{- end }} {{- end }} +{{- end }} diff --git a/charts/gameplane/templates/namespaces.yaml b/charts/gameplane/templates/namespaces.yaml index cf6204696..6a7376641 100644 --- a/charts/gameplane/templates/namespaces.yaml +++ b/charts/gameplane/templates/namespaces.yaml @@ -19,5 +19,8 @@ metadata: # StatefulSet, PVC and per-server Secret Kubernetes garbage-collects # underneath it). Same pattern as the API PVC in api.yaml. helm.sh/resource-policy: keep + {{- if (.Values.argoCD | default dict).enabled }} + argocd.argoproj.io/sync-options: Prune=false,Delete=false + {{- end }} {{- end }} diff --git a/charts/gameplane/templates/validate-argocd.yaml b/charts/gameplane/templates/validate-argocd.yaml new file mode 100644 index 000000000..ddb84587b --- /dev/null +++ b/charts/gameplane/templates/validate-argocd.yaml @@ -0,0 +1,21 @@ +{{- if (.Values.argoCD | default dict).enabled }} +{{- if or .Values.api.standalone (ne (include "gameplane.operatorEnabled" .) "true") }} +{{- fail "argoCD.enabled is the native operator delivery profile" }} +{{- end }} +{{- if .Values.crds.autoApply.enabled }} +{{- fail "argoCD.enabled requires crds.autoApply.enabled=false and one external CRD owner" }} +{{- end }} +{{- $mtls := .Values.api.agentMTLS }} +{{- $ca := $mtls.caSecretRef.name | default "" }} +{{- $client := $mtls.clientCertRef.name | default "" }} +{{- $generated := list "" "gameplane-agent-ca" "gameplane-agent-client" }} +{{- if or (has $ca $generated) (has $client $generated) (eq $ca $client) }} +{{- fail "argoCD.enabled requires distinct external CA and client Secret names in api.agentMTLS" }} +{{- end }} +{{- range $kind := list "ValidatingAdmissionPolicy" "ValidatingAdmissionPolicyBinding" }} +{{- $api := printf "admissionregistration.k8s.io/v1/%s" $kind }} +{{- if not ($.Capabilities.APIVersions.Has $api) }} +{{- fail (printf "argoCD.enabled requires target API capability %s for waker enforcement" $api) }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index 889ebaea8..14fddcea3 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -11,6 +11,11 @@ ## serviceMonitors — Prometheus Operator integration ## podSecurity — cluster-wide hardening knobs +# Offline rendering and resource retention for an Argo CD native installation. +# See examples/argocd-values.yaml and docs/argocd.md. +argoCD: + enabled: false + # Native Kubernetes grants. The defaults retain the full native dashboard. # See docs/native-rbac.md for single-cluster and GitOps profiles. nativeRBAC: diff --git a/charts/tests/check_argocd.py b/charts/tests/check_argocd.py new file mode 100644 index 000000000..c4f965a9e --- /dev/null +++ b/charts/tests/check_argocd.py @@ -0,0 +1,82 @@ +"""CI contract for Argo CD's offline native Helm profile.""" +import subprocess + +import yaml + +from check_gateway import CHART, one + + +PROFILE = CHART / "examples" / "argocd-values.yaml" +ADMISSION = "admissionregistration.k8s.io/v1/" +POLICY = ADMISSION + "ValidatingAdmissionPolicy" +BINDING = ADMISSION + "ValidatingAdmissionPolicyBinding" + + +def render(*flags, capabilities=(POLICY, BINDING), failure=None): + cmd = ["helm", "template", "gameplane", str(CHART), "--namespace", "gameplane-system", + "--kube-version", "1.34.0", "--values", str(PROFILE)] + for capability in capabilities: + cmd.extend(["--api-versions", capability]) + result = subprocess.run(cmd + list(flags), capture_output=True, check=False) + if failure: + assert result.returncode and failure.encode() in result.stderr, result.stderr.decode() + else: + result.check_returncode() + return result.stdout + + +def objects(output): + return [obj for obj in yaml.safe_load_all(output) if obj] + + +def check_trust(resources, ca, client): + checksums = [] + for name in ("gameplane-api", "gameplane-operator"): + template = one(resources, "Deployment", name)["spec"]["template"] + volumes = {v["name"]: v for v in template["spec"]["volumes"]} + assert volumes["agent-ca"]["secret"]["secretName"] == ca + assert volumes["agent-client"]["secret"]["secretName"] == client + checksums.append(template["metadata"]["annotations"]["checksum/mtls"]) + assert checksums[0] == checksums[1] + return checksums[0] + + +def main(): + output = render() + assert output == render() == render("--is-upgrade"), "render depends on time, randomness or release state" + resources = objects(output) + for obj in resources: + assert obj["kind"] != "CustomResourceDefinition", obj["metadata"]["name"] + assert "helm.sh/hook" not in obj["metadata"].get("annotations", {}), obj["metadata"]["name"] + assert not (obj["kind"] == "Secret" and {"ca.key", "tls.key"} & obj.get("data", {}).keys()) + policy = one(resources, "ValidatingAdmissionPolicy", "gameplane-waker-wake-only") + binding = one(resources, "ValidatingAdmissionPolicyBinding", "gameplane-waker-wake-only") + assert policy["spec"]["failurePolicy"] == "Fail" + assert binding["spec"]["policyName"] == policy["metadata"]["name"] + assert "Deny" in binding["spec"]["validationActions"] + for kind, name in (("PersistentVolumeClaim", "gameplane-api-data"), ("Namespace", "gameplane-games")): + annotations = one(resources, kind, name)["metadata"]["annotations"] + assert set(annotations["argocd.argoproj.io/sync-options"].split(",")) == {"Prune=false", "Delete=false"} + current = check_trust(resources, "gameplane-external-ca-v1", "gameplane-external-client-v1") + rotated = objects(render("--set", "api.agentMTLS.caSecretRef.name=external-ca-v2", + "--set", "api.agentMTLS.clientCertRef.name=external-client-v2")) + assert check_trust(rotated, "external-ca-v2", "external-client-v2") != current + for capabilities, missing in (((), POLICY), ((POLICY,), BINDING), ((BINDING,), POLICY)): + render(capabilities=capabilities, failure=missing) + for path, value, reason in ( + ("api.agentMTLS.caSecretRef.name", "gameplane-agent-ca", "distinct external"), + ("api.agentMTLS.clientCertRef.name", "gameplane-agent-client", "distinct external"), + ("api.agentMTLS.clientCertRef.name", "gameplane-external-ca-v1", "distinct external"), + ("crds.autoApply.enabled", "true", "one external CRD owner"), + ): + render("--set", f"{path}={value}", failure=reason) + handoff = objects(render("--is-upgrade", "--set", "api.storage.existingClaim=gameplane-api-data")) + assert not any(obj["kind"] == "PersistentVolumeClaim" and obj["metadata"]["name"] == "gameplane-api-data" for obj in handoff) + api = one(handoff, "Deployment", "gameplane-api") + claims = [vol["persistentVolumeClaim"]["claimName"] for vol in api["spec"]["template"]["spec"]["volumes"] + if "persistentVolumeClaim" in vol] + assert "gameplane-api-data" in claims + + +if __name__ == "__main__": + main() diff --git a/deploy/kind/argocd.sh b/deploy/kind/argocd.sh new file mode 100644 index 000000000..b398d1e47 --- /dev/null +++ b/deploy/kind/argocd.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Install the upstream core controller only on the disposable upgrade cluster. +set -euo pipefail +context="${CTX:?kind upgrade context required}" +[[ "$context" == "kind-${CLUSTER:?kind upgrade cluster required}" ]] +kubectl() { command kubectl --context "$context" "$@"; } +bundle="$(mktemp)" +trap 'rm -f "$bundle"' EXIT +# argoproj/argo-cd v3.6.1 +curl --fail --location --silent --show-error --retry 4 --max-time 80 \ + https://raw.githubusercontent.com/argoproj/argo-cd/94037c4d4e1af4bd0651d37a179fbd9e347e668c/manifests/core-install.yaml \ + --output "$bundle" +echo "0b38189d6e1fd76ef5c3db10355026c48a02020d82bf21a1a1b5ab16dffe4f3c $bundle" | sha256sum --check +kubectl create namespace argocd +kubectl apply --server-side --namespace argocd --filename "$bundle" +kubectl set env -n argocd deployment/argocd-repo-server --containers=argocd-repo-server ARGOCD_GIT_MODULES_ENABLED=false +# Immediately after creation a CRD can report null conditions. Wait for +# the API to publish Established=True on both resources before discovery. +crds=(crd/applications.argoproj.io crd/appprojects.argoproj.io) +deadline=$((SECONDS + 120)) +until kubectl get "${crds[@]}" --request-timeout=5s -o json | + jq -e '.items | length == 2 and all(.[]; any(.status.conditions[]?; .type == "Established" and .status == "True"))' >/dev/null; do + if ((SECONDS >= deadline)); then + kubectl get "${crds[@]}" -o json | jq '.items[] | {name: .metadata.name, conditions: .status.conditions}' >&2 + exit 1 + fi + sleep 1 +done +kubectl rollout status -n argocd deployment/argocd-repo-server --timeout=4m +kubectl rollout status -n argocd statefulset/argocd-application-controller --timeout=4m diff --git a/docs/argocd.md b/docs/argocd.md new file mode 100644 index 000000000..f3931a5cb --- /dev/null +++ b/docs/argocd.md @@ -0,0 +1,50 @@ +# Argo CD delivery of a native Gameplane installation + +Use the chart’s Argo CD values for a combined API and operator installation. The profile uses two externally managed mTLS Secrets and requires discovery of the GA admission policy and binding APIs. It also gives the games Namespace and chart-managed SQLite claim explicit Argo CD retention annotations. + +## One owner for each resource + +Pin both Applications to the same Gameplane Git commit. Sync the CRD Application first and wait for its definitions to be Established before syncing the workload Application. + +| Application | Source | Settings | +| --- | --- | --- | +| CRDs | Plain directory `charts/gameplane/charts/gameplane-crds/crds` | Use `ServerSideApply=true` for the large schemas. Keep automated pruning off and sync this Application without prune. For Application deletion, preserve its resources by using non-cascading deletion. | +| Workload | Helm chart `charts/gameplane` | Set `spec.source.helm.skipCrds: true` and include `examples/argocd-values.yaml` in `spec.source.helm.valueFiles`. The profile disables the CRD apply hook. Set the release name to `gameplane` and the destination namespace to `gameplane-system`. | + +Provision the release namespace, CA and client Secret before syncing. Keep ownership of the release namespace and external Secrets outside the workload Application. The example expects `gameplane-external-ca-v1` with `ca.crt` and `ca.key`, and a different Secret named `gameplane-external-client-v1` with `tls.crt` and `tls.key`. Use immutable, versioned Secret names. The operator signs per-game serving certificates with this CA, and the API and operator present the client certificate to the agents. + +Carry the installation's `nativeRBAC` settings, additional game namespaces, and credentials namespace into the workload Application. For a single-cluster panel where GitOps owns catalogs, configure the reduced Kubernetes permissions described in the [native RBAC guide](native-rbac.md). + +Argo CD ordinarily passes the destination cluster’s API capabilities to Helm. Verify that both `admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy` and `admissionregistration.k8s.io/v1/ValidatingAdmissionPolicyBinding` are served and that admission is enabled. For an offline comparison, pass the discovered capabilities and the destination Kubernetes version. This example assumes both kinds on Kubernetes 1.34. + +```sh +for run in 1 2; do + helm template gameplane charts/gameplane -n gameplane-system \ + -f charts/gameplane/examples/argocd-values.yaml --kube-version 1.34.0 \ + --api-versions admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy \ + --api-versions admissionregistration.k8s.io/v1/ValidatingAdmissionPolicyBinding \ + > "gameplane-$run.yaml" +done +cmp gameplane-1.yaml gameplane-2.yaml +``` + +The chart fails the profile render if either kind is missing. Supplying a capability flag locally does not install that API on the destination. See the [Argo CD Helm source parameters](https://argo-cd.readthedocs.io/en/stable/user-guide/application-specification/) and [Helm hook mapping](https://argo-cd.readthedocs.io/en/stable/user-guide/helm/#helm-hooks). + +## Adopting a populated installation + +1. Back up the database and game data. Record the API and game claim UIDs, the current CA public certificate fingerprint, and each Gameplane CRD’s UID and `status.storedVersions`. Verify each stored version appears in the target CRD’s `spec.versions`. A removed stored version needs a storage-version migration first. +2. Copy the current CA and client material into the two external Secrets through your secret manager. Compare the CA’s public certificate digest and test the client against a live agent. Keep the existing trust pair for the initial handoff. Disable the prior delivery owner so it cannot concurrently change these resources. +3. Check both Deployments’ live update strategy before handoff. Offline rendering cannot detect an old server-defaulted `spec.strategy.rollingUpdate`. When the desired strategy is `Recreate`, migrate the live Deployment through the existing Helm upgrade path or a controlled merge patch that sets `type: Recreate` and `rollingUpdate: null` together. +4. Sync with pruning disabled. Keep the SQLite claim name and size from the populated release. After the first sync, verify the live games Namespace and the live API PVC both carry `argocd.argoproj.io/sync-options: Prune=false,Delete=false`. If switching to `api.storage.existingClaim`, verify retention on the **old live claim before** the sync that removes its manifest. Point the API at that same populated PVC, or migrate and verify the data separately first. +5. Review the workload prune preview. Compare the UIDs and CA fingerprint from step 1, log in with a pre-upgrade account, read a persisted game marker and make an authenticated call to the running game’s agent. On a wake-enabled test GameServer, impersonate its waker ServiceAccount for server-side dry-run patches. Its wake annotation should succeed and a spec mutation should be denied by the wake policy. Verify the policy and binding names and denial reason. +6. Enable workload pruning after reviewing the retained objects. Argo CD may report retained resources as OutOfSync. `Prune=false` preserves an object removed from Git during sync. `Delete=false` preserves it on Application deletion. Helm’s `helm.sh/resource-policy: keep` maps to deletion protection in Argo CD. Deleting a containing namespace can still remove its claims. See [Argo CD sync options](https://argo-cd.readthedocs.io/en/stable/user-guide/sync-options/). + +## Rotating external certificates + +For a client renewal under the same CA, stage a new immutable client Secret and change its reference in Git. For a CA rollover, schedule a maintenance window, stage a new CA and matching client Secret, then change both references in one revision. The reference change rolls the API and operator. The operator reissues each GameServer’s agent TLS Secret against the active CA. Kubelet projection and the next TLS handshake determine when each agent starts using the new trust. + +Finish the rollout by checking the new CA fingerprint on each `-agent-tls` Secret and making a fresh authenticated agent call for every running game. A stopped game needs that check on its next start. Retain the previous Secret pair until the checks finish. If rolling back the CA, restore both references together and repeat the per-agent checks while certificates are reissued. + +## CI qualification + +The amd64 upgrade job installs Argo CD Core v3.6.1 on a disposable kind cluster. Two Applications sync the PR commit from a read-only Git endpoint on the kind bridge. After adoption, the workload Application renews the client, switches to the previously populated database claim and prunes a tracked dashboard. A second sync replaces the CA and client together. At each stage the fixture verifies the claim and CRD identities, validates stored versions, authenticates with the original account and reads a saved world marker through the agent. Server-side dry runs check the waker's allowed annotation and denied spec mutation. [Populated upgrade fixture](../test/e2e/argocd_upgrade_e2e_test.go) diff --git a/docs/install.md b/docs/install.md index 68dfff30e..ef9ee7f1e 100644 --- a/docs/install.md +++ b/docs/install.md @@ -5,6 +5,8 @@ For a central dashboard/API on a host without Kubernetes, use the panel-only Helm profile. The instructions below describe the default combined installation, with an operator and a local game cluster. +For an offline Helm renderer, see the [Argo CD delivery guide](argocd.md). + ## Prerequisites - Kubernetes 1.30+ with ValidatingAdmissionPolicy and PodSecurity admission enabled diff --git a/test/e2e/argocd_git_e2e_test.go b/test/e2e/argocd_git_e2e_test.go new file mode 100644 index 000000000..81367e5c2 --- /dev/null +++ b/test/e2e/argocd_git_e2e_test.go @@ -0,0 +1,90 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "net" + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// Serve exactly the tested commit, including fork PRs, without giving the +// controller a GitHub credential or making runner credentials part of a clone. +func argoGitSource(t *testing.T) (address, revision string) { + t.Helper() + run := func(program string, args ...string) string { + t.Helper() + out, err := exec.CommandContext(t.Context(), program, args...).CombinedOutput() + if err != nil { + t.Fatalf("%s: %v: %s", program, err, out) + } + return strings.TrimSpace(string(out)) + } + revision = run("git", "rev-parse", "HEAD") + directory := t.TempDir() + repo := filepath.Join(directory, "candidate.git") + if run("git", "rev-parse", "--is-shallow-repository") == "true" { + t.Fatal("the Argo CD fixture needs a full checkout to serve the candidate branch") + } + run("git", "init", "--quiet", "--bare", repo) + // Share the checkout's objects. The temporary repository publishes one + // ref and has its own configuration, with no runner credentials. + objects := run("git", "rev-parse", "--path-format=absolute", "--git-path", "objects") + if err := os.WriteFile(filepath.Join(repo, "objects", "info", "alternates"), []byte(objects+"\n"), 0o600); err != nil { + t.Fatal(err) + } + run("git", "-C", repo, "update-ref", "refs/heads/candidate", revision) + run("git", "-C", repo, "symbolic-ref", "HEAD", "refs/heads/candidate") + gitHTTP := &cgi.Handler{Path: filepath.Join(run("git", "--exec-path"), "git-http-backend"), Dir: directory, + Env: []string{"GIT_PROJECT_ROOT=" + directory, "GIT_HTTP_EXPORT_ALL=1"}} + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + discover := r.Method == http.MethodGet && r.URL.Path == "/candidate.git/info/refs" && r.URL.Query().Get("service") == "git-upload-pack" + fetch := r.Method == http.MethodPost && r.URL.Path == "/candidate.git/git-upload-pack" + if !discover && !fetch { + http.NotFound(w, r) + return + } + gitHTTP.ServeHTTP(w, r) + }) + // Docker may list the IPv6 bridge first even on an IPv4 Pod network. + // Use the same Git client and network namespace that will fetch the chart. + gateways := run("docker", "network", "inspect", "kind", "--format", "{{range .IPAM.Config}}{{println .Gateway}}{{end}}") + var failures []string + for _, gateway := range strings.Fields(gateways) { + if ip := net.ParseIP(gateway); ip == nil || !ip.IsPrivate() { + continue + } + var listen net.ListenConfig + listener, err := listen.Listen(t.Context(), "tcp", net.JoinHostPort(gateway, "0")) + if err != nil { + failures = append(failures, err.Error()) + continue + } + srv := httptest.NewUnstartedServer(handler) + _ = srv.Listener.Close() + srv.Listener = listener + srv.Start() + address := srv.URL + "/candidate.git" + probe, cancel := context.WithTimeout(t.Context(), 10*time.Second) + out, err := envInstance.Kubectl(probe, "-n", "argocd", "exec", "deploy/argocd-repo-server", "-c", "argocd-repo-server", + "--", "git", "ls-remote", "--exit-code", address, "refs/heads/candidate") + cancel() + if err == nil && strings.TrimSpace(out) == revision+"\trefs/heads/candidate" { + t.Cleanup(srv.Close) + return address, revision + } + srv.Close() + failures = append(failures, fmt.Sprintf("%s: %v %s", gateway, err, strings.TrimSpace(out))) + } + t.Fatalf("Argo CD repo-server could not read revision %s through the private kind gateways %q: %v", revision, gateways, failures) + return "", "" +} diff --git a/test/e2e/argocd_upgrade_e2e_test.go b/test/e2e/argocd_upgrade_e2e_test.go new file mode 100644 index 000000000..b4f08a3de --- /dev/null +++ b/test/e2e/argocd_upgrade_e2e_test.go @@ -0,0 +1,344 @@ +//go:build e2e + +package e2e + +import ( + "bytes" + "context" + "crypto" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "fmt" + "math/big" + "net/http" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" +) + +var argoAppGVR = schema.GroupVersionResource{Group: "argoproj.io", Version: "v1alpha1", Resource: "applications"} +var argoSyncOptions = []any{"ServerSideApply=true", "ClientSideApplyMigration=false", "FailOnSharedResource=true"} + +func checkArgoUpgrade(t *testing.T, ns, game, marker, username, password, credentialsNS string) { + t.Helper() + const system = "gameplane-system" + const workload = "gameplane" + ctx := t.Context() + repo, revision := argoGitSource(t) + projectName := "upgrade-fixture" + destination := func(namespace string) map[string]any { + return map[string]any{"server": "https://kubernetes.default.svc", "namespace": namespace} + } + projectGVR := schema.GroupVersionResource{Group: "argoproj.io", Version: "v1alpha1", Resource: "appprojects"} + // The preceding RBAC phase left a reduced privilege profile and a second + // managed namespace. Carry those settings through adoption without copying + // arbitrary release values, such as credentials, into an Application. + var permissions struct { + NativeRBAC map[string]any `json:"nativeRBAC"` + Operator struct { + GameNamespaces []string `json:"gameNamespaces"` + } `json:"operator"` + } + rawValues, err := exec.CommandContext(ctx, "helm", "get", "values", workload, "--namespace", system, + "--kube-context", envInstance.Context, "-o", "json").Output() + if err != nil { + t.Fatalf("read installed privilege profile: %v", err) + } + if err := json.Unmarshal(rawValues, &permissions); err != nil { + t.Fatal(err) + } + if permissions.NativeRBAC["catalogAdmin"] != false { + t.Fatal("expected the reduced native RBAC profile before GitOps adoption") + } + destinations := []any{destination(system), destination(ns), destination(credentialsNS)} + var gameNamespaces []any + for _, n := range permissions.Operator.GameNamespaces { + destinations = append(destinations, destination(n)) + gameNamespaces = append(gameNamespaces, n) + } + var clusterResources []any + for _, resource := range [][2]string{ + {"", "Namespace"}, + {"apiextensions.k8s.io", "CustomResourceDefinition"}, + {"rbac.authorization.k8s.io", "ClusterRole"}, + {"rbac.authorization.k8s.io", "ClusterRoleBinding"}, + {"admissionregistration.k8s.io", "ValidatingAdmissionPolicy"}, + {"admissionregistration.k8s.io", "ValidatingAdmissionPolicyBinding"}, + } { + clusterResources = append(clusterResources, map[string]any{"group": resource[0], "kind": resource[1]}) + } + create := func(gvr schema.GroupVersionResource, kind, name string, spec map[string]any) { + t.Helper() + resource := envInstance.Dyn.Resource(gvr).Namespace("argocd") + obj := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "argoproj.io/v1alpha1", "kind": kind, + "metadata": map[string]any{"name": name, "namespace": "argocd"}, "spec": spec}} + if _, err := resource.Create(ctx, obj, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + // Applications have no cascading-deletion finalizer; the later Helm + // reinstall phase owns teardown of the populated world. + t.Cleanup(func() { + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + defer cancel() + if err := resource.Delete(cleanup, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + t.Errorf("delete Argo CD %s %s: %v", kind, name, err) + } + }) + } + create(projectGVR, "AppProject", projectName, map[string]any{ + "sourceRepos": []any{repo}, + "destinations": destinations, + "clusterResourceWhitelist": clusterResources, + }) + app := func(name, path string, helm map[string]any) { + t.Helper() + source := map[string]any{"repoURL": repo, "targetRevision": revision, "path": path} + if helm != nil { + source["helm"] = helm + } + create(argoAppGVR, "Application", name, map[string]any{ + "project": projectName, "source": source, "destination": destination(system), + "syncPolicy": map[string]any{"syncOptions": argoSyncOptions}, + }) + } + identity := readUpgradeIdentity(ctx, t, "gameplane-agent-ca") + claim, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(ns).Get(ctx, game+"-data", metav1.GetOptions{}) + if err != nil || claim.UID == "" { + t.Fatalf("read populated game PVC: %v", err) + } + currentSecret := func(name string) *corev1.Secret { + t.Helper() + s, err := envInstance.K8s.CoreV1().Secrets(system).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + return s + } + immutable := func(name string, kind corev1.SecretType, material map[string][]byte) { + t.Helper() + _, err := envInstance.K8s.CoreV1().Secrets(system).Create(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}, + Type: kind, Immutable: ptr.To(true), Data: material}, metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + } + firstCA, firstClient := currentSecret("gameplane-agent-ca"), currentSecret("gameplane-agent-client") + issuer, clientCert := "gameplane-external-ca-v1", "gameplane-external-client-v1" + immutable(issuer, firstCA.Type, firstCA.Data) + immutable(clientCert, firstClient.Type, firstClient.Data) + // Reauthenticate against the current API on every rollout; a stale + // port-forward or cached /players response cannot verify an agent's mTLS. + checkWorld := func(issuerSecret, clientSecret string, expectedCA []byte) { + t.Helper() + now := readUpgradeIdentity(ctx, t, issuerSecret) + expected := fmt.Sprintf("%x", sha256.Sum256(expectedCA)) + if now["agent CA fingerprint"] != expected { + t.Fatal("active issuer fingerprint differs from staged certificate") + } + for field, value := range identity { + if field != "agent CA fingerprint" && now[field] != value { + t.Fatalf("changed %s: %s -> %s", field, value, now[field]) + } + } + sameClaim, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(ns).Get(ctx, game+"-data", metav1.GetOptions{}) + if err != nil || sameClaim.UID != claim.UID { + t.Fatalf("game PVC was replaced: %v", err) + } + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + cert, err := envInstance.K8s.CoreV1().Secrets(ns).Get(ctx, game+"-agent-tls", metav1.GetOptions{}) + return err == nil && bytes.Equal(cert.Data["ca.crt"], expectedCA), "waiting for agent TLS issuance under the selected CA" + }) + for _, deployment := range []string{"gameplane-api", "gameplane-operator"} { + out, err := envInstance.Kubectl(ctx, "rollout", "status", "-n", system, "deployment/"+deployment, "--timeout=3m") + if err != nil { + t.Fatalf("%s rollout: %v %s", deployment, err, out) + } + dep, err := envInstance.K8s.AppsV1().Deployments(system).Get(ctx, deployment, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + mounted := map[string]bool{} + for _, volume := range dep.Spec.Template.Spec.Volumes { + if volume.Secret != nil { + mounted[volume.Secret.SecretName] = true + } + } + if !mounted[issuerSecret] || !mounted[clientSecret] { + t.Fatalf("%s rolled out without the intended issuer/client Secret references", deployment) + } + } + cli := envInstance.apiClientContext(ctx, t, username, password) + defer cli.Close() + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + resp, body, err := cli.DoContext(ctx, http.MethodGet, "/servers/"+game+"/files/read?path=%2Fmarker.txt", nil) + if err != nil { + return false, err.Error() + } + defer resp.Body.Close() + return resp.StatusCode == http.StatusOK && string(body) == marker, fmt.Sprintf("reading the seeded world through agent mTLS: HTTP %d", resp.StatusCode) + }) + } + checkWorld("gameplane-agent-ca", "gameplane-agent-client", firstCA.Data["ca.crt"]) + app("gameplane-crds", "charts/gameplane/charts/gameplane-crds/crds", nil) + syncArgo(t, "gameplane-crds", revision, "crds", false, nil) + trust := map[string]any{ + "caSecretRef": map[string]any{"name": issuer}, + "clientCertRef": map[string]any{"name": clientCert}, + } + apiValues := map[string]any{ + "credentialsNamespace": credentialsNS, + "resources": map[string]any{"limits": map[string]any{"memory": "1Gi"}}, + "agentMTLS": trust, + } + values := map[string]any{ + "image": map[string]any{"registry": "gameplane-test", "tag": envInstance.Tag}, + "operator": map[string]any{"leaderElect": false, "agentImage": "gameplane-test/agent:" + envInstance.Tag, "gameNamespaces": gameNamespaces}, + "nativeRBAC": permissions.NativeRBAC, + "podSecurity": map[string]any{"enforceRestricted": false}, + "web": map[string]any{"enabled": false}, + "ingress": map[string]any{"enabled": false}, + "defaultModuleSource": map[string]any{"enabled": false}, + "uploadModuleSource": map[string]any{"enabled": false}, + "grafanaDashboards": map[string]any{"enabled": true}, + "api": apiValues, + } + helm := map[string]any{"releaseName": "gameplane", "skipCrds": true, "valueFiles": []any{"examples/argocd-values.yaml"}, "valuesObject": values} + app(workload, "charts/gameplane", helm) + syncArgo(t, workload, revision, "adopt", false, nil) + checkWorld(issuer, clientCert, firstCA.Data["ca.crt"]) + checkArgoWaker(t, ns, game) + dashboards := envInstance.K8s.CoreV1().ConfigMaps(system) + dashboard, err := dashboards.Get(ctx, "gameplane-operator-dashboard", metav1.GetOptions{}) + if err != nil || dashboard.Annotations["argocd.argoproj.io/tracking-id"] == "" { + t.Fatalf("expected Argo-owned disposable dashboard before prune: %v", err) + } + dbClaim, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(system).Get(ctx, "gameplane-api-data", metav1.GetOptions{}) + if err != nil || !strings.Contains(dbClaim.Annotations["argocd.argoproj.io/sync-options"], "Prune=false") || dbClaim.Annotations["argocd.argoproj.io/tracking-id"] == "" { + t.Fatalf("expected a tracked, protected SQLite claim before prune: %v", err) + } + newClient := "gameplane-external-client-v2" + immutable(newClient, corev1.SecretTypeTLS, renewArgoClient(t, firstCA)) + values["grafanaDashboards"] = map[string]any{"enabled": false} + apiValues["storage"] = map[string]any{"existingClaim": "gameplane-api-data"} + trust["clientCertRef"] = map[string]any{"name": newClient} + syncArgo(t, workload, revision, "prune-and-renew-client", true, values) + if _, err := dashboards.Get(ctx, dashboard.Name, metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("expected Argo to prune the dashboard: %v", err) + } + checkWorld(issuer, newClient, firstCA.Data["ca.crt"]) + replacement, leaf := harvestMTLSPair(ctx, t, filepath.Join("..", "..", "charts", "gameplane"), "argo-rotation-donor") + issuer, newClient = "gameplane-external-ca-v2", "gameplane-external-client-v3" + immutable(issuer, corev1.SecretTypeOpaque, map[string][]byte{"ca.crt": []byte(replacement.crt), "ca.key": []byte(replacement.key)}) + immutable(newClient, corev1.SecretTypeTLS, map[string][]byte{"tls.crt": []byte(leaf.crt), "tls.key": []byte(leaf.key)}) + trust["caSecretRef"] = map[string]any{"name": issuer} + trust["clientCertRef"] = map[string]any{"name": newClient} + syncArgo(t, workload, revision, "rotate-issuer", true, values) + checkWorld(issuer, newClient, []byte(replacement.crt)) + checkArgoWaker(t, ns, game) +} + +// A unique operation marker distinguishes successive syncs of the same pinned +// Git revision with different Helm inputs. OutOfSync is expected for retained PVCs. +func syncArgo(t *testing.T, name, revision, step string, prune bool, values map[string]any) { + t.Helper() + apps := envInstance.Dyn.Resource(argoAppGVR).Namespace("argocd") + marker := []any{map[string]any{"name": "fixture-step", "value": step}} + update := map[string]any{"operation": map[string]any{ + "info": marker, + "sync": map[string]any{"revision": revision, "prune": prune, "syncOptions": argoSyncOptions}, + }} + if values != nil { + update["spec"] = map[string]any{"source": map[string]any{"helm": map[string]any{"valuesObject": values}}} + } + raw, err := json.Marshal(update) + if err != nil { + t.Fatal(err) + } + if _, err := apps.Patch(t.Context(), name, types.MergePatchType, raw, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + envInstance.Eventually(t, 5*time.Minute, func() (bool, string) { + app, err := apps.Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + info, _, _ := unstructured.NestedSlice(app.Object, "status", "operationState", "operation", "info") + if !reflect.DeepEqual(info, marker) { + return false, "waiting for " + step + } + phase, _, _ := unstructured.NestedString(app.Object, "status", "operationState", "phase") + msg, _, _ := unstructured.NestedString(app.Object, "status", "operationState", "message") + if phase == "Failed" || phase == "Error" { + t.Fatalf("%s/%s: %s: %s", name, step, phase, msg) + } + applied, _, _ := unstructured.NestedString(app.Object, "status", "operationState", "syncResult", "revision") + return phase == "Succeeded" && applied == revision, phase + ": " + msg + }) +} + +func checkArgoWaker(t *testing.T, ns, game string) { + t.Helper() + sa := "system:serviceaccount:" + ns + ":" + game + "-waker-sa" + patch := func(data string) (string, error) { + return envInstance.Kubectl(t.Context(), "--as", sa, "-n", ns, "patch", "gameserver", game, "--dry-run=server", "--type=merge", "-p", data) + } + if out, err := patch(`{"metadata":{"annotations":{"gameplane.local/idle-wake-requested":"argo-check"}}}`); err != nil { + t.Fatalf("waker annotation denied: %v %s", err, out) + } + out, err := patch(`{"spec":{"suspend":true}}`) + if err == nil || !strings.Contains(out, "gameplane-waker-wake-only") || !strings.Contains(out, "may not change the GameServer spec") { + t.Fatalf("expected wake-only admission to deny the spec patch: %v %s", err, out) + } +} + +func renewArgoClient(t *testing.T, ca *corev1.Secret) map[string][]byte { + t.Helper() + pair, err := tls.X509KeyPair(ca.Data["ca.crt"], ca.Data["ca.key"]) + if err != nil { + t.Fatal(err) + } + issuer, err := x509.ParseCertificate(pair.Certificate[0]) + if err != nil { + t.Fatal(err) + } + key, ok := pair.PrivateKey.(crypto.Signer) + if !ok { + t.Fatal("issuer key cannot sign") + } + public, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + t.Fatal(err) + } + cert := &x509.Certificate{SerialNumber: serial, Subject: pkix.Name{CommonName: "gameplane-api-client"}, NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(6 * time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}} + der, err := x509.CreateCertificate(rand.Reader, cert, issuer, public, key) + if err != nil { + t.Fatal(err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(private) + if err != nil { + t.Fatal(err) + } + return map[string][]byte{"tls.crt": pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), "tls.key": pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})} +} diff --git a/test/e2e/env.go b/test/e2e/env.go index f8cdf886d..70cf21f49 100644 --- a/test/e2e/env.go +++ b/test/e2e/env.go @@ -341,6 +341,11 @@ func (e *Env) WriteAdminPasswordFile(t *testing.T, pw string) { // // `target` follows kubectl conventions ("svc/foo" or "pod/foo"). func (e *Env) PortForward(t *testing.T, ns, target string, remotePort int) (int, func()) { + t.Helper() + return e.portForwardContext(t.Context(), t, ns, target, remotePort) +} + +func (e *Env) portForwardContext(ctx context.Context, t *testing.T, ns, target string, remotePort int) (int, func()) { t.Helper() // Retry the whole spawn: on a loaded self-hosted runner, kubectl // port-forward can take a while to wire the pod tunnel, or fail @@ -351,7 +356,7 @@ func (e *Env) PortForward(t *testing.T, ns, target string, remotePort int) (int, const attempts = 4 var lastErr error for attempt := 1; attempt <= attempts; attempt++ { - local, stop, err := e.tryPortForward(t.Context(), ns, target, remotePort) + local, stop, err := e.tryPortForward(ctx, ns, target, remotePort) if err == nil { return local, stop } @@ -497,7 +502,12 @@ type APIClient struct { // tear the port-forward down. func (e *Env) APIClient(t *testing.T, username, password string) *APIClient { t.Helper() - local, stop := e.PortForward(t, "gameplane-system", "svc/gameplane-api", 80) + return e.apiClientContext(t.Context(), t, username, password) +} + +func (e *Env) apiClientContext(ctx context.Context, t *testing.T, username, password string) *APIClient { + t.Helper() + local, stop := e.portForwardContext(ctx, t, "gameplane-system", "svc/gameplane-api", 80) base := fmt.Sprintf("http://127.0.0.1:%d", local) // 90s tolerates the few legitimately-slow endpoints (notably :restart, @@ -527,7 +537,7 @@ func (e *Env) APIClient(t *testing.T, username, password string) *APIClient { const maxAttempts = 7 reconnected := false for attempt := 0; attempt < maxAttempts; attempt++ { - req, rerr := http.NewRequestWithContext(t.Context(), http.MethodPost, base+"/auth/login", bytes.NewReader(body)) + req, rerr := http.NewRequestWithContext(ctx, http.MethodPost, base+"/auth/login", bytes.NewReader(body)) if rerr != nil { stop() t.Fatalf("new login request: %v", rerr) @@ -539,8 +549,8 @@ func (e *Env) APIClient(t *testing.T, username, password string) *APIClient { // A vanished local listener cannot have received this request. // Reopen once within the existing attempt budget; never replay an // ambiguous read/write failure or reconnect an established session. - if !reconnected && attempt+1 < maxAttempts && t.Context().Err() == nil && loginDialRefused(err) { - local, stop = e.PortForward(t, "gameplane-system", "svc/gameplane-api", 80) + if !reconnected && attempt+1 < maxAttempts && ctx.Err() == nil && loginDialRefused(err) { + local, stop = e.portForwardContext(ctx, t, "gameplane-system", "svc/gameplane-api", 80) base = fmt.Sprintf("http://127.0.0.1:%d", local) reconnected = true continue diff --git a/test/e2e/upgrade_e2e_test.go b/test/e2e/upgrade_e2e_test.go index 263f98c04..09fbf6b9a 100644 --- a/test/e2e/upgrade_e2e_test.go +++ b/test/e2e/upgrade_e2e_test.go @@ -52,7 +52,7 @@ import ( // also bring them up to date. It runs last so steps 1-6 start from the old // release's genuinely stale schema. func TestUpgrade_FromPreviousRelease(t *testing.T) { - ctx := context.Background() + ctx := t.Context() ns := "gameplane-games" tmpl := "e2e-upgrade-busybox" gs := "e2e-upgrade-target" @@ -74,6 +74,7 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { applyBusyboxGameServer(t, ns, gs, tmpl) waitPVCBound(t, ns, gs+"-data", 90*time.Second) waitGameContainerReady(t, ns, gs+"-0", 2*time.Minute) + beforeIdentity := readUpgradeIdentity(ctx, t, "gameplane-agent-ca") legacyReadBinding := nativeReadBinding(t) checkNativeCatalogAccess(t, tmpl, true) @@ -160,6 +161,13 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { t.Fatalf("operator strategy migration retained RollingUpdate fields: %+v", operator.Spec.Strategy) } + afterIdentity := readUpgradeIdentity(ctx, t, "gameplane-agent-ca") + for name, before := range beforeIdentity { + if afterIdentity[name] != before { + t.Errorf("upgrade changed %s identity: before %s, after %s", name, before, afterIdentity[name]) + } + } + // ---- 4. the CRDs were actually updated ------------------------------- // This is the hook's literal contract. Without it Helm leaves the old @@ -278,6 +286,13 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { waitAgentReachable(t, localClient, gs) verifyLocalClusterDiscovery(t, localClient) + if os.Getenv("GAMEPLANE_E2E_ARGOCD") == "1" { + t.Run("ArgoCD", func(t *testing.T) { + checkArgoUpgrade(t, ns, gs, marker, adminUser, adminPass, credentialsNS) + checkNativeCatalogAccess(t, tmpl, false) + }) + } + // ---- 7. F-218: a fresh `helm install` over leftover, STALE CRDs ------ // // Deliberately LAST, after steps 3-6 have proven the pre-upgrade hook diff --git a/test/e2e/upgrade_identity_e2e_test.go b/test/e2e/upgrade_identity_e2e_test.go new file mode 100644 index 000000000..f95a62a22 --- /dev/null +++ b/test/e2e/upgrade_identity_e2e_test.go @@ -0,0 +1,62 @@ +//go:build e2e + +package e2e + +import ( + "context" + "crypto/sha256" + "fmt" + "slices" + "strings" + "testing" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// Snapshot the populated installation's storage and trust identities. Remaining +// Gameplane CRDs must include every version still recorded as stored by Kubernetes. +func readUpgradeIdentity(ctx context.Context, t *testing.T, issuerSecret string) map[string]string { + t.Helper() + identity := map[string]string{} + pvc, err := envInstance.K8s.CoreV1().PersistentVolumeClaims("gameplane-system").Get(ctx, "gameplane-api-data", metav1.GetOptions{}) + if err != nil || pvc.UID == "" { + t.Fatalf("read gameplane-system/gameplane-api-data PVC identity: %v", err) + } + identity["gameplane-system/gameplane-api-data"] = string(pvc.UID) + ca, err := envInstance.K8s.CoreV1().Secrets("gameplane-system").Get(ctx, issuerSecret, metav1.GetOptions{}) + if err != nil || len(ca.Data["ca.crt"]) == 0 { + t.Fatalf("read public agent CA certificate: %v", err) + } + identity["agent CA fingerprint"] = fmt.Sprintf("%x", sha256.Sum256(ca.Data["ca.crt"])) + crds, err := envInstance.Dyn.Resource(crdGVR).List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + for _, crd := range crds.Items { + if !strings.HasSuffix(crd.GetName(), ".gameplane.local") { + continue + } + if crd.GetUID() == "" { + t.Fatalf("%s has no UID", crd.GetName()) + } + identity[crd.GetName()] = string(crd.GetUID()) + var typed apiextensionsv1.CustomResourceDefinition + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(crd.Object, &typed); err != nil { + t.Fatal(err) + } + if len(typed.Status.StoredVersions) == 0 { + t.Fatalf("%s has no storedVersions", crd.GetName()) + } + for _, stored := range typed.Status.StoredVersions { + if !slices.ContainsFunc(typed.Spec.Versions, func(v apiextensionsv1.CustomResourceDefinitionVersion) bool { return v.Name == stored }) { + t.Fatalf("%s stored version %s is absent from the schema", crd.GetName(), stored) + } + } + } + if identity["gameservers.gameplane.local"] == "" { + t.Fatal("missing GameServer CRD") + } + return identity +}