From baf76118e8e9d6093a277c3d90adc58c6aad534d Mon Sep 17 00:00:00 2001 From: Baylem Date: Tue, 10 Nov 2026 21:07:47 -0500 Subject: [PATCH 1/3] fix(chart): make native Argo CD renders deterministic Signed-off-by: Baylem --- .github/workflows/ci.yaml | 2 + charts/gameplane/examples/argocd-values.yaml | 13 +++ charts/gameplane/templates/_helpers.tpl | 5 ++ charts/gameplane/templates/api.yaml | 3 + charts/gameplane/templates/mtls.yaml | 2 + charts/gameplane/templates/namespaces.yaml | 3 + .../gameplane/templates/validate-argocd.yaml | 21 +++++ charts/gameplane/values.yaml | 5 ++ charts/tests/check_argocd.py | 82 +++++++++++++++++++ docs/argocd.md | 44 ++++++++++ docs/install.md | 2 + test/e2e/upgrade_e2e_test.go | 61 ++++++++++++++ 12 files changed, 243 insertions(+) create mode 100644 charts/gameplane/examples/argocd-values.yaml create mode 100644 charts/gameplane/templates/validate-argocd.yaml create mode 100644 charts/tests/check_argocd.py create mode 100644 docs/argocd.md diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8ce0fa057..af1f7b002 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1261,6 +1261,8 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - run: helm template gameplane charts/gameplane > /tmp/rendered.yaml + - name: deterministic offline native chart and upgrade renders + run: python3 charts/tests/check_argocd.py - name: gateway isolation and API-disabled chart renders run: python3 charts/tests/check_gateway.py - name: standalone panel isolation and persistence chart renders diff --git a/charts/gameplane/examples/argocd-values.yaml b/charts/gameplane/examples/argocd-values.yaml new file mode 100644 index 000000000..0dfe686c7 --- /dev/null +++ b/charts/gameplane/examples/argocd-values.yaml @@ -0,0 +1,13 @@ +# Native Argo CD profile. Pre-create these immutable Secrets and set the +# workload Application's source.helm.skipCrds=true (see docs/argocd.md). +argoCD: + enabled: true +crds: + autoApply: + enabled: false +api: + agentMTLS: + caSecretRef: + name: gameplane-external-ca-v1 + clientCertRef: + name: gameplane-external-client-v1 diff --git a/charts/gameplane/templates/_helpers.tpl b/charts/gameplane/templates/_helpers.tpl index 44bf31ee3..0682af346 100644 --- a/charts/gameplane/templates/_helpers.tpl +++ b/charts/gameplane/templates/_helpers.tpl @@ -107,6 +107,10 @@ CA or client Secret it references. mtls.yaml does not render those Secrets, so without them rotating a custom Secret would not roll the API and operator. */}} {{- define "gameplane.mtlsChecksum" -}} +{{- if (.Values.argoCD | default dict).enabled -}} +{{- /* Immutable external Secret names are the rotation inputs available to an offline renderer. */ -}} +{{- list .Values.api.agentMTLS.caSecretRef.name .Values.api.agentMTLS.clientCertRef.name | toJson | sha256sum -}} +{{- else -}} {{- $parts := list (include (print .Template.BasePath "/mtls.yaml") .) -}} {{- $m := .Values.api.agentMTLS -}} {{- range $name := list ($m.caSecretRef.name | default "") ($m.clientCertRef.name | default "") -}} @@ -117,6 +121,7 @@ without them rotating a custom Secret would not roll the API and operator. {{- end -}} {{- join "\n---\n" $parts | sha256sum -}} {{- end -}} +{{- end -}} {{- /* Absent on releases predating optional gateway installs (--reuse-values). */}} {{- define "gameplane.apiEnabled" -}} diff --git a/charts/gameplane/templates/api.yaml b/charts/gameplane/templates/api.yaml index 1ba3e8209..dd0285fae 100644 --- a/charts/gameplane/templates/api.yaml +++ b/charts/gameplane/templates/api.yaml @@ -212,6 +212,9 @@ metadata: labels: {{- include "gameplane.labels" . | nindent 4 }} annotations: helm.sh/resource-policy: keep + {{- if (.Values.argoCD | default dict).enabled }} + argocd.argoproj.io/sync-options: Prune=false,Delete=false + {{- end }} spec: accessModes: [ReadWriteOnce] resources: diff --git a/charts/gameplane/templates/mtls.yaml b/charts/gameplane/templates/mtls.yaml index 3eda0bbcd..27fe933c3 100644 --- a/charts/gameplane/templates/mtls.yaml +++ b/charts/gameplane/templates/mtls.yaml @@ -1,4 +1,5 @@ {{- if or (eq (include "gameplane.operatorEnabled" .) "true") (and (eq (include "gameplane.apiEnabled" .) "true") (not .Values.api.standalone)) }} +{{- if not (and (.Values.argoCD | default dict).enabled (ne (include "gameplane.liveCluster" .) "true")) }} {{- /* Agent mTLS material. @@ -112,3 +113,4 @@ data: {{- end }} {{- end }} +{{- end }} diff --git a/charts/gameplane/templates/namespaces.yaml b/charts/gameplane/templates/namespaces.yaml index b73cce0bd..a60ee9427 100644 --- a/charts/gameplane/templates/namespaces.yaml +++ b/charts/gameplane/templates/namespaces.yaml @@ -15,5 +15,8 @@ metadata: # StatefulSet, PVC and per-server Secret Kubernetes garbage-collects # underneath it). Same pattern as the API PVC in api.yaml. helm.sh/resource-policy: keep + {{- if (.Values.argoCD | default dict).enabled }} + argocd.argoproj.io/sync-options: Prune=false,Delete=false + {{- end }} {{- end }} diff --git a/charts/gameplane/templates/validate-argocd.yaml b/charts/gameplane/templates/validate-argocd.yaml new file mode 100644 index 000000000..ddb84587b --- /dev/null +++ b/charts/gameplane/templates/validate-argocd.yaml @@ -0,0 +1,21 @@ +{{- if (.Values.argoCD | default dict).enabled }} +{{- if or .Values.api.standalone (ne (include "gameplane.operatorEnabled" .) "true") }} +{{- fail "argoCD.enabled is the native operator delivery profile" }} +{{- end }} +{{- if .Values.crds.autoApply.enabled }} +{{- fail "argoCD.enabled requires crds.autoApply.enabled=false and one external CRD owner" }} +{{- end }} +{{- $mtls := .Values.api.agentMTLS }} +{{- $ca := $mtls.caSecretRef.name | default "" }} +{{- $client := $mtls.clientCertRef.name | default "" }} +{{- $generated := list "" "gameplane-agent-ca" "gameplane-agent-client" }} +{{- if or (has $ca $generated) (has $client $generated) (eq $ca $client) }} +{{- fail "argoCD.enabled requires distinct external CA and client Secret names in api.agentMTLS" }} +{{- end }} +{{- range $kind := list "ValidatingAdmissionPolicy" "ValidatingAdmissionPolicyBinding" }} +{{- $api := printf "admissionregistration.k8s.io/v1/%s" $kind }} +{{- if not ($.Capabilities.APIVersions.Has $api) }} +{{- fail (printf "argoCD.enabled requires target API capability %s for waker enforcement" $api) }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index 503c95481..9aaa4ed2b 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -11,6 +11,11 @@ ## serviceMonitors — Prometheus Operator integration ## podSecurity — cluster-wide hardening knobs +# Offline rendering and resource retention for an Argo CD native installation. +# See examples/argocd-values.yaml and docs/argocd.md. +argoCD: + enabled: false + image: # Component images are published to {registry}/{operator,api,agent} by the # release workflow. The release matches this default; override only to pin a diff --git a/charts/tests/check_argocd.py b/charts/tests/check_argocd.py new file mode 100644 index 000000000..c4f965a9e --- /dev/null +++ b/charts/tests/check_argocd.py @@ -0,0 +1,82 @@ +"""CI contract for Argo CD's offline native Helm profile.""" +import subprocess + +import yaml + +from check_gateway import CHART, one + + +PROFILE = CHART / "examples" / "argocd-values.yaml" +ADMISSION = "admissionregistration.k8s.io/v1/" +POLICY = ADMISSION + "ValidatingAdmissionPolicy" +BINDING = ADMISSION + "ValidatingAdmissionPolicyBinding" + + +def render(*flags, capabilities=(POLICY, BINDING), failure=None): + cmd = ["helm", "template", "gameplane", str(CHART), "--namespace", "gameplane-system", + "--kube-version", "1.34.0", "--values", str(PROFILE)] + for capability in capabilities: + cmd.extend(["--api-versions", capability]) + result = subprocess.run(cmd + list(flags), capture_output=True, check=False) + if failure: + assert result.returncode and failure.encode() in result.stderr, result.stderr.decode() + else: + result.check_returncode() + return result.stdout + + +def objects(output): + return [obj for obj in yaml.safe_load_all(output) if obj] + + +def check_trust(resources, ca, client): + checksums = [] + for name in ("gameplane-api", "gameplane-operator"): + template = one(resources, "Deployment", name)["spec"]["template"] + volumes = {v["name"]: v for v in template["spec"]["volumes"]} + assert volumes["agent-ca"]["secret"]["secretName"] == ca + assert volumes["agent-client"]["secret"]["secretName"] == client + checksums.append(template["metadata"]["annotations"]["checksum/mtls"]) + assert checksums[0] == checksums[1] + return checksums[0] + + +def main(): + output = render() + assert output == render() == render("--is-upgrade"), "render depends on time, randomness or release state" + resources = objects(output) + for obj in resources: + assert obj["kind"] != "CustomResourceDefinition", obj["metadata"]["name"] + assert "helm.sh/hook" not in obj["metadata"].get("annotations", {}), obj["metadata"]["name"] + assert not (obj["kind"] == "Secret" and {"ca.key", "tls.key"} & obj.get("data", {}).keys()) + policy = one(resources, "ValidatingAdmissionPolicy", "gameplane-waker-wake-only") + binding = one(resources, "ValidatingAdmissionPolicyBinding", "gameplane-waker-wake-only") + assert policy["spec"]["failurePolicy"] == "Fail" + assert binding["spec"]["policyName"] == policy["metadata"]["name"] + assert "Deny" in binding["spec"]["validationActions"] + for kind, name in (("PersistentVolumeClaim", "gameplane-api-data"), ("Namespace", "gameplane-games")): + annotations = one(resources, kind, name)["metadata"]["annotations"] + assert set(annotations["argocd.argoproj.io/sync-options"].split(",")) == {"Prune=false", "Delete=false"} + current = check_trust(resources, "gameplane-external-ca-v1", "gameplane-external-client-v1") + rotated = objects(render("--set", "api.agentMTLS.caSecretRef.name=external-ca-v2", + "--set", "api.agentMTLS.clientCertRef.name=external-client-v2")) + assert check_trust(rotated, "external-ca-v2", "external-client-v2") != current + for capabilities, missing in (((), POLICY), ((POLICY,), BINDING), ((BINDING,), POLICY)): + render(capabilities=capabilities, failure=missing) + for path, value, reason in ( + ("api.agentMTLS.caSecretRef.name", "gameplane-agent-ca", "distinct external"), + ("api.agentMTLS.clientCertRef.name", "gameplane-agent-client", "distinct external"), + ("api.agentMTLS.clientCertRef.name", "gameplane-external-ca-v1", "distinct external"), + ("crds.autoApply.enabled", "true", "one external CRD owner"), + ): + render("--set", f"{path}={value}", failure=reason) + handoff = objects(render("--is-upgrade", "--set", "api.storage.existingClaim=gameplane-api-data")) + assert not any(obj["kind"] == "PersistentVolumeClaim" and obj["metadata"]["name"] == "gameplane-api-data" for obj in handoff) + api = one(handoff, "Deployment", "gameplane-api") + claims = [vol["persistentVolumeClaim"]["claimName"] for vol in api["spec"]["template"]["spec"]["volumes"] + if "persistentVolumeClaim" in vol] + assert "gameplane-api-data" in claims + + +if __name__ == "__main__": + main() diff --git a/docs/argocd.md b/docs/argocd.md new file mode 100644 index 000000000..4d44a0afc --- /dev/null +++ b/docs/argocd.md @@ -0,0 +1,44 @@ +# Argo CD delivery of a native Gameplane installation + +Use the chart’s Argo CD values for a combined API and operator installation. The profile uses two externally managed mTLS Secrets and requires discovery of the GA admission policy and binding APIs. It also gives the games Namespace and chart-managed SQLite claim explicit Argo CD retention annotations. The fixture covers Helm rendering and the populated Helm upgrade path. A populated Argo CD sync and prune drill is tracked in [#882](https://github.com/GameplanePanel/Gameplane/issues/882). + +## One owner for each resource + +Pin both Applications to the same Gameplane Git commit. Sync the CRD Application first and wait for its definitions to be Established before syncing the workload Application. + +| Application | Source | Settings | +| --- | --- | --- | +| CRDs | Plain directory `charts/gameplane/charts/gameplane-crds/crds` | Use server-side apply for the large schemas. Set `spec.syncPolicy.syncOptions` to include `ServerSideApply=true`, `Prune=false` and `Delete=false`. | +| Workload | Helm chart `charts/gameplane` | Set `spec.source.helm.skipCrds: true` and include `examples/argocd-values.yaml` in `spec.source.helm.valueFiles`. The profile disables the CRD apply hook. Set the release name to `gameplane` and the destination namespace to `gameplane-system`. | + +Provision the release namespace, CA and client Secret before syncing. Keep ownership of the release namespace and external Secrets outside the workload Application. The example expects `gameplane-external-ca-v1` with `ca.crt` and `ca.key`, and a different Secret named `gameplane-external-client-v1` with `tls.crt` and `tls.key`. Use immutable, versioned Secret names. The operator signs per-game serving certificates with this CA, and the API and operator present the client certificate to the agents. + +Argo CD ordinarily passes the destination cluster’s API capabilities to Helm. Verify that both `admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy` and `admissionregistration.k8s.io/v1/ValidatingAdmissionPolicyBinding` are served and that admission is enabled. For an offline comparison, pass the discovered capabilities and the destination Kubernetes version. This example assumes both kinds on Kubernetes 1.34. + +```sh +for run in 1 2; do + helm template gameplane charts/gameplane -n gameplane-system \ + -f charts/gameplane/examples/argocd-values.yaml --kube-version 1.34.0 \ + --api-versions admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy \ + --api-versions admissionregistration.k8s.io/v1/ValidatingAdmissionPolicyBinding \ + > "gameplane-$run.yaml" +done +cmp gameplane-1.yaml gameplane-2.yaml +``` + +The chart fails the profile render if either kind is missing. Supplying a capability flag locally does not install that API on the destination. See the [Argo CD Helm source parameters](https://argo-cd.readthedocs.io/en/stable/user-guide/application-specification/) and [Helm hook mapping](https://argo-cd.readthedocs.io/en/stable/user-guide/helm/#helm-hooks). + +## Adopting a populated installation + +1. Back up the database and game data. Record the API and game claim UIDs, the current CA public certificate fingerprint, and each Gameplane CRD’s UID and `status.storedVersions`. Verify each stored version appears in the target CRD’s `spec.versions`. A removed stored version needs a storage-version migration first. +2. Copy the current CA and client material into the two external Secrets through your secret manager. Compare the CA’s public certificate digest and test the client against a live agent. Keep the existing trust pair for the initial handoff. Disable the prior delivery owner so it cannot concurrently change these resources. +3. Check both Deployments’ live update strategy before handoff. Offline rendering cannot detect an old server-defaulted `spec.strategy.rollingUpdate`. When the desired strategy is `Recreate`, migrate the live Deployment through the existing Helm upgrade path or a controlled merge patch that sets `type: Recreate` and `rollingUpdate: null` together. +4. Sync with pruning disabled. Keep the SQLite claim name and size from the populated release. After the first sync, verify the live games Namespace and the live API PVC both carry `argocd.argoproj.io/sync-options: Prune=false,Delete=false`. If switching to `api.storage.existingClaim`, verify retention on the **old live claim before** the sync that removes its manifest. Point the API at that same populated PVC, or migrate and verify the data separately first. +5. Review the workload prune preview. Compare the UIDs and CA fingerprint from step 1, log in with a pre-upgrade account, read a persisted game marker and make an authenticated call to the running game’s agent. On a wake-enabled test GameServer, impersonate its waker ServiceAccount for server-side dry-run patches. Its wake annotation should succeed and a spec mutation should be denied by the wake policy. Verify the policy and binding names and denial reason. +6. Enable workload pruning after reviewing the retained objects. Argo CD may report retained resources as OutOfSync. `Prune=false` preserves an object removed from Git during sync. `Delete=false` preserves it on Application deletion. Helm’s `helm.sh/resource-policy: keep` maps to deletion protection in Argo CD. Deleting a containing namespace can still remove its claims. See [Argo CD sync options](https://argo-cd.readthedocs.io/en/stable/user-guide/sync-options/). + +## Rotating external certificates + +For a client renewal under the same CA, stage a new immutable client Secret and change its reference in Git. For a CA rollover, schedule a maintenance window, stage a new CA and matching client Secret, then change both references in one revision. The reference change rolls the API and operator. The operator reissues each GameServer’s agent TLS Secret against the active CA. Kubelet projection and the next TLS handshake determine when each agent starts using the new trust. + +Finish the rollout by checking the new CA fingerprint on each `-agent-tls` Secret and making a fresh authenticated agent call for every running game. A stopped game needs that check on its next start. Retain the previous Secret pair until the checks finish. If rolling back the CA, restore both references together and repeat the per-agent checks while certificates are reissued. diff --git a/docs/install.md b/docs/install.md index f0b5767d7..dd0f5e426 100644 --- a/docs/install.md +++ b/docs/install.md @@ -5,6 +5,8 @@ For a central dashboard/API on a host without Kubernetes, use the panel-only Helm profile. The instructions below describe the default combined installation, with an operator and a local game cluster. +For an offline Helm renderer, see the [Argo CD delivery guide](argocd.md). + ## Prerequisites - Kubernetes 1.28+ diff --git a/test/e2e/upgrade_e2e_test.go b/test/e2e/upgrade_e2e_test.go index 56736b34c..75a24f763 100644 --- a/test/e2e/upgrade_e2e_test.go +++ b/test/e2e/upgrade_e2e_test.go @@ -4,18 +4,23 @@ package e2e import ( "context" + "crypto/sha256" "encoding/json" + "fmt" "os" "os/exec" "path/filepath" + "slices" "strconv" "strings" "testing" "time" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/yaml" ) @@ -69,6 +74,7 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { applyBusyboxGameServer(t, ns, gs, tmpl) waitPVCBound(t, ns, gs+"-data", 90*time.Second) waitGameContainerReady(t, ns, gs+"-0", 2*time.Minute) + beforeIdentity := readUpgradeIdentity(t, ns, gs) // `sync` so the bytes are on the PVC's backing store before the pod is // restarted by the upgrade, not sitting in the page cache. @@ -147,6 +153,13 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { t.Fatalf("operator strategy migration retained RollingUpdate fields: %+v", operator.Spec.Strategy) } + afterIdentity := readUpgradeIdentity(t, ns, gs) + for name, before := range beforeIdentity { + if afterIdentity[name] != before { + t.Errorf("upgrade changed %s identity: before %s, after %s", name, before, afterIdentity[name]) + } + } + // ---- 4. the CRDs were actually updated ------------------------------- // This is the hook's literal contract. Without it Helm leaves the old @@ -448,6 +461,54 @@ func waitGameContainerReady(t *testing.T, ns, pod string, timeout time.Duration) }) } +// Snapshot the populated installation's storage and trust identities. Remaining +// Gameplane CRDs must include every version still recorded as stored by Kubernetes. +func readUpgradeIdentity(t *testing.T, gamesNamespace, gs string) map[string]string { + t.Helper() + identity := map[string]string{} + for _, claim := range [][2]string{{"gameplane-system", "gameplane-api-data"}, {gamesNamespace, gs + "-data"}} { + pvc, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(claim[0]).Get(t.Context(), claim[1], metav1.GetOptions{}) + if err != nil || pvc.UID == "" { + t.Fatalf("read PVC identity %s/%s: %v", claim[0], claim[1], err) + } + identity[claim[0]+"/"+claim[1]] = string(pvc.UID) + } + ca, err := envInstance.K8s.CoreV1().Secrets("gameplane-system").Get(t.Context(), "gameplane-agent-ca", metav1.GetOptions{}) + if err != nil || len(ca.Data["ca.crt"]) == 0 { + t.Fatalf("read public agent CA certificate: %v", err) + } + identity["agent CA fingerprint"] = fmt.Sprintf("%x", sha256.Sum256(ca.Data["ca.crt"])) + crds, err := envInstance.Dyn.Resource(crdGVR).List(t.Context(), metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + for _, crd := range crds.Items { + if !strings.HasSuffix(crd.GetName(), ".gameplane.local") { + continue + } + if crd.GetUID() == "" { + t.Fatalf("%s has no UID", crd.GetName()) + } + identity[crd.GetName()] = string(crd.GetUID()) + var typed apiextensionsv1.CustomResourceDefinition + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(crd.Object, &typed); err != nil { + t.Fatal(err) + } + if len(typed.Status.StoredVersions) == 0 { + t.Fatalf("%s has no storedVersions", crd.GetName()) + } + for _, stored := range typed.Status.StoredVersions { + if !slices.ContainsFunc(typed.Spec.Versions, func(v apiextensionsv1.CustomResourceDefinitionVersion) bool { return v.Name == stored }) { + t.Fatalf("%s stored version %s is absent from the schema", crd.GetName(), stored) + } + } + } + if identity["gameservers.gameplane.local"] == "" { + t.Fatal("missing GameServer CRD") + } + return identity +} + // crdSpecProperties returns the top-level property names under the CRD's // spec schema, as the apiserver currently serves it. func crdSpecProperties(t *testing.T, crdName string) map[string]struct{} { From 48b20d5a7373ae09cf90e275901e7af1f8bb3352 Mon Sep 17 00:00:00 2001 From: Baylem Date: Tue, 10 Nov 2026 21:18:48 -0500 Subject: [PATCH 2/3] test(e2e): isolate upgrade identity checks from security fixtures Signed-off-by: Baylem --- test/e2e/upgrade_e2e_test.go | 53 ---------------------- test/e2e/upgrade_identity_e2e_test.go | 63 +++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 53 deletions(-) create mode 100644 test/e2e/upgrade_identity_e2e_test.go diff --git a/test/e2e/upgrade_e2e_test.go b/test/e2e/upgrade_e2e_test.go index 75a24f763..9fadced50 100644 --- a/test/e2e/upgrade_e2e_test.go +++ b/test/e2e/upgrade_e2e_test.go @@ -4,23 +4,18 @@ package e2e import ( "context" - "crypto/sha256" "encoding/json" - "fmt" "os" "os/exec" "path/filepath" - "slices" "strconv" "strings" "testing" "time" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/yaml" ) @@ -461,54 +456,6 @@ func waitGameContainerReady(t *testing.T, ns, pod string, timeout time.Duration) }) } -// Snapshot the populated installation's storage and trust identities. Remaining -// Gameplane CRDs must include every version still recorded as stored by Kubernetes. -func readUpgradeIdentity(t *testing.T, gamesNamespace, gs string) map[string]string { - t.Helper() - identity := map[string]string{} - for _, claim := range [][2]string{{"gameplane-system", "gameplane-api-data"}, {gamesNamespace, gs + "-data"}} { - pvc, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(claim[0]).Get(t.Context(), claim[1], metav1.GetOptions{}) - if err != nil || pvc.UID == "" { - t.Fatalf("read PVC identity %s/%s: %v", claim[0], claim[1], err) - } - identity[claim[0]+"/"+claim[1]] = string(pvc.UID) - } - ca, err := envInstance.K8s.CoreV1().Secrets("gameplane-system").Get(t.Context(), "gameplane-agent-ca", metav1.GetOptions{}) - if err != nil || len(ca.Data["ca.crt"]) == 0 { - t.Fatalf("read public agent CA certificate: %v", err) - } - identity["agent CA fingerprint"] = fmt.Sprintf("%x", sha256.Sum256(ca.Data["ca.crt"])) - crds, err := envInstance.Dyn.Resource(crdGVR).List(t.Context(), metav1.ListOptions{}) - if err != nil { - t.Fatal(err) - } - for _, crd := range crds.Items { - if !strings.HasSuffix(crd.GetName(), ".gameplane.local") { - continue - } - if crd.GetUID() == "" { - t.Fatalf("%s has no UID", crd.GetName()) - } - identity[crd.GetName()] = string(crd.GetUID()) - var typed apiextensionsv1.CustomResourceDefinition - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(crd.Object, &typed); err != nil { - t.Fatal(err) - } - if len(typed.Status.StoredVersions) == 0 { - t.Fatalf("%s has no storedVersions", crd.GetName()) - } - for _, stored := range typed.Status.StoredVersions { - if !slices.ContainsFunc(typed.Spec.Versions, func(v apiextensionsv1.CustomResourceDefinitionVersion) bool { return v.Name == stored }) { - t.Fatalf("%s stored version %s is absent from the schema", crd.GetName(), stored) - } - } - } - if identity["gameservers.gameplane.local"] == "" { - t.Fatal("missing GameServer CRD") - } - return identity -} - // crdSpecProperties returns the top-level property names under the CRD's // spec schema, as the apiserver currently serves it. func crdSpecProperties(t *testing.T, crdName string) map[string]struct{} { diff --git a/test/e2e/upgrade_identity_e2e_test.go b/test/e2e/upgrade_identity_e2e_test.go new file mode 100644 index 000000000..1f0038e1e --- /dev/null +++ b/test/e2e/upgrade_identity_e2e_test.go @@ -0,0 +1,63 @@ +//go:build e2e + +package e2e + +import ( + "crypto/sha256" + "fmt" + "slices" + "strings" + "testing" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// Snapshot the populated installation's storage and trust identities. Remaining +// Gameplane CRDs must include every version still recorded as stored by Kubernetes. +func readUpgradeIdentity(t *testing.T, gamesNamespace, gs string) map[string]string { + t.Helper() + identity := map[string]string{} + for _, claim := range [][2]string{{"gameplane-system", "gameplane-api-data"}, {gamesNamespace, gs + "-data"}} { + pvc, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(claim[0]).Get(t.Context(), claim[1], metav1.GetOptions{}) + if err != nil || pvc.UID == "" { + t.Fatalf("read PVC identity %s/%s: %v", claim[0], claim[1], err) + } + identity[claim[0]+"/"+claim[1]] = string(pvc.UID) + } + ca, err := envInstance.K8s.CoreV1().Secrets("gameplane-system").Get(t.Context(), "gameplane-agent-ca", metav1.GetOptions{}) + if err != nil || len(ca.Data["ca.crt"]) == 0 { + t.Fatalf("read public agent CA certificate: %v", err) + } + identity["agent CA fingerprint"] = fmt.Sprintf("%x", sha256.Sum256(ca.Data["ca.crt"])) + crds, err := envInstance.Dyn.Resource(crdGVR).List(t.Context(), metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + for _, crd := range crds.Items { + if !strings.HasSuffix(crd.GetName(), ".gameplane.local") { + continue + } + if crd.GetUID() == "" { + t.Fatalf("%s has no UID", crd.GetName()) + } + identity[crd.GetName()] = string(crd.GetUID()) + var typed apiextensionsv1.CustomResourceDefinition + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(crd.Object, &typed); err != nil { + t.Fatal(err) + } + if len(typed.Status.StoredVersions) == 0 { + t.Fatalf("%s has no storedVersions", crd.GetName()) + } + for _, stored := range typed.Status.StoredVersions { + if !slices.ContainsFunc(typed.Spec.Versions, func(v apiextensionsv1.CustomResourceDefinitionVersion) bool { return v.Name == stored }) { + t.Fatalf("%s stored version %s is absent from the schema", crd.GetName(), stored) + } + } + } + if identity["gameservers.gameplane.local"] == "" { + t.Fatal("missing GameServer CRD") + } + return identity +} From b171d6c139f5ba4ca5f165f98ba0495fd3ba3dc3 Mon Sep 17 00:00:00 2001 From: Baylem Date: Tue, 10 Nov 2026 21:22:44 -0500 Subject: [PATCH 3/3] test(e2e): keep game claim identity in the shared release check Signed-off-by: Baylem --- test/e2e/upgrade_e2e_test.go | 4 ++-- test/e2e/upgrade_identity_e2e_test.go | 12 +++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/test/e2e/upgrade_e2e_test.go b/test/e2e/upgrade_e2e_test.go index 9fadced50..5003aea4b 100644 --- a/test/e2e/upgrade_e2e_test.go +++ b/test/e2e/upgrade_e2e_test.go @@ -69,7 +69,7 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { applyBusyboxGameServer(t, ns, gs, tmpl) waitPVCBound(t, ns, gs+"-data", 90*time.Second) waitGameContainerReady(t, ns, gs+"-0", 2*time.Minute) - beforeIdentity := readUpgradeIdentity(t, ns, gs) + beforeIdentity := readUpgradeIdentity(t) // `sync` so the bytes are on the PVC's backing store before the pod is // restarted by the upgrade, not sitting in the page cache. @@ -148,7 +148,7 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { t.Fatalf("operator strategy migration retained RollingUpdate fields: %+v", operator.Spec.Strategy) } - afterIdentity := readUpgradeIdentity(t, ns, gs) + afterIdentity := readUpgradeIdentity(t) for name, before := range beforeIdentity { if afterIdentity[name] != before { t.Errorf("upgrade changed %s identity: before %s, after %s", name, before, afterIdentity[name]) diff --git a/test/e2e/upgrade_identity_e2e_test.go b/test/e2e/upgrade_identity_e2e_test.go index 1f0038e1e..2a7a73bd9 100644 --- a/test/e2e/upgrade_identity_e2e_test.go +++ b/test/e2e/upgrade_identity_e2e_test.go @@ -16,16 +16,14 @@ import ( // Snapshot the populated installation's storage and trust identities. Remaining // Gameplane CRDs must include every version still recorded as stored by Kubernetes. -func readUpgradeIdentity(t *testing.T, gamesNamespace, gs string) map[string]string { +func readUpgradeIdentity(t *testing.T) map[string]string { t.Helper() identity := map[string]string{} - for _, claim := range [][2]string{{"gameplane-system", "gameplane-api-data"}, {gamesNamespace, gs + "-data"}} { - pvc, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(claim[0]).Get(t.Context(), claim[1], metav1.GetOptions{}) - if err != nil || pvc.UID == "" { - t.Fatalf("read PVC identity %s/%s: %v", claim[0], claim[1], err) - } - identity[claim[0]+"/"+claim[1]] = string(pvc.UID) + pvc, err := envInstance.K8s.CoreV1().PersistentVolumeClaims("gameplane-system").Get(t.Context(), "gameplane-api-data", metav1.GetOptions{}) + if err != nil || pvc.UID == "" { + t.Fatalf("read gameplane-system/gameplane-api-data PVC identity: %v", err) } + identity["gameplane-system/gameplane-api-data"] = string(pvc.UID) ca, err := envInstance.K8s.CoreV1().Secrets("gameplane-system").Get(t.Context(), "gameplane-agent-ca", metav1.GetOptions{}) if err != nil || len(ca.Data["ca.crt"]) == 0 { t.Fatalf("read public agent CA certificate: %v", err)