From baf76118e8e9d6093a277c3d90adc58c6aad534d Mon Sep 17 00:00:00 2001 From: Baylem Date: Tue, 10 Nov 2026 21:07:47 -0500 Subject: [PATCH 1/7] fix(chart): make native Argo CD renders deterministic Signed-off-by: Baylem --- .github/workflows/ci.yaml | 2 + charts/gameplane/examples/argocd-values.yaml | 13 +++ charts/gameplane/templates/_helpers.tpl | 5 ++ charts/gameplane/templates/api.yaml | 3 + charts/gameplane/templates/mtls.yaml | 2 + charts/gameplane/templates/namespaces.yaml | 3 + .../gameplane/templates/validate-argocd.yaml | 21 +++++ charts/gameplane/values.yaml | 5 ++ charts/tests/check_argocd.py | 82 +++++++++++++++++++ docs/argocd.md | 44 ++++++++++ docs/install.md | 2 + test/e2e/upgrade_e2e_test.go | 61 ++++++++++++++ 12 files changed, 243 insertions(+) create mode 100644 charts/gameplane/examples/argocd-values.yaml create mode 100644 charts/gameplane/templates/validate-argocd.yaml create mode 100644 charts/tests/check_argocd.py create mode 100644 docs/argocd.md diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8ce0fa057..af1f7b002 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1261,6 +1261,8 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - run: helm template gameplane charts/gameplane > /tmp/rendered.yaml + - name: deterministic offline native chart and upgrade renders + run: python3 charts/tests/check_argocd.py - name: gateway isolation and API-disabled chart renders run: python3 charts/tests/check_gateway.py - name: standalone panel isolation and persistence chart renders diff --git a/charts/gameplane/examples/argocd-values.yaml b/charts/gameplane/examples/argocd-values.yaml new file mode 100644 index 000000000..0dfe686c7 --- /dev/null +++ b/charts/gameplane/examples/argocd-values.yaml @@ -0,0 +1,13 @@ +# Native Argo CD profile. Pre-create these immutable Secrets and set the +# workload Application's source.helm.skipCrds=true (see docs/argocd.md). +argoCD: + enabled: true +crds: + autoApply: + enabled: false +api: + agentMTLS: + caSecretRef: + name: gameplane-external-ca-v1 + clientCertRef: + name: gameplane-external-client-v1 diff --git a/charts/gameplane/templates/_helpers.tpl b/charts/gameplane/templates/_helpers.tpl index 44bf31ee3..0682af346 100644 --- a/charts/gameplane/templates/_helpers.tpl +++ b/charts/gameplane/templates/_helpers.tpl @@ -107,6 +107,10 @@ CA or client Secret it references. mtls.yaml does not render those Secrets, so without them rotating a custom Secret would not roll the API and operator. */}} {{- define "gameplane.mtlsChecksum" -}} +{{- if (.Values.argoCD | default dict).enabled -}} +{{- /* Immutable external Secret names are the rotation inputs available to an offline renderer. */ -}} +{{- list .Values.api.agentMTLS.caSecretRef.name .Values.api.agentMTLS.clientCertRef.name | toJson | sha256sum -}} +{{- else -}} {{- $parts := list (include (print .Template.BasePath "/mtls.yaml") .) -}} {{- $m := .Values.api.agentMTLS -}} {{- range $name := list ($m.caSecretRef.name | default "") ($m.clientCertRef.name | default "") -}} @@ -117,6 +121,7 @@ without them rotating a custom Secret would not roll the API and operator. {{- end -}} {{- join "\n---\n" $parts | sha256sum -}} {{- end -}} +{{- end -}} {{- /* Absent on releases predating optional gateway installs (--reuse-values). */}} {{- define "gameplane.apiEnabled" -}} diff --git a/charts/gameplane/templates/api.yaml b/charts/gameplane/templates/api.yaml index 1ba3e8209..dd0285fae 100644 --- a/charts/gameplane/templates/api.yaml +++ b/charts/gameplane/templates/api.yaml @@ -212,6 +212,9 @@ metadata: labels: {{- include "gameplane.labels" . | nindent 4 }} annotations: helm.sh/resource-policy: keep + {{- if (.Values.argoCD | default dict).enabled }} + argocd.argoproj.io/sync-options: Prune=false,Delete=false + {{- end }} spec: accessModes: [ReadWriteOnce] resources: diff --git a/charts/gameplane/templates/mtls.yaml b/charts/gameplane/templates/mtls.yaml index 3eda0bbcd..27fe933c3 100644 --- a/charts/gameplane/templates/mtls.yaml +++ b/charts/gameplane/templates/mtls.yaml @@ -1,4 +1,5 @@ {{- if or (eq (include "gameplane.operatorEnabled" .) "true") (and (eq (include "gameplane.apiEnabled" .) "true") (not .Values.api.standalone)) }} +{{- if not (and (.Values.argoCD | default dict).enabled (ne (include "gameplane.liveCluster" .) "true")) }} {{- /* Agent mTLS material. @@ -112,3 +113,4 @@ data: {{- end }} {{- end }} +{{- end }} diff --git a/charts/gameplane/templates/namespaces.yaml b/charts/gameplane/templates/namespaces.yaml index b73cce0bd..a60ee9427 100644 --- a/charts/gameplane/templates/namespaces.yaml +++ b/charts/gameplane/templates/namespaces.yaml @@ -15,5 +15,8 @@ metadata: # StatefulSet, PVC and per-server Secret Kubernetes garbage-collects # underneath it). Same pattern as the API PVC in api.yaml. helm.sh/resource-policy: keep + {{- if (.Values.argoCD | default dict).enabled }} + argocd.argoproj.io/sync-options: Prune=false,Delete=false + {{- end }} {{- end }} diff --git a/charts/gameplane/templates/validate-argocd.yaml b/charts/gameplane/templates/validate-argocd.yaml new file mode 100644 index 000000000..ddb84587b --- /dev/null +++ b/charts/gameplane/templates/validate-argocd.yaml @@ -0,0 +1,21 @@ +{{- if (.Values.argoCD | default dict).enabled }} +{{- if or .Values.api.standalone (ne (include "gameplane.operatorEnabled" .) "true") }} +{{- fail "argoCD.enabled is the native operator delivery profile" }} +{{- end }} +{{- if .Values.crds.autoApply.enabled }} +{{- fail "argoCD.enabled requires crds.autoApply.enabled=false and one external CRD owner" }} +{{- end }} +{{- $mtls := .Values.api.agentMTLS }} +{{- $ca := $mtls.caSecretRef.name | default "" }} +{{- $client := $mtls.clientCertRef.name | default "" }} +{{- $generated := list "" "gameplane-agent-ca" "gameplane-agent-client" }} +{{- if or (has $ca $generated) (has $client $generated) (eq $ca $client) }} +{{- fail "argoCD.enabled requires distinct external CA and client Secret names in api.agentMTLS" }} +{{- end }} +{{- range $kind := list "ValidatingAdmissionPolicy" "ValidatingAdmissionPolicyBinding" }} +{{- $api := printf "admissionregistration.k8s.io/v1/%s" $kind }} +{{- if not ($.Capabilities.APIVersions.Has $api) }} +{{- fail (printf "argoCD.enabled requires target API capability %s for waker enforcement" $api) }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index 503c95481..9aaa4ed2b 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -11,6 +11,11 @@ ## serviceMonitors — Prometheus Operator integration ## podSecurity — cluster-wide hardening knobs +# Offline rendering and resource retention for an Argo CD native installation. +# See examples/argocd-values.yaml and docs/argocd.md. +argoCD: + enabled: false + image: # Component images are published to {registry}/{operator,api,agent} by the # release workflow. The release matches this default; override only to pin a diff --git a/charts/tests/check_argocd.py b/charts/tests/check_argocd.py new file mode 100644 index 000000000..c4f965a9e --- /dev/null +++ b/charts/tests/check_argocd.py @@ -0,0 +1,82 @@ +"""CI contract for Argo CD's offline native Helm profile.""" +import subprocess + +import yaml + +from check_gateway import CHART, one + + +PROFILE = CHART / "examples" / "argocd-values.yaml" +ADMISSION = "admissionregistration.k8s.io/v1/" +POLICY = ADMISSION + "ValidatingAdmissionPolicy" +BINDING = ADMISSION + "ValidatingAdmissionPolicyBinding" + + +def render(*flags, capabilities=(POLICY, BINDING), failure=None): + cmd = ["helm", "template", "gameplane", str(CHART), "--namespace", "gameplane-system", + "--kube-version", "1.34.0", "--values", str(PROFILE)] + for capability in capabilities: + cmd.extend(["--api-versions", capability]) + result = subprocess.run(cmd + list(flags), capture_output=True, check=False) + if failure: + assert result.returncode and failure.encode() in result.stderr, result.stderr.decode() + else: + result.check_returncode() + return result.stdout + + +def objects(output): + return [obj for obj in yaml.safe_load_all(output) if obj] + + +def check_trust(resources, ca, client): + checksums = [] + for name in ("gameplane-api", "gameplane-operator"): + template = one(resources, "Deployment", name)["spec"]["template"] + volumes = {v["name"]: v for v in template["spec"]["volumes"]} + assert volumes["agent-ca"]["secret"]["secretName"] == ca + assert volumes["agent-client"]["secret"]["secretName"] == client + checksums.append(template["metadata"]["annotations"]["checksum/mtls"]) + assert checksums[0] == checksums[1] + return checksums[0] + + +def main(): + output = render() + assert output == render() == render("--is-upgrade"), "render depends on time, randomness or release state" + resources = objects(output) + for obj in resources: + assert obj["kind"] != "CustomResourceDefinition", obj["metadata"]["name"] + assert "helm.sh/hook" not in obj["metadata"].get("annotations", {}), obj["metadata"]["name"] + assert not (obj["kind"] == "Secret" and {"ca.key", "tls.key"} & obj.get("data", {}).keys()) + policy = one(resources, "ValidatingAdmissionPolicy", "gameplane-waker-wake-only") + binding = one(resources, "ValidatingAdmissionPolicyBinding", "gameplane-waker-wake-only") + assert policy["spec"]["failurePolicy"] == "Fail" + assert binding["spec"]["policyName"] == policy["metadata"]["name"] + assert "Deny" in binding["spec"]["validationActions"] + for kind, name in (("PersistentVolumeClaim", "gameplane-api-data"), ("Namespace", "gameplane-games")): + annotations = one(resources, kind, name)["metadata"]["annotations"] + assert set(annotations["argocd.argoproj.io/sync-options"].split(",")) == {"Prune=false", "Delete=false"} + current = check_trust(resources, "gameplane-external-ca-v1", "gameplane-external-client-v1") + rotated = objects(render("--set", "api.agentMTLS.caSecretRef.name=external-ca-v2", + "--set", "api.agentMTLS.clientCertRef.name=external-client-v2")) + assert check_trust(rotated, "external-ca-v2", "external-client-v2") != current + for capabilities, missing in (((), POLICY), ((POLICY,), BINDING), ((BINDING,), POLICY)): + render(capabilities=capabilities, failure=missing) + for path, value, reason in ( + ("api.agentMTLS.caSecretRef.name", "gameplane-agent-ca", "distinct external"), + ("api.agentMTLS.clientCertRef.name", "gameplane-agent-client", "distinct external"), + ("api.agentMTLS.clientCertRef.name", "gameplane-external-ca-v1", "distinct external"), + ("crds.autoApply.enabled", "true", "one external CRD owner"), + ): + render("--set", f"{path}={value}", failure=reason) + handoff = objects(render("--is-upgrade", "--set", "api.storage.existingClaim=gameplane-api-data")) + assert not any(obj["kind"] == "PersistentVolumeClaim" and obj["metadata"]["name"] == "gameplane-api-data" for obj in handoff) + api = one(handoff, "Deployment", "gameplane-api") + claims = [vol["persistentVolumeClaim"]["claimName"] for vol in api["spec"]["template"]["spec"]["volumes"] + if "persistentVolumeClaim" in vol] + assert "gameplane-api-data" in claims + + +if __name__ == "__main__": + main() diff --git a/docs/argocd.md b/docs/argocd.md new file mode 100644 index 000000000..4d44a0afc --- /dev/null +++ b/docs/argocd.md @@ -0,0 +1,44 @@ +# Argo CD delivery of a native Gameplane installation + +Use the chart’s Argo CD values for a combined API and operator installation. The profile uses two externally managed mTLS Secrets and requires discovery of the GA admission policy and binding APIs. It also gives the games Namespace and chart-managed SQLite claim explicit Argo CD retention annotations. The fixture covers Helm rendering and the populated Helm upgrade path. A populated Argo CD sync and prune drill is tracked in [#882](https://github.com/GameplanePanel/Gameplane/issues/882). + +## One owner for each resource + +Pin both Applications to the same Gameplane Git commit. Sync the CRD Application first and wait for its definitions to be Established before syncing the workload Application. + +| Application | Source | Settings | +| --- | --- | --- | +| CRDs | Plain directory `charts/gameplane/charts/gameplane-crds/crds` | Use server-side apply for the large schemas. Set `spec.syncPolicy.syncOptions` to include `ServerSideApply=true`, `Prune=false` and `Delete=false`. | +| Workload | Helm chart `charts/gameplane` | Set `spec.source.helm.skipCrds: true` and include `examples/argocd-values.yaml` in `spec.source.helm.valueFiles`. The profile disables the CRD apply hook. Set the release name to `gameplane` and the destination namespace to `gameplane-system`. | + +Provision the release namespace, CA and client Secret before syncing. Keep ownership of the release namespace and external Secrets outside the workload Application. The example expects `gameplane-external-ca-v1` with `ca.crt` and `ca.key`, and a different Secret named `gameplane-external-client-v1` with `tls.crt` and `tls.key`. Use immutable, versioned Secret names. The operator signs per-game serving certificates with this CA, and the API and operator present the client certificate to the agents. + +Argo CD ordinarily passes the destination cluster’s API capabilities to Helm. Verify that both `admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy` and `admissionregistration.k8s.io/v1/ValidatingAdmissionPolicyBinding` are served and that admission is enabled. For an offline comparison, pass the discovered capabilities and the destination Kubernetes version. This example assumes both kinds on Kubernetes 1.34. + +```sh +for run in 1 2; do + helm template gameplane charts/gameplane -n gameplane-system \ + -f charts/gameplane/examples/argocd-values.yaml --kube-version 1.34.0 \ + --api-versions admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy \ + --api-versions admissionregistration.k8s.io/v1/ValidatingAdmissionPolicyBinding \ + > "gameplane-$run.yaml" +done +cmp gameplane-1.yaml gameplane-2.yaml +``` + +The chart fails the profile render if either kind is missing. Supplying a capability flag locally does not install that API on the destination. See the [Argo CD Helm source parameters](https://argo-cd.readthedocs.io/en/stable/user-guide/application-specification/) and [Helm hook mapping](https://argo-cd.readthedocs.io/en/stable/user-guide/helm/#helm-hooks). + +## Adopting a populated installation + +1. Back up the database and game data. Record the API and game claim UIDs, the current CA public certificate fingerprint, and each Gameplane CRD’s UID and `status.storedVersions`. Verify each stored version appears in the target CRD’s `spec.versions`. A removed stored version needs a storage-version migration first. +2. Copy the current CA and client material into the two external Secrets through your secret manager. Compare the CA’s public certificate digest and test the client against a live agent. Keep the existing trust pair for the initial handoff. Disable the prior delivery owner so it cannot concurrently change these resources. +3. Check both Deployments’ live update strategy before handoff. Offline rendering cannot detect an old server-defaulted `spec.strategy.rollingUpdate`. When the desired strategy is `Recreate`, migrate the live Deployment through the existing Helm upgrade path or a controlled merge patch that sets `type: Recreate` and `rollingUpdate: null` together. +4. Sync with pruning disabled. Keep the SQLite claim name and size from the populated release. After the first sync, verify the live games Namespace and the live API PVC both carry `argocd.argoproj.io/sync-options: Prune=false,Delete=false`. If switching to `api.storage.existingClaim`, verify retention on the **old live claim before** the sync that removes its manifest. Point the API at that same populated PVC, or migrate and verify the data separately first. +5. Review the workload prune preview. Compare the UIDs and CA fingerprint from step 1, log in with a pre-upgrade account, read a persisted game marker and make an authenticated call to the running game’s agent. On a wake-enabled test GameServer, impersonate its waker ServiceAccount for server-side dry-run patches. Its wake annotation should succeed and a spec mutation should be denied by the wake policy. Verify the policy and binding names and denial reason. +6. Enable workload pruning after reviewing the retained objects. Argo CD may report retained resources as OutOfSync. `Prune=false` preserves an object removed from Git during sync. `Delete=false` preserves it on Application deletion. Helm’s `helm.sh/resource-policy: keep` maps to deletion protection in Argo CD. Deleting a containing namespace can still remove its claims. See [Argo CD sync options](https://argo-cd.readthedocs.io/en/stable/user-guide/sync-options/). + +## Rotating external certificates + +For a client renewal under the same CA, stage a new immutable client Secret and change its reference in Git. For a CA rollover, schedule a maintenance window, stage a new CA and matching client Secret, then change both references in one revision. The reference change rolls the API and operator. The operator reissues each GameServer’s agent TLS Secret against the active CA. Kubelet projection and the next TLS handshake determine when each agent starts using the new trust. + +Finish the rollout by checking the new CA fingerprint on each `-agent-tls` Secret and making a fresh authenticated agent call for every running game. A stopped game needs that check on its next start. Retain the previous Secret pair until the checks finish. If rolling back the CA, restore both references together and repeat the per-agent checks while certificates are reissued. diff --git a/docs/install.md b/docs/install.md index f0b5767d7..dd0f5e426 100644 --- a/docs/install.md +++ b/docs/install.md @@ -5,6 +5,8 @@ For a central dashboard/API on a host without Kubernetes, use the panel-only Helm profile. The instructions below describe the default combined installation, with an operator and a local game cluster. +For an offline Helm renderer, see the [Argo CD delivery guide](argocd.md). + ## Prerequisites - Kubernetes 1.28+ diff --git a/test/e2e/upgrade_e2e_test.go b/test/e2e/upgrade_e2e_test.go index 56736b34c..75a24f763 100644 --- a/test/e2e/upgrade_e2e_test.go +++ b/test/e2e/upgrade_e2e_test.go @@ -4,18 +4,23 @@ package e2e import ( "context" + "crypto/sha256" "encoding/json" + "fmt" "os" "os/exec" "path/filepath" + "slices" "strconv" "strings" "testing" "time" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/yaml" ) @@ -69,6 +74,7 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { applyBusyboxGameServer(t, ns, gs, tmpl) waitPVCBound(t, ns, gs+"-data", 90*time.Second) waitGameContainerReady(t, ns, gs+"-0", 2*time.Minute) + beforeIdentity := readUpgradeIdentity(t, ns, gs) // `sync` so the bytes are on the PVC's backing store before the pod is // restarted by the upgrade, not sitting in the page cache. @@ -147,6 +153,13 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { t.Fatalf("operator strategy migration retained RollingUpdate fields: %+v", operator.Spec.Strategy) } + afterIdentity := readUpgradeIdentity(t, ns, gs) + for name, before := range beforeIdentity { + if afterIdentity[name] != before { + t.Errorf("upgrade changed %s identity: before %s, after %s", name, before, afterIdentity[name]) + } + } + // ---- 4. the CRDs were actually updated ------------------------------- // This is the hook's literal contract. Without it Helm leaves the old @@ -448,6 +461,54 @@ func waitGameContainerReady(t *testing.T, ns, pod string, timeout time.Duration) }) } +// Snapshot the populated installation's storage and trust identities. Remaining +// Gameplane CRDs must include every version still recorded as stored by Kubernetes. +func readUpgradeIdentity(t *testing.T, gamesNamespace, gs string) map[string]string { + t.Helper() + identity := map[string]string{} + for _, claim := range [][2]string{{"gameplane-system", "gameplane-api-data"}, {gamesNamespace, gs + "-data"}} { + pvc, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(claim[0]).Get(t.Context(), claim[1], metav1.GetOptions{}) + if err != nil || pvc.UID == "" { + t.Fatalf("read PVC identity %s/%s: %v", claim[0], claim[1], err) + } + identity[claim[0]+"/"+claim[1]] = string(pvc.UID) + } + ca, err := envInstance.K8s.CoreV1().Secrets("gameplane-system").Get(t.Context(), "gameplane-agent-ca", metav1.GetOptions{}) + if err != nil || len(ca.Data["ca.crt"]) == 0 { + t.Fatalf("read public agent CA certificate: %v", err) + } + identity["agent CA fingerprint"] = fmt.Sprintf("%x", sha256.Sum256(ca.Data["ca.crt"])) + crds, err := envInstance.Dyn.Resource(crdGVR).List(t.Context(), metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + for _, crd := range crds.Items { + if !strings.HasSuffix(crd.GetName(), ".gameplane.local") { + continue + } + if crd.GetUID() == "" { + t.Fatalf("%s has no UID", crd.GetName()) + } + identity[crd.GetName()] = string(crd.GetUID()) + var typed apiextensionsv1.CustomResourceDefinition + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(crd.Object, &typed); err != nil { + t.Fatal(err) + } + if len(typed.Status.StoredVersions) == 0 { + t.Fatalf("%s has no storedVersions", crd.GetName()) + } + for _, stored := range typed.Status.StoredVersions { + if !slices.ContainsFunc(typed.Spec.Versions, func(v apiextensionsv1.CustomResourceDefinitionVersion) bool { return v.Name == stored }) { + t.Fatalf("%s stored version %s is absent from the schema", crd.GetName(), stored) + } + } + } + if identity["gameservers.gameplane.local"] == "" { + t.Fatal("missing GameServer CRD") + } + return identity +} + // crdSpecProperties returns the top-level property names under the CRD's // spec schema, as the apiserver currently serves it. func crdSpecProperties(t *testing.T, crdName string) map[string]struct{} { From 48b20d5a7373ae09cf90e275901e7af1f8bb3352 Mon Sep 17 00:00:00 2001 From: Baylem Date: Tue, 10 Nov 2026 21:18:48 -0500 Subject: [PATCH 2/7] test(e2e): isolate upgrade identity checks from security fixtures Signed-off-by: Baylem --- test/e2e/upgrade_e2e_test.go | 53 ---------------------- test/e2e/upgrade_identity_e2e_test.go | 63 +++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 53 deletions(-) create mode 100644 test/e2e/upgrade_identity_e2e_test.go diff --git a/test/e2e/upgrade_e2e_test.go b/test/e2e/upgrade_e2e_test.go index 75a24f763..9fadced50 100644 --- a/test/e2e/upgrade_e2e_test.go +++ b/test/e2e/upgrade_e2e_test.go @@ -4,23 +4,18 @@ package e2e import ( "context" - "crypto/sha256" "encoding/json" - "fmt" "os" "os/exec" "path/filepath" - "slices" "strconv" "strings" "testing" "time" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/yaml" ) @@ -461,54 +456,6 @@ func waitGameContainerReady(t *testing.T, ns, pod string, timeout time.Duration) }) } -// Snapshot the populated installation's storage and trust identities. Remaining -// Gameplane CRDs must include every version still recorded as stored by Kubernetes. -func readUpgradeIdentity(t *testing.T, gamesNamespace, gs string) map[string]string { - t.Helper() - identity := map[string]string{} - for _, claim := range [][2]string{{"gameplane-system", "gameplane-api-data"}, {gamesNamespace, gs + "-data"}} { - pvc, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(claim[0]).Get(t.Context(), claim[1], metav1.GetOptions{}) - if err != nil || pvc.UID == "" { - t.Fatalf("read PVC identity %s/%s: %v", claim[0], claim[1], err) - } - identity[claim[0]+"/"+claim[1]] = string(pvc.UID) - } - ca, err := envInstance.K8s.CoreV1().Secrets("gameplane-system").Get(t.Context(), "gameplane-agent-ca", metav1.GetOptions{}) - if err != nil || len(ca.Data["ca.crt"]) == 0 { - t.Fatalf("read public agent CA certificate: %v", err) - } - identity["agent CA fingerprint"] = fmt.Sprintf("%x", sha256.Sum256(ca.Data["ca.crt"])) - crds, err := envInstance.Dyn.Resource(crdGVR).List(t.Context(), metav1.ListOptions{}) - if err != nil { - t.Fatal(err) - } - for _, crd := range crds.Items { - if !strings.HasSuffix(crd.GetName(), ".gameplane.local") { - continue - } - if crd.GetUID() == "" { - t.Fatalf("%s has no UID", crd.GetName()) - } - identity[crd.GetName()] = string(crd.GetUID()) - var typed apiextensionsv1.CustomResourceDefinition - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(crd.Object, &typed); err != nil { - t.Fatal(err) - } - if len(typed.Status.StoredVersions) == 0 { - t.Fatalf("%s has no storedVersions", crd.GetName()) - } - for _, stored := range typed.Status.StoredVersions { - if !slices.ContainsFunc(typed.Spec.Versions, func(v apiextensionsv1.CustomResourceDefinitionVersion) bool { return v.Name == stored }) { - t.Fatalf("%s stored version %s is absent from the schema", crd.GetName(), stored) - } - } - } - if identity["gameservers.gameplane.local"] == "" { - t.Fatal("missing GameServer CRD") - } - return identity -} - // crdSpecProperties returns the top-level property names under the CRD's // spec schema, as the apiserver currently serves it. func crdSpecProperties(t *testing.T, crdName string) map[string]struct{} { diff --git a/test/e2e/upgrade_identity_e2e_test.go b/test/e2e/upgrade_identity_e2e_test.go new file mode 100644 index 000000000..1f0038e1e --- /dev/null +++ b/test/e2e/upgrade_identity_e2e_test.go @@ -0,0 +1,63 @@ +//go:build e2e + +package e2e + +import ( + "crypto/sha256" + "fmt" + "slices" + "strings" + "testing" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// Snapshot the populated installation's storage and trust identities. Remaining +// Gameplane CRDs must include every version still recorded as stored by Kubernetes. +func readUpgradeIdentity(t *testing.T, gamesNamespace, gs string) map[string]string { + t.Helper() + identity := map[string]string{} + for _, claim := range [][2]string{{"gameplane-system", "gameplane-api-data"}, {gamesNamespace, gs + "-data"}} { + pvc, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(claim[0]).Get(t.Context(), claim[1], metav1.GetOptions{}) + if err != nil || pvc.UID == "" { + t.Fatalf("read PVC identity %s/%s: %v", claim[0], claim[1], err) + } + identity[claim[0]+"/"+claim[1]] = string(pvc.UID) + } + ca, err := envInstance.K8s.CoreV1().Secrets("gameplane-system").Get(t.Context(), "gameplane-agent-ca", metav1.GetOptions{}) + if err != nil || len(ca.Data["ca.crt"]) == 0 { + t.Fatalf("read public agent CA certificate: %v", err) + } + identity["agent CA fingerprint"] = fmt.Sprintf("%x", sha256.Sum256(ca.Data["ca.crt"])) + crds, err := envInstance.Dyn.Resource(crdGVR).List(t.Context(), metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + for _, crd := range crds.Items { + if !strings.HasSuffix(crd.GetName(), ".gameplane.local") { + continue + } + if crd.GetUID() == "" { + t.Fatalf("%s has no UID", crd.GetName()) + } + identity[crd.GetName()] = string(crd.GetUID()) + var typed apiextensionsv1.CustomResourceDefinition + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(crd.Object, &typed); err != nil { + t.Fatal(err) + } + if len(typed.Status.StoredVersions) == 0 { + t.Fatalf("%s has no storedVersions", crd.GetName()) + } + for _, stored := range typed.Status.StoredVersions { + if !slices.ContainsFunc(typed.Spec.Versions, func(v apiextensionsv1.CustomResourceDefinitionVersion) bool { return v.Name == stored }) { + t.Fatalf("%s stored version %s is absent from the schema", crd.GetName(), stored) + } + } + } + if identity["gameservers.gameplane.local"] == "" { + t.Fatal("missing GameServer CRD") + } + return identity +} From b171d6c139f5ba4ca5f165f98ba0495fd3ba3dc3 Mon Sep 17 00:00:00 2001 From: Baylem Date: Tue, 10 Nov 2026 21:22:44 -0500 Subject: [PATCH 3/7] test(e2e): keep game claim identity in the shared release check Signed-off-by: Baylem --- test/e2e/upgrade_e2e_test.go | 4 ++-- test/e2e/upgrade_identity_e2e_test.go | 12 +++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/test/e2e/upgrade_e2e_test.go b/test/e2e/upgrade_e2e_test.go index 9fadced50..5003aea4b 100644 --- a/test/e2e/upgrade_e2e_test.go +++ b/test/e2e/upgrade_e2e_test.go @@ -69,7 +69,7 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { applyBusyboxGameServer(t, ns, gs, tmpl) waitPVCBound(t, ns, gs+"-data", 90*time.Second) waitGameContainerReady(t, ns, gs+"-0", 2*time.Minute) - beforeIdentity := readUpgradeIdentity(t, ns, gs) + beforeIdentity := readUpgradeIdentity(t) // `sync` so the bytes are on the PVC's backing store before the pod is // restarted by the upgrade, not sitting in the page cache. @@ -148,7 +148,7 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { t.Fatalf("operator strategy migration retained RollingUpdate fields: %+v", operator.Spec.Strategy) } - afterIdentity := readUpgradeIdentity(t, ns, gs) + afterIdentity := readUpgradeIdentity(t) for name, before := range beforeIdentity { if afterIdentity[name] != before { t.Errorf("upgrade changed %s identity: before %s, after %s", name, before, afterIdentity[name]) diff --git a/test/e2e/upgrade_identity_e2e_test.go b/test/e2e/upgrade_identity_e2e_test.go index 1f0038e1e..2a7a73bd9 100644 --- a/test/e2e/upgrade_identity_e2e_test.go +++ b/test/e2e/upgrade_identity_e2e_test.go @@ -16,16 +16,14 @@ import ( // Snapshot the populated installation's storage and trust identities. Remaining // Gameplane CRDs must include every version still recorded as stored by Kubernetes. -func readUpgradeIdentity(t *testing.T, gamesNamespace, gs string) map[string]string { +func readUpgradeIdentity(t *testing.T) map[string]string { t.Helper() identity := map[string]string{} - for _, claim := range [][2]string{{"gameplane-system", "gameplane-api-data"}, {gamesNamespace, gs + "-data"}} { - pvc, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(claim[0]).Get(t.Context(), claim[1], metav1.GetOptions{}) - if err != nil || pvc.UID == "" { - t.Fatalf("read PVC identity %s/%s: %v", claim[0], claim[1], err) - } - identity[claim[0]+"/"+claim[1]] = string(pvc.UID) + pvc, err := envInstance.K8s.CoreV1().PersistentVolumeClaims("gameplane-system").Get(t.Context(), "gameplane-api-data", metav1.GetOptions{}) + if err != nil || pvc.UID == "" { + t.Fatalf("read gameplane-system/gameplane-api-data PVC identity: %v", err) } + identity["gameplane-system/gameplane-api-data"] = string(pvc.UID) ca, err := envInstance.K8s.CoreV1().Secrets("gameplane-system").Get(t.Context(), "gameplane-agent-ca", metav1.GetOptions{}) if err != nil || len(ca.Data["ca.crt"]) == 0 { t.Fatalf("read public agent CA certificate: %v", err) From bb229bb3b6b552b4c427aa449d8994e079274307 Mon Sep 17 00:00:00 2001 From: Baylem Date: Tue, 10 Nov 2026 23:39:25 -0500 Subject: [PATCH 4/7] test(gitops): exercise populated Argo CD adoption, prune and trust rotation Signed-off-by: Baylem --- .github/workflows/ci.yaml | 7 +- deploy/kind/argocd.sh | 19 ++ docs/argocd.md | 8 +- test/e2e/argocd_git_e2e_test.go | 59 +++++ test/e2e/argocd_upgrade_e2e_test.go | 309 ++++++++++++++++++++++++++ test/e2e/upgrade_e2e_test.go | 10 +- test/e2e/upgrade_identity_e2e_test.go | 4 +- 7 files changed, 409 insertions(+), 7 deletions(-) create mode 100644 deploy/kind/argocd.sh create mode 100644 test/e2e/argocd_git_e2e_test.go create mode 100644 test/e2e/argocd_upgrade_e2e_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index af1f7b002..ad84528cf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2175,13 +2175,18 @@ jobs: arch: ${{ matrix.arch }} - name: bring up cluster on the PREVIOUS release run: ./deploy/kind/upgrade.sh up "$CLUSTER" e2e + - name: install Argo CD for the populated GitOps upgrade + if: matrix.arch == 'amd64' + run: bash deploy/kind/argocd.sh - name: run go e2e (upgrade) + env: + GAMEPLANE_E2E_ARGOCD: ${{ matrix.arch == 'amd64' && '1' || '' }} run: | RUN_REGEX="$(./test/e2e/buckets.sh regex upgrade)" cd test/e2e && \ GAMEPLANE_E2E_REUSE_CLUSTER=1 \ GAMEPLANE_E2E_CLUSTER="$CLUSTER" \ - go test -tags=e2e -timeout 25m -v -run "$RUN_REGEX" ./... + go test -tags=e2e -timeout 45m -v -run "$RUN_REGEX" ./... - name: dump cluster state on failure (e2e-upgrade) if: failure() uses: ./.github/actions/dump-cluster-state diff --git a/deploy/kind/argocd.sh b/deploy/kind/argocd.sh new file mode 100644 index 000000000..e5c5324f6 --- /dev/null +++ b/deploy/kind/argocd.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Install the upstream core controller only on the disposable upgrade cluster. +set -euo pipefail +context="${CTX:?kind upgrade context required}" +[[ "$context" == "kind-${CLUSTER:?kind upgrade cluster required}" ]] +kubectl() { command kubectl --context "$context" "$@"; } +bundle="$(mktemp)" +trap 'rm -f "$bundle"' EXIT +# argoproj/argo-cd v3.6.1 +curl --fail --location --silent --show-error --retry 4 --max-time 80 \ + https://raw.githubusercontent.com/argoproj/argo-cd/94037c4d4e1af4bd0651d37a179fbd9e347e668c/manifests/core-install.yaml \ + --output "$bundle" +echo "0b38189d6e1fd76ef5c3db10355026c48a02020d82bf21a1a1b5ab16dffe4f3c $bundle" | sha256sum --check +kubectl create namespace argocd +kubectl apply --server-side --namespace argocd --filename "$bundle" +kubectl set env -n argocd deployment/argocd-repo-server --containers=argocd-repo-server ARGOCD_GIT_MODULES_ENABLED=false +kubectl wait --for=condition=Established crd/applications.argoproj.io crd/appprojects.argoproj.io --timeout=2m +kubectl rollout status -n argocd deployment/argocd-repo-server --timeout=4m +kubectl rollout status -n argocd statefulset/argocd-application-controller --timeout=4m diff --git a/docs/argocd.md b/docs/argocd.md index 4d44a0afc..cbd08f03b 100644 --- a/docs/argocd.md +++ b/docs/argocd.md @@ -1,6 +1,6 @@ # Argo CD delivery of a native Gameplane installation -Use the chart’s Argo CD values for a combined API and operator installation. The profile uses two externally managed mTLS Secrets and requires discovery of the GA admission policy and binding APIs. It also gives the games Namespace and chart-managed SQLite claim explicit Argo CD retention annotations. The fixture covers Helm rendering and the populated Helm upgrade path. A populated Argo CD sync and prune drill is tracked in [#882](https://github.com/GameplanePanel/Gameplane/issues/882). +Use the chart’s Argo CD values for a combined API and operator installation. The profile uses two externally managed mTLS Secrets and requires discovery of the GA admission policy and binding APIs. It also gives the games Namespace and chart-managed SQLite claim explicit Argo CD retention annotations. ## One owner for each resource @@ -8,7 +8,7 @@ Pin both Applications to the same Gameplane Git commit. Sync the CRD Application | Application | Source | Settings | | --- | --- | --- | -| CRDs | Plain directory `charts/gameplane/charts/gameplane-crds/crds` | Use server-side apply for the large schemas. Set `spec.syncPolicy.syncOptions` to include `ServerSideApply=true`, `Prune=false` and `Delete=false`. | +| CRDs | Plain directory `charts/gameplane/charts/gameplane-crds/crds` | Use `ServerSideApply=true` for the large schemas. Keep automated pruning off and sync this Application without prune. For Application deletion, preserve its resources by using non-cascading deletion. | | Workload | Helm chart `charts/gameplane` | Set `spec.source.helm.skipCrds: true` and include `examples/argocd-values.yaml` in `spec.source.helm.valueFiles`. The profile disables the CRD apply hook. Set the release name to `gameplane` and the destination namespace to `gameplane-system`. | Provision the release namespace, CA and client Secret before syncing. Keep ownership of the release namespace and external Secrets outside the workload Application. The example expects `gameplane-external-ca-v1` with `ca.crt` and `ca.key`, and a different Secret named `gameplane-external-client-v1` with `tls.crt` and `tls.key`. Use immutable, versioned Secret names. The operator signs per-game serving certificates with this CA, and the API and operator present the client certificate to the agents. @@ -42,3 +42,7 @@ The chart fails the profile render if either kind is missing. Supplying a capabi For a client renewal under the same CA, stage a new immutable client Secret and change its reference in Git. For a CA rollover, schedule a maintenance window, stage a new CA and matching client Secret, then change both references in one revision. The reference change rolls the API and operator. The operator reissues each GameServer’s agent TLS Secret against the active CA. Kubelet projection and the next TLS handshake determine when each agent starts using the new trust. Finish the rollout by checking the new CA fingerprint on each `-agent-tls` Secret and making a fresh authenticated agent call for every running game. A stopped game needs that check on its next start. Retain the previous Secret pair until the checks finish. If rolling back the CA, restore both references together and repeat the per-agent checks while certificates are reissued. + +## CI qualification + +The amd64 upgrade job installs Argo CD Core v3.6.1 on a disposable kind cluster. Two Applications sync the PR commit from a read-only Git endpoint on the kind bridge. After adoption, the workload Application renews the client, switches to the previously populated database claim and prunes a tracked dashboard. A second sync replaces the CA and client together. At each stage the fixture verifies the claim and CRD identities, validates stored versions, authenticates with the original account and reads a saved world marker through the agent. Server-side dry runs check the waker's allowed annotation and denied spec mutation. [Populated upgrade fixture](../test/e2e/argocd_upgrade_e2e_test.go) diff --git a/test/e2e/argocd_git_e2e_test.go b/test/e2e/argocd_git_e2e_test.go new file mode 100644 index 000000000..f48a18992 --- /dev/null +++ b/test/e2e/argocd_git_e2e_test.go @@ -0,0 +1,59 @@ +//go:build e2e + +package e2e + +import ( + "net" + "net/http" + "net/http/cgi" + "net/http/httptest" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// Serve exactly the tested commit, including fork PRs, without giving the +// controller a GitHub credential or making runner credentials part of a clone. +func argoGitSource(t *testing.T) (address, revision string) { + t.Helper() + run := func(program string, args ...string) string { + t.Helper() + out, err := exec.CommandContext(t.Context(), program, args...).CombinedOutput() + if err != nil { + t.Fatalf("%s: %v: %s", program, err, out) + } + return strings.TrimSpace(string(out)) + } + root := run("git", "rev-parse", "--show-toplevel") + revision = run("git", "rev-parse", "HEAD") + directory := t.TempDir() + repo := filepath.Join(directory, "candidate.git") + run("git", "init", "--quiet", "--bare", repo) + run("git", "-C", repo, "fetch", "--quiet", "--depth=1", root, revision+":refs/heads/candidate") + run("git", "-C", repo, "symbolic-ref", "HEAD", "refs/heads/candidate") + gateway := run("docker", "network", "inspect", "kind", "--format", "{{(index .IPAM.Config 0).Gateway}}") + if ip := net.ParseIP(gateway); ip == nil || !ip.IsPrivate() { + t.Fatalf("expected a private kind bridge gateway, got %q", gateway) + } + listener, err := net.Listen("tcp", net.JoinHostPort(gateway, "0")) + if err != nil { + t.Fatal(err) + } + gitHTTP := &cgi.Handler{Path: filepath.Join(run("git", "--exec-path"), "git-http-backend"), Dir: directory, + Env: []string{"GIT_PROJECT_ROOT=" + directory, "GIT_HTTP_EXPORT_ALL=1"}} + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + discover := r.Method == http.MethodGet && r.URL.Path == "/candidate.git/info/refs" && r.URL.Query().Get("service") == "git-upload-pack" + fetch := r.Method == http.MethodPost && r.URL.Path == "/candidate.git/git-upload-pack" + if !discover && !fetch { + http.NotFound(w, r) + return + } + gitHTTP.ServeHTTP(w, r) + })) + _ = srv.Listener.Close() + srv.Listener = listener + srv.Start() + t.Cleanup(srv.Close) + return srv.URL + "/candidate.git", revision +} diff --git a/test/e2e/argocd_upgrade_e2e_test.go b/test/e2e/argocd_upgrade_e2e_test.go new file mode 100644 index 000000000..042200598 --- /dev/null +++ b/test/e2e/argocd_upgrade_e2e_test.go @@ -0,0 +1,309 @@ +//go:build e2e + +package e2e + +import ( + "bytes" + "context" + "crypto" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "fmt" + "math/big" + "net/http" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" +) + +var argoAppGVR = schema.GroupVersionResource{Group: "argoproj.io", Version: "v1alpha1", Resource: "applications"} +var argoSyncOptions = []any{"ServerSideApply=true", "ClientSideApplyMigration=false", "FailOnSharedResource=true"} + +func checkArgoUpgrade(t *testing.T, ns, game, marker, username, password, credentialsNS string) { + t.Helper() + const system = "gameplane-system" + const workload = "gameplane" + ctx := t.Context() + repo, revision := argoGitSource(t) + projectName := "upgrade-fixture" + destination := func(namespace string) map[string]any { + return map[string]any{"server": "https://kubernetes.default.svc", "namespace": namespace} + } + projectGVR := schema.GroupVersionResource{Group: "argoproj.io", Version: "v1alpha1", Resource: "appprojects"} + var clusterResources []any + for _, resource := range [][2]string{ + {"", "Namespace"}, + {"apiextensions.k8s.io", "CustomResourceDefinition"}, + {"rbac.authorization.k8s.io", "ClusterRole"}, + {"rbac.authorization.k8s.io", "ClusterRoleBinding"}, + {"admissionregistration.k8s.io", "ValidatingAdmissionPolicy"}, + {"admissionregistration.k8s.io", "ValidatingAdmissionPolicyBinding"}, + } { + clusterResources = append(clusterResources, map[string]any{"group": resource[0], "kind": resource[1]}) + } + create := func(gvr schema.GroupVersionResource, kind, name string, spec map[string]any) { + t.Helper() + resource := envInstance.Dyn.Resource(gvr).Namespace("argocd") + obj := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "argoproj.io/v1alpha1", "kind": kind, + "metadata": map[string]any{"name": name, "namespace": "argocd"}, "spec": spec}} + if _, err := resource.Create(ctx, obj, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + // Applications have no cascading-deletion finalizer; the later Helm + // reinstall phase owns teardown of the populated world. + t.Cleanup(func() { _ = resource.Delete(context.Background(), name, metav1.DeleteOptions{}) }) + } + create(projectGVR, "AppProject", projectName, map[string]any{ + "sourceRepos": []any{repo}, + "destinations": []any{destination(system), destination(ns), destination(credentialsNS)}, + "clusterResourceWhitelist": clusterResources, + }) + app := func(name, path string, helm map[string]any) { + t.Helper() + source := map[string]any{"repoURL": repo, "targetRevision": revision, "path": path} + if helm != nil { + source["helm"] = helm + } + create(argoAppGVR, "Application", name, map[string]any{ + "project": projectName, "source": source, "destination": destination(system), + "syncPolicy": map[string]any{"syncOptions": argoSyncOptions}, + }) + } + identity := readUpgradeIdentity(t, "gameplane-agent-ca") + claim, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(ns).Get(ctx, game+"-data", metav1.GetOptions{}) + if err != nil || claim.UID == "" { + t.Fatalf("read populated game PVC: %v", err) + } + currentSecret := func(name string) *corev1.Secret { + t.Helper() + s, err := envInstance.K8s.CoreV1().Secrets(system).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + return s + } + immutable := func(name string, kind corev1.SecretType, material map[string][]byte) { + t.Helper() + _, err := envInstance.K8s.CoreV1().Secrets(system).Create(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}, + Type: kind, Immutable: ptr.To(true), Data: material}, metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + } + firstCA, firstClient := currentSecret("gameplane-agent-ca"), currentSecret("gameplane-agent-client") + issuer, clientCert := "gameplane-external-ca-v1", "gameplane-external-client-v1" + immutable(issuer, firstCA.Type, firstCA.Data) + immutable(clientCert, firstClient.Type, firstClient.Data) + // Reauthenticate against the current API on every rollout; a stale + // port-forward or cached /players response cannot verify an agent's mTLS. + checkWorld := func(issuerSecret, clientSecret string, expectedCA []byte) { + t.Helper() + now := readUpgradeIdentity(t, issuerSecret) + expected := fmt.Sprintf("%x", sha256.Sum256(expectedCA)) + if now["agent CA fingerprint"] != expected { + t.Fatal("active issuer fingerprint differs from staged certificate") + } + for field, value := range identity { + if field != "agent CA fingerprint" && now[field] != value { + t.Fatalf("changed %s: %s -> %s", field, value, now[field]) + } + } + sameClaim, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(ns).Get(ctx, game+"-data", metav1.GetOptions{}) + if err != nil || sameClaim.UID != claim.UID { + t.Fatalf("game PVC was replaced: %v", err) + } + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + cert, err := envInstance.K8s.CoreV1().Secrets(ns).Get(ctx, game+"-agent-tls", metav1.GetOptions{}) + return err == nil && bytes.Equal(cert.Data["ca.crt"], expectedCA), "waiting for agent TLS issuance under the selected CA" + }) + for _, deployment := range []string{"gameplane-api", "gameplane-operator"} { + out, err := envInstance.Kubectl(ctx, "rollout", "status", "-n", system, "deployment/"+deployment, "--timeout=3m") + if err != nil { + t.Fatalf("%s rollout: %v %s", deployment, err, out) + } + dep, err := envInstance.K8s.AppsV1().Deployments(system).Get(ctx, deployment, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + mounted := map[string]bool{} + for _, volume := range dep.Spec.Template.Spec.Volumes { + if volume.Secret != nil { + mounted[volume.Secret.SecretName] = true + } + } + if !mounted[issuerSecret] || !mounted[clientSecret] { + t.Fatalf("%s rolled out without the intended issuer/client Secret references", deployment) + } + } + cli := envInstance.APIClient(t, username, password) + defer cli.Close() + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + resp, body, err := cli.Get("/servers/" + game + "/files/read?path=%2Fmarker.txt") + if err != nil { + return false, err.Error() + } + defer resp.Body.Close() + return resp.StatusCode == http.StatusOK && string(body) == marker, fmt.Sprintf("reading the seeded world through agent mTLS: HTTP %d", resp.StatusCode) + }) + } + checkWorld("gameplane-agent-ca", "gameplane-agent-client", firstCA.Data["ca.crt"]) + app("gameplane-crds", "charts/gameplane/charts/gameplane-crds/crds", nil) + syncArgo(t, "gameplane-crds", revision, "crds", false, nil) + trust := map[string]any{ + "caSecretRef": map[string]any{"name": issuer}, + "clientCertRef": map[string]any{"name": clientCert}, + } + apiValues := map[string]any{ + "credentialsNamespace": credentialsNS, + "resources": map[string]any{"limits": map[string]any{"memory": "1Gi"}}, + "agentMTLS": trust, + } + values := map[string]any{ + "image": map[string]any{"registry": "gameplane-test", "tag": envInstance.Tag}, + "operator": map[string]any{"leaderElect": false, "agentImage": "gameplane-test/agent:" + envInstance.Tag}, + "podSecurity": map[string]any{"enforceRestricted": false}, + "web": map[string]any{"enabled": false}, + "ingress": map[string]any{"enabled": false}, + "defaultModuleSource": map[string]any{"enabled": false}, + "grafanaDashboards": map[string]any{"enabled": true}, + "api": apiValues, + } + helm := map[string]any{"releaseName": "gameplane", "skipCrds": true, "valueFiles": []any{"examples/argocd-values.yaml"}, "valuesObject": values} + app(workload, "charts/gameplane", helm) + syncArgo(t, workload, revision, "adopt", false, nil) + checkWorld(issuer, clientCert, firstCA.Data["ca.crt"]) + checkArgoWaker(t, ns, game) + dashboards := envInstance.K8s.CoreV1().ConfigMaps(system) + dashboard, err := dashboards.Get(ctx, "gameplane-operator-dashboard", metav1.GetOptions{}) + if err != nil || dashboard.Annotations["argocd.argoproj.io/tracking-id"] == "" { + t.Fatalf("expected Argo-owned disposable dashboard before prune: %v", err) + } + dbClaim, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(system).Get(ctx, "gameplane-api-data", metav1.GetOptions{}) + if err != nil || !strings.Contains(dbClaim.Annotations["argocd.argoproj.io/sync-options"], "Prune=false") || dbClaim.Annotations["argocd.argoproj.io/tracking-id"] == "" { + t.Fatalf("expected a tracked, protected SQLite claim before prune: %v", err) + } + newClient := "gameplane-external-client-v2" + immutable(newClient, corev1.SecretTypeTLS, renewArgoClient(t, firstCA)) + values["grafanaDashboards"] = map[string]any{"enabled": false} + apiValues["storage"] = map[string]any{"existingClaim": "gameplane-api-data"} + trust["clientCertRef"] = map[string]any{"name": newClient} + syncArgo(t, workload, revision, "prune-and-renew-client", true, values) + if _, err := dashboards.Get(ctx, dashboard.Name, metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("expected Argo to prune the dashboard: %v", err) + } + checkWorld(issuer, newClient, firstCA.Data["ca.crt"]) + replacement, leaf := harvestMTLSPair(ctx, t, filepath.Join("..", "..", "charts", "gameplane"), "argo-rotation-donor") + issuer, newClient = "gameplane-external-ca-v2", "gameplane-external-client-v3" + immutable(issuer, corev1.SecretTypeOpaque, map[string][]byte{"ca.crt": []byte(replacement.crt), "ca.key": []byte(replacement.key)}) + immutable(newClient, corev1.SecretTypeTLS, map[string][]byte{"tls.crt": []byte(leaf.crt), "tls.key": []byte(leaf.key)}) + trust["caSecretRef"] = map[string]any{"name": issuer} + trust["clientCertRef"] = map[string]any{"name": newClient} + syncArgo(t, workload, revision, "rotate-issuer", true, values) + checkWorld(issuer, newClient, []byte(replacement.crt)) + checkArgoWaker(t, ns, game) +} + +// A unique operation marker distinguishes successive syncs of the same pinned +// Git revision with different Helm inputs. OutOfSync is expected for retained PVCs. +func syncArgo(t *testing.T, name, revision, step string, prune bool, values map[string]any) { + t.Helper() + apps := envInstance.Dyn.Resource(argoAppGVR).Namespace("argocd") + marker := []any{map[string]any{"name": "fixture-step", "value": step}} + update := map[string]any{"operation": map[string]any{ + "info": marker, + "sync": map[string]any{"revision": revision, "prune": prune, "syncOptions": argoSyncOptions}, + }} + if values != nil { + update["spec"] = map[string]any{"source": map[string]any{"helm": map[string]any{"valuesObject": values}}} + } + raw, err := json.Marshal(update) + if err != nil { + t.Fatal(err) + } + if _, err := apps.Patch(t.Context(), name, types.MergePatchType, raw, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + envInstance.Eventually(t, 5*time.Minute, func() (bool, string) { + app, err := apps.Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + info, _, _ := unstructured.NestedSlice(app.Object, "status", "operationState", "operation", "info") + if !reflect.DeepEqual(info, marker) { + return false, "waiting for " + step + } + phase, _, _ := unstructured.NestedString(app.Object, "status", "operationState", "phase") + msg, _, _ := unstructured.NestedString(app.Object, "status", "operationState", "message") + if phase == "Failed" || phase == "Error" { + t.Fatalf("%s/%s: %s: %s", name, step, phase, msg) + } + applied, _, _ := unstructured.NestedString(app.Object, "status", "operationState", "syncResult", "revision") + return phase == "Succeeded" && applied == revision, phase + ": " + msg + }) +} + +func checkArgoWaker(t *testing.T, ns, game string) { + t.Helper() + sa := "system:serviceaccount:" + ns + ":" + game + "-waker-sa" + patch := func(data string) (string, error) { + return envInstance.Kubectl(t.Context(), "--as", sa, "-n", ns, "patch", "gameserver", game, "--dry-run=server", "--type=merge", "-p", data) + } + if out, err := patch(`{"metadata":{"annotations":{"gameplane.local/idle-wake-requested":"argo-check"}}}`); err != nil { + t.Fatalf("waker annotation denied: %v %s", err, out) + } + out, err := patch(`{"spec":{"suspend":true}}`) + if err == nil || !strings.Contains(out, "gameplane-waker-wake-only") || !strings.Contains(out, "may not change the GameServer spec") { + t.Fatalf("expected wake-only admission to deny the spec patch: %v %s", err, out) + } +} + +func renewArgoClient(t *testing.T, ca *corev1.Secret) map[string][]byte { + t.Helper() + pair, err := tls.X509KeyPair(ca.Data["ca.crt"], ca.Data["ca.key"]) + if err != nil { + t.Fatal(err) + } + issuer, err := x509.ParseCertificate(pair.Certificate[0]) + if err != nil { + t.Fatal(err) + } + key, ok := pair.PrivateKey.(crypto.Signer) + if !ok { + t.Fatal("issuer key cannot sign") + } + public, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + t.Fatal(err) + } + cert := &x509.Certificate{SerialNumber: serial, Subject: pkix.Name{CommonName: "gameplane-api-client"}, NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(6 * time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}} + der, err := x509.CreateCertificate(rand.Reader, cert, issuer, public, key) + if err != nil { + t.Fatal(err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(private) + if err != nil { + t.Fatal(err) + } + return map[string][]byte{"tls.crt": pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), "tls.key": pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})} +} diff --git a/test/e2e/upgrade_e2e_test.go b/test/e2e/upgrade_e2e_test.go index 0a5c106fc..f1ba8ac51 100644 --- a/test/e2e/upgrade_e2e_test.go +++ b/test/e2e/upgrade_e2e_test.go @@ -74,7 +74,7 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { applyBusyboxGameServer(t, ns, gs, tmpl) waitPVCBound(t, ns, gs+"-data", 90*time.Second) waitGameContainerReady(t, ns, gs+"-0", 2*time.Minute) - beforeIdentity := readUpgradeIdentity(t) + beforeIdentity := readUpgradeIdentity(t, "gameplane-agent-ca") // `sync` so the bytes are on the PVC's backing store before the pod is // restarted by the upgrade, not sitting in the page cache. @@ -159,7 +159,7 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { t.Fatalf("operator strategy migration retained RollingUpdate fields: %+v", operator.Spec.Strategy) } - afterIdentity := readUpgradeIdentity(t) + afterIdentity := readUpgradeIdentity(t, "gameplane-agent-ca") for name, before := range beforeIdentity { if afterIdentity[name] != before { t.Errorf("upgrade changed %s identity: before %s, after %s", name, before, afterIdentity[name]) @@ -278,6 +278,12 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { t.Errorf("telemetry sendMetrics is true after an upgrade with no saved choice; body=%s", body) } + if os.Getenv("GAMEPLANE_E2E_ARGOCD") == "1" { + t.Run("ArgoCD", func(t *testing.T) { + checkArgoUpgrade(t, ns, gs, marker, adminUser, adminPass, credentialsNS) + }) + } + // ---- 7. F-218: a fresh `helm install` over leftover, STALE CRDs ------ // // Deliberately LAST, after steps 3-6 have proven the pre-upgrade hook diff --git a/test/e2e/upgrade_identity_e2e_test.go b/test/e2e/upgrade_identity_e2e_test.go index 2a7a73bd9..762abc7f3 100644 --- a/test/e2e/upgrade_identity_e2e_test.go +++ b/test/e2e/upgrade_identity_e2e_test.go @@ -16,7 +16,7 @@ import ( // Snapshot the populated installation's storage and trust identities. Remaining // Gameplane CRDs must include every version still recorded as stored by Kubernetes. -func readUpgradeIdentity(t *testing.T) map[string]string { +func readUpgradeIdentity(t *testing.T, issuerSecret string) map[string]string { t.Helper() identity := map[string]string{} pvc, err := envInstance.K8s.CoreV1().PersistentVolumeClaims("gameplane-system").Get(t.Context(), "gameplane-api-data", metav1.GetOptions{}) @@ -24,7 +24,7 @@ func readUpgradeIdentity(t *testing.T) map[string]string { t.Fatalf("read gameplane-system/gameplane-api-data PVC identity: %v", err) } identity["gameplane-system/gameplane-api-data"] = string(pvc.UID) - ca, err := envInstance.K8s.CoreV1().Secrets("gameplane-system").Get(t.Context(), "gameplane-agent-ca", metav1.GetOptions{}) + ca, err := envInstance.K8s.CoreV1().Secrets("gameplane-system").Get(t.Context(), issuerSecret, metav1.GetOptions{}) if err != nil || len(ca.Data["ca.crt"]) == 0 { t.Fatalf("read public agent CA certificate: %v", err) } From bc1dbfe8d0b1eb26029746e0864d25fe856dc3c8 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 11 Nov 2026 00:02:17 -0500 Subject: [PATCH 5/7] test(gitops): propagate the Argo upgrade context Signed-off-by: Baylem --- test/e2e/argocd_git_e2e_test.go | 3 ++- test/e2e/argocd_upgrade_e2e_test.go | 16 +++++++++++----- test/e2e/env.go | 20 +++++++++++++++----- test/e2e/upgrade_e2e_test.go | 6 +++--- test/e2e/upgrade_identity_e2e_test.go | 9 +++++---- 5 files changed, 36 insertions(+), 18 deletions(-) diff --git a/test/e2e/argocd_git_e2e_test.go b/test/e2e/argocd_git_e2e_test.go index f48a18992..938e0e101 100644 --- a/test/e2e/argocd_git_e2e_test.go +++ b/test/e2e/argocd_git_e2e_test.go @@ -36,7 +36,8 @@ func argoGitSource(t *testing.T) (address, revision string) { if ip := net.ParseIP(gateway); ip == nil || !ip.IsPrivate() { t.Fatalf("expected a private kind bridge gateway, got %q", gateway) } - listener, err := net.Listen("tcp", net.JoinHostPort(gateway, "0")) + var listen net.ListenConfig + listener, err := listen.Listen(t.Context(), "tcp", net.JoinHostPort(gateway, "0")) if err != nil { t.Fatal(err) } diff --git a/test/e2e/argocd_upgrade_e2e_test.go b/test/e2e/argocd_upgrade_e2e_test.go index c68bac365..b4f08a3de 100644 --- a/test/e2e/argocd_upgrade_e2e_test.go +++ b/test/e2e/argocd_upgrade_e2e_test.go @@ -94,7 +94,13 @@ func checkArgoUpgrade(t *testing.T, ns, game, marker, username, password, creden } // Applications have no cascading-deletion finalizer; the later Helm // reinstall phase owns teardown of the populated world. - t.Cleanup(func() { _ = resource.Delete(context.Background(), name, metav1.DeleteOptions{}) }) + t.Cleanup(func() { + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + defer cancel() + if err := resource.Delete(cleanup, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + t.Errorf("delete Argo CD %s %s: %v", kind, name, err) + } + }) } create(projectGVR, "AppProject", projectName, map[string]any{ "sourceRepos": []any{repo}, @@ -112,7 +118,7 @@ func checkArgoUpgrade(t *testing.T, ns, game, marker, username, password, creden "syncPolicy": map[string]any{"syncOptions": argoSyncOptions}, }) } - identity := readUpgradeIdentity(t, "gameplane-agent-ca") + identity := readUpgradeIdentity(ctx, t, "gameplane-agent-ca") claim, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(ns).Get(ctx, game+"-data", metav1.GetOptions{}) if err != nil || claim.UID == "" { t.Fatalf("read populated game PVC: %v", err) @@ -141,7 +147,7 @@ func checkArgoUpgrade(t *testing.T, ns, game, marker, username, password, creden // port-forward or cached /players response cannot verify an agent's mTLS. checkWorld := func(issuerSecret, clientSecret string, expectedCA []byte) { t.Helper() - now := readUpgradeIdentity(t, issuerSecret) + now := readUpgradeIdentity(ctx, t, issuerSecret) expected := fmt.Sprintf("%x", sha256.Sum256(expectedCA)) if now["agent CA fingerprint"] != expected { t.Fatal("active issuer fingerprint differs from staged certificate") @@ -178,10 +184,10 @@ func checkArgoUpgrade(t *testing.T, ns, game, marker, username, password, creden t.Fatalf("%s rolled out without the intended issuer/client Secret references", deployment) } } - cli := envInstance.APIClient(t, username, password) + cli := envInstance.apiClientContext(ctx, t, username, password) defer cli.Close() envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { - resp, body, err := cli.Get("/servers/" + game + "/files/read?path=%2Fmarker.txt") + resp, body, err := cli.DoContext(ctx, http.MethodGet, "/servers/"+game+"/files/read?path=%2Fmarker.txt", nil) if err != nil { return false, err.Error() } diff --git a/test/e2e/env.go b/test/e2e/env.go index f8cdf886d..70cf21f49 100644 --- a/test/e2e/env.go +++ b/test/e2e/env.go @@ -341,6 +341,11 @@ func (e *Env) WriteAdminPasswordFile(t *testing.T, pw string) { // // `target` follows kubectl conventions ("svc/foo" or "pod/foo"). func (e *Env) PortForward(t *testing.T, ns, target string, remotePort int) (int, func()) { + t.Helper() + return e.portForwardContext(t.Context(), t, ns, target, remotePort) +} + +func (e *Env) portForwardContext(ctx context.Context, t *testing.T, ns, target string, remotePort int) (int, func()) { t.Helper() // Retry the whole spawn: on a loaded self-hosted runner, kubectl // port-forward can take a while to wire the pod tunnel, or fail @@ -351,7 +356,7 @@ func (e *Env) PortForward(t *testing.T, ns, target string, remotePort int) (int, const attempts = 4 var lastErr error for attempt := 1; attempt <= attempts; attempt++ { - local, stop, err := e.tryPortForward(t.Context(), ns, target, remotePort) + local, stop, err := e.tryPortForward(ctx, ns, target, remotePort) if err == nil { return local, stop } @@ -497,7 +502,12 @@ type APIClient struct { // tear the port-forward down. func (e *Env) APIClient(t *testing.T, username, password string) *APIClient { t.Helper() - local, stop := e.PortForward(t, "gameplane-system", "svc/gameplane-api", 80) + return e.apiClientContext(t.Context(), t, username, password) +} + +func (e *Env) apiClientContext(ctx context.Context, t *testing.T, username, password string) *APIClient { + t.Helper() + local, stop := e.portForwardContext(ctx, t, "gameplane-system", "svc/gameplane-api", 80) base := fmt.Sprintf("http://127.0.0.1:%d", local) // 90s tolerates the few legitimately-slow endpoints (notably :restart, @@ -527,7 +537,7 @@ func (e *Env) APIClient(t *testing.T, username, password string) *APIClient { const maxAttempts = 7 reconnected := false for attempt := 0; attempt < maxAttempts; attempt++ { - req, rerr := http.NewRequestWithContext(t.Context(), http.MethodPost, base+"/auth/login", bytes.NewReader(body)) + req, rerr := http.NewRequestWithContext(ctx, http.MethodPost, base+"/auth/login", bytes.NewReader(body)) if rerr != nil { stop() t.Fatalf("new login request: %v", rerr) @@ -539,8 +549,8 @@ func (e *Env) APIClient(t *testing.T, username, password string) *APIClient { // A vanished local listener cannot have received this request. // Reopen once within the existing attempt budget; never replay an // ambiguous read/write failure or reconnect an established session. - if !reconnected && attempt+1 < maxAttempts && t.Context().Err() == nil && loginDialRefused(err) { - local, stop = e.PortForward(t, "gameplane-system", "svc/gameplane-api", 80) + if !reconnected && attempt+1 < maxAttempts && ctx.Err() == nil && loginDialRefused(err) { + local, stop = e.portForwardContext(ctx, t, "gameplane-system", "svc/gameplane-api", 80) base = fmt.Sprintf("http://127.0.0.1:%d", local) reconnected = true continue diff --git a/test/e2e/upgrade_e2e_test.go b/test/e2e/upgrade_e2e_test.go index 9ea17c245..09fbf6b9a 100644 --- a/test/e2e/upgrade_e2e_test.go +++ b/test/e2e/upgrade_e2e_test.go @@ -52,7 +52,7 @@ import ( // also bring them up to date. It runs last so steps 1-6 start from the old // release's genuinely stale schema. func TestUpgrade_FromPreviousRelease(t *testing.T) { - ctx := context.Background() + ctx := t.Context() ns := "gameplane-games" tmpl := "e2e-upgrade-busybox" gs := "e2e-upgrade-target" @@ -74,7 +74,7 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { applyBusyboxGameServer(t, ns, gs, tmpl) waitPVCBound(t, ns, gs+"-data", 90*time.Second) waitGameContainerReady(t, ns, gs+"-0", 2*time.Minute) - beforeIdentity := readUpgradeIdentity(t, "gameplane-agent-ca") + beforeIdentity := readUpgradeIdentity(ctx, t, "gameplane-agent-ca") legacyReadBinding := nativeReadBinding(t) checkNativeCatalogAccess(t, tmpl, true) @@ -161,7 +161,7 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { t.Fatalf("operator strategy migration retained RollingUpdate fields: %+v", operator.Spec.Strategy) } - afterIdentity := readUpgradeIdentity(t, "gameplane-agent-ca") + afterIdentity := readUpgradeIdentity(ctx, t, "gameplane-agent-ca") for name, before := range beforeIdentity { if afterIdentity[name] != before { t.Errorf("upgrade changed %s identity: before %s, after %s", name, before, afterIdentity[name]) diff --git a/test/e2e/upgrade_identity_e2e_test.go b/test/e2e/upgrade_identity_e2e_test.go index 762abc7f3..f95a62a22 100644 --- a/test/e2e/upgrade_identity_e2e_test.go +++ b/test/e2e/upgrade_identity_e2e_test.go @@ -3,6 +3,7 @@ package e2e import ( + "context" "crypto/sha256" "fmt" "slices" @@ -16,20 +17,20 @@ import ( // Snapshot the populated installation's storage and trust identities. Remaining // Gameplane CRDs must include every version still recorded as stored by Kubernetes. -func readUpgradeIdentity(t *testing.T, issuerSecret string) map[string]string { +func readUpgradeIdentity(ctx context.Context, t *testing.T, issuerSecret string) map[string]string { t.Helper() identity := map[string]string{} - pvc, err := envInstance.K8s.CoreV1().PersistentVolumeClaims("gameplane-system").Get(t.Context(), "gameplane-api-data", metav1.GetOptions{}) + pvc, err := envInstance.K8s.CoreV1().PersistentVolumeClaims("gameplane-system").Get(ctx, "gameplane-api-data", metav1.GetOptions{}) if err != nil || pvc.UID == "" { t.Fatalf("read gameplane-system/gameplane-api-data PVC identity: %v", err) } identity["gameplane-system/gameplane-api-data"] = string(pvc.UID) - ca, err := envInstance.K8s.CoreV1().Secrets("gameplane-system").Get(t.Context(), issuerSecret, metav1.GetOptions{}) + ca, err := envInstance.K8s.CoreV1().Secrets("gameplane-system").Get(ctx, issuerSecret, metav1.GetOptions{}) if err != nil || len(ca.Data["ca.crt"]) == 0 { t.Fatalf("read public agent CA certificate: %v", err) } identity["agent CA fingerprint"] = fmt.Sprintf("%x", sha256.Sum256(ca.Data["ca.crt"])) - crds, err := envInstance.Dyn.Resource(crdGVR).List(t.Context(), metav1.ListOptions{}) + crds, err := envInstance.Dyn.Resource(crdGVR).List(ctx, metav1.ListOptions{}) if err != nil { t.Fatal(err) } From 37e290110cd126ef04837f5686b9bca30efdfec6 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 11 Nov 2026 02:40:27 -0500 Subject: [PATCH 6/7] fix(e2e): wait for both Argo CD APIs to become established Signed-off-by: Baylem --- deploy/kind/argocd.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/deploy/kind/argocd.sh b/deploy/kind/argocd.sh index e5c5324f6..b398d1e47 100644 --- a/deploy/kind/argocd.sh +++ b/deploy/kind/argocd.sh @@ -14,6 +14,17 @@ echo "0b38189d6e1fd76ef5c3db10355026c48a02020d82bf21a1a1b5ab16dffe4f3c $bundle" kubectl create namespace argocd kubectl apply --server-side --namespace argocd --filename "$bundle" kubectl set env -n argocd deployment/argocd-repo-server --containers=argocd-repo-server ARGOCD_GIT_MODULES_ENABLED=false -kubectl wait --for=condition=Established crd/applications.argoproj.io crd/appprojects.argoproj.io --timeout=2m +# Immediately after creation a CRD can report null conditions. Wait for +# the API to publish Established=True on both resources before discovery. +crds=(crd/applications.argoproj.io crd/appprojects.argoproj.io) +deadline=$((SECONDS + 120)) +until kubectl get "${crds[@]}" --request-timeout=5s -o json | + jq -e '.items | length == 2 and all(.[]; any(.status.conditions[]?; .type == "Established" and .status == "True"))' >/dev/null; do + if ((SECONDS >= deadline)); then + kubectl get "${crds[@]}" -o json | jq '.items[] | {name: .metadata.name, conditions: .status.conditions}' >&2 + exit 1 + fi + sleep 1 +done kubectl rollout status -n argocd deployment/argocd-repo-server --timeout=4m kubectl rollout status -n argocd statefulset/argocd-application-controller --timeout=4m From 6a24ee488da68370f00f64ab0018f674b661250c Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 11 Nov 2026 05:22:27 -0500 Subject: [PATCH 7/7] fix(e2e): verify the Argo repo-server can fetch the candidate chart Signed-off-by: Baylem --- .github/workflows/ci.yaml | 4 +++ test/e2e/argocd_git_e2e_test.go | 64 ++++++++++++++++++++++++--------- 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3736a6962..01029934d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2161,6 +2161,10 @@ jobs: GAMEPLANE_UPGRADE_FROM: 0.3.0 steps: &e2e-upgrade-steps - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # Argo fetches the exact tested commit from a read-only Git server. + # A shallow server can advertise a ref that a fresh fetch rejects. + fetch-depth: ${{ matrix.arch == 'amd64' && '0' || '1' }} - uses: ./.github/actions/go-cache with: key-suffix: e2e diff --git a/test/e2e/argocd_git_e2e_test.go b/test/e2e/argocd_git_e2e_test.go index 938e0e101..81367e5c2 100644 --- a/test/e2e/argocd_git_e2e_test.go +++ b/test/e2e/argocd_git_e2e_test.go @@ -3,14 +3,18 @@ package e2e import ( + "context" + "fmt" "net" "net/http" "net/http/cgi" "net/http/httptest" + "os" "os/exec" "path/filepath" "strings" "testing" + "time" ) // Serve exactly the tested commit, including fork PRs, without giving the @@ -25,25 +29,24 @@ func argoGitSource(t *testing.T) (address, revision string) { } return strings.TrimSpace(string(out)) } - root := run("git", "rev-parse", "--show-toplevel") revision = run("git", "rev-parse", "HEAD") directory := t.TempDir() repo := filepath.Join(directory, "candidate.git") - run("git", "init", "--quiet", "--bare", repo) - run("git", "-C", repo, "fetch", "--quiet", "--depth=1", root, revision+":refs/heads/candidate") - run("git", "-C", repo, "symbolic-ref", "HEAD", "refs/heads/candidate") - gateway := run("docker", "network", "inspect", "kind", "--format", "{{(index .IPAM.Config 0).Gateway}}") - if ip := net.ParseIP(gateway); ip == nil || !ip.IsPrivate() { - t.Fatalf("expected a private kind bridge gateway, got %q", gateway) + if run("git", "rev-parse", "--is-shallow-repository") == "true" { + t.Fatal("the Argo CD fixture needs a full checkout to serve the candidate branch") } - var listen net.ListenConfig - listener, err := listen.Listen(t.Context(), "tcp", net.JoinHostPort(gateway, "0")) - if err != nil { + run("git", "init", "--quiet", "--bare", repo) + // Share the checkout's objects. The temporary repository publishes one + // ref and has its own configuration, with no runner credentials. + objects := run("git", "rev-parse", "--path-format=absolute", "--git-path", "objects") + if err := os.WriteFile(filepath.Join(repo, "objects", "info", "alternates"), []byte(objects+"\n"), 0o600); err != nil { t.Fatal(err) } + run("git", "-C", repo, "update-ref", "refs/heads/candidate", revision) + run("git", "-C", repo, "symbolic-ref", "HEAD", "refs/heads/candidate") gitHTTP := &cgi.Handler{Path: filepath.Join(run("git", "--exec-path"), "git-http-backend"), Dir: directory, Env: []string{"GIT_PROJECT_ROOT=" + directory, "GIT_HTTP_EXPORT_ALL=1"}} - srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { discover := r.Method == http.MethodGet && r.URL.Path == "/candidate.git/info/refs" && r.URL.Query().Get("service") == "git-upload-pack" fetch := r.Method == http.MethodPost && r.URL.Path == "/candidate.git/git-upload-pack" if !discover && !fetch { @@ -51,10 +54,37 @@ func argoGitSource(t *testing.T) (address, revision string) { return } gitHTTP.ServeHTTP(w, r) - })) - _ = srv.Listener.Close() - srv.Listener = listener - srv.Start() - t.Cleanup(srv.Close) - return srv.URL + "/candidate.git", revision + }) + // Docker may list the IPv6 bridge first even on an IPv4 Pod network. + // Use the same Git client and network namespace that will fetch the chart. + gateways := run("docker", "network", "inspect", "kind", "--format", "{{range .IPAM.Config}}{{println .Gateway}}{{end}}") + var failures []string + for _, gateway := range strings.Fields(gateways) { + if ip := net.ParseIP(gateway); ip == nil || !ip.IsPrivate() { + continue + } + var listen net.ListenConfig + listener, err := listen.Listen(t.Context(), "tcp", net.JoinHostPort(gateway, "0")) + if err != nil { + failures = append(failures, err.Error()) + continue + } + srv := httptest.NewUnstartedServer(handler) + _ = srv.Listener.Close() + srv.Listener = listener + srv.Start() + address := srv.URL + "/candidate.git" + probe, cancel := context.WithTimeout(t.Context(), 10*time.Second) + out, err := envInstance.Kubectl(probe, "-n", "argocd", "exec", "deploy/argocd-repo-server", "-c", "argocd-repo-server", + "--", "git", "ls-remote", "--exit-code", address, "refs/heads/candidate") + cancel() + if err == nil && strings.TrimSpace(out) == revision+"\trefs/heads/candidate" { + t.Cleanup(srv.Close) + return address, revision + } + srv.Close() + failures = append(failures, fmt.Sprintf("%s: %v %s", gateway, err, strings.TrimSpace(out))) + } + t.Fatalf("Argo CD repo-server could not read revision %s through the private kind gateways %q: %v", revision, gateways, failures) + return "", "" }