test(release): verify published charts and images from a fresh consumer

.github/workflows/release.yaml

@@ -271,6 +271,31 @@ jobs:
           cosign public-key --key env://COSIGN_PRIVATE_KEY --outfile "$RUNNER_TEMP/cosign.pub"
           cosign verify --key "$RUNNER_TEMP/cosign.pub" --new-bundle-format=false "$REF"
 
+  consumer:
+    name: qualify published artifacts as an anonymous consumer
+    runs-on: ubuntu-latest
+    needs: chart
+    permissions:
+      contents: read
+    timeout-minutes: 50
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/go-cache
+        with:
+          key-suffix: e2e
+      - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
+      - uses: helm/kind-action@ea822c7bf1874f08e5a491df991c228edffc3376 # v1.15.2
+        with:
+          install_only: true
+      - uses: oras-project/setup-oras@5e2f2482b30b0cb3e14e67f89a80c88c659a7c63 # v2.0.3
+      - uses: sigstore/cosign-installer@756fe2d284c7292c4e9b62c8a28f1e5629c59532 # v4.1.3
+        with:
+          cosign-release: 'v3.1.2'
+      - name: verify, install and upgrade using published chart and images
+        run: bash deploy/kind/qualify-published-release.sh
+
   # Publish the GitHub Release page for the tag, with the matching CHANGELOG
   # section as the notes. beta.3's page was created by hand and beta.4's was
   # forgotten entirely — this makes the page part of the release, not a
@@ -281,7 +306,7 @@ jobs:
     timeout-minutes: 15 # applied by extension per D-A, pending maintainer confirmation
     permissions:
       contents: write
-    needs: [images, chart]
+    needs: consumer
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: extract changelog section

deploy/kind/qualify-published-release.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Release CI runs this on a fresh amd64 runner without a registry login.
+set -euo pipefail
+version="${GITHUB_REF_NAME#v}"
+owner="${GITHUB_REPOSITORY_OWNER,,}"
+registry="ghcr.io/$owner"
+scratch="$(mktemp -d)"
+export DOCKER_CONFIG="$scratch/docker" HELM_REGISTRY_CONFIG="$scratch/helm.json"
+export GAMEPLANE_PUBLISHED_CHART="$scratch/gameplane" GAMEPLANE_RELEASE_DIGESTS="$scratch/images.tsv"
+mkdir -p "$DOCKER_CONFIG"
+cosign initialize
+
+chart="$registry/charts/gameplane"
+digest="$(oras resolve "$chart:$version")"
+cosign verify --key signing/cosign.pub --new-bundle-format=false "$chart@$digest" >/dev/null
+helm pull "oci://$chart@$digest" --untar --untardir "$scratch"
+test "$(yq -r '.version' "$GAMEPLANE_PUBLISHED_CHART/Chart.yaml")" = "$version"
+test "$(yq -r '.appVersion' "$GAMEPLANE_PUBLISHED_CHART/Chart.yaml")" = "$version"
+printf 'Verified chart `%s@%s`\n' "$chart" "$digest" >> "$GITHUB_STEP_SUMMARY"
+
+# Read the publisher's complete matrix, including optional images. Keep the
+# signed index and its amd64 manifest/config IDs for containerd comparisons.
+components="$(yq -r '.jobs.images.strategy.matrix.include[].component' .github/workflows/release.yaml)"
+test -n "$components"
+while read -r component; do
+  image="$registry/gameplane/$component"
+  index="$(oras resolve "$image:$version")"
+  ref="$image@$index"
+  cosign verify --key signing/cosign.pub --new-bundle-format=false "$ref" >/dev/null
+  docker pull --platform linux/amd64 "$ref"
+  manifest="$(oras manifest fetch "$ref" | jq -er '[.manifests[] | select(.platform.os == "linux" and .platform.architecture == "amd64") | .digest] | if length == 1 then .[0] else error("expected one amd64 image") end')"
+  config="$(oras manifest fetch "$image@$manifest" | jq -er '.config.digest')"
+  printf '%s\t%s\t%s\t%s\t%s\n' "$component" "$image:$version" "$index" "$manifest" "$config" >> "$GAMEPLANE_RELEASE_DIGESTS"
+  printf '\n- `%s`\n' "$ref" >> "$GITHUB_STEP_SUMMARY"
+done <<< "$components"
+
+kind create cluster --name gp-consumer-fresh --wait 90s
+helm install gameplane "$GAMEPLANE_PUBLISHED_CHART" --kube-context kind-gp-consumer-fresh \
+  --namespace gameplane-system --create-namespace --set ingress.enabled=false \
+  --set defaultModuleSource.enabled=false --wait --timeout 6m
+bash deploy/kind/verify-release-runtime.sh kind-gp-consumer-fresh api operator web
+kind delete cluster --name gp-consumer-fresh
+
+# The regular upgrade fixture seeds the old API database and game PVC,
+# exercises the CRD hooks, and checks their contents after the upgrade.
+export GAMEPLANE_E2E_CLUSTER=gp-consumer-upgrade GAMEPLANE_E2E_REUSE_CLUSTER=1
+deploy/kind/upgrade.sh up "$GAMEPLANE_E2E_CLUSTER"
+(cd test/e2e && go test -tags=e2e -timeout 25m -v -run '^TestUpgrade_FromPreviousRelease$' .)

deploy/kind/upgrade.sh

@@ -63,14 +63,16 @@ EOF
     # Load the NEW (working-tree) images now, so the upgrade the test performs
     # later needs no network. The OLD release's images are pulled from GHCR by
     # the install below — that is the point of the test.
-    echo "loading working-tree gameplane-test/{operator,api,agent,sentinel}:${TAG} into kind"
-    for img in operator api agent sentinel; do
-        if ! docker image inspect "gameplane-test/${img}:${TAG}" >/dev/null 2>&1; then
-            echo "  missing local image gameplane-test/${img}:${TAG} — building"
-            docker build -t "gameplane-test/${img}:${TAG}" -f "${REPO}/${img}/Dockerfile" "${REPO}"
-        fi
-        kind load docker-image "gameplane-test/${img}:${TAG}" --name "${CLUSTER}"
-    done
+    if [[ -z "${GAMEPLANE_PUBLISHED_CHART:-}" ]]; then
+        echo "loading working-tree gameplane-test/{operator,api,agent,sentinel}:${TAG} into kind"
+        for img in operator api agent sentinel; do
+            if ! docker image inspect "gameplane-test/${img}:${TAG}" >/dev/null 2>&1; then
+                echo "  missing local image gameplane-test/${img}:${TAG} — building"
+                docker build -t "gameplane-test/${img}:${TAG}" -f "${REPO}/${img}/Dockerfile" "${REPO}"
+            fi
+            kind load docker-image "gameplane-test/${img}:${TAG}" --name "${CLUSTER}"
+        done
+    fi
 
     echo "installing PUBLISHED chart ${CHART_REF} --version ${FROM_VERSION}"
     # Value overrides mirror e2e.sh's, and are all about the harness rather

deploy/kind/verify-release-runtime.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Accept only the signed index, native-platform manifest or image config
+# digest derived from that index. Runtimes differ in which ID they expose.
+set -euo pipefail
+context="$1"
+shift
+for component in "$@"; do
+  row="$(awk -F '\t' -v name="$component" '$1 == name' "$GAMEPLANE_RELEASE_DIGESTS")"
+  test -n "$row"
+  IFS=$'\t' read -r _ image index manifest config <<< "$row"
+  for ((attempt=0; attempt<60; attempt++)); do
+    pods="$(kubectl --context "$context" get pods -A -o json)"
+    ids="$(jq -r --arg image "$image" '.items[] | select(.metadata.deletionTimestamp == null)
+      | .status.containerStatuses[]? | select(.ready and .image == $image) | .imageID' <<< "$pods")"
+    [[ -n "$ids" ]] && break
+    sleep 2
+  done
+  if [[ -z "$ids" ]]; then
+    echo "No Ready container running $image in $context" >&2
+    exit 1
+  fi
+  while read -r id; do
+    digest="${id##*@}"
+    digest="${digest##*://}"
+    if [[ "$digest" != "$index" && "$digest" != "$manifest" && "$digest" != "$config" ]]; then
+      echo "Unexpected runtime image ID for $image: $id" >&2
+      exit 1
+    fi
+    printf '%s %s %s\n' "$context" "$image" "$id"
+    printf '\n- `%s` ran `%s` as `%s`\n' "$context" "$image" "$id" >> "$GITHUB_STEP_SUMMARY"
+  done <<< "$ids"
+done

docs/install.md

@@ -951,6 +951,12 @@ test and asserts that
 - the pre-upgrade admin can still log in — i.e. the new binary's database
   migrations ran against the populated SQLite volume without data loss.
 
+The release workflow also pulls the signed chart and release images from GHCR
+on a fresh amd64 runner using anonymous access and the public signing key.
+It installs the verified chart in a clean Kind cluster, then runs the populated
+upgrade fixture in a second cluster using the published candidate. The job
+summary records the chart and image digests and the image IDs actually running.
+
 See `test/e2e/upgrade_e2e_test.go`. What this does **not** yet cover: upgrades
 that skip several releases at once, and Postgres (still an experimental
 driver — see [`roadmap.md`](roadmap.md)). Take a backup before upgrading

test/e2e/upgrade_e2e_test.go

@@ -77,6 +77,17 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) {
 		t.Fatalf("write marker: %v\n%s", err, out)
 	}
 
+	claim, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(ns).Get(ctx, gs+"-data", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if chart := os.Getenv("GAMEPLANE_PUBLISHED_CHART"); chart != "" {
+		upgradePublishedChart(t, chart)
+		verifyPopulatedUpgrade(t, ns, gs, marker, adminUser, adminPass, claim.UID)
+		checkPublishedRuntime(t)
+		return
+	}
+
 	// ---- 2. work out what the upgrade is supposed to change --------------
 
 	// Rather than hardcoding a field name that goes stale, compute the drift:
@@ -165,68 +176,14 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) {
 		}
 	}
 
-	// ---- 5. the workload and its data survived ---------------------------
-
-	got, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).
-		Get(ctx, gs, metav1.GetOptions{})
-	if err != nil {
-		t.Fatalf("GameServer %s/%s is gone after the upgrade: %v", ns, gs, err)
-	}
-	if suspend, _, _ := unstructured.NestedBool(got.Object, "spec", "suspend"); suspend {
-		t.Errorf("GameServer was suspended by the upgrade; spec.suspend should be untouched")
-	}
-
-	waitPVCBound(t, ns, gs+"-data", 2*time.Minute)
-	waitGameContainerReady(t, ns, gs+"-0", 4*time.Minute)
-
-	// Retry the read rather than exec'ing once. An upgrade re-renders the
-	// StatefulSet, so the pod keeps its name but is legitimately replaced —
-	// a Ready observation can be of the outgoing pod, and the exec then
-	// lands mid-restart ("container not found", "no running task found").
-	// That is the workload behaving correctly, not a regression, so the
-	// assertion tolerates the churn and waits for the volume to be readable.
-	// What is being proven is that the bytes survived, not that they were
-	// readable at one exact instant.
-	envInstance.Eventually(t, 3*time.Minute, func() (bool, string) {
-		out, err := envInstance.KubectlExec(t, ns, "pod/"+gs+"-0", "cat", "/data/marker.txt")
-		if err != nil {
-			return false, "exec: " + err.Error() + " out=" + out
-		}
-		// kubectl mixes its "Defaulted container ..." stderr preamble into
-		// CombinedOutput, so check for the marker as a substring.
-		if !strings.Contains(out, marker) {
-			return false, "marker not in output yet: " + out
-		}
-		return true, ""
-	})
-
-	// ---- 6. the database and its migrations survived ---------------------
-
-	// Deliberately after the upgrade: the API pod was replaced, so this both
-	// re-establishes a port-forward against the NEW pod and proves the new
-	// binary's migrations ran against the old release's populated SQLite file
-	// without dropping the user rows.
-	client := envInstance.APIClient(t, adminUser, adminPass)
-	defer client.Close()
-
-	resp, body, err := client.Get("/servers/" + gs + "?namespace=" + ns)
-	if err != nil {
-		t.Fatalf("read the pre-upgrade server through the upgraded API: %v", err)
-	}
-	if resp.StatusCode != 200 {
-		t.Fatalf("GET /servers/%s after upgrade: status %d body %s", gs, resp.StatusCode, body)
-	}
-	if !strings.Contains(string(body), gs) {
-		t.Errorf("upgraded API does not return the pre-upgrade GameServer %q; body=%s", gs, body)
-	}
-	resp.Body.Close()
+	client := verifyPopulatedUpgrade(t, ns, gs, marker, adminUser, adminPass, claim.UID)
 
 	// ---- 6b. telemetry stays off for an install that never opted in ------
 	//
 	// The previous release had no telemetry choice saved for this install, so
 	// the upgrade must not turn reporting on or show the notice (spec 022 US1
 	// scenario 5): migration 015 seeds consent_source "legacy".
-	resp, body, err = client.Get("/admin/telemetry/notice")
+	resp, body, err := client.Get("/admin/telemetry/notice")
 	if err != nil || resp.StatusCode != 200 {
 		t.Fatalf("GET /admin/telemetry/notice after upgrade: %v status %d body %s", err, resp.StatusCode, body)
 	}

test/e2e/upgrade_published_e2e_test.go

@@ -0,0 +1,110 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"context"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+// The previous published chart was installed by deploy/kind/upgrade.sh. Use
+// the downloaded candidate and let it choose its own registry and image tags.
+func upgradePublishedChart(t *testing.T, chart string) {
+	t.Helper()
+	cmd := exec.CommandContext(t.Context(), "helm", "upgrade", "gameplane", chart,
+		"--kube-context", envInstance.Context, "--namespace", "gameplane-system",
+		"--set", "web.enabled=true", "--set", "ingress.enabled=false",
+		"--set", "api.resources.limits.memory=1Gi", "--set", "defaultModuleSource.enabled=false",
+		// The seeded legacy template uses the original busybox security profile.
+		"--set", "podSecurity.enforceRestricted=false", "--wait", "--timeout", "6m")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("upgrade to published chart: %v\n%s", err, out)
+	}
+}
+
+func checkPublishedRuntime(t *testing.T) {
+	t.Helper()
+	check := exec.CommandContext(t.Context(), "bash", filepath.Join("..", "..", "deploy", "kind", "verify-release-runtime.sh"),
+		envInstance.Context, "api", "operator", "web", "agent")
+	out, err := check.CombinedOutput()
+	if err != nil {
+		t.Fatalf("published upgrade runtime image IDs: %v\n%s", err, out)
+	}
+	t.Logf("published upgrade image IDs:\n%s", out)
+}
+
+// Both PR CI and release CI must prove that the seeded world, PVC and account
+// survived the upgrade. A Ready replacement pod alone does not establish this.
+func verifyPopulatedUpgrade(t *testing.T, ns, gs, marker, adminUser, adminPass string, previousClaim types.UID) *APIClient {
+	t.Helper()
+	ctx := context.Background()
+	// ---- 5. the workload and its data survived ---------------------------
+
+	got, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).
+		Get(ctx, gs, metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("GameServer %s/%s is gone after the upgrade: %v", ns, gs, err)
+	}
+	if suspend, _, _ := unstructured.NestedBool(got.Object, "spec", "suspend"); suspend {
+		t.Errorf("GameServer was suspended by the upgrade; spec.suspend should be untouched")
+	}
+
+	waitPVCBound(t, ns, gs+"-data", 2*time.Minute)
+	claim, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(ns).Get(ctx, gs+"-data", metav1.GetOptions{})
+	if err != nil || claim.UID != previousClaim {
+		t.Fatalf("upgrade replaced the game data claim or made it unavailable: %v", err)
+	}
+	waitGameContainerReady(t, ns, gs+"-0", 4*time.Minute)
+
+	// Retry the read rather than exec'ing once. An upgrade re-renders the
+	// StatefulSet, so the pod keeps its name but is legitimately replaced —
+	// a Ready observation can be of the outgoing pod, and the exec then
+	// lands mid-restart ("container not found", "no running task found").
+	// That is the workload behaving correctly, not a regression, so the
+	// assertion tolerates the churn and waits for the volume to be readable.
+	// What is being proven is that the bytes survived, not that they were
+	// readable at one exact instant.
+	envInstance.Eventually(t, 3*time.Minute, func() (bool, string) {
+		out, err := envInstance.KubectlExec(t, ns, "pod/"+gs+"-0", "cat", "/data/marker.txt")
+		if err != nil {
+			return false, "exec: " + err.Error() + " out=" + out
+		}
+		// kubectl mixes its "Defaulted container ..." stderr preamble into
+		// CombinedOutput, so check for the marker as a substring.
+		if !strings.Contains(out, marker) {
+			return false, "marker not in output yet: " + out
+		}
+		return true, ""
+	})
+
+	// ---- 6. the database and its migrations survived ---------------------
+
+	// Deliberately after the upgrade: the API pod was replaced, so this both
+	// re-establishes a port-forward against the NEW pod and proves the new
+	// binary's migrations ran against the old release's populated SQLite file
+	// without dropping the user rows.
+	client := envInstance.APIClient(t, adminUser, adminPass)
+	t.Cleanup(client.Close)
+
+	resp, body, err := client.Get("/servers/" + gs + "?namespace=" + ns)
+	if err != nil {
+		t.Fatalf("read the pre-upgrade server through the upgraded API: %v", err)
+	}
+	if resp.StatusCode != 200 {
+		t.Fatalf("GET /servers/%s after upgrade: status %d body %s", gs, resp.StatusCode, body)
+	}
+	if !strings.Contains(string(body), gs) {
+		t.Errorf("upgraded API does not return the pre-upgrade GameServer %q; body=%s", gs, body)
+	}
+	resp.Body.Close()
+
+	return client
+}