diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7cb866a6a..f9965c7f9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -271,6 +271,31 @@ jobs: cosign public-key --key env://COSIGN_PRIVATE_KEY --outfile "$RUNNER_TEMP/cosign.pub" cosign verify --key "$RUNNER_TEMP/cosign.pub" --new-bundle-format=false "$REF" + consumer: + name: qualify published artifacts as an anonymous consumer + runs-on: ubuntu-latest + needs: chart + permissions: + contents: read + timeout-minutes: 50 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/go-cache + with: + key-suffix: e2e + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + - uses: helm/kind-action@ea822c7bf1874f08e5a491df991c228edffc3376 # v1.15.2 + with: + install_only: true + - uses: oras-project/setup-oras@5e2f2482b30b0cb3e14e67f89a80c88c659a7c63 # v2.0.3 + - uses: sigstore/cosign-installer@756fe2d284c7292c4e9b62c8a28f1e5629c59532 # v4.1.3 + with: + cosign-release: 'v3.1.2' + - name: verify, install and upgrade using published chart and images + run: bash deploy/kind/qualify-published-release.sh + # Publish the GitHub Release page for the tag, with the matching CHANGELOG # section as the notes. beta.3's page was created by hand and beta.4's was # forgotten entirely — this makes the page part of the release, not a @@ -281,7 +306,7 @@ jobs: timeout-minutes: 15 # applied by extension per D-A, pending maintainer confirmation permissions: contents: write - needs: [images, chart] + needs: consumer steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: extract changelog section diff --git a/deploy/kind/qualify-published-release.sh b/deploy/kind/qualify-published-release.sh new file mode 100644 index 000000000..73836bdf3 --- /dev/null +++ b/deploy/kind/qualify-published-release.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Release CI runs this on a fresh amd64 runner without a registry login. +set -euo pipefail +version="${GITHUB_REF_NAME#v}" +owner="${GITHUB_REPOSITORY_OWNER,,}" +registry="ghcr.io/$owner" +scratch="$(mktemp -d)" +export DOCKER_CONFIG="$scratch/docker" HELM_REGISTRY_CONFIG="$scratch/helm.json" +export GAMEPLANE_PUBLISHED_CHART="$scratch/gameplane" GAMEPLANE_RELEASE_DIGESTS="$scratch/images.tsv" +mkdir -p "$DOCKER_CONFIG" +cosign initialize + +chart="$registry/charts/gameplane" +digest="$(oras resolve "$chart:$version")" +cosign verify --key signing/cosign.pub --new-bundle-format=false "$chart@$digest" >/dev/null +helm pull "oci://$chart@$digest" --untar --untardir "$scratch" +test "$(yq -r '.version' "$GAMEPLANE_PUBLISHED_CHART/Chart.yaml")" = "$version" +test "$(yq -r '.appVersion' "$GAMEPLANE_PUBLISHED_CHART/Chart.yaml")" = "$version" +printf 'Verified chart `%s@%s`\n' "$chart" "$digest" >> "$GITHUB_STEP_SUMMARY" + +# Read the publisher's complete matrix, including optional images. Keep the +# signed index and its amd64 manifest/config IDs for containerd comparisons. +components="$(yq -r '.jobs.images.strategy.matrix.include[].component' .github/workflows/release.yaml)" +test -n "$components" +while read -r component; do + image="$registry/gameplane/$component" + index="$(oras resolve "$image:$version")" + ref="$image@$index" + cosign verify --key signing/cosign.pub --new-bundle-format=false "$ref" >/dev/null + docker pull --platform linux/amd64 "$ref" + manifest="$(oras manifest fetch "$ref" | jq -er '[.manifests[] | select(.platform.os == "linux" and .platform.architecture == "amd64") | .digest] | if length == 1 then .[0] else error("expected one amd64 image") end')" + config="$(oras manifest fetch "$image@$manifest" | jq -er '.config.digest')" + printf '%s\t%s\t%s\t%s\t%s\n' "$component" "$image:$version" "$index" "$manifest" "$config" >> "$GAMEPLANE_RELEASE_DIGESTS" + printf '\n- `%s`\n' "$ref" >> "$GITHUB_STEP_SUMMARY" +done <<< "$components" + +kind create cluster --name gp-consumer-fresh --wait 90s +helm install gameplane "$GAMEPLANE_PUBLISHED_CHART" --kube-context kind-gp-consumer-fresh \ + --namespace gameplane-system --create-namespace --set ingress.enabled=false \ + --set defaultModuleSource.enabled=false --wait --timeout 6m +bash deploy/kind/verify-release-runtime.sh kind-gp-consumer-fresh api operator web +kind delete cluster --name gp-consumer-fresh + +# The regular upgrade fixture seeds the old API database and game PVC, +# exercises the CRD hooks, and checks their contents after the upgrade. +export GAMEPLANE_E2E_CLUSTER=gp-consumer-upgrade GAMEPLANE_E2E_REUSE_CLUSTER=1 +deploy/kind/upgrade.sh up "$GAMEPLANE_E2E_CLUSTER" +(cd test/e2e && go test -tags=e2e -timeout 25m -v -run '^TestUpgrade_FromPreviousRelease$' .) diff --git a/deploy/kind/upgrade.sh b/deploy/kind/upgrade.sh index 0a7315716..8e14e76fc 100755 --- a/deploy/kind/upgrade.sh +++ b/deploy/kind/upgrade.sh @@ -63,14 +63,16 @@ EOF # Load the NEW (working-tree) images now, so the upgrade the test performs # later needs no network. The OLD release's images are pulled from GHCR by # the install below — that is the point of the test. - echo "loading working-tree gameplane-test/{operator,api,agent,sentinel}:${TAG} into kind" - for img in operator api agent sentinel; do - if ! docker image inspect "gameplane-test/${img}:${TAG}" >/dev/null 2>&1; then - echo " missing local image gameplane-test/${img}:${TAG} — building" - docker build -t "gameplane-test/${img}:${TAG}" -f "${REPO}/${img}/Dockerfile" "${REPO}" - fi - kind load docker-image "gameplane-test/${img}:${TAG}" --name "${CLUSTER}" - done + if [[ -z "${GAMEPLANE_PUBLISHED_CHART:-}" ]]; then + echo "loading working-tree gameplane-test/{operator,api,agent,sentinel}:${TAG} into kind" + for img in operator api agent sentinel; do + if ! docker image inspect "gameplane-test/${img}:${TAG}" >/dev/null 2>&1; then + echo " missing local image gameplane-test/${img}:${TAG} — building" + docker build -t "gameplane-test/${img}:${TAG}" -f "${REPO}/${img}/Dockerfile" "${REPO}" + fi + kind load docker-image "gameplane-test/${img}:${TAG}" --name "${CLUSTER}" + done + fi echo "installing PUBLISHED chart ${CHART_REF} --version ${FROM_VERSION}" # Value overrides mirror e2e.sh's, and are all about the harness rather diff --git a/deploy/kind/verify-release-runtime.sh b/deploy/kind/verify-release-runtime.sh new file mode 100644 index 000000000..d32f9ec63 --- /dev/null +++ b/deploy/kind/verify-release-runtime.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Accept only the signed index, native-platform manifest or image config +# digest derived from that index. Runtimes differ in which ID they expose. +set -euo pipefail +context="$1" +shift +for component in "$@"; do + row="$(awk -F '\t' -v name="$component" '$1 == name' "$GAMEPLANE_RELEASE_DIGESTS")" + test -n "$row" + IFS=$'\t' read -r _ image index manifest config <<< "$row" + for ((attempt=0; attempt<60; attempt++)); do + pods="$(kubectl --context "$context" get pods -A -o json)" + ids="$(jq -r --arg image "$image" '.items[] | select(.metadata.deletionTimestamp == null) + | .status.containerStatuses[]? | select(.ready and .image == $image) | .imageID' <<< "$pods")" + [[ -n "$ids" ]] && break + sleep 2 + done + if [[ -z "$ids" ]]; then + echo "No Ready container running $image in $context" >&2 + exit 1 + fi + while read -r id; do + digest="${id##*@}" + digest="${digest##*://}" + if [[ "$digest" != "$index" && "$digest" != "$manifest" && "$digest" != "$config" ]]; then + echo "Unexpected runtime image ID for $image: $id" >&2 + exit 1 + fi + printf '%s %s %s\n' "$context" "$image" "$id" + printf '\n- `%s` ran `%s` as `%s`\n' "$context" "$image" "$id" >> "$GITHUB_STEP_SUMMARY" + done <<< "$ids" +done diff --git a/docs/install.md b/docs/install.md index f0b5767d7..16cf7735c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -951,6 +951,12 @@ test and asserts that - the pre-upgrade admin can still log in — i.e. the new binary's database migrations ran against the populated SQLite volume without data loss. +The release workflow also pulls the signed chart and release images from GHCR +on a fresh amd64 runner using anonymous access and the public signing key. +It installs the verified chart in a clean Kind cluster, then runs the populated +upgrade fixture in a second cluster using the published candidate. The job +summary records the chart and image digests and the image IDs actually running. + See `test/e2e/upgrade_e2e_test.go`. What this does **not** yet cover: upgrades that skip several releases at once, and Postgres (still an experimental driver — see [`roadmap.md`](roadmap.md)). Take a backup before upgrading diff --git a/test/e2e/upgrade_e2e_test.go b/test/e2e/upgrade_e2e_test.go index 56736b34c..7c3fc8214 100644 --- a/test/e2e/upgrade_e2e_test.go +++ b/test/e2e/upgrade_e2e_test.go @@ -77,6 +77,17 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { t.Fatalf("write marker: %v\n%s", err, out) } + claim, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(ns).Get(ctx, gs+"-data", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if chart := os.Getenv("GAMEPLANE_PUBLISHED_CHART"); chart != "" { + upgradePublishedChart(t, chart) + verifyPopulatedUpgrade(t, ns, gs, marker, adminUser, adminPass, claim.UID) + checkPublishedRuntime(t) + return + } + // ---- 2. work out what the upgrade is supposed to change -------------- // Rather than hardcoding a field name that goes stale, compute the drift: @@ -165,68 +176,14 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { } } - // ---- 5. the workload and its data survived --------------------------- - - got, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns). - Get(ctx, gs, metav1.GetOptions{}) - if err != nil { - t.Fatalf("GameServer %s/%s is gone after the upgrade: %v", ns, gs, err) - } - if suspend, _, _ := unstructured.NestedBool(got.Object, "spec", "suspend"); suspend { - t.Errorf("GameServer was suspended by the upgrade; spec.suspend should be untouched") - } - - waitPVCBound(t, ns, gs+"-data", 2*time.Minute) - waitGameContainerReady(t, ns, gs+"-0", 4*time.Minute) - - // Retry the read rather than exec'ing once. An upgrade re-renders the - // StatefulSet, so the pod keeps its name but is legitimately replaced — - // a Ready observation can be of the outgoing pod, and the exec then - // lands mid-restart ("container not found", "no running task found"). - // That is the workload behaving correctly, not a regression, so the - // assertion tolerates the churn and waits for the volume to be readable. - // What is being proven is that the bytes survived, not that they were - // readable at one exact instant. - envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { - out, err := envInstance.KubectlExec(t, ns, "pod/"+gs+"-0", "cat", "/data/marker.txt") - if err != nil { - return false, "exec: " + err.Error() + " out=" + out - } - // kubectl mixes its "Defaulted container ..." stderr preamble into - // CombinedOutput, so check for the marker as a substring. - if !strings.Contains(out, marker) { - return false, "marker not in output yet: " + out - } - return true, "" - }) - - // ---- 6. the database and its migrations survived --------------------- - - // Deliberately after the upgrade: the API pod was replaced, so this both - // re-establishes a port-forward against the NEW pod and proves the new - // binary's migrations ran against the old release's populated SQLite file - // without dropping the user rows. - client := envInstance.APIClient(t, adminUser, adminPass) - defer client.Close() - - resp, body, err := client.Get("/servers/" + gs + "?namespace=" + ns) - if err != nil { - t.Fatalf("read the pre-upgrade server through the upgraded API: %v", err) - } - if resp.StatusCode != 200 { - t.Fatalf("GET /servers/%s after upgrade: status %d body %s", gs, resp.StatusCode, body) - } - if !strings.Contains(string(body), gs) { - t.Errorf("upgraded API does not return the pre-upgrade GameServer %q; body=%s", gs, body) - } - resp.Body.Close() + client := verifyPopulatedUpgrade(t, ns, gs, marker, adminUser, adminPass, claim.UID) // ---- 6b. telemetry stays off for an install that never opted in ------ // // The previous release had no telemetry choice saved for this install, so // the upgrade must not turn reporting on or show the notice (spec 022 US1 // scenario 5): migration 015 seeds consent_source "legacy". - resp, body, err = client.Get("/admin/telemetry/notice") + resp, body, err := client.Get("/admin/telemetry/notice") if err != nil || resp.StatusCode != 200 { t.Fatalf("GET /admin/telemetry/notice after upgrade: %v status %d body %s", err, resp.StatusCode, body) } diff --git a/test/e2e/upgrade_published_e2e_test.go b/test/e2e/upgrade_published_e2e_test.go new file mode 100644 index 000000000..4050dcd32 --- /dev/null +++ b/test/e2e/upgrade_published_e2e_test.go @@ -0,0 +1,110 @@ +//go:build e2e + +package e2e + +import ( + "context" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +// The previous published chart was installed by deploy/kind/upgrade.sh. Use +// the downloaded candidate and let it choose its own registry and image tags. +func upgradePublishedChart(t *testing.T, chart string) { + t.Helper() + cmd := exec.CommandContext(t.Context(), "helm", "upgrade", "gameplane", chart, + "--kube-context", envInstance.Context, "--namespace", "gameplane-system", + "--set", "web.enabled=true", "--set", "ingress.enabled=false", + "--set", "api.resources.limits.memory=1Gi", "--set", "defaultModuleSource.enabled=false", + // The seeded legacy template uses the original busybox security profile. + "--set", "podSecurity.enforceRestricted=false", "--wait", "--timeout", "6m") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("upgrade to published chart: %v\n%s", err, out) + } +} + +func checkPublishedRuntime(t *testing.T) { + t.Helper() + check := exec.CommandContext(t.Context(), "bash", filepath.Join("..", "..", "deploy", "kind", "verify-release-runtime.sh"), + envInstance.Context, "api", "operator", "web", "agent") + out, err := check.CombinedOutput() + if err != nil { + t.Fatalf("published upgrade runtime image IDs: %v\n%s", err, out) + } + t.Logf("published upgrade image IDs:\n%s", out) +} + +// Both PR CI and release CI must prove that the seeded world, PVC and account +// survived the upgrade. A Ready replacement pod alone does not establish this. +func verifyPopulatedUpgrade(t *testing.T, ns, gs, marker, adminUser, adminPass string, previousClaim types.UID) *APIClient { + t.Helper() + ctx := context.Background() + // ---- 5. the workload and its data survived --------------------------- + + got, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns). + Get(ctx, gs, metav1.GetOptions{}) + if err != nil { + t.Fatalf("GameServer %s/%s is gone after the upgrade: %v", ns, gs, err) + } + if suspend, _, _ := unstructured.NestedBool(got.Object, "spec", "suspend"); suspend { + t.Errorf("GameServer was suspended by the upgrade; spec.suspend should be untouched") + } + + waitPVCBound(t, ns, gs+"-data", 2*time.Minute) + claim, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(ns).Get(ctx, gs+"-data", metav1.GetOptions{}) + if err != nil || claim.UID != previousClaim { + t.Fatalf("upgrade replaced the game data claim or made it unavailable: %v", err) + } + waitGameContainerReady(t, ns, gs+"-0", 4*time.Minute) + + // Retry the read rather than exec'ing once. An upgrade re-renders the + // StatefulSet, so the pod keeps its name but is legitimately replaced — + // a Ready observation can be of the outgoing pod, and the exec then + // lands mid-restart ("container not found", "no running task found"). + // That is the workload behaving correctly, not a regression, so the + // assertion tolerates the churn and waits for the volume to be readable. + // What is being proven is that the bytes survived, not that they were + // readable at one exact instant. + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + out, err := envInstance.KubectlExec(t, ns, "pod/"+gs+"-0", "cat", "/data/marker.txt") + if err != nil { + return false, "exec: " + err.Error() + " out=" + out + } + // kubectl mixes its "Defaulted container ..." stderr preamble into + // CombinedOutput, so check for the marker as a substring. + if !strings.Contains(out, marker) { + return false, "marker not in output yet: " + out + } + return true, "" + }) + + // ---- 6. the database and its migrations survived --------------------- + + // Deliberately after the upgrade: the API pod was replaced, so this both + // re-establishes a port-forward against the NEW pod and proves the new + // binary's migrations ran against the old release's populated SQLite file + // without dropping the user rows. + client := envInstance.APIClient(t, adminUser, adminPass) + t.Cleanup(client.Close) + + resp, body, err := client.Get("/servers/" + gs + "?namespace=" + ns) + if err != nil { + t.Fatalf("read the pre-upgrade server through the upgraded API: %v", err) + } + if resp.StatusCode != 200 { + t.Fatalf("GET /servers/%s after upgrade: status %d body %s", gs, resp.StatusCode, body) + } + if !strings.Contains(string(body), gs) { + t.Errorf("upgraded API does not return the pre-upgrade GameServer %q; body=%s", gs, body) + } + resp.Body.Close() + + return client +}