diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a0d6e05af..313332f7b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -38,12 +38,17 @@ on: # Feature catalog and the website gitlink feed the features sync gate (spec 024). - "FEATURES.md" - "website" - # Manual full-suite run on any branch (spec 023 OD-7). + # An empty dispatch runs the full suite (spec 023 OD-7). A tag runs the release consumer. workflow_dispatch: + inputs: + published_release: + description: Run only release qualification for this published tag (for example v1.0.0) + required: false + type: string # Cancel superseded runs so rapid pushes to the same ref don't waste # runner minutes on stale commits. concurrency: - group: ci-${{ github.ref }} + group: ci-${{ inputs.published_release && format('published-{0}', inputs.published_release) || 'suite' }}-${{ github.ref }} cancel-in-progress: true # The Go module mirror and checksum database have repeatedly dropped HTTP/2 @@ -69,6 +74,15 @@ permissions: # its own kind cluster, so the slowest suite — not the sum of all of # them — sets the wall-clock time. jobs: + published-release: + name: qualify a published release + if: github.event_name == 'workflow_dispatch' && inputs.published_release != '' + permissions: + contents: read + uses: ./.github/workflows/qualify-release.yaml + with: + release_tag: ${{ inputs.published_release }} + # Path-filter gate. Computes which parts of the tree a change touches so the # heavy jobs below can skip when they can't be affected (a web-only PR drops # from ~22 jobs to a handful). Any shared/CI-config path (this workflow, the @@ -79,6 +93,7 @@ jobs: # selection table to this job's summary. changes: name: detect changes + if: github.event_name != 'workflow_dispatch' || inputs.published_release == '' runs-on: ubuntu-latest timeout-minutes: 5 permissions: @@ -849,6 +864,7 @@ jobs: # separate: not part of `make lint`, and not in the `report` job's needs. telemetry-default-gate: name: telemetry default gate + if: github.event_name != 'workflow_dispatch' || inputs.published_release == '' runs-on: ubuntu-latest timeout-minutes: 5 permissions: @@ -2189,6 +2205,9 @@ jobs: uses: ./.github/actions/e2e-images with: arch: ${{ matrix.arch }} + - name: verify release consumer provenance and cluster ownership guards + if: matrix.arch == 'amd64' + run: bash test/e2e/release-consumer-checks.sh - name: bring up cluster on the PREVIOUS release run: ./deploy/kind/upgrade.sh up "$CLUSTER" e2e - name: install Argo CD for the populated GitOps upgrade @@ -2683,7 +2702,7 @@ jobs: e2e-postgres, e2e-kind-ingress-smoke, ] - if: always() + if: always() && (github.event_name != 'workflow_dispatch' || inputs.published_release == '') runs-on: ubuntu-latest timeout-minutes: 5 permissions: diff --git a/.github/workflows/qualify-release.yaml b/.github/workflows/qualify-release.yaml new file mode 100644 index 000000000..297f1ba07 --- /dev/null +++ b/.github/workflows/qualify-release.yaml @@ -0,0 +1,37 @@ +name: published release qualification + +on: + workflow_call: + inputs: + release_tag: + description: Published Gameplane version tag + required: true + type: string + +permissions: + contents: read + +jobs: + consumer: + name: verify published chart and images + runs-on: ubuntu-latest + timeout-minutes: 50 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/go-cache + with: + key-suffix: e2e + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + - uses: helm/kind-action@ea822c7bf1874f08e5a491df991c228edffc3376 # v1.15.2 + with: + install_only: true + - uses: oras-project/setup-oras@5e2f2482b30b0cb3e14e67f89a80c88c659a7c63 # v2.0.3 + - uses: sigstore/cosign-installer@756fe2d284c7292c4e9b62c8a28f1e5629c59532 # v4.1.3 + with: + cosign-release: 'v3.1.2' + - name: verify, install and upgrade using published chart and images + env: + RELEASE_TAG: ${{ inputs.release_tag }} + run: bash deploy/kind/qualify-published-release.sh "$RELEASE_TAG" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7cb866a6a..4c1f080bc 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -271,6 +271,15 @@ jobs: cosign public-key --key env://COSIGN_PRIVATE_KEY --outfile "$RUNNER_TEMP/cosign.pub" cosign verify --key "$RUNNER_TEMP/cosign.pub" --new-bundle-format=false "$REF" + consumer: + name: qualify published artifacts as an anonymous consumer + needs: chart + permissions: + contents: read + uses: ./.github/workflows/qualify-release.yaml + with: + release_tag: ${{ github.ref_name }} + # Publish the GitHub Release page for the tag, with the matching CHANGELOG # section as the notes. beta.3's page was created by hand and beta.4's was # forgotten entirely — this makes the page part of the release, not a @@ -281,7 +290,7 @@ jobs: timeout-minutes: 15 # applied by extension per D-A, pending maintainer confirmation permissions: contents: write - needs: [images, chart] + needs: consumer steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: extract changelog section diff --git a/changelog/unreleased/906-published-release-check.md b/changelog/unreleased/906-published-release-check.md new file mode 100644 index 000000000..2478009bc --- /dev/null +++ b/changelog/unreleased/906-published-release-check.md @@ -0,0 +1,5 @@ +# Published release verification + +Release qualification now verifies the signed Helm chart and component image digests as an +anonymous consumer. It exercises a clean install and a populated upgrade from the previous +supported release. [PR #906](https://github.com/GameplanePanel/Gameplane/pull/906) diff --git a/deploy/kind/qualify-published-release.sh b/deploy/kind/qualify-published-release.sh new file mode 100644 index 000000000..242783588 --- /dev/null +++ b/deploy/kind/qualify-published-release.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Release CI runs this on a fresh amd64 runner without a registry login. +set -euo pipefail +source deploy/kind/release-consumer-lib.sh +tag=${1:?pass the published release tag} +[[ $tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z]+([.-][0-9A-Za-z]+)*)?$ ]] || { + echo "expected a published vMAJOR.MINOR.PATCH tag, optionally with a prerelease suffix" >&2 + exit 2 +} +version=${tag#v} +owner="${GITHUB_REPOSITORY_OWNER,,}" +registry="ghcr.io/$owner" +scratch="$(mktemp -d)" +export DOCKER_CONFIG="$scratch/docker" HELM_REGISTRY_CONFIG="$scratch/helm.json" +export KUBECONFIG="$scratch/kubeconfig" KIND_EXPERIMENTAL_PROVIDER=docker +owned_node='' +cleanup() { + local status=$? + delete_consumer_cluster || status=1 + rm -rf -- "$scratch" + exit "$status" +} +trap cleanup EXIT +export GAMEPLANE_PUBLISHED_CHART="$scratch/gameplane" GAMEPLANE_RELEASE_DIGESTS="$scratch/images.tsv" +export GAMEPLANE_PREVIOUS_RELEASE_DIGESTS="$scratch/previous-images.tsv" +mkdir -p "$DOCKER_CONFIG" +cosign initialize + +chart="$registry/charts/gameplane" +digest="$(oras resolve "$chart:$version")" +verify_release_signature "$chart" "$digest" +helm pull "oci://$chart@$digest" --untar --untardir "$scratch" +test "$(yq -r '.version' "$GAMEPLANE_PUBLISHED_CHART/Chart.yaml")" = "$version" +test "$(yq -r '.appVersion' "$GAMEPLANE_PUBLISHED_CHART/Chart.yaml")" = "$version" +printf 'Verified chart `%s@%s`\n' "$chart" "$digest" >> "$GITHUB_STEP_SUMMARY" + +# Read the publisher's complete matrix, including optional images. Keep the +# signed index and its amd64 manifest/config IDs for containerd comparisons. +git fetch --quiet --no-tags --depth=1 origin "refs/tags/$tag" +revision=$(git rev-parse --verify 'FETCH_HEAD^{commit}') +git show "$revision:.github/workflows/release.yaml" > "$scratch/published-release.yaml" +printf 'Release `%s` at Git commit `%s`\n' "$tag" "$revision" >> "$GITHUB_STEP_SUMMARY" +components="$(yq -r '.jobs.images.strategy.matrix.include[].component' "$scratch/published-release.yaml")" +test -n "$components" +while read -r component; do + verify_release_image "$registry/gameplane/$component" "$version" "$revision" "$GAMEPLANE_RELEASE_DIGESTS" +done <<< "$components" + +run_id=$(cat /proc/sys/kernel/random/uuid) +fresh="gp-fresh-$run_id" +create_consumer_cluster "$fresh" +helm install gameplane "$GAMEPLANE_PUBLISHED_CHART" --kube-context "kind-$fresh" \ + --namespace gameplane-system --create-namespace --set ingress.enabled=false \ + --set defaultModuleSource.enabled=false --wait --timeout 6m +bash deploy/kind/verify-release-runtime.sh "kind-$fresh" "$GAMEPLANE_RELEASE_DIGESTS" api operator web +delete_consumer_cluster + +# The regular upgrade fixture seeds the old API database and game PVC, +# exercises the CRD hooks, and checks their contents after the upgrade. +export GAMEPLANE_E2E_CLUSTER="gp-upgrade-$run_id" GAMEPLANE_E2E_REUSE_CLUSTER=1 +export GAMEPLANE_E2E_CONTEXT="kind-$GAMEPLANE_E2E_CLUSTER" +create_consumer_cluster "$GAMEPLANE_E2E_CLUSTER" +deploy/kind/upgrade.sh install "$GAMEPLANE_E2E_CLUSTER" +(cd test/e2e && go test -tags=e2e -timeout 25m -v -run '^TestUpgrade_FromPreviousRelease$' .) diff --git a/deploy/kind/release-consumer-lib.sh b/deploy/kind/release-consumer-lib.sh new file mode 100644 index 000000000..dcb02cdfc --- /dev/null +++ b/deploy/kind/release-consumer-lib.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Classic keyed cosign verification checks the digest. Also check the +# repository named in the signed payload before trusting a component. +verify_release_signature() { + local repository=$1 digest=$2 + if ! cosign verify --key signing/cosign.pub --new-bundle-format=false "$repository@$digest" | + jq -e --arg repo "$repository" --arg digest "$digest" 'any(.[]; + .critical.identity["docker-reference"] == $repo and + .critical.image["docker-manifest-digest"] == $digest)' >/dev/null; then + echo "Missing valid release signature for $repository@$digest" >&2 + return 1 + fi +} + +# The config blob is addressed through the verified release index. +verify_release_config() { + jq -e --arg arch "$1" --arg revision "$2" --arg version "$3" ' + .os == "linux" and .architecture == $arch and + .config.Labels["org.opencontainers.image.revision"] == $revision and + .config.Labels["org.opencontainers.image.version"] == $version' >/dev/null +} + +verify_release_image() { + local image=$1 version=$2 revision=$3 digest_file=$4 + local index ref index_json arch manifest config + index=$(oras resolve "$image:$version") + ref="$image@$index" + verify_release_signature "$image" "$index" + docker pull --platform linux/amd64 "$ref" + printf '\n- `%s`\n' "$ref" >> "$GITHUB_STEP_SUMMARY" + index_json=$(oras manifest fetch "$ref") + for arch in amd64 arm64; do + manifest=$(jq -er --arg arch "$arch" ' + [.manifests[] | select(.platform.os == "linux" and .platform.architecture == $arch) | .digest] + | if length == 1 then .[0] else error("expected one linux/" + $arch + " manifest") end' <<< "$index_json") + config=$(oras manifest fetch "$image@$manifest" | jq -er '.config.digest') + if ! oras blob fetch --output - "$image@$config" | verify_release_config "$arch" "$revision" "$version"; then + echo "$image@$config must identify linux/$arch, version $version and Git commit $revision" >&2 + return 1 + fi + printf ' - linux/%s manifest `%s`, config `%s`\n' "$arch" "$manifest" "$config" >> "$GITHUB_STEP_SUMMARY" + if [[ $arch == amd64 ]]; then + printf '%s\t%s\t%s\t%s\t%s\n' "${image##*/}" "$image:$version" "$index" "$manifest" "$config" >> "$digest_file" + fi + done +} + +# Each consumer cluster has one Docker control-plane node. Capture its +# immutable container ID only after its CA matches the private kubeconfig +# written by this creation. A concurrent name replacement stays untouched. +create_consumer_cluster() { + local name=$1 existing id node_ca kube_ca + [[ -z $owned_node ]] || return 1 + existing=$(kind get clusters) || return 1 + if grep -Fxq -- "$name" <<< "$existing"; then + echo "Kind cluster $name already exists" >&2 + return 1 + fi + kind create cluster --name "$name" --kubeconfig "$KUBECONFIG" --wait 90s || return 1 + id=$(docker inspect --type container --format '{{.Id}}' "$name-control-plane") || return 1 + [[ $id =~ ^[a-f0-9]{64}$ ]] || return 1 + node_ca=$(docker exec "$id" cat /etc/kubernetes/pki/ca.crt | base64 -w0) || return 1 + kube_ca=$(kubectl config view --kubeconfig "$KUBECONFIG" --context "kind-$name" \ + --minify --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}') || return 1 + if [[ -z $node_ca || $node_ca != "$kube_ca" ]]; then + echo "Control-plane container for $name does not match its kubeconfig" >&2 + return 1 + fi + owned_node=$id +} + +delete_consumer_cluster() { + if [[ -n $owned_node ]]; then + docker rm --force --volumes "$owned_node" >/dev/null || return 1 + owned_node='' + fi +} diff --git a/deploy/kind/upgrade.sh b/deploy/kind/upgrade.sh index 0a7315716..680f8ec85 100755 --- a/deploy/kind/upgrade.sh +++ b/deploy/kind/upgrade.sh @@ -12,6 +12,7 @@ # # Usage: # deploy/kind/upgrade.sh up [cluster-name] [tag] +# deploy/kind/upgrade.sh install [tag] # deploy/kind/upgrade.sh down [cluster-name] set -euo pipefail @@ -35,44 +36,65 @@ REPO="$(cd "${HERE}/../.." && pwd)" FROM_VERSION="${GAMEPLANE_UPGRADE_FROM:-0.3.0}" # Releases up to v0.3.0 were published before the repo moved to the # GameplanePanel org, so they live under the original owner on GHCR. -CHART_REF="oci://ghcr.io/valgulnecron/charts/gameplane" +FROM_REGISTRY="ghcr.io/valgulnecron" +CHART_REF="oci://$FROM_REGISTRY/charts/gameplane" need() { command -v "$1" >/dev/null 2>&1 || { echo "missing: $1" >&2; exit 1; }; } case "${ACTION}" in -up) +up|install) need kind need kubectl need helm need docker - if kind get clusters | grep -qx "${CLUSTER}"; then - echo "removing pre-existing cluster ${CLUSTER} for a clean slate" - kind delete cluster --name "${CLUSTER}" || true + if [[ "$ACTION" == up ]]; then + # Kind's default is a single control-plane node. Creation rejects + # collisions and leaves the existing cluster and its volumes alone. + kind create cluster --name "${CLUSTER}" --wait 90s + elif [[ -z ${2:-} ]]; then + echo "install requires the name of the existing Kind cluster" >&2 + exit 2 fi - echo "creating kind cluster ${CLUSTER} (single-node)" - kind create cluster --name "${CLUSTER}" --wait 90s --config - </dev/null # Load the NEW (working-tree) images now, so the upgrade the test performs # later needs no network. The OLD release's images are pulled from GHCR by # the install below — that is the point of the test. - echo "loading working-tree gameplane-test/{operator,api,agent,sentinel}:${TAG} into kind" - for img in operator api agent sentinel; do - if ! docker image inspect "gameplane-test/${img}:${TAG}" >/dev/null 2>&1; then - echo " missing local image gameplane-test/${img}:${TAG} — building" - docker build -t "gameplane-test/${img}:${TAG}" -f "${REPO}/${img}/Dockerfile" "${REPO}" - fi - kind load docker-image "gameplane-test/${img}:${TAG}" --name "${CLUSTER}" - done + if [[ -z "${GAMEPLANE_PUBLISHED_CHART:-}" ]]; then + echo "loading working-tree gameplane-test/{operator,api,agent,sentinel}:${TAG} into kind" + for img in operator api agent sentinel; do + if ! docker image inspect "gameplane-test/${img}:${TAG}" >/dev/null 2>&1; then + echo " missing local image gameplane-test/${img}:${TAG} — building" + docker build -t "gameplane-test/${img}:${TAG}" -f "${REPO}/${img}/Dockerfile" "${REPO}" + fi + kind load docker-image "gameplane-test/${img}:${TAG}" --name "${CLUSTER}" + done + fi - echo "installing PUBLISHED chart ${CHART_REF} --version ${FROM_VERSION}" + chart_args=(--version "$FROM_VERSION") + if [[ -n ${GAMEPLANE_PUBLISHED_CHART:-} ]]; then + # Resolve the documented predecessor in its original registry. Record + # what should be running before the seeded game and database migrate. + source "$HERE/release-consumer-lib.sh" + git -C "$REPO" fetch --quiet --no-tags --depth=1 origin "refs/tags/v$FROM_VERSION" + previous_commit=$(git -C "$REPO" rev-parse --verify 'FETCH_HEAD^{commit}') + repository=${CHART_REF#oci://} + digest=$(oras resolve "$repository:$FROM_VERSION") + verify_release_signature "$repository" "$digest" + CHART_REF="$CHART_REF@$digest" + chart_args=() + metadata=$(helm show chart "$CHART_REF") + [[ $(yq -r '.version' <<< "$metadata") == "$FROM_VERSION" && + $(yq -r '.appVersion' <<< "$metadata") == "$FROM_VERSION" ]] + printf '\nPrevious chart `%s@%s`, version `%s`, Git commit `%s`\n' \ + "$repository" "$digest" "$FROM_VERSION" "$previous_commit" >> "$GITHUB_STEP_SUMMARY" + for component in api operator agent; do + verify_release_image "$FROM_REGISTRY/gameplane/$component" "$FROM_VERSION" "$previous_commit" "$GAMEPLANE_PREVIOUS_RELEASE_DIGESTS" + done + fi + echo "installing PUBLISHED chart ${CHART_REF}, version ${FROM_VERSION}" # Value overrides mirror e2e.sh's, and are all about the harness rather # than the version under test: # - api memory 1Gi: argon2id uses ~64Mi per hash and the 256Mi default @@ -85,7 +107,8 @@ EOF # Note what is NOT overridden: image.registry/image.tag. The old release # must pull its own published GHCR images, or this tests nothing. helm install gameplane "${CHART_REF}" \ - --version "${FROM_VERSION}" \ + --kube-context "kind-${CLUSTER}" \ + "${chart_args[@]}" \ --namespace gameplane-system --create-namespace \ --set "ingress.enabled=false" \ --set "web.enabled=false" \ @@ -108,7 +131,7 @@ down) ;; *) - echo "usage: $0 {up|down} [cluster-name] [tag]" >&2 + echo "usage: $0 {up|install|down} [cluster-name] [tag]" >&2 exit 2 ;; esac diff --git a/deploy/kind/verify-release-runtime.sh b/deploy/kind/verify-release-runtime.sh new file mode 100644 index 000000000..c5484377b --- /dev/null +++ b/deploy/kind/verify-release-runtime.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Accept only the signed index, native-platform manifest or image config +# digest derived from that index. Runtimes differ in which ID they expose. +set -euo pipefail +context="$1" +digests="$2" +shift 2 +for component in "$@"; do + row="$(awk -F '\t' -v name="$component" '$1 == name' "$digests")" + test -n "$row" + IFS=$'\t' read -r _ image index manifest config <<< "$row" + namespace=gameplane-system + label="gameplane-$component" + container="$component" + case "$component" in + operator) container=manager ;; + agent) namespace=gameplane-games; label=gameplane-game ;; + api|web) ;; + *) echo "Unsupported runtime component: $component" >&2; exit 1 ;; + esac + ids="" + for ((attempt=0; attempt<60; attempt++)); do + pods="$(kubectl --context "$context" -n "$namespace" get pods -l "app.kubernetes.io/name=$label" -o json)" + # A matching new replica must not mask another replica still running an old image. + if ids="$(jq -er --arg image "$image" --arg name "$container" ' + [.items[] | select(.metadata.deletionTimestamp == null) | { + spec: [.spec.containers[] | select(.name == $name) | .image], + status: [.status.containerStatuses[]? | select(.name == $name)] + }] | select(length > 0 and all(.[]; + .spec == [$image] and (.status | length) == 1 and + .status[0].ready and (.status[0].imageID | type == "string" and length > 0))) + | .[].status[0].imageID' <<< "$pods")"; then + break + fi + sleep 2 + done + if [[ -z "$ids" ]]; then + echo "Expected every $label pod to run a Ready $container from $image in $context" >&2 + exit 1 + fi + while read -r id; do + digest="${id##*@}" + digest="${digest##*://}" + if [[ "$digest" != "$index" && "$digest" != "$manifest" && "$digest" != "$config" ]]; then + echo "Unexpected runtime image ID for $image: $id" >&2 + exit 1 + fi + printf '%s %s %s\n' "$context" "$image" "$id" + printf '\n- `%s` ran `%s` as `%s`\n' "$context" "$image" "$id" >> "$GITHUB_STEP_SUMMARY" + done <<< "$ids" +done diff --git a/docs/install.md b/docs/install.md index ef9ee7f1e..1d311896f 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1035,6 +1035,26 @@ test and asserts that - the pre-upgrade admin can still log in — i.e. the new binary's database migrations ran against the populated SQLite volume without data loss. +The release workflow also pulls the signed chart and release images from GHCR +on a fresh amd64 runner using anonymous access and the public signing key. +It checks the repository and digest in the signed payload. For each image +index, it verifies that the amd64 and arm64 configs name +the release version and the commit referenced by the Git tag. It installs the +chart in a clean Kind cluster, then runs the populated upgrade fixture in a +second cluster. It verifies the previous chart in its original registry and +checks the running API, operator, and seeded game agent against the previous +release before upgrading. Both clusters use a private kubeconfig. The job +summary records the Git commit, chart digest, image index and platform digests, +and running image IDs. + +To qualify an already published candidate from a review branch, run the CI +workflow on that branch with its `published_release` input set to the actual +version tag. With `REVIEW_REF` set to the branch under review +and `TAG` set to the published tag, run +`gh workflow run ci.yaml --ref "$REVIEW_REF" -f published_release="$TAG"`. This runs only the read-only +consumer, which uses the tagged publisher's component list and reports chart and image digests +in its Actions summary. Omit `published_release` to run the ordinary full CI suite. + See `test/e2e/upgrade_e2e_test.go`. What this does **not** yet cover: upgrades that skip several releases at once. The `postgres-availability` bucket tests a PostgreSQL combined-install deployment-strategy change and outage recovery diff --git a/test/e2e/release-consumer-checks.sh b/test/e2e/release-consumer-checks.sh new file mode 100644 index 000000000..abba835a5 --- /dev/null +++ b/test/e2e/release-consumer-checks.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# CI checks for the release consumer's artifact and cluster ownership gates. +set -euo pipefail +source deploy/kind/release-consumer-lib.sh + +# A different component in the same release can have a valid project-key +# signature and identical revision/version labels. Its signed name differs. +cosign() { + echo '[{"critical":{"identity":{"docker-reference":"ghcr.io/example/gameplane/web"},"image":{"docker-manifest-digest":"sha256:abc"}}}]' +} +verify_release_signature ghcr.io/example/gameplane/web sha256:abc +if verify_release_signature ghcr.io/example/gameplane/api sha256:abc || + verify_release_signature ghcr.io/example/gameplane/web sha256:def; then + echo "Accepted a signature for another repository or digest" >&2 + exit 1 +fi +unset -f cosign + +revision=040e10ee16253f6ead593bd076ed90835b68e2c1 +config='{"os":"linux","architecture":"amd64","config":{"Labels":{"org.opencontainers.image.revision":"040e10ee16253f6ead593bd076ed90835b68e2c1","org.opencontainers.image.version":"0.3.0"}}}' +verify_release_config amd64 "$revision" 0.3.0 <<< "$config" +for args in "amd64 07bee47bf01a7ced0114898605155499065dfb26 0.3.0" "arm64 $revision 0.3.0" "amd64 $revision 0.4.1"; do + read -r arch commit version <<< "$args" + if verify_release_config "$arch" "$commit" "$version" <<< "$config"; then + echo "Accepted mismatched image config for $args" >&2 + exit 1 + fi +done + +scratch=$(mktemp -d) +trap 'rm -rf -- "$scratch"' EXIT +export KUBECONFIG="$scratch/kubeconfig" +node_id=$(printf 'a%.0s' {1..64}) +owned_node='' +clusters='occupied' +kind() { + case "$1 $2" in + 'get clusters') echo "$clusters" ;; + 'create cluster') echo create >> "$scratch/actions" ;; + *) return 1 ;; + esac +} +docker() { + case "$1" in + inspect) echo "$node_id" ;; + exec) printf 'fixture CA' ;; + rm) printf '%s\n' "$*" >> "$scratch/actions" ;; + *) return 1 ;; + esac +} +kubectl() { printf 'fixture CA' | base64 -w0; } +if create_consumer_cluster occupied; then exit 1; fi +delete_consumer_cluster +test ! -e "$scratch/actions" + +kubectl() { printf 'different CA' | base64 -w0; } +if create_consumer_cluster replaced; then exit 1; fi +delete_consumer_cluster +test "$(cat "$scratch/actions")" = create + +kubectl() { printf 'fixture CA' | base64 -w0; } +create_consumer_cluster mine +original=$node_id +node_id=$(printf 'b%.0s' {1..64}) +delete_consumer_cluster +test "$(tail -1 "$scratch/actions")" = "rm --force --volumes $original" + +# Exercise Kind's real CA data and Docker node identity on the same runner +# used for release consumption before the populated upgrade cluster starts. +unset -f kind docker kubectl +export KIND_EXPERIMENTAL_PROVIDER=docker +trap 'delete_consumer_cluster; rm -rf -- "$scratch"' EXIT +name="gp-check-$(cat /proc/sys/kernel/random/uuid)" +create_consumer_cluster "$name" +original=$owned_node +kubectl --kubeconfig "$KUBECONFIG" --context "kind-$name" get node "$name-control-plane" +delete_consumer_cluster +if docker inspect --type container "$original" >/dev/null 2>&1; then + echo "Owned control plane still exists after cleanup" >&2 + exit 1 +fi diff --git a/test/e2e/upgrade_e2e_test.go b/test/e2e/upgrade_e2e_test.go index 09fbf6b9a..7368ebc74 100644 --- a/test/e2e/upgrade_e2e_test.go +++ b/test/e2e/upgrade_e2e_test.go @@ -85,6 +85,18 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { t.Fatalf("write marker: %v\n%s", err, out) } + claim, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(ns).Get(ctx, gs+"-data", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if chart := os.Getenv("GAMEPLANE_PUBLISHED_CHART"); chart != "" { + checkPublishedRuntime(t, os.Getenv("GAMEPLANE_PREVIOUS_RELEASE_DIGESTS"), "api", "operator", "agent") + upgradePublishedChart(t, chart) + verifyPopulatedUpgrade(t, ns, gs, marker, adminUser, adminPass, claim.UID) + checkPublishedRuntime(t, os.Getenv("GAMEPLANE_RELEASE_DIGESTS"), "api", "operator", "web", "agent") + return + } + // ---- 2. work out what the upgrade is supposed to change -------------- // Rather than hardcoding a field name that goes stale, compute the drift: @@ -186,68 +198,14 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { } } - // ---- 5. the workload and its data survived --------------------------- - - got, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns). - Get(ctx, gs, metav1.GetOptions{}) - if err != nil { - t.Fatalf("GameServer %s/%s is gone after the upgrade: %v", ns, gs, err) - } - if suspend, _, _ := unstructured.NestedBool(got.Object, "spec", "suspend"); suspend { - t.Errorf("GameServer was suspended by the upgrade; spec.suspend should be untouched") - } - - waitPVCBound(t, ns, gs+"-data", 2*time.Minute) - waitGameContainerReady(t, ns, gs+"-0", 4*time.Minute) - - // Retry the read rather than exec'ing once. An upgrade re-renders the - // StatefulSet, so the pod keeps its name but is legitimately replaced — - // a Ready observation can be of the outgoing pod, and the exec then - // lands mid-restart ("container not found", "no running task found"). - // That is the workload behaving correctly, not a regression, so the - // assertion tolerates the churn and waits for the volume to be readable. - // What is being proven is that the bytes survived, not that they were - // readable at one exact instant. - envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { - out, err := envInstance.KubectlExec(t, ns, "pod/"+gs+"-0", "cat", "/data/marker.txt") - if err != nil { - return false, "exec: " + err.Error() + " out=" + out - } - // kubectl mixes its "Defaulted container ..." stderr preamble into - // CombinedOutput, so check for the marker as a substring. - if !strings.Contains(out, marker) { - return false, "marker not in output yet: " + out - } - return true, "" - }) - - // ---- 6. the database and its migrations survived --------------------- - - // Deliberately after the upgrade: the API pod was replaced, so this both - // re-establishes a port-forward against the NEW pod and proves the new - // binary's migrations ran against the old release's populated SQLite file - // without dropping the user rows. - client := envInstance.APIClient(t, adminUser, adminPass) - defer client.Close() - - resp, body, err := client.Get("/servers/" + gs + "?namespace=" + ns) - if err != nil { - t.Fatalf("read the pre-upgrade server through the upgraded API: %v", err) - } - if resp.StatusCode != 200 { - t.Fatalf("GET /servers/%s after upgrade: status %d body %s", gs, resp.StatusCode, body) - } - if !strings.Contains(string(body), gs) { - t.Errorf("upgraded API does not return the pre-upgrade GameServer %q; body=%s", gs, body) - } - resp.Body.Close() + client := verifyPopulatedUpgrade(t, ns, gs, marker, adminUser, adminPass, claim.UID) // ---- 6b. telemetry stays off for an install that never opted in ------ // // The previous release had no telemetry choice saved for this install, so // the upgrade must not turn reporting on or show the notice (spec 022 US1 // scenario 5): migration 015 seeds consent_source "legacy". - resp, body, err = client.Get("/admin/telemetry/notice") + resp, body, err := client.Get("/admin/telemetry/notice") if err != nil || resp.StatusCode != 200 { t.Fatalf("GET /admin/telemetry/notice after upgrade: %v status %d body %s", err, resp.StatusCode, body) } diff --git a/test/e2e/upgrade_published_e2e_test.go b/test/e2e/upgrade_published_e2e_test.go new file mode 100644 index 000000000..1e70a89ca --- /dev/null +++ b/test/e2e/upgrade_published_e2e_test.go @@ -0,0 +1,110 @@ +//go:build e2e + +package e2e + +import ( + "context" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +// The previous published chart was installed by deploy/kind/upgrade.sh. Use +// the downloaded candidate and let it choose its own registry and image tags. +func upgradePublishedChart(t *testing.T, chart string) { + t.Helper() + cmd := exec.CommandContext(t.Context(), "helm", "upgrade", "gameplane", chart, + "--kube-context", envInstance.Context, "--namespace", "gameplane-system", + "--set", "web.enabled=true", "--set", "ingress.enabled=false", + "--set", "api.resources.limits.memory=1Gi", "--set", "defaultModuleSource.enabled=false", + // The seeded legacy template uses the original busybox security profile. + "--set", "podSecurity.enforceRestricted=false", "--wait", "--timeout", "6m") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("upgrade to published chart: %v\n%s", err, out) + } +} + +func checkPublishedRuntime(t *testing.T, digestFile string, components ...string) { + t.Helper() + args := []string{filepath.Join("..", "..", "deploy", "kind", "verify-release-runtime.sh"), envInstance.Context, digestFile} + check := exec.CommandContext(t.Context(), "bash", append(args, components...)...) + out, err := check.CombinedOutput() + if err != nil { + t.Fatalf("published upgrade runtime image IDs: %v\n%s", err, out) + } + t.Logf("published upgrade image IDs:\n%s", out) +} + +// Both PR CI and release CI must prove that the seeded world, PVC and account +// survived the upgrade. A Ready replacement pod alone does not establish this. +func verifyPopulatedUpgrade(t *testing.T, ns, gs, marker, adminUser, adminPass string, previousClaim types.UID) *APIClient { + t.Helper() + ctx := context.Background() + // ---- 5. the workload and its data survived --------------------------- + + got, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns). + Get(ctx, gs, metav1.GetOptions{}) + if err != nil { + t.Fatalf("GameServer %s/%s is gone after the upgrade: %v", ns, gs, err) + } + if suspend, _, _ := unstructured.NestedBool(got.Object, "spec", "suspend"); suspend { + t.Errorf("GameServer was suspended by the upgrade; spec.suspend should be untouched") + } + + waitPVCBound(t, ns, gs+"-data", 2*time.Minute) + claim, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(ns).Get(ctx, gs+"-data", metav1.GetOptions{}) + if err != nil || claim.UID != previousClaim { + t.Fatalf("upgrade replaced the game data claim or made it unavailable: %v", err) + } + waitGameContainerReady(t, ns, gs+"-0", 4*time.Minute) + + // Retry the read rather than exec'ing once. An upgrade re-renders the + // StatefulSet, so the pod keeps its name but is legitimately replaced — + // a Ready observation can be of the outgoing pod, and the exec then + // lands mid-restart ("container not found", "no running task found"). + // That is the workload behaving correctly, not a regression, so the + // assertion tolerates the churn and waits for the volume to be readable. + // What is being proven is that the bytes survived, not that they were + // readable at one exact instant. + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + out, err := envInstance.KubectlExec(t, ns, "pod/"+gs+"-0", "cat", "/data/marker.txt") + if err != nil { + return false, "exec: " + err.Error() + " out=" + out + } + // kubectl mixes its "Defaulted container ..." stderr preamble into + // CombinedOutput, so check for the marker as a substring. + if !strings.Contains(out, marker) { + return false, "marker not in output yet: " + out + } + return true, "" + }) + + // ---- 6. the database and its migrations survived --------------------- + + // Deliberately after the upgrade: the API pod was replaced, so this both + // re-establishes a port-forward against the NEW pod and proves the new + // binary's migrations ran against the old release's populated SQLite file + // without dropping the user rows. + client := envInstance.APIClient(t, adminUser, adminPass) + t.Cleanup(client.Close) + + resp, body, err := client.Get("/servers/" + gs + "?namespace=" + ns) + if err != nil { + t.Fatalf("read the pre-upgrade server through the upgraded API: %v", err) + } + if resp.StatusCode != 200 { + t.Fatalf("GET /servers/%s after upgrade: status %d body %s", gs, resp.StatusCode, body) + } + if !strings.Contains(string(body), gs) { + t.Errorf("upgraded API does not return the pre-upgrade GameServer %q; body=%s", gs, body) + } + resp.Body.Close() + + return client +}