From a939e6835e0b54506b60b5315f497a455654b053 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Fri, 30 Oct 2026 08:41:45 +0000 Subject: [PATCH] e2e: Install the released chart in the upgrade test Since the move to Helm, TestOperatorUpgrade installed the released operator image with the chart from the current checkout. That only works while the current chart stays compatible with the released binary: a chart that passes a new controller flag or narrows the RBAC makes the released controller crash or fail to sync, and the test never gets to the upgrade. Install the released version from its published chart instead, the way users upgrading from it have it deployed, then upgrade to the local chart as before. The chart comes from RELEASED_OPERATOR_CHART, so downstream builds can point elsewhere, and its version is RELEASED_OPERATOR_TAG without the leading "v", matching how the release workflow publishes it. The released image is now the one the chart deploys, so RELEASED_OPERATOR_IMG is gone. On bink, push-released-operator-image reads that image from the chart and still copies it to the local registry, which keeps the test fast and avoids pulls from a remote registry timing out; the test overrides the chart image with that copy. The restore cleanup is now registered right after the uninstall: when the released operator failed to come up, the cleanup was never registered and every later test ran without an operator. Assisted-by: AI Signed-off-by: Alice Frosi --- Makefile | 30 +++++++++++-------- test/e2e/upgrade_test.go | 65 ++++++++++++++++++++++++++-------------- 2 files changed, 60 insertions(+), 35 deletions(-) diff --git a/Makefile b/Makefile index 52605716..648af029 100644 --- a/Makefile +++ b/Makefile @@ -17,10 +17,12 @@ BINK_NODE_DISK_IMAGE ?= ghcr.io/bootc-dev/bink/node:v$(DEFAULT_KUBE_MINOR)-fedor BINK_LOCAL_REGISTRY_NODE_IMAGE ?= registry.cluster.local:5000/node E2E_REGISTRY_USER ?= e2e-user E2E_REGISTRY_PASSWORD ?= e2e-password -# Released operator for upgrade testing (TestOperatorUpgrade). -# Override RELEASED_OPERATOR_IMG for downstream images. +# Released operator for upgrade testing (TestOperatorUpgrade): the published +# chart and the release tag, whose version without the leading "v" is the +# chart version. The released image is the one the chart deploys. Override +# RELEASED_OPERATOR_CHART for downstream charts. RELEASED_OPERATOR_TAG ?= v0.2.2 -RELEASED_OPERATOR_IMG ?= ghcr.io/bootc-dev/bootc-operator:$(RELEASED_OPERATOR_TAG) +RELEASED_OPERATOR_CHART ?= oci://ghcr.io/bootc-dev/bootc-operator/charts/bootc-operator IMG_BINK_RELEASED ?= registry.cluster.local:5000/bootc-operator-released:latest # YEAR defines the year value used for substituting the YEAR placeholder in the boilerplate header. YEAR ?= $(shell date +%Y) @@ -120,7 +122,8 @@ e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN= E2E_NODE_IMAGE_UPDATE2_DIGEST=$$(skopeo inspect --tls-verify=false docker://localhost:5000/node:update2 | jq -r '.Digest') \ E2E_REGISTRY_USER=$(E2E_REGISTRY_USER) E2E_REGISTRY_PASSWORD=$(E2E_REGISTRY_PASSWORD) \ $(if $(RELEASED_OPERATOR_TAG),E2E_OPERATOR_RELEASE_TAG=$(RELEASED_OPERATOR_TAG)) \ - $(if $(RELEASED_OPERATOR_IMG),E2E_OPERATOR_RELEASED_IMG=$(IMG_BINK_RELEASED)) \ + $(if $(RELEASED_OPERATOR_TAG),E2E_OPERATOR_RELEASED_IMG=$(IMG_BINK_RELEASED)) \ + $(if $(RELEASED_OPERATOR_CHART),E2E_OPERATOR_RELEASED_CHART=$(RELEASED_OPERATOR_CHART)) \ go test -timeout 90m -count=1 -parallel 2 $(if $(V),-v) $(if $(RUN),-run '$(RUN)') . # EKS e2e settings @@ -152,7 +155,7 @@ e2e-eks: ## Run e2e tests against EKS. V=1 for verbose. RUN= to filter. E2E_REGISTRY_USER="$(EKS_REGISTRY_USER)" \ E2E_REGISTRY_PASSWORD="$(EKS_REGISTRY_PASSWORD)" \ $(if $(RELEASED_OPERATOR_TAG),E2E_OPERATOR_RELEASE_TAG=$(RELEASED_OPERATOR_TAG)) \ - $(if $(RELEASED_OPERATOR_IMG),E2E_OPERATOR_RELEASED_IMG=$(RELEASED_OPERATOR_IMG)) \ + $(if $(RELEASED_OPERATOR_CHART),E2E_OPERATOR_RELEASED_CHART=$(RELEASED_OPERATOR_CHART)) \ $(if $(EKS_IMAGE_PULL_SECRET),E2E_IMAGE_PULL_SECRET=$(EKS_IMAGE_PULL_SECRET)) \ E2E_NODE_PULL_SECRET="$(EKS_NODE_PULL_SECRET)" \ ARTIFACTS="$(ARTIFACTS)" \ @@ -192,13 +195,14 @@ build-update-image: ## Build derived node images for update testing and push to podman push --tls-verify=false localhost:5000/node:update2 .PHONY: push-released-operator-image -push-released-operator-image: ## Pull released operator image and push to bink registry for upgrade testing. - @if [ -z "$(RELEASED_OPERATOR_IMG)" ]; then \ - echo "Error: RELEASED_OPERATOR_IMG must be set (e.g. ghcr.io/bootc-dev/bootc-operator:v0.1.0)"; \ - exit 1; \ - fi - podman pull $(RELEASED_OPERATOR_IMG) - podman push --tls-verify=false $(RELEASED_OPERATOR_IMG) localhost:5000/bootc-operator-released:latest +push-released-operator-image: yq ## Copy the image of the released chart to the bink registry for upgrade testing. + img=$$(helm template bootc-operator $(RELEASED_OPERATOR_CHART) \ + --version $(RELEASED_OPERATOR_TAG:v%=%) \ + --kube-version $(DEFAULT_KUBE_MINOR).0 | \ + $(YQ) 'select(.kind == "Deployment") | .spec.template.spec.containers[] | select(.name == "manager") | .image') && \ + test -n "$$img" && \ + podman pull "$$img" && \ + podman push --tls-verify=false "$$img" localhost:5000/bootc-operator-released:latest ##@ Deployment @@ -256,7 +260,7 @@ start-bink: seed-node-image ## Start a bink cluster (idempotent). kubectl --kubeconfig $(KUBECONFIG_BINK) wait --for=condition=Ready node/controller --timeout=5m .PHONY: deploy-bink -deploy-bink: start-bink build-update-image $(if $(RELEASED_OPERATOR_IMG),push-released-operator-image) helm ## Deploy to a bink cluster (requires: buildimg). +deploy-bink: start-bink build-update-image $(if $(RELEASED_OPERATOR_TAG),push-released-operator-image) helm ## Deploy to a bink cluster (requires: buildimg). podman push --tls-verify=false $(IMG) localhost:5000/bootc-operator-e2e:latest helm dependency build ./chart/bootc-operator img="$(IMG_BINK)" && \ diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go index da7ff675..957539b1 100644 --- a/test/e2e/upgrade_test.go +++ b/test/e2e/upgrade_test.go @@ -32,9 +32,9 @@ var operatorCRDNames = []string{ } // TestOperatorUpgrade installs the released version of the operator -// via Helm, verifies basic functionality, then upgrades by running -// helm upgrade with the current image and verifies the operator still -// works. +// from its published Helm chart, verifies basic functionality, then +// upgrades by running helm upgrade with the current chart and image and +// verifies the operator still works. func TestOperatorUpgrade(t *testing.T) { e2eutil.Providers(t, "bink", "eks") @@ -42,10 +42,14 @@ func TestOperatorUpgrade(t *testing.T) { if releaseTag == "" { t.Skip("E2E_OPERATOR_RELEASE_TAG not set") } + // Optional: overrides the image the released chart deploys, e.g. + // with a copy in a local registry. releasedImg := os.Getenv("E2E_OPERATOR_RELEASED_IMG") - if releasedImg == "" { - t.Skip("E2E_OPERATOR_RELEASED_IMG not set") + releasedChart := os.Getenv("E2E_OPERATOR_RELEASED_CHART") + if releasedChart == "" { + t.Skip("E2E_OPERATOR_RELEASED_CHART not set") } + releasedChartVersion := strings.TrimPrefix(releaseTag, "v") g := NewWithT(t) g.SetDefaultEventuallyTimeout(pollTimeout) @@ -64,16 +68,17 @@ func TestOperatorUpgrade(t *testing.T) { currentImg := containerImage(t, origDeploy.Spec.Template.Spec.Containers, "manager") extraArgs := controllerExtraArgs(t, origDeploy.Spec.Template.Spec.Containers) t.Logf("Current operator image: %s", currentImg) - t.Logf("Released operator tag: %s, image: %s", releaseTag, releasedImg) + t.Logf("Released operator tag: %s, image: %s, chart: %s", + releaseTag, releasedImg, releasedChart) - // Phase 1: Uninstall the current operator and re-install with - // the released image via Helm. + // Phase 1: Uninstall the current operator and re-install the + // released version from its published chart, the way users + // upgrading from it have it deployed. The current chart may not work + // with the released image. uninstallOperator(t, g, ctx, env) - helmInstallOperator(t, chartPath, releasedImg, extraArgs) - waitForOperatorReady(t, g, ctx, env.Client) - - t.Logf("Released operator %s installed via Helm", releaseTag) + // Restore the current operator even if the released one fails to + // install, so the tests that run after this one have an operator. t.Cleanup(func() { t.Logf("Restoring operator to current version...") helmUpgradeOperator(t, chartPath, currentImg, extraArgs) @@ -81,6 +86,12 @@ func TestOperatorUpgrade(t *testing.T) { t.Logf("Operator restored") }) + helmInstallOperator(t, releasedChart, releasedChartVersion, + releasedImg, extraArgs) + waitForOperatorReady(t, g, ctx, env.Client) + + t.Logf("Released operator %s installed via Helm", releaseTag) + // Phase 2: Verify the released version works. nodeName := env.AddNode(t) @@ -181,12 +192,17 @@ func uninstallOperator( t.Logf("Operator uninstalled") } -// helmInstallOperator installs the operator via helm install. -func helmInstallOperator(t *testing.T, chartPath, img string, extraArgs []string) { +// helmInstallOperator installs the given version of a published chart +// via helm install. +func helmInstallOperator( + t *testing.T, + chartRef, chartVersion, img string, + extraArgs []string, +) { t.Helper() - helmRun(t, "dependency", "build", chartPath) args := []string{ - "install", helmReleaseName, chartPath, + "install", helmReleaseName, chartRef, + "--version", chartVersion, "--namespace", testutil.OperatorNamespaceName, "--create-namespace", } @@ -194,9 +210,11 @@ func helmInstallOperator(t *testing.T, chartPath, img string, extraArgs []string helmRun(t, args...) } -// helmUpgradeOperator upgrades the operator via helm upgrade. +// helmUpgradeOperator upgrades the operator to the local chart via helm +// upgrade. func helmUpgradeOperator(t *testing.T, chartPath, img string, extraArgs []string) { t.Helper() + helmRun(t, "dependency", "build", chartPath) args := []string{ "upgrade", "--install", helmReleaseName, chartPath, "--namespace", testutil.OperatorNamespaceName, @@ -206,13 +224,16 @@ func helmUpgradeOperator(t *testing.T, chartPath, img string, extraArgs []string } // helmImageArgs returns the --set flags for image and controller args. +// An empty img keeps the image of the chart. func helmImageArgs(t *testing.T, img string, extraArgs []string) []string { t.Helper() - repo, tag := splitImage(img) - args := []string{ - "--set-string", "image.repository=" + repo, - "--set-string", "image.tag=" + tag, - "--set-string", "image.pullPolicy=Always", + args := []string{"--set-string", "image.pullPolicy=Always"} + if img != "" { + repo, tag := splitImage(img) + args = append(args, + "--set-string", "image.repository="+repo, + "--set-string", "image.tag="+tag, + ) } if secret := os.Getenv("E2E_IMAGE_PULL_SECRET"); secret != "" { args = append(args, "--set", "imagePullSecrets[0].name="+secret)