From 32f6865de845244828f96a15d4e2536d7297f4d7 Mon Sep 17 00:00:00 2001 From: Charlie Le Date: Wed, 19 Aug 2026 14:16:50 -0700 Subject: [PATCH 1/2] Resolve the latest release image to a published version integration/util.go derived the "latest release" image straight from the VERSION file. That holds on master, where VERSION is the last GA, but not on a release branch: VERSION is bumped to the version being prepared (e.g. 1.22.0-rc.0) long before the deploy job publishes that tag, and the integration job is a dependency of deploy. So the query fuzz leg would pull an image that does not exist yet. Resolve a pre-release version to the release preceding it instead, and add CORTEX_LATEST_RELEASE_IMAGE as an escape hatch for the cases the version math cannot cover (a major pre-release). The preload step in test-build-deploy.yml mirrors the same rule. Signed-off-by: Charlie Le --- .github/workflows/test-build-deploy.yml | 32 ++++++++- integration/util.go | 83 ++++++++++++++++++++-- integration/util_test.go | 94 +++++++++++++++++++++++++ 3 files changed, 204 insertions(+), 5 deletions(-) create mode 100644 integration/util_test.go diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index 6950325c945..e446933562e 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -317,6 +317,36 @@ jobs: done } + # Mirror of latestReleaseVersion() in integration/util.go: VERSION names the version + # being prepared, which on a release branch is not published yet, so a pre-release + # resolves to the release preceding it. Keep the two implementations in sync. + latest_release_image() { + if [ -n "${CORTEX_LATEST_RELEASE_IMAGE:-}" ]; then + echo "$CORTEX_LATEST_RELEASE_IMAGE" + return 0 + fi + + local version major minor patch + version=$(cat testdata/VERSION) + case "$version" in + *-*) + IFS='.' read -r major minor patch <<< "${version%%-*}" + if [ "$patch" -gt 0 ]; then + patch=$((patch - 1)) + elif [ "$minor" -gt 0 ]; then + minor=$((minor - 1)) + patch=0 + else + echo "ERROR: cannot resolve the release preceding major pre-release version ${version};" \ + "set CORTEX_LATEST_RELEASE_IMAGE to the latest published release image." >&2 + return 1 + fi + version="${major}.${minor}.${patch}" + ;; + esac + echo "quay.io/cortexproject/cortex:v${version}" + } + retry docker pull minio/minio:RELEASE.2024-05-28T17-19-04Z retry docker pull consul:1.8.4 retry docker pull quay.io/coreos/etcd:v3.5.29 @@ -329,7 +359,7 @@ jobs: retry docker pull quay.io/cortexproject/cortex:v1.21.0 retry docker pull quay.io/cortexproject/cortex:v1.21.1 elif [ "$TEST_TAGS" = "integration_query_fuzz" ]; then - retry docker pull quay.io/cortexproject/cortex:v$(cat testdata/VERSION) + retry docker pull "$(latest_release_image)" retry docker pull quay.io/prometheus/prometheus:v3.9.1 elif [ "$TEST_TAGS" = "integration_configs_db" ]; then retry docker pull postgres:9.6.16 diff --git a/integration/util.go b/integration/util.go index 0ec7721838c..0c3e802b831 100644 --- a/integration/util.go +++ b/integration/util.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "github.com/pkg/errors" @@ -36,20 +37,94 @@ func getCortexProjectDir() string { return os.Getenv("GOPATH") + "/src/github.com/cortexproject/cortex" } -// getLatestReleaseImage returns the Cortex image reference for the latest release, -// derived from the VERSION file at the project root. +// getLatestReleaseImage returns the Cortex image reference for the latest published +// release, derived from the VERSION file at the project root. +// +// Set CORTEX_LATEST_RELEASE_IMAGE to override the resolution entirely. +// +// If you change how this resolves, remember to update the preloading done by GitHub +// Actions too (see .github/workflows/test-build-deploy.yml). func getLatestReleaseImage() (string, error) { + if image := os.Getenv("CORTEX_LATEST_RELEASE_IMAGE"); image != "" { + return image, nil + } + content, err := os.ReadFile(filepath.Join(getCortexProjectDir(), "VERSION")) if err != nil { return "", errors.Wrap(err, "unable to read VERSION file") } - version := strings.TrimSpace(string(content)) + version, err := latestReleaseVersion(strings.TrimSpace(string(content))) + if err != nil { + return "", err + } + + return fmt.Sprintf("quay.io/cortexproject/cortex:v%s", version), nil +} + +// latestReleaseVersion maps the contents of the VERSION file to a version that has +// actually been published to the container registries. +// +// VERSION does not always name a published release. On a release branch it is bumped to +// the version being prepared (e.g. "1.22.0-rc.0") long before the deploy job publishes +// that tag, and the integration job runs before deploy. So a pre-release version resolves +// to the release preceding it, which is always already published by then: +// +// 1.21.1 -> 1.21.1 (VERSION on master is the last GA, whose image exists) +// 1.22.0-rc.0 -> 1.21.0 (the previous minor always shipped a .0) +// 1.22.2-rc.1 -> 1.22.1 (the preceding patch of the same minor) +func latestReleaseVersion(version string) (string, error) { if version == "" { return "", errors.New("VERSION file is empty") } - return fmt.Sprintf("quay.io/cortexproject/cortex:v%s", version), nil + // Anything after the first "-" is a pre-release identifier (e.g. "-rc.0"). + base, preRelease, isPreRelease := strings.Cut(version, "-") + if !isPreRelease { + return version, nil + } + + major, minor, patch, err := parseVersion(base) + if err != nil { + return "", errors.Wrapf(err, "unable to resolve the release preceding pre-release version %q", version) + } + + switch { + case patch > 0: + // A patch pre-release: the preceding patch of the same minor is published. + patch-- + case minor > 0: + // A minor pre-release: the previous minor's initial release is published. Using + // .0 rather than its latest patch keeps this derivable from VERSION alone. + minor-- + patch = 0 + default: + // A major pre-release (e.g. "2.0.0-rc.0"). The last release of the previous major + // is not derivable from VERSION, so the maintainer has to say which one it is. + return "", errors.Errorf("cannot resolve the release preceding major pre-release version %q (base %q, pre-release %q):"+ + " set CORTEX_LATEST_RELEASE_IMAGE to the latest published release image", version, base, preRelease) + } + + return fmt.Sprintf("%d.%d.%d", major, minor, patch), nil +} + +func parseVersion(version string) (major, minor, patch int, err error) { + parts := strings.Split(version, ".") + if len(parts) != 3 { + return 0, 0, 0, errors.Errorf("expected a major.minor.patch version, got %q", version) + } + + out := make([]int, len(parts)) + for i, part := range parts { + if out[i], err = strconv.Atoi(part); err != nil { + return 0, 0, 0, errors.Wrapf(err, "invalid version %q", version) + } + if out[i] < 0 { + return 0, 0, 0, errors.Errorf("invalid version %q", version) + } + } + + return out[0], out[1], out[2], nil } func writeFileToSharedDir(s *e2e.Scenario, dst string, content []byte) error { diff --git a/integration/util_test.go b/integration/util_test.go new file mode 100644 index 00000000000..8b84d988562 --- /dev/null +++ b/integration/util_test.go @@ -0,0 +1,94 @@ +//go:build integration + +package integration + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLatestReleaseVersion(t *testing.T) { + tests := map[string]struct { + version string + expected string + expectedErr bool + }{ + "a GA version is already published": { + version: "1.21.1", + expected: "1.21.1", + }, + "a GA version with a zero patch is already published": { + version: "1.21.0", + expected: "1.21.0", + }, + "a minor release candidate falls back to the previous minor": { + version: "1.22.0-rc.0", + expected: "1.21.0", + }, + "a later minor release candidate falls back to the same previous minor": { + version: "1.22.0-rc.3", + expected: "1.21.0", + }, + "a patch release candidate falls back to the preceding patch": { + version: "1.22.1-rc.0", + expected: "1.22.0", + }, + "a later patch release candidate falls back to the preceding patch": { + version: "1.22.3-rc.1", + expected: "1.22.2", + }, + "a major release candidate cannot be resolved": { + version: "2.0.0-rc.0", + expectedErr: true, + }, + "an empty VERSION is rejected": { + version: "", + expectedErr: true, + }, + "a malformed pre-release base is rejected": { + version: "1.22-rc.0", + expectedErr: true, + }, + "a non-numeric pre-release base is rejected": { + version: "1.x.0-rc.0", + expectedErr: true, + }, + } + + for name, testData := range tests { + t.Run(name, func(t *testing.T) { + actual, err := latestReleaseVersion(testData.version) + if testData.expectedErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + assert.Equal(t, testData.expected, actual) + }) + } +} + +func TestGetLatestReleaseImage(t *testing.T) { + // Point getCortexProjectDir() at a scratch checkout so we can exercise the VERSION file + // contents a release branch would actually have. + dir := t.TempDir() + t.Setenv("CORTEX_CHECKOUT_DIR", dir) + require.NoError(t, os.WriteFile(filepath.Join(dir, "VERSION"), []byte("1.22.0-rc.0\n"), 0o600)) + + image, err := getLatestReleaseImage() + require.NoError(t, err) + assert.Equal(t, "quay.io/cortexproject/cortex:v1.21.0", image) +} + +func TestGetLatestReleaseImage_HonorsOverride(t *testing.T) { + t.Setenv("CORTEX_LATEST_RELEASE_IMAGE", "quay.io/cortexproject/cortex:v1.20.1") + + image, err := getLatestReleaseImage() + require.NoError(t, err) + assert.Equal(t, "quay.io/cortexproject/cortex:v1.20.1", image) +} From f4182fd4238916633f3c357c8806082e6e98fa34 Mon Sep 17 00:00:00 2001 From: Charlie Le Date: Tue, 8 Sep 2026 11:02:16 -0700 Subject: [PATCH 2/2] Resolve the latest release image from the registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deriving the previous release from VERSION alone left one case broken, as SungJin1212 pointed out on #7786: on the GA tag push VERSION is 1.22.0 with no pre-release suffix, so it resolves to v1.22.0 — an image that only `deploy` publishes, and `deploy` needs `integration` to pass first. Ask quay.io what actually exists instead. The registry is the only source of truth for "published", so list the GA tags (^v\d+\.\d+\.\d+$) and take the highest one that does not exceed VERSION, then export it as CORTEX_LATEST_RELEASE_IMAGE for the preload and test steps. The <= bound only changes the result when a newer release already exists on quay than the branch being tested, e.g. preparing 1.21.2 on release-1.21 after v1.22.0 has shipped. This also drops the bash mirror of the Go derivation, and stops guessing at the previous minor's .0: with the registry answering, 1.19.5-rc.0 resolves to the v1.19.1 that exists rather than a v1.19.4 that never shipped, and a major pre-release such as 2.0.0-rc.0 no longer needs a manual override. The derivation stays in integration/util.go as the offline fallback for local runs, where no env var is set and no network call is wanted. Signed-off-by: Charlie Le --- .github/workflows/test-build-deploy.yml | 107 +++++++++++++++++------- integration/util.go | 20 +++-- 2 files changed, 90 insertions(+), 37 deletions(-) diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index e446933562e..ab90fcb567b 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -294,6 +294,81 @@ jobs: name: integration-tests-${{ matrix.arch }} - name: Extract Integration Tests Archive run: tar -xzvf integration-tests-${{ matrix.arch }}.tar.gz + - name: Resolve Latest Release Image + # The query fuzz tests compare the build under test against the latest *published* release. + # VERSION cannot answer "what is published" on its own: on a release branch it is bumped to + # the version being prepared (e.g. 1.22.0-rc.0) long before anything pushes that tag, and + # even on the GA tag push the v1.22.0 image is only pushed by `deploy`, which needs this job + # to pass first. The registry is the only source of truth, so ask it which GA tags exist and + # take the highest one that does not exceed VERSION. + # + # The <= bound (rather than simply "the highest published GA tag") only changes the result + # when a newer release already exists on quay than the branch being tested, e.g. preparing + # 1.21.2 on release-1.21 after v1.22.0 has shipped. + # + # Set the CORTEX_LATEST_RELEASE_IMAGE repository variable to bypass the lookup entirely. + if: matrix.tags == 'integration_query_fuzz' + env: + CORTEX_LATEST_RELEASE_IMAGE: ${{ vars.CORTEX_LATEST_RELEASE_IMAGE }} + run: | + if [ -n "${CORTEX_LATEST_RELEASE_IMAGE:-}" ]; then + echo "Using the CORTEX_LATEST_RELEASE_IMAGE override: ${CORTEX_LATEST_RELEASE_IMAGE}" + echo "CORTEX_LATEST_RELEASE_IMAGE=${CORTEX_LATEST_RELEASE_IMAGE}" >> "$GITHUB_ENV" + exit 0 + fi + + version=$(tr -d '[:space:]' < testdata/VERSION) + base=${version%%-*} + if ! printf '%s' "$base" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then + echo "ERROR: VERSION '${version}' does not begin with a major.minor.patch version." >&2 + exit 1 + fi + + # List the GA tags published to quay.io. filter_tag_name keeps the release tags and drops + # the per-commit master-* ones; the API pages at 100 tags, so follow has_additional. + tags_file=$(mktemp) + page=1 + while [ "$page" -le 20 ]; do + body="" + for attempt in 1 2 3; do + if body=$(curl -sSf --max-time 30 \ + "https://quay.io/api/v1/repository/cortexproject/cortex/tag/?onlyActiveTags=true&limit=100&page=${page}&filter_tag_name=like:v"); then + break + fi + echo "WARNING: listing quay.io tags page ${page} failed (attempt ${attempt}/3); retrying..." >&2 + body="" + sleep $((attempt * 5)) + done + if [ -z "$body" ]; then + echo "ERROR: unable to list the published tags from quay.io." >&2 + exit 1 + fi + printf '%s' "$body" | jq -r '.tags[].name' >> "$tags_file" + [ "$(printf '%s' "$body" | jq -r '.has_additional')" = "true" ] || break + page=$((page + 1)) + done + + published=$(grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' "$tags_file" | sed 's/^v//' | sort -u -V) + if [ -z "$published" ]; then + echo "ERROR: quay.io reported no published GA release tags." >&2 + exit 1 + fi + + if printf '%s\n' "$published" | grep -qxF "$base"; then + # VERSION itself names a published release, which is the steady state on master. + resolved="$base" + else + # Splice the (unpublished) base into the sorted list and take the entry just below it. + resolved=$(printf '%s\n%s\n' "$published" "$base" | sort -V | + awk -v base="$base" '$0 == base { exit } { previous = $0 } END { print previous }') + fi + if [ -z "$resolved" ]; then + echo "ERROR: quay.io has no published GA release at or below ${base}." >&2 + exit 1 + fi + + echo "VERSION is ${version}; the latest release published at or below ${base} is v${resolved}." + echo "CORTEX_LATEST_RELEASE_IMAGE=quay.io/cortexproject/cortex:v${resolved}" >> "$GITHUB_ENV" - name: Preload Images # We download docker images used by integration tests so that all images are available # locally and the download time doesn't account in the test execution time, which is subject @@ -317,36 +392,6 @@ jobs: done } - # Mirror of latestReleaseVersion() in integration/util.go: VERSION names the version - # being prepared, which on a release branch is not published yet, so a pre-release - # resolves to the release preceding it. Keep the two implementations in sync. - latest_release_image() { - if [ -n "${CORTEX_LATEST_RELEASE_IMAGE:-}" ]; then - echo "$CORTEX_LATEST_RELEASE_IMAGE" - return 0 - fi - - local version major minor patch - version=$(cat testdata/VERSION) - case "$version" in - *-*) - IFS='.' read -r major minor patch <<< "${version%%-*}" - if [ "$patch" -gt 0 ]; then - patch=$((patch - 1)) - elif [ "$minor" -gt 0 ]; then - minor=$((minor - 1)) - patch=0 - else - echo "ERROR: cannot resolve the release preceding major pre-release version ${version};" \ - "set CORTEX_LATEST_RELEASE_IMAGE to the latest published release image." >&2 - return 1 - fi - version="${major}.${minor}.${patch}" - ;; - esac - echo "quay.io/cortexproject/cortex:v${version}" - } - retry docker pull minio/minio:RELEASE.2024-05-28T17-19-04Z retry docker pull consul:1.8.4 retry docker pull quay.io/coreos/etcd:v3.5.29 @@ -359,7 +404,7 @@ jobs: retry docker pull quay.io/cortexproject/cortex:v1.21.0 retry docker pull quay.io/cortexproject/cortex:v1.21.1 elif [ "$TEST_TAGS" = "integration_query_fuzz" ]; then - retry docker pull "$(latest_release_image)" + retry docker pull "$CORTEX_LATEST_RELEASE_IMAGE" retry docker pull quay.io/prometheus/prometheus:v3.9.1 elif [ "$TEST_TAGS" = "integration_configs_db" ]; then retry docker pull postgres:9.6.16 diff --git a/integration/util.go b/integration/util.go index 0c3e802b831..89e4e248f10 100644 --- a/integration/util.go +++ b/integration/util.go @@ -38,12 +38,15 @@ func getCortexProjectDir() string { } // getLatestReleaseImage returns the Cortex image reference for the latest published -// release, derived from the VERSION file at the project root. +// release. // -// Set CORTEX_LATEST_RELEASE_IMAGE to override the resolution entirely. +// CORTEX_LATEST_RELEASE_IMAGE short-circuits the resolution. CI always sets it: the +// integration workflow asks quay.io which GA tags actually exist and picks the highest one +// that does not exceed VERSION, because the registry is the only source of truth for what +// is published (see .github/workflows/test-build-deploy.yml). // -// If you change how this resolves, remember to update the preloading done by GitHub -// Actions too (see .github/workflows/test-build-deploy.yml). +// Without it — a local run — fall back to deriving the version from the VERSION file at the +// project root, which needs no network but cannot see what the registry holds. func getLatestReleaseImage() (string, error) { if image := os.Getenv("CORTEX_LATEST_RELEASE_IMAGE"); image != "" { return image, nil @@ -62,8 +65,9 @@ func getLatestReleaseImage() (string, error) { return fmt.Sprintf("quay.io/cortexproject/cortex:v%s", version), nil } -// latestReleaseVersion maps the contents of the VERSION file to a version that has -// actually been published to the container registries. +// latestReleaseVersion maps the contents of the VERSION file to a version that has very +// likely been published to the container registries. It is the offline fallback for +// getLatestReleaseImage; CI resolves against the registry instead. // // VERSION does not always name a published release. On a release branch it is bumped to // the version being prepared (e.g. "1.22.0-rc.0") long before the deploy job publishes @@ -73,6 +77,10 @@ func getLatestReleaseImage() (string, error) { // 1.21.1 -> 1.21.1 (VERSION on master is the last GA, whose image exists) // 1.22.0-rc.0 -> 1.21.0 (the previous minor always shipped a .0) // 1.22.2-rc.1 -> 1.22.1 (the preceding patch of the same minor) +// +// A GA VERSION is assumed published, which holds everywhere except the GA tag build itself +// — there v1.22.0 is only pushed by deploy, after this runs. That case is why CI consults +// the registry rather than relying on this. func latestReleaseVersion(version string) (string, error) { if version == "" { return "", errors.New("VERSION file is empty")