test: support validating published ANC hotfix version in e2e test

.pipelines/scripts/e2e_run.sh

@@ -14,6 +14,8 @@ set -euo pipefail
 # * IGNORE_SCENARIOS_WITH_MISSING_VHD: a true/false flag that indicates if the build should fail if the VHD is missing.
 # * BUILD_SRC_DIR: the src directory for the repository. Probably the same as DefaultWorkingDirectory.
 # * E2E_FAILED_TESTS_RETRY_COUNT: the number of times the runner retries failed scenarios. Defaults to 0.
+# * ANC_HOTFIX_TARGET_VERSION: exact published ANC package version; empty keeps the fixed regression target.
+# * REQUIRE_ANC_HOTFIX_VALIDATION: require an explicit ANC hotfix target version.
 
 # In addition, the e2e test framework reads a whole lot of environment variables.
 # These are defined in: e2e/config/config.go
@@ -58,6 +60,8 @@ E2E_GO_TEST_TIMEOUT="${E2E_GO_TEST_TIMEOUT:-80m}"
 E2E_FAILED_TESTS_RETRY_COUNT="${E2E_FAILED_TESTS_RETRY_COUNT:-0}"
 GALLERY_NAME="${GALLERY_NAME:-}"
 SIG_GALLERY_NAME="${SIG_GALLERY_NAME:-}"
+export ANC_HOTFIX_TARGET_VERSION="${ANC_HOTFIX_TARGET_VERSION:-}"
+export REQUIRE_ANC_HOTFIX_VALIDATION="${REQUIRE_ANC_HOTFIX_VALIDATION:-false}"
 
 case "${E2E_FAILED_TESTS_RETRY_COUNT}" in
   '' | *[!0-9]*)

e2e/README.md

@@ -85,6 +85,32 @@ coverage matching production. A Scriptless tag or an AKSNodeConfig mutator alone
 does not opt into Phase 3. Existing scriptless opt-outs, image restrictions, and
 VHD-caching behavior remain in effect.
 
+### ANC hotfix package validation
+
+`Ubuntu2204_ANCHotfixFlow` and `AzureLinuxV3_ANCHotfixFlow` seed a hotfix pointer
+and run the production authenticated repository download, verification/extraction,
+binary selection, and execution on E2E VMs. They do not validate LPS rollout or
+require embedded-script application.
+
+Ordinary runs keep target `202608.21.1` and stamp the test ANC binary `202608.21.0`.
+Set `--anc-hotfix-target-version` / `ANC_HOTFIX_TARGET_VERSION` to an exact
+published package version (`YYYYMM.DD.PATCH`, valid date and positive patch) to
+test that package instead. The test ANC stamp uses the same base and patch `0`;
+compilation, pointer injection, and version assertions share the resolved pair.
+
+Release validation must also set `--require-anc-hotfix-validation` /
+`REQUIRE_ANC_HOTFIX_VALIDATION=true`. Missing/invalid targets fail before Azure
+initialization. The calling pipeline selects the scenarios and supplies their
+VHDs; the runner retains its normal filtering, skip, failure, and retry behavior.
+Do not enable two-stage VHD caching or disable scriptless compilation for these
+scenarios.
+
+Queue-time variables `ANC_HOTFIX_TARGET_VERSION` and
+`REQUIRE_ANC_HOTFIX_VALIDATION` are automatically available to the pipeline task
+environment and are consumed by `e2e_run.sh` and the runner. With those
+variables absent, the script defaults them to empty and `false`, preserving
+ordinary runs.
+
 ### LocalDNS fallback checks
 
 The existing LocalDNS scenarios use short-lived DNS pods on the scenario node

e2e/config/anc_hotfix.go

@@ -0,0 +1,47 @@
+package config
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type ANCHotfixVersions struct {
+	Base   string
+	Target string
+}
+
+// Ordinary runs exercise a known published package; release validation must supply its own.
+var defaultANCHotfixVersions = ANCHotfixVersions{Base: "202608.21.0", Target: "202608.21.1"}
+
+var ancHotfixVersionPattern = regexp.MustCompile(`^[0-9]{6}\.[0-9]{2}\.[1-9][0-9]*$`)
+
+// ResolveANCHotfixVersions runs after CLI parsing, before scenario setup or Azure initialization.
+func (c *Configuration) ResolveANCHotfixVersions() error {
+	target := c.ANCHotfixTargetVersion
+	if target == "" {
+		if c.RequireANCHotfixValidation {
+			return fmt.Errorf("ANC_HOTFIX_TARGET_VERSION is required when REQUIRE_ANC_HOTFIX_VALIDATION=true")
+		}
+		c.ANCHotfixVersions = defaultANCHotfixVersions
+		return nil
+	}
+	invalid := func() error {
+		return fmt.Errorf("invalid ANC_HOTFIX_TARGET_VERSION %q: expected YYYYMM.DD.PATCH with a valid date and positive patch", target)
+	}
+	if !ancHotfixVersionPattern.MatchString(target) {
+		return invalid()
+	}
+	parts := strings.Split(target, ".")
+	base := parts[0] + "." + parts[1]
+	if date, err := time.Parse("200601.02", base); err != nil || date.Year() < 1 {
+		return invalid()
+	}
+	if _, err := strconv.ParseUint(parts[2], 10, 64); err != nil {
+		return invalid()
+	}
+	c.ANCHotfixVersions = ANCHotfixVersions{Base: base + ".0", Target: target}
+	return nil
+}

e2e/config/anc_hotfix_test.go

@@ -0,0 +1,126 @@
+package config
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v3"
+	"gopkg.in/yaml.v3"
+)
+
+func TestResolveANCHotfixVersions(t *testing.T) {
+	for _, tc := range []struct {
+		target   string
+		required bool
+		base     string
+		want     string
+	}{
+		{base: "202608.21.0", want: "202608.21.1"},
+		{target: "202608.26.2", base: "202608.26.0", want: "202608.26.2"},
+		{target: "202610.20.15", required: true, base: "202610.20.0", want: "202610.20.15"},
+		{required: true},
+		{target: "202608.26.0"},
+		{target: "202608.26.-1"},
+		{target: "202608.26.01"},
+		{target: "202608.26"},
+		{target: "v202608.26.2"},
+		{target: "202608.26.2-ubuntu22.04u1"},
+		{target: "202613.26.2"},
+		{target: "202602.30.2"},
+		{target: "000001.01.2"},
+		{target: "202608.01.2", base: "202608.01.0", want: "202608.01.2"},
+		{target: "202802.29.2", base: "202802.29.0", want: "202802.29.2"},
+		{target: "202608.26.18446744073709551616"},
+		{target: " 202608.26.2"},
+		{target: "$(ANC_HOTFIX_TARGET_VERSION)"},
+	} {
+		t.Run(tc.target+map[bool]string{true: "/required", false: "/optional"}[tc.required], func(t *testing.T) {
+			cfg := DefaultConfiguration()
+			cfg.ANCHotfixTargetVersion = tc.target
+			cfg.RequireANCHotfixValidation = tc.required
+			err := cfg.ResolveANCHotfixVersions()
+			if tc.want == "" {
+				require.ErrorContains(t, err, "ANC_HOTFIX_TARGET_VERSION")
+				return
+			}
+			require.NoError(t, err)
+			require.Equal(t, ANCHotfixVersions{Base: tc.base, Target: tc.want}, cfg.ANCHotfixVersions)
+		})
+	}
+}
+
+func TestANCHotfixQueueVariablesAreNotOverridden(t *testing.T) {
+	type pipeline struct {
+		Variables  map[string]string
+		Parameters []struct{ Name string }
+		Jobs       []struct {
+			Parameters map[string]string
+			Steps      []struct {
+				Env map[string]any
+			}
+		}
+	}
+	read := func(name string) pipeline {
+		t.Helper()
+		data, err := os.ReadFile(filepath.Join("..", "..", ".pipelines", name))
+		require.NoError(t, err)
+		var p pipeline
+		require.NoError(t, yaml.Unmarshal(data, &p))
+		return p
+	}
+	template := read(filepath.Join("templates", "e2e-template.yaml"))
+	parameters := make(map[string]bool)
+	for _, parameter := range template.Parameters {
+		parameters[parameter.Name] = true
+	}
+	require.NotContains(t, parameters, "ancHotfixTargetVersion")
+	require.NotContains(t, parameters, "requireANCHotfixValidation")
+	for _, step := range template.Jobs[0].Steps {
+		require.NotContains(t, step.Env, "ANC_HOTFIX_TARGET_VERSION")
+		require.NotContains(t, step.Env, "REQUIRE_ANC_HOTFIX_VALIDATION")
+	}
+	for _, name := range []string{"e2e-tme.yaml", "e2e.yaml"} {
+		p := read(name)
+		require.NotContains(t, p.Variables, "ANC_HOTFIX_TARGET_VERSION")
+		require.NotContains(t, p.Variables, "REQUIRE_ANC_HOTFIX_VALIDATION")
+		require.NotContains(t, p.Jobs[0].Parameters, "ancHotfixTargetVersion")
+		require.NotContains(t, p.Jobs[0].Parameters, "requireANCHotfixValidation")
+	}
+}
+
+func TestANCHotfixFlagsResolveAfterParsing(t *testing.T) {
+	saved := *Config
+	t.Cleanup(func() { *Config = saved })
+	t.Setenv("ANC_HOTFIX_TARGET_VERSION", "202608.26.2")
+	t.Setenv("REQUIRE_ANC_HOTFIX_VALIDATION", "true")
+	for _, tc := range []struct {
+		name     string
+		args     []string
+		target   string
+		required bool
+	}{
+		{name: "environment", target: "202608.26.2", required: true},
+		{name: "flag override", args: []string{"--anc-hotfix-target-version", "202610.20.3"}, target: "202610.20.3", required: true},
+		{name: "optional flag override", args: []string{"--require-anc-hotfix-validation=false"}, target: "202608.26.2"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			*Config = *DefaultConfiguration()
+			cmd := &cli.Command{
+				Flags: Flags(),
+				Action: func(context.Context, *cli.Command) error {
+					require.Equal(t, tc.required, Config.RequireANCHotfixValidation)
+					require.Equal(t, tc.target, Config.ANCHotfixTargetVersion)
+					require.Equal(t, defaultANCHotfixVersions, Config.ANCHotfixVersions)
+					require.NoError(t, Config.ResolveANCHotfixVersions())
+					require.Equal(t, tc.target, Config.ANCHotfixVersions.Target)
+					require.Equal(t, tc.target[:9]+".0", Config.ANCHotfixVersions.Base)
+					return nil
+				},
+			}
+			require.NoError(t, cmd.Run(context.Background(), append([]string{"e2e"}, tc.args...)))
+		})
+	}
+}

e2e/config/config.go

@@ -42,6 +42,9 @@ func PrivateACRName(location string) string {
 }
 
 type Configuration struct {
+	ANCHotfixTargetVersion                 string
+	RequireANCHotfixValidation             bool
+	ANCHotfixVersions                      ANCHotfixVersions
 	ACRSecretName                          string
 	AzureContainerRegistrytargetRepository string
 	BlobContainer                          string
@@ -91,6 +94,7 @@ type Configuration struct {
 
 func DefaultConfiguration() *Configuration {
 	return &Configuration{
+		ANCHotfixVersions:                      defaultANCHotfixVersions,
 		ACRSecretName:                          "acr-secret-code2",
 		AzureContainerRegistrytargetRepository: "aks-managed-repository/*",
 		BlobContainer:                          "abe2e",

e2e/config/flags.go

@@ -5,6 +5,8 @@ import "github.com/urfave/cli/v3"
 func Flags() []cli.Flag {
 	defaults := DefaultConfiguration()
 	return []cli.Flag{
+		&cli.StringFlag{Name: "anc-hotfix-target-version", Destination: &Config.ANCHotfixTargetVersion, Sources: cli.EnvVars("ANC_HOTFIX_TARGET_VERSION"), Usage: "Exact published ANC hotfix package version (YYYYMM.DD.PATCH, positive patch)"},
+		&cli.BoolFlag{Name: "require-anc-hotfix-validation", Destination: &Config.RequireANCHotfixValidation, Sources: cli.EnvVars("REQUIRE_ANC_HOTFIX_VALIDATION"), Usage: "Require an explicit ANC hotfix target version"},
 		&cli.StringFlag{Name: "acr-secret-name", Value: defaults.ACRSecretName, Destination: &Config.ACRSecretName, Sources: cli.EnvVars("ACR_SECRET_NAME")},
 		&cli.StringFlag{Name: "acr-target-repository", Value: defaults.AzureContainerRegistrytargetRepository, Destination: &Config.AzureContainerRegistrytargetRepository, Sources: cli.EnvVars("ACR_TARGET_REPOSITORY")},
 		&cli.StringFlag{Name: "blob-container", Value: defaults.BlobContainer, Destination: &Config.BlobContainer, Sources: cli.EnvVars("BLOB_CONTAINER")},

e2e/runner/app.go

@@ -92,6 +92,9 @@ func (a *App) Run(ctx context.Context, args []string) int {
 }
 
 func (a *App) run(ctx context.Context, opts runOptions) error {
+	if err := config.Config.ResolveANCHotfixVersions(); err != nil {
+		return &usageError{message: err.Error()}
+	}
 	if opts.parallel < 1 {
 		return &usageError{message: "--parallel must be at least 1"}
 	}

e2e/runner/filter_test.go

@@ -119,6 +119,25 @@ func restoreRunnerConfig(t *testing.T) {
 	t.Helper()
 	saved := *config.Config
 	t.Cleanup(func() { *config.Config = saved })
+	t.Setenv("ANC_HOTFIX_TARGET_VERSION", "")
+	t.Setenv("REQUIRE_ANC_HOTFIX_VALIDATION", "false")
+}
+
+func TestRequiredANCHotfixTargetFailsBeforeAzureInitialization(t *testing.T) {
+	for _, target := range []string{"", "202608.26.0"} {
+		t.Run(target, func(t *testing.T) {
+			restoreRunnerConfig(t)
+			t.Setenv("ANC_HOTFIX_TARGET_VERSION", target)
+			before := azureInitProbe()
+			var stderr bytes.Buffer
+			code := NewApp(&bytes.Buffer{}, &stderr).Run(context.Background(), []string{
+				"e2e", "run", "--require-anc-hotfix-validation",
+			})
+			require.Equal(t, exitUsage, code)
+			require.Contains(t, stderr.String(), "ANC_HOTFIX_TARGET_VERSION")
+			require.Equal(t, before, azureInitProbe())
+		})
+	}
 }
 
 func TestMatchFiltersPreservesTagPolicy(t *testing.T) {

e2e/scenario/scenario_anc_hotfix.go

@@ -28,19 +28,6 @@ const (
 	// exists in cse_timing.go.)
 	provisionCompletePath = "/opt/azure/containers/provision.complete"
 	ancNBCCmdMarker       = "anc-hotfix-flow-nbc-executed"
-
-	// ancHotfixFlowTargetVersion is a real published ANC hotfix (git tag
-	// aks-node-controller/hotfix/v202608.21.1), so download-hotfix performs a real PMC
-	// download rather than hitting a synthetic artifact. The validators pin the run to the
-	// authenticated repository fast path, which extracts and stages the package itself, so
-	// no apt/dnf/tdnf installation is exercised here.
-	//
-	// The PR-built ANC is stamped to ancHotfixFlowBaseVersion so it shares the "202608.21"
-	// base and sits at a strictly lower patch, which is what makes the pointer applicable:
-	// download-hotfix derives the base from the running binary's version and only upgrades
-	// when the base matches and the target patch is higher.
-	ancHotfixFlowBaseVersion   = "202608.21.0"
-	ancHotfixFlowTargetVersion = "202608.21.1"
 )
 
 // The fixture writes the hotfix pointer directly instead of letting check-hotfix fetch it
@@ -85,6 +72,7 @@ func newANCHotfixFlowScenario(name, description string, vhd *config.Image) *Scen
 			// The no-op NBC command validates hotfix execution, not node readiness.
 			SkipDefaultValidation: true,
 			Validator: func(ctx context.Context, s *Scenario) error {
+				versions := config.Config.ANCHotfixVersions
 				// The boothook starts the launcher under nohup and SkipDefaultValidation
 				// removes the node-readiness wait, so SSH can come up while download-hotfix
 				// is still running. Every assertion below is a one-shot read, and the
@@ -94,8 +82,8 @@ func newANCHotfixFlowScenario(name, description string, vhd *config.Image) *Scen
 					return err
 				}
 				return errors.Join(
-					ValidateANCBakedBinaryVersion(ctx, s, ancHotfixFlowBaseVersion),
-					ValidateFileHasContent(ctx, s, ancHotfixPointerPath, fmt.Sprintf(`"%s":"%s"`, hotfixBaseVersion(ancHotfixFlowBaseVersion), ancHotfixFlowTargetVersion)),
+					ValidateANCBakedBinaryVersion(ctx, s, versions.Base),
+					ValidateFileHasContent(ctx, s, ancHotfixPointerPath, fmt.Sprintf(`"%s":"%s"`, hotfixBaseVersion(versions.Base), versions.Target)),
 					ValidateFileHasContent(ctx, s, ancLauncherOutput, "Found ANC hotfix config"),
 					ValidateFileHasContent(ctx, s, ancLogPath, "downloading ANC hotfix"),
 					// Assert the fast path's full message rather than the "downloaded ANC hotfix"
@@ -112,7 +100,7 @@ func newANCHotfixFlowScenario(name, description string, vhd *config.Image) *Scen
 					ValidateFileExists(ctx, s, ancHotfixBinaryPath),
 					ValidateFileHasContent(ctx, s, ancLauncherOutput, "Using hotfix binary"),
 					ValidateFileHasContent(ctx, s, ancLauncherOutput, "aks-node-controller completed successfully"),
-					ValidateANCHotfixBinaryVersion(ctx, s, ancHotfixFlowTargetVersion),
+					ValidateANCHotfixBinaryVersion(ctx, s, versions.Target),
 					// The no-op NBC command publishes the provision status itself, since it
 					// bypasses cse_start.sh which normally writes both artifacts. Without them
 					// the CSE's provision-wait never unblocks. Assert the marker rather than