feat(vhd): add isolated AMD GPU image and diagnostics

.github/renovate.json

@@ -737,6 +737,30 @@
         "rashmichandrashekar",
         "bragi92"
       ]
+    },
+    {
+      "description": "AMD image updates always require coordinated pin review and fresh MI300X qualification",
+      "matchFileNames": [
+        "vhdbuilder/packer/amd-gpu-components.json"
+      ],
+      "matchManagers": [
+        "custom.regex"
+      ],
+      "enabled": true,
+      "automerge": false,
+      "ignoreUnstable": false,
+      "groupName": "amd-gpu-image",
+      "separateMajorMinor": false,
+      "separateMinorPatch": false,
+      "assignees": [
+        "team:aks-node-lifecycle"
+      ],
+      "reviewers": [
+        "team:aks-node-lifecycle"
+      ],
+      "prBodyNotes": [
+        "Update AMD repository, driver, firmware, module and DKMS pins together; AMD CUE validation rejects mismatches. Rebuild the separate AMD VHD and qualify it on MI300X before merging."
+      ]
     }
   ],
   "customManagers": [
@@ -964,6 +988,78 @@
       "matchStrings": [
         "#\\s*renovate:\\s*datasource=(?<datasource>\\S+)\\s+depName=(?<depName>\\S+)(?:\\s+versioning=(?<versioning>\\S+))?(?:\\s+registryUrl=(?<registryUrl>\\S+))?\\s*\\n[^\\n]*VERSION=\"(?<currentValue>[^\"]+)\""
       ]
+    },
+    {
+      "customType": "regex",
+      "description": "Track amdgpu-dkms in the isolated AMD VHD manifest",
+      "managerFilePatterns": [
+        "/^vhdbuilder/packer/amd-gpu-components\\.json$/"
+      ],
+      "matchStrings": [
+        "\"AMDGPUDriver\":\\s*\\{[^}]*?\"repositoryURL\":\\s*\"(?<amdRepository>https://repo\\.radeon\\.com/amdgpu/[0-9.]+/ubuntu)\"[^}]*\\}",
+        "\"packageVersion\":\\s*\"(?<currentValue>[^\"]+)\""
+      ],
+      "depNameTemplate": "amdgpu-dkms",
+      "datasourceTemplate": "deb",
+      "versioningTemplate": "deb",
+      "registryUrlTemplate": "{{{amdRepository}}}?suite=noble&components=main&binaryArch=amd64",
+      "matchStringsStrategy": "recursive"
+    },
+    {
+      "customType": "regex",
+      "description": "Track amdgpu-dkms-firmware in the isolated AMD VHD manifest",
+      "managerFilePatterns": [
+        "/^vhdbuilder/packer/amd-gpu-components\\.json$/"
+      ],
+      "matchStrings": [
+        "\"AMDGPUDriver\":\\s*\\{[^}]*?\"repositoryURL\":\\s*\"(?<amdRepository>https://repo\\.radeon\\.com/amdgpu/[0-9.]+/ubuntu)\"[^}]*\\}",
+        "\"firmwarePackageVersion\":\\s*\"(?<currentValue>[^\"]+)\""
+      ],
+      "depNameTemplate": "amdgpu-dkms-firmware",
+      "datasourceTemplate": "deb",
+      "versioningTemplate": "deb",
+      "registryUrlTemplate": "{{{amdRepository}}}?suite=noble&components=main&binaryArch=amd64",
+      "matchStringsStrategy": "recursive"
+    },
+    {
+      "customType": "regex",
+      "description": "Track AMD amdsmi in the isolated AMD VHD manifest",
+      "managerFilePatterns": [
+        "/^vhdbuilder/packer/amd-gpu-components\\.json$/"
+      ],
+      "matchStrings": [
+        "\"amdsmiPackage\":\\s*\"(?<depName>amdrocm-amdsmi[0-9]+\\.[0-9]+)\"[^}]*?\"amdsmiVersion\":\\s*\"(?<currentValue>[^\"]+)\""
+      ],
+      "datasourceTemplate": "deb",
+      "versioningTemplate": "deb",
+      "registryUrlTemplate": "https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/?suite=stable&components=main&binaryArch=amd64"
+    },
+    {
+      "customType": "regex",
+      "description": "Track AMD sysdeps in the isolated AMD VHD manifest",
+      "managerFilePatterns": [
+        "/^vhdbuilder/packer/amd-gpu-components\\.json$/"
+      ],
+      "matchStrings": [
+        "\"sysdepsPackage\":\\s*\"(?<depName>amdrocm-sysdeps[0-9]+\\.[0-9]+)\"[^}]*?\"sysdepsVersion\":\\s*\"(?<currentValue>[^\"]+)\""
+      ],
+      "datasourceTemplate": "deb",
+      "versioningTemplate": "deb",
+      "registryUrlTemplate": "https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/?suite=stable&components=main&binaryArch=amd64"
+    },
+    {
+      "customType": "regex",
+      "description": "Detect new AMD driver release directories; coordinated pins require review",
+      "managerFilePatterns": [
+        "/^vhdbuilder/packer/amd-gpu-components\\.json$/"
+      ],
+      "matchStrings": [
+        "\"repositoryURL\":\\s*\"https://repo\\.radeon\\.com/amdgpu/(?<currentValue>[0-9.]+)/ubuntu\""
+      ],
+      "depNameTemplate": "amdgpu-repository",
+      "datasourceTemplate": "custom.amdgpu-releases",
+      "versioningTemplate": "loose",
+      "extractVersionTemplate": "^(?<version>[0-9]+(?:\\.[0-9]+)+)/$"
     }
   ],
   "customDatasources": {
@@ -1022,6 +1118,10 @@
       "transformTemplates": [
         "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}"
       ]
+    },
+    "amdgpu-releases": {
+      "defaultRegistryUrlTemplate": "https://repo.radeon.com/amdgpu/",
+      "format": "html"
     }
   }
 }

.github/workflows/validate-components.yml

@@ -2,6 +2,22 @@ name: Validate Components
 on: pull_request
 
 jobs:
+  amd-components:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version: '1.26'
+      - name: Validate dedicated AMD manifest and coordinated package pins
+        run: |
+          go install cuelang.org/go/cmd/cue@v0.15.4
+          "$(go env GOPATH)/bin/cue" vet -c ./schemas/amd-gpu-components.cue ./vhdbuilder/packer/amd-gpu-components.json
+
   cue:
     runs-on: ubuntu-latest
     permissions:

.pipelines/.vsts-vhd-builder-amd.yaml

@@ -0,0 +1,83 @@
+# Register this manual pipeline with the existing nonproduction Linux VHD
+# builder's pool, service connection and build-environment variables.
+# Its single image job is independent of the shared PR and release pipelines.
+name: $(Date:yyyyMMdd)$(Rev:.r)_AMD_$(BuildID)
+trigger: none
+pr: none
+
+parameters:
+  - name: replicationRegions
+    displayName: Capture replication targets (region=replicas, comma separated)
+    type: string
+    default: francecentral=1
+
+pool:
+  name: $(POOL_NAME)
+
+variables:
+  - name: MODE
+    value: linuxVhdMode
+  - group: aks-vuln-to-kusto
+  - group: "AKS Node SIG UA Token (KV)"
+  - group: build_performance
+  - group: aks-vuln-to-kusto-tme
+  - name: BUILDER
+    value: packer
+  # Capture a test image and retain it for explicit MI300X qualification.
+  # Publishing/export and shared-gallery garbage collection are not part of this pipeline.
+  - name: GENERATE_PUBLISHING_INFO
+    value: 'False'
+  - name: RUN_PREFETCH_OPTIMIZATION
+    value: 'False'
+  - name: DRY_RUN
+    value: 'False'
+  - name: SKIP_CLEANUP
+    # Shared cleanup includes subscription-wide garbage collection. The final
+    # step below removes only this build's recorded and ownership-verified group.
+    value: 'True'
+  - name: REPLICATIONS
+    value: ${{ parameters.replicationRegions }}
+
+stages:
+  - stage: build_amd
+    displayName: Build dedicated AMD GPU VHD
+    jobs:
+      - job: build2404amdgpugen2containerd
+        timeoutInMinutes: 360
+        steps:
+          - bash: |
+              echo '##vso[task.setvariable variable=OS_SKU]Ubuntu'
+              echo '##vso[task.setvariable variable=OS_VERSION]24.04'
+              echo '##vso[task.setvariable variable=IMG_PUBLISHER]Canonical'
+              echo '##vso[task.setvariable variable=IMG_OFFER]ubuntu-24_04-lts'
+              echo '##vso[task.setvariable variable=IMG_SKU]server'
+              echo '##vso[task.setvariable variable=IMG_VERSION]latest'
+              echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2'
+              echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5'
+              echo '##vso[task.setvariable variable=FEATURE_FLAGS]AMD_GPU'
+              echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64'
+              echo '##vso[task.setvariable variable=ENABLE_FIPS]False'
+              echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False'
+              echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]False'
+            displayName: Setup AMD Build Variables
+          - template: ./templates/.builder-release-template.yaml
+            parameters:
+              artifactName: 2404-amdgpu-gen2-containerd
+              # Retries overwrite the Packer output used for scoped cleanup.
+              packerBuildRetryCount: 0
+          - task: AzureCLI@2
+            displayName: Clean up this AMD build's Packer resource group
+            condition: always()
+            inputs:
+              azureSubscription: $(VHD_ARM_SERVICE_CONNECTION)
+              scriptType: bash
+              scriptLocation: inlineScript
+              inlineScript: |
+                set -euo pipefail
+                source vhdbuilder/packer/cleanup-amd-gpu-build.sh
+                cleanup_amd_gpu_build_resource_group \
+                  vhdbuilder/packer/settings.json \
+                  "${PACKER_RESOURCE_GROUP}" "${AMD_BUILD_ID}"
+            env:
+              PACKER_RESOURCE_GROUP: $(PKR_RG_NAME)
+              AMD_BUILD_ID: $(Build.BuildId)

.pipelines/templates/.builder-release-template.yaml

@@ -8,6 +8,9 @@ parameters:
   - name: overrideBranch
     type: string
     default: master
+  - name: packerBuildRetryCount
+    type: number
+    default: 3
 
 steps:
   - template: ./.template-override-components-json.yaml
@@ -74,18 +77,30 @@ steps:
     displayName: Setup Cue
 
   - bash: |
-      SKU_NAME=${OS_VERSION} && \
-      if grep -q "minimal" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}minimal"; fi && \
-      if [[ "${HYPERV_GENERATION,,}" == "v2" ]]; then SKU_NAME="${SKU_NAME}gen2"; fi && \
-      if [[ "${ARCHITECTURE,,}" == "arm64" ]]; then SKU_NAME="${SKU_NAME}arm64"; fi && \
-      if grep -q "NVIDIA_GB" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}gb"; fi && \
-      if [[ "${ENABLE_FIPS,,}" == "true" ]]; then SKU_NAME="${SKU_NAME}fips"; fi && \
-      if grep -q "cvm" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}CVM"; fi && \
-      if [[ "${ENABLE_TRUSTED_LAUNCH}" == "True" ]]; then SKU_NAME="${SKU_NAME}TL"; fi && \
-      if [[ ${OS_SKU} != "CBLMariner" && ${OS_SKU} != "AzureLinux" && ${OS_SKU} != "AzureLinuxOSGuard" && ${OS_SKU} != "Flatcar" && ${OS_SKU} != "AzureContainerLinux" ]]; then SKU_NAME="${SKU_NAME}containerd"; fi && \
-      SKU_NAME=$(echo ${SKU_NAME} | tr -d '.') && \
-      echo "##vso[task.setvariable variable=SKU_NAME]$SKU_NAME"
-      echo "Set SKU_NAME to $SKU_NAME"
+      if [[ "${FEATURE_FLAGS}" == *AMD_GPU* ]]; then
+        "$(go env GOPATH)/bin/cue" vet -c ./schemas/amd-gpu-components.cue ./vhdbuilder/packer/amd-gpu-components.json || exit 1
+        source vhdbuilder/packer/amd-gpu-build-settings.sh
+        SKU_NAME=$(get_amd_gpu_sku_name) || exit 1
+        # Prepare only this AMD worker checkout after the template's checkout step.
+        prepare_amd_gpu_packer_template \
+          vhdbuilder/packer/vhd-image-builder-base.json \
+          vhdbuilder/packer/amd-gpu-packer-files.json || exit 1
+        echo "##vso[task.setvariable variable=SKU_NAME]$SKU_NAME"
+        echo "Set SKU_NAME to $SKU_NAME"
+      else
+        SKU_NAME=${OS_VERSION} && \
+        if grep -q "minimal" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}minimal"; fi && \
+        if [[ "${HYPERV_GENERATION,,}" == "v2" ]]; then SKU_NAME="${SKU_NAME}gen2"; fi && \
+        if [[ "${ARCHITECTURE,,}" == "arm64" ]]; then SKU_NAME="${SKU_NAME}arm64"; fi && \
+        if grep -q "NVIDIA_GB" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}gb"; fi && \
+        if [[ "${ENABLE_FIPS,,}" == "true" ]]; then SKU_NAME="${SKU_NAME}fips"; fi && \
+        if grep -q "cvm" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}CVM"; fi && \
+        if [[ "${ENABLE_TRUSTED_LAUNCH}" == "True" ]]; then SKU_NAME="${SKU_NAME}TL"; fi && \
+        if [[ ${OS_SKU} != "CBLMariner" && ${OS_SKU} != "AzureLinux" && ${OS_SKU} != "AzureLinuxOSGuard" && ${OS_SKU} != "Flatcar" && ${OS_SKU} != "AzureContainerLinux" ]]; then SKU_NAME="${SKU_NAME}containerd"; fi && \
+        SKU_NAME=$(echo ${SKU_NAME} | tr -d '.') && \
+        echo "##vso[task.setvariable variable=SKU_NAME]$SKU_NAME"
+        echo "Set SKU_NAME to $SKU_NAME"
+      fi
     displayName: Set SKU Name
 
   - task: AzureCLI@2
@@ -97,7 +112,7 @@ steps:
         make -f packer.mk run-packer
     condition: ne(variables['BUILDER'], 'imagecustomizer')
     displayName: Build VHD
-    retryCountOnTaskFailure: 3
+    retryCountOnTaskFailure: ${{ parameters.packerBuildRetryCount }}
     env:
       OS_TYPE: Linux
       GIT_VERSION: $(Build.SourceVersion)
@@ -391,7 +406,7 @@ steps:
       TargetFolder: '$(Build.ArtifactStagingDirectory)'
 
   - task: CopyFiles@2
-    condition: and(eq(variables.OS_SKU, 'Ubuntu'), in(variables.OS_VERSION, '22.04', '24.04'), in(variables.FEATURE_FLAGS, 'None', 'cvm', 'NVIDIA_GB', 'NVIDIA_CUDA_PREBAKE'))
+    condition: and(eq(variables.OS_SKU, 'Ubuntu'), in(variables.OS_VERSION, '22.04', '24.04'), in(variables.FEATURE_FLAGS, 'None', 'cvm', 'NVIDIA_GB', 'NVIDIA_CUDA_PREBAKE', 'AMD_GPU'))
     displayName: Copy CIS Reports
     inputs:
       SourceFolder: '$(System.DefaultWorkingDirectory)'

Makefile

@@ -265,6 +265,10 @@ unit-tests:
 update-windows-base-versions:
 	./vhdbuilder/packer/windows/update_windows_base_versions.sh
 
+.PHONY: validate-amd-components
+validate-amd-components:
+	@./hack/tools/bin/cue vet -c ./schemas/amd-gpu-components.cue ./vhdbuilder/packer/amd-gpu-components.json
+
 .PHONY: validate-components
 validate-components:
 	@./hack/tools/bin/cue vet -c ./schemas/components.cue ./parts/common/components.json

aks-node-controller/parser/parser.go

@@ -96,6 +96,7 @@ func getCSEEnv(ctx context.Context, config *aksnodeconfigv1.Configuration, gpuCo
 		"API_SERVER_NAME":                                      config.GetApiServerConfig().GetApiServerName(),
 		"IS_VHD":                                               fmt.Sprintf("%v", getIsVHD(config.IsVhd)),
 		"GPU_NODE":                                             fmt.Sprintf("%v", getEnableNvidia(config)),
+		"AMD_GPU_NODE":                                         fmt.Sprintf("%v", config.GetGpuConfig().GetEnableAmdGpu()),
 		"SGX_NODE":                                             fmt.Sprintf("%v", getIsSgxEnabledSKU(config.GetVmSize())),
 		"MIG_NODE":                                             fmt.Sprintf("%v", isMIGNode),
 		"CONFIG_GPU_DRIVER_IF_NEEDED":                          fmt.Sprintf("%v", config.GetGpuConfig().GetConfigGpuDriver()),

aks-node-controller/parser/parser_test.go

@@ -26,6 +26,27 @@ func TestBuildCSECmd(t *testing.T) {
 		aksNodeConfigUpdator func(*aksnodeconfigv1.Configuration)
 		validator            func(cmd *exec.Cmd)
 	}{
+		{
+			name:       "AMD GPU uses the baked driver and the standard container runtime",
+			folder:     "AKSUbuntu2204+Containerd+MIG",
+			k8sVersion: "1.36.3",
+			aksNodeConfigUpdator: func(config *aksnodeconfigv1.Configuration) {
+				config.GpuConfig.EnableNvidia = to.Ptr(false)
+				config.GpuConfig.EnableAmdGpu = to.Ptr(true)
+				config.GpuConfig.GpuInstanceProfile = ""
+				config.VmSize = "Standard_ND96isr_MI300X_v5"
+			},
+			validator: func(cmd *exec.Cmd) {
+				vars := environToMap(cmd.Env)
+				assert.Equal(t, "true", vars["AMD_GPU_NODE"])
+				assert.Equal(t, "false", vars["GPU_NODE"])
+				assert.Equal(t, "false", vars["GPU_NEEDS_FABRIC_MANAGER"])
+				containerdConfig, err := getBase64DecodedValue([]byte(vars["CONTAINERD_CONFIG_NO_GPU_CONTENT"]))
+				require.NoError(t, err)
+				assert.NotContains(t, containerdConfig, "nvidia")
+				assert.Contains(t, containerdConfig, `default_runtime_name = "runc"`)
+			},
+		},
 		{
 			name:       "AKSUbuntu2204 containerd with multi-instance GPU",
 			folder:     "AKSUbuntu2204+Containerd+MIG",
@@ -41,6 +62,7 @@ func TestBuildCSECmd(t *testing.T) {
 				vars := environToMap(cmd.Env)
 				assertHasKeyWithValue(t, vars, "LOCATION", "southcentralus")
 				assert.Equal(t, "false", vars["GPU_NODE"])
+				assert.Equal(t, "false", vars["AMD_GPU_NODE"])
 				assertHasKeyWithValue(t, vars, "MIG_NODE", "true")
 				assertHasKeyWithValue(t, vars, "GPU_INSTANCE_PROFILE", "MIG7g")
 				assertHasKeyWithValue(t, vars, "NVIDIA_MIG_PROFILE_LAYOUT", "")