diff --git a/.github/renovate.json b/.github/renovate.json index 3d94540846d..1d140dfaa35 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -737,6 +737,30 @@ "rashmichandrashekar", "bragi92" ] + }, + { + "description": "AMD image updates always require coordinated pin review and fresh MI300X qualification", + "matchFileNames": [ + "vhdbuilder/packer/amd-gpu-components.json" + ], + "matchManagers": [ + "custom.regex" + ], + "enabled": true, + "automerge": false, + "ignoreUnstable": false, + "groupName": "amd-gpu-image", + "separateMajorMinor": false, + "separateMinorPatch": false, + "assignees": [ + "team:aks-node-lifecycle" + ], + "reviewers": [ + "team:aks-node-lifecycle" + ], + "prBodyNotes": [ + "Update AMD repository, driver, firmware, module and DKMS pins together; AMD CUE validation rejects mismatches. Rebuild the separate AMD VHD and qualify it on MI300X before merging." + ] } ], "customManagers": [ @@ -964,6 +988,78 @@ "matchStrings": [ "#\\s*renovate:\\s*datasource=(?\\S+)\\s+depName=(?\\S+)(?:\\s+versioning=(?\\S+))?(?:\\s+registryUrl=(?\\S+))?\\s*\\n[^\\n]*VERSION=\"(?[^\"]+)\"" ] + }, + { + "customType": "regex", + "description": "Track amdgpu-dkms in the isolated AMD VHD manifest", + "managerFilePatterns": [ + "/^vhdbuilder/packer/amd-gpu-components\\.json$/" + ], + "matchStrings": [ + "\"AMDGPUDriver\":\\s*\\{[^}]*?\"repositoryURL\":\\s*\"(?https://repo\\.radeon\\.com/amdgpu/[0-9.]+/ubuntu)\"[^}]*\\}", + "\"packageVersion\":\\s*\"(?[^\"]+)\"" + ], + "depNameTemplate": "amdgpu-dkms", + "datasourceTemplate": "deb", + "versioningTemplate": "deb", + "registryUrlTemplate": "{{{amdRepository}}}?suite=noble&components=main&binaryArch=amd64", + "matchStringsStrategy": "recursive" + }, + { + "customType": "regex", + "description": "Track amdgpu-dkms-firmware in the isolated AMD VHD manifest", + "managerFilePatterns": [ + "/^vhdbuilder/packer/amd-gpu-components\\.json$/" + ], + "matchStrings": [ + "\"AMDGPUDriver\":\\s*\\{[^}]*?\"repositoryURL\":\\s*\"(?https://repo\\.radeon\\.com/amdgpu/[0-9.]+/ubuntu)\"[^}]*\\}", + "\"firmwarePackageVersion\":\\s*\"(?[^\"]+)\"" + ], + "depNameTemplate": "amdgpu-dkms-firmware", + "datasourceTemplate": "deb", + "versioningTemplate": "deb", + "registryUrlTemplate": "{{{amdRepository}}}?suite=noble&components=main&binaryArch=amd64", + "matchStringsStrategy": "recursive" + }, + { + "customType": "regex", + "description": "Track AMD amdsmi in the isolated AMD VHD manifest", + "managerFilePatterns": [ + "/^vhdbuilder/packer/amd-gpu-components\\.json$/" + ], + "matchStrings": [ + "\"amdsmiPackage\":\\s*\"(?amdrocm-amdsmi[0-9]+\\.[0-9]+)\"[^}]*?\"amdsmiVersion\":\\s*\"(?[^\"]+)\"" + ], + "datasourceTemplate": "deb", + "versioningTemplate": "deb", + "registryUrlTemplate": "https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/?suite=stable&components=main&binaryArch=amd64" + }, + { + "customType": "regex", + "description": "Track AMD sysdeps in the isolated AMD VHD manifest", + "managerFilePatterns": [ + "/^vhdbuilder/packer/amd-gpu-components\\.json$/" + ], + "matchStrings": [ + "\"sysdepsPackage\":\\s*\"(?amdrocm-sysdeps[0-9]+\\.[0-9]+)\"[^}]*?\"sysdepsVersion\":\\s*\"(?[^\"]+)\"" + ], + "datasourceTemplate": "deb", + "versioningTemplate": "deb", + "registryUrlTemplate": "https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/?suite=stable&components=main&binaryArch=amd64" + }, + { + "customType": "regex", + "description": "Detect new AMD driver release directories; coordinated pins require review", + "managerFilePatterns": [ + "/^vhdbuilder/packer/amd-gpu-components\\.json$/" + ], + "matchStrings": [ + "\"repositoryURL\":\\s*\"https://repo\\.radeon\\.com/amdgpu/(?[0-9.]+)/ubuntu\"" + ], + "depNameTemplate": "amdgpu-repository", + "datasourceTemplate": "custom.amdgpu-releases", + "versioningTemplate": "loose", + "extractVersionTemplate": "^(?[0-9]+(?:\\.[0-9]+)+)/$" } ], "customDatasources": { @@ -1022,6 +1118,10 @@ "transformTemplates": [ "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}" ] + }, + "amdgpu-releases": { + "defaultRegistryUrlTemplate": "https://repo.radeon.com/amdgpu/", + "format": "html" } } } diff --git a/.github/workflows/validate-components.yml b/.github/workflows/validate-components.yml index 3993267b126..f5eedaf00d0 100644 --- a/.github/workflows/validate-components.yml +++ b/.github/workflows/validate-components.yml @@ -2,6 +2,22 @@ name: Validate Components on: pull_request jobs: + amd-components: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: '1.26' + - name: Validate dedicated AMD manifest and coordinated package pins + run: | + go install cuelang.org/go/cmd/cue@v0.15.4 + "$(go env GOPATH)/bin/cue" vet -c ./schemas/amd-gpu-components.cue ./vhdbuilder/packer/amd-gpu-components.json + cue: runs-on: ubuntu-latest permissions: diff --git a/.pipelines/.vsts-vhd-builder-amd.yaml b/.pipelines/.vsts-vhd-builder-amd.yaml new file mode 100644 index 00000000000..7eddcf251b1 --- /dev/null +++ b/.pipelines/.vsts-vhd-builder-amd.yaml @@ -0,0 +1,83 @@ +# Register this manual pipeline with the existing nonproduction Linux VHD +# builder's pool, service connection and build-environment variables. +# Its single image job is independent of the shared PR and release pipelines. +name: $(Date:yyyyMMdd)$(Rev:.r)_AMD_$(BuildID) +trigger: none +pr: none + +parameters: + - name: replicationRegions + displayName: Capture replication targets (region=replicas, comma separated) + type: string + default: francecentral=1 + +pool: + name: $(POOL_NAME) + +variables: + - name: MODE + value: linuxVhdMode + - group: aks-vuln-to-kusto + - group: "AKS Node SIG UA Token (KV)" + - group: build_performance + - group: aks-vuln-to-kusto-tme + - name: BUILDER + value: packer + # Capture a test image and retain it for explicit MI300X qualification. + # Publishing/export and shared-gallery garbage collection are not part of this pipeline. + - name: GENERATE_PUBLISHING_INFO + value: 'False' + - name: RUN_PREFETCH_OPTIMIZATION + value: 'False' + - name: DRY_RUN + value: 'False' + - name: SKIP_CLEANUP + # Shared cleanup includes subscription-wide garbage collection. The final + # step below removes only this build's recorded and ownership-verified group. + value: 'True' + - name: REPLICATIONS + value: ${{ parameters.replicationRegions }} + +stages: + - stage: build_amd + displayName: Build dedicated AMD GPU VHD + jobs: + - job: build2404amdgpugen2containerd + timeoutInMinutes: 360 + steps: + - bash: | + echo '##vso[task.setvariable variable=OS_SKU]Ubuntu' + echo '##vso[task.setvariable variable=OS_VERSION]24.04' + echo '##vso[task.setvariable variable=IMG_PUBLISHER]Canonical' + echo '##vso[task.setvariable variable=IMG_OFFER]ubuntu-24_04-lts' + echo '##vso[task.setvariable variable=IMG_SKU]server' + echo '##vso[task.setvariable variable=IMG_VERSION]latest' + echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' + echo '##vso[task.setvariable variable=FEATURE_FLAGS]AMD_GPU' + echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' + echo '##vso[task.setvariable variable=ENABLE_FIPS]False' + echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' + echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]False' + displayName: Setup AMD Build Variables + - template: ./templates/.builder-release-template.yaml + parameters: + artifactName: 2404-amdgpu-gen2-containerd + # Retries overwrite the Packer output used for scoped cleanup. + packerBuildRetryCount: 0 + - task: AzureCLI@2 + displayName: Clean up this AMD build's Packer resource group + condition: always() + inputs: + azureSubscription: $(VHD_ARM_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -euo pipefail + source vhdbuilder/packer/cleanup-amd-gpu-build.sh + cleanup_amd_gpu_build_resource_group \ + vhdbuilder/packer/settings.json \ + "${PACKER_RESOURCE_GROUP}" "${AMD_BUILD_ID}" + env: + PACKER_RESOURCE_GROUP: $(PKR_RG_NAME) + AMD_BUILD_ID: $(Build.BuildId) diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index be55156ce30..7a0d8365de0 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -8,6 +8,9 @@ parameters: - name: overrideBranch type: string default: master + - name: packerBuildRetryCount + type: number + default: 3 steps: - template: ./.template-override-components-json.yaml @@ -74,18 +77,30 @@ steps: displayName: Setup Cue - bash: | - SKU_NAME=${OS_VERSION} && \ - if grep -q "minimal" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}minimal"; fi && \ - if [[ "${HYPERV_GENERATION,,}" == "v2" ]]; then SKU_NAME="${SKU_NAME}gen2"; fi && \ - if [[ "${ARCHITECTURE,,}" == "arm64" ]]; then SKU_NAME="${SKU_NAME}arm64"; fi && \ - if grep -q "NVIDIA_GB" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}gb"; fi && \ - if [[ "${ENABLE_FIPS,,}" == "true" ]]; then SKU_NAME="${SKU_NAME}fips"; fi && \ - if grep -q "cvm" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}CVM"; fi && \ - if [[ "${ENABLE_TRUSTED_LAUNCH}" == "True" ]]; then SKU_NAME="${SKU_NAME}TL"; fi && \ - if [[ ${OS_SKU} != "CBLMariner" && ${OS_SKU} != "AzureLinux" && ${OS_SKU} != "AzureLinuxOSGuard" && ${OS_SKU} != "Flatcar" && ${OS_SKU} != "AzureContainerLinux" ]]; then SKU_NAME="${SKU_NAME}containerd"; fi && \ - SKU_NAME=$(echo ${SKU_NAME} | tr -d '.') && \ - echo "##vso[task.setvariable variable=SKU_NAME]$SKU_NAME" - echo "Set SKU_NAME to $SKU_NAME" + if [[ "${FEATURE_FLAGS}" == *AMD_GPU* ]]; then + "$(go env GOPATH)/bin/cue" vet -c ./schemas/amd-gpu-components.cue ./vhdbuilder/packer/amd-gpu-components.json || exit 1 + source vhdbuilder/packer/amd-gpu-build-settings.sh + SKU_NAME=$(get_amd_gpu_sku_name) || exit 1 + # Prepare only this AMD worker checkout after the template's checkout step. + prepare_amd_gpu_packer_template \ + vhdbuilder/packer/vhd-image-builder-base.json \ + vhdbuilder/packer/amd-gpu-packer-files.json || exit 1 + echo "##vso[task.setvariable variable=SKU_NAME]$SKU_NAME" + echo "Set SKU_NAME to $SKU_NAME" + else + SKU_NAME=${OS_VERSION} && \ + if grep -q "minimal" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}minimal"; fi && \ + if [[ "${HYPERV_GENERATION,,}" == "v2" ]]; then SKU_NAME="${SKU_NAME}gen2"; fi && \ + if [[ "${ARCHITECTURE,,}" == "arm64" ]]; then SKU_NAME="${SKU_NAME}arm64"; fi && \ + if grep -q "NVIDIA_GB" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}gb"; fi && \ + if [[ "${ENABLE_FIPS,,}" == "true" ]]; then SKU_NAME="${SKU_NAME}fips"; fi && \ + if grep -q "cvm" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}CVM"; fi && \ + if [[ "${ENABLE_TRUSTED_LAUNCH}" == "True" ]]; then SKU_NAME="${SKU_NAME}TL"; fi && \ + if [[ ${OS_SKU} != "CBLMariner" && ${OS_SKU} != "AzureLinux" && ${OS_SKU} != "AzureLinuxOSGuard" && ${OS_SKU} != "Flatcar" && ${OS_SKU} != "AzureContainerLinux" ]]; then SKU_NAME="${SKU_NAME}containerd"; fi && \ + SKU_NAME=$(echo ${SKU_NAME} | tr -d '.') && \ + echo "##vso[task.setvariable variable=SKU_NAME]$SKU_NAME" + echo "Set SKU_NAME to $SKU_NAME" + fi displayName: Set SKU Name - task: AzureCLI@2 @@ -97,7 +112,7 @@ steps: make -f packer.mk run-packer condition: ne(variables['BUILDER'], 'imagecustomizer') displayName: Build VHD - retryCountOnTaskFailure: 3 + retryCountOnTaskFailure: ${{ parameters.packerBuildRetryCount }} env: OS_TYPE: Linux GIT_VERSION: $(Build.SourceVersion) @@ -391,7 +406,7 @@ steps: TargetFolder: '$(Build.ArtifactStagingDirectory)' - task: CopyFiles@2 - condition: and(eq(variables.OS_SKU, 'Ubuntu'), in(variables.OS_VERSION, '22.04', '24.04'), in(variables.FEATURE_FLAGS, 'None', 'cvm', 'NVIDIA_GB', 'NVIDIA_CUDA_PREBAKE')) + condition: and(eq(variables.OS_SKU, 'Ubuntu'), in(variables.OS_VERSION, '22.04', '24.04'), in(variables.FEATURE_FLAGS, 'None', 'cvm', 'NVIDIA_GB', 'NVIDIA_CUDA_PREBAKE', 'AMD_GPU')) displayName: Copy CIS Reports inputs: SourceFolder: '$(System.DefaultWorkingDirectory)' diff --git a/Makefile b/Makefile index deb1212a203..247a225a95e 100644 --- a/Makefile +++ b/Makefile @@ -265,6 +265,10 @@ unit-tests: update-windows-base-versions: ./vhdbuilder/packer/windows/update_windows_base_versions.sh +.PHONY: validate-amd-components +validate-amd-components: + @./hack/tools/bin/cue vet -c ./schemas/amd-gpu-components.cue ./vhdbuilder/packer/amd-gpu-components.json + .PHONY: validate-components validate-components: @./hack/tools/bin/cue vet -c ./schemas/components.cue ./parts/common/components.json diff --git a/aks-node-controller/parser/parser.go b/aks-node-controller/parser/parser.go index cd8124e01f5..53371808cb9 100644 --- a/aks-node-controller/parser/parser.go +++ b/aks-node-controller/parser/parser.go @@ -96,6 +96,7 @@ func getCSEEnv(ctx context.Context, config *aksnodeconfigv1.Configuration, gpuCo "API_SERVER_NAME": config.GetApiServerConfig().GetApiServerName(), "IS_VHD": fmt.Sprintf("%v", getIsVHD(config.IsVhd)), "GPU_NODE": fmt.Sprintf("%v", getEnableNvidia(config)), + "AMD_GPU_NODE": fmt.Sprintf("%v", config.GetGpuConfig().GetEnableAmdGpu()), "SGX_NODE": fmt.Sprintf("%v", getIsSgxEnabledSKU(config.GetVmSize())), "MIG_NODE": fmt.Sprintf("%v", isMIGNode), "CONFIG_GPU_DRIVER_IF_NEEDED": fmt.Sprintf("%v", config.GetGpuConfig().GetConfigGpuDriver()), diff --git a/aks-node-controller/parser/parser_test.go b/aks-node-controller/parser/parser_test.go index 9a2341f67ed..0ea79adc825 100644 --- a/aks-node-controller/parser/parser_test.go +++ b/aks-node-controller/parser/parser_test.go @@ -26,6 +26,27 @@ func TestBuildCSECmd(t *testing.T) { aksNodeConfigUpdator func(*aksnodeconfigv1.Configuration) validator func(cmd *exec.Cmd) }{ + { + name: "AMD GPU uses the baked driver and the standard container runtime", + folder: "AKSUbuntu2204+Containerd+MIG", + k8sVersion: "1.36.3", + aksNodeConfigUpdator: func(config *aksnodeconfigv1.Configuration) { + config.GpuConfig.EnableNvidia = to.Ptr(false) + config.GpuConfig.EnableAmdGpu = to.Ptr(true) + config.GpuConfig.GpuInstanceProfile = "" + config.VmSize = "Standard_ND96isr_MI300X_v5" + }, + validator: func(cmd *exec.Cmd) { + vars := environToMap(cmd.Env) + assert.Equal(t, "true", vars["AMD_GPU_NODE"]) + assert.Equal(t, "false", vars["GPU_NODE"]) + assert.Equal(t, "false", vars["GPU_NEEDS_FABRIC_MANAGER"]) + containerdConfig, err := getBase64DecodedValue([]byte(vars["CONTAINERD_CONFIG_NO_GPU_CONTENT"])) + require.NoError(t, err) + assert.NotContains(t, containerdConfig, "nvidia") + assert.Contains(t, containerdConfig, `default_runtime_name = "runc"`) + }, + }, { name: "AKSUbuntu2204 containerd with multi-instance GPU", folder: "AKSUbuntu2204+Containerd+MIG", @@ -41,6 +62,7 @@ func TestBuildCSECmd(t *testing.T) { vars := environToMap(cmd.Env) assertHasKeyWithValue(t, vars, "LOCATION", "southcentralus") assert.Equal(t, "false", vars["GPU_NODE"]) + assert.Equal(t, "false", vars["AMD_GPU_NODE"]) assertHasKeyWithValue(t, vars, "MIG_NODE", "true") assertHasKeyWithValue(t, vars, "GPU_INSTANCE_PROFILE", "MIG7g") assertHasKeyWithValue(t, vars, "NVIDIA_MIG_PROFILE_LAYOUT", "") diff --git a/docs/amd-gpu-vhd.md b/docs/amd-gpu-vhd.md new file mode 100644 index 00000000000..69fa4fcb016 --- /dev/null +++ b/docs/amd-gpu-vhd.md @@ -0,0 +1,229 @@ +# Dedicated AMD GPU VHD + +The `2404gen2amdgpucontainerd` image is an opt-in Ubuntu 24.04, x86-64, +Generation 2 image for `Standard_ND96isr_MI300X_v5` and +`Standard_ND96is_MI300X_v5`. Its build flag is `AMD_GPU` and its pipeline +artifact is `2404-amdgpu-gen2-containerd`. FIPS, Trusted Launch, confidential +VMs, ARM64, and combinations with NVIDIA image flags are rejected. + +## Host and container boundary + +The host contains the official AMDGPU DKMS driver, firmware, AMD SMI diagnostics, +matching kernel headers, and build prerequisites. Exact AMD package versions, +module version, repository, and signing-key fingerprint live in +`vhdbuilder/packer/amd-gpu-components.json`, validated separately by +`schemas/amd-gpu-components.cue`. The ordinary component manifest is unchanged. +The build authenticates AMD repository metadata, installs only the selected +driver, firmware, SMI and SMI sysdeps packages, builds the module for the image +kernel, and removes its temporary repository and package-download state. It writes +`/opt/azure/amd-gpu/driver.json` only after successful verification. + +The ROCm compute runtime, HIP/RCCL, PyTorch, math libraries, compilers for AI workloads, and heavier +tools such as `rocminfo` belong in containers. Workloads use the normal `runc` +container runtime; the AMD Kubernetes device plugin advertises `amd.com/gpu` and supplies access to +the kernel devices. No NVIDIA container runtime is required. The AMD image +does not cache NVIDIA driver or plugin payloads. + +Host operators can run `amd-smi` directly through `/usr/local/bin/amd-smi`. +The image pins `amdrocm-amdsmi10.0` and `amdrocm-sysdeps10.0`, both `10.0.0-4`, +in `AMDGPUDiagnostics`. The installer verifies the separate AMD ROCm repository +key and signed metadata, then installs just these packages with Ubuntu Python +and C++ runtime dependencies. It does not install the ROCm compute runtime, +SDK, or Python packages from pip. `pciutils` (`lspci`) and `numactl` are retained +for PCI and NUMA diagnosis. + +Read-only examples on a GPU node (the default SSH user needs `sudo` for AMD +SMI hardware access): + +```bash +sudo amd-smi version +sudo amd-smi list --json +sudo amd-smi static --json +sudo amd-smi metric --json +lspci -nn +numactl --hardware +``` + +AMD SMI initializes hardware even for `--help`, so CPU VHD builds validate its +Python bindings and management library without initializing GPUs. GPU E2E +checks require its CLI to discover eight GPUs matching kernel PCI addresses. +Some telemetry fields are unavailable (`N/A`) on MI300X virtual functions. + +Keep DKMS sources, the host compiler, and matching headers installed: kernel +updates need them to rebuild the driver. Do not delete these files just to +reduce the image size. A future, smaller alternative is an AKS-built signed +module package for each exact supported kernel ABI. That requires coordinated +kernel/module updates and qualification; a module copied from a different +kernel is not sufficient. + +## Measured package footprint + +Official package metadata was measured on an Ubuntu 24.04 MI300X test VM on +2026-09-19. Decimal MB/GB are used. Installed sizes below are package metadata, +not compressed VHD sizes. + +| Payload | Download | Installed | +| --- | ---: | ---: | +| AMDGPU 31.50 driver and firmware | 29.84 MB | 671.67 MB | +| ROCm 10 gfx942 SDK plus runtime development package closure | 2.198 GB | 9.626 GB | +| Included AMD SMI and its AMD sysdeps package | 18.18 MB | 100.38 MB | + +The driver also requires Ubuntu build/kernel dependencies and creates module +and initramfs files. The measured driver/build dependency closure was 1.287 GB, +including packages already present in Ubuntu; this is not an incremental VHD +size estimate. The full SDK package set is 9.626 GB; the host includes only +its 100.38 MB AMD SMI diagnostics subset. +Driver, firmware and SMI together account for 772.05 MB of declared AMD package +contents, before Ubuntu dependencies and generated module/initramfs files. +Even ROCm 10's base/runtime packages pull LLVM; they are unnecessary for AMD SMI. + +The full ROCm PyTorch development image tested separately has 20.455 GB of +compressed OCI layers. Keeping that image out of the VHD does not make its +first pull smaller. For production workloads, use a separately qualified +runtime image containing only the required framework/runtime libraries and +GPU architecture, and keep compilers and build caches in a separate build +stage. Measure that image's first-pull time and digest before adopting it. + +## Provisioning and image selection + +Legacy provisioning sets `NodeBootstrappingConfiguration.EnableAMDGPU=true`; +ANC sets `GpuConfig.enable_amd_gpu=true`. Leave NVIDIA enablement false. AMD +driver configuration also requires `ConfigGPUDriverIfNeeded=true` (ANC: +`GpuConfig.config_gpu_driver=true`), preserving an explicit driver opt-out. +Validation runs in `nodePrep`, including when a PIS image skips `basePrep`. +It verifies installed package versions, the DKMS module for the running kernel, +the loaded module version, `/dev/kfd`, and eight active KFD GPUs. Preallocated +DRM render nodes are not counted as GPUs. Missing or incompatible drivers fail +provisioning; there is no vendor download or compile fallback on node startup. + +Build this flavor with the separate manual pipeline +`.pipelines/.vsts-vhd-builder-amd.yaml` (`trigger: none`, `pr: none`). Register it +with the existing nonproduction VHD builder pool, service connection, and +build-environment variables. It captures only `2404gen2amdgpucontainerd` and +replicates to France Central by default; it does not publish a production image. +The normal PR and release build matrices are unchanged and have no dependency +on the AMD pipeline. A failed AMD bake fails only that separate run. + +AMD installers, package metadata, content checks, and bootstrap validation live +in AMD-specific files. Shared scripts load them only behind `AMD_GPU` or +`AMD_GPU_NODE` guards. Node provisioning sources the validator baked into this +image, so normal VHDs do not execute or depend on AMD package installation. +Existing CPU/NVIDIA SKU naming and image-selection defaults remain unchanged. + +Use the captured AMD gallery image explicitly for testing. Its existing Ubuntu +24.04 Gen2 `Distro` describes OS behavior; its distinct gallery image name +selects the driver payload. Do not point the production SIG map at an image +before that image has been published and qualified. + +The AMD flags configure bootstrap validation; they do not select an image. +`GetNodeBootstrapping` and `GetLatestSigImageConfig` still return ordinary Ubuntu +image metadata for the existing `Distro`. An opt-in caller must set the VM or +VMSS `ImageReference.ID` to the captured AMD gallery image version explicitly, +as the E2E scenario does, instead of using the returned ordinary +`SigImageConfig` for image selection. + +Native AKS node-pool availability also requires service-side work: permit the +supported AMD SKUs/driver policy, select this separate gallery image, propagate +AMD enablement to both provisioning contracts, and arrange the AMD device +plugin. This AgentBaker change alone does not enable a public AKS SKU or change +the AKS resource provider's existing AMD driver-policy validation. + +## Qualification + +On 2026-09-19, dedicated AMD build `181874913` +successfully captured and tested `2404gen2amdgpucontainerd/1.1789808022.4162` +and completed France Central replication. Its image inputs match PR commit +`3ce6d173`; validation commit `72e925a9` changes only the pipeline entrypoint. +The E2E runner includes the AMD test invocation fix in `4de7f56d`; that change +does not alter the captured VHD payload. + +A fresh MI300X node booted this exact image, joined AKS, and passed +`Ubuntu2404_MI300X_AMDGPU` with kernel `6.8.0-1067-azure` and AMDGPU module +`7.1.3.31500000`. AMD SMI and the device plugin discovered eight GPUs. Using +the [scenario's digest-pinned images](../e2e/scenario/scenario_gpu_amd.go), +PyTorch 2.13.0 / ROCm 10.0 completed 40 FP32 training steps per GPU, checking +outputs, loss, gradients, and updated parameters against a CPU reference. + +Additional checks passed before and after restarting that same instance. A +changed boot ID and node readiness were required before the second round: + +- Eight-rank all-to-all transfers checked every received element at + 64/80/128/256 MiB per peer: 42,631 measured calls across both rounds, 291.16 TB + received including 254.77 TB between GPUs, and zero mismatched elements. + Independent calculation verified complete timing samples and byte totals. +- A 49,016,832-parameter Transformer completed 100 BF16 DDP/AdamW steps across + eight GPUs in each round, processing 1,638,400 synthetic tokens per round. + Mean loss decreased from 6.519 to 0.000249; losses, gradients, and weights + remained finite, and all eight ranks produced the same full-model SHA-256. +- The 40-step CPU-reference training check passed again on all eight GPUs + after reboot. +- Host driver, AMD SMI, and runtime health checks passed before and after these + workloads; the host had no ROCm compute SDK installed. + +Remote-send bandwidth per GPU (decimal GB/s): + +| MiB per peer | Median before reboot | Median after reboot | Effective before | Effective after | +| --- | ---: | ---: | ---: | ---: | +| 64 | 278.7 | 278.8 | 269.9 | 272.6 | +| 80 | 281.9 | 282.0 | 281.2 | 277.7 | +| 128 | 286.5 | 286.5 | 286.3 | 286.3 | +| 256 | 291.1 | 291.4 | 289.6 | 290.5 | + +Rates use decimal GB/s and seven remote sends per GPU divided by the slowest +rank's call time, without send/receive double-counting. Effective rates include +all timed calls, including outliers; setup and data comparison are outside the +timed calls. These bounded tests do not establish peak throughput, model +quality, or integrity of untested transfers and network/storage paths. + +After capturing the image and replicating it to `francecentral`, use the +repository's normal E2E Azure configuration and its captured build metadata: + +The test nodes need outbound HTTPS access to the pinned images on Docker Hub. +The standard E2E firewall does not include this access. For an isolated AMD lab, +scope an additional rule to its GPU nodes for `registry-1.docker.io`, +`auth.docker.io`, and the image-layer CDN. The current image digests were served +by `production.cloudfront.docker.com`; recheck CDN destinations when updating +them. Remove temporary rules with the lab resources. A recurring AMD pipeline +can instead mirror the exact digests into an approved reachable registry and +verify the copied manifests and layers before use. + +```bash +cd e2e +AGENTBAKER_E2E_ENABLE_MI300X=true ./e2e-local.sh \ + --subscription-id "$SUBSCRIPTION_ID" \ + --vm-sku Standard_ND96isr_MI300X_v5 \ + --vhd-metadata-file /absolute/path/to/vhd-build-metadata.json \ + --parallel 1 --disable-scriptless=false \ + --disable-scriptless-compilation=true \ + --ignore-missing-vhd=false --skip-capacity-errors=false \ + --tags '' --skip-tags '' \ + Ubuntu2404_MI300X_AMDGPU +``` + +The metadata must contain `2404gen2amdgpucontainerd`, its real image-version +resource ID, and the replication region. This scenario allocates MI300X +capacity and is skipped unless explicitly enabled. The existing `--vm-sku` +option sets the tested node size before capability queries; the scenario keeps +the shared AKS system pool on the ordinary CPU SKU. It validates the driver, +deploys the digest-pinned AMD device plugin, requires eight advertised GPUs, +and requires completed reference-checked training. A managed 256 GiB OS disk +provides room to unpack the development workload image. The scenario does not +yet automate reboot, serviced-kernel, or large all-to-all qualification. + +The command above uses scriptless NBC provisioning with the scripts and ANC +binary baked into the image. It avoids embedding the full legacy script payload +in Azure custom data, which can exceed Azure's size limit. This path still +executes an NBC command; it does not independently qualify native ANC JSON +provisioning. AMD ANC environment generation is covered by parser tests, but +native ANC boot requires separate qualification. + +AgentBaker's Ubuntu 24.04 build currently uses the 6.8 Azure LTS kernel policy. +Native ANC JSON and legacy provisioning, PIS, and a serviced-kernel update with +DKMS rebuild and subsequent GPU workloads require separate qualification. +Record the image version, kernel, driver, +workload image digests, and results for each path. + +Production release additionally requires vendor driver security qualification. +A successful workload run is not evidence that all driver security issues are +resolved. Secure Boot support needs a separately qualified signing and trust +chain; this initial flavor does not enable it. diff --git a/e2e/config/vhd_amd.go b/e2e/config/vhd_amd.go new file mode 100644 index 00000000000..bf4459cdc32 --- /dev/null +++ b/e2e/config/vhd_amd.go @@ -0,0 +1,16 @@ +package config + +import "github.com/Azure/agentbaker/pkg/agent/datamodel" + +var ( + // The dedicated AMD image is selected only by opt-in GPU scenarios. It uses + // the existing Ubuntu distro and does not change production SIG defaults. + VHDUbuntu2404Gen2AMDGPUContainerd = &Image{ + Name: "2404gen2amdgpucontainerd", + OS: OSUbuntu, + Arch: "amd64", + Distro: datamodel.AKSUbuntuContainerd2404Gen2, + Gallery: &Config.GalleryLinux, + OSDiskSizeGB: 256, // The on-demand PyTorch image is 20.46 GB compressed. + } +) diff --git a/e2e/scenario/fixtures/amd_gpu_host_check.py b/e2e/scenario/fixtures/amd_gpu_host_check.py new file mode 100644 index 00000000000..9c48126edc4 --- /dev/null +++ b/e2e/scenario/fixtures/amd_gpu_host_check.py @@ -0,0 +1,74 @@ +"""Read-only validation of the baked driver, AMD SMI, and MI300X devices.""" + +import base64 +import json +from pathlib import Path +import os +import platform +import re +import subprocess +import sys + + +def command(*args): + return subprocess.check_output(args, text=True).strip() + + +manifest = json.loads(base64.b64decode(sys.argv[1])) +expected = manifest["AMDGPUDriver"] +diagnostics = manifest["AMDGPUDiagnostics"] +marker = json.loads(Path("/opt/azure/amd-gpu/driver.json").read_text()) +assert marker["schema_version"] == 1, marker +for marker_key, manifest_key in (("package_version", "packageVersion"), + ("firmware_package_version", "firmwarePackageVersion"), + ("module_version", "moduleVersion")): + assert marker[marker_key] == expected[manifest_key], (marker_key, marker) +for package, key in (("amdgpu-dkms", "packageVersion"), + ("amdgpu-dkms-firmware", "firmwarePackageVersion")): + assert command("dpkg-query", "-W", "-f=${Status} ${Version}", package) == f"install ok installed {expected[key]}", package +for package_key, version_key in (("amdsmiPackage", "amdsmiVersion"), + ("sysdepsPackage", "sysdepsVersion")): + package = diagnostics[package_key] + assert command("dpkg-query", "-W", "-f=${Status} ${Version}", package) == f"install ok installed {diagnostics[version_key]}", package +cli = Path("/usr/local/bin/amd-smi") +assert cli.is_symlink() and cli.resolve() == Path(diagnostics["cliPath"]).resolve(), cli +assert os.access(cli, os.X_OK), cli +kernel = platform.release() +module_path = command("modinfo", "-F", "filename", "amdgpu") +assert module_path.startswith(f"/lib/modules/{kernel}/updates/dkms/"), module_path +assert command("modinfo", "-F", "version", "amdgpu") == expected["moduleVersion"] +assert Path("/sys/module/amdgpu/version").read_text().strip() == expected["moduleVersion"] +assert command("modinfo", "-F", "vermagic", "amdgpu").split()[0] == kernel +assert Path("/sys/module/amdgpu/srcversion").read_text().strip() == command("modinfo", "-F", "srcversion", "amdgpu") +assert f"amdgpu/{expected['dkmsVersion']}, {kernel}, x86_64: installed" in command("dkms", "status") +assert Path("/dev/kfd").is_char_device(), "/dev/kfd missing" +devices = [] +for device in Path("/sys/bus/pci/devices").iterdir(): + if (device / "vendor").read_text().strip() == "0x1002" and (device / "device").read_text().strip() == "0x74b5": + assert (device / "driver").resolve().name == "amdgpu", device + devices.append(device.name) +assert len(devices) == 8, devices +amdsmi_version = json.loads(command(str(cli), "version", "--json")) +assert isinstance(amdsmi_version, list) and len(amdsmi_version) == 1, amdsmi_version +version = amdsmi_version[0] +assert version["tool"] == "AMDSMI Tool" and version["version"] and version["amdsmi_library_version"], version +assert version["amdgpu_version"] == expected["moduleVersion"], version +amdsmi_devices = json.loads(command(str(cli), "list", "--json")) +assert isinstance(amdsmi_devices, list) and len(amdsmi_devices) == 8, amdsmi_devices +assert {device["gpu"] for device in amdsmi_devices} == set(range(8)), amdsmi_devices +assert {device["bdf"].lower() for device in amdsmi_devices} == set(devices), amdsmi_devices +packages = command("dpkg-query", "-W", "-f=${Package}\\t${Status}\\n") +diagnostics_packages = {diagnostics["amdsmiPackage"], diagnostics["sysdepsPackage"]} +unexpected_packages = [] +for line in packages.splitlines(): + package, status = line.split("\t", 1) + package = package.split(":", 1)[0] + if status == "install ok installed" and package not in diagnostics_packages and re.match( + r"^(amdrocm|rocm|hip|hsa-rocr|rocblas|rocfft|rocrand|rocsolver|rocsparse|miopen|migraphx|" + r"amdgpu-(core|lib|install|pro)|lib.*-amdgpu-|nvidia-|libnvidia-|cuda-|" + r"datacenter-gpu-manager-|dcgm-exporter)", package): + unexpected_packages.append(package) +assert not unexpected_packages, f"Unexpected GPU userspace packages: {unexpected_packages}" +print(json.dumps({"kernel": kernel, "module": expected["moduleVersion"], "gpu_functions": devices, + "baked_kernel": marker["kernel_version"], "host_rocm_sdk": False, + "amdsmi_version": version, "amdsmi_devices": amdsmi_devices})) diff --git a/e2e/scenario/fixtures/amd_gpu_training.py b/e2e/scenario/fixtures/amd_gpu_training.py new file mode 100644 index 00000000000..3f9881e43fe --- /dev/null +++ b/e2e/scenario/fixtures/amd_gpu_training.py @@ -0,0 +1,76 @@ +"""Small FP32 training/reference check on every GPU in a driver-only VHD. + +The container supplies all ROCm userspace. No host ROCm libraries are mounted. +This is a functional smoke test, not a throughput or model-quality benchmark. +""" + +import copy +import json + +import torch + + +def train_and_compare(device): + torch.manual_seed(20260919) + x = torch.randn(256, 32) + target = x @ torch.randn(32, 16) * 0.1 + reference = torch.nn.Linear(32, 16) + actual = copy.deepcopy(reference).to(device) + reference_optimizer = torch.optim.SGD(reference.parameters(), lr=0.2) + actual_optimizer = torch.optim.SGD(actual.parameters(), lr=0.2) + gpu_x, gpu_target = x.to(device), target.to(device) + losses = [] + for step in range(40): + reference_optimizer.zero_grad(set_to_none=True) + actual_optimizer.zero_grad(set_to_none=True) + expected_output = reference(x) + actual_output = actual(gpu_x) + expected_loss = torch.nn.functional.mse_loss(expected_output, target) + actual_loss = torch.nn.functional.mse_loss(actual_output, gpu_target) + torch.testing.assert_close(actual_output.cpu(), expected_output, rtol=5e-5, atol=5e-6) + torch.testing.assert_close(actual_loss.cpu(), expected_loss, rtol=5e-5, atol=5e-6) + assert torch.isfinite(actual_loss).item(), (device, step, "nonfinite loss") + expected_loss.backward() + actual_loss.backward() + for actual_parameter, expected_parameter in zip(actual.parameters(), reference.parameters()): + torch.testing.assert_close(actual_parameter.grad.cpu(), expected_parameter.grad, + rtol=5e-5, atol=5e-6) + reference_optimizer.step() + actual_optimizer.step() + for actual_parameter, expected_parameter in zip(actual.parameters(), reference.parameters()): + torch.testing.assert_close(actual_parameter.cpu(), expected_parameter, + rtol=5e-5, atol=5e-6) + losses.append(actual_loss.item()) + assert losses[-1] < losses[0] * 0.8, (device, "training did not converge", losses) + return {"gpu": device.index, "steps": len(losses), "initial_loss": losses[0], "final_loss": losses[-1]} + + +def main(): + assert torch.__version__ == "2.13.0+rocm10.0.0", torch.__version__ + assert torch.version.hip == "7.15.26333", torch.version.hip + assert torch.cuda.is_available(), "ROCm GPU unavailable" + assert torch.cuda.device_count() == 8, torch.cuda.device_count() + torch.set_num_threads(2) + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + results = [] + for index in range(8): + properties = torch.cuda.get_device_properties(index) + assert properties.gcnArchName.split(":")[0] == "gfx942", properties + assert "MI300X" in properties.name, properties + for peer in range(8): + if peer != index: + assert torch.cuda.can_device_access_peer(index, peer), (index, peer) + result = train_and_compare(torch.device("cuda", index)) + results.append(result) + print(json.dumps(result), flush=True) + print("AMDGPU_TRAINING_PASS " + json.dumps({ + "torch": torch.__version__, "hip": torch.version.hip, + "gpu_count": len(results), "steps_per_gpu": 40, + "checked": ["outputs", "loss", "gradients", "updated_parameters", "loss_decrease"], + "rtol": 5e-5, "atol": 5e-6, + }), flush=True) + + +if __name__ == "__main__": + main() diff --git a/e2e/scenario/scenario_gpu_amd.go b/e2e/scenario/scenario_gpu_amd.go new file mode 100644 index 00000000000..6bc7620eed5 --- /dev/null +++ b/e2e/scenario/scenario_gpu_amd.go @@ -0,0 +1,309 @@ +package scenario + +import ( + "context" + _ "embed" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "strings" + "time" + + aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1" + "github.com/Azure/agentbaker/e2e/config" + "github.com/Azure/agentbaker/e2e/logging" + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +const ( + amdMI300XVMSize = "Standard_ND96isr_MI300X_v5" + amdMI300XOptIn = "AGENTBAKER_E2E_ENABLE_MI300X" + // Lab-validated upstream images: device plugin 1.31.0.11 and PyTorch 2.13.0 / ROCm 10.0. + amdDevicePluginImage = "rocm/k8s-device-plugin@sha256:e4df5dc9a7fa34e2344852256dcc5762171a6d68f1f9a34026ce26786ae335e2" + amdPyTorchImage = "rocm/pytorch@sha256:bbdaba66029f905321be3bbc95206a2d9a56a2bc5300877d146ad24d40db9a12" + amdTrainingSuccessMarker = "AMDGPU_TRAINING_PASS " +) + +//go:embed fixtures/amd_gpu_training.py +var amdGPUTrainingScript string + +//go:embed fixtures/amd_gpu_host_check.py +var amdGPUHostCheckScript string + +var _ = Register(newUbuntu2404MI300XAMDGPUScenario()) + +func newUbuntu2404MI300XAMDGPUScenario() *Scenario { + return &Scenario{ + Name: "Ubuntu2404_MI300X_AMDGPU", + Description: "Opt-in Ubuntu 24.04 MI300X image with AMDGPU and AMD SMI: eight AMD GPUs, upstream device plugin, and containerized FP32 training checked against a CPU reference", + Tags: Tags{GPU: true}, + Location: "francecentral", + K8sSystemPoolSKU: config.DEFAULT_VMSKU, + SkipIf: func(context.Context) string { + if os.Getenv(amdMI300XOptIn) != "true" { + return "set " + amdMI300XOptIn + "=true after publishing the dedicated VHD and confirming MI300X quota" + } + return "" + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2404Gen2AMDGPUContainerd, + SkipNVMeOSDiskPlacement: true, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = amdMI300XVMSize + nbc.EnableAMDGPU = true + nbc.EnableNvidia = false + nbc.ConfigGPUDriverIfNeeded = true + nbc.EnableGPUDevicePluginIfNeeded = false + }, + BootstrapConfigMutatorWithError: func(context.Context, *Cluster, *datamodel.NodeBootstrappingConfiguration) error { + if config.Config.VMSKU() != amdMI300XVMSize { + return fmt.Errorf("AMD scenario requires --vm-sku %s", amdMI300XVMSize) + } + return nil + }, + AKSNodeConfigMutator: func(_ *Cluster, cfg *aksnodeconfigv1.Configuration) { + cfg.VmSize = amdMI300XVMSize + cfg.GpuConfig.EnableAmdGpu = to.Ptr(true) + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr(amdMI300XVMSize) + // Qualify the driver without depending on the SKU's ephemeral disk placement or size. + osDisk := vmss.Properties.VirtualMachineProfile.StorageProfile.OSDisk + osDisk.DiffDiskSettings = nil + if osDisk.ManagedDisk == nil { + osDisk.ManagedDisk = &armcompute.VirtualMachineScaleSetManagedDiskParameters{} + } + osDisk.ManagedDisk.StorageAccountType = to.Ptr(armcompute.StorageAccountTypesPremiumLRS) + }, + Validator: validateAMDMI300X, + }, + } +} + +func validateAMDMI300X(ctx context.Context, s *Scenario) error { + for _, check := range []struct { + name string + run func(context.Context, *Scenario) error + }{ + {"driver/amd-baked-driver", validateAMDGPUHost}, + {"bootstrap/kubelet-continuity", ValidateKubeletHasNotStopped}, + {"plugin/amd-deploy", deployAMDDevicePluginDaemonset}, + {"plugin/amd-eight-gpus", validateAMDGPUResources}, + {"workload/amd-pytorch-training", validateAMDGPUTraining}, + } { + if err := runGPUCheck(ctx, s, check.name, check.run); err != nil { + return err + } + } + return nil +} + +func validateAMDGPUHost(ctx context.Context, s *Scenario) error { + contents, err := os.ReadFile(repoPath("vhdbuilder/packer/amd-gpu-components.json")) + if err != nil { + return err + } + script, err := amdGPUHostCheckCommand(contents) + if err != nil { + return err + } + result, err := execScriptOnVMForScenarioValidateExitCode(ctx, s, script, 0, "baked AMDGPU driver, AMD SMI diagnostics or MI300X devices did not match") + if err == nil { + logging.Logf(ctx, "AMDGPU host validation: %s", result.stdout) + } + return err +} + +func amdGPUHostCheckCommand(contents []byte) (string, error) { + var manifest struct { + AMDGPUDriver map[string]string + AMDGPUDiagnostics map[string]string + } + if err := json.Unmarshal(contents, &manifest); err != nil { + return "", err + } + for _, key := range []string{"packageVersion", "firmwarePackageVersion", "moduleVersion", "dkmsVersion"} { + if manifest.AMDGPUDriver[key] == "" { + return "", fmt.Errorf("AMDGPUDriver.%s missing from amd-gpu-components.json", key) + } + } + for _, key := range []string{"amdsmiPackage", "amdsmiVersion", "sysdepsPackage", "sysdepsVersion", "cliPath"} { + if manifest.AMDGPUDiagnostics[key] == "" { + return "", fmt.Errorf("AMDGPUDiagnostics.%s missing from amd-gpu-components.json", key) + } + } + encoded, err := json.Marshal(manifest) + if err != nil { + return "", err + } + // AMD SMI needs KFD/render device access; the SSH user is not in the GPU groups. + return "sudo -n python3 - '" + base64.StdEncoding.EncodeToString(encoded) + "' <<'AMDGPU_CHECK'\n" + amdGPUHostCheckScript + "\nAMDGPU_CHECK\n", nil +} + +func amdGPUTolerations() []corev1.Toleration { + return []corev1.Toleration{ + {Key: "CriticalAddonsOnly", Operator: corev1.TolerationOpExists}, + {Key: "amd.com/gpu", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, + } +} + +func amdDevicePluginDaemonset(nodeName string, owner metav1.OwnerReference) *appsv1.DaemonSet { + name := uniqueKubernetesResourceName("amddp-" + nodeName) + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "kube-system", OwnerReferences: []metav1.OwnerReference{owner}}, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"name": name}}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"name": name}}, + Spec: corev1.PodSpec{ + NodeSelector: map[string]string{"kubernetes.io/hostname": nodeName, "kubernetes.io/arch": "amd64"}, + AutomountServiceAccountToken: to.Ptr(false), + PriorityClassName: "system-node-critical", + Tolerations: amdGPUTolerations(), + Containers: []corev1.Container{{ + Name: "device-plugin", Image: amdDevicePluginImage, + SecurityContext: &corev1.SecurityContext{ + Privileged: to.Ptr(true), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m"), corev1.ResourceMemory: resource.MustParse("128Mi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("512Mi")}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "device-plugins", MountPath: "/var/lib/kubelet/device-plugins"}, + {Name: "sys", MountPath: "/sys"}, + }, + }}, + Volumes: []corev1.Volume{ + {Name: "device-plugins", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/kubelet/device-plugins", Type: to.Ptr(corev1.HostPathDirectory)}}}, + {Name: "sys", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys", Type: to.Ptr(corev1.HostPathDirectory)}}}, + }, + }, + }, + }, + } +} + +func deployAMDDevicePluginDaemonset(ctx context.Context, s *Scenario) error { + owner, err := scenarioNodeOwnerReference(ctx, s) + if err != nil { + return err + } + ds := amdDevicePluginDaemonset(s.Runtime.VM.KubeName, owner) + created, err := s.Runtime.Kube.Typed.AppsV1().DaemonSets(ds.Namespace).Create(ctx, ds, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create AMD device plugin: %w", err) + } + s.Cleanup(func(ctx context.Context) error { + err := s.Runtime.Kube.Typed.AppsV1().DaemonSets(created.Namespace).Delete(ctx, created.Name, metav1.DeleteOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + return err + }) + _, err = s.Runtime.Kube.WaitUntilPodRunning(ctx, created.Namespace, "name="+created.Name, "spec.nodeName="+s.Runtime.VM.KubeName) + return err +} + +func validateAMDGPUResources(ctx context.Context, s *Scenario) error { + return wait.PollUntilContextTimeout(ctx, 3*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + node, err := s.Runtime.Kube.Typed.CoreV1().Nodes().Get(ctx, s.Runtime.VM.KubeName, metav1.GetOptions{}) + if err != nil { + return false, err + } + return amdGPUResourcesReady(node), nil + }) +} + +func amdGPUResourcesReady(node *corev1.Node) bool { + capacity, allocatable := node.Status.Capacity["amd.com/gpu"], node.Status.Allocatable["amd.com/gpu"] + return capacity.Value() == 8 && allocatable.Value() == 8 +} + +func amdGPUTrainingPod(nodeName string, owner metav1.OwnerReference) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: uniqueKubernetesResourceName("amd-train-" + nodeName), Namespace: "default", OwnerReferences: []metav1.OwnerReference{owner}}, + Spec: corev1.PodSpec{ + AutomountServiceAccountToken: to.Ptr(false), + RestartPolicy: corev1.RestartPolicyNever, + ActiveDeadlineSeconds: to.Ptr(int64(1800)), + NodeSelector: map[string]string{"kubernetes.io/hostname": nodeName}, + Tolerations: amdGPUTolerations(), + Containers: []corev1.Container{{ + Name: "training", Image: amdPyTorchImage, ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"python3", "-u", "-c", amdGPUTrainingScript}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{"amd.com/gpu": resource.MustParse("8"), corev1.ResourceCPU: resource.MustParse("8"), corev1.ResourceMemory: resource.MustParse("16Gi")}, + Limits: corev1.ResourceList{"amd.com/gpu": resource.MustParse("8"), corev1.ResourceMemory: resource.MustParse("32Gi")}, + }, + }}, + }, + } +} + +// Completion is required: a merely Running GPU pod does not prove training passed. +func amdGPUTrainingPodCompleted(pod *corev1.Pod) (bool, error) { + switch pod.Status.Phase { + case corev1.PodSucceeded: + return true, nil + case corev1.PodFailed: + return false, fmt.Errorf("AMD training pod %s failed: %s %s", pod.Name, pod.Status.Reason, pod.Status.Message) + default: + for _, container := range pod.Status.ContainerStatuses { + if state := container.State.Terminated; state != nil && state.ExitCode != 0 { + return false, fmt.Errorf("AMD training container %s exited %d: %s", container.Name, state.ExitCode, state.Message) + } + } + return false, nil + } +} + +func validateAMDGPUTraining(ctx context.Context, s *Scenario) error { + owner, err := scenarioNodeOwnerReference(ctx, s) + if err != nil { + return err + } + pod := amdGPUTrainingPod(s.Runtime.VM.KubeName, owner) + created, err := s.Runtime.Kube.Typed.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create AMD training pod: %w", err) + } + s.Cleanup(func(ctx context.Context) error { + err := s.Runtime.Kube.Typed.CoreV1().Pods(created.Namespace).Delete(ctx, created.Name, metav1.DeleteOptions{GracePeriodSeconds: to.Ptr(int64(0))}) + if apierrors.IsNotFound(err) { + return nil + } + return err + }) + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 30*time.Minute, true, func(ctx context.Context) (bool, error) { + pod, err := s.Runtime.Kube.Typed.CoreV1().Pods(created.Namespace).Get(ctx, created.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + return amdGPUTrainingPodCompleted(pod) + }) + logs, logErr := s.Runtime.Kube.Typed.CoreV1().Pods(created.Namespace).GetLogs(created.Name, &corev1.PodLogOptions{LimitBytes: to.Ptr(int64(512 * 1024))}).DoRaw(ctx) + logging.Logf(ctx, "AMD training pod %s: %s", created.Name, logs) + if err != nil { + logPodDebugInfo(ctx, s.Runtime.Kube, created) + return fmt.Errorf("AMD training did not complete: %w", err) + } + if logErr != nil { + return fmt.Errorf("read AMD training result: %w", logErr) + } + if !strings.Contains(string(logs), amdTrainingSuccessMarker) { + return fmt.Errorf("AMD training pod completed without its success record") + } + return nil +} diff --git a/e2e/scenario/scenario_gpu_amd_test.go b/e2e/scenario/scenario_gpu_amd_test.go new file mode 100644 index 00000000000..6d8ab5378dc --- /dev/null +++ b/e2e/scenario/scenario_gpu_amd_test.go @@ -0,0 +1,277 @@ +package scenario + +import ( + "context" + "encoding/base64" + "encoding/json" + "os" + "strings" + "testing" + + aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1" + "github.com/Azure/agentbaker/e2e/config" + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestAMDGPUHostValidationRequiresPinnedDiagnostics(t *testing.T) { + contents, err := os.ReadFile(repoPath("vhdbuilder/packer/amd-gpu-components.json")) + require.NoError(t, err) + script, err := amdGPUHostCheckCommand(contents) + require.NoError(t, err) + require.True(t, strings.HasPrefix(script, "sudo -n python3 - '"), "host diagnostics require noninteractive access to GPU devices") + encoded := strings.SplitN(script, "'", 3)[1] + decoded, err := base64.StdEncoding.DecodeString(encoded) + require.NoError(t, err) + var metadata map[string]map[string]string + require.NoError(t, json.Unmarshal(decoded, &metadata)) + require.NotEmpty(t, metadata["AMDGPUDriver"]["moduleVersion"]) + require.Equal(t, "amdrocm-amdsmi10.0", metadata["AMDGPUDiagnostics"]["amdsmiPackage"]) + require.Equal(t, "amdrocm-sysdeps10.0", metadata["AMDGPUDiagnostics"]["sysdepsPackage"]) + + for _, key := range []string{"amdsmiPackage", "amdsmiVersion", "sysdepsPackage", "sysdepsVersion", "cliPath"} { + t.Run(key, func(t *testing.T) { + var manifest map[string]map[string]string + require.NoError(t, json.Unmarshal(decoded, &manifest)) + delete(manifest["AMDGPUDiagnostics"], key) + incomplete, err := json.Marshal(manifest) + require.NoError(t, err) + script, err := amdGPUHostCheckCommand(incomplete) + require.Empty(t, script) + require.ErrorContains(t, err, "AMDGPUDiagnostics."+key) + }) + } +} + +func TestAMDMI300XScenarioSkipsBeforeProvisioning(t *testing.T) { + t.Setenv(amdMI300XOptIn, "") + s := newUbuntu2404MI300XAMDGPUScenario() + outcome := runExecution(t.Context(), s.Name, s.Name, s, + func(context.Context, string, *Scenario) error { + t.Fatal("must skip before the provisioning flow resolves the unpublished VHD or creates resources") + return nil + }) + require.NoError(t, outcome.Error) + require.Contains(t, outcome.SkipReason, amdMI300XOptIn) +} + +func TestAMDMI300XScenarioIsOptInAndUsesDedicatedImage(t *testing.T) { + s := newUbuntu2404MI300XAMDGPUScenario() + for _, value := range []string{"", "false", "1"} { + t.Setenv(amdMI300XOptIn, value) + require.NotEmpty(t, s.SkipIf(t.Context()), "MI300X allocation needs explicit opt-in") + } + t.Setenv(amdMI300XOptIn, "true") + require.Empty(t, s.SkipIf(t.Context())) + require.True(t, s.Tags.GPU) + require.Equal(t, config.DEFAULT_VMSKU, s.K8sSystemPoolSKU, "the shared AKS system pool must stay on a CPU SKU") + require.True(t, s.SkipNVMeOSDiskPlacement, "managed OS disks have no ephemeral NVMe placement") + require.Equal(t, "2404gen2amdgpucontainerd", s.VHD.Name) + require.Equal(t, datamodel.AKSUbuntuContainerd2404Gen2, s.VHD.Distro) + require.Equal(t, "2404gen2containerd", config.VHDUbuntu2404Gen2Containerd.Name, "generic image selection must stay unchanged") + require.GreaterOrEqual(t, s.VHD.OSDiskSizeGB, int32(128), "allow space to unpack the PyTorch image") + require.False(t, s.SkipDefaultValidation) + + nbc := &datamodel.NodeBootstrappingConfiguration{AgentPoolProfile: &datamodel.AgentPoolProfile{}} + s.BootstrapConfigMutator(nil, nbc) + require.Equal(t, amdMI300XVMSize, nbc.AgentPoolProfile.VMSize) + require.True(t, nbc.EnableAMDGPU) + require.True(t, nbc.ConfigGPUDriverIfNeeded) + require.False(t, nbc.EnableNvidia) + require.False(t, nbc.EnableGPUDevicePluginIfNeeded) + + vmss := &armcompute.VirtualMachineScaleSet{ + SKU: &armcompute.SKU{Name: to.Ptr("Standard_D2s_v3")}, + Properties: &armcompute.VirtualMachineScaleSetProperties{ + VirtualMachineProfile: &armcompute.VirtualMachineScaleSetVMProfile{ + StorageProfile: &armcompute.VirtualMachineScaleSetStorageProfile{ + OSDisk: &armcompute.VirtualMachineScaleSetOSDisk{}, + }, + }, + }, + } + s.VMConfigMutator(vmss) + require.Equal(t, amdMI300XVMSize, *vmss.SKU.Name) + anc := &aksnodeconfigv1.Configuration{GpuConfig: &aksnodeconfigv1.GpuConfig{}} + s.AKSNodeConfigMutator(nil, anc) + require.Equal(t, amdMI300XVMSize, anc.VmSize) + require.True(t, anc.GpuConfig.GetEnableAmdGpu()) +} + +func TestAMDMI300XScenarioUsesManagedOSDisk(t *testing.T) { + for _, test := range []struct { + name string + managedDisk *armcompute.VirtualMachineScaleSetManagedDiskParameters + }{ + {name: "default ephemeral disk without managed parameters"}, + { + name: "existing managed parameters", + managedDisk: &armcompute.VirtualMachineScaleSetManagedDiskParameters{ + StorageAccountType: to.Ptr(armcompute.StorageAccountTypesStandardLRS), + DiskEncryptionSet: &armcompute.DiskEncryptionSetParameters{ + ID: to.Ptr("/subscriptions/test/resourceGroups/test/providers/Microsoft.Compute/diskEncryptionSets/test"), + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + s := newUbuntu2404MI300XAMDGPUScenario() + osDisk := &armcompute.VirtualMachineScaleSetOSDisk{ + CreateOption: to.Ptr(armcompute.DiskCreateOptionTypesFromImage), + DiskSizeGB: to.Ptr(s.VHD.OSDiskSizeGB), + Caching: to.Ptr(armcompute.CachingTypesReadOnly), + DiffDiskSettings: &armcompute.DiffDiskSettings{ + Option: to.Ptr(armcompute.DiffDiskOptionsLocal), + Placement: to.Ptr(armcompute.DiffDiskPlacementResourceDisk), + }, + ManagedDisk: test.managedDisk, + } + vmss := &armcompute.VirtualMachineScaleSet{ + SKU: &armcompute.SKU{}, + Properties: &armcompute.VirtualMachineScaleSetProperties{ + VirtualMachineProfile: &armcompute.VirtualMachineScaleSetVMProfile{ + StorageProfile: &armcompute.VirtualMachineScaleSetStorageProfile{OSDisk: osDisk}, + }, + }, + } + s.VMConfigMutator(vmss) + require.True(t, s.SkipNVMeOSDiskPlacement, "only this scenario opts out of ephemeral NVMe placement") + require.Nil(t, osDisk.DiffDiskSettings, "the 256 GiB image must not depend on ephemeral ResourceDisk capacity") + require.NotNil(t, osDisk.ManagedDisk) + require.Equal(t, armcompute.StorageAccountTypesPremiumLRS, *osDisk.ManagedDisk.StorageAccountType) + require.Equal(t, int32(256), *osDisk.DiskSizeGB) + require.Equal(t, armcompute.DiskCreateOptionTypesFromImage, *osDisk.CreateOption) + require.Equal(t, armcompute.CachingTypesReadOnly, *osDisk.Caching) + if test.managedDisk != nil { + require.Same(t, test.managedDisk, osDisk.ManagedDisk, "preserve other managed disk settings") + require.NotNil(t, osDisk.ManagedDisk.DiskEncryptionSet) + } + }) + } +} + +func TestAMDScenarioEnablesANCFlag(t *testing.T) { + nbc, err := baseTemplateLinux("francecentral", "1.34.0", "amd64") + require.NoError(t, err) + anc, err := nbcToAKSNodeConfigV1(nbc) + require.NoError(t, err) + require.Nil(t, anc.GpuConfig.EnableAmdGpu, "default conversion stays unchanged") + newUbuntu2404MI300XAMDGPUScenario().AKSNodeConfigMutator(nil, anc) + require.True(t, anc.GpuConfig.GetEnableAmdGpu()) + require.True(t, anc.GpuConfig.ConfigGpuDriver) +} + +func TestAMDGPUManifestsUseAssignedDevicesAndPinnedUserspace(t *testing.T) { + owner := metav1.OwnerReference{APIVersion: "v1", Kind: "Node", Name: "mi300x-node", UID: "node-uid"} + ds := amdDevicePluginDaemonset(owner.Name, owner) + require.Equal(t, []metav1.OwnerReference{owner}, ds.OwnerReferences) + require.Equal(t, ds.Spec.Selector.MatchLabels, ds.Spec.Template.Labels) + require.Equal(t, owner.Name, ds.Spec.Template.Spec.NodeSelector["kubernetes.io/hostname"]) + require.False(t, *ds.Spec.Template.Spec.AutomountServiceAccountToken) + require.Len(t, ds.Spec.Template.Spec.Containers, 1) + plugin := ds.Spec.Template.Spec.Containers[0] + require.Equal(t, "rocm/k8s-device-plugin@sha256:e4df5dc9a7fa34e2344852256dcc5762171a6d68f1f9a34026ce26786ae335e2", plugin.Image) + require.True(t, *plugin.SecurityContext.Privileged) + require.NotNil(t, plugin.SecurityContext.Capabilities) + require.Equal(t, []corev1.Capability{"ALL"}, plugin.SecurityContext.Capabilities.Drop) + require.Len(t, ds.Spec.Template.Spec.Volumes, 2) + paths := map[string]bool{} + for _, volume := range ds.Spec.Template.Spec.Volumes { + require.NotNil(t, volume.HostPath) + paths[volume.HostPath.Path] = true + } + require.Equal(t, map[string]bool{"/sys": true, "/var/lib/kubelet/device-plugins": true}, paths) + + pod := amdGPUTrainingPod(owner.Name, owner) + require.Equal(t, []metav1.OwnerReference{owner}, pod.OwnerReferences) + require.Equal(t, owner.Name, pod.Spec.NodeSelector["kubernetes.io/hostname"]) + require.Empty(t, pod.Spec.NodeName, "the scheduler must allocate extended GPU resources") + require.Equal(t, corev1.RestartPolicyNever, pod.Spec.RestartPolicy) + require.NotNil(t, pod.Spec.ActiveDeadlineSeconds) + require.False(t, *pod.Spec.AutomountServiceAccountToken) + require.Len(t, pod.Spec.Containers, 1) + workload := pod.Spec.Containers[0] + require.Equal(t, "rocm/pytorch@sha256:bbdaba66029f905321be3bbc95206a2d9a56a2bc5300877d146ad24d40db9a12", workload.Image) + requested, limited := workload.Resources.Requests["amd.com/gpu"], workload.Resources.Limits["amd.com/gpu"] + require.Equal(t, int64(8), requested.Value()) + require.Equal(t, int64(8), limited.Value()) + require.Empty(t, pod.Spec.Volumes, "workload must use container ROCm and plugin-assigned devices") + require.Empty(t, workload.VolumeMounts, "do not mount host ROCm libraries or bypass the plugin") + require.Equal(t, []string{"python3", "-u", "-c", amdGPUTrainingScript}, workload.Command) + require.NotEmpty(t, amdGPUTrainingScript) + + other := amdGPUTrainingPod(owner.Name, owner) + require.NotEqual(t, pod.Name, other.Name, "parallel/repeated scenarios must not share pods") +} + +func TestAMDGPUResourcesRequireEightHealthyGPUs(t *testing.T) { + for _, test := range []struct { + name string + capacity string + allocatable string + ready bool + }{ + {"ready", "8", "8", true}, + {"unhealthy GPU", "8", "7", false}, + {"missing GPU", "7", "7", false}, + {"inactive render nodes counted", "64", "64", false}, + {"not registered", "0", "0", false}, + } { + t.Run(test.name, func(t *testing.T) { + node := &corev1.Node{Status: corev1.NodeStatus{ + Capacity: corev1.ResourceList{"amd.com/gpu": resource.MustParse(test.capacity)}, + Allocatable: corev1.ResourceList{"amd.com/gpu": resource.MustParse(test.allocatable)}, + }} + require.Equal(t, test.ready, amdGPUResourcesReady(node)) + }) + } +} + +func TestAMDGPUTrainingRequiresSuccessfulCompletion(t *testing.T) { + for _, test := range []struct { + phase corev1.PodPhase + done bool + failed bool + }{ + {corev1.PodPending, false, false}, + {corev1.PodRunning, false, false}, + {corev1.PodSucceeded, true, false}, + {corev1.PodFailed, false, true}, + } { + t.Run(string(test.phase), func(t *testing.T) { + pod := &corev1.Pod{Status: corev1.PodStatus{Phase: test.phase}} + done, err := amdGPUTrainingPodCompleted(pod) + require.Equal(t, test.done, done) + if test.failed { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } + failedContainer := &corev1.Pod{Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{{Name: "training", State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ExitCode: 1, Message: "reference mismatch"}, + }}}, + }} + done, err := amdGPUTrainingPodCompleted(failedContainer) + require.False(t, done) + require.ErrorContains(t, err, "reference mismatch") +} + +func TestAMDMI300XRequiresExplicitRunnerSKU(t *testing.T) { + original := config.Config.DefaultVMSKU + t.Cleanup(func() { config.Config.DefaultVMSKU = original }) + s := newUbuntu2404MI300XAMDGPUScenario() + config.Config.DefaultVMSKU = config.DEFAULT_VMSKU + require.ErrorContains(t, s.BootstrapConfigMutatorWithError(t.Context(), nil, nil), "--vm-sku") + config.Config.DefaultVMSKU = amdMI300XVMSize + require.NoError(t, s.BootstrapConfigMutatorWithError(t.Context(), nil, nil)) + require.Equal(t, amdMI300XVMSize, scenarioVMSize(s), "existing runner SKU selection handles AMD before capability queries") +} diff --git a/e2e/scenario/types.go b/e2e/scenario/types.go index b298d62dc1c..ea1163cef7e 100644 --- a/e2e/scenario/types.go +++ b/e2e/scenario/types.go @@ -199,6 +199,9 @@ type Config struct { // UseNVMe indicates whether to use NVMe-based disk placement/controller. This is required for certain VM sizes (e.g., v6 and v7 series) which only support NVMe disk controllers. UseNVMe bool + // SkipNVMeOSDiskPlacement leaves placement to scenarios using a managed OS disk. + SkipNVMeOSDiskPlacement bool + // EagerCSETimingExtraction when true causes CSE timing events to be extracted // immediately after SSH is established, before other validators run. // This prevents the Guest Agent from sweeping events before they can be read. diff --git a/e2e/scenario/vmss.go b/e2e/scenario/vmss.go index 3070a7b007d..a2b79ca0565 100644 --- a/e2e/scenario/vmss.go +++ b/e2e/scenario/vmss.go @@ -487,7 +487,7 @@ func createVMSSModel(ctx context.Context, s *Scenario) (armcompute.VirtualMachin return armcompute.VirtualMachineScaleSet{}, err } - if s.Config.UseNVMe { + if s.Config.UseNVMe && !s.Config.SkipNVMeOSDiskPlacement { if model.Properties == nil || model.Properties.VirtualMachineProfile == nil || model.Properties.VirtualMachineProfile.StorageProfile == nil || model.Properties.VirtualMachineProfile.StorageProfile.OSDisk == nil || diff --git a/parts/linux/cloud-init/artifacts/cse_amd_gpu.sh b/parts/linux/cloud-init/artifacts/cse_amd_gpu.sh new file mode 100644 index 00000000000..57d1a0d5aa4 --- /dev/null +++ b/parts/linux/cloud-init/artifacts/cse_amd_gpu.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Validate the dedicated AMD image on each real node, including PIS nodes. There +# is deliberately no apt/DKMS installation fallback during node provisioning. +ensureAmdGpuDrivers() { + local vm_sku + vm_sku=$(get_compute_sku) || return $ERR_AMD_GPU_VALIDATE_FAIL + if [ "${OS}" != "${UBUNTU_OS_NAME}" ] || [ "${OS_VERSION}" != "24.04" ] || [ "$(uname -m)" != "x86_64" ] || [ "${GPU_NODE:-false}" = "true" ]; then + echo "AMD GPU requires the dedicated Ubuntu 24.04 amd64 image and exclusive AMD configuration" + return $ERR_AMD_GPU_UNSUPPORTED + fi + case "${vm_sku,,}" in + standard_nd96isr_mi300x_v5|standard_nd96is_mi300x_v5) ;; + *) echo "Unsupported AMD GPU VM SKU: ${vm_sku}"; return $ERR_AMD_GPU_UNSUPPORTED ;; + esac + + if ! validateAmdGpuDriver; then + echo "Baked AMD GPU driver validation failed; use a qualified AMD GPU VHD" + return $ERR_AMD_GPU_VALIDATE_FAIL + fi +} + +validateAmdGpuDriver() { + local marker=/opt/azure/amd-gpu/driver.json + local package_version firmware_version module_version module_path kernel_version loaded_version + [ -s "${marker}" ] || return 1 + jq -e '.schema_version == 1' "${marker}" >/dev/null || return 1 + package_version=$(jq -er '.package_version | strings | select(length > 0)' "${marker}") || return 1 + firmware_version=$(jq -er '.firmware_package_version | strings | select(length > 0)' "${marker}") || return 1 + module_version=$(jq -er '.module_version | strings | select(length > 0)' "${marker}") || return 1 + [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms)" = "install ok installed ${package_version}" ] || return 1 + [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms-firmware)" = "install ok installed ${firmware_version}" ] || return 1 + + kernel_version=$(uname -r) + module_path=$(modinfo -k "${kernel_version}" -F filename amdgpu) || return 1 + # Reject the inbox driver even if modprobe succeeds. DKMS must have built the + # pinned driver for the running kernel, which may differ from the bake kernel. + case "${module_path}" in + /lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*|/usr/lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*) ;; + *) return 1 ;; + esac + [ "$(modinfo -k "${kernel_version}" -F version amdgpu)" = "${module_version}" ] || return 1 + retrycmd_if_failure 12 5 30 modprobe amdgpu || return 1 + loaded_version=$(cat /sys/module/amdgpu/version) || return 1 + [ "${loaded_version}" = "${module_version}" ] || return 1 + retrycmd_if_failure 12 5 5 test -c /dev/kfd || return 1 + export -f validateAmdGpuDevices + retrycmd_if_failure 12 5 5 bash -c validateAmdGpuDevices || return 1 + echo "AMD GPU driver ${module_version} ready on ${kernel_version}; eight GPUs detected" +} + +validateAmdGpuDevices() { + local gpu_id_file gpu_id count=0 + # Count active KFD agents, not DRM render nodes: MI300X VFs can preallocate + # many more render nodes than the eight GPUs assigned to the VM. + for gpu_id_file in /sys/class/kfd/kfd/topology/nodes/*/gpu_id; do + [ -r "${gpu_id_file}" ] || continue + gpu_id=$(cat "${gpu_id_file}") || return 1 + case "${gpu_id}" in + ''|*[!0-9]*) return 1 ;; + 0) continue ;; + esac + count=$((count + 1)) + done + [ "${count}" -eq 8 ] +} diff --git a/parts/linux/cloud-init/artifacts/cse_cmd.sh b/parts/linux/cloud-init/artifacts/cse_cmd.sh index 2079e1f1664..7adc4eb264d 100644 --- a/parts/linux/cloud-init/artifacts/cse_cmd.sh +++ b/parts/linux/cloud-init/artifacts/cse_cmd.sh @@ -77,6 +77,7 @@ IDENTITY_BINDINGS_LOCAL_AUTHORITY_SNI={{GetVariable "identityBindingsLocalAuthor API_SERVER_NAME={{GetKubernetesEndpoint}} IS_VHD={{GetVariable "isVHD"}} GPU_NODE={{GetVariable "gpuNode"}} +AMD_GPU_NODE={{GetVariable "amdGpuNode"}} SGX_NODE={{GetVariable "sgxNode"}} MIG_NODE={{GetVariable "migNode"}} CONFIG_GPU_DRIVER_IF_NEEDED={{GetVariable "configGPUDriverIfNeeded"}} diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index 8862606a704..3d601ce4c7b 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -162,6 +162,8 @@ ERR_AKS_NODE_CONTROLLER_ERROR=240 # Generic error in AKS Node Controller ERR_AZNFS_RPM_DOWNLOAD_TIMEOUT=241 # Timeout downloading aznfs RPM from PMC ERR_AZNFS_INSTALL_FAIL=242 # Failed to install aznfs RPM package ERR_SECONDARY_NIC_CONFIG_FAIL=243 # Error configuring secondary NIC network interface +ERR_AMD_GPU_UNSUPPORTED=244 # AMD GPU image is only supported on Ubuntu 24.04 amd64 MI300X nodes +ERR_AMD_GPU_VALIDATE_FAIL=245 # Baked AMD GPU driver or device validation failed # ----------------------------------------------------------------------------- # This probably wasn't launched via a login shell, so ensure the PATH is correct. diff --git a/parts/linux/cloud-init/artifacts/cse_main.sh b/parts/linux/cloud-init/artifacts/cse_main.sh index ae432248934..fbd7b1a105f 100755 --- a/parts/linux/cloud-init/artifacts/cse_main.sh +++ b/parts/linux/cloud-init/artifacts/cse_main.sh @@ -487,6 +487,13 @@ function nodePrep { REBOOTREQUIRED=false # Install and configure GPU drivers if this is a GPU node + # Hardware validation belongs in nodePrep so it also runs on PIS nodes. + # The dedicated AMD image contains the driver; provisioning never downloads it. + if [ "${AMD_GPU_NODE:-false}" = "true" ] && [ "${CONFIG_GPU_DRIVER_IF_NEEDED:-false}" = "true" ]; then + source /opt/azure/containers/amd-gpu-validate.sh || exit $ERR_AMD_GPU_VALIDATE_FAIL + logs_to_events "AKS.CSE.ensureAmdGpuDrivers" ensureAmdGpuDrivers || exit $? + fi + if [ "${GPU_NODE}" = "true" ] && [ "${skip_nvidia_driver_install}" != "true" ]; then echo $(date),$(hostname), "Start configuring GPU drivers" diff --git a/pkg/agent/variables.go b/pkg/agent/variables.go index 3e16b778874..2639285de09 100644 --- a/pkg/agent/variables.go +++ b/pkg/agent/variables.go @@ -137,6 +137,7 @@ func getCSECommandVariables(config *datamodel.NodeBootstrappingConfiguration) pa "userAssignedIdentityID": config.UserAssignedIdentityClientID, "isVHD": isVHD(profile), "gpuNode": strconv.FormatBool(config.EnableNvidia), + "amdGpuNode": strconv.FormatBool(config.EnableAMDGPU), "sgxNode": strconv.FormatBool(datamodel.IsSgxEnabledSKU(profile.VMSize)), "configGPUDriverIfNeeded": config.ConfigGPUDriverIfNeeded, "enableGPUDevicePluginIfNeeded": config.EnableGPUDevicePluginIfNeeded, diff --git a/pkg/agent/variables_test.go b/pkg/agent/variables_test.go index 556511c56a3..cafdf82ee2c 100644 --- a/pkg/agent/variables_test.go +++ b/pkg/agent/variables_test.go @@ -438,6 +438,15 @@ var _ = Describe("Windows CSE variables check", func() { config = getDefaultNBC() }) + It("keeps AMD GPU bootstrap explicitly opt-in and independent of NVIDIA", func() { + vars := getCSECommandVariables(config) + Expect(vars["amdGpuNode"]).To(Equal("false")) + config.EnableAMDGPU = true + vars = getCSECommandVariables(config) + Expect(vars["amdGpuNode"]).To(Equal("true")) + Expect(vars["gpuNode"]).To(Equal("false")) + }) + It("sets maximumLoadBalancerRuleCount", func() { config.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.MaximumLoadBalancerRuleCount = 5 vars := getCSECommandVariables(config) diff --git a/schemas/amd-gpu-components.cue b/schemas/amd-gpu-components.cue new file mode 100644 index 00000000000..09e8d21f831 --- /dev/null +++ b/schemas/amd-gpu-components.cue @@ -0,0 +1,55 @@ +package amdgpu + +import ( + "regexp" + "strings" +) + +#AMDGPUDriver: { + repositoryURL: string & =~"^https://repo[.]radeon[.]com/amdgpu/[0-9]+([.][0-9]+){1,3}/ubuntu$" + signingKeyURL: string & =~"^https://repo[.]radeon[.]com/[^?[:space:]]+$" + signingKeyFingerprint: string & =~"^[A-F0-9]{40}$" + distribution: "noble" + component: "main" + packageVersion: string & =~"^[0-9]+:[0-9]+[.][0-9]+[.][0-9]+[.][0-9]+-[0-9]+[.]24[.]04$" + firmwarePackageVersion: string & =~"^[0-9]+:[0-9][0-9A-Za-z.+~-]+$" + moduleVersion: string & =~"^[0-9][0-9.]+$" + dkmsVersion: string & =~"^[0-9][0-9A-Za-z.+~-]+$" + + // These fields describe one driver release, not independent update targets. + // Reject partial Renovate proposals until a reviewer supplies coherent pins. + _package: regexp.FindSubmatch("^([0-9]+):([0-9]+[.][0-9]+[.][0-9]+)[.]([0-9]+)-([0-9]+[.]24[.]04)$", packageVersion) + _release: strings.TrimSuffix(strings.TrimPrefix(repositoryURL, "https://repo.radeon.com/amdgpu/"), "/ubuntu") + _releaseParts: strings.Split(_release, ".") + _firmwareRelease: _release + strings.Repeat(".0", 4-len(_releaseParts)) + moduleVersion: "\(_package[2]).\(_package[3])" + dkmsVersion: "\(_package[2])-\(_package[4])" + firmwarePackageVersion: "\(_package[1]):\(_firmwareRelease).\(_package[3])-\(_package[4])" +} + +#AMDGPUDiagnostics: { + repositoryURL: "https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/" + signingKeyURL: "https://stable.repo.amd.com/rocm/gpg/packages.gpg" + signingKeyFingerprint: string & =~"^[A-F0-9]{40}$" + distribution: "stable" + component: "main" + amdsmiPackage: string & =~"^amdrocm-amdsmi[0-9]+[.][0-9]+$" + amdsmiVersion: string & =~"^[0-9][0-9A-Za-z.+~-]+$" + sysdepsPackage: string & =~"^amdrocm-sysdeps[0-9]+[.][0-9]+$" + sysdepsVersion: string & =~"^[0-9][0-9A-Za-z.+~-]+$" + cliPath: string & =~"^/opt/rocm/core-[0-9]+[.][0-9]+/bin/amd-smi$" + + // Keep package names, versions and the installed CLI in the same ROCm line. + _core: strings.TrimPrefix(amdsmiPackage, "amdrocm-amdsmi") + sysdepsPackage: "amdrocm-sysdeps\(_core)" + amdsmiVersion: =~"^\(regexp.QuoteMeta(_core))[.]" + sysdepsVersion: =~"^\(regexp.QuoteMeta(_core))[.]" + cliPath: "/opt/rocm/core-\(_core)/bin/amd-smi" +} + +#Components: { + AMDGPUDriver: #AMDGPUDriver + AMDGPUDiagnostics: #AMDGPUDiagnostics +} + +#Components diff --git a/spec/parts/linux/cloud-init/artifacts/cse_install_amd_gpu_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_install_amd_gpu_spec.sh new file mode 100644 index 00000000000..bdda86c64ef --- /dev/null +++ b/spec/parts/linux/cloud-init/artifacts/cse_install_amd_gpu_spec.sh @@ -0,0 +1,193 @@ +#!/bin/bash + +Describe 'dedicated AMD GPU image provisioning' + Include ./parts/linux/cloud-init/artifacts/cse_amd_gpu.sh + ERR_AMD_GPU_UNSUPPORTED=244 + ERR_AMD_GPU_VALIDATE_FAIL=245 + OS=UBUNTU + UBUNTU_OS_NAME=UBUNTU + OS_VERSION=24.04 + GPU_NODE=false + get_compute_sku() { echo Standard_ND96isr_MI300X_v5; } + uname() { + case "$1" in + -m) echo x86_64 ;; + -r) echo 6.17.0-test-azure ;; + esac + } + + Describe 'nodePrep opt-in dispatch' + run_amd_node_prep() { + # Execute the real dispatch block, restricted to nodePrep so moving + # it into basePrep (which PIS skips) also fails the enabled test. + eval "$(awk ' + /^function nodePrep/ { node_prep = 1 } + node_prep && /if .*AMD_GPU_NODE/ { block = 1 } + block { print } + block && /^[[:space:]]*fi$/ { exit } + ' parts/linux/cloud-init/artifacts/cse_main.sh)" + } + source() { test "$1" = /opt/azure/containers/amd-gpu-validate.sh; } + logs_to_events() { shift; "$@"; } + ensureAmdGpuDrivers() { echo amd-driver-validated; } + + It 'runs hardware checks in nodePrep with both explicit flags' + AMD_GPU_NODE=true + CONFIG_GPU_DRIVER_IF_NEEDED=true + When run run_amd_node_prep + The status should be success + The output should equal amd-driver-validated + End + + It 'fails the AMD node if its dedicated image validator is missing' + AMD_GPU_NODE=true + CONFIG_GPU_DRIVER_IF_NEEDED=true + source() { return 1; } + When run run_amd_node_prep + The status should equal 245 + The output should equal '' + End + + It 'honors driver configuration opt-out' + AMD_GPU_NODE=true + CONFIG_GPU_DRIVER_IF_NEEDED=false + When run run_amd_node_prep + The status should be success + The output should equal '' + End + + It 'does nothing on existing nodes without AMD enablement' + unset AMD_GPU_NODE + CONFIG_GPU_DRIVER_IF_NEEDED=true + When run run_amd_node_prep + The status should be success + The output should equal '' + End + End + + Describe 'ensureAmdGpuDrivers' + validateAmdGpuDriver() { echo driver-validated; } + + It 'validates the AMD image without installing anything' + When call ensureAmdGpuDrivers + The status should be success + The output should equal driver-validated + End + + It 'rejects another GPU SKU' + get_compute_sku() { echo Standard_ND96asr_v4; } + When call ensureAmdGpuDrivers + The status should equal 244 + The output should include 'Unsupported AMD GPU VM SKU' + End + + It 'rejects mixed NVIDIA and AMD configuration' + GPU_NODE=true + When call ensureAmdGpuDrivers + The status should equal 244 + The output should include 'exclusive AMD configuration' + End + + It 'rejects unsupported Ubuntu versions' + OS_VERSION=22.04 + When call ensureAmdGpuDrivers + The status should equal 244 + The output should include 'Ubuntu 24.04' + End + + It 'fails closed when the baked driver is missing or invalid' + validateAmdGpuDriver() { return 1; } + When call ensureAmdGpuDrivers + The status should equal 245 + The output should include 'use a qualified AMD GPU VHD' + End + End + + Describe 'validateAmdGpuDriver and active devices' + setup() { + AMD_TEST_ROOT=$(mktemp -d) + export AMD_TEST_ROOT + mkdir -p "${AMD_TEST_ROOT}/nodes" "${AMD_TEST_ROOT}/module" + echo 7.1.3.test > "${AMD_TEST_ROOT}/module/version" + echo '{"schema_version":1,"package_version":"1:driver-test","firmware_package_version":"firmware-test","module_version":"7.1.3.test","kernel_version":"older-bake-kernel"}' > "${AMD_TEST_ROOT}/driver.json" + local i + for i in {0..8}; do + mkdir -p "${AMD_TEST_ROOT}/nodes/${i}" + echo "${i}" > "${AMD_TEST_ROOT}/nodes/${i}/gpu_id" + done + eval "$(sed -n '/^validateAmdGpuDriver()/,/^}/p; /^validateAmdGpuDevices()/,/^}/p' parts/linux/cloud-init/artifacts/cse_amd_gpu.sh | + sed "s|/opt/azure/amd-gpu|${AMD_TEST_ROOT}|g; s|/sys/module/amdgpu|${AMD_TEST_ROOT}/module|g; s|/sys/class/kfd/kfd/topology/nodes|${AMD_TEST_ROOT}/nodes|g")" + } + cleanup() { rm -rf "${AMD_TEST_ROOT}"; } + BeforeEach setup + AfterEach cleanup + + dpkg-query() { + case "${*: -1}" in + amdgpu-dkms) echo 'install ok installed 1:driver-test' ;; + amdgpu-dkms-firmware) echo 'install ok installed firmware-test' ;; + esac + } + modinfo() { + case "$4" in + filename) echo /lib/modules/6.17.0-test-azure/updates/dkms/amdgpu.ko.zst ;; + version) echo 7.1.3.test ;; + esac + } + retrycmd_if_failure() { + shift 3 + case "$1" in + modprobe) return 0 ;; + test) return 0 ;; + *) "$@" ;; + esac + } + + It 'accepts a rebuilt pinned driver on a newer kernel and ignores the CPU KFD node' + When call validateAmdGpuDriver + The status should be success + The output should include 'eight GPUs detected' + End + + It 'rejects the inbox module' + modinfo() { echo /lib/modules/6.17.0-test-azure/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu.ko.zst; } + When call validateAmdGpuDriver + The status should be failure + The output should equal '' + End + + It 'rejects a different already-loaded module' + echo old-driver > "${AMD_TEST_ROOT}/module/version" + When call validateAmdGpuDriver + The status should be failure + The output should equal '' + End + + It 'rejects removed packages with leftover dpkg records' + dpkg-query() { echo 'deinstall ok config-files 1:driver-test'; } + When call validateAmdGpuDriver + The status should be failure + The output should equal '' + End + + It 'rejects an unknown marker format' + echo '{"schema_version":2}' > "${AMD_TEST_ROOT}/driver.json" + When call validateAmdGpuDriver + The status should be failure + The output should equal '' + End + + It 'rejects missing GPUs' + echo 0 > "${AMD_TEST_ROOT}/nodes/8/gpu_id" + When call validateAmdGpuDriver + The status should be failure + The output should equal '' + End + + It 'rejects malformed GPU topology rather than counting it as a GPU' + echo invalid > "${AMD_TEST_ROOT}/nodes/8/gpu_id" + When call validateAmdGpuDevices + The status should be failure + End + End +End diff --git a/spec/shellspec.Dockerfile b/spec/shellspec.Dockerfile index a7d9313712e..39bdb503d6d 100644 --- a/spec/shellspec.Dockerfile +++ b/spec/shellspec.Dockerfile @@ -30,7 +30,7 @@ FROM aksdataplanedev.azurecr.io/shellspec/shellspec-debian:0.28.1 RUN sed -i -e 's/\(deb\|security\).debian.org/archive.debian.org/g' /etc/apt/sources.list && \ apt-get update && \ apt-get install -y --no-install-recommends \ - gawk jq curl dnsutils openssl \ + gawk jq curl dnsutils python3 openssl \ libcurl4 libdw1 libelf1 libssl1.1 zlib1g && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/spec/vhdbuilder/packer/amd_gpu_diagnostics_spec.sh b/spec/vhdbuilder/packer/amd_gpu_diagnostics_spec.sh new file mode 100644 index 00000000000..6703b60e090 --- /dev/null +++ b/spec/vhdbuilder/packer/amd_gpu_diagnostics_spec.sh @@ -0,0 +1,278 @@ +#!/bin/bash +# shellcheck disable=SC2034,SC2329 + +Describe 'Minimal AMDGPU host diagnostics bake' + setup_amd_diagnostics() { + TEST_DIR=$(mktemp -d) + TRACE="${TEST_DIR}/trace" + : > "${TRACE}" + OS=UBUNTU OS_VERSION=24.04 CPU_ARCH=amd64 HYPERV_GENERATION=v2 ENABLE_FIPS=false + AMD_COMPONENTS_FILEPATH="${TEST_DIR}/components.json" + VHD_LOGS_FILEPATH="${TEST_DIR}/vhd.log" + FAIL_STAGE="" + PACKAGE_ARCH=amd64 + mkdir -p "${TEST_DIR}/work" "${TEST_DIR}/bin" "${TEST_DIR}/rocm/core-10.0/bin" "${TEST_DIR}/rocm/core-10.0/share/amd_smi" + jq --arg cli_path "${TEST_DIR}/rocm/core-10.0/bin/amd-smi" '.AMDGPUDiagnostics.cliPath = $cli_path' \ + vhdbuilder/packer/amd-gpu-components.json > "${AMD_COMPONENTS_FILEPATH}" + AMDSMI_PACKAGE=$(jq -r '.AMDGPUDiagnostics.amdsmiPackage' "${AMD_COMPONENTS_FILEPATH}") + AMDSMI_VERSION=$(jq -r '.AMDGPUDiagnostics.amdsmiVersion' "${AMD_COMPONENTS_FILEPATH}") + SYSDEPS_PACKAGE=$(jq -r '.AMDGPUDiagnostics.sysdepsPackage' "${AMD_COMPONENTS_FILEPATH}") + SYSDEPS_VERSION=$(jq -r '.AMDGPUDiagnostics.sysdepsVersion' "${AMD_COMPONENTS_FILEPATH}") + # Run the production function, remapping only owned filesystem locations. + # Network, package metadata and package installation are mocked below. + eval "$(sed -n '/^installAMDGPUDiagnostics()/,/^}$/p' vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh | + sed -e "s|/opt/rocm|${TEST_DIR}/rocm|g" -e "s|/usr/local/bin|${TEST_DIR}/bin|g" \ + -e "s|/tmp/amd-gpu-diagnostics|${TEST_DIR}/work/amd-gpu-diagnostics|g")" + } + cleanup_amd_diagnostics() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_amd_diagnostics + AfterEach cleanup_amd_diagnostics + + retrycmd_if_failure() { shift 3; "$@"; } + apt_get_update() { echo ubuntu-update >> "${TRACE}"; } + apt_get_install() { + shift 3 + echo "ubuntu-install $*" >> "${TRACE}" + case " $* " in + *'.deb '*) + echo vendor-install >> "${TRACE}" + [ "${FAIL_STAGE}" != install ] || return 1 + [ "${FAIL_STAGE}" != missing_cli ] || return 0 + # A CPU builder cannot run CLI commands that initialize GPU hardware. + printf '#!/bin/bash\necho "amd-smi $*" >> "%s"\nexit 99\n' \ + "${TRACE}" > "${TEST_DIR}/rocm/core-10.0/bin/amd-smi" + [ "${FAIL_STAGE}" = nonexecutable_cli ] || chmod +x "${TEST_DIR}/rocm/core-10.0/bin/amd-smi" + if [ "${FAIL_STAGE}" = python_import ]; then + echo 'raise RuntimeError("diagnostics import failed")' > "${TEST_DIR}/rocm/core-10.0/share/amd_smi/amdsmi.py" + elif [ "${FAIL_STAGE}" = library_version ]; then + printf 'def amdsmi_get_lib_version():\n return {"major": 0}\n' > "${TEST_DIR}/rocm/core-10.0/share/amd_smi/amdsmi.py" + else + printf '__version__ = "26.0.0"\ndef amdsmi_get_lib_version():\n with open("%s", "a") as trace:\n trace.write("amdsmi-library-version\\n")\n return {"major": 26, "minor": 0, "release": 0}\n' \ + "${TRACE}" > "${TEST_DIR}/rocm/core-10.0/share/amd_smi/amdsmi.py" + fi + ;; + *) [ "${FAIL_STAGE}" != prerequisites ] ;; + esac + } + apt-mark() { + echo "apt-mark $*" >> "${TRACE}" + printf 'retained-package %s\n' "$@" >> "${TRACE}" + [ "${FAIL_STAGE}" != retain ] + } + apt-get() { + echo "apt-get $*" >> "${TRACE}" + local option + for option in "$@"; do + case "${option}" in + Dir::Etc::sourcelist=*) printf 'repository %s\n' "$(cat "${option#*=}")" >> "${TRACE}" ;; + esac + done + case " $* " in + *' update '*) [ "${FAIL_STAGE}" != metadata ] ;; + *' download '*) + [ "${FAIL_STAGE}" != download ] || return 1 + touch amdsmi.deb sysdeps.deb + [ "${FAIL_STAGE}" != extra ] || touch runtime.deb + ;; + *) return 99 ;; + esac + } + curl() { + echo key-download >> "${TRACE}" + while [ "$#" -gt 0 ]; do + if [ "$1" = -o ]; then printf 'test-key\n' > "$2"; return; fi + shift + done + return 99 + } + gpg() { + case " $* " in + *' --show-keys '*) + echo 'pub:::::::::' + if [ "${FAIL_STAGE}" = key ]; then + echo 'fpr:::::::::WRONG:' + else + echo 'fpr:::::::::D0F004A0025A1145C7807FCD0701EAC4D5E02107:' + fi + echo 'sub:::::::::' + echo 'fpr:::::::::UNRELATED_SUBKEY:' + ;; + *) + while [ "$#" -gt 0 ]; do + if [ "$1" = --output ]; then touch "$2"; return; fi + shift + done + return 99 + ;; + esac + } + dpkg-deb() { + case "$2:$3" in + */amdsmi.deb:Package) echo "${AMDSMI_PACKAGE}" ;; + */sysdeps.deb:Package) echo "${SYSDEPS_PACKAGE}" ;; + */runtime.deb:Package) echo amdrocm-runtime10.0 ;; + *:Architecture) echo "${PACKAGE_ARCH}" ;; + */amdsmi.deb:Version) + if [ "${FAIL_STAGE}" = amdsmi_version ]; then echo wrong-version; else echo "${AMDSMI_VERSION}"; fi + ;; + */sysdeps.deb:Version) + if [ "${FAIL_STAGE}" = sysdeps_version ]; then echo wrong-version; else echo "${SYSDEPS_VERSION}"; fi + ;; + *) return 99 ;; + esac + } + dpkg-query() { + case "${*: -1}" in + "${AMDSMI_PACKAGE}") + if [ "${FAIL_STAGE}" = installed_version ]; then echo 'install ok installed wrong-version'; + else echo "install ok installed ${AMDSMI_VERSION}"; fi + ;; + "${SYSDEPS_PACKAGE}") echo "install ok installed ${SYSDEPS_VERSION}" ;; + *) return 99 ;; + esac + } + run_diagnostics_installer() { + local status=0 + # Conditional callers disable errexit: installation must still fail closed. + installAMDGPUDiagnostics || status=$? + echo "remaining-work-files=$(find "${TEST_DIR}/work" -mindepth 1 | wc -l | tr -d ' ')" + return "${status}" + } + repeat_diagnostics_installer() { + installAMDGPUDiagnostics && installAMDGPUDiagnostics && + [ "$(readlink "${TEST_DIR}/bin/amd-smi")" = "${TEST_DIR}/rocm/core-10.0/bin/amd-smi" ] + } + + It 'installs only the pinned AMD SMI subset from authenticated isolated metadata' + When run run_diagnostics_installer + The status should be success + The output should include '"python_module": "26.0.0"' + The output should include 'remaining-work-files=0' + The stderr should eq '' + The contents of file "${TRACE}" should include "download ${AMDSMI_PACKAGE}=${AMDSMI_VERSION} ${SYSDEPS_PACKAGE}=${SYSDEPS_VERSION}" + The contents of file "${TRACE}" should include 'https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/ stable main' + The contents of file "${TRACE}" should include 'Dir::Etc::sourceparts=-' + The contents of file "${TRACE}" should include 'AllowUnauthenticated=false' + The contents of file "${TRACE}" should include 'AllowInsecureRepositories=false' + The contents of file "${TRACE}" should include 'apt-mark manual' + The contents of file "${TRACE}" should include 'retained-package pciutils' + The contents of file "${TRACE}" should include 'retained-package numactl' + The contents of file "${TRACE}" should include 'retained-package python3' + The contents of file "${TRACE}" should include 'retained-package libstdc++6' + The contents of file "${TRACE}" should include 'retained-package libgcc-s1' + The contents of file "${TRACE}" should include "${AMDSMI_PACKAGE} ${SYSDEPS_PACKAGE}" + The contents of file "${TRACE}" should include 'amdsmi-library-version' + The contents of file "${TRACE}" should not include 'amd-smi --help' + The contents of file "${TRACE}" should not include 'amdrocm-runtime' + The contents of file "${TRACE}" should not include 'amdrocm10.0-gfx942' + The contents of file "${TRACE}" should not include 'amdrocm-llvm' + The path "${TEST_DIR}/bin/amd-smi" should be symlink + End + + It 'rejects a signing key mismatch before querying the AMD repository' + FAIL_STAGE=key + When run run_diagnostics_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'signing key fingerprint mismatch' + The contents of file "${TRACE}" should not include 'apt-get ' + The path "${TEST_DIR}/bin/amd-smi" should not be exist + End + + Describe 'failed repository and package transactions' + Parameters + prerequisites + metadata + download + amdsmi_version + sysdeps_version + install + retain + installed_version + End + It 'fails and removes temporary repository state' + FAIL_STAGE="$1" + When run run_diagnostics_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should eq '' + End + End + + It 'rejects an unexpected extra vendor archive before installing it' + FAIL_STAGE=extra + When run run_diagnostics_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'Unexpected' + The contents of file "${TRACE}" should not include 'vendor-install' + End + + It 'rejects a package for the wrong architecture before installing it' + PACKAGE_ARCH=arm64 + When run run_diagnostics_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'architecture' + The contents of file "${TRACE}" should not include 'vendor-install' + End + + Describe 'a broken installed diagnostic CLI' + Parameters + missing_cli + nonexecutable_cli + End + It 'fails its CPU-only executable smoke check' + FAIL_STAGE="$1" + When run run_diagnostics_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should eq '' + End + End + + It 'fails when the installed Python binding cannot import' + FAIL_STAGE=python_import + When run run_diagnostics_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'diagnostics import failed' + End + + It 'fails when the diagnostic library returns an invalid version' + FAIL_STAGE=library_version + When run run_diagnostics_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'AssertionError' + End + + It 'replaces a stale command symlink and repeats installation idempotently' + ln -s "${TEST_DIR}/missing" "${TEST_DIR}/bin/amd-smi" + When run repeat_diagnostics_installer + The status should be success + The output should include '"python_module": "26.0.0"' + The stderr should eq '' + The path "${TEST_DIR}/bin/amd-smi" should be symlink + End + + It 'does not overwrite an existing regular file at the command location' + printf 'owned-by-another-package\n' > "${TEST_DIR}/bin/amd-smi" + When run run_diagnostics_installer + The status should be failure + The output should include 'remaining-work-files=0' + The stderr should include 'amd-smi' + The contents of file "${TEST_DIR}/bin/amd-smi" should eq 'owned-by-another-package' + End + + It 'does not replace or write into an existing directory at the command location' + mkdir "${TEST_DIR}/bin/amd-smi" + printf 'preserve\n' > "${TEST_DIR}/bin/amd-smi/owned-file" + When run run_diagnostics_installer + The status should be failure + The output should include 'remaining-work-files=0' + The stderr should include 'Refusing to replace a non-symlink' + The contents of file "${TEST_DIR}/bin/amd-smi/owned-file" should eq 'preserve' + The path "${TEST_DIR}/bin/amd-smi/amd-smi" should not be exist + End +End diff --git a/spec/vhdbuilder/packer/amd_gpu_driver_spec.sh b/spec/vhdbuilder/packer/amd_gpu_driver_spec.sh new file mode 100644 index 00000000000..f73a2955722 --- /dev/null +++ b/spec/vhdbuilder/packer/amd_gpu_driver_spec.sh @@ -0,0 +1,393 @@ +#!/bin/bash +# shellcheck disable=SC2034,SC2329 + +Describe 'CPU-only AMDGPU driver bake' + setup_amd_driver() { + TEST_DIR=$(mktemp -d) + TRACE="${TEST_DIR}/trace" + : > "${TRACE}" + OS=UBUNTU OS_VERSION=24.04 CPU_ARCH=amd64 HYPERV_GENERATION=v2 ENABLE_FIPS=false + AMD_COMPONENTS_FILEPATH="${PWD}/vhdbuilder/packer/amd-gpu-components.json" + VHD_LOGS_FILEPATH="${TEST_DIR}/vhd.log" + FAIL_STAGE="" + PACKAGE_ARCH=all + KERNEL=6.8.0-test-azure + PACKAGE_VERSION=$(jq -r '.AMDGPUDriver.packageVersion' "${AMD_COMPONENTS_FILEPATH}") + FIRMWARE_VERSION=$(jq -r '.AMDGPUDriver.firmwarePackageVersion' "${AMD_COMPONENTS_FILEPATH}") + MODULE_VERSION=$(jq -r '.AMDGPUDriver.moduleVersion' "${AMD_COMPONENTS_FILEPATH}") + DKMS_VERSION=$(jq -r '.AMDGPUDriver.dkmsVersion' "${AMD_COMPONENTS_FILEPATH}") + mkdir -p "${TEST_DIR}/work" "${TEST_DIR}/marker" "${TEST_DIR}/modprobe" "${TEST_DIR}/vendor-modprobe" \ + "${TEST_DIR}/modules/${KERNEL}/build" + touch "${TEST_DIR}/modules/${KERNEL}/build/Makefile" + printf 'blacklist amdgpu\nblacklist nouveau\n# retain this comment\n' > "${TEST_DIR}/modprobe/cloud.conf" + # Remap owned filesystem locations only. Real sed exercises the cloud image + # blacklist edit; apt, GPG, module commands and network are all mocked below. + eval "$(sed -n '/^installAMDGPUDriver()/,/^}$/p' vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh | + sed -e "s|/opt/azure/amd-gpu|${TEST_DIR}/marker|g" -e "s|/tmp/amd-gpu|${TEST_DIR}/work/amd-gpu|g" \ + -e "s|/etc/modprobe.d|${TEST_DIR}/modprobe|g" -e "s|/usr/lib/modprobe.d|${TEST_DIR}/vendor-modprobe|g" \ + -e "s|/lib/modules|${TEST_DIR}/modules|g")" + } + cleanup_amd_driver() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_amd_driver + AfterEach cleanup_amd_driver + + uname() { printf '%s\n' "${KERNEL}"; } + retrycmd_if_failure() { shift 3; "$@"; } + apt_get_update() { echo ubuntu-update >> "${TRACE}"; } + apt_get_install() { + shift 3 + echo "ubuntu-install $*" >> "${TRACE}" + [ "${FAIL_STAGE}" != install ] + } + apt-mark() { echo "apt-mark $*" >> "${TRACE}"; } + apt-get() { + echo "apt-get $*" >> "${TRACE}" + if [ "${FAIL_STAGE}" = metadata ]; then return 1; fi + case " $* " in + *' download '*) touch driver.deb firmware.deb ;; + esac + } + curl() { + echo key-download >> "${TRACE}" + while [ "$#" -gt 0 ]; do + if [ "$1" = -o ]; then printf 'test-key\n' > "$2"; return; fi + shift + done + return 99 + } + gpg() { + case " $* " in + *' --show-keys '*) + echo 'pub:::::::::' + if [ "${FAIL_STAGE}" = key ]; then + echo 'fpr:::::::::WRONG:' + else + echo 'fpr:::::::::CA8BB4727A47B4D09B4EE8969386B48A1A693C5C:' + fi + echo 'sub:::::::::' + echo 'fpr:::::::::2B2AFE47A094DC2D1013777C30C07AF01A6D36BA:' + ;; + *) + while [ "$#" -gt 0 ]; do + if [ "$1" = --output ]; then touch "$2"; return; fi + shift + done + return 99 + ;; + esac + } + dpkg-deb() { + case "$2:$3" in + */driver.deb:Package) echo amdgpu-dkms ;; + */firmware.deb:Package) echo amdgpu-dkms-firmware ;; + *:Architecture) echo "${PACKAGE_ARCH:-all}" ;; + */driver.deb:Version) if [ "${FAIL_STAGE}" = package ]; then echo wrong-version; else echo "${PACKAGE_VERSION}"; fi ;; + */firmware.deb:Version) echo "${FIRMWARE_VERSION}" ;; + *) return 99 ;; + esac + } + dpkg-query() { + case "${*: -1}" in + amdgpu-dkms) echo "install ok installed ${PACKAGE_VERSION}" ;; + amdgpu-dkms-firmware) echo "install ok installed ${FIRMWARE_VERSION}" ;; + *) return 99 ;; + esac + } + # ShellSpec parameters below also contain the literal command name. + # shellcheck disable=SC2120 + dkms() { + echo "dkms $*" >> "${TRACE}" + [ "${FAIL_STAGE}" != dkms ] || return 1 + if [ "$1" = status ]; then + echo "amdgpu/${DKMS_VERSION}, ${KERNEL}, x86_64: installed" + fi + } + depmod() { echo "depmod $*" >> "${TRACE}"; } + modinfo() { + echo "modinfo $*" >> "${TRACE}" + case "$4" in + filename) + if [ "${FAIL_STAGE}" = inbox ]; then + echo "${TEST_DIR}/modules/${KERNEL}/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu.ko" + else + echo "${TEST_DIR}/modules/${KERNEL}/updates/dkms/amdgpu.ko.zst" + fi + ;; + version) if [ "${FAIL_STAGE}" = module ]; then echo wrong-version; else echo "${MODULE_VERSION}"; fi ;; + vermagic) if [ "${FAIL_STAGE}" = kernel ]; then echo wrong-kernel; else echo "${KERNEL} SMP mod_unload modversions "; fi ;; + *) return 99 ;; + esac + } + modprobe() { + echo "modprobe $*" >> "${TRACE}" + [ "$*" = -c ] || return 99 + cat "${TEST_DIR}/modprobe/cloud.conf" + } + update-initramfs() { + echo "update-initramfs $*" >> "${TRACE}" + [ "${FAIL_STAGE}" != initramfs ] + } + run_installer() { + local status=0 + # Conditional callers disable errexit: every mutation must still fail closed. + installAMDGPUDriver || status=$? + echo "remaining-work-files=$(find "${TEST_DIR}/work" -mindepth 1 | wc -l | tr -d ' ')" + return "${status}" + } + + It 'authenticates two pinned packages and validates a DKMS module without GPU hardware' + When run run_installer + The status should be success + The output should eq 'remaining-work-files=0' + The stderr should eq '' + The contents of file "${TEST_DIR}/marker/driver.json" should include '"schema_version": 1' + The contents of file "${TEST_DIR}/marker/driver.json" should include "\"module_version\": \"${MODULE_VERSION}\"" + The contents of file "${TRACE}" should include "download amdgpu-dkms=${PACKAGE_VERSION} amdgpu-dkms-firmware=${FIRMWARE_VERSION}" + The contents of file "${TRACE}" should include 'Dir::Etc::sourceparts=-' + The contents of file "${TRACE}" should include 'AllowUnauthenticated=false' + The contents of file "${TRACE}" should include 'apt-mark manual ca-certificates curl gnupg build-essential dkms autoconf automake initramfs-tools' + The contents of file "${TRACE}" should include 'modprobe -c' + The contents of file "${TRACE}" should not include 'modprobe amdgpu' + The contents of file "${TRACE}" should not include 'rocm-dev' + The contents of file "${TEST_DIR}/modprobe/cloud.conf" should eq "blacklist nouveau +# retain this comment" + End + + It 'rejects a key mismatch before querying the AMD repository and removes stale success state' + FAIL_STAGE=key + echo stale > "${TEST_DIR}/marker/driver.json" + When run run_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'signing key fingerprint mismatch' + The contents of file "${TRACE}" should not include 'apt-get ' + The path "${TEST_DIR}/marker/driver.json" should not be exist + End + + Describe 'installation and validation failures' + Parameters + install + metadata + package + dkms + module + kernel + initramfs + End + It 'cleans temporary repository state and never writes a driver marker' + FAIL_STAGE="$1" + When run run_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should eq '' + The path "${TEST_DIR}/marker/driver.json" should not be exist + End + End + + It 'rejects the inbox kernel module even when its version looks correct' + FAIL_STAGE=inbox + When run run_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'unexpected module' + The path "${TEST_DIR}/marker/driver.json" should not be exist + End + + It 'rejects an archive for the wrong architecture before installation' + PACKAGE_ARCH=arm64 + When run run_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'Unexpected AMDGPU package architecture' + The path "${TEST_DIR}/marker/driver.json" should not be exist + End + + It 'rejects an effective install deny rule without removing that rule' + printf 'install amdgpu /bin/false\n' >> "${TEST_DIR}/modprobe/cloud.conf" + When run run_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'remains disabled' + The contents of file "${TEST_DIR}/modprobe/cloud.conf" should include 'install amdgpu /bin/false' + The path "${TEST_DIR}/marker/driver.json" should not be exist + End + + Describe 'unsupported bake configurations' + Parameters + MARINER 24.04 amd64 v2 false + UBUNTU 22.04 amd64 v2 false + UBUNTU 24.04 arm64 v2 false + UBUNTU 24.04 amd64 v1 false + UBUNTU 24.04 amd64 v2 true + UBUNTU 24.04 amd64 v2 True + End + It 'fails before package or filesystem work' + OS="$1" OS_VERSION="$2" CPU_ARCH="$3" HYPERV_GENERATION="$4" ENABLE_FIPS="$5" + When run run_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'requires Ubuntu 24.04 amd64 Gen2 without FIPS' + The contents of file "${TRACE}" should eq '' + End + End +End + +Describe 'AMD-specific build dispatch' + setup_amd_dispatch() { + TEST_DIR=$(mktemp -d) + TRACE="${TEST_DIR}/trace" + : > "${TRACE}" + FEATURE_FLAGS=AMD_GPU OS=UBUNTU OS_VERSION=24.04 OS_VARIANT='' IS_KATA=false + COMPONENTS_FILEPATH="${TEST_DIR}/components.json" + jq '{Packages: ([.Packages[] | select(.name | test("nvidia|datacenter-gpu-manager|dcgm-exporter"))] + [{name: "retained-package"}])}' \ + parts/common/components.json > "${COMPONENTS_FILEPATH}" + eval "$(sed -n '/^isAMDGPUSkippedPackage()/,/^}$/p' vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh)" + eval "$(sed -n '/^cachePackageAndBinaryComponents()/,/^}$/p' vhdbuilder/packer/install-dependencies.sh)" + } + cleanup_amd_dispatch() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_amd_dispatch + AfterEach cleanup_amd_dispatch + isMariner() { return 1; } + isAzureLinux() { return 1; } + updatePackageVersions() { jq -r .name <<< "$1" >> "${TRACE}"; PACKAGE_VERSIONS=(); } + updatePackageDownloadURL() { PACKAGE_DOWNLOAD_URL=''; } + + It 'excludes all declared NVIDIA payloads before resolving package versions or repositories' + When run cachePackageAndBinaryComponents + The status should be success + The output should include 'Skipping NVIDIA package' + The stderr should eq '' + The contents of file "${TRACE}" should eq 'retained-package' + End + + It 'continues processing NVIDIA packages on existing image SKUs' + FEATURE_FLAGS=NVIDIA_CUDA_PREBAKE + unset -f isAMDGPUSkippedPackage + When run cachePackageAndBinaryComponents + The status should be success + The output should include 'processing components.packages' + The stderr should eq '' + The contents of file "${TRACE}" should include 'nvidia-device-plugin' + The contents of file "${TRACE}" should include 'datacenter-gpu-manager-4-core' + The contents of file "${TRACE}" should include 'retained-package' + End +End + +Describe 'Dedicated AMD image installation hook' + setup_amd_image() { + TEST_DIR=$(mktemp -d) + TRACE="${TEST_DIR}/trace" + : > "${TRACE}" + FEATURE_FLAGS=AMD_GPU OS=UBUNTU OS_VERSION=24.04 CPU_ARCH=amd64 HYPERV_GENERATION=v2 ENABLE_FIPS=false + SCRIPT_NAME=test-build FAIL_STAGE='' + AMD_COMPONENTS_FILEPATH="${TEST_DIR}/amd-gpu/components.json" + eval "$(sed -n '/^validateAMDGPUImageConfiguration()/,/^}$/p' vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh)" + eval "$(sed -n '/^installAMDGPUImage()/,/^}$/p' vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh | + sed 's/^ install / amd_test_install /')" + } + cleanup_amd_image() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_amd_image + AfterEach cleanup_amd_image + amd_test_install() { + case "$2" in + /home/packer/amd-gpu-components.json) + echo "metadata $*" >> "${TRACE}" + [ "${FAIL_STAGE}" != metadata ] + ;; + /home/packer/amd-gpu-validate.sh) + echo "validator $*" >> "${TRACE}" + [ "${FAIL_STAGE}" != validator ] + ;; + *) return 99 ;; + esac + } + installAMDGPUDriver() { echo driver >> "${TRACE}"; [ "${FAIL_STAGE}" != driver ]; } + installAMDGPUDiagnostics() { echo diagnostics >> "${TRACE}"; [ "${FAIL_STAGE}" != diagnostics ]; } + capture_benchmark() { echo "benchmark $1" >> "${TRACE}"; [ "${FAIL_STAGE}" != benchmark ]; } + + It 'copies only dedicated metadata and the baked validator around the AMD installation' + When run installAMDGPUImage + The status should be success + The output should eq '' + The stderr should eq '' + The contents of file "${TRACE}" should eq "metadata -Dm0644 /home/packer/amd-gpu-components.json ${AMD_COMPONENTS_FILEPATH} +driver +benchmark test-build_build_amd_gpu_kernel_module +diagnostics +benchmark test-build_install_amd_gpu_diagnostics +validator -Dm0755 /home/packer/amd-gpu-validate.sh /opt/azure/containers/amd-gpu-validate.sh" + End + + Describe 'incomplete image installation' + Parameters + metadata + driver + diagnostics + benchmark + validator + End + It 'propagates each installation failure' + FAIL_STAGE="$1" + When run installAMDGPUImage + The status should be failure + The output should eq '' + The stderr should eq '' + End + End + + Describe 'unsupported image configuration' + Parameters + NVIDIA_CUDA_PREBAKE UBUNTU 24.04 amd64 v2 false + AMD_GPU,NVIDIA_CUDA_PREBAKE UBUNTU 24.04 amd64 v2 false + AMD_GPU UBUNTU 22.04 amd64 v2 false + AMD_GPU MARINER 24.04 amd64 v2 false + AMD_GPU UBUNTU 24.04 arm64 v2 false + AMD_GPU UBUNTU 24.04 amd64 v1 false + AMD_GPU UBUNTU 24.04 amd64 v2 True + End + It 'fails before changing image content' + FEATURE_FLAGS="$1" OS="$2" OS_VERSION="$3" CPU_ARCH="$4" HYPERV_GENERATION="$5" ENABLE_FIPS="$6" + When run installAMDGPUImage + The status should be failure + The output should eq '' + The stderr should include 'requires the dedicated Ubuntu 24.04 amd64 Gen2 non-FIPS image' + The contents of file "${TRACE}" should eq '' + End + End +End + +Describe 'AMD installer loading does not affect existing images' + setup_shared_bake_hook() { + TEST_DIR=$(mktemp -d) + TRACE="${TEST_DIR}/trace" + : > "${TRACE}" + AMD_SUPPORT_FILE="${TEST_DIR}/amd_gpu.sh" + # Match the literal feature flag expression in the shared source hook. + # shellcheck disable=SC2016 + SHARED_HOOK=$(sed -n '/^case "${FEATURE_FLAGS:-}" in$/,/^esac$/p' vhdbuilder/packer/install-dependencies.sh | + sed "s|/home/packer/amd_gpu.sh|${AMD_SUPPORT_FILE}|g") + [ -n "${SHARED_HOOK}" ] + } + cleanup_shared_bake_hook() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_shared_bake_hook + AfterEach cleanup_shared_bake_hook + run_shared_bake_hook() { eval "${SHARED_HOOK}"; } + + Parameters + None missing + None broken + NVIDIA_CUDA_PREBAKE missing + NVIDIA_CUDA_PREBAKE broken + End + It 'does not load a missing or broken AMD support file on non-AMD images' + FEATURE_FLAGS="$1" + if [ "$2" = broken ]; then + printf 'echo AMD-file-was-sourced >> "%s"\nreturn 99\n' "${TRACE}" > "${AMD_SUPPORT_FILE}" + fi + When run run_shared_bake_hook + The status should be success + The output should eq '' + The stderr should eq '' + The contents of file "${TRACE}" should eq '' + End +End diff --git a/spec/vhdbuilder/packer/amd_gpu_pipeline_spec.sh b/spec/vhdbuilder/packer/amd_gpu_pipeline_spec.sh new file mode 100644 index 00000000000..213e5b15659 --- /dev/null +++ b/spec/vhdbuilder/packer/amd_gpu_pipeline_spec.sh @@ -0,0 +1,194 @@ +#!/bin/bash +# shellcheck disable=SC2034,SC2329 + +Describe 'AMD-only Packer template preparation' + Include './vhdbuilder/packer/amd-gpu-build-settings.sh' + + setup_template() { + TEST_DIR=$(mktemp -d) + TEMPLATE="${TEST_DIR}/template.json" + MAPPINGS="${PWD}/vhdbuilder/packer/amd-gpu-packer-files.json" + cp vhdbuilder/packer/vhd-image-builder-base.json "${TEMPLATE}" + OS_SKU=Ubuntu OS_VERSION=24.04 ARCHITECTURE=X86_64 HYPERV_GENERATION=V2 + FEATURE_FLAGS=AMD_GPU ENABLE_FIPS=False ENABLE_TRUSTED_LAUNCH=False TRUSTED_LAUNCH_SUPPORTED=False + SKU_NAME='' SIG_IMAGE_NAME='' + } + cleanup_template() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_template + AfterEach cleanup_template + + verify_upload_order() { + prepare_amd_gpu_packer_template "${TEMPLATE}" "${MAPPINGS}" || return 1 + jq -e --slurpfile mappings "${MAPPINGS}" --slurpfile original vhdbuilder/packer/vhd-image-builder-base.json ' + [.provisioners | to_entries[] | select(.value.type == "shell") | + select(any(.value.inline[]?; contains(" /home/packer/install-dependencies.sh"))) | .key][0] as $install | + .provisioners[$install - 3:$install] == $mappings[0] and + (.provisioners | map(. as $entry | select(all($mappings[0][]; . != $entry)))) == $original[0].provisioners and + del(.provisioners) == ($original[0] | del(.provisioners)) + ' "${TEMPLATE}" + } + + It 'uploads all three AMD files before installation and preserves the original template content' + When call verify_upload_order + The status should be success + The output should equal true + End + + verify_idempotency() { + prepare_amd_gpu_packer_template "${TEMPLATE}" "${MAPPINGS}" || return 1 + cp "${TEMPLATE}" "${TEST_DIR}/once.json" + prepare_amd_gpu_packer_template "${TEMPLATE}" "${MAPPINGS}" || return 1 + cmp "${TEMPLATE}" "${TEST_DIR}/once.json" + } + + It 'can prepare the worker checkout repeatedly without duplicate uploads' + When call verify_idempotency + The status should be success + The output should be blank + End + + break_template() { + case "$1" in + missing) + jq '.provisioners |= map(select(.type != "shell" or + (any(.inline[]?; contains(" /home/packer/install-dependencies.sh")) | not)))' "${TEMPLATE}" ;; + duplicate) + jq '.provisioners += [.provisioners[] | select(.type == "shell") | + select(any(.inline[]?; contains(" /home/packer/install-dependencies.sh")))]' "${TEMPLATE}" ;; + conflict) + jq '.provisioners += [{"type":"file", "source":"unexpected.sh", "destination":"/home/packer/amd_gpu.sh"}]' "${TEMPLATE}" ;; + esac > "${TEST_DIR}/broken.json" + mv "${TEST_DIR}/broken.json" "${TEMPLATE}" + } + + reject_without_truncation() { + local status=0 + cp "${TEMPLATE}" "${TEST_DIR}/before.json" + prepare_amd_gpu_packer_template "${TEMPLATE}" "${MAPPINGS}" || status=$? + cmp "${TEMPLATE}" "${TEST_DIR}/before.json" || return 99 + return "${status}" + } + + Describe 'invalid base templates' + Parameters + missing 'expected exactly one install-dependencies.sh provisioner' + duplicate 'expected exactly one install-dependencies.sh provisioner' + conflict 'conflicting AMD Packer file destination' + End + + It 'rejects an invalid template without replacing or truncating it' + break_template "$1" + When call reject_without_truncation + The status should equal 5 + The stderr should include "$2" + The output should be blank + End + End + + It 'rejects duplicate AMD destinations without changing the worker template' + jq '. + [.[0]]' "${MAPPINGS}" > "${TEST_DIR}/duplicates.json" + MAPPINGS="${TEST_DIR}/duplicates.json" + When call reject_without_truncation + The status should equal 5 + The stderr should include 'invalid AMD Packer file mappings' + The output should be blank + End + + It 'rejects preparation for an ordinary CPU image' + FEATURE_FLAGS=None + When call reject_without_truncation + The status should equal 1 + The stderr should include 'AMD_GPU requires Ubuntu 24.04 x86_64 Gen2' + The output should be blank + End +End + +Describe 'AMD build resource group cleanup' + Include './vhdbuilder/packer/cleanup-amd-gpu-build.sh' + + setup_cleanup() { + TEST_DIR=$(mktemp -d) + TRACE="${TEST_DIR}/trace" + SETTINGS="${TEST_DIR}/settings.json" + RESOURCE_GROUP=image-builder-recorded BUILD_ID=1234 + GROUP_JSON='{"name":"image-builder-recorded","id":"/subscriptions/subscription-build/resourceGroups/image-builder-recorded","tags":{"buildId":"1234","createdBy":"aks-vhd-pipeline"}}' + SHOW_STATUS=0 + printf '{"subscription_id":"subscription-build"}\n' > "${SETTINGS}" + : > "${TRACE}" + } + cleanup_test() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_cleanup + AfterEach cleanup_test + + az() { + printf '%s\n' "$*" >> "${TRACE}" + case "$1 $2" in + 'group show') printf '%s\n' "${GROUP_JSON}"; return "${SHOW_STATUS}" ;; + 'group delete') return 0 ;; + *) return 99 ;; + esac + } + cleanup_recorded_group() { + cleanup_amd_gpu_build_resource_group "${SETTINGS}" "${RESOURCE_GROUP}" "${BUILD_ID}" + } + + It 'deletes only the recorded group with matching build ownership in the recorded subscription' + When call cleanup_recorded_group + The status should be success + The output should include 'Deleting verified Packer resource group image-builder-recorded for build 1234' + The contents of file "${TRACE}" should equal 'group show --name image-builder-recorded --subscription subscription-build --output json +group delete --name image-builder-recorded --subscription subscription-build --yes --only-show-errors' + End + + Describe 'unverified resource ownership' + Parameters + build_id '.tags.buildId = "another-build"' + missing_tag 'del(.tags.buildId)' + owner '.tags.createdBy = "another-owner"' + name '.name = "another-resource-group"' + subscription '.id = "/subscriptions/another-subscription/resourceGroups/image-builder-recorded"' + End + + It 'does not delete a group without exact recorded ownership' + GROUP_JSON=$(jq "$2" <<< "${GROUP_JSON}") + When call cleanup_recorded_group + The status should be success + The output should include 'ownership does not match this build' + The contents of file "${TRACE}" should include 'group show ' + The contents of file "${TRACE}" should not include 'group delete ' + End + End + + Describe 'missing pipeline identity' + Parameters + RESOURCE_GROUP '' + RESOURCE_GROUP "\$(PKR_RG_NAME)" + BUILD_ID '' + BUILD_ID "\$(Build.BuildId)" + End + + It 'does not call Azure when the recorded group or build ID is unavailable' + printf -v "$1" '%s' "$2" + When call cleanup_recorded_group + The status should be success + The output should include 'skipping AMD build cleanup' + The contents of file "${TRACE}" should be blank + End + End + + It 'does not delete anything when the recorded subscription is missing' + printf '{}\n' > "${SETTINGS}" + When call cleanup_recorded_group + The status should be success + The output should include 'No recorded build subscription' + The contents of file "${TRACE}" should be blank + End + + It 'does not delete anything when the group cannot be read' + SHOW_STATUS=1 + When call cleanup_recorded_group + The status should be success + The output should include 'absent or could not be read' + The contents of file "${TRACE}" should not include 'group delete ' + End +End diff --git a/spec/vhdbuilder/packer/linux_sku_name_spec.sh b/spec/vhdbuilder/packer/linux_sku_name_spec.sh new file mode 100644 index 00000000000..16bd2820c47 --- /dev/null +++ b/spec/vhdbuilder/packer/linux_sku_name_spec.sh @@ -0,0 +1,141 @@ +#!/bin/bash + +Describe 'Linux VHD flavor naming and AMD GPU build constraints' + Include './vhdbuilder/packer/amd-gpu-build-settings.sh' + Include './vhdbuilder/packer/produce-packer-settings-functions.sh' + + setup_environment() { + OS_SKU=Ubuntu + OS_VERSION=24.04 + ARCHITECTURE=X86_64 + HYPERV_GENERATION=V2 + FEATURE_FLAGS=AMD_GPU + ENABLE_FIPS=False + ENABLE_TRUSTED_LAUNCH=False + TRUSTED_LAUNCH_SUPPORTED=False + SKU_NAME='' + SIG_IMAGE_NAME='' + SIG_GALLERY_NAME='' + IMG_OFFER=ubuntu-24_04-lts + } + BeforeEach 'setup_environment' + + It 'uses a dedicated AMD GPU SKU instead of the shared Ubuntu image' + When call get_amd_gpu_sku_name + The status should be success + The output should equal '2404gen2amdgpucontainerd' + End + + derive_sig_name() { + SKU_NAME=$(get_amd_gpu_sku_name) || return 1 + ensure_sig_image_name_linux + } + + It 'uses the dedicated SKU as the default staging gallery image definition' + When call derive_sig_name + The status should be success + The variable SIG_IMAGE_NAME should equal '2404gen2amdgpucontainerd' + The variable SIG_GALLERY_NAME should equal 'PackerSigGalleryEastUS' + The output should be present + End + + Describe 'unsupported AMD configurations' + Parameters + OS_SKU AzureLinux + OS_VERSION 22.04 + OS_VERSION 26.04 + ARCHITECTURE ARM64 + HYPERV_GENERATION V1 + ENABLE_FIPS True + ENABLE_TRUSTED_LAUNCH True + TRUSTED_LAUNCH_SUPPORTED True + FEATURE_FLAGS 'AMD_GPU,NVIDIA_CUDA_PREBAKE' + FEATURE_FLAGS 'AMD_GPU,NVIDIA_GB' + FEATURE_FLAGS 'AMD_GPU,cvm' + FEATURE_FLAGS 'AMD_GPU,minimal' + FEATURE_FLAGS 'NOT_AMD_GPU' + FEATURE_FLAGS None + End + + It 'rejects the unsupported AMD image configuration' + printf -v "$1" '%s' "$2" + When call get_amd_gpu_sku_name + The status should be failure + The stderr should include 'AMD_GPU requires Ubuntu 24.04 x86_64 Gen2' + The output should be blank + End + End + + It 'rejects reusing the existing shared SKU for an AMD build' + SKU_NAME=2404gen2containerd + When call validate_amd_gpu_build + The status should be failure + The stderr should include 'requires the dedicated SKU_NAME' + End + + It 'accepts the generated dedicated SKU during Packer settings generation' + SKU_NAME=2404gen2amdgpucontainerd + When call validate_amd_gpu_build + The status should be success + The output should be blank + End + + It 'rejects overriding the capture definition with the existing shared image' + SKU_NAME=2404gen2amdgpucontainerd + SIG_IMAGE_NAME=2404gen2containerd + When call validate_amd_gpu_build + The status should be failure + The stderr should include 'requires the dedicated SIG_IMAGE_NAME' + End + + It 'accepts an explicit dedicated capture definition' + SKU_NAME=2404gen2amdgpucontainerd + SIG_IMAGE_NAME=2404gen2amdgpucontainerd + When call validate_amd_gpu_build + The status should be success + The output should be blank + End + + # Run the actual pipeline step to verify that ordinary image names still use + # their existing algorithm without loading or invoking the AMD helper/schema. + derive_pipeline_sku_name() { + local script + script=$(awk ' + /^ - bash: \|$/ { script = ""; in_script = 1; next } + /^ displayName: Set SKU Name$/ { printf "%s", script; exit } + in_script && /^ / { sub(/^ /, ""); script = script $0 "\n" } + ' .pipelines/templates/.builder-release-template.yaml) + eval "${script}" >/dev/null || return 1 + printf '%s\n' "${SKU_NAME}" + } + + Describe 'existing image flavors' + Parameters + Ubuntu 22.04 X86_64 V1 None False False 2204containerd + Ubuntu 22.04 X86_64 V2 NVIDIA_CUDA_PREBAKE False False 2204gen2containerd + Ubuntu 24.04 X86_64 V2 NVIDIA_CUDA_PREBAKE False False 2404gen2containerd + Ubuntu 24.04 ARM64 V2 NVIDIA_GB False False 2404gen2arm64gbcontainerd + Ubuntu 22.04 X86_64 V2 None True True 2204gen2fipsTLcontainerd + Ubuntu 24.04 X86_64 V2 cvm False False 2404gen2CVMcontainerd + Ubuntu 26.04 ARM64 V2 minimal False False 2604minimalgen2arm64containerd + AzureLinux V3kata X86_64 V2 kata False False V3katagen2 + AzureLinuxOSGuard V3 X86_64 V2 None True True V3gen2fipsTL + CBLMariner V2 X86_64 V1 None False False V2 + Flatcar '' ARM64 V2 None False False gen2arm64 + AzureContainerLinux acl ARM64 V2 None True True aclgen2arm64fipsTL + End + + It 'preserves the existing SKU name' + OS_SKU="$1" + OS_VERSION="$2" + ARCHITECTURE="$3" + HYPERV_GENERATION="$4" + FEATURE_FLAGS="$5" + ENABLE_FIPS="$6" + ENABLE_TRUSTED_LAUNCH="$7" + When call derive_pipeline_sku_name + The status should be success + The output should equal "$8" + End + End +End diff --git a/spec/vhdbuilder/packer/test/amd_gpu_content_spec.sh b/spec/vhdbuilder/packer/test/amd_gpu_content_spec.sh new file mode 100644 index 00000000000..380f3a2d8e9 --- /dev/null +++ b/spec/vhdbuilder/packer/test/amd_gpu_content_spec.sh @@ -0,0 +1,340 @@ +#!/bin/bash +# shellcheck disable=SC2034,SC2329 + +Describe 'AMD VHD content validation' + setup_amd_content() { + TEST_DIR=$(mktemp -d) + FEATURE_FLAGS=AMD_GPU OS_SKU=Ubuntu OS_VERSION=24.04 ENABLE_FIPS=false + AMD_COMPONENTS_FILEPATH="${PWD}/vhdbuilder/packer/amd-gpu-components.json" + KERNEL=6.8.0-test-azure + FAIL_STAGE="" + PACKAGE_VERSION=$(jq -r '.AMDGPUDriver.packageVersion' "${AMD_COMPONENTS_FILEPATH}") + FIRMWARE_VERSION=$(jq -r '.AMDGPUDriver.firmwarePackageVersion' "${AMD_COMPONENTS_FILEPATH}") + MODULE_VERSION=$(jq -r '.AMDGPUDriver.moduleVersion' "${AMD_COMPONENTS_FILEPATH}") + DKMS_VERSION=$(jq -r '.AMDGPUDriver.dkmsVersion' "${AMD_COMPONENTS_FILEPATH}") + AMDSMI_PACKAGE=$(jq -r '.AMDGPUDiagnostics.amdsmiPackage' "${AMD_COMPONENTS_FILEPATH}") + SYSDEPS_PACKAGE=$(jq -r '.AMDGPUDiagnostics.sysdepsPackage' "${AMD_COMPONENTS_FILEPATH}") + mkdir -p "${TEST_DIR}/marker" "${TEST_DIR}/modules/${KERNEL}/build" "${TEST_DIR}/sources/amdgpu-${DKMS_VERSION}" + touch "${TEST_DIR}/modules/${KERNEL}/build/Makefile" "${TEST_DIR}/sources/amdgpu-${DKMS_VERSION}/dkms.conf" + jq -n --arg driver "${PACKAGE_VERSION}" --arg firmware "${FIRMWARE_VERSION}" --arg module_version "${MODULE_VERSION}" \ + '{schema_version: 1, package_version: $driver, firmware_package_version: $firmware, module_version: $module_version, + kernel_version: "earlier-bake-kernel"}' > "${TEST_DIR}/marker/driver.json" + eval "$(sed -n '/^testAMDGPUDriver()/,/^}$/p' vhdbuilder/packer/test/amd-gpu-vhd-content-test.sh | + sed -e "s|/opt/azure/amd-gpu|${TEST_DIR}/marker|g" -e "s|/lib/modules|${TEST_DIR}/modules|g" \ + -e "s|/usr/src|${TEST_DIR}/sources|g")" + } + cleanup_amd_content() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_amd_content + AfterEach cleanup_amd_content + err() { printf '%s: %s\n' "$1" "$2" >&2; } + uname() { if [ "$1" = -m ]; then echo x86_64; else echo "${KERNEL}"; fi; } + dpkg-query() { + if [ "$#" -eq 2 ]; then + printf 'ii amdgpu-dkms\nii amdgpu-dkms-firmware\n' + printf 'ii %s\nii %s:amd64\n' "${AMDSMI_PACKAGE}" "${SYSDEPS_PACKAGE}" + if [ "${FAIL_STAGE}" = sdk ]; then echo 'ii amdrocm-hip-sdk'; fi + if [ "${FAIL_STAGE}" = similar_package ]; then echo "ii ${AMDSMI_PACKAGE}-dev"; fi + if [ "${FAIL_STAGE}" = older_diagnostics ]; then echo 'ii amdrocm-amdsmi9.0'; fi + return + fi + case "${*: -1}" in + amdgpu-dkms) echo "install ok installed ${PACKAGE_VERSION}" ;; + amdgpu-dkms-firmware) + if [ "${FAIL_STAGE}" = firmware ]; then echo 'install ok installed wrong'; else echo "install ok installed ${FIRMWARE_VERSION}"; fi + ;; + build-essential) if [ "${FAIL_STAGE}" = dependencies ]; then return 1; else echo 'install ok installed'; fi ;; + *) echo 'install ok installed' ;; + esac + } + dkms() { + if [ "${FAIL_STAGE}" = registration ]; then + echo "amdgpu/${DKMS_VERSION}, ${KERNEL}, x86_64: added" + else + echo "amdgpu/${DKMS_VERSION}, ${KERNEL}, x86_64: installed" + fi + } + modinfo() { + case "$4" in + filename) + if [ "${FAIL_STAGE}" = inbox ]; then echo /kernel/inbox/amdgpu.ko; + else echo "${TEST_DIR}/modules/${KERNEL}/updates/dkms/amdgpu.ko.zst"; fi + ;; + version) if [ "${FAIL_STAGE}" = version ]; then echo wrong; else echo "${MODULE_VERSION}"; fi ;; + vermagic) if [ "${FAIL_STAGE}" = kernel ]; then echo wrong; else echo "${KERNEL} SMP mod_unload modversions"; fi ;; + esac + } + modprobe() { + [ "$*" = -c ] || { echo 'unexpected hardware-dependent command' >&2; return 99; } + if [ "${FAIL_STAGE}" = blacklist ]; then echo 'blacklist amdgpu'; fi + } + + It 'validates retained DKMS state on a CPU machine, treating bake kernel as provenance' + When run testAMDGPUDriver + The status should be success + The output should include 'rebuild dependencies verified' + The stderr should eq '' + End + + It 'skips ordinary images without inspecting AMD state' + FEATURE_FLAGS=NVIDIA_CUDA_PREBAKE + rm "${TEST_DIR}/marker/driver.json" + When run testAMDGPUDriver + The status should be success + The output should eq '' + The stderr should eq '' + End + + Describe 'unqualified image contents' + Parameters + firmware 'differs from pinned versions' + registration 'not installed for the running kernel' + inbox 'inbox or a missing module' + version 'version or vermagic mismatch' + kernel 'version or vermagic mismatch' + dependencies 'Missing retained build dependency' + sdk 'Unexpected GPU userspace' + similar_package 'Unexpected GPU userspace' + older_diagnostics 'Unexpected GPU userspace' + blacklist 'disabled in modprobe configuration' + End + It 'reports every failure on stderr as required by the VM content-test runner' + FAIL_STAGE="$1" + When run testAMDGPUDriver + The status should be failure + The output should eq '' + The stderr should include "$2" + End + End + + It 'reports missing marker metadata on stderr' + printf '{}\n' > "${TEST_DIR}/marker/driver.json" + When run testAMDGPUDriver + The status should be failure + The output should eq '' + The stderr should include 'Missing or inconsistent AMDGPU driver marker' + End + + It 'reports removed DKMS source even if the compiled module still exists' + rm "${TEST_DIR}/sources/amdgpu-${DKMS_VERSION}/dkms.conf" + When run testAMDGPUDriver + The status should be failure + The output should eq '' + The stderr should include 'Headers or AMDGPU sources' + End +End + +Describe 'AMD SMI VHD content validation' + setup_amd_diagnostics_content() { + TEST_DIR=$(mktemp -d) + FEATURE_FLAGS=AMD_GPU + FAIL_STAGE="" + AMD_COMPONENTS_FILEPATH="${TEST_DIR}/components.json" + CLI_PATH="${TEST_DIR}/rocm/core-10.0/bin/amd-smi" + CLI_TARGET="${TEST_DIR}/rocm/core-10.0/libexec/amdsmi_cli/amdsmi_cli.py" + MODULE_PATH="${TEST_DIR}/rocm/core-10.0/share/amd_smi/amdsmi/__init__.py" + mkdir -p "$(dirname "${CLI_PATH}")" "$(dirname "${CLI_TARGET}")" "$(dirname "${MODULE_PATH}")" \ + "${TEST_DIR}/bin" "${TEST_DIR}/environment" + printf '#!/bin/sh\nexit 99\n' > "${CLI_TARGET}" + ln -s ../libexec/amdsmi_cli/amdsmi_cli.py "${CLI_PATH}" + chmod +x "${CLI_PATH}" + ln -s "${CLI_PATH}" "${TEST_DIR}/bin/amd-smi" + cat > "${MODULE_PATH}" <<'PY' +from pathlib import Path +__version__ = "27.0.0-test" +def amdsmi_get_lib_version(): + Path(__file__).with_name("library-queried").touch() + return {"major": 27, "minor": 0, "release": 0} +def amdsmi_init(): + raise RuntimeError("GPU initialization must not run on the CPU bake machine") +PY + printf 'raise ImportError("unrelated PYTHONPATH must not affect the bake check")\n' > "${TEST_DIR}/environment/amdsmi.py" + jq --arg cli "${CLI_PATH}" '.AMDGPUDiagnostics.cliPath = $cli' vhdbuilder/packer/amd-gpu-components.json > "${AMD_COMPONENTS_FILEPATH}" + AMDSMI_PACKAGE=$(jq -r '.AMDGPUDiagnostics.amdsmiPackage' "${AMD_COMPONENTS_FILEPATH}") + AMDSMI_VERSION=$(jq -r '.AMDGPUDiagnostics.amdsmiVersion' "${AMD_COMPONENTS_FILEPATH}") + SYSDEPS_PACKAGE=$(jq -r '.AMDGPUDiagnostics.sysdepsPackage' "${AMD_COMPONENTS_FILEPATH}") + SYSDEPS_VERSION=$(jq -r '.AMDGPUDiagnostics.sysdepsVersion' "${AMD_COMPONENTS_FILEPATH}") + eval "$(sed -n '/^testAMDGPUDiagnostics()/,/^}$/p' vhdbuilder/packer/test/amd-gpu-vhd-content-test.sh | + sed -e "s|/usr/local/bin/amd-smi|${TEST_DIR}/bin/amd-smi|g" \ + -e "s|/opt/rocm/|${TEST_DIR}/rocm/|g")" + } + cleanup_amd_diagnostics_content() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_amd_diagnostics_content + AfterEach cleanup_amd_diagnostics_content + err() { printf '%s: %s\n' "$1" "$2" >&2; } + dpkg-query() { + case "${*: -1}" in + "${AMDSMI_PACKAGE}") + if [ "${FAIL_STAGE}" = amdsmi ]; then echo 'install ok installed wrong'; else echo "install ok installed ${AMDSMI_VERSION}"; fi + ;; + "${SYSDEPS_PACKAGE}") + if [ "${FAIL_STAGE}" = sysdeps ]; then echo 'deinstall ok config-files'; else echo "install ok installed ${SYSDEPS_VERSION}"; fi + ;; + pciutils|numactl) + if [ "${FAIL_STAGE}" = "${*: -1}_package" ]; then echo 'deinstall ok config-files'; else echo 'install ok installed'; fi + ;; + esac + } + lspci() { + [ "$*" = --version ] || return 99 + [ "${FAIL_STAGE}" != lspci_cli ] || return 1 + echo 'lspci version 3.10.0' + } + numactl() { + [ "$*" = --version ] || return 99 + [ "${FAIL_STAGE}" != numactl_cli ] || return 1 + echo 'numactl version 2.0.18' + } + + It 'imports the library beside the vendor bin-to-libexec symlink with isolated Python' + PYTHONPATH="${TEST_DIR}/environment" + export PYTHONPATH + When run testAMDGPUDiagnostics + The status should be success + The output should include 'hardware-independent library loading verified' + The stderr should eq '' + The path "${TEST_DIR}/rocm/core-10.0/share/amd_smi/amdsmi/library-queried" should be exist + End + + It 'skips ordinary images' + FEATURE_FLAGS=NVIDIA_CUDA_PREBAKE + rm "${TEST_DIR}/bin/amd-smi" + When run testAMDGPUDiagnostics + The status should be success + The output should eq '' + The stderr should eq '' + End + + Describe 'incomplete diagnostics packages' + Parameters + amdsmi 'differ from pinned packages' + sysdeps 'differ from pinned packages' + library 'library failed to load' + pciutils_package 'Missing host diagnostics package pciutils' + numactl_package 'Missing host diagnostics package numactl' + lspci_cli 'Host diagnostics command lspci failed' + numactl_cli 'Host diagnostics command numactl failed' + End + It 'fails the content check' + FAIL_STAGE="$1" + if [ "${FAIL_STAGE}" = library ]; then printf 'raise ImportError("missing AMD SMI library")\n' > "${MODULE_PATH}"; fi + When run testAMDGPUDiagnostics + The status should be failure + The output should eq '' + The stderr should include "$2" + End + End + + It 'rejects a command symlink pointing to a different executable' + ln -sfn /bin/false "${TEST_DIR}/bin/amd-smi" + When run testAMDGPUDiagnostics + The status should be failure + The output should eq '' + The stderr should include 'executable or command symlink' + End + + It 'rejects a non-executable CLI' + chmod -x "${CLI_PATH}" + When run testAMDGPUDiagnostics + The status should be failure + The output should eq '' + The stderr should include 'executable or command symlink' + End + + It 'reports missing diagnostics metadata' + printf '{}\n' > "${AMD_COMPONENTS_FILEPATH}" + When run testAMDGPUDiagnostics + The status should be failure + The output should eq '' + The stderr should include 'Missing AMDGPU diagnostics component metadata' + End +End + +Describe 'Dedicated AMD image content hook' + setup_amd_content_hook() { + TEST_DIR=$(mktemp -d) + TRACE="${TEST_DIR}/trace" + : > "${TRACE}" + touch "${TEST_DIR}/amd-gpu-validate.sh" + chmod +x "${TEST_DIR}/amd-gpu-validate.sh" + FAIL_STAGE='' + eval "$(sed -n '/^testAMDGPUImage()/,/^}$/p' vhdbuilder/packer/test/amd-gpu-vhd-content-test.sh | + sed "s|/opt/azure/containers/amd-gpu-validate.sh|${TEST_DIR}/amd-gpu-validate.sh|g")" + } + cleanup_amd_content_hook() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_amd_content_hook + AfterEach cleanup_amd_content_hook + err() { printf '%s: %s\n' "$1" "$2" >&2; } + testAMDGPUDriver() { echo driver >> "${TRACE}"; [ "${FAIL_STAGE}" != driver ]; } + testAMDGPUDiagnostics() { echo diagnostics >> "${TRACE}"; [ "${FAIL_STAGE}" != diagnostics ]; } + + It 'checks the driver, diagnostics and executable bootstrap validator' + When run testAMDGPUImage + The status should be success + The output should eq '' + The stderr should eq '' + The contents of file "${TRACE}" should eq "driver +diagnostics" + End + + It 'fails a missing bootstrap validator while still checking installed content' + rm "${TEST_DIR}/amd-gpu-validate.sh" + When run testAMDGPUImage + The status should be failure + The output should eq '' + The stderr should include 'bootstrap validator is missing or not executable' + The contents of file "${TRACE}" should eq "driver +diagnostics" + End + + Describe 'failed component checks' + Parameters + driver + diagnostics + End + It 'preserves failure even if the other component succeeds' + FAIL_STAGE="$1" + When run testAMDGPUImage + The status should be failure + The output should eq '' + The stderr should eq '' + The contents of file "${TRACE}" should eq "driver +diagnostics" + End + End +End + +Describe 'AMD content checks do not affect existing images' + setup_shared_content_hook() { + TEST_DIR=$(mktemp -d) + TRACE="${TEST_DIR}/trace" + : > "${TRACE}" + AMD_SUPPORT_FILE="${TEST_DIR}/amd-gpu-vhd-content-test.sh" + # Match the literal feature flag expression in the shared source hook. + # shellcheck disable=SC2016 + SHARED_HOOK=$(sed -n '/^if \[ "${FEATURE_FLAGS:-}" = "AMD_GPU" \]; then$/,/^fi$/p' vhdbuilder/packer/test/linux-vhd-content-test.sh | + sed "s|./AgentBaker/vhdbuilder/packer/test/amd-gpu-vhd-content-test.sh|${AMD_SUPPORT_FILE}|g") + [ -n "${SHARED_HOOK}" ] + } + cleanup_shared_content_hook() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_shared_content_hook + AfterEach cleanup_shared_content_hook + run_shared_content_hook() { eval "${SHARED_HOOK}"; } + + Parameters + None missing + None broken + NVIDIA_CUDA_PREBAKE missing + NVIDIA_CUDA_PREBAKE broken + End + It 'does not load missing or broken AMD tests on non-AMD images' + FEATURE_FLAGS="$1" + if [ "$2" = broken ]; then + printf 'echo AMD-file-was-sourced >> "%s"\nreturn 99\n' "${TRACE}" > "${AMD_SUPPORT_FILE}" + fi + When run run_shared_content_hook + The status should be success + The output should eq '' + The stderr should eq '' + The contents of file "${TRACE}" should eq '' + End +End diff --git a/vhdbuilder/packer/amd-gpu-build-settings.sh b/vhdbuilder/packer/amd-gpu-build-settings.sh new file mode 100644 index 00000000000..6e4a4ee050e --- /dev/null +++ b/vhdbuilder/packer/amd-gpu-build-settings.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Loaded only by an explicit AMD build. Shared CPU/NVIDIA settings stay in their +# existing path and do not depend on this image's configuration or validation. +function validate_amd_gpu_build() { + if [ "${FEATURE_FLAGS:-}" != "AMD_GPU" ] || [ "${OS_SKU:-}" != "Ubuntu" ] || + [ "${OS_VERSION:-}" != "24.04" ] || [ "${ARCHITECTURE,,}" != "x86_64" ] || + [ "${HYPERV_GENERATION,,}" != "v2" ] || [ "${ENABLE_FIPS,,}" = "true" ] || + [ "${ENABLE_TRUSTED_LAUNCH,,}" = "true" ] || [ "${TRUSTED_LAUNCH_SUPPORTED,,}" = "true" ]; then + echo "AMD_GPU requires Ubuntu 24.04 x86_64 Gen2, without other feature flags, FIPS or Trusted Launch" >&2 + return 1 + fi + if [ -n "${SKU_NAME:-}" ] && [ "${SKU_NAME}" != "2404gen2amdgpucontainerd" ]; then + echo "AMD_GPU requires the dedicated SKU_NAME 2404gen2amdgpucontainerd" >&2 + return 1 + fi + if [ -n "${SIG_IMAGE_NAME:-}" ] && [ "${SIG_IMAGE_NAME}" != "2404gen2amdgpucontainerd" ]; then + echo "AMD_GPU requires the dedicated SIG_IMAGE_NAME 2404gen2amdgpucontainerd" >&2 + return 1 + fi +} + +function get_amd_gpu_sku_name() { + validate_amd_gpu_build || return 1 + printf '%s\n' '2404gen2amdgpucontainerd' +} + +# Only the manual AMD pipeline calls this function. The shared checked-in Packer +# template stays unchanged, and ordinary image builds never upload AMD files. +function prepare_amd_gpu_packer_template() { + local template="$1" mappings="$2" temporary status + validate_amd_gpu_build || return 1 + temporary=$(mktemp "${template}.amd.XXXXXX") || return 1 + jq --slurpfile mappings "${mappings}" ' + def install_boundary: + .type == "shell" and any(.inline[]?; + test("(^|[[:space:]])/home/packer/install-dependencies\\.sh([[:space:]]|$)")); + $mappings[0] as $uploads | + if ($mappings | length) != 1 or ($uploads | type) != "array" or + ($uploads | length) == 0 or + (all($uploads[]; .type == "file" and (.source | type) == "string" and + (.destination | type) == "string") | not) or + ([$uploads[].destination] | length) != ([$uploads[].destination] | unique | length) + then error("invalid AMD Packer file mappings") + elif any(.provisioners[]; . as $existing | + any($uploads[]; .destination == $existing.destination and . != $existing)) + then error("conflicting AMD Packer file destination") + else .provisioners |= ( + map(. as $existing | select(all($uploads[]; . != $existing))) | + [to_entries[] | select(.value | install_boundary) | .key] as $boundaries | + if ($boundaries | length) != 1 + then error("expected exactly one install-dependencies.sh provisioner") + else .[:$boundaries[0]] + $uploads + .[$boundaries[0]:] + end) + end + ' "${template}" >"${temporary}" && mv "${temporary}" "${template}" + status=$? + rm -f "${temporary}" + return "${status}" +} diff --git a/vhdbuilder/packer/amd-gpu-components.json b/vhdbuilder/packer/amd-gpu-components.json new file mode 100644 index 00000000000..5edd901a079 --- /dev/null +++ b/vhdbuilder/packer/amd-gpu-components.json @@ -0,0 +1,25 @@ +{ + "AMDGPUDriver": { + "repositoryURL": "https://repo.radeon.com/amdgpu/31.50/ubuntu", + "signingKeyURL": "https://repo.radeon.com/rocm/rocm.gpg.key", + "signingKeyFingerprint": "CA8BB4727A47B4D09B4EE8969386B48A1A693C5C", + "distribution": "noble", + "component": "main", + "packageVersion": "1:7.1.3.31500000-2390945.24.04", + "firmwarePackageVersion": "1:31.50.0.0.31500000-2390945.24.04", + "moduleVersion": "7.1.3.31500000", + "dkmsVersion": "7.1.3-2390945.24.04" + }, + "AMDGPUDiagnostics": { + "repositoryURL": "https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/", + "signingKeyURL": "https://stable.repo.amd.com/rocm/gpg/packages.gpg", + "signingKeyFingerprint": "D0F004A0025A1145C7807FCD0701EAC4D5E02107", + "distribution": "stable", + "component": "main", + "amdsmiPackage": "amdrocm-amdsmi10.0", + "amdsmiVersion": "10.0.0-4", + "sysdepsPackage": "amdrocm-sysdeps10.0", + "sysdepsVersion": "10.0.0-4", + "cliPath": "/opt/rocm/core-10.0/bin/amd-smi" + } +} diff --git a/vhdbuilder/packer/amd-gpu-packer-files.json b/vhdbuilder/packer/amd-gpu-packer-files.json new file mode 100644 index 00000000000..41cf7ae4ebd --- /dev/null +++ b/vhdbuilder/packer/amd-gpu-packer-files.json @@ -0,0 +1,17 @@ +[ + { + "type": "file", + "source": "vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh", + "destination": "/home/packer/amd_gpu.sh" + }, + { + "type": "file", + "source": "vhdbuilder/packer/amd-gpu-components.json", + "destination": "/home/packer/amd-gpu-components.json" + }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/cse_amd_gpu.sh", + "destination": "/home/packer/amd-gpu-validate.sh" + } +] diff --git a/vhdbuilder/packer/cleanup-amd-gpu-build.sh b/vhdbuilder/packer/cleanup-amd-gpu-build.sh new file mode 100644 index 00000000000..334335eaffa --- /dev/null +++ b/vhdbuilder/packer/cleanup-amd-gpu-build.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# The shared cleanup script also collects unrelated old images, galleries and +# resource groups. This manual pipeline cleans only its recorded Packer group. +function cleanup_amd_gpu_build_resource_group() { + local settings_file="$1" resource_group="$2" build_id="$3" + local subscription group + case "${resource_group}" in + '' | *"\$("*) + echo 'No recorded Packer resource group; skipping AMD build cleanup' + return 0 + ;; + esac + case "${build_id}" in + '' | *[!0-9]*) + echo 'No valid build ID; skipping AMD build cleanup' + return 0 + ;; + esac + subscription=$(jq -er '.subscription_id | select(type == "string" and length > 0)' "${settings_file}") || { + echo 'No recorded build subscription; skipping AMD build cleanup' + return 0 + } + case "${subscription}" in + *"\$("*) + echo 'Unresolved build subscription; skipping AMD build cleanup' + return 0 + ;; + esac + group=$(az group show --name "${resource_group}" --subscription "${subscription}" --output json) || { + echo 'Packer resource group is absent or could not be read; skipping AMD build cleanup' + return 0 + } + if ! jq -e --arg build_id "${build_id}" --arg resource_group "${resource_group}" --arg subscription "${subscription}" ' + .tags.buildId == $build_id and .tags.createdBy == "aks-vhd-pipeline" and + (.name | ascii_downcase) == ($resource_group | ascii_downcase) and + (.id | ascii_downcase) == + ("/subscriptions/" + $subscription + "/resourceGroups/" + $resource_group | ascii_downcase) + ' <<<"${group}" >/dev/null; then + echo 'Packer resource group ownership does not match this build; skipping AMD build cleanup' + return 0 + fi + echo "Deleting verified Packer resource group ${resource_group} for build ${build_id}" + az group delete --name "${resource_group}" --subscription "${subscription}" --yes --only-show-errors +} diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index dd4d4cd18a8..14461471b49 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -42,6 +42,13 @@ LOCALDNS_BINARY_PATH="/opt/azure/containers/localdns/binary" PERFORMANCE_DATA_FILE=/opt/azure/vhd-build-performance-data.json GRID_COMPATIBILITY_DATA_FILE=/opt/azure/vhd-grid-compatibility-data.json +case "${FEATURE_FLAGS:-}" in + *AMD_GPU*) + source /home/packer/amd_gpu.sh + validateAMDGPUImageConfiguration || exit 1 + ;; +esac + string_replace() { echo ${1//\*/$2} } @@ -505,6 +512,10 @@ cachePackageAndBinaryComponents() { while IFS= read -r p; do #getting metadata for each package name=$(echo "${p}" | jq .name -r) + if [ "${FEATURE_FLAGS:-}" = "AMD_GPU" ] && isAMDGPUSkippedPackage "${name}"; then + echo "Skipping NVIDIA package ${name} on the AMD GPU image" + continue + fi os=${OS} # TODO(mheberling): Remove this once kata uses standard containerd. This OS is referenced # in file `parts/common/component.json` with the same ${MARINER_KATA_OS_NAME}. @@ -1211,7 +1222,9 @@ if isMarinerOrAzureLinux "$OS" && ! isAzureLinuxOSGuard "$OS" "$OS_VARIANT"; the activateNfConntrack elif [ "${OS}" = "${UBUNTU_OS_NAME}" ]; then updateAptWithMicrosoftPkg - updateAptWithNvidiaPkg + if [ "${FEATURE_FLAGS:-}" != "AMD_GPU" ]; then + updateAptWithNvidiaPkg + fi fi capture_benchmark "${SCRIPT_NAME}_handle_os_specific_configurations" @@ -1228,9 +1241,13 @@ ctr namespace create k8s.io # running them here (near-empty disk, BCC not yet started) avoids exhausting the 30GB packer build disk. # Running them after the container-image cache and/or concurrently with the BCC build fills the disk # (worse on 24.04), failing at the nvidia.ko link or the driver lib copy with "No space left on device". -cacheGPUContainerImageComponents -buildNVIDIAKernelModule -capture_benchmark "${SCRIPT_NAME}_caching_gpu_container_images_and_build_nvidia_kernel_module" +if [ "${FEATURE_FLAGS:-}" = "AMD_GPU" ]; then + installAMDGPUImage || exit 1 +else + cacheGPUContainerImageComponents + buildNVIDIAKernelModule + capture_benchmark "${SCRIPT_NAME}_caching_gpu_container_images_and_build_nvidia_kernel_module" +fi # Start eBPF tool installation in the background while we pull container images in the foreground startEBPFToolsInstallation diff --git a/vhdbuilder/packer/produce-packer-settings.sh b/vhdbuilder/packer/produce-packer-settings.sh index 248f63ed65d..be6b2817eb6 100755 --- a/vhdbuilder/packer/produce-packer-settings.sh +++ b/vhdbuilder/packer/produce-packer-settings.sh @@ -4,6 +4,15 @@ set -e SCRIPT_DIR=$(dirname "$0") source "$SCRIPT_DIR/produce-packer-settings-functions.sh" +if [ "${MODE}" = "linuxVhdMode" ]; then + case "${FEATURE_FLAGS:-}" in + *AMD_GPU*) + source "$SCRIPT_DIR/amd-gpu-build-settings.sh" + validate_amd_gpu_build + ;; + esac +fi + CDIR=$(dirname "${BASH_SOURCE}") SETTINGS_JSON="${SETTINGS_JSON:-./packer/settings.json}" PUBLISHER_BASE_IMAGE_VERSION_JSON="${PUBLISHER_BASE_IMAGE_VERSION_JSON:-./vhdbuilder/publisher_base_image_version.json}" diff --git a/vhdbuilder/packer/test/amd-gpu-vhd-content-test.sh b/vhdbuilder/packer/test/amd-gpu-vhd-content-test.sh new file mode 100644 index 00000000000..ea8a35bfb7d --- /dev/null +++ b/vhdbuilder/packer/test/amd-gpu-vhd-content-test.sh @@ -0,0 +1,153 @@ +#!/bin/bash +# Sourced by linux-vhd-content-test.sh only for the dedicated AMD_GPU image. + +testAMDGPUImage() { + local status=0 + if [ ! -x /opt/azure/containers/amd-gpu-validate.sh ]; then + err testAMDGPUImage "Baked AMD GPU bootstrap validator is missing or not executable" + status=1 + fi + testAMDGPUDriver || status=1 + testAMDGPUDiagnostics || status=1 + return "${status}" +} + +# Hardware-independent validation: the AMD VHD is baked and content-tested on a +# CPU VM. Loading the module and checking KFD/GPU devices belongs to node CSE/E2E. +# shellcheck disable=SC2016 +testAMDGPUDriver() { + [ "${FEATURE_FLAGS:-}" = "AMD_GPU" ] || return 0 + local components_file="${AMD_COMPONENTS_FILEPATH:-/opt/azure/amd-gpu/components.json}" + local test=testAMDGPUDriver marker=/opt/azure/amd-gpu/driver.json + local metadata kernel_version module_path module_vermagic package_version firmware_version module_version dkms_version + local installed_packages modprobe_config package diagnostics_amdsmi diagnostics_sysdeps + if [ "${OS_SKU}" != "Ubuntu" ] || [ "${OS_VERSION}" != "24.04" ] || + [ "${ENABLE_FIPS,,}" = "true" ] || [ "$(uname -m)" != "x86_64" ]; then + err "${test}" "Unsupported AMD GPU image configuration" + return 1 + fi + if ! metadata=$(jq -ce '.AMDGPUDriver | select(type == "object")' "${components_file}") || + ! package_version=$(jq -er '.packageVersion | strings | select(length > 0)' <<< "${metadata}") || + ! firmware_version=$(jq -er '.firmwarePackageVersion | strings | select(length > 0)' <<< "${metadata}") || + ! module_version=$(jq -er '.moduleVersion | strings | select(length > 0)' <<< "${metadata}") || + ! dkms_version=$(jq -er '.dkmsVersion | strings | select(length > 0)' <<< "${metadata}"); then + err "${test}" "Missing AMDGPU component metadata" + return 1 + fi + if ! jq -e --arg package "${package_version}" --arg firmware "${firmware_version}" --arg module_version "${module_version}" \ + '.schema_version == 1 and .package_version == $package and .firmware_package_version == $firmware and + .module_version == $module_version and (.kernel_version | type == "string" and length > 0)' "${marker}" >/dev/null; then + err "${test}" "Missing or inconsistent AMDGPU driver marker" + return 1 + fi + if [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms)" != "install ok installed ${package_version}" ] || + [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms-firmware)" != "install ok installed ${firmware_version}" ]; then + err "${test}" "Installed AMDGPU driver/firmware differs from pinned versions" + return 1 + fi + kernel_version=$(uname -r) + if ! dkms status -m amdgpu -v "${dkms_version}" -k "${kernel_version}" | + grep -Fqx "amdgpu/${dkms_version}, ${kernel_version}, x86_64: installed"; then + err "${test}" "AMDGPU DKMS is not installed for the running kernel" + return 1 + fi + module_path=$(modinfo -k "${kernel_version}" -F filename amdgpu) + case "${module_path}" in + /lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*|/usr/lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*) ;; + *) err "${test}" "AMDGPU resolves to the inbox or a missing module"; return 1 ;; + esac + module_vermagic=$(modinfo -k "${kernel_version}" -F vermagic amdgpu) + if [ "$(modinfo -k "${kernel_version}" -F version amdgpu)" != "${module_version}" ] || + [ "${module_vermagic%% *}" != "${kernel_version}" ]; then + err "${test}" "AMDGPU module version or vermagic mismatch" + return 1 + fi + if [ ! -f "/lib/modules/${kernel_version}/build/Makefile" ] || [ ! -f "/usr/src/amdgpu-${dkms_version}/dkms.conf" ]; then + err "${test}" "Headers or AMDGPU sources required by future DKMS rebuilds are missing" + return 1 + fi + for package in build-essential dkms autoconf automake initramfs-tools "linux-headers-${kernel_version}" "linux-modules-extra-${kernel_version}"; do + if [ "$(dpkg-query -W -f='${Status}' "${package}")" != 'install ok installed' ]; then + err "${test}" "Missing retained build dependency ${package}" + return 1 + fi + done + if ! installed_packages=$(dpkg-query -W -f='${db:Status-Abbrev} ${binary:Package}\n'); then + err "${test}" "Unable to inspect installed packages" + return 1 + fi + if ! diagnostics_amdsmi=$(jq -er '.AMDGPUDiagnostics.amdsmiPackage | strings | select(length > 0)' "${components_file}") || + ! diagnostics_sysdeps=$(jq -er '.AMDGPUDiagnostics.sysdepsPackage | strings | select(length > 0)' "${components_file}"); then + err "${test}" "Missing AMDGPU diagnostics package allowlist" + return 1 + fi + if awk -v amdsmi="${diagnostics_amdsmi}" -v sysdeps="${diagnostics_sysdeps}" \ + '$1 == "ii" { sub(/:.*/, "", $2); if ($2 != amdsmi && $2 != sysdeps) print $2 }' <<< "${installed_packages}" | + grep -Eq '^(amdrocm|rocm|hip|hsa-rocr|rocblas|rocfft|rocrand|rocsolver|rocsparse|miopen|migraphx|amdgpu-(core|lib|install|pro)|lib.*-amdgpu-|nvidia-|libnvidia-|cuda-|datacenter-gpu-manager-|dcgm-exporter)'; then + err "${test}" "Unexpected GPU userspace or NVIDIA package on the AMD driver-and-diagnostics image" + return 1 + fi + if ! modprobe_config=$(modprobe -c) || + grep -Eq '^[[:space:]]*(blacklist|install)[[:space:]]+amdgpu([[:space:]]|$)' <<< "${modprobe_config}"; then + err "${test}" "AMDGPU is disabled in modprobe configuration" + return 1 + fi + echo "${test}: pinned driver and firmware, DKMS module and rebuild dependencies verified" +} + +# AMD SMI initializes GPU drivers even for --help. CPU bake validation checks +# contents and loads the library without amdsmi_init; device discovery runs in GPU E2E. +# shellcheck disable=SC2016 +testAMDGPUDiagnostics() { + [ "${FEATURE_FLAGS:-}" = "AMD_GPU" ] || return 0 + local components_file="${AMD_COMPONENTS_FILEPATH:-/opt/azure/amd-gpu/components.json}" + local test=testAMDGPUDiagnostics metadata amdsmi_package amdsmi_version sysdeps_package sysdeps_version cli_path library_output + local package tool tool_output + if ! metadata=$(jq -ce '.AMDGPUDiagnostics | select(type == "object")' "${components_file}") || + ! amdsmi_package=$(jq -er '.amdsmiPackage | strings | select(length > 0)' <<< "${metadata}") || + ! amdsmi_version=$(jq -er '.amdsmiVersion | strings | select(length > 0)' <<< "${metadata}") || + ! sysdeps_package=$(jq -er '.sysdepsPackage | strings | select(length > 0)' <<< "${metadata}") || + ! sysdeps_version=$(jq -er '.sysdepsVersion | strings | select(length > 0)' <<< "${metadata}") || + ! cli_path=$(jq -er '.cliPath | strings | select(startswith("/opt/rocm/"))' <<< "${metadata}"); then + err "${test}" "Missing AMDGPU diagnostics component metadata" + return 1 + fi + if [ "$(dpkg-query -W -f='${Status} ${Version}' "${amdsmi_package}")" != "install ok installed ${amdsmi_version}" ] || + [ "$(dpkg-query -W -f='${Status} ${Version}' "${sysdeps_package}")" != "install ok installed ${sysdeps_version}" ]; then + err "${test}" "Installed AMD SMI diagnostics differ from pinned packages" + return 1 + fi + for package in pciutils numactl; do + if [ "$(dpkg-query -W -f='${Status}' "${package}")" != 'install ok installed' ]; then + err "${test}" "Missing host diagnostics package ${package}" + return 1 + fi + done + for tool in lspci numactl; do + if ! tool_output=$("${tool}" --version 2>&1); then + err "${test}" "Host diagnostics command ${tool} failed: ${tool_output}" + return 1 + fi + done + if [ ! -x "${cli_path}" ] || [ ! -L /usr/local/bin/amd-smi ] || + [ "$(readlink -f /usr/local/bin/amd-smi)" != "$(readlink -f "${cli_path}")" ]; then + err "${test}" "AMD SMI executable or command symlink is missing or inconsistent" + return 1 + fi + if ! library_output=$(/usr/bin/python3 -I - "${cli_path}" <<'PY' +import json +from pathlib import Path +import sys +# The vendor bin/amd-smi symlink points into libexec; preserve its lexical bin path. +sys.path.insert(0, str(Path(sys.argv[1]).parents[1] / "share/amd_smi")) +import amdsmi +version = amdsmi.amdsmi_get_lib_version() +assert version["major"] > 0, version +print(json.dumps({"python_module": amdsmi.__version__, "library": version})) +PY + ); then + err "${test}" "AMD SMI library failed to load without GPU hardware: ${library_output}" + return 1 + fi + echo "${test}: pinned AMD SMI packages, command symlink, PCI/NUMA tools and hardware-independent library loading verified" +} diff --git a/vhdbuilder/packer/test/linux-vhd-content-test.sh b/vhdbuilder/packer/test/linux-vhd-content-test.sh index 805f0461828..7637e9d20df 100644 --- a/vhdbuilder/packer/test/linux-vhd-content-test.sh +++ b/vhdbuilder/packer/test/linux-vhd-content-test.sh @@ -244,6 +244,9 @@ testPackagesInstalled() { while IFS= read -r p; do name=$(echo "${p}" | jq .name -r) + if [ "${FEATURE_FLAGS:-}" = "AMD_GPU" ]; then + case "${name}" in nvidia-*|dra-driver-nvidia-*|datacenter-gpu-manager-*|dcgm-exporter) continue ;; esac + fi downloadLocation=$(echo "${p}" | jq .downloadLocation -r) if [ "$downloadLocation" = "" ] || [ "$downloadLocation" = "null" ]; then continue @@ -2799,3 +2802,7 @@ testFileOwnership $OS_SKU testDiskQueueServiceIsActive testVulnerableKernelModulesDisabled $OS_SKU $OS_VERSION testArtifactStreamingPackagesCleanedUp +if [ "${FEATURE_FLAGS:-}" = "AMD_GPU" ]; then + source ./AgentBaker/vhdbuilder/packer/test/amd-gpu-vhd-content-test.sh && + testAMDGPUImage || err testAMDGPUImage "AMD VHD content validation failed" +fi diff --git a/vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh b/vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh new file mode 100644 index 00000000000..77077f72f48 --- /dev/null +++ b/vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh @@ -0,0 +1,254 @@ +#!/bin/bash +# Dedicated AMD_GPU Ubuntu VHD support. Sourced only by the AMD image build. + +validateAMDGPUImageConfiguration() { + local generation=${HYPERV_GENERATION:-} enable_fips=${ENABLE_FIPS:-false} + if [ "${FEATURE_FLAGS:-}" != "AMD_GPU" ] || [ "${OS:-}" != "UBUNTU" ] || + [ "${OS_VERSION:-}" != "24.04" ] || [ "${CPU_ARCH:-}" != "amd64" ] || + [ "${generation,,}" != "v2" ] || [ "${enable_fips,,}" = "true" ]; then + echo "AMD_GPU requires the dedicated Ubuntu 24.04 amd64 Gen2 non-FIPS image" >&2 + return 1 + fi +} + +isAMDGPUSkippedPackage() { + case "$1" in + nvidia-*|dra-driver-nvidia-*|datacenter-gpu-manager-*|dcgm-exporter) return 0 ;; + *) return 1 ;; + esac +} + +installAMDGPUImage() { + local components_file="${AMD_COMPONENTS_FILEPATH:-/opt/azure/amd-gpu/components.json}" + validateAMDGPUImageConfiguration || return 1 + install -Dm0644 /home/packer/amd-gpu-components.json "${components_file}" || return 1 + installAMDGPUDriver || return 1 + capture_benchmark "${SCRIPT_NAME}_build_amd_gpu_kernel_module" || return 1 + installAMDGPUDiagnostics || return 1 + capture_benchmark "${SCRIPT_NAME}_install_amd_gpu_diagnostics" || return 1 + install -Dm0755 /home/packer/amd-gpu-validate.sh /opt/azure/containers/amd-gpu-validate.sh || return 1 +} + +# Bake on a CPU VM: keep DKMS and its build dependencies for kernel servicing, +# but never load the driver or install the ROCm compute SDK on the host. +# The subshell owns its cleanup trap without replacing the caller's traps. +installAMDGPUDriver() { + ( + set -o pipefail + local components_file="${AMD_COMPONENTS_FILEPATH:-/opt/azure/amd-gpu/components.json}" + local metadata repository key_url fingerprint package_version firmware_version module_version dkms_version + local kernel_version work_dir marker_dir=/opt/azure/amd-gpu marker_tmp="" + local generation=${HYPERV_GENERATION:-} enable_fips=${ENABLE_FIPS:-false} + local actual_fingerprint archive package_name package_arch module_path module_vermagic modprobe_config conf + local driver_deb="" firmware_deb="" + local -a apt_options build_packages + if [ "${OS:-}" != "UBUNTU" ] || [ "${OS_VERSION:-}" != "24.04" ] || + [ "${CPU_ARCH:-}" != "amd64" ] || [ "${generation,,}" != "v2" ] || + [ "${enable_fips,,}" = "true" ]; then + echo "AMDGPU requires Ubuntu 24.04 amd64 Gen2 without FIPS" >&2 + exit 1 + fi + # A failed re-bake must not leave a success marker from an earlier attempt. + rm -f "${marker_dir}/driver.json" || exit 1 + metadata=$(jq -ce '.AMDGPUDriver | select(type == "object")' "${components_file}") || exit 1 + repository=$(jq -er '.repositoryURL | select(test("^https://repo[.]radeon[.]com/amdgpu/[0-9.]+/ubuntu$"))' <<< "${metadata}") || exit 1 + key_url=$(jq -er '.signingKeyURL | select(test("^https://repo[.]radeon[.]com/[^?[:space:]]+$"))' <<< "${metadata}") || exit 1 + fingerprint=$(jq -er '.signingKeyFingerprint | select(test("^[A-F0-9]{40}$"))' <<< "${metadata}") || exit 1 + jq -e '.distribution == "noble" and .component == "main"' <<< "${metadata}" >/dev/null || exit 1 + package_version=$(jq -er '.packageVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 + firmware_version=$(jq -er '.firmwarePackageVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 + module_version=$(jq -er '.moduleVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 + dkms_version=$(jq -er '.dkmsVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 + kernel_version=$(uname -r) || exit 1 + work_dir=$(mktemp -d /tmp/amd-gpu.XXXXXX) || exit 1 + trap 'rm -rf "${work_dir}"; if [ -n "${marker_tmp}" ]; then rm -f "${marker_tmp}"; fi' EXIT + build_packages=(ca-certificates curl gnupg build-essential dkms autoconf automake initramfs-tools + "linux-headers-${kernel_version}" "linux-modules-extra-${kernel_version}") + apt_get_update || exit 1 + apt_get_install 20 5 1800 "${build_packages[@]}" || exit 1 + test -f "/lib/modules/${kernel_version}/build/Makefile" || exit 1 + + mkdir -p "${work_dir}/gnupg" "${work_dir}/lists/partial" "${work_dir}/packages" || exit 1 + chmod 755 "${work_dir}" || exit 1 + chmod 700 "${work_dir}/gnupg" || exit 1 + retrycmd_if_failure 5 5 60 curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' \ + "${key_url}" -o "${work_dir}/key.asc" || exit 1 + # Compare primary fingerprints only; the legitimate key also has an encryption subkey. + actual_fingerprint=$(gpg --batch --homedir "${work_dir}/gnupg" --show-keys --with-colons "${work_dir}/key.asc" | + awk -F: '$1 == "pub" { primary = 1; next } primary && $1 == "fpr" { print $10; primary = 0 }') || exit 1 + if [ "${actual_fingerprint}" != "${fingerprint}" ]; then + echo "AMDGPU signing key fingerprint mismatch" >&2 + exit 1 + fi + gpg --batch --homedir "${work_dir}/gnupg" --dearmor --output "${work_dir}/key.gpg" "${work_dir}/key.asc" || exit 1 + chmod 644 "${work_dir}/key.gpg" || exit 1 + printf 'deb [arch=amd64 signed-by=%s/key.gpg] %s noble main\n' "${work_dir}" "${repository}" > "${work_dir}/amdgpu.list" || exit 1 + # AMD's signed metadata is isolated from normal APT state. Only these two + # exact packages come from AMD; Ubuntu supplies all remaining dependencies. + apt_options=(-o "Dir::Etc::sourcelist=${work_dir}/amdgpu.list" -o Dir::Etc::sourceparts=- + -o "Dir::State::lists=${work_dir}/lists" -o APT::Get::AllowUnauthenticated=false + -o Acquire::AllowInsecureRepositories=false -o Acquire::AllowDowngradeToInsecureRepositories=false) + retrycmd_if_failure 5 5 300 apt-get "${apt_options[@]}" update || exit 1 + cd "${work_dir}/packages" || exit 1 + retrycmd_if_failure 5 5 300 apt-get "${apt_options[@]}" download \ + "amdgpu-dkms=${package_version}" "amdgpu-dkms-firmware=${firmware_version}" || exit 1 + for archive in "${work_dir}/packages/"*.deb; do + package_name=$(dpkg-deb -f "${archive}" Package) || exit 1 + package_arch=$(dpkg-deb -f "${archive}" Architecture) || exit 1 + case "${package_arch}" in all|amd64) ;; *) echo "Unexpected AMDGPU package architecture" >&2; exit 1 ;; esac + case "${package_name}" in + amdgpu-dkms) + [ -z "${driver_deb}" ] && [ "$(dpkg-deb -f "${archive}" Version)" = "${package_version}" ] || exit 1 + driver_deb=${archive} + ;; + amdgpu-dkms-firmware) + [ -z "${firmware_deb}" ] && [ "$(dpkg-deb -f "${archive}" Version)" = "${firmware_version}" ] || exit 1 + firmware_deb=${archive} + ;; + *) echo "Unexpected AMDGPU package: ${package_name}" >&2; exit 1 ;; + esac + done + [ -n "${driver_deb}" ] && [ -n "${firmware_deb}" ] || exit 1 + apt_get_install 20 5 1800 "${driver_deb}" "${firmware_deb}" || exit 1 + # Keep the compiler, matching headers, firmware and DKMS sources through + # image cleanup. No vendor repository/key is persisted on the image. + apt-mark manual "${build_packages[@]}" amdgpu-dkms amdgpu-dkms-firmware || exit 1 + # shellcheck disable=SC2016 + [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms)" = "install ok installed ${package_version}" ] || exit 1 + # shellcheck disable=SC2016 + [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms-firmware)" = "install ok installed ${firmware_version}" ] || exit 1 + dkms install -m amdgpu -v "${dkms_version}" -k "${kernel_version}" || exit 1 + dkms status -m amdgpu -v "${dkms_version}" -k "${kernel_version}" | grep -Fqx "amdgpu/${dkms_version}, ${kernel_version}, x86_64: installed" || exit 1 + depmod -a "${kernel_version}" || exit 1 + module_path=$(modinfo -k "${kernel_version}" -F filename amdgpu) || exit 1 + case "${module_path}" in + /lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*|/usr/lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*) ;; + *) echo "AMDGPU resolves to an unexpected module: ${module_path}" >&2; exit 1 ;; + esac + [ "$(modinfo -k "${kernel_version}" -F version amdgpu)" = "${module_version}" ] || exit 1 + module_vermagic=$(modinfo -k "${kernel_version}" -F vermagic amdgpu) || exit 1 + [ "${module_vermagic%% *}" = "${kernel_version}" ] || exit 1 + + # Azure's CPU base image may blacklist the inbox GPU driver. Remove only + # the AMD entry, retaining unrelated cloud-image module deny rules. + for conf in /etc/modprobe.d/*.conf /usr/lib/modprobe.d/*.conf; do + [ -f "${conf}" ] || continue + sed -i -E '/^[[:space:]]*blacklist[[:space:]]+amdgpu([[:space:]]|$)/d' "${conf}" || exit 1 + done + modprobe_config=$(modprobe -c) || exit 1 + if grep -Eq '^[[:space:]]*(blacklist|install)[[:space:]]+amdgpu([[:space:]]|$)' <<< "${modprobe_config}"; then + echo "AMDGPU remains disabled in modprobe configuration" >&2 + exit 1 + fi + update-initramfs -u -k "${kernel_version}" || exit 1 + mkdir -p "${marker_dir}" || exit 1 + marker_tmp=$(mktemp "${marker_dir}/.driver.json.XXXXXX") || exit 1 + jq -n --arg package_version "${package_version}" --arg firmware_package_version "${firmware_version}" \ + --arg module_version "${module_version}" --arg kernel_version "${kernel_version}" \ + '{schema_version: 1, package_version: $package_version, firmware_package_version: $firmware_package_version, + module_version: $module_version, kernel_version: $kernel_version}' > "${marker_tmp}" || exit 1 + chmod 644 "${marker_tmp}" || exit 1 + echo " - amdgpu-dkms version ${package_version}; module ${module_version}; kernel ${kernel_version}" >> "${VHD_LOGS_FILEPATH}" || exit 1 + mv -f "${marker_tmp}" "${marker_dir}/driver.json" || exit 1 + ) +} + +# Host diagnostics are intentionally separate from the compute SDK. AMD SMI +# uses only its management library and sysdeps package; no pip installation. +installAMDGPUDiagnostics() { + ( + set -o pipefail + local components_file="${AMD_COMPONENTS_FILEPATH:-/opt/azure/amd-gpu/components.json}" + local metadata repository key_url fingerprint actual_fingerprint work_dir + local amdsmi_package amdsmi_version sysdeps_package sysdeps_version cli_path python_path + local archive package_name package_arch amdsmi_deb="" sysdeps_deb="" + local generation=${HYPERV_GENERATION:-} enable_fips=${ENABLE_FIPS:-false} + local -a apt_options ubuntu_packages + if [ "${OS:-}" != "UBUNTU" ] || [ "${OS_VERSION:-}" != "24.04" ] || + [ "${CPU_ARCH:-}" != "amd64" ] || [ "${generation,,}" != "v2" ] || + [ "${enable_fips,,}" = "true" ]; then + echo "AMD GPU diagnostics require Ubuntu 24.04 amd64 Gen2 without FIPS" >&2 + exit 1 + fi + metadata=$(jq -ce '.AMDGPUDiagnostics | select(type == "object")' "${components_file}") || exit 1 + repository=$(jq -er '.repositoryURL | select(. == "https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/")' <<< "${metadata}") || exit 1 + key_url=$(jq -er '.signingKeyURL | select(. == "https://stable.repo.amd.com/rocm/gpg/packages.gpg")' <<< "${metadata}") || exit 1 + fingerprint=$(jq -er '.signingKeyFingerprint | select(test("^[A-F0-9]{40}$"))' <<< "${metadata}") || exit 1 + jq -e '.distribution == "stable" and .component == "main"' <<< "${metadata}" >/dev/null || exit 1 + amdsmi_package=$(jq -er '.amdsmiPackage | select(test("^amdrocm-amdsmi[0-9]+[.][0-9]+$"))' <<< "${metadata}") || exit 1 + sysdeps_package=$(jq -er '.sysdepsPackage | select(test("^amdrocm-sysdeps[0-9]+[.][0-9]+$"))' <<< "${metadata}") || exit 1 + amdsmi_version=$(jq -er '.amdsmiVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 + sysdeps_version=$(jq -er '.sysdepsVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 + cli_path=$(jq -er '.cliPath | select(test("^/opt/rocm/core-[0-9]+[.][0-9]+/bin/amd-smi$"))' <<< "${metadata}") || exit 1 + python_path="${cli_path%/bin/amd-smi}/share/amd_smi" + work_dir=$(mktemp -d /tmp/amd-gpu-diagnostics.XXXXXX) || exit 1 + trap 'rm -rf "${work_dir}"' EXIT + # The vendor package omits the C++ runtime dependency from its metadata. + ubuntu_packages=(ca-certificates curl gnupg python3 libstdc++6 libgcc-s1 pciutils numactl) + apt_get_update || exit 1 + apt_get_install 20 5 600 "${ubuntu_packages[@]}" || exit 1 + mkdir -p "${work_dir}/gnupg" "${work_dir}/lists/partial" "${work_dir}/packages" || exit 1 + chmod 755 "${work_dir}" || exit 1 + chmod 700 "${work_dir}/gnupg" || exit 1 + retrycmd_if_failure 5 5 60 curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' \ + "${key_url}" -o "${work_dir}/key.asc" || exit 1 + actual_fingerprint=$(gpg --batch --homedir "${work_dir}/gnupg" --show-keys --with-colons "${work_dir}/key.asc" | + awk -F: '$1 == "pub" { primary = 1; next } primary && $1 == "fpr" { print $10; primary = 0 }') || exit 1 + if [ "${actual_fingerprint}" != "${fingerprint}" ]; then + echo "AMD GPU diagnostics signing key fingerprint mismatch" >&2 + exit 1 + fi + gpg --batch --homedir "${work_dir}/gnupg" --dearmor --output "${work_dir}/key.gpg" "${work_dir}/key.asc" || exit 1 + chmod 644 "${work_dir}/key.gpg" || exit 1 + printf 'deb [arch=amd64 signed-by=%s/key.gpg] %s stable main\n' "${work_dir}" "${repository}" > "${work_dir}/diagnostics.list" || exit 1 + apt_options=(-o "Dir::Etc::sourcelist=${work_dir}/diagnostics.list" -o Dir::Etc::sourceparts=- + -o "Dir::State::lists=${work_dir}/lists" -o APT::Get::AllowUnauthenticated=false + -o Acquire::AllowInsecureRepositories=false -o Acquire::AllowDowngradeToInsecureRepositories=false) + retrycmd_if_failure 5 5 300 apt-get "${apt_options[@]}" update || exit 1 + cd "${work_dir}/packages" || exit 1 + retrycmd_if_failure 5 5 300 apt-get "${apt_options[@]}" download \ + "${amdsmi_package}=${amdsmi_version}" "${sysdeps_package}=${sysdeps_version}" || exit 1 + for archive in "${work_dir}/packages/"*.deb; do + package_name=$(dpkg-deb -f "${archive}" Package) || exit 1 + package_arch=$(dpkg-deb -f "${archive}" Architecture) || exit 1 + [ "${package_arch}" = amd64 ] || { echo "Unexpected AMD diagnostics package architecture" >&2; exit 1; } + case "${package_name}" in + "${amdsmi_package}") + [ -z "${amdsmi_deb}" ] && [ "$(dpkg-deb -f "${archive}" Version)" = "${amdsmi_version}" ] || exit 1 + amdsmi_deb=${archive} + ;; + "${sysdeps_package}") + [ -z "${sysdeps_deb}" ] && [ "$(dpkg-deb -f "${archive}" Version)" = "${sysdeps_version}" ] || exit 1 + sysdeps_deb=${archive} + ;; + *) echo "Unexpected AMD diagnostics package: ${package_name}" >&2; exit 1 ;; + esac + done + [ -n "${amdsmi_deb}" ] && [ -n "${sysdeps_deb}" ] || exit 1 + apt_get_install 20 5 600 "${amdsmi_deb}" "${sysdeps_deb}" || exit 1 + apt-mark manual "${ubuntu_packages[@]}" "${amdsmi_package}" "${sysdeps_package}" || exit 1 + # shellcheck disable=SC2016 + [ "$(dpkg-query -W -f='${Status} ${Version}' "${amdsmi_package}")" = "install ok installed ${amdsmi_version}" ] || exit 1 + # shellcheck disable=SC2016 + [ "$(dpkg-query -W -f='${Status} ${Version}' "${sysdeps_package}")" = "install ok installed ${sysdeps_version}" ] || exit 1 + [ -x "${cli_path}" ] || exit 1 + # Even --help initializes hardware in AMD SMI. Verify the Python bindings + # and shared-library dependency closure without initialization on CPU bakes. + python3 -I - "${python_path}" <<'AMDSMI_CHECK' || exit 1 +import json +import sys +sys.path.insert(0, sys.argv[1]) +import amdsmi +version = amdsmi.amdsmi_get_lib_version() +assert version["major"] > 0, version +print(json.dumps({"amd_smi_library": version, "python_module": amdsmi.__version__})) +AMDSMI_CHECK + if [ -e /usr/local/bin/amd-smi ] && [ ! -L /usr/local/bin/amd-smi ]; then + echo "Refusing to replace a non-symlink at /usr/local/bin/amd-smi" >&2 + exit 1 + fi + mkdir -p /usr/local/bin || exit 1 + ln -sfnT "${cli_path}" /usr/local/bin/amd-smi || exit 1 + echo " - ${amdsmi_package} version ${amdsmi_version}; ${sysdeps_package} version ${sysdeps_version}" >> "${VHD_LOGS_FILEPATH}" || exit 1 + ) +}