From 12a924da1c9f1687947deafdc8587fe574ee668c Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 19 Sep 2026 07:48:19 +0000 Subject: [PATCH 1/7] feat(vhd): add dedicated MI300X AMD GPU image and diagnostics --- .pipelines/.vsts-vhd-builder-release.yaml | 29 ++ .pipelines/.vsts-vhd-builder.yaml | 23 ++ .../templates/.builder-release-template.yaml | 15 +- aks-node-controller/parser/parser.go | 1 + aks-node-controller/parser/parser_test.go | 22 ++ docs/amd-gpu-vhd.md | 180 +++++++++++ e2e/config/vhd.go | 11 + e2e/scenario/fixtures/amd_gpu_host_check.py | 74 +++++ e2e/scenario/fixtures/amd_gpu_training.py | 76 +++++ e2e/scenario/node_config.go | 1 + e2e/scenario/provision.go | 2 +- e2e/scenario/scenario_gpu_amd.go | 297 ++++++++++++++++++ e2e/scenario/scenario_gpu_amd_test.go | 264 ++++++++++++++++ e2e/scenario/types.go | 3 + e2e/scenario/vmss.go | 24 +- e2e/scenario/vmss_test.go | 49 ++- parts/common/components.json | 23 ++ parts/linux/cloud-init/artifacts/cse_cmd.sh | 1 + .../linux/cloud-init/artifacts/cse_helpers.sh | 2 + .../linux/cloud-init/artifacts/cse_install.sh | 65 ++++ parts/linux/cloud-init/artifacts/cse_main.sh | 6 + pkg/agent/variables.go | 1 + pkg/agent/variables_test.go | 9 + schemas/components.cue | 29 ++ .../artifacts/cse_install_amd_gpu_spec.sh | 183 +++++++++++ .../packer/amd_gpu_diagnostics_spec.sh | 278 ++++++++++++++++ spec/vhdbuilder/packer/amd_gpu_driver_spec.sh | 273 ++++++++++++++++ spec/vhdbuilder/packer/linux_sku_name_spec.sh | 127 ++++++++ .../packer/test/amd_gpu_content_spec.sh | 250 +++++++++++++++ vhdbuilder/packer/install-dependencies.sh | 38 ++- .../produce-packer-settings-functions.sh | 42 +++ vhdbuilder/packer/produce-packer-settings.sh | 4 + .../packer/test/linux-vhd-content-test.sh | 143 +++++++++ .../linux/ubuntu/tool_installs_ubuntu.sh | 222 +++++++++++++ 34 files changed, 2742 insertions(+), 25 deletions(-) create mode 100644 docs/amd-gpu-vhd.md create mode 100644 e2e/scenario/fixtures/amd_gpu_host_check.py create mode 100644 e2e/scenario/fixtures/amd_gpu_training.py create mode 100644 e2e/scenario/scenario_gpu_amd.go create mode 100644 e2e/scenario/scenario_gpu_amd_test.go create mode 100644 spec/parts/linux/cloud-init/artifacts/cse_install_amd_gpu_spec.sh create mode 100644 spec/vhdbuilder/packer/amd_gpu_diagnostics_spec.sh create mode 100644 spec/vhdbuilder/packer/amd_gpu_driver_spec.sh create mode 100644 spec/vhdbuilder/packer/linux_sku_name_spec.sh create mode 100644 spec/vhdbuilder/packer/test/amd_gpu_content_spec.sh diff --git a/.pipelines/.vsts-vhd-builder-release.yaml b/.pipelines/.vsts-vhd-builder-release.yaml index a6e1c33f959..1e66877e542 100644 --- a/.pipelines/.vsts-vhd-builder-release.yaml +++ b/.pipelines/.vsts-vhd-builder-release.yaml @@ -105,6 +105,10 @@ parameters: displayName: Build 2404 Gen2 Containerd type: boolean default: true + - name: build2404amdgpugen2containerd + displayName: Build 2404 AMD GPU Gen2 Containerd (preview) + type: boolean + default: false - name: build2404tlgen2containerd displayName: Build 2404 TL Gen2 Containerd type: boolean @@ -717,6 +721,31 @@ stages: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: 2404-gen2-containerd + - job: build2404amdgpugen2containerd + condition: eq('${{ parameters.build2404amdgpugen2containerd }}', true) + dependsOn: [ ] + timeoutInMinutes: 360 + steps: + - bash: | + echo '##vso[task.setvariable variable=OS_SKU]Ubuntu' + echo '##vso[task.setvariable variable=OS_VERSION]24.04' + echo '##vso[task.setvariable variable=IMG_PUBLISHER]Canonical' + echo '##vso[task.setvariable variable=IMG_OFFER]ubuntu-24_04-lts' + echo '##vso[task.setvariable variable=IMG_SKU]server' + echo '##vso[task.setvariable variable=IMG_VERSION]latest' + echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' + echo '##vso[task.setvariable variable=FEATURE_FLAGS]AMD_GPU' + echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' + echo '##vso[task.setvariable variable=ENABLE_FIPS]False' + echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' + echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]False' + displayName: Setup Build Variables + - template: ./templates/.builder-release-template.yaml + parameters: + useOverrides: ${{ parameters.useOverrides }} + overrideBranch: ${{ parameters.overrideBranch }} + artifactName: 2404-amdgpu-gen2-containerd - job: build2404tlgen2containerd condition: eq('${{ parameters.build2404tlgen2containerd }}', true) dependsOn: [ ] diff --git a/.pipelines/.vsts-vhd-builder.yaml b/.pipelines/.vsts-vhd-builder.yaml index a7e955095e3..5e1e2e14c7d 100644 --- a/.pipelines/.vsts-vhd-builder.yaml +++ b/.pipelines/.vsts-vhd-builder.yaml @@ -144,6 +144,29 @@ stages: parameters: artifactName: 2404-gen2-containerd + - job: build2404amdgpugen2containerd + timeoutInMinutes: 360 + steps: + - bash: | + echo '##vso[task.setvariable variable=OS_SKU]Ubuntu' + echo '##vso[task.setvariable variable=OS_VERSION]24.04' + echo '##vso[task.setvariable variable=IMG_PUBLISHER]Canonical' + echo '##vso[task.setvariable variable=IMG_OFFER]ubuntu-24_04-lts' + echo '##vso[task.setvariable variable=IMG_SKU]server' + echo '##vso[task.setvariable variable=IMG_VERSION]latest' + echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' + # Build the kernel module on a CPU VM; MI300X validation uses the captured image. + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' + echo '##vso[task.setvariable variable=FEATURE_FLAGS]AMD_GPU' + echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' + echo '##vso[task.setvariable variable=ENABLE_FIPS]False' + echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' + echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]False' + displayName: Setup Build Variables + - template: ./templates/.builder-release-template.yaml + parameters: + artifactName: 2404-amdgpu-gen2-containerd + - job: build2604minimalgen2containerd timeoutInMinutes: 360 steps: diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index be55156ce30..40465d516da 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -74,16 +74,9 @@ steps: displayName: Setup Cue - bash: | - SKU_NAME=${OS_VERSION} && \ - if grep -q "minimal" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}minimal"; fi && \ - if [[ "${HYPERV_GENERATION,,}" == "v2" ]]; then SKU_NAME="${SKU_NAME}gen2"; fi && \ - if [[ "${ARCHITECTURE,,}" == "arm64" ]]; then SKU_NAME="${SKU_NAME}arm64"; fi && \ - if grep -q "NVIDIA_GB" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}gb"; fi && \ - if [[ "${ENABLE_FIPS,,}" == "true" ]]; then SKU_NAME="${SKU_NAME}fips"; fi && \ - if grep -q "cvm" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}CVM"; fi && \ - if [[ "${ENABLE_TRUSTED_LAUNCH}" == "True" ]]; then SKU_NAME="${SKU_NAME}TL"; fi && \ - if [[ ${OS_SKU} != "CBLMariner" && ${OS_SKU} != "AzureLinux" && ${OS_SKU} != "AzureLinuxOSGuard" && ${OS_SKU} != "Flatcar" && ${OS_SKU} != "AzureContainerLinux" ]]; then SKU_NAME="${SKU_NAME}containerd"; fi && \ - SKU_NAME=$(echo ${SKU_NAME} | tr -d '.') && \ + set -e + source vhdbuilder/packer/produce-packer-settings-functions.sh + SKU_NAME=$(get_linux_sku_name) echo "##vso[task.setvariable variable=SKU_NAME]$SKU_NAME" echo "Set SKU_NAME to $SKU_NAME" displayName: Set SKU Name @@ -391,7 +384,7 @@ steps: TargetFolder: '$(Build.ArtifactStagingDirectory)' - task: CopyFiles@2 - condition: and(eq(variables.OS_SKU, 'Ubuntu'), in(variables.OS_VERSION, '22.04', '24.04'), in(variables.FEATURE_FLAGS, 'None', 'cvm', 'NVIDIA_GB', 'NVIDIA_CUDA_PREBAKE')) + condition: and(eq(variables.OS_SKU, 'Ubuntu'), in(variables.OS_VERSION, '22.04', '24.04'), in(variables.FEATURE_FLAGS, 'None', 'cvm', 'NVIDIA_GB', 'NVIDIA_CUDA_PREBAKE', 'AMD_GPU')) displayName: Copy CIS Reports inputs: SourceFolder: '$(System.DefaultWorkingDirectory)' diff --git a/aks-node-controller/parser/parser.go b/aks-node-controller/parser/parser.go index cd8124e01f5..53371808cb9 100644 --- a/aks-node-controller/parser/parser.go +++ b/aks-node-controller/parser/parser.go @@ -96,6 +96,7 @@ func getCSEEnv(ctx context.Context, config *aksnodeconfigv1.Configuration, gpuCo "API_SERVER_NAME": config.GetApiServerConfig().GetApiServerName(), "IS_VHD": fmt.Sprintf("%v", getIsVHD(config.IsVhd)), "GPU_NODE": fmt.Sprintf("%v", getEnableNvidia(config)), + "AMD_GPU_NODE": fmt.Sprintf("%v", config.GetGpuConfig().GetEnableAmdGpu()), "SGX_NODE": fmt.Sprintf("%v", getIsSgxEnabledSKU(config.GetVmSize())), "MIG_NODE": fmt.Sprintf("%v", isMIGNode), "CONFIG_GPU_DRIVER_IF_NEEDED": fmt.Sprintf("%v", config.GetGpuConfig().GetConfigGpuDriver()), diff --git a/aks-node-controller/parser/parser_test.go b/aks-node-controller/parser/parser_test.go index 9a2341f67ed..0ea79adc825 100644 --- a/aks-node-controller/parser/parser_test.go +++ b/aks-node-controller/parser/parser_test.go @@ -26,6 +26,27 @@ func TestBuildCSECmd(t *testing.T) { aksNodeConfigUpdator func(*aksnodeconfigv1.Configuration) validator func(cmd *exec.Cmd) }{ + { + name: "AMD GPU uses the baked driver and the standard container runtime", + folder: "AKSUbuntu2204+Containerd+MIG", + k8sVersion: "1.36.3", + aksNodeConfigUpdator: func(config *aksnodeconfigv1.Configuration) { + config.GpuConfig.EnableNvidia = to.Ptr(false) + config.GpuConfig.EnableAmdGpu = to.Ptr(true) + config.GpuConfig.GpuInstanceProfile = "" + config.VmSize = "Standard_ND96isr_MI300X_v5" + }, + validator: func(cmd *exec.Cmd) { + vars := environToMap(cmd.Env) + assert.Equal(t, "true", vars["AMD_GPU_NODE"]) + assert.Equal(t, "false", vars["GPU_NODE"]) + assert.Equal(t, "false", vars["GPU_NEEDS_FABRIC_MANAGER"]) + containerdConfig, err := getBase64DecodedValue([]byte(vars["CONTAINERD_CONFIG_NO_GPU_CONTENT"])) + require.NoError(t, err) + assert.NotContains(t, containerdConfig, "nvidia") + assert.Contains(t, containerdConfig, `default_runtime_name = "runc"`) + }, + }, { name: "AKSUbuntu2204 containerd with multi-instance GPU", folder: "AKSUbuntu2204+Containerd+MIG", @@ -41,6 +62,7 @@ func TestBuildCSECmd(t *testing.T) { vars := environToMap(cmd.Env) assertHasKeyWithValue(t, vars, "LOCATION", "southcentralus") assert.Equal(t, "false", vars["GPU_NODE"]) + assert.Equal(t, "false", vars["AMD_GPU_NODE"]) assertHasKeyWithValue(t, vars, "MIG_NODE", "true") assertHasKeyWithValue(t, vars, "GPU_INSTANCE_PROFILE", "MIG7g") assertHasKeyWithValue(t, vars, "NVIDIA_MIG_PROFILE_LAYOUT", "") diff --git a/docs/amd-gpu-vhd.md b/docs/amd-gpu-vhd.md new file mode 100644 index 00000000000..04e06a7481b --- /dev/null +++ b/docs/amd-gpu-vhd.md @@ -0,0 +1,180 @@ +# Dedicated AMD GPU VHD + +The `2404gen2amdgpucontainerd` image is an opt-in Ubuntu 24.04, x86-64, +Generation 2 image for `Standard_ND96isr_MI300X_v5` and +`Standard_ND96is_MI300X_v5`. Its build flag is `AMD_GPU` and its pipeline +artifact is `2404-amdgpu-gen2-containerd`. FIPS, Trusted Launch, confidential +VMs, ARM64, and combinations with NVIDIA image flags are rejected. + +## Host and container boundary + +The host contains the official AMDGPU DKMS driver, firmware, AMD SMI diagnostics, +matching kernel headers, and build prerequisites. Exact AMD package versions, +module version, repository, and signing-key fingerprint live in +`parts/common/components.json`. +The build authenticates AMD repository metadata, installs only the selected +driver, firmware, SMI and SMI sysdeps packages, builds the module for the image +kernel, and removes its temporary repository and package-download state. It writes +`/opt/azure/amd-gpu/driver.json` only after successful verification. + +The ROCm compute runtime, HIP/RCCL, PyTorch, math libraries, compilers for AI workloads, and heavier +tools such as `rocminfo` belong in containers. Workloads use the normal `runc` +container runtime; the AMD Kubernetes device plugin advertises `amd.com/gpu` and supplies access to +the kernel devices. No NVIDIA container runtime is required. The AMD image +does not cache NVIDIA driver or plugin payloads. + +Host operators can run `amd-smi` directly through `/usr/local/bin/amd-smi`. +The image pins `amdrocm-amdsmi10.0` and `amdrocm-sysdeps10.0`, both `10.0.0-4`, +in `AMDGPUDiagnostics`. The installer verifies the separate AMD ROCm repository +key and signed metadata, then installs just these packages with Ubuntu Python +and C++ runtime dependencies. It does not install the ROCm compute runtime, +SDK, or Python packages from pip. `pciutils` (`lspci`) and `numactl` are retained +for PCI and NUMA diagnosis. + +Read-only examples on a GPU node: + +```bash +amd-smi version +amd-smi list --json +amd-smi static --json +amd-smi metric --json +lspci -nn +numactl --hardware +``` + +AMD SMI initializes hardware even for `--help`, so CPU VHD builds validate its +Python bindings and management library without initializing GPUs. GPU E2E +checks require its CLI to discover eight GPUs matching kernel PCI addresses. +Some telemetry fields are unavailable (`N/A`) on MI300X virtual functions. + +Keep DKMS sources, the host compiler, and matching headers installed: kernel +updates need them to rebuild the driver. Do not delete these files just to +reduce the image size. A future, smaller alternative is an AKS-built signed +module package for each exact supported kernel ABI. That requires coordinated +kernel/module updates and qualification; a module copied from a different +kernel is not sufficient. + +## Measured package footprint + +Official package metadata was measured on an Ubuntu 24.04 MI300X test VM on +2026-09-19. Decimal MB/GB are used. Installed sizes below are package metadata, +not compressed VHD sizes. + +| Payload | Download | Installed | +| --- | ---: | ---: | +| AMDGPU 31.50 driver and firmware | 29.84 MB | 671.67 MB | +| ROCm 10 gfx942 SDK plus runtime development package closure | 2.198 GB | 9.626 GB | +| Included AMD SMI and its AMD sysdeps package | 18.18 MB | 100.38 MB | + +The driver also requires Ubuntu build/kernel dependencies and creates module +and initramfs files. The measured driver/build dependency closure was 1.287 GB, +including packages already present in Ubuntu; this is not an incremental VHD +size estimate. The full SDK package set is 9.626 GB; the host includes only +its 100.38 MB AMD SMI diagnostics subset. +Driver, firmware and SMI together account for 772.05 MB of declared AMD package +contents, before Ubuntu dependencies and generated module/initramfs files. +Even ROCm 10's base/runtime packages pull LLVM; they are unnecessary for AMD SMI. + +The full ROCm PyTorch development image tested separately has 20.455 GB of +compressed OCI layers. Keeping that image out of the VHD does not make its +first pull smaller. For production workloads, use a separately qualified +runtime image containing only the required framework/runtime libraries and +GPU architecture, and keep compilers and build caches in a separate build +stage. Measure that image's first-pull time and digest before adopting it. + +## Provisioning and image selection + +Legacy provisioning sets `NodeBootstrappingConfiguration.EnableAMDGPU=true`; +ANC sets `GpuConfig.enable_amd_gpu=true`. Leave NVIDIA enablement false. AMD +driver configuration also requires `ConfigGPUDriverIfNeeded=true` (ANC: +`GpuConfig.config_gpu_driver=true`), preserving an explicit driver opt-out. +Validation runs in `nodePrep`, including when a PIS image skips `basePrep`. +It verifies installed package versions, the DKMS module for the running kernel, +the loaded module version, `/dev/kfd`, and eight active KFD GPUs. Preallocated +DRM render nodes are not counted as GPUs. Missing or incompatible drivers fail +provisioning; there is no vendor download or compile fallback on node startup. + +Build this flavor through the AMD job in `.pipelines/.vsts-vhd-builder.yaml`, +or enable `build2404amdgpugen2containerd` in the release pipeline. The release +parameter defaults to false. Existing CPU/NVIDIA image identities and +production image-selection defaults remain unchanged. + +Use the captured AMD gallery image explicitly for testing. Its existing Ubuntu +24.04 Gen2 `Distro` describes OS behavior; its distinct gallery image name +selects the driver payload. Do not point the production SIG map at an image +before that image has been published and qualified. + +The AMD flags configure bootstrap validation; they do not select an image. +`GetNodeBootstrapping` and `GetLatestSigImageConfig` still return ordinary Ubuntu +image metadata for the existing `Distro`. An opt-in caller must set the VM or +VMSS `ImageReference.ID` to the captured AMD gallery image version explicitly, +as the E2E scenario does, instead of using the returned ordinary +`SigImageConfig` for image selection. + +Native AKS node-pool availability also requires service-side work: permit the +supported AMD SKUs/driver policy, select this separate gallery image, propagate +AMD enablement to both provisioning contracts, and arrange the AMD device +plugin. This AgentBaker change alone does not enable a public AKS SKU or change +the AKS resource provider's existing AMD driver-policy validation. + +## Qualification + +Local repository generation, Go tests, lint, and focused ShellSpec checks +passed. The actual installer also compiled the pinned module for +`6.8.0-1067-azure` in an isolated Ubuntu 24.04 CPU container. An idempotent +reinstall, APT autoremove, and the actual VHD driver content check passed; +the installed module and DKMS rebuild dependencies survived cleanup. The +container test supplied the target kernel to the installer; it did not boot +that kernel or build/capture a complete VHD. + +The AMD SMI addition also passed an actual minimal Ubuntu 24.04 installation, +repeat installation, APT autoremove, and the diagnostics content test with +`lspci`/`numactl`. Only the two selected AMD diagnostics packages were installed; +its Python binding and library loaded without GPU hardware or the compute SDK. +On the existing MI300X lab, `amd-smi` version, discovery, static information and +metrics succeeded. The new `/usr/local/bin/amd-smi` symlink was also tested with +a clean environment and detected eight GPUs. + +The existing MI300X lab used AMDGPU 31.50 with ROCm 10 containers on kernel +6.17.0-1022-azure. Its eight-GPU training and exact all-to-all integrity tests +passed. Those results establish this host/container combination; they do not +establish that a newly baked AgentBaker image has booted successfully. +The new training fixture also passed 40 steps on each of eight GPUs, comparing +outputs, losses, gradients, and updated parameters with a CPU reference. + +After capturing the image and replicating it to `francecentral`, use the +repository's normal E2E Azure configuration and its captured build metadata: + +```bash +cd e2e +AGENTBAKER_E2E_ENABLE_MI300X=true ./e2e-local.sh \ + --subscription-id "$SUBSCRIPTION_ID" \ + --vhd-metadata-file /absolute/path/to/vhd-build-metadata.json \ + --parallel 1 --disable-scriptless \ + Ubuntu2404_MI300X_AMDGPU +``` + +The metadata must contain `2404gen2amdgpucontainerd`, its real image-version +resource ID, and the replication region. This scenario allocates MI300X +capacity and is skipped unless explicitly enabled. It validates the driver, +deploys the digest-pinned AMD device plugin, requires eight advertised GPUs, +and requires completed reference-checked training. A managed 256 GiB OS disk +provides room to unpack the development workload image. The scenario does not +yet automate reboot, serviced-kernel, or large all-to-all qualification. + +The command above exercises legacy NBC provisioning. The current E2E runner's +scriptless path still executes an NBC command; it does not independently +qualify native ANC JSON provisioning. AMD ANC environment generation is +covered by parser tests, but native ANC boot requires separate qualification. + +AgentBaker's Ubuntu 24.04 build currently uses the 6.8 Azure LTS kernel policy. +Require a successful build and boot of the captured image on MI300X, including +node readiness, eight advertised GPUs, AI/reference checks, large all-to-all +integrity/bandwidth checks, reboot, and kernel-update/rebuild validation. Run +both legacy and ANC provisioning, and PIS validation before supporting PIS. +Record the image ID, kernel, driver, workload image digests, and results. + +Production release additionally requires vendor driver security qualification. +A successful workload run is not evidence that all driver security issues are +resolved. Secure Boot support needs a separately qualified signing and trust +chain; this initial flavor does not enable it. diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index dd44c1e13be..217be73ec74 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -154,6 +154,17 @@ var ( Gallery: &Config.GalleryLinux, } + // The dedicated AMD image is selected only by opt-in GPU scenarios. It uses + // the existing Ubuntu distro and does not change production SIG defaults. + VHDUbuntu2404Gen2AMDGPUContainerd = &Image{ + Name: "2404gen2amdgpucontainerd", + OS: OSUbuntu, + Arch: "amd64", + Distro: datamodel.AKSUbuntuContainerd2404Gen2, + Gallery: &Config.GalleryLinux, + OSDiskSizeGB: 256, // The on-demand PyTorch image is 20.46 GB compressed. + } + VHDUbuntu2404ArmContainerd = &Image{ Name: "2404gen2arm64containerd", OS: OSUbuntu, diff --git a/e2e/scenario/fixtures/amd_gpu_host_check.py b/e2e/scenario/fixtures/amd_gpu_host_check.py new file mode 100644 index 00000000000..9c48126edc4 --- /dev/null +++ b/e2e/scenario/fixtures/amd_gpu_host_check.py @@ -0,0 +1,74 @@ +"""Read-only validation of the baked driver, AMD SMI, and MI300X devices.""" + +import base64 +import json +from pathlib import Path +import os +import platform +import re +import subprocess +import sys + + +def command(*args): + return subprocess.check_output(args, text=True).strip() + + +manifest = json.loads(base64.b64decode(sys.argv[1])) +expected = manifest["AMDGPUDriver"] +diagnostics = manifest["AMDGPUDiagnostics"] +marker = json.loads(Path("/opt/azure/amd-gpu/driver.json").read_text()) +assert marker["schema_version"] == 1, marker +for marker_key, manifest_key in (("package_version", "packageVersion"), + ("firmware_package_version", "firmwarePackageVersion"), + ("module_version", "moduleVersion")): + assert marker[marker_key] == expected[manifest_key], (marker_key, marker) +for package, key in (("amdgpu-dkms", "packageVersion"), + ("amdgpu-dkms-firmware", "firmwarePackageVersion")): + assert command("dpkg-query", "-W", "-f=${Status} ${Version}", package) == f"install ok installed {expected[key]}", package +for package_key, version_key in (("amdsmiPackage", "amdsmiVersion"), + ("sysdepsPackage", "sysdepsVersion")): + package = diagnostics[package_key] + assert command("dpkg-query", "-W", "-f=${Status} ${Version}", package) == f"install ok installed {diagnostics[version_key]}", package +cli = Path("/usr/local/bin/amd-smi") +assert cli.is_symlink() and cli.resolve() == Path(diagnostics["cliPath"]).resolve(), cli +assert os.access(cli, os.X_OK), cli +kernel = platform.release() +module_path = command("modinfo", "-F", "filename", "amdgpu") +assert module_path.startswith(f"/lib/modules/{kernel}/updates/dkms/"), module_path +assert command("modinfo", "-F", "version", "amdgpu") == expected["moduleVersion"] +assert Path("/sys/module/amdgpu/version").read_text().strip() == expected["moduleVersion"] +assert command("modinfo", "-F", "vermagic", "amdgpu").split()[0] == kernel +assert Path("/sys/module/amdgpu/srcversion").read_text().strip() == command("modinfo", "-F", "srcversion", "amdgpu") +assert f"amdgpu/{expected['dkmsVersion']}, {kernel}, x86_64: installed" in command("dkms", "status") +assert Path("/dev/kfd").is_char_device(), "/dev/kfd missing" +devices = [] +for device in Path("/sys/bus/pci/devices").iterdir(): + if (device / "vendor").read_text().strip() == "0x1002" and (device / "device").read_text().strip() == "0x74b5": + assert (device / "driver").resolve().name == "amdgpu", device + devices.append(device.name) +assert len(devices) == 8, devices +amdsmi_version = json.loads(command(str(cli), "version", "--json")) +assert isinstance(amdsmi_version, list) and len(amdsmi_version) == 1, amdsmi_version +version = amdsmi_version[0] +assert version["tool"] == "AMDSMI Tool" and version["version"] and version["amdsmi_library_version"], version +assert version["amdgpu_version"] == expected["moduleVersion"], version +amdsmi_devices = json.loads(command(str(cli), "list", "--json")) +assert isinstance(amdsmi_devices, list) and len(amdsmi_devices) == 8, amdsmi_devices +assert {device["gpu"] for device in amdsmi_devices} == set(range(8)), amdsmi_devices +assert {device["bdf"].lower() for device in amdsmi_devices} == set(devices), amdsmi_devices +packages = command("dpkg-query", "-W", "-f=${Package}\\t${Status}\\n") +diagnostics_packages = {diagnostics["amdsmiPackage"], diagnostics["sysdepsPackage"]} +unexpected_packages = [] +for line in packages.splitlines(): + package, status = line.split("\t", 1) + package = package.split(":", 1)[0] + if status == "install ok installed" and package not in diagnostics_packages and re.match( + r"^(amdrocm|rocm|hip|hsa-rocr|rocblas|rocfft|rocrand|rocsolver|rocsparse|miopen|migraphx|" + r"amdgpu-(core|lib|install|pro)|lib.*-amdgpu-|nvidia-|libnvidia-|cuda-|" + r"datacenter-gpu-manager-|dcgm-exporter)", package): + unexpected_packages.append(package) +assert not unexpected_packages, f"Unexpected GPU userspace packages: {unexpected_packages}" +print(json.dumps({"kernel": kernel, "module": expected["moduleVersion"], "gpu_functions": devices, + "baked_kernel": marker["kernel_version"], "host_rocm_sdk": False, + "amdsmi_version": version, "amdsmi_devices": amdsmi_devices})) diff --git a/e2e/scenario/fixtures/amd_gpu_training.py b/e2e/scenario/fixtures/amd_gpu_training.py new file mode 100644 index 00000000000..3f9881e43fe --- /dev/null +++ b/e2e/scenario/fixtures/amd_gpu_training.py @@ -0,0 +1,76 @@ +"""Small FP32 training/reference check on every GPU in a driver-only VHD. + +The container supplies all ROCm userspace. No host ROCm libraries are mounted. +This is a functional smoke test, not a throughput or model-quality benchmark. +""" + +import copy +import json + +import torch + + +def train_and_compare(device): + torch.manual_seed(20260919) + x = torch.randn(256, 32) + target = x @ torch.randn(32, 16) * 0.1 + reference = torch.nn.Linear(32, 16) + actual = copy.deepcopy(reference).to(device) + reference_optimizer = torch.optim.SGD(reference.parameters(), lr=0.2) + actual_optimizer = torch.optim.SGD(actual.parameters(), lr=0.2) + gpu_x, gpu_target = x.to(device), target.to(device) + losses = [] + for step in range(40): + reference_optimizer.zero_grad(set_to_none=True) + actual_optimizer.zero_grad(set_to_none=True) + expected_output = reference(x) + actual_output = actual(gpu_x) + expected_loss = torch.nn.functional.mse_loss(expected_output, target) + actual_loss = torch.nn.functional.mse_loss(actual_output, gpu_target) + torch.testing.assert_close(actual_output.cpu(), expected_output, rtol=5e-5, atol=5e-6) + torch.testing.assert_close(actual_loss.cpu(), expected_loss, rtol=5e-5, atol=5e-6) + assert torch.isfinite(actual_loss).item(), (device, step, "nonfinite loss") + expected_loss.backward() + actual_loss.backward() + for actual_parameter, expected_parameter in zip(actual.parameters(), reference.parameters()): + torch.testing.assert_close(actual_parameter.grad.cpu(), expected_parameter.grad, + rtol=5e-5, atol=5e-6) + reference_optimizer.step() + actual_optimizer.step() + for actual_parameter, expected_parameter in zip(actual.parameters(), reference.parameters()): + torch.testing.assert_close(actual_parameter.cpu(), expected_parameter, + rtol=5e-5, atol=5e-6) + losses.append(actual_loss.item()) + assert losses[-1] < losses[0] * 0.8, (device, "training did not converge", losses) + return {"gpu": device.index, "steps": len(losses), "initial_loss": losses[0], "final_loss": losses[-1]} + + +def main(): + assert torch.__version__ == "2.13.0+rocm10.0.0", torch.__version__ + assert torch.version.hip == "7.15.26333", torch.version.hip + assert torch.cuda.is_available(), "ROCm GPU unavailable" + assert torch.cuda.device_count() == 8, torch.cuda.device_count() + torch.set_num_threads(2) + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + results = [] + for index in range(8): + properties = torch.cuda.get_device_properties(index) + assert properties.gcnArchName.split(":")[0] == "gfx942", properties + assert "MI300X" in properties.name, properties + for peer in range(8): + if peer != index: + assert torch.cuda.can_device_access_peer(index, peer), (index, peer) + result = train_and_compare(torch.device("cuda", index)) + results.append(result) + print(json.dumps(result), flush=True) + print("AMDGPU_TRAINING_PASS " + json.dumps({ + "torch": torch.__version__, "hip": torch.version.hip, + "gpu_count": len(results), "steps_per_gpu": 40, + "checked": ["outputs", "loss", "gradients", "updated_parameters", "loss_decrease"], + "rtol": 5e-5, "atol": 5e-6, + }), flush=True) + + +if __name__ == "__main__": + main() diff --git a/e2e/scenario/node_config.go b/e2e/scenario/node_config.go index f8f53457056..d897a905a8a 100644 --- a/e2e/scenario/node_config.go +++ b/e2e/scenario/node_config.go @@ -282,6 +282,7 @@ func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) (*aksno VnetCniPluginsUrl: vnetCNIPluginURL, }, GpuConfig: &aksnodeconfigv1.GpuConfig{ + EnableAmdGpu: to.Ptr(nbc.EnableAMDGPU), ConfigGpuDriver: true, GpuDevicePlugin: false, GpuInstanceProfile: nbc.GPUInstanceProfile, diff --git a/e2e/scenario/provision.go b/e2e/scenario/provision.go index 6620562fc32..35e5ccab109 100644 --- a/e2e/scenario/provision.go +++ b/e2e/scenario/provision.go @@ -239,7 +239,7 @@ func runScenario(ctx context.Context, scenarioName string, s *Scenario) (runErr s.Runtime = &ScenarioRuntime{} } s.Runtime.Cluster = cluster - s.Runtime.VMSize = config.Config.VMSKU() + s.Runtime.VMSize = scenarioVMSize(s) s.Runtime.VMSSName = generateVMSSName(s) testKube, err := cluster.NewKubeclientForTest() diff --git a/e2e/scenario/scenario_gpu_amd.go b/e2e/scenario/scenario_gpu_amd.go new file mode 100644 index 00000000000..c11ecd72ce8 --- /dev/null +++ b/e2e/scenario/scenario_gpu_amd.go @@ -0,0 +1,297 @@ +package scenario + +import ( + "context" + _ "embed" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "strings" + "time" + + aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1" + "github.com/Azure/agentbaker/e2e/config" + "github.com/Azure/agentbaker/e2e/logging" + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +const ( + amdMI300XVMSize = "Standard_ND96isr_MI300X_v5" + amdMI300XOptIn = "AGENTBAKER_E2E_ENABLE_MI300X" + // Lab-validated upstream images: device plugin 1.31.0.11 and PyTorch 2.13.0 / ROCm 10.0. + amdDevicePluginImage = "rocm/k8s-device-plugin@sha256:e4df5dc9a7fa34e2344852256dcc5762171a6d68f1f9a34026ce26786ae335e2" + amdPyTorchImage = "rocm/pytorch@sha256:bbdaba66029f905321be3bbc95206a2d9a56a2bc5300877d146ad24d40db9a12" + amdTrainingSuccessMarker = "AMDGPU_TRAINING_PASS " +) + +//go:embed fixtures/amd_gpu_training.py +var amdGPUTrainingScript string + +//go:embed fixtures/amd_gpu_host_check.py +var amdGPUHostCheckScript string + +var _ = Register(newUbuntu2404MI300XAMDGPUScenario()) + +func newUbuntu2404MI300XAMDGPUScenario() *Scenario { + return &Scenario{ + Name: "Ubuntu2404_MI300X_AMDGPU", + Description: "Opt-in Ubuntu 24.04 MI300X image with AMDGPU and AMD SMI: eight AMD GPUs, upstream device plugin, and containerized FP32 training checked against a CPU reference", + Tags: Tags{GPU: true}, + Location: "francecentral", + SkipIf: func(context.Context) string { + if os.Getenv(amdMI300XOptIn) != "true" { + return "set " + amdMI300XOptIn + "=true after publishing the dedicated VHD and confirming MI300X quota" + } + return "" + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2404Gen2AMDGPUContainerd, + VMSize: amdMI300XVMSize, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = amdMI300XVMSize + nbc.EnableAMDGPU = true + nbc.EnableNvidia = false + nbc.ConfigGPUDriverIfNeeded = true + nbc.EnableGPUDevicePluginIfNeeded = false + }, + AKSNodeConfigMutator: func(_ *Cluster, cfg *aksnodeconfigv1.Configuration) { + cfg.VmSize = amdMI300XVMSize + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr(amdMI300XVMSize) + // Qualify the driver without depending on the SKU's ephemeral disk placement or size. + osDisk := vmss.Properties.VirtualMachineProfile.StorageProfile.OSDisk + osDisk.DiffDiskSettings = nil + if osDisk.ManagedDisk == nil { + osDisk.ManagedDisk = &armcompute.VirtualMachineScaleSetManagedDiskParameters{} + } + osDisk.ManagedDisk.StorageAccountType = to.Ptr(armcompute.StorageAccountTypesPremiumLRS) + }, + Validator: validateAMDMI300X, + }, + } +} + +func validateAMDMI300X(ctx context.Context, s *Scenario) error { + for _, check := range []struct { + name string + run func(context.Context, *Scenario) error + }{ + {"driver/amd-baked-driver", validateAMDGPUHost}, + {"bootstrap/kubelet-continuity", ValidateKubeletHasNotStopped}, + {"plugin/amd-deploy", deployAMDDevicePluginDaemonset}, + {"plugin/amd-eight-gpus", validateAMDGPUResources}, + {"workload/amd-pytorch-training", validateAMDGPUTraining}, + } { + if err := runGPUCheck(ctx, s, check.name, check.run); err != nil { + return err + } + } + return nil +} + +func validateAMDGPUHost(ctx context.Context, s *Scenario) error { + contents, err := os.ReadFile(repoPath("parts/common/components.json")) + if err != nil { + return err + } + script, err := amdGPUHostCheckCommand(contents) + if err != nil { + return err + } + result, err := execScriptOnVMForScenarioValidateExitCode(ctx, s, script, 0, "baked AMDGPU driver, AMD SMI diagnostics or MI300X devices did not match") + if err == nil { + logging.Logf(ctx, "AMDGPU host validation: %s", result.stdout) + } + return err +} + +func amdGPUHostCheckCommand(contents []byte) (string, error) { + var manifest struct { + AMDGPUDriver map[string]string + AMDGPUDiagnostics map[string]string + } + if err := json.Unmarshal(contents, &manifest); err != nil { + return "", err + } + for _, key := range []string{"packageVersion", "firmwarePackageVersion", "moduleVersion", "dkmsVersion"} { + if manifest.AMDGPUDriver[key] == "" { + return "", fmt.Errorf("AMDGPUDriver.%s missing from components.json", key) + } + } + for _, key := range []string{"amdsmiPackage", "amdsmiVersion", "sysdepsPackage", "sysdepsVersion", "cliPath"} { + if manifest.AMDGPUDiagnostics[key] == "" { + return "", fmt.Errorf("AMDGPUDiagnostics.%s missing from components.json", key) + } + } + encoded, err := json.Marshal(manifest) + if err != nil { + return "", err + } + return "python3 - '" + base64.StdEncoding.EncodeToString(encoded) + "' <<'AMDGPU_CHECK'\n" + amdGPUHostCheckScript + "\nAMDGPU_CHECK\n", nil +} + +func amdGPUTolerations() []corev1.Toleration { + return []corev1.Toleration{ + {Key: "CriticalAddonsOnly", Operator: corev1.TolerationOpExists}, + {Key: "amd.com/gpu", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, + } +} + +func amdDevicePluginDaemonset(nodeName string, owner metav1.OwnerReference) *appsv1.DaemonSet { + name := uniqueKubernetesResourceName("amddp-" + nodeName) + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "kube-system", OwnerReferences: []metav1.OwnerReference{owner}}, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"name": name}}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"name": name}}, + Spec: corev1.PodSpec{ + NodeSelector: map[string]string{"kubernetes.io/hostname": nodeName, "kubernetes.io/arch": "amd64"}, + AutomountServiceAccountToken: to.Ptr(false), + PriorityClassName: "system-node-critical", + Tolerations: amdGPUTolerations(), + Containers: []corev1.Container{{ + Name: "device-plugin", Image: amdDevicePluginImage, + SecurityContext: &corev1.SecurityContext{Privileged: to.Ptr(true)}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m"), corev1.ResourceMemory: resource.MustParse("128Mi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("512Mi")}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "device-plugins", MountPath: "/var/lib/kubelet/device-plugins"}, + {Name: "sys", MountPath: "/sys"}, + }, + }}, + Volumes: []corev1.Volume{ + {Name: "device-plugins", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/kubelet/device-plugins", Type: to.Ptr(corev1.HostPathDirectory)}}}, + {Name: "sys", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys", Type: to.Ptr(corev1.HostPathDirectory)}}}, + }, + }, + }, + }, + } +} + +func deployAMDDevicePluginDaemonset(ctx context.Context, s *Scenario) error { + owner, err := scenarioNodeOwnerReference(ctx, s) + if err != nil { + return err + } + ds := amdDevicePluginDaemonset(s.Runtime.VM.KubeName, owner) + created, err := s.Runtime.Kube.Typed.AppsV1().DaemonSets(ds.Namespace).Create(ctx, ds, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create AMD device plugin: %w", err) + } + s.Cleanup(func(ctx context.Context) error { + err := s.Runtime.Kube.Typed.AppsV1().DaemonSets(created.Namespace).Delete(ctx, created.Name, metav1.DeleteOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + return err + }) + _, err = s.Runtime.Kube.WaitUntilPodRunning(ctx, created.Namespace, "name="+created.Name, "spec.nodeName="+s.Runtime.VM.KubeName) + return err +} + +func validateAMDGPUResources(ctx context.Context, s *Scenario) error { + return wait.PollUntilContextTimeout(ctx, 3*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + node, err := s.Runtime.Kube.Typed.CoreV1().Nodes().Get(ctx, s.Runtime.VM.KubeName, metav1.GetOptions{}) + if err != nil { + return false, err + } + return amdGPUResourcesReady(node), nil + }) +} + +func amdGPUResourcesReady(node *corev1.Node) bool { + capacity, allocatable := node.Status.Capacity["amd.com/gpu"], node.Status.Allocatable["amd.com/gpu"] + return capacity.Value() == 8 && allocatable.Value() == 8 +} + +func amdGPUTrainingPod(nodeName string, owner metav1.OwnerReference) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: uniqueKubernetesResourceName("amd-train-" + nodeName), Namespace: "default", OwnerReferences: []metav1.OwnerReference{owner}}, + Spec: corev1.PodSpec{ + AutomountServiceAccountToken: to.Ptr(false), + RestartPolicy: corev1.RestartPolicyNever, + ActiveDeadlineSeconds: to.Ptr(int64(1800)), + NodeSelector: map[string]string{"kubernetes.io/hostname": nodeName}, + Tolerations: amdGPUTolerations(), + Containers: []corev1.Container{{ + Name: "training", Image: amdPyTorchImage, ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"python3", "-u", "-c", amdGPUTrainingScript}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{"amd.com/gpu": resource.MustParse("8"), corev1.ResourceCPU: resource.MustParse("8"), corev1.ResourceMemory: resource.MustParse("16Gi")}, + Limits: corev1.ResourceList{"amd.com/gpu": resource.MustParse("8"), corev1.ResourceMemory: resource.MustParse("32Gi")}, + }, + }}, + }, + } +} + +// Completion is required: a merely Running GPU pod does not prove training passed. +func amdGPUTrainingPodCompleted(pod *corev1.Pod) (bool, error) { + switch pod.Status.Phase { + case corev1.PodSucceeded: + return true, nil + case corev1.PodFailed: + return false, fmt.Errorf("AMD training pod %s failed: %s %s", pod.Name, pod.Status.Reason, pod.Status.Message) + default: + for _, container := range pod.Status.ContainerStatuses { + if state := container.State.Terminated; state != nil && state.ExitCode != 0 { + return false, fmt.Errorf("AMD training container %s exited %d: %s", container.Name, state.ExitCode, state.Message) + } + } + return false, nil + } +} + +func validateAMDGPUTraining(ctx context.Context, s *Scenario) error { + owner, err := scenarioNodeOwnerReference(ctx, s) + if err != nil { + return err + } + pod := amdGPUTrainingPod(s.Runtime.VM.KubeName, owner) + created, err := s.Runtime.Kube.Typed.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create AMD training pod: %w", err) + } + s.Cleanup(func(ctx context.Context) error { + err := s.Runtime.Kube.Typed.CoreV1().Pods(created.Namespace).Delete(ctx, created.Name, metav1.DeleteOptions{GracePeriodSeconds: to.Ptr(int64(0))}) + if apierrors.IsNotFound(err) { + return nil + } + return err + }) + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 30*time.Minute, true, func(ctx context.Context) (bool, error) { + pod, err := s.Runtime.Kube.Typed.CoreV1().Pods(created.Namespace).Get(ctx, created.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + return amdGPUTrainingPodCompleted(pod) + }) + logs, logErr := s.Runtime.Kube.Typed.CoreV1().Pods(created.Namespace).GetLogs(created.Name, &corev1.PodLogOptions{LimitBytes: to.Ptr(int64(512 * 1024))}).DoRaw(ctx) + logging.Logf(ctx, "AMD training pod %s: %s", created.Name, logs) + if err != nil { + logPodDebugInfo(ctx, s.Runtime.Kube, created) + return fmt.Errorf("AMD training did not complete: %w", err) + } + if logErr != nil { + return fmt.Errorf("read AMD training result: %w", logErr) + } + if !strings.Contains(string(logs), amdTrainingSuccessMarker) { + return fmt.Errorf("AMD training pod completed without its success record") + } + return nil +} diff --git a/e2e/scenario/scenario_gpu_amd_test.go b/e2e/scenario/scenario_gpu_amd_test.go new file mode 100644 index 00000000000..1d6254cd788 --- /dev/null +++ b/e2e/scenario/scenario_gpu_amd_test.go @@ -0,0 +1,264 @@ +package scenario + +import ( + "context" + "encoding/base64" + "encoding/json" + "os" + "strings" + "testing" + + aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1" + "github.com/Azure/agentbaker/e2e/config" + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestAMDGPUHostValidationRequiresPinnedDiagnostics(t *testing.T) { + contents, err := os.ReadFile(repoPath("parts/common/components.json")) + require.NoError(t, err) + script, err := amdGPUHostCheckCommand(contents) + require.NoError(t, err) + encoded := strings.SplitN(script, "'", 3)[1] + decoded, err := base64.StdEncoding.DecodeString(encoded) + require.NoError(t, err) + var metadata map[string]map[string]string + require.NoError(t, json.Unmarshal(decoded, &metadata)) + require.NotEmpty(t, metadata["AMDGPUDriver"]["moduleVersion"]) + require.Equal(t, "amdrocm-amdsmi10.0", metadata["AMDGPUDiagnostics"]["amdsmiPackage"]) + require.Equal(t, "amdrocm-sysdeps10.0", metadata["AMDGPUDiagnostics"]["sysdepsPackage"]) + + for _, key := range []string{"amdsmiPackage", "amdsmiVersion", "sysdepsPackage", "sysdepsVersion", "cliPath"} { + t.Run(key, func(t *testing.T) { + var manifest map[string]map[string]string + require.NoError(t, json.Unmarshal(decoded, &manifest)) + delete(manifest["AMDGPUDiagnostics"], key) + incomplete, err := json.Marshal(manifest) + require.NoError(t, err) + script, err := amdGPUHostCheckCommand(incomplete) + require.Empty(t, script) + require.ErrorContains(t, err, "AMDGPUDiagnostics."+key) + }) + } +} + +func TestAMDMI300XScenarioSkipsBeforeProvisioning(t *testing.T) { + t.Setenv(amdMI300XOptIn, "") + s := newUbuntu2404MI300XAMDGPUScenario() + outcome := runExecution(t.Context(), s.Name, s.Name, s, + func(context.Context, string, *Scenario) error { + t.Fatal("must skip before the provisioning flow resolves the unpublished VHD or creates resources") + return nil + }) + require.NoError(t, outcome.Error) + require.Contains(t, outcome.SkipReason, amdMI300XOptIn) +} + +func TestAMDMI300XScenarioIsOptInAndUsesDedicatedImage(t *testing.T) { + s := newUbuntu2404MI300XAMDGPUScenario() + for _, value := range []string{"", "false", "1"} { + t.Setenv(amdMI300XOptIn, value) + require.NotEmpty(t, s.SkipIf(t.Context()), "MI300X allocation needs explicit opt-in") + } + t.Setenv(amdMI300XOptIn, "true") + require.Empty(t, s.SkipIf(t.Context())) + require.True(t, s.Tags.GPU) + require.Equal(t, amdMI300XVMSize, s.Config.VMSize) + require.Equal(t, amdMI300XVMSize, scenarioVMSize(s), "capability queries must use MI300X before bootstrap or VMSS mutation") + require.Equal(t, "2404gen2amdgpucontainerd", s.VHD.Name) + require.Equal(t, datamodel.AKSUbuntuContainerd2404Gen2, s.VHD.Distro) + require.Equal(t, "2404gen2containerd", config.VHDUbuntu2404Gen2Containerd.Name, "generic image selection must stay unchanged") + require.GreaterOrEqual(t, s.VHD.OSDiskSizeGB, int32(128), "allow space to unpack the PyTorch image") + require.False(t, s.SkipDefaultValidation) + + nbc := &datamodel.NodeBootstrappingConfiguration{AgentPoolProfile: &datamodel.AgentPoolProfile{}} + s.BootstrapConfigMutator(nil, nbc) + require.Equal(t, amdMI300XVMSize, nbc.AgentPoolProfile.VMSize) + require.True(t, nbc.EnableAMDGPU) + require.True(t, nbc.ConfigGPUDriverIfNeeded) + require.False(t, nbc.EnableNvidia) + require.False(t, nbc.EnableGPUDevicePluginIfNeeded) + + vmss := &armcompute.VirtualMachineScaleSet{ + SKU: &armcompute.SKU{Name: to.Ptr("Standard_D2s_v3")}, + Properties: &armcompute.VirtualMachineScaleSetProperties{ + VirtualMachineProfile: &armcompute.VirtualMachineScaleSetVMProfile{ + StorageProfile: &armcompute.VirtualMachineScaleSetStorageProfile{ + OSDisk: &armcompute.VirtualMachineScaleSetOSDisk{}, + }, + }, + }, + } + s.VMConfigMutator(vmss) + require.Equal(t, amdMI300XVMSize, *vmss.SKU.Name) + anc := &aksnodeconfigv1.Configuration{} + s.AKSNodeConfigMutator(nil, anc) + require.Equal(t, amdMI300XVMSize, anc.VmSize) +} + +func TestAMDMI300XScenarioUsesManagedOSDisk(t *testing.T) { + for _, test := range []struct { + name string + managedDisk *armcompute.VirtualMachineScaleSetManagedDiskParameters + }{ + {name: "default ephemeral disk without managed parameters"}, + { + name: "existing managed parameters", + managedDisk: &armcompute.VirtualMachineScaleSetManagedDiskParameters{ + StorageAccountType: to.Ptr(armcompute.StorageAccountTypesStandardLRS), + DiskEncryptionSet: &armcompute.DiskEncryptionSetParameters{ + ID: to.Ptr("/subscriptions/test/resourceGroups/test/providers/Microsoft.Compute/diskEncryptionSets/test"), + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + s := newUbuntu2404MI300XAMDGPUScenario() + osDisk := &armcompute.VirtualMachineScaleSetOSDisk{ + CreateOption: to.Ptr(armcompute.DiskCreateOptionTypesFromImage), + DiskSizeGB: to.Ptr(s.VHD.OSDiskSizeGB), + Caching: to.Ptr(armcompute.CachingTypesReadOnly), + DiffDiskSettings: &armcompute.DiffDiskSettings{ + Option: to.Ptr(armcompute.DiffDiskOptionsLocal), + Placement: to.Ptr(armcompute.DiffDiskPlacementResourceDisk), + }, + ManagedDisk: test.managedDisk, + } + vmss := &armcompute.VirtualMachineScaleSet{ + SKU: &armcompute.SKU{}, + Properties: &armcompute.VirtualMachineScaleSetProperties{ + VirtualMachineProfile: &armcompute.VirtualMachineScaleSetVMProfile{ + StorageProfile: &armcompute.VirtualMachineScaleSetStorageProfile{OSDisk: osDisk}, + }, + }, + } + s.VMConfigMutator(vmss) + require.NoError(t, configureNVMeOSDiskPlacement(vmss), "a managed OS disk remains valid if the SKU reports NVMe support") + require.Nil(t, osDisk.DiffDiskSettings, "the 256 GiB image must not depend on ephemeral ResourceDisk capacity") + require.NotNil(t, osDisk.ManagedDisk) + require.Equal(t, armcompute.StorageAccountTypesPremiumLRS, *osDisk.ManagedDisk.StorageAccountType) + require.Equal(t, int32(256), *osDisk.DiskSizeGB) + require.Equal(t, armcompute.DiskCreateOptionTypesFromImage, *osDisk.CreateOption) + require.Equal(t, armcompute.CachingTypesReadOnly, *osDisk.Caching) + if test.managedDisk != nil { + require.Same(t, test.managedDisk, osDisk.ManagedDisk, "preserve other managed disk settings") + require.NotNil(t, osDisk.ManagedDisk.DiskEncryptionSet) + } + }) + } +} + +func TestAMDGPUBootstrapFlagReachesANC(t *testing.T) { + for _, enabled := range []bool{false, true} { + nbc, err := baseTemplateLinux("francecentral", "1.34.0", "amd64") + require.NoError(t, err) + nbc.EnableAMDGPU = enabled + anc, err := nbcToAKSNodeConfigV1(nbc) + require.NoError(t, err) + require.NotNil(t, anc.GpuConfig.EnableAmdGpu) + require.Equal(t, enabled, anc.GpuConfig.GetEnableAmdGpu()) + require.True(t, anc.GpuConfig.ConfigGpuDriver) + } +} + +func TestAMDGPUManifestsUseAssignedDevicesAndPinnedUserspace(t *testing.T) { + owner := metav1.OwnerReference{APIVersion: "v1", Kind: "Node", Name: "mi300x-node", UID: "node-uid"} + ds := amdDevicePluginDaemonset(owner.Name, owner) + require.Equal(t, []metav1.OwnerReference{owner}, ds.OwnerReferences) + require.Equal(t, ds.Spec.Selector.MatchLabels, ds.Spec.Template.Labels) + require.Equal(t, owner.Name, ds.Spec.Template.Spec.NodeSelector["kubernetes.io/hostname"]) + require.False(t, *ds.Spec.Template.Spec.AutomountServiceAccountToken) + require.Len(t, ds.Spec.Template.Spec.Containers, 1) + plugin := ds.Spec.Template.Spec.Containers[0] + require.Equal(t, "rocm/k8s-device-plugin@sha256:e4df5dc9a7fa34e2344852256dcc5762171a6d68f1f9a34026ce26786ae335e2", plugin.Image) + require.True(t, *plugin.SecurityContext.Privileged) + require.Len(t, ds.Spec.Template.Spec.Volumes, 2) + paths := map[string]bool{} + for _, volume := range ds.Spec.Template.Spec.Volumes { + require.NotNil(t, volume.HostPath) + paths[volume.HostPath.Path] = true + } + require.Equal(t, map[string]bool{"/sys": true, "/var/lib/kubelet/device-plugins": true}, paths) + + pod := amdGPUTrainingPod(owner.Name, owner) + require.Equal(t, []metav1.OwnerReference{owner}, pod.OwnerReferences) + require.Equal(t, owner.Name, pod.Spec.NodeSelector["kubernetes.io/hostname"]) + require.Empty(t, pod.Spec.NodeName, "the scheduler must allocate extended GPU resources") + require.Equal(t, corev1.RestartPolicyNever, pod.Spec.RestartPolicy) + require.NotNil(t, pod.Spec.ActiveDeadlineSeconds) + require.False(t, *pod.Spec.AutomountServiceAccountToken) + require.Len(t, pod.Spec.Containers, 1) + workload := pod.Spec.Containers[0] + require.Equal(t, "rocm/pytorch@sha256:bbdaba66029f905321be3bbc95206a2d9a56a2bc5300877d146ad24d40db9a12", workload.Image) + requested, limited := workload.Resources.Requests["amd.com/gpu"], workload.Resources.Limits["amd.com/gpu"] + require.Equal(t, int64(8), requested.Value()) + require.Equal(t, int64(8), limited.Value()) + require.Empty(t, pod.Spec.Volumes, "workload must use container ROCm and plugin-assigned devices") + require.Empty(t, workload.VolumeMounts, "do not mount host ROCm libraries or bypass the plugin") + require.Equal(t, []string{"python3", "-u", "-c", amdGPUTrainingScript}, workload.Command) + require.NotEmpty(t, amdGPUTrainingScript) + + other := amdGPUTrainingPod(owner.Name, owner) + require.NotEqual(t, pod.Name, other.Name, "parallel/repeated scenarios must not share pods") +} + +func TestAMDGPUResourcesRequireEightHealthyGPUs(t *testing.T) { + for _, test := range []struct { + name string + capacity string + allocatable string + ready bool + }{ + {"ready", "8", "8", true}, + {"unhealthy GPU", "8", "7", false}, + {"missing GPU", "7", "7", false}, + {"inactive render nodes counted", "64", "64", false}, + {"not registered", "0", "0", false}, + } { + t.Run(test.name, func(t *testing.T) { + node := &corev1.Node{Status: corev1.NodeStatus{ + Capacity: corev1.ResourceList{"amd.com/gpu": resource.MustParse(test.capacity)}, + Allocatable: corev1.ResourceList{"amd.com/gpu": resource.MustParse(test.allocatable)}, + }} + require.Equal(t, test.ready, amdGPUResourcesReady(node)) + }) + } +} + +func TestAMDGPUTrainingRequiresSuccessfulCompletion(t *testing.T) { + for _, test := range []struct { + phase corev1.PodPhase + done bool + failed bool + }{ + {corev1.PodPending, false, false}, + {corev1.PodRunning, false, false}, + {corev1.PodSucceeded, true, false}, + {corev1.PodFailed, false, true}, + } { + t.Run(string(test.phase), func(t *testing.T) { + pod := &corev1.Pod{Status: corev1.PodStatus{Phase: test.phase}} + done, err := amdGPUTrainingPodCompleted(pod) + require.Equal(t, test.done, done) + if test.failed { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } + failedContainer := &corev1.Pod{Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{{Name: "training", State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ExitCode: 1, Message: "reference mismatch"}, + }}}, + }} + done, err := amdGPUTrainingPodCompleted(failedContainer) + require.False(t, done) + require.ErrorContains(t, err, "reference mismatch") +} diff --git a/e2e/scenario/types.go b/e2e/scenario/types.go index b298d62dc1c..e8afc07a0be 100644 --- a/e2e/scenario/types.go +++ b/e2e/scenario/types.go @@ -141,6 +141,9 @@ type Config struct { // VHD is the node image used by the scenario. VHD *config.Image + // VMSize overrides the default VM size before capability queries and VMSS model creation. + VMSize string + // BootstrapConfigMutator is a function which mutates the base NodeBootstrappingConfig according to the scenario's requirements BootstrapConfigMutator func(*Cluster, *datamodel.NodeBootstrappingConfiguration) diff --git a/e2e/scenario/vmss.go b/e2e/scenario/vmss.go index 3070a7b007d..a379e8c65f2 100644 --- a/e2e/scenario/vmss.go +++ b/e2e/scenario/vmss.go @@ -488,17 +488,26 @@ func createVMSSModel(ctx context.Context, s *Scenario) (armcompute.VirtualMachin } if s.Config.UseNVMe { - if model.Properties == nil || model.Properties.VirtualMachineProfile == nil || - model.Properties.VirtualMachineProfile.StorageProfile == nil || - model.Properties.VirtualMachineProfile.StorageProfile.OSDisk == nil || - model.Properties.VirtualMachineProfile.StorageProfile.OSDisk.DiffDiskSettings == nil { - return armcompute.VirtualMachineScaleSet{}, fmt.Errorf("VMSS model is missing diff disk settings required for NVMe placement") + if err := configureNVMeOSDiskPlacement(&model); err != nil { + return armcompute.VirtualMachineScaleSet{}, err } - model.Properties.VirtualMachineProfile.StorageProfile.OSDisk.DiffDiskSettings.Placement = to.Ptr(armcompute.DiffDiskPlacementNvmeDisk) } return model, nil } +func configureNVMeOSDiskPlacement(model *armcompute.VirtualMachineScaleSet) error { + if model == nil || model.Properties == nil || model.Properties.VirtualMachineProfile == nil || + model.Properties.VirtualMachineProfile.StorageProfile == nil || + model.Properties.VirtualMachineProfile.StorageProfile.OSDisk == nil { + return fmt.Errorf("VMSS model is missing OS disk settings required for NVMe placement") + } + // Managed OS disks have no ephemeral placement, even on a SKU supporting NVMe. + if diffDisk := model.Properties.VirtualMachineProfile.StorageProfile.OSDisk.DiffDiskSettings; diffDisk != nil { + diffDisk.Placement = to.Ptr(armcompute.DiffDiskPlacementNvmeDisk) + } + return nil +} + func usesScriptlessNBCCSECmd(s *Scenario) bool { if s == nil || s.Runtime == nil || s.Runtime.NBC == nil || s.VHD == nil { return false @@ -1670,6 +1679,9 @@ func scenarioVMSize(s *Scenario) string { if s.Runtime != nil && s.Runtime.VMSize != "" { return s.Runtime.VMSize } + if s.Config.VMSize != "" { + return s.Config.VMSize + } return config.Config.VMSKU() } diff --git a/e2e/scenario/vmss_test.go b/e2e/scenario/vmss_test.go index e11d4092797..d0c5f8551c1 100644 --- a/e2e/scenario/vmss_test.go +++ b/e2e/scenario/vmss_test.go @@ -395,10 +395,53 @@ func TestWriteScriptHotfixFixtureRejectsInvalidData(t *testing.T) { // ERR_OUTBOUND_CONN_FAIL in parts/linux/cloud-init/artifacts/cse_helpers.sh. If the // product error code changes, this test forces the harness mitigation to be updated. func TestGetBaseVMSSModelUsesScenarioVMSize(t *testing.T) { - s := &Scenario{ - Runtime: &ScenarioRuntime{VMSize: config.DEFAULT_VMSKU}, + for _, test := range []struct { + name string + s *Scenario + want string + }{ + {name: "default", s: &Scenario{}, want: config.Config.VMSKU()}, + {name: "configured before runtime initialization", s: &Scenario{Config: Config{VMSize: "Standard_ND96isr_MI300X_v5"}}, want: "Standard_ND96isr_MI300X_v5"}, + {name: "runtime compatibility fallback wins", s: &Scenario{Config: Config{VMSize: "Standard_ND96isr_MI300X_v5"}, Runtime: &ScenarioRuntime{VMSize: config.DEFAULT_VMSKU}}, want: config.DEFAULT_VMSKU}, + } { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.want, scenarioVMSize(test.s)) + }) + } +} + +func TestConfigureNVMeOSDiskPlacement(t *testing.T) { + for _, managed := range []bool{false, true} { + t.Run(fmt.Sprintf("managed=%t", managed), func(t *testing.T) { + osDisk := &armcompute.VirtualMachineScaleSetOSDisk{DiskSizeGB: to.Ptr[int32](256)} + if managed { + osDisk.ManagedDisk = &armcompute.VirtualMachineScaleSetManagedDiskParameters{ + StorageAccountType: to.Ptr(armcompute.StorageAccountTypesPremiumLRS), + } + } else { + osDisk.DiffDiskSettings = &armcompute.DiffDiskSettings{ + Option: to.Ptr(armcompute.DiffDiskOptionsLocal), + Placement: to.Ptr(armcompute.DiffDiskPlacementResourceDisk), + } + } + vmss := &armcompute.VirtualMachineScaleSet{Properties: &armcompute.VirtualMachineScaleSetProperties{ + VirtualMachineProfile: &armcompute.VirtualMachineScaleSetVMProfile{ + StorageProfile: &armcompute.VirtualMachineScaleSetStorageProfile{OSDisk: osDisk}, + }, + }} + require.NoError(t, configureNVMeOSDiskPlacement(vmss)) + require.Equal(t, int32(256), *osDisk.DiskSizeGB) + if managed { + require.Nil(t, osDisk.DiffDiskSettings) + require.Equal(t, armcompute.StorageAccountTypesPremiumLRS, *osDisk.ManagedDisk.StorageAccountType) + } else { + require.Equal(t, armcompute.DiffDiskPlacementNvmeDisk, *osDisk.DiffDiskSettings.Placement) + require.Equal(t, armcompute.DiffDiskOptionsLocal, *osDisk.DiffDiskSettings.Option) + } + }) } - assert.Equal(t, s.Runtime.VMSize, scenarioVMSize(s)) + require.Error(t, configureNVMeOSDiskPlacement(nil)) + require.Error(t, configureNVMeOSDiskPlacement(&armcompute.VirtualMachineScaleSet{})) } func TestCSEExitCodeOutboundConnFail(t *testing.T) { diff --git a/parts/common/components.json b/parts/common/components.json index 930f6d4cd8e..187d6ab2dbb 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -1,4 +1,27 @@ { + "AMDGPUDriver": { + "repositoryURL": "https://repo.radeon.com/amdgpu/31.50/ubuntu", + "signingKeyURL": "https://repo.radeon.com/rocm/rocm.gpg.key", + "signingKeyFingerprint": "CA8BB4727A47B4D09B4EE8969386B48A1A693C5C", + "distribution": "noble", + "component": "main", + "packageVersion": "1:7.1.3.31500000-2390945.24.04", + "firmwarePackageVersion": "1:31.50.0.0.31500000-2390945.24.04", + "moduleVersion": "7.1.3.31500000", + "dkmsVersion": "7.1.3-2390945.24.04" + }, + "AMDGPUDiagnostics": { + "repositoryURL": "https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/", + "signingKeyURL": "https://stable.repo.amd.com/rocm/gpg/packages.gpg", + "signingKeyFingerprint": "D0F004A0025A1145C7807FCD0701EAC4D5E02107", + "distribution": "stable", + "component": "main", + "amdsmiPackage": "amdrocm-amdsmi10.0", + "amdsmiVersion": "10.0.0-4", + "sysdepsPackage": "amdrocm-sysdeps10.0", + "sysdepsVersion": "10.0.0-4", + "cliPath": "/opt/rocm/core-10.0/bin/amd-smi" + }, "ContainerImages": [ { "downloadURL": "mcr.microsoft.com/windows/servercore:*", diff --git a/parts/linux/cloud-init/artifacts/cse_cmd.sh b/parts/linux/cloud-init/artifacts/cse_cmd.sh index 2079e1f1664..7adc4eb264d 100644 --- a/parts/linux/cloud-init/artifacts/cse_cmd.sh +++ b/parts/linux/cloud-init/artifacts/cse_cmd.sh @@ -77,6 +77,7 @@ IDENTITY_BINDINGS_LOCAL_AUTHORITY_SNI={{GetVariable "identityBindingsLocalAuthor API_SERVER_NAME={{GetKubernetesEndpoint}} IS_VHD={{GetVariable "isVHD"}} GPU_NODE={{GetVariable "gpuNode"}} +AMD_GPU_NODE={{GetVariable "amdGpuNode"}} SGX_NODE={{GetVariable "sgxNode"}} MIG_NODE={{GetVariable "migNode"}} CONFIG_GPU_DRIVER_IF_NEEDED={{GetVariable "configGPUDriverIfNeeded"}} diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index 8862606a704..3d601ce4c7b 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -162,6 +162,8 @@ ERR_AKS_NODE_CONTROLLER_ERROR=240 # Generic error in AKS Node Controller ERR_AZNFS_RPM_DOWNLOAD_TIMEOUT=241 # Timeout downloading aznfs RPM from PMC ERR_AZNFS_INSTALL_FAIL=242 # Failed to install aznfs RPM package ERR_SECONDARY_NIC_CONFIG_FAIL=243 # Error configuring secondary NIC network interface +ERR_AMD_GPU_UNSUPPORTED=244 # AMD GPU image is only supported on Ubuntu 24.04 amd64 MI300X nodes +ERR_AMD_GPU_VALIDATE_FAIL=245 # Baked AMD GPU driver or device validation failed # ----------------------------------------------------------------------------- # This probably wasn't launched via a login shell, so ensure the PATH is correct. diff --git a/parts/linux/cloud-init/artifacts/cse_install.sh b/parts/linux/cloud-init/artifacts/cse_install.sh index f3331d66864..710c508538f 100755 --- a/parts/linux/cloud-init/artifacts/cse_install.sh +++ b/parts/linux/cloud-init/artifacts/cse_install.sh @@ -886,4 +886,69 @@ EOF systemctl restart nvidia-persistenced.service || exit 1 } +# Validate the dedicated AMD image on each real node, including PIS nodes. There +# is deliberately no apt/DKMS installation fallback during node provisioning. +ensureAmdGpuDrivers() { + local vm_sku + vm_sku=$(get_compute_sku) || return $ERR_AMD_GPU_VALIDATE_FAIL + if [ "${OS}" != "${UBUNTU_OS_NAME}" ] || [ "${OS_VERSION}" != "24.04" ] || [ "$(uname -m)" != "x86_64" ] || [ "${GPU_NODE:-false}" = "true" ]; then + echo "AMD GPU requires the dedicated Ubuntu 24.04 amd64 image and exclusive AMD configuration" + return $ERR_AMD_GPU_UNSUPPORTED + fi + case "${vm_sku,,}" in + standard_nd96isr_mi300x_v5|standard_nd96is_mi300x_v5) ;; + *) echo "Unsupported AMD GPU VM SKU: ${vm_sku}"; return $ERR_AMD_GPU_UNSUPPORTED ;; + esac + + if ! validateAmdGpuDriver; then + echo "Baked AMD GPU driver validation failed; use a qualified AMD GPU VHD" + return $ERR_AMD_GPU_VALIDATE_FAIL + fi +} + +validateAmdGpuDriver() { + local marker=/opt/azure/amd-gpu/driver.json + local package_version firmware_version module_version module_path kernel_version loaded_version + [ -s "${marker}" ] || return 1 + jq -e '.schema_version == 1' "${marker}" >/dev/null || return 1 + package_version=$(jq -er '.package_version | strings | select(length > 0)' "${marker}") || return 1 + firmware_version=$(jq -er '.firmware_package_version | strings | select(length > 0)' "${marker}") || return 1 + module_version=$(jq -er '.module_version | strings | select(length > 0)' "${marker}") || return 1 + [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms)" = "install ok installed ${package_version}" ] || return 1 + [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms-firmware)" = "install ok installed ${firmware_version}" ] || return 1 + + kernel_version=$(uname -r) + module_path=$(modinfo -k "${kernel_version}" -F filename amdgpu) || return 1 + # Reject the inbox driver even if modprobe succeeds. DKMS must have built the + # pinned driver for the running kernel, which may differ from the bake kernel. + case "${module_path}" in + /lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*|/usr/lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*) ;; + *) return 1 ;; + esac + [ "$(modinfo -k "${kernel_version}" -F version amdgpu)" = "${module_version}" ] || return 1 + retrycmd_if_failure 12 5 30 modprobe amdgpu || return 1 + loaded_version=$(cat /sys/module/amdgpu/version) || return 1 + [ "${loaded_version}" = "${module_version}" ] || return 1 + retrycmd_if_failure 12 5 5 test -c /dev/kfd || return 1 + export -f validateAmdGpuDevices + retrycmd_if_failure 12 5 5 bash -c validateAmdGpuDevices || return 1 + echo "AMD GPU driver ${module_version} ready on ${kernel_version}; eight GPUs detected" +} + +validateAmdGpuDevices() { + local gpu_id_file gpu_id count=0 + # Count active KFD agents, not DRM render nodes: MI300X VFs can preallocate + # many more render nodes than the eight GPUs assigned to the VM. + for gpu_id_file in /sys/class/kfd/kfd/topology/nodes/*/gpu_id; do + [ -r "${gpu_id_file}" ] || continue + gpu_id=$(cat "${gpu_id_file}") || return 1 + case "${gpu_id}" in + ''|*[!0-9]*) return 1 ;; + 0) continue ;; + esac + count=$((count + 1)) + done + [ "${count}" -eq 8 ] +} + #EOF diff --git a/parts/linux/cloud-init/artifacts/cse_main.sh b/parts/linux/cloud-init/artifacts/cse_main.sh index ae432248934..0543f672e03 100755 --- a/parts/linux/cloud-init/artifacts/cse_main.sh +++ b/parts/linux/cloud-init/artifacts/cse_main.sh @@ -487,6 +487,12 @@ function nodePrep { REBOOTREQUIRED=false # Install and configure GPU drivers if this is a GPU node + # Hardware validation belongs in nodePrep so it also runs on PIS nodes. + # The dedicated AMD image contains the driver; provisioning never downloads it. + if [ "${AMD_GPU_NODE:-false}" = "true" ] && [ "${CONFIG_GPU_DRIVER_IF_NEEDED:-false}" = "true" ]; then + logs_to_events "AKS.CSE.ensureAmdGpuDrivers" ensureAmdGpuDrivers || exit $? + fi + if [ "${GPU_NODE}" = "true" ] && [ "${skip_nvidia_driver_install}" != "true" ]; then echo $(date),$(hostname), "Start configuring GPU drivers" diff --git a/pkg/agent/variables.go b/pkg/agent/variables.go index 3e16b778874..2639285de09 100644 --- a/pkg/agent/variables.go +++ b/pkg/agent/variables.go @@ -137,6 +137,7 @@ func getCSECommandVariables(config *datamodel.NodeBootstrappingConfiguration) pa "userAssignedIdentityID": config.UserAssignedIdentityClientID, "isVHD": isVHD(profile), "gpuNode": strconv.FormatBool(config.EnableNvidia), + "amdGpuNode": strconv.FormatBool(config.EnableAMDGPU), "sgxNode": strconv.FormatBool(datamodel.IsSgxEnabledSKU(profile.VMSize)), "configGPUDriverIfNeeded": config.ConfigGPUDriverIfNeeded, "enableGPUDevicePluginIfNeeded": config.EnableGPUDevicePluginIfNeeded, diff --git a/pkg/agent/variables_test.go b/pkg/agent/variables_test.go index 556511c56a3..cafdf82ee2c 100644 --- a/pkg/agent/variables_test.go +++ b/pkg/agent/variables_test.go @@ -438,6 +438,15 @@ var _ = Describe("Windows CSE variables check", func() { config = getDefaultNBC() }) + It("keeps AMD GPU bootstrap explicitly opt-in and independent of NVIDIA", func() { + vars := getCSECommandVariables(config) + Expect(vars["amdGpuNode"]).To(Equal("false")) + config.EnableAMDGPU = true + vars = getCSECommandVariables(config) + Expect(vars["amdGpuNode"]).To(Equal("true")) + Expect(vars["gpuNode"]).To(Equal("false")) + }) + It("sets maximumLoadBalancerRuleCount", func() { config.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.MaximumLoadBalancerRuleCount = 5 vars := getCSECommandVariables(config) diff --git a/schemas/components.cue b/schemas/components.cue index 89a9fe9a062..5136bea9209 100644 --- a/schemas/components.cue +++ b/schemas/components.cue @@ -125,11 +125,40 @@ package components windowsVersions: [...#WindowsVersion] } +// The dedicated AMD image installs the kernel driver, firmware and diagnostics. Keeping +// this metadata outside Packages prevents generic images from caching it. +#AMDGPUDriver: { + repositoryURL: string & =~"^https://repo[.]radeon[.]com/amdgpu/[0-9.]+/ubuntu$" + signingKeyURL: string & =~"^https://repo[.]radeon[.]com/[^?[:space:]]+$" + signingKeyFingerprint: string & =~"^[A-F0-9]{40}$" + distribution: "noble" + component: "main" + packageVersion: string & =~"^[0-9]+:[0-9][0-9A-Za-z.+~-]+$" + firmwarePackageVersion: string & =~"^[0-9]+:[0-9][0-9A-Za-z.+~-]+$" + moduleVersion: string & =~"^[0-9][0-9.]+$" + dkmsVersion: string & =~"^[0-9][0-9A-Za-z.+~-]+$" +} + +#AMDGPUDiagnostics: { + repositoryURL: "https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/" + signingKeyURL: "https://stable.repo.amd.com/rocm/gpg/packages.gpg" + signingKeyFingerprint: string & =~"^[A-F0-9]{40}$" + distribution: "stable" + component: "main" + amdsmiPackage: string & =~"^amdrocm-amdsmi[0-9]+[.][0-9]+$" + amdsmiVersion: string & =~"^[0-9][0-9A-Za-z.+~-]+$" + sysdepsPackage: string & =~"^amdrocm-sysdeps[0-9]+[.][0-9]+$" + sysdepsVersion: string & =~"^[0-9][0-9A-Za-z.+~-]+$" + cliPath: string & =~"^/opt/rocm/core-[0-9]+[.][0-9]+/bin/amd-smi$" +} + #Components: { ContainerImages: #Images Packages: #Packages GPUContainerImages?: #GPUImages OCIArtifacts?: #OCIArtifacts + AMDGPUDriver?: #AMDGPUDriver + AMDGPUDiagnostics?: #AMDGPUDiagnostics } #Components diff --git a/spec/parts/linux/cloud-init/artifacts/cse_install_amd_gpu_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_install_amd_gpu_spec.sh new file mode 100644 index 00000000000..fb31656f28a --- /dev/null +++ b/spec/parts/linux/cloud-init/artifacts/cse_install_amd_gpu_spec.sh @@ -0,0 +1,183 @@ +#!/bin/bash + +Describe 'dedicated AMD GPU image provisioning' + Include ./parts/linux/cloud-init/artifacts/cse_install.sh + ERR_AMD_GPU_UNSUPPORTED=244 + ERR_AMD_GPU_VALIDATE_FAIL=245 + OS=UBUNTU + UBUNTU_OS_NAME=UBUNTU + OS_VERSION=24.04 + GPU_NODE=false + get_compute_sku() { echo Standard_ND96isr_MI300X_v5; } + uname() { + case "$1" in + -m) echo x86_64 ;; + -r) echo 6.17.0-test-azure ;; + esac + } + + Describe 'nodePrep opt-in dispatch' + run_amd_node_prep() { + # Execute the real dispatch block, restricted to nodePrep so moving + # it into basePrep (which PIS skips) also fails the enabled test. + eval "$(awk ' + /^function nodePrep/ { node_prep = 1 } + node_prep && /if .*AMD_GPU_NODE/ { block = 1 } + block { print } + block && /^[[:space:]]*fi$/ { exit } + ' parts/linux/cloud-init/artifacts/cse_main.sh)" + } + logs_to_events() { shift; "$@"; } + ensureAmdGpuDrivers() { echo amd-driver-validated; } + + It 'runs hardware checks in nodePrep with both explicit flags' + AMD_GPU_NODE=true + CONFIG_GPU_DRIVER_IF_NEEDED=true + When call run_amd_node_prep + The status should be success + The output should equal amd-driver-validated + End + + It 'honors driver configuration opt-out' + AMD_GPU_NODE=true + CONFIG_GPU_DRIVER_IF_NEEDED=false + When call run_amd_node_prep + The status should be success + The output should equal '' + End + + It 'does nothing on existing nodes without AMD enablement' + unset AMD_GPU_NODE + CONFIG_GPU_DRIVER_IF_NEEDED=true + When call run_amd_node_prep + The status should be success + The output should equal '' + End + End + + Describe 'ensureAmdGpuDrivers' + validateAmdGpuDriver() { echo driver-validated; } + + It 'validates the AMD image without installing anything' + When call ensureAmdGpuDrivers + The status should be success + The output should equal driver-validated + End + + It 'rejects another GPU SKU' + get_compute_sku() { echo Standard_ND96asr_v4; } + When call ensureAmdGpuDrivers + The status should equal 244 + The output should include 'Unsupported AMD GPU VM SKU' + End + + It 'rejects mixed NVIDIA and AMD configuration' + GPU_NODE=true + When call ensureAmdGpuDrivers + The status should equal 244 + The output should include 'exclusive AMD configuration' + End + + It 'rejects unsupported Ubuntu versions' + OS_VERSION=22.04 + When call ensureAmdGpuDrivers + The status should equal 244 + The output should include 'Ubuntu 24.04' + End + + It 'fails closed when the baked driver is missing or invalid' + validateAmdGpuDriver() { return 1; } + When call ensureAmdGpuDrivers + The status should equal 245 + The output should include 'use a qualified AMD GPU VHD' + End + End + + Describe 'validateAmdGpuDriver and active devices' + setup() { + AMD_TEST_ROOT=$(mktemp -d) + export AMD_TEST_ROOT + mkdir -p "${AMD_TEST_ROOT}/nodes" "${AMD_TEST_ROOT}/module" + echo 7.1.3.test > "${AMD_TEST_ROOT}/module/version" + echo '{"schema_version":1,"package_version":"1:driver-test","firmware_package_version":"firmware-test","module_version":"7.1.3.test","kernel_version":"older-bake-kernel"}' > "${AMD_TEST_ROOT}/driver.json" + local i + for i in {0..8}; do + mkdir -p "${AMD_TEST_ROOT}/nodes/${i}" + echo "${i}" > "${AMD_TEST_ROOT}/nodes/${i}/gpu_id" + done + eval "$(sed -n '/^validateAmdGpuDriver()/,/^}/p; /^validateAmdGpuDevices()/,/^}/p' parts/linux/cloud-init/artifacts/cse_install.sh | + sed "s|/opt/azure/amd-gpu|${AMD_TEST_ROOT}|g; s|/sys/module/amdgpu|${AMD_TEST_ROOT}/module|g; s|/sys/class/kfd/kfd/topology/nodes|${AMD_TEST_ROOT}/nodes|g")" + } + cleanup() { rm -rf "${AMD_TEST_ROOT}"; } + BeforeEach setup + AfterEach cleanup + + dpkg-query() { + case "${*: -1}" in + amdgpu-dkms) echo 'install ok installed 1:driver-test' ;; + amdgpu-dkms-firmware) echo 'install ok installed firmware-test' ;; + esac + } + modinfo() { + case "$4" in + filename) echo /lib/modules/6.17.0-test-azure/updates/dkms/amdgpu.ko.zst ;; + version) echo 7.1.3.test ;; + esac + } + retrycmd_if_failure() { + shift 3 + case "$1" in + modprobe) return 0 ;; + test) return 0 ;; + *) "$@" ;; + esac + } + + It 'accepts a rebuilt pinned driver on a newer kernel and ignores the CPU KFD node' + When call validateAmdGpuDriver + The status should be success + The output should include 'eight GPUs detected' + End + + It 'rejects the inbox module' + modinfo() { echo /lib/modules/6.17.0-test-azure/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu.ko.zst; } + When call validateAmdGpuDriver + The status should be failure + The output should equal '' + End + + It 'rejects a different already-loaded module' + echo old-driver > "${AMD_TEST_ROOT}/module/version" + When call validateAmdGpuDriver + The status should be failure + The output should equal '' + End + + It 'rejects removed packages with leftover dpkg records' + dpkg-query() { echo 'deinstall ok config-files 1:driver-test'; } + When call validateAmdGpuDriver + The status should be failure + The output should equal '' + End + + It 'rejects an unknown marker format' + echo '{"schema_version":2}' > "${AMD_TEST_ROOT}/driver.json" + When call validateAmdGpuDriver + The status should be failure + The output should equal '' + End + + It 'rejects missing GPUs' + echo 0 > "${AMD_TEST_ROOT}/nodes/8/gpu_id" + When call validateAmdGpuDriver + The status should be failure + The output should equal '' + End + + It 'rejects malformed GPU topology rather than counting it as a GPU' + echo invalid > "${AMD_TEST_ROOT}/nodes/8/gpu_id" + When call validateAmdGpuDevices + The status should be failure + End + End +End diff --git a/spec/vhdbuilder/packer/amd_gpu_diagnostics_spec.sh b/spec/vhdbuilder/packer/amd_gpu_diagnostics_spec.sh new file mode 100644 index 00000000000..4878d3ba9b0 --- /dev/null +++ b/spec/vhdbuilder/packer/amd_gpu_diagnostics_spec.sh @@ -0,0 +1,278 @@ +#!/bin/bash +# shellcheck disable=SC2034,SC2329 + +Describe 'Minimal AMDGPU host diagnostics bake' + setup_amd_diagnostics() { + TEST_DIR=$(mktemp -d) + TRACE="${TEST_DIR}/trace" + : > "${TRACE}" + OS=UBUNTU OS_VERSION=24.04 CPU_ARCH=amd64 HYPERV_GENERATION=v2 ENABLE_FIPS=false + COMPONENTS_FILEPATH="${TEST_DIR}/components.json" + VHD_LOGS_FILEPATH="${TEST_DIR}/vhd.log" + FAIL_STAGE="" + PACKAGE_ARCH=amd64 + mkdir -p "${TEST_DIR}/work" "${TEST_DIR}/bin" "${TEST_DIR}/rocm/core-10.0/bin" "${TEST_DIR}/rocm/core-10.0/share/amd_smi" + jq --arg cli_path "${TEST_DIR}/rocm/core-10.0/bin/amd-smi" '.AMDGPUDiagnostics.cliPath = $cli_path' \ + parts/common/components.json > "${COMPONENTS_FILEPATH}" + AMDSMI_PACKAGE=$(jq -r '.AMDGPUDiagnostics.amdsmiPackage' "${COMPONENTS_FILEPATH}") + AMDSMI_VERSION=$(jq -r '.AMDGPUDiagnostics.amdsmiVersion' "${COMPONENTS_FILEPATH}") + SYSDEPS_PACKAGE=$(jq -r '.AMDGPUDiagnostics.sysdepsPackage' "${COMPONENTS_FILEPATH}") + SYSDEPS_VERSION=$(jq -r '.AMDGPUDiagnostics.sysdepsVersion' "${COMPONENTS_FILEPATH}") + # Run the production function, remapping only owned filesystem locations. + # Network, package metadata and package installation are mocked below. + eval "$(sed -n '/^installAMDGPUDiagnostics()/,/^}$/p' vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh | + sed -e "s|/opt/rocm|${TEST_DIR}/rocm|g" -e "s|/usr/local/bin|${TEST_DIR}/bin|g" \ + -e "s|/tmp/amd-gpu-diagnostics|${TEST_DIR}/work/amd-gpu-diagnostics|g")" + } + cleanup_amd_diagnostics() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_amd_diagnostics + AfterEach cleanup_amd_diagnostics + + retrycmd_if_failure() { shift 3; "$@"; } + apt_get_update() { echo ubuntu-update >> "${TRACE}"; } + apt_get_install() { + shift 3 + echo "ubuntu-install $*" >> "${TRACE}" + case " $* " in + *'.deb '*) + echo vendor-install >> "${TRACE}" + [ "${FAIL_STAGE}" != install ] || return 1 + [ "${FAIL_STAGE}" != missing_cli ] || return 0 + # A CPU builder cannot run CLI commands that initialize GPU hardware. + printf '#!/bin/bash\necho "amd-smi $*" >> "%s"\nexit 99\n' \ + "${TRACE}" > "${TEST_DIR}/rocm/core-10.0/bin/amd-smi" + [ "${FAIL_STAGE}" = nonexecutable_cli ] || chmod +x "${TEST_DIR}/rocm/core-10.0/bin/amd-smi" + if [ "${FAIL_STAGE}" = python_import ]; then + echo 'raise RuntimeError("diagnostics import failed")' > "${TEST_DIR}/rocm/core-10.0/share/amd_smi/amdsmi.py" + elif [ "${FAIL_STAGE}" = library_version ]; then + printf 'def amdsmi_get_lib_version():\n return {"major": 0}\n' > "${TEST_DIR}/rocm/core-10.0/share/amd_smi/amdsmi.py" + else + printf '__version__ = "26.0.0"\ndef amdsmi_get_lib_version():\n with open("%s", "a") as trace:\n trace.write("amdsmi-library-version\\n")\n return {"major": 26, "minor": 0, "release": 0}\n' \ + "${TRACE}" > "${TEST_DIR}/rocm/core-10.0/share/amd_smi/amdsmi.py" + fi + ;; + *) [ "${FAIL_STAGE}" != prerequisites ] ;; + esac + } + apt-mark() { + echo "apt-mark $*" >> "${TRACE}" + printf 'retained-package %s\n' "$@" >> "${TRACE}" + [ "${FAIL_STAGE}" != retain ] + } + apt-get() { + echo "apt-get $*" >> "${TRACE}" + local option + for option in "$@"; do + case "${option}" in + Dir::Etc::sourcelist=*) printf 'repository %s\n' "$(cat "${option#*=}")" >> "${TRACE}" ;; + esac + done + case " $* " in + *' update '*) [ "${FAIL_STAGE}" != metadata ] ;; + *' download '*) + [ "${FAIL_STAGE}" != download ] || return 1 + touch amdsmi.deb sysdeps.deb + [ "${FAIL_STAGE}" != extra ] || touch runtime.deb + ;; + *) return 99 ;; + esac + } + curl() { + echo key-download >> "${TRACE}" + while [ "$#" -gt 0 ]; do + if [ "$1" = -o ]; then printf 'test-key\n' > "$2"; return; fi + shift + done + return 99 + } + gpg() { + case " $* " in + *' --show-keys '*) + echo 'pub:::::::::' + if [ "${FAIL_STAGE}" = key ]; then + echo 'fpr:::::::::WRONG:' + else + echo 'fpr:::::::::D0F004A0025A1145C7807FCD0701EAC4D5E02107:' + fi + echo 'sub:::::::::' + echo 'fpr:::::::::UNRELATED_SUBKEY:' + ;; + *) + while [ "$#" -gt 0 ]; do + if [ "$1" = --output ]; then touch "$2"; return; fi + shift + done + return 99 + ;; + esac + } + dpkg-deb() { + case "$2:$3" in + */amdsmi.deb:Package) echo "${AMDSMI_PACKAGE}" ;; + */sysdeps.deb:Package) echo "${SYSDEPS_PACKAGE}" ;; + */runtime.deb:Package) echo amdrocm-runtime10.0 ;; + *:Architecture) echo "${PACKAGE_ARCH}" ;; + */amdsmi.deb:Version) + if [ "${FAIL_STAGE}" = amdsmi_version ]; then echo wrong-version; else echo "${AMDSMI_VERSION}"; fi + ;; + */sysdeps.deb:Version) + if [ "${FAIL_STAGE}" = sysdeps_version ]; then echo wrong-version; else echo "${SYSDEPS_VERSION}"; fi + ;; + *) return 99 ;; + esac + } + dpkg-query() { + case "${*: -1}" in + "${AMDSMI_PACKAGE}") + if [ "${FAIL_STAGE}" = installed_version ]; then echo 'install ok installed wrong-version'; + else echo "install ok installed ${AMDSMI_VERSION}"; fi + ;; + "${SYSDEPS_PACKAGE}") echo "install ok installed ${SYSDEPS_VERSION}" ;; + *) return 99 ;; + esac + } + run_diagnostics_installer() { + local status=0 + # Conditional callers disable errexit: installation must still fail closed. + installAMDGPUDiagnostics || status=$? + echo "remaining-work-files=$(find "${TEST_DIR}/work" -mindepth 1 | wc -l | tr -d ' ')" + return "${status}" + } + repeat_diagnostics_installer() { + installAMDGPUDiagnostics && installAMDGPUDiagnostics && + [ "$(readlink "${TEST_DIR}/bin/amd-smi")" = "${TEST_DIR}/rocm/core-10.0/bin/amd-smi" ] + } + + It 'installs only the pinned AMD SMI subset from authenticated isolated metadata' + When run run_diagnostics_installer + The status should be success + The output should include '"python_module": "26.0.0"' + The output should include 'remaining-work-files=0' + The stderr should eq '' + The contents of file "${TRACE}" should include "download ${AMDSMI_PACKAGE}=${AMDSMI_VERSION} ${SYSDEPS_PACKAGE}=${SYSDEPS_VERSION}" + The contents of file "${TRACE}" should include 'https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/ stable main' + The contents of file "${TRACE}" should include 'Dir::Etc::sourceparts=-' + The contents of file "${TRACE}" should include 'AllowUnauthenticated=false' + The contents of file "${TRACE}" should include 'AllowInsecureRepositories=false' + The contents of file "${TRACE}" should include 'apt-mark manual' + The contents of file "${TRACE}" should include 'retained-package pciutils' + The contents of file "${TRACE}" should include 'retained-package numactl' + The contents of file "${TRACE}" should include 'retained-package python3' + The contents of file "${TRACE}" should include 'retained-package libstdc++6' + The contents of file "${TRACE}" should include 'retained-package libgcc-s1' + The contents of file "${TRACE}" should include "${AMDSMI_PACKAGE} ${SYSDEPS_PACKAGE}" + The contents of file "${TRACE}" should include 'amdsmi-library-version' + The contents of file "${TRACE}" should not include 'amd-smi --help' + The contents of file "${TRACE}" should not include 'amdrocm-runtime' + The contents of file "${TRACE}" should not include 'amdrocm10.0-gfx942' + The contents of file "${TRACE}" should not include 'amdrocm-llvm' + The path "${TEST_DIR}/bin/amd-smi" should be symlink + End + + It 'rejects a signing key mismatch before querying the AMD repository' + FAIL_STAGE=key + When run run_diagnostics_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'signing key fingerprint mismatch' + The contents of file "${TRACE}" should not include 'apt-get ' + The path "${TEST_DIR}/bin/amd-smi" should not be exist + End + + Describe 'failed repository and package transactions' + Parameters + prerequisites + metadata + download + amdsmi_version + sysdeps_version + install + retain + installed_version + End + It 'fails and removes temporary repository state' + FAIL_STAGE="$1" + When run run_diagnostics_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should eq '' + End + End + + It 'rejects an unexpected extra vendor archive before installing it' + FAIL_STAGE=extra + When run run_diagnostics_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'Unexpected' + The contents of file "${TRACE}" should not include 'vendor-install' + End + + It 'rejects a package for the wrong architecture before installing it' + PACKAGE_ARCH=arm64 + When run run_diagnostics_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'architecture' + The contents of file "${TRACE}" should not include 'vendor-install' + End + + Describe 'a broken installed diagnostic CLI' + Parameters + missing_cli + nonexecutable_cli + End + It 'fails its CPU-only executable smoke check' + FAIL_STAGE="$1" + When run run_diagnostics_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should eq '' + End + End + + It 'fails when the installed Python binding cannot import' + FAIL_STAGE=python_import + When run run_diagnostics_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'diagnostics import failed' + End + + It 'fails when the diagnostic library returns an invalid version' + FAIL_STAGE=library_version + When run run_diagnostics_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'AssertionError' + End + + It 'replaces a stale command symlink and repeats installation idempotently' + ln -s "${TEST_DIR}/missing" "${TEST_DIR}/bin/amd-smi" + When run repeat_diagnostics_installer + The status should be success + The output should include '"python_module": "26.0.0"' + The stderr should eq '' + The path "${TEST_DIR}/bin/amd-smi" should be symlink + End + + It 'does not overwrite an existing regular file at the command location' + printf 'owned-by-another-package\n' > "${TEST_DIR}/bin/amd-smi" + When run run_diagnostics_installer + The status should be failure + The output should include 'remaining-work-files=0' + The stderr should include 'amd-smi' + The contents of file "${TEST_DIR}/bin/amd-smi" should eq 'owned-by-another-package' + End + + It 'does not replace or write into an existing directory at the command location' + mkdir "${TEST_DIR}/bin/amd-smi" + printf 'preserve\n' > "${TEST_DIR}/bin/amd-smi/owned-file" + When run run_diagnostics_installer + The status should be failure + The output should include 'remaining-work-files=0' + The stderr should include 'Refusing to replace a non-symlink' + The contents of file "${TEST_DIR}/bin/amd-smi/owned-file" should eq 'preserve' + The path "${TEST_DIR}/bin/amd-smi/amd-smi" should not be exist + End +End diff --git a/spec/vhdbuilder/packer/amd_gpu_driver_spec.sh b/spec/vhdbuilder/packer/amd_gpu_driver_spec.sh new file mode 100644 index 00000000000..bf9f07ce3b8 --- /dev/null +++ b/spec/vhdbuilder/packer/amd_gpu_driver_spec.sh @@ -0,0 +1,273 @@ +#!/bin/bash +# shellcheck disable=SC2034,SC2329 + +Describe 'CPU-only AMDGPU driver bake' + setup_amd_driver() { + TEST_DIR=$(mktemp -d) + TRACE="${TEST_DIR}/trace" + : > "${TRACE}" + OS=UBUNTU OS_VERSION=24.04 CPU_ARCH=amd64 HYPERV_GENERATION=v2 ENABLE_FIPS=false + COMPONENTS_FILEPATH="${PWD}/parts/common/components.json" + VHD_LOGS_FILEPATH="${TEST_DIR}/vhd.log" + FAIL_STAGE="" + PACKAGE_ARCH=all + KERNEL=6.8.0-test-azure + PACKAGE_VERSION=$(jq -r '.AMDGPUDriver.packageVersion' "${COMPONENTS_FILEPATH}") + FIRMWARE_VERSION=$(jq -r '.AMDGPUDriver.firmwarePackageVersion' "${COMPONENTS_FILEPATH}") + MODULE_VERSION=$(jq -r '.AMDGPUDriver.moduleVersion' "${COMPONENTS_FILEPATH}") + DKMS_VERSION=$(jq -r '.AMDGPUDriver.dkmsVersion' "${COMPONENTS_FILEPATH}") + mkdir -p "${TEST_DIR}/work" "${TEST_DIR}/marker" "${TEST_DIR}/modprobe" "${TEST_DIR}/vendor-modprobe" \ + "${TEST_DIR}/modules/${KERNEL}/build" + touch "${TEST_DIR}/modules/${KERNEL}/build/Makefile" + printf 'blacklist amdgpu\nblacklist nouveau\n# retain this comment\n' > "${TEST_DIR}/modprobe/cloud.conf" + # Remap owned filesystem locations only. Real sed exercises the cloud image + # blacklist edit; apt, GPG, module commands and network are all mocked below. + eval "$(sed -n '/^installAMDGPUDriver()/,/^}$/p' vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh | + sed -e "s|/opt/azure/amd-gpu|${TEST_DIR}/marker|g" -e "s|/tmp/amd-gpu|${TEST_DIR}/work/amd-gpu|g" \ + -e "s|/etc/modprobe.d|${TEST_DIR}/modprobe|g" -e "s|/usr/lib/modprobe.d|${TEST_DIR}/vendor-modprobe|g" \ + -e "s|/lib/modules|${TEST_DIR}/modules|g")" + } + cleanup_amd_driver() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_amd_driver + AfterEach cleanup_amd_driver + + uname() { printf '%s\n' "${KERNEL}"; } + retrycmd_if_failure() { shift 3; "$@"; } + apt_get_update() { echo ubuntu-update >> "${TRACE}"; } + apt_get_install() { + shift 3 + echo "ubuntu-install $*" >> "${TRACE}" + [ "${FAIL_STAGE}" != install ] + } + apt-mark() { echo "apt-mark $*" >> "${TRACE}"; } + apt-get() { + echo "apt-get $*" >> "${TRACE}" + if [ "${FAIL_STAGE}" = metadata ]; then return 1; fi + case " $* " in + *' download '*) touch driver.deb firmware.deb ;; + esac + } + curl() { + echo key-download >> "${TRACE}" + while [ "$#" -gt 0 ]; do + if [ "$1" = -o ]; then printf 'test-key\n' > "$2"; return; fi + shift + done + return 99 + } + gpg() { + case " $* " in + *' --show-keys '*) + echo 'pub:::::::::' + if [ "${FAIL_STAGE}" = key ]; then + echo 'fpr:::::::::WRONG:' + else + echo 'fpr:::::::::CA8BB4727A47B4D09B4EE8969386B48A1A693C5C:' + fi + echo 'sub:::::::::' + echo 'fpr:::::::::2B2AFE47A094DC2D1013777C30C07AF01A6D36BA:' + ;; + *) + while [ "$#" -gt 0 ]; do + if [ "$1" = --output ]; then touch "$2"; return; fi + shift + done + return 99 + ;; + esac + } + dpkg-deb() { + case "$2:$3" in + */driver.deb:Package) echo amdgpu-dkms ;; + */firmware.deb:Package) echo amdgpu-dkms-firmware ;; + *:Architecture) echo "${PACKAGE_ARCH:-all}" ;; + */driver.deb:Version) if [ "${FAIL_STAGE}" = package ]; then echo wrong-version; else echo "${PACKAGE_VERSION}"; fi ;; + */firmware.deb:Version) echo "${FIRMWARE_VERSION}" ;; + *) return 99 ;; + esac + } + dpkg-query() { + case "${*: -1}" in + amdgpu-dkms) echo "install ok installed ${PACKAGE_VERSION}" ;; + amdgpu-dkms-firmware) echo "install ok installed ${FIRMWARE_VERSION}" ;; + *) return 99 ;; + esac + } + # ShellSpec parameters below also contain the literal command name. + # shellcheck disable=SC2120 + dkms() { + echo "dkms $*" >> "${TRACE}" + [ "${FAIL_STAGE}" != dkms ] || return 1 + if [ "$1" = status ]; then + echo "amdgpu/${DKMS_VERSION}, ${KERNEL}, x86_64: installed" + fi + } + depmod() { echo "depmod $*" >> "${TRACE}"; } + modinfo() { + echo "modinfo $*" >> "${TRACE}" + case "$4" in + filename) + if [ "${FAIL_STAGE}" = inbox ]; then + echo "${TEST_DIR}/modules/${KERNEL}/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu.ko" + else + echo "${TEST_DIR}/modules/${KERNEL}/updates/dkms/amdgpu.ko.zst" + fi + ;; + version) if [ "${FAIL_STAGE}" = module ]; then echo wrong-version; else echo "${MODULE_VERSION}"; fi ;; + vermagic) if [ "${FAIL_STAGE}" = kernel ]; then echo wrong-kernel; else echo "${KERNEL} SMP mod_unload modversions "; fi ;; + *) return 99 ;; + esac + } + modprobe() { + echo "modprobe $*" >> "${TRACE}" + [ "$*" = -c ] || return 99 + cat "${TEST_DIR}/modprobe/cloud.conf" + } + update-initramfs() { + echo "update-initramfs $*" >> "${TRACE}" + [ "${FAIL_STAGE}" != initramfs ] + } + run_installer() { + local status=0 + # Conditional callers disable errexit: every mutation must still fail closed. + installAMDGPUDriver || status=$? + echo "remaining-work-files=$(find "${TEST_DIR}/work" -mindepth 1 | wc -l | tr -d ' ')" + return "${status}" + } + + It 'authenticates two pinned packages and validates a DKMS module without GPU hardware' + When run run_installer + The status should be success + The output should eq 'remaining-work-files=0' + The stderr should eq '' + The contents of file "${TEST_DIR}/marker/driver.json" should include '"schema_version": 1' + The contents of file "${TEST_DIR}/marker/driver.json" should include "\"module_version\": \"${MODULE_VERSION}\"" + The contents of file "${TRACE}" should include "download amdgpu-dkms=${PACKAGE_VERSION} amdgpu-dkms-firmware=${FIRMWARE_VERSION}" + The contents of file "${TRACE}" should include 'Dir::Etc::sourceparts=-' + The contents of file "${TRACE}" should include 'AllowUnauthenticated=false' + The contents of file "${TRACE}" should include 'apt-mark manual ca-certificates curl gnupg build-essential dkms autoconf automake initramfs-tools' + The contents of file "${TRACE}" should include 'modprobe -c' + The contents of file "${TRACE}" should not include 'modprobe amdgpu' + The contents of file "${TRACE}" should not include 'rocm-dev' + The contents of file "${TEST_DIR}/modprobe/cloud.conf" should eq "blacklist nouveau +# retain this comment" + End + + It 'rejects a key mismatch before querying the AMD repository and removes stale success state' + FAIL_STAGE=key + echo stale > "${TEST_DIR}/marker/driver.json" + When run run_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'signing key fingerprint mismatch' + The contents of file "${TRACE}" should not include 'apt-get ' + The path "${TEST_DIR}/marker/driver.json" should not be exist + End + + Describe 'installation and validation failures' + Parameters + install + metadata + package + dkms + module + kernel + initramfs + End + It 'cleans temporary repository state and never writes a driver marker' + FAIL_STAGE="$1" + When run run_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should eq '' + The path "${TEST_DIR}/marker/driver.json" should not be exist + End + End + + It 'rejects the inbox kernel module even when its version looks correct' + FAIL_STAGE=inbox + When run run_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'unexpected module' + The path "${TEST_DIR}/marker/driver.json" should not be exist + End + + It 'rejects an archive for the wrong architecture before installation' + PACKAGE_ARCH=arm64 + When run run_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'Unexpected AMDGPU package architecture' + The path "${TEST_DIR}/marker/driver.json" should not be exist + End + + It 'rejects an effective install deny rule without removing that rule' + printf 'install amdgpu /bin/false\n' >> "${TEST_DIR}/modprobe/cloud.conf" + When run run_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'remains disabled' + The contents of file "${TEST_DIR}/modprobe/cloud.conf" should include 'install amdgpu /bin/false' + The path "${TEST_DIR}/marker/driver.json" should not be exist + End + + Describe 'unsupported bake configurations' + Parameters + MARINER 24.04 amd64 v2 false + UBUNTU 22.04 amd64 v2 false + UBUNTU 24.04 arm64 v2 false + UBUNTU 24.04 amd64 v1 false + UBUNTU 24.04 amd64 v2 true + UBUNTU 24.04 amd64 v2 True + End + It 'fails before package or filesystem work' + OS="$1" OS_VERSION="$2" CPU_ARCH="$3" HYPERV_GENERATION="$4" ENABLE_FIPS="$5" + When run run_installer + The status should be failure + The output should eq 'remaining-work-files=0' + The stderr should include 'requires Ubuntu 24.04 amd64 Gen2 without FIPS' + The contents of file "${TRACE}" should eq '' + End + End +End + +Describe 'AMD-specific build dispatch' + setup_amd_dispatch() { + TEST_DIR=$(mktemp -d) + TRACE="${TEST_DIR}/trace" + : > "${TRACE}" + FEATURE_FLAGS=AMD_GPU OS=UBUNTU OS_VERSION=24.04 OS_VARIANT='' IS_KATA=false + COMPONENTS_FILEPATH="${TEST_DIR}/components.json" + jq '{Packages: ([.Packages[] | select(.name | test("nvidia|datacenter-gpu-manager|dcgm-exporter"))] + [{name: "retained-package"}])}' \ + parts/common/components.json > "${COMPONENTS_FILEPATH}" + eval "$(sed -n '/^isAMDGPUBuild()/,/^}$/p' vhdbuilder/packer/install-dependencies.sh)" + eval "$(sed -n '/^cachePackageAndBinaryComponents()/,/^}$/p' vhdbuilder/packer/install-dependencies.sh)" + } + cleanup_amd_dispatch() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_amd_dispatch + AfterEach cleanup_amd_dispatch + isMariner() { return 1; } + isAzureLinux() { return 1; } + updatePackageVersions() { jq -r .name <<< "$1" >> "${TRACE}"; PACKAGE_VERSIONS=(); } + updatePackageDownloadURL() { PACKAGE_DOWNLOAD_URL=''; } + + It 'excludes all declared NVIDIA payloads before resolving package versions or repositories' + When run cachePackageAndBinaryComponents + The status should be success + The output should include 'Skipping NVIDIA package' + The stderr should eq '' + The contents of file "${TRACE}" should eq 'retained-package' + End + + It 'continues processing NVIDIA packages on existing image SKUs' + FEATURE_FLAGS=NVIDIA_CUDA_PREBAKE + When run cachePackageAndBinaryComponents + The status should be success + The output should include 'processing components.packages' + The stderr should eq '' + The contents of file "${TRACE}" should include 'nvidia-device-plugin' + The contents of file "${TRACE}" should include 'datacenter-gpu-manager-4-core' + The contents of file "${TRACE}" should include 'retained-package' + End +End diff --git a/spec/vhdbuilder/packer/linux_sku_name_spec.sh b/spec/vhdbuilder/packer/linux_sku_name_spec.sh new file mode 100644 index 00000000000..1ba3356e3d5 --- /dev/null +++ b/spec/vhdbuilder/packer/linux_sku_name_spec.sh @@ -0,0 +1,127 @@ +#!/bin/bash + +Describe 'Linux VHD flavor naming and AMD GPU build constraints' + Include './vhdbuilder/packer/produce-packer-settings-functions.sh' + + setup_environment() { + OS_SKU=Ubuntu + OS_VERSION=24.04 + ARCHITECTURE=X86_64 + HYPERV_GENERATION=V2 + FEATURE_FLAGS=AMD_GPU + ENABLE_FIPS=False + ENABLE_TRUSTED_LAUNCH=False + TRUSTED_LAUNCH_SUPPORTED=False + SKU_NAME='' + SIG_IMAGE_NAME='' + SIG_GALLERY_NAME='' + IMG_OFFER=ubuntu-24_04-lts + } + BeforeEach 'setup_environment' + + It 'uses a dedicated AMD GPU SKU instead of the shared Ubuntu image' + When call get_linux_sku_name + The status should be success + The output should equal '2404gen2amdgpucontainerd' + End + + derive_sig_name() { + SKU_NAME=$(get_linux_sku_name) || return 1 + ensure_sig_image_name_linux + } + + It 'uses the dedicated SKU as the default staging gallery image definition' + When call derive_sig_name + The status should be success + The variable SIG_IMAGE_NAME should equal '2404gen2amdgpucontainerd' + The variable SIG_GALLERY_NAME should equal 'PackerSigGalleryEastUS' + The output should be present + End + + Describe 'unsupported AMD configurations' + Parameters + OS_SKU AzureLinux + OS_VERSION 22.04 + OS_VERSION 26.04 + ARCHITECTURE ARM64 + HYPERV_GENERATION V1 + ENABLE_FIPS True + ENABLE_TRUSTED_LAUNCH True + TRUSTED_LAUNCH_SUPPORTED True + FEATURE_FLAGS 'AMD_GPU,NVIDIA_CUDA_PREBAKE' + FEATURE_FLAGS 'AMD_GPU,NVIDIA_GB' + FEATURE_FLAGS 'AMD_GPU,cvm' + FEATURE_FLAGS 'AMD_GPU,minimal' + FEATURE_FLAGS 'NOT_AMD_GPU' + End + + It 'rejects the unsupported AMD image configuration' + printf -v "$1" '%s' "$2" + When call get_linux_sku_name + The status should be failure + The stderr should include 'AMD_GPU requires Ubuntu 24.04 x86_64 Gen2' + The output should be blank + End + End + + It 'rejects reusing the existing shared SKU for an AMD build' + SKU_NAME=2404gen2containerd + When call validate_amd_gpu_build + The status should be failure + The stderr should include 'requires the dedicated SKU_NAME' + End + + It 'accepts the generated dedicated SKU during Packer settings generation' + SKU_NAME=2404gen2amdgpucontainerd + When call validate_amd_gpu_build + The status should be success + The output should be blank + End + + It 'rejects overriding the capture definition with the existing shared image' + SKU_NAME=2404gen2amdgpucontainerd + SIG_IMAGE_NAME=2404gen2containerd + When call validate_amd_gpu_build + The status should be failure + The stderr should include 'requires the dedicated SIG_IMAGE_NAME' + End + + It 'accepts an explicit dedicated capture definition' + SKU_NAME=2404gen2amdgpucontainerd + SIG_IMAGE_NAME=2404gen2amdgpucontainerd + When call validate_amd_gpu_build + The status should be success + The output should be blank + End + + # Preserve the existing public naming contract when extracting the pipeline logic. + Describe 'existing image flavors' + Parameters + Ubuntu 22.04 X86_64 V1 None False False 2204containerd + Ubuntu 22.04 X86_64 V2 NVIDIA_CUDA_PREBAKE False False 2204gen2containerd + Ubuntu 24.04 X86_64 V2 NVIDIA_CUDA_PREBAKE False False 2404gen2containerd + Ubuntu 24.04 ARM64 V2 NVIDIA_GB False False 2404gen2arm64gbcontainerd + Ubuntu 22.04 X86_64 V2 None True True 2204gen2fipsTLcontainerd + Ubuntu 24.04 X86_64 V2 cvm False False 2404gen2CVMcontainerd + Ubuntu 26.04 ARM64 V2 minimal False False 2604minimalgen2arm64containerd + AzureLinux V3kata X86_64 V2 kata False False V3katagen2 + AzureLinuxOSGuard V3 X86_64 V2 None True True V3gen2fipsTL + CBLMariner V2 X86_64 V1 None False False V2 + Flatcar '' ARM64 V2 None False False gen2arm64 + AzureContainerLinux acl ARM64 V2 None True True aclgen2arm64fipsTL + End + + It 'preserves the existing SKU name' + OS_SKU="$1" + OS_VERSION="$2" + ARCHITECTURE="$3" + HYPERV_GENERATION="$4" + FEATURE_FLAGS="$5" + ENABLE_FIPS="$6" + ENABLE_TRUSTED_LAUNCH="$7" + When call get_linux_sku_name + The status should be success + The output should equal "$8" + End + End +End diff --git a/spec/vhdbuilder/packer/test/amd_gpu_content_spec.sh b/spec/vhdbuilder/packer/test/amd_gpu_content_spec.sh new file mode 100644 index 00000000000..6945e6ebeff --- /dev/null +++ b/spec/vhdbuilder/packer/test/amd_gpu_content_spec.sh @@ -0,0 +1,250 @@ +#!/bin/bash +# shellcheck disable=SC2034,SC2329 + +Describe 'AMD VHD content validation' + setup_amd_content() { + TEST_DIR=$(mktemp -d) + FEATURE_FLAGS=AMD_GPU OS_SKU=Ubuntu OS_VERSION=24.04 ENABLE_FIPS=false + COMPONENTS_FILEPATH="${PWD}/parts/common/components.json" + KERNEL=6.8.0-test-azure + FAIL_STAGE="" + PACKAGE_VERSION=$(jq -r '.AMDGPUDriver.packageVersion' "${COMPONENTS_FILEPATH}") + FIRMWARE_VERSION=$(jq -r '.AMDGPUDriver.firmwarePackageVersion' "${COMPONENTS_FILEPATH}") + MODULE_VERSION=$(jq -r '.AMDGPUDriver.moduleVersion' "${COMPONENTS_FILEPATH}") + DKMS_VERSION=$(jq -r '.AMDGPUDriver.dkmsVersion' "${COMPONENTS_FILEPATH}") + AMDSMI_PACKAGE=$(jq -r '.AMDGPUDiagnostics.amdsmiPackage' "${COMPONENTS_FILEPATH}") + SYSDEPS_PACKAGE=$(jq -r '.AMDGPUDiagnostics.sysdepsPackage' "${COMPONENTS_FILEPATH}") + mkdir -p "${TEST_DIR}/marker" "${TEST_DIR}/modules/${KERNEL}/build" "${TEST_DIR}/sources/amdgpu-${DKMS_VERSION}" + touch "${TEST_DIR}/modules/${KERNEL}/build/Makefile" "${TEST_DIR}/sources/amdgpu-${DKMS_VERSION}/dkms.conf" + jq -n --arg driver "${PACKAGE_VERSION}" --arg firmware "${FIRMWARE_VERSION}" --arg module "${MODULE_VERSION}" \ + '{schema_version: 1, package_version: $driver, firmware_package_version: $firmware, module_version: $module, + kernel_version: "earlier-bake-kernel"}' > "${TEST_DIR}/marker/driver.json" + eval "$(sed -n '/^testAMDGPUDriver()/,/^}$/p' vhdbuilder/packer/test/linux-vhd-content-test.sh | + sed -e "s|/opt/azure/amd-gpu|${TEST_DIR}/marker|g" -e "s|/lib/modules|${TEST_DIR}/modules|g" \ + -e "s|/usr/src|${TEST_DIR}/sources|g")" + } + cleanup_amd_content() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_amd_content + AfterEach cleanup_amd_content + err() { printf '%s: %s\n' "$1" "$2" >&2; } + uname() { if [ "$1" = -m ]; then echo x86_64; else echo "${KERNEL}"; fi; } + dpkg-query() { + if [ "$#" -eq 2 ]; then + printf 'ii amdgpu-dkms\nii amdgpu-dkms-firmware\n' + printf 'ii %s\nii %s:amd64\n' "${AMDSMI_PACKAGE}" "${SYSDEPS_PACKAGE}" + if [ "${FAIL_STAGE}" = sdk ]; then echo 'ii amdrocm-hip-sdk'; fi + if [ "${FAIL_STAGE}" = similar_package ]; then echo "ii ${AMDSMI_PACKAGE}-dev"; fi + if [ "${FAIL_STAGE}" = older_diagnostics ]; then echo 'ii amdrocm-amdsmi9.0'; fi + return + fi + case "${*: -1}" in + amdgpu-dkms) echo "install ok installed ${PACKAGE_VERSION}" ;; + amdgpu-dkms-firmware) + if [ "${FAIL_STAGE}" = firmware ]; then echo 'install ok installed wrong'; else echo "install ok installed ${FIRMWARE_VERSION}"; fi + ;; + build-essential) if [ "${FAIL_STAGE}" = dependencies ]; then return 1; else echo 'install ok installed'; fi ;; + *) echo 'install ok installed' ;; + esac + } + dkms() { + if [ "${FAIL_STAGE}" = registration ]; then + echo "amdgpu/${DKMS_VERSION}, ${KERNEL}, x86_64: added" + else + echo "amdgpu/${DKMS_VERSION}, ${KERNEL}, x86_64: installed" + fi + } + modinfo() { + case "$4" in + filename) + if [ "${FAIL_STAGE}" = inbox ]; then echo /kernel/inbox/amdgpu.ko; + else echo "${TEST_DIR}/modules/${KERNEL}/updates/dkms/amdgpu.ko.zst"; fi + ;; + version) if [ "${FAIL_STAGE}" = version ]; then echo wrong; else echo "${MODULE_VERSION}"; fi ;; + vermagic) if [ "${FAIL_STAGE}" = kernel ]; then echo wrong; else echo "${KERNEL} SMP mod_unload modversions"; fi ;; + esac + } + modprobe() { + [ "$*" = -c ] || { echo 'unexpected hardware-dependent command' >&2; return 99; } + if [ "${FAIL_STAGE}" = blacklist ]; then echo 'blacklist amdgpu'; fi + } + + It 'validates retained DKMS state on a CPU machine, treating bake kernel as provenance' + When run testAMDGPUDriver + The status should be success + The output should include 'rebuild dependencies verified' + The stderr should eq '' + End + + It 'skips ordinary images without inspecting AMD state' + FEATURE_FLAGS=NVIDIA_CUDA_PREBAKE + rm "${TEST_DIR}/marker/driver.json" + When run testAMDGPUDriver + The status should be success + The output should eq '' + The stderr should eq '' + End + + Describe 'unqualified image contents' + Parameters + firmware 'differs from pinned versions' + registration 'not installed for the running kernel' + inbox 'inbox or a missing module' + version 'version or vermagic mismatch' + kernel 'version or vermagic mismatch' + dependencies 'Missing retained build dependency' + sdk 'Unexpected GPU userspace' + similar_package 'Unexpected GPU userspace' + older_diagnostics 'Unexpected GPU userspace' + blacklist 'disabled in modprobe configuration' + End + It 'reports every failure on stderr as required by the VM content-test runner' + FAIL_STAGE="$1" + When run testAMDGPUDriver + The status should be failure + The output should eq '' + The stderr should include "$2" + End + End + + It 'reports missing marker metadata on stderr' + printf '{}\n' > "${TEST_DIR}/marker/driver.json" + When run testAMDGPUDriver + The status should be failure + The output should eq '' + The stderr should include 'Missing or inconsistent AMDGPU driver marker' + End + + It 'reports removed DKMS source even if the compiled module still exists' + rm "${TEST_DIR}/sources/amdgpu-${DKMS_VERSION}/dkms.conf" + When run testAMDGPUDriver + The status should be failure + The output should eq '' + The stderr should include 'Headers or AMDGPU sources' + End +End + +Describe 'AMD SMI VHD content validation' + setup_amd_diagnostics_content() { + TEST_DIR=$(mktemp -d) + FEATURE_FLAGS=AMD_GPU + FAIL_STAGE="" + COMPONENTS_FILEPATH="${TEST_DIR}/components.json" + CLI_PATH="${TEST_DIR}/rocm/core-10.0/bin/amd-smi" + CLI_TARGET="${TEST_DIR}/rocm/core-10.0/libexec/amdsmi_cli/amdsmi_cli.py" + MODULE_PATH="${TEST_DIR}/rocm/core-10.0/share/amd_smi/amdsmi/__init__.py" + mkdir -p "$(dirname "${CLI_PATH}")" "$(dirname "${CLI_TARGET}")" "$(dirname "${MODULE_PATH}")" \ + "${TEST_DIR}/bin" "${TEST_DIR}/environment" + printf '#!/bin/sh\nexit 99\n' > "${CLI_TARGET}" + ln -s ../libexec/amdsmi_cli/amdsmi_cli.py "${CLI_PATH}" + chmod +x "${CLI_PATH}" + ln -s "${CLI_PATH}" "${TEST_DIR}/bin/amd-smi" + cat > "${MODULE_PATH}" <<'PY' +from pathlib import Path +__version__ = "27.0.0-test" +def amdsmi_get_lib_version(): + Path(__file__).with_name("library-queried").touch() + return {"major": 27, "minor": 0, "release": 0} +def amdsmi_init(): + raise RuntimeError("GPU initialization must not run on the CPU bake machine") +PY + printf 'raise ImportError("unrelated PYTHONPATH must not affect the bake check")\n' > "${TEST_DIR}/environment/amdsmi.py" + jq --arg cli "${CLI_PATH}" '.AMDGPUDiagnostics.cliPath = $cli' parts/common/components.json > "${COMPONENTS_FILEPATH}" + AMDSMI_PACKAGE=$(jq -r '.AMDGPUDiagnostics.amdsmiPackage' "${COMPONENTS_FILEPATH}") + AMDSMI_VERSION=$(jq -r '.AMDGPUDiagnostics.amdsmiVersion' "${COMPONENTS_FILEPATH}") + SYSDEPS_PACKAGE=$(jq -r '.AMDGPUDiagnostics.sysdepsPackage' "${COMPONENTS_FILEPATH}") + SYSDEPS_VERSION=$(jq -r '.AMDGPUDiagnostics.sysdepsVersion' "${COMPONENTS_FILEPATH}") + eval "$(sed -n '/^testAMDGPUDiagnostics()/,/^}$/p' vhdbuilder/packer/test/linux-vhd-content-test.sh | + sed -e "s|/usr/local/bin/amd-smi|${TEST_DIR}/bin/amd-smi|g" \ + -e "s|/opt/rocm/|${TEST_DIR}/rocm/|g")" + } + cleanup_amd_diagnostics_content() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_amd_diagnostics_content + AfterEach cleanup_amd_diagnostics_content + err() { printf '%s: %s\n' "$1" "$2" >&2; } + dpkg-query() { + case "${*: -1}" in + "${AMDSMI_PACKAGE}") + if [ "${FAIL_STAGE}" = amdsmi ]; then echo 'install ok installed wrong'; else echo "install ok installed ${AMDSMI_VERSION}"; fi + ;; + "${SYSDEPS_PACKAGE}") + if [ "${FAIL_STAGE}" = sysdeps ]; then echo 'deinstall ok config-files'; else echo "install ok installed ${SYSDEPS_VERSION}"; fi + ;; + pciutils|numactl) + if [ "${FAIL_STAGE}" = "${*: -1}_package" ]; then echo 'deinstall ok config-files'; else echo 'install ok installed'; fi + ;; + esac + } + lspci() { + [ "$*" = --version ] || return 99 + [ "${FAIL_STAGE}" != lspci_cli ] || return 1 + echo 'lspci version 3.10.0' + } + numactl() { + [ "$*" = --version ] || return 99 + [ "${FAIL_STAGE}" != numactl_cli ] || return 1 + echo 'numactl version 2.0.18' + } + + It 'imports the library beside the vendor bin-to-libexec symlink with isolated Python' + PYTHONPATH="${TEST_DIR}/environment" + export PYTHONPATH + When run testAMDGPUDiagnostics + The status should be success + The output should include 'hardware-independent library loading verified' + The stderr should eq '' + The path "${TEST_DIR}/rocm/core-10.0/share/amd_smi/amdsmi/library-queried" should be exist + End + + It 'skips ordinary images' + FEATURE_FLAGS=NVIDIA_CUDA_PREBAKE + rm "${TEST_DIR}/bin/amd-smi" + When run testAMDGPUDiagnostics + The status should be success + The output should eq '' + The stderr should eq '' + End + + Describe 'incomplete diagnostics packages' + Parameters + amdsmi 'differ from pinned packages' + sysdeps 'differ from pinned packages' + library 'library failed to load' + pciutils_package 'Missing host diagnostics package pciutils' + numactl_package 'Missing host diagnostics package numactl' + lspci_cli 'Host diagnostics command lspci failed' + numactl_cli 'Host diagnostics command numactl failed' + End + It 'fails the content check' + FAIL_STAGE="$1" + if [ "${FAIL_STAGE}" = library ]; then printf 'raise ImportError("missing AMD SMI library")\n' > "${MODULE_PATH}"; fi + When run testAMDGPUDiagnostics + The status should be failure + The output should eq '' + The stderr should include "$2" + End + End + + It 'rejects a command symlink pointing to a different executable' + ln -sfn /bin/false "${TEST_DIR}/bin/amd-smi" + When run testAMDGPUDiagnostics + The status should be failure + The output should eq '' + The stderr should include 'executable or command symlink' + End + + It 'rejects a non-executable CLI' + chmod -x "${CLI_PATH}" + When run testAMDGPUDiagnostics + The status should be failure + The output should eq '' + The stderr should include 'executable or command symlink' + End + + It 'reports missing diagnostics metadata' + printf '{}\n' > "${COMPONENTS_FILEPATH}" + When run testAMDGPUDiagnostics + The status should be failure + The output should eq '' + The stderr should include 'Missing AMDGPU diagnostics component metadata' + End +End diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index dd4d4cd18a8..6840418c4d2 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -42,6 +42,19 @@ LOCALDNS_BINARY_PATH="/opt/azure/containers/localdns/binary" PERFORMANCE_DATA_FILE=/opt/azure/vhd-build-performance-data.json GRID_COMPATIBILITY_DATA_FILE=/opt/azure/vhd-grid-compatibility-data.json +isAMDGPUBuild() { + case "${FEATURE_FLAGS:-}" in *AMD_GPU*) return 0 ;; *) return 1 ;; esac +} + +if isAMDGPUBuild; then + if [ "${FEATURE_FLAGS}" != "AMD_GPU" ] || [ "${OS}" != "${UBUNTU_OS_NAME}" ] || + [ "${OS_VERSION}" != "24.04" ] || [ "${CPU_ARCH}" != "amd64" ] || + [ "${HYPERV_GENERATION,,}" != "v2" ] || [ "${ENABLE_FIPS,,}" = "true" ]; then + echo "AMD_GPU requires the dedicated Ubuntu 24.04 amd64 Gen2 non-FIPS image" >&2 + exit 1 + fi +fi + string_replace() { echo ${1//\*/$2} } @@ -505,6 +518,14 @@ cachePackageAndBinaryComponents() { while IFS= read -r p; do #getting metadata for each package name=$(echo "${p}" | jq .name -r) + if isAMDGPUBuild; then + case "${name}" in + nvidia-*|dra-driver-nvidia-*|datacenter-gpu-manager-*|dcgm-exporter) + echo "Skipping NVIDIA package ${name} on the AMD GPU image" + continue + ;; + esac + fi os=${OS} # TODO(mheberling): Remove this once kata uses standard containerd. This OS is referenced # in file `parts/common/component.json` with the same ${MARINER_KATA_OS_NAME}. @@ -1211,7 +1232,9 @@ if isMarinerOrAzureLinux "$OS" && ! isAzureLinuxOSGuard "$OS" "$OS_VARIANT"; the activateNfConntrack elif [ "${OS}" = "${UBUNTU_OS_NAME}" ]; then updateAptWithMicrosoftPkg - updateAptWithNvidiaPkg + if ! isAMDGPUBuild; then + updateAptWithNvidiaPkg + fi fi capture_benchmark "${SCRIPT_NAME}_handle_os_specific_configurations" @@ -1228,9 +1251,16 @@ ctr namespace create k8s.io # running them here (near-empty disk, BCC not yet started) avoids exhausting the 30GB packer build disk. # Running them after the container-image cache and/or concurrently with the BCC build fills the disk # (worse on 24.04), failing at the nvidia.ko link or the driver lib copy with "No space left on device". -cacheGPUContainerImageComponents -buildNVIDIAKernelModule -capture_benchmark "${SCRIPT_NAME}_caching_gpu_container_images_and_build_nvidia_kernel_module" +if isAMDGPUBuild; then + installAMDGPUDriver || exit 1 + capture_benchmark "${SCRIPT_NAME}_build_amd_gpu_kernel_module" + installAMDGPUDiagnostics || exit 1 + capture_benchmark "${SCRIPT_NAME}_install_amd_gpu_diagnostics" +else + cacheGPUContainerImageComponents + buildNVIDIAKernelModule + capture_benchmark "${SCRIPT_NAME}_caching_gpu_container_images_and_build_nvidia_kernel_module" +fi # Start eBPF tool installation in the background while we pull container images in the foreground startEBPFToolsInstallation diff --git a/vhdbuilder/packer/produce-packer-settings-functions.sh b/vhdbuilder/packer/produce-packer-settings-functions.sh index 76572ca6531..e61f066dd0e 100644 --- a/vhdbuilder/packer/produce-packer-settings-functions.sh +++ b/vhdbuilder/packer/produce-packer-settings-functions.sh @@ -1,5 +1,47 @@ #!/bin/bash +function validate_amd_gpu_build() { + case "${FEATURE_FLAGS:-}" in + *AMD_GPU*) ;; + *) return 0 ;; + esac + + # This is a separate driver-only image, not an addition to a shared NVIDIA VHD. + if [ "${FEATURE_FLAGS}" != "AMD_GPU" ] || [ "${OS_SKU}" != "Ubuntu" ] || + [ "${OS_VERSION}" != "24.04" ] || [ "${ARCHITECTURE,,}" != "x86_64" ] || + [ "${HYPERV_GENERATION,,}" != "v2" ] || [ "${ENABLE_FIPS,,}" = "true" ] || + [ "${ENABLE_TRUSTED_LAUNCH,,}" = "true" ] || [ "${TRUSTED_LAUNCH_SUPPORTED,,}" = "true" ]; then + echo "AMD_GPU requires Ubuntu 24.04 x86_64 Gen2, without other feature flags, FIPS or Trusted Launch" >&2 + return 1 + fi + if [ -n "${SKU_NAME:-}" ] && [ "${SKU_NAME}" != "2404gen2amdgpucontainerd" ]; then + echo "AMD_GPU requires the dedicated SKU_NAME 2404gen2amdgpucontainerd" >&2 + return 1 + fi + if [ -n "${SIG_IMAGE_NAME:-}" ] && [ "${SIG_IMAGE_NAME}" != "2404gen2amdgpucontainerd" ]; then + echo "AMD_GPU requires the dedicated SIG_IMAGE_NAME 2404gen2amdgpucontainerd" >&2 + return 1 + fi +} + +function get_linux_sku_name() { + validate_amd_gpu_build || return 1 + local sku_name="${OS_VERSION}" + if grep -q "minimal" <<<"$FEATURE_FLAGS"; then sku_name+="minimal"; fi + if [ "${HYPERV_GENERATION,,}" = "v2" ]; then sku_name+="gen2"; fi + if [ "${ARCHITECTURE,,}" = "arm64" ]; then sku_name+="arm64"; fi + if grep -q "NVIDIA_GB" <<<"$FEATURE_FLAGS"; then sku_name+="gb"; fi + if [ "${FEATURE_FLAGS}" = "AMD_GPU" ]; then sku_name+="amdgpu"; fi + if [ "${ENABLE_FIPS,,}" = "true" ]; then sku_name+="fips"; fi + if grep -q "cvm" <<<"$FEATURE_FLAGS"; then sku_name+="CVM"; fi + if [ "${ENABLE_TRUSTED_LAUNCH}" = "True" ]; then sku_name+="TL"; fi + case "${OS_SKU}" in + CBLMariner | AzureLinux | AzureLinuxOSGuard | Flatcar | AzureContainerLinux) ;; + *) sku_name+="containerd" ;; + esac + printf '%s\n' "${sku_name//./}" +} + function compute_msi_resource_strings() { # Populates the caller-declared array named by $1 (an explicit output parameter, via # nameref) with the UAMI resource string to attach to the VHD build VM, or leaves it diff --git a/vhdbuilder/packer/produce-packer-settings.sh b/vhdbuilder/packer/produce-packer-settings.sh index 248f63ed65d..9fa8036fa92 100755 --- a/vhdbuilder/packer/produce-packer-settings.sh +++ b/vhdbuilder/packer/produce-packer-settings.sh @@ -4,6 +4,10 @@ set -e SCRIPT_DIR=$(dirname "$0") source "$SCRIPT_DIR/produce-packer-settings-functions.sh" +if [ "${MODE}" = "linuxVhdMode" ]; then + validate_amd_gpu_build +fi + CDIR=$(dirname "${BASH_SOURCE}") SETTINGS_JSON="${SETTINGS_JSON:-./packer/settings.json}" PUBLISHER_BASE_IMAGE_VERSION_JSON="${PUBLISHER_BASE_IMAGE_VERSION_JSON:-./vhdbuilder/publisher_base_image_version.json}" diff --git a/vhdbuilder/packer/test/linux-vhd-content-test.sh b/vhdbuilder/packer/test/linux-vhd-content-test.sh index 805f0461828..ab6fdd640e8 100644 --- a/vhdbuilder/packer/test/linux-vhd-content-test.sh +++ b/vhdbuilder/packer/test/linux-vhd-content-test.sh @@ -244,6 +244,9 @@ testPackagesInstalled() { while IFS= read -r p; do name=$(echo "${p}" | jq .name -r) + if [ "${FEATURE_FLAGS:-}" = "AMD_GPU" ]; then + case "${name}" in nvidia-*|dra-driver-nvidia-*|datacenter-gpu-manager-*|dcgm-exporter) continue ;; esac + fi downloadLocation=$(echo "${p}" | jq .downloadLocation -r) if [ "$downloadLocation" = "" ] || [ "$downloadLocation" = "null" ]; then continue @@ -2729,6 +2732,144 @@ testContainerNetworkingPluginsInstalled() { return 0 } +# Hardware-independent validation: the AMD VHD is baked and content-tested on a +# CPU VM. Loading the module and checking KFD/GPU devices belongs to node CSE/E2E. +# shellcheck disable=SC2016 +testAMDGPUDriver() { + [ "${FEATURE_FLAGS:-}" = "AMD_GPU" ] || return 0 + local test=testAMDGPUDriver marker=/opt/azure/amd-gpu/driver.json + local metadata kernel_version module_path module_vermagic package_version firmware_version module_version dkms_version + local installed_packages modprobe_config package diagnostics_amdsmi diagnostics_sysdeps + if [ "${OS_SKU}" != "Ubuntu" ] || [ "${OS_VERSION}" != "24.04" ] || + [ "${ENABLE_FIPS,,}" = "true" ] || [ "$(uname -m)" != "x86_64" ]; then + err "${test}" "Unsupported AMD GPU image configuration" + return 1 + fi + if ! metadata=$(jq -ce '.AMDGPUDriver | select(type == "object")' "${COMPONENTS_FILEPATH}") || + ! package_version=$(jq -er '.packageVersion | strings | select(length > 0)' <<< "${metadata}") || + ! firmware_version=$(jq -er '.firmwarePackageVersion | strings | select(length > 0)' <<< "${metadata}") || + ! module_version=$(jq -er '.moduleVersion | strings | select(length > 0)' <<< "${metadata}") || + ! dkms_version=$(jq -er '.dkmsVersion | strings | select(length > 0)' <<< "${metadata}"); then + err "${test}" "Missing AMDGPU component metadata" + return 1 + fi + if ! jq -e --arg package "${package_version}" --arg firmware "${firmware_version}" --arg module "${module_version}" \ + '.schema_version == 1 and .package_version == $package and .firmware_package_version == $firmware and + .module_version == $module and (.kernel_version | type == "string" and length > 0)' "${marker}" >/dev/null; then + err "${test}" "Missing or inconsistent AMDGPU driver marker" + return 1 + fi + if [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms)" != "install ok installed ${package_version}" ] || + [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms-firmware)" != "install ok installed ${firmware_version}" ]; then + err "${test}" "Installed AMDGPU driver/firmware differs from pinned versions" + return 1 + fi + kernel_version=$(uname -r) + if ! dkms status -m amdgpu -v "${dkms_version}" -k "${kernel_version}" | + grep -Fqx "amdgpu/${dkms_version}, ${kernel_version}, x86_64: installed"; then + err "${test}" "AMDGPU DKMS is not installed for the running kernel" + return 1 + fi + module_path=$(modinfo -k "${kernel_version}" -F filename amdgpu) + case "${module_path}" in + /lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*|/usr/lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*) ;; + *) err "${test}" "AMDGPU resolves to the inbox or a missing module"; return 1 ;; + esac + module_vermagic=$(modinfo -k "${kernel_version}" -F vermagic amdgpu) + if [ "$(modinfo -k "${kernel_version}" -F version amdgpu)" != "${module_version}" ] || + [ "${module_vermagic%% *}" != "${kernel_version}" ]; then + err "${test}" "AMDGPU module version or vermagic mismatch" + return 1 + fi + if [ ! -f "/lib/modules/${kernel_version}/build/Makefile" ] || [ ! -f "/usr/src/amdgpu-${dkms_version}/dkms.conf" ]; then + err "${test}" "Headers or AMDGPU sources required by future DKMS rebuilds are missing" + return 1 + fi + for package in build-essential dkms autoconf automake initramfs-tools "linux-headers-${kernel_version}" "linux-modules-extra-${kernel_version}"; do + if [ "$(dpkg-query -W -f='${Status}' "${package}")" != 'install ok installed' ]; then + err "${test}" "Missing retained build dependency ${package}" + return 1 + fi + done + if ! installed_packages=$(dpkg-query -W -f='${db:Status-Abbrev} ${binary:Package}\n'); then + err "${test}" "Unable to inspect installed packages" + return 1 + fi + if ! diagnostics_amdsmi=$(jq -er '.AMDGPUDiagnostics.amdsmiPackage | strings | select(length > 0)' "${COMPONENTS_FILEPATH}") || + ! diagnostics_sysdeps=$(jq -er '.AMDGPUDiagnostics.sysdepsPackage | strings | select(length > 0)' "${COMPONENTS_FILEPATH}"); then + err "${test}" "Missing AMDGPU diagnostics package allowlist" + return 1 + fi + if awk -v amdsmi="${diagnostics_amdsmi}" -v sysdeps="${diagnostics_sysdeps}" \ + '$1 == "ii" { sub(/:.*/, "", $2); if ($2 != amdsmi && $2 != sysdeps) print $2 }' <<< "${installed_packages}" | + grep -Eq '^(amdrocm|rocm|hip|hsa-rocr|rocblas|rocfft|rocrand|rocsolver|rocsparse|miopen|migraphx|amdgpu-(core|lib|install|pro)|lib.*-amdgpu-|nvidia-|libnvidia-|cuda-|datacenter-gpu-manager-|dcgm-exporter)'; then + err "${test}" "Unexpected GPU userspace or NVIDIA package on the AMD driver-and-diagnostics image" + return 1 + fi + if ! modprobe_config=$(modprobe -c) || + grep -Eq '^[[:space:]]*(blacklist|install)[[:space:]]+amdgpu([[:space:]]|$)' <<< "${modprobe_config}"; then + err "${test}" "AMDGPU is disabled in modprobe configuration" + return 1 + fi + echo "${test}: pinned driver and firmware, DKMS module and rebuild dependencies verified" +} + +# AMD SMI initializes GPU drivers even for --help. CPU bake validation checks +# contents and loads the library without amdsmi_init; device discovery runs in GPU E2E. +# shellcheck disable=SC2016 +testAMDGPUDiagnostics() { + [ "${FEATURE_FLAGS:-}" = "AMD_GPU" ] || return 0 + local test=testAMDGPUDiagnostics metadata amdsmi_package amdsmi_version sysdeps_package sysdeps_version cli_path library_output + local package tool tool_output + if ! metadata=$(jq -ce '.AMDGPUDiagnostics | select(type == "object")' "${COMPONENTS_FILEPATH}") || + ! amdsmi_package=$(jq -er '.amdsmiPackage | strings | select(length > 0)' <<< "${metadata}") || + ! amdsmi_version=$(jq -er '.amdsmiVersion | strings | select(length > 0)' <<< "${metadata}") || + ! sysdeps_package=$(jq -er '.sysdepsPackage | strings | select(length > 0)' <<< "${metadata}") || + ! sysdeps_version=$(jq -er '.sysdepsVersion | strings | select(length > 0)' <<< "${metadata}") || + ! cli_path=$(jq -er '.cliPath | strings | select(startswith("/opt/rocm/"))' <<< "${metadata}"); then + err "${test}" "Missing AMDGPU diagnostics component metadata" + return 1 + fi + if [ "$(dpkg-query -W -f='${Status} ${Version}' "${amdsmi_package}")" != "install ok installed ${amdsmi_version}" ] || + [ "$(dpkg-query -W -f='${Status} ${Version}' "${sysdeps_package}")" != "install ok installed ${sysdeps_version}" ]; then + err "${test}" "Installed AMD SMI diagnostics differ from pinned packages" + return 1 + fi + for package in pciutils numactl; do + if [ "$(dpkg-query -W -f='${Status}' "${package}")" != 'install ok installed' ]; then + err "${test}" "Missing host diagnostics package ${package}" + return 1 + fi + done + for tool in lspci numactl; do + if ! tool_output=$("${tool}" --version 2>&1); then + err "${test}" "Host diagnostics command ${tool} failed: ${tool_output}" + return 1 + fi + done + if [ ! -x "${cli_path}" ] || [ ! -L /usr/local/bin/amd-smi ] || + [ "$(readlink -f /usr/local/bin/amd-smi)" != "$(readlink -f "${cli_path}")" ]; then + err "${test}" "AMD SMI executable or command symlink is missing or inconsistent" + return 1 + fi + if ! library_output=$(/usr/bin/python3 -I - "${cli_path}" <<'PY' +import json +from pathlib import Path +import sys +# The vendor bin/amd-smi symlink points into libexec; preserve its lexical bin path. +sys.path.insert(0, str(Path(sys.argv[1]).parents[1] / "share/amd_smi")) +import amdsmi +version = amdsmi.amdsmi_get_lib_version() +assert version["major"] > 0, version +print(json.dumps({"python_module": amdsmi.__version__, "library": version})) +PY + ); then + err "${test}" "AMD SMI library failed to load without GPU hardware: ${library_output}" + return 1 + fi + echo "${test}: pinned AMD SMI packages, command symlink, PCI/NUMA tools and hardware-independent library loading verified" +} + # As we call these tests, we need to bear in mind how the test results are processed by the # the caller in run-tests.sh. That code uses az vm run-command invoke to run this script # on a VM. It then looks at stderr to see if any errors were reported. Notably it doesn't @@ -2799,3 +2940,5 @@ testFileOwnership $OS_SKU testDiskQueueServiceIsActive testVulnerableKernelModulesDisabled $OS_SKU $OS_VERSION testArtifactStreamingPackagesCleanedUp +testAMDGPUDriver +testAMDGPUDiagnostics diff --git a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh index 4d6cba7f9e5..3c54b6aedf4 100755 --- a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh +++ b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh @@ -20,6 +20,228 @@ ERR_CHRONY_START_TIMEOUT=15 # Unable to start CHRONY echo "Sourcing tool_installs_ubuntu.sh" +# Bake on a CPU VM: keep DKMS and its build dependencies for kernel servicing, +# but never load the driver or install the ROCm compute SDK on the host. +# The subshell owns its cleanup trap without replacing the caller's traps. +installAMDGPUDriver() { + ( + set -o pipefail + local metadata repository key_url fingerprint package_version firmware_version module_version dkms_version + local kernel_version work_dir marker_dir=/opt/azure/amd-gpu marker_tmp="" + local generation=${HYPERV_GENERATION:-} enable_fips=${ENABLE_FIPS:-false} + local actual_fingerprint archive package_name package_arch module_path module_vermagic modprobe_config conf + local driver_deb="" firmware_deb="" + local -a apt_options build_packages + if [ "${OS:-}" != "UBUNTU" ] || [ "${OS_VERSION:-}" != "24.04" ] || + [ "${CPU_ARCH:-}" != "amd64" ] || [ "${generation,,}" != "v2" ] || + [ "${enable_fips,,}" = "true" ]; then + echo "AMDGPU requires Ubuntu 24.04 amd64 Gen2 without FIPS" >&2 + exit 1 + fi + # A failed re-bake must not leave a success marker from an earlier attempt. + rm -f "${marker_dir}/driver.json" || exit 1 + metadata=$(jq -ce '.AMDGPUDriver | select(type == "object")' "${COMPONENTS_FILEPATH}") || exit 1 + repository=$(jq -er '.repositoryURL | select(test("^https://repo[.]radeon[.]com/amdgpu/[0-9.]+/ubuntu$"))' <<< "${metadata}") || exit 1 + key_url=$(jq -er '.signingKeyURL | select(test("^https://repo[.]radeon[.]com/[^?[:space:]]+$"))' <<< "${metadata}") || exit 1 + fingerprint=$(jq -er '.signingKeyFingerprint | select(test("^[A-F0-9]{40}$"))' <<< "${metadata}") || exit 1 + jq -e '.distribution == "noble" and .component == "main"' <<< "${metadata}" >/dev/null || exit 1 + package_version=$(jq -er '.packageVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 + firmware_version=$(jq -er '.firmwarePackageVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 + module_version=$(jq -er '.moduleVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 + dkms_version=$(jq -er '.dkmsVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 + kernel_version=$(uname -r) || exit 1 + work_dir=$(mktemp -d /tmp/amd-gpu.XXXXXX) || exit 1 + trap 'rm -rf "${work_dir}"; if [ -n "${marker_tmp}" ]; then rm -f "${marker_tmp}"; fi' EXIT + build_packages=(ca-certificates curl gnupg build-essential dkms autoconf automake initramfs-tools + "linux-headers-${kernel_version}" "linux-modules-extra-${kernel_version}") + apt_get_update || exit 1 + apt_get_install 20 5 1800 "${build_packages[@]}" || exit 1 + test -f "/lib/modules/${kernel_version}/build/Makefile" || exit 1 + + mkdir -p "${work_dir}/gnupg" "${work_dir}/lists/partial" "${work_dir}/packages" || exit 1 + chmod 755 "${work_dir}" || exit 1 + chmod 700 "${work_dir}/gnupg" || exit 1 + retrycmd_if_failure 5 5 60 curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' \ + "${key_url}" -o "${work_dir}/key.asc" || exit 1 + # Compare primary fingerprints only; the legitimate key also has an encryption subkey. + actual_fingerprint=$(gpg --batch --homedir "${work_dir}/gnupg" --show-keys --with-colons "${work_dir}/key.asc" | + awk -F: '$1 == "pub" { primary = 1; next } primary && $1 == "fpr" { print $10; primary = 0 }') || exit 1 + if [ "${actual_fingerprint}" != "${fingerprint}" ]; then + echo "AMDGPU signing key fingerprint mismatch" >&2 + exit 1 + fi + gpg --batch --homedir "${work_dir}/gnupg" --dearmor --output "${work_dir}/key.gpg" "${work_dir}/key.asc" || exit 1 + chmod 644 "${work_dir}/key.gpg" || exit 1 + printf 'deb [arch=amd64 signed-by=%s/key.gpg] %s noble main\n' "${work_dir}" "${repository}" > "${work_dir}/amdgpu.list" || exit 1 + # AMD's signed metadata is isolated from normal APT state. Only these two + # exact packages come from AMD; Ubuntu supplies all remaining dependencies. + apt_options=(-o "Dir::Etc::sourcelist=${work_dir}/amdgpu.list" -o Dir::Etc::sourceparts=- + -o "Dir::State::lists=${work_dir}/lists" -o APT::Get::AllowUnauthenticated=false + -o Acquire::AllowInsecureRepositories=false -o Acquire::AllowDowngradeToInsecureRepositories=false) + retrycmd_if_failure 5 5 300 apt-get "${apt_options[@]}" update || exit 1 + cd "${work_dir}/packages" || exit 1 + retrycmd_if_failure 5 5 300 apt-get "${apt_options[@]}" download \ + "amdgpu-dkms=${package_version}" "amdgpu-dkms-firmware=${firmware_version}" || exit 1 + for archive in "${work_dir}/packages/"*.deb; do + package_name=$(dpkg-deb -f "${archive}" Package) || exit 1 + package_arch=$(dpkg-deb -f "${archive}" Architecture) || exit 1 + case "${package_arch}" in all|amd64) ;; *) echo "Unexpected AMDGPU package architecture" >&2; exit 1 ;; esac + case "${package_name}" in + amdgpu-dkms) + [ -z "${driver_deb}" ] && [ "$(dpkg-deb -f "${archive}" Version)" = "${package_version}" ] || exit 1 + driver_deb=${archive} + ;; + amdgpu-dkms-firmware) + [ -z "${firmware_deb}" ] && [ "$(dpkg-deb -f "${archive}" Version)" = "${firmware_version}" ] || exit 1 + firmware_deb=${archive} + ;; + *) echo "Unexpected AMDGPU package: ${package_name}" >&2; exit 1 ;; + esac + done + [ -n "${driver_deb}" ] && [ -n "${firmware_deb}" ] || exit 1 + apt_get_install 20 5 1800 "${driver_deb}" "${firmware_deb}" || exit 1 + # Keep the compiler, matching headers, firmware and DKMS sources through + # image cleanup. No vendor repository/key is persisted on the image. + apt-mark manual "${build_packages[@]}" amdgpu-dkms amdgpu-dkms-firmware || exit 1 + # shellcheck disable=SC2016 + [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms)" = "install ok installed ${package_version}" ] || exit 1 + # shellcheck disable=SC2016 + [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms-firmware)" = "install ok installed ${firmware_version}" ] || exit 1 + dkms install -m amdgpu -v "${dkms_version}" -k "${kernel_version}" || exit 1 + dkms status -m amdgpu -v "${dkms_version}" -k "${kernel_version}" | grep -Fqx "amdgpu/${dkms_version}, ${kernel_version}, x86_64: installed" || exit 1 + depmod -a "${kernel_version}" || exit 1 + module_path=$(modinfo -k "${kernel_version}" -F filename amdgpu) || exit 1 + case "${module_path}" in + /lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*|/usr/lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*) ;; + *) echo "AMDGPU resolves to an unexpected module: ${module_path}" >&2; exit 1 ;; + esac + [ "$(modinfo -k "${kernel_version}" -F version amdgpu)" = "${module_version}" ] || exit 1 + module_vermagic=$(modinfo -k "${kernel_version}" -F vermagic amdgpu) || exit 1 + [ "${module_vermagic%% *}" = "${kernel_version}" ] || exit 1 + + # Azure's CPU base image may blacklist the inbox GPU driver. Remove only + # the AMD entry, retaining unrelated cloud-image module deny rules. + for conf in /etc/modprobe.d/*.conf /usr/lib/modprobe.d/*.conf; do + [ -f "${conf}" ] || continue + sed -i -E '/^[[:space:]]*blacklist[[:space:]]+amdgpu([[:space:]]|$)/d' "${conf}" || exit 1 + done + modprobe_config=$(modprobe -c) || exit 1 + if grep -Eq '^[[:space:]]*(blacklist|install)[[:space:]]+amdgpu([[:space:]]|$)' <<< "${modprobe_config}"; then + echo "AMDGPU remains disabled in modprobe configuration" >&2 + exit 1 + fi + update-initramfs -u -k "${kernel_version}" || exit 1 + mkdir -p "${marker_dir}" || exit 1 + marker_tmp=$(mktemp "${marker_dir}/.driver.json.XXXXXX") || exit 1 + jq -n --arg package_version "${package_version}" --arg firmware_package_version "${firmware_version}" \ + --arg module_version "${module_version}" --arg kernel_version "${kernel_version}" \ + '{schema_version: 1, package_version: $package_version, firmware_package_version: $firmware_package_version, + module_version: $module_version, kernel_version: $kernel_version}' > "${marker_tmp}" || exit 1 + chmod 644 "${marker_tmp}" || exit 1 + echo " - amdgpu-dkms version ${package_version}; module ${module_version}; kernel ${kernel_version}" >> "${VHD_LOGS_FILEPATH}" || exit 1 + mv -f "${marker_tmp}" "${marker_dir}/driver.json" || exit 1 + ) +} + +# Host diagnostics are intentionally separate from the compute SDK. AMD SMI +# uses only its management library and sysdeps package; no pip installation. +installAMDGPUDiagnostics() { + ( + set -o pipefail + local metadata repository key_url fingerprint actual_fingerprint work_dir + local amdsmi_package amdsmi_version sysdeps_package sysdeps_version cli_path python_path + local archive package_name package_arch amdsmi_deb="" sysdeps_deb="" + local generation=${HYPERV_GENERATION:-} enable_fips=${ENABLE_FIPS:-false} + local -a apt_options ubuntu_packages + if [ "${OS:-}" != "UBUNTU" ] || [ "${OS_VERSION:-}" != "24.04" ] || + [ "${CPU_ARCH:-}" != "amd64" ] || [ "${generation,,}" != "v2" ] || + [ "${enable_fips,,}" = "true" ]; then + echo "AMD GPU diagnostics require Ubuntu 24.04 amd64 Gen2 without FIPS" >&2 + exit 1 + fi + metadata=$(jq -ce '.AMDGPUDiagnostics | select(type == "object")' "${COMPONENTS_FILEPATH}") || exit 1 + repository=$(jq -er '.repositoryURL | select(. == "https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/")' <<< "${metadata}") || exit 1 + key_url=$(jq -er '.signingKeyURL | select(. == "https://stable.repo.amd.com/rocm/gpg/packages.gpg")' <<< "${metadata}") || exit 1 + fingerprint=$(jq -er '.signingKeyFingerprint | select(test("^[A-F0-9]{40}$"))' <<< "${metadata}") || exit 1 + jq -e '.distribution == "stable" and .component == "main"' <<< "${metadata}" >/dev/null || exit 1 + amdsmi_package=$(jq -er '.amdsmiPackage | select(test("^amdrocm-amdsmi[0-9]+[.][0-9]+$"))' <<< "${metadata}") || exit 1 + sysdeps_package=$(jq -er '.sysdepsPackage | select(test("^amdrocm-sysdeps[0-9]+[.][0-9]+$"))' <<< "${metadata}") || exit 1 + amdsmi_version=$(jq -er '.amdsmiVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 + sysdeps_version=$(jq -er '.sysdepsVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 + cli_path=$(jq -er '.cliPath | select(test("^/opt/rocm/core-[0-9]+[.][0-9]+/bin/amd-smi$"))' <<< "${metadata}") || exit 1 + python_path="${cli_path%/bin/amd-smi}/share/amd_smi" + work_dir=$(mktemp -d /tmp/amd-gpu-diagnostics.XXXXXX) || exit 1 + trap 'rm -rf "${work_dir}"' EXIT + # The vendor package omits the C++ runtime dependency from its metadata. + ubuntu_packages=(ca-certificates curl gnupg python3 libstdc++6 libgcc-s1 pciutils numactl) + apt_get_update || exit 1 + apt_get_install 20 5 600 "${ubuntu_packages[@]}" || exit 1 + mkdir -p "${work_dir}/gnupg" "${work_dir}/lists/partial" "${work_dir}/packages" || exit 1 + chmod 755 "${work_dir}" || exit 1 + chmod 700 "${work_dir}/gnupg" || exit 1 + retrycmd_if_failure 5 5 60 curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' \ + "${key_url}" -o "${work_dir}/key.asc" || exit 1 + actual_fingerprint=$(gpg --batch --homedir "${work_dir}/gnupg" --show-keys --with-colons "${work_dir}/key.asc" | + awk -F: '$1 == "pub" { primary = 1; next } primary && $1 == "fpr" { print $10; primary = 0 }') || exit 1 + if [ "${actual_fingerprint}" != "${fingerprint}" ]; then + echo "AMD GPU diagnostics signing key fingerprint mismatch" >&2 + exit 1 + fi + gpg --batch --homedir "${work_dir}/gnupg" --dearmor --output "${work_dir}/key.gpg" "${work_dir}/key.asc" || exit 1 + chmod 644 "${work_dir}/key.gpg" || exit 1 + printf 'deb [arch=amd64 signed-by=%s/key.gpg] %s stable main\n' "${work_dir}" "${repository}" > "${work_dir}/diagnostics.list" || exit 1 + apt_options=(-o "Dir::Etc::sourcelist=${work_dir}/diagnostics.list" -o Dir::Etc::sourceparts=- + -o "Dir::State::lists=${work_dir}/lists" -o APT::Get::AllowUnauthenticated=false + -o Acquire::AllowInsecureRepositories=false -o Acquire::AllowDowngradeToInsecureRepositories=false) + retrycmd_if_failure 5 5 300 apt-get "${apt_options[@]}" update || exit 1 + cd "${work_dir}/packages" || exit 1 + retrycmd_if_failure 5 5 300 apt-get "${apt_options[@]}" download \ + "${amdsmi_package}=${amdsmi_version}" "${sysdeps_package}=${sysdeps_version}" || exit 1 + for archive in "${work_dir}/packages/"*.deb; do + package_name=$(dpkg-deb -f "${archive}" Package) || exit 1 + package_arch=$(dpkg-deb -f "${archive}" Architecture) || exit 1 + [ "${package_arch}" = amd64 ] || { echo "Unexpected AMD diagnostics package architecture" >&2; exit 1; } + case "${package_name}" in + "${amdsmi_package}") + [ -z "${amdsmi_deb}" ] && [ "$(dpkg-deb -f "${archive}" Version)" = "${amdsmi_version}" ] || exit 1 + amdsmi_deb=${archive} + ;; + "${sysdeps_package}") + [ -z "${sysdeps_deb}" ] && [ "$(dpkg-deb -f "${archive}" Version)" = "${sysdeps_version}" ] || exit 1 + sysdeps_deb=${archive} + ;; + *) echo "Unexpected AMD diagnostics package: ${package_name}" >&2; exit 1 ;; + esac + done + [ -n "${amdsmi_deb}" ] && [ -n "${sysdeps_deb}" ] || exit 1 + apt_get_install 20 5 600 "${amdsmi_deb}" "${sysdeps_deb}" || exit 1 + apt-mark manual "${ubuntu_packages[@]}" "${amdsmi_package}" "${sysdeps_package}" || exit 1 + # shellcheck disable=SC2016 + [ "$(dpkg-query -W -f='${Status} ${Version}' "${amdsmi_package}")" = "install ok installed ${amdsmi_version}" ] || exit 1 + # shellcheck disable=SC2016 + [ "$(dpkg-query -W -f='${Status} ${Version}' "${sysdeps_package}")" = "install ok installed ${sysdeps_version}" ] || exit 1 + [ -x "${cli_path}" ] || exit 1 + # Even --help initializes hardware in AMD SMI. Verify the Python bindings + # and shared-library dependency closure without initialization on CPU bakes. + python3 -I - "${python_path}" <<'AMDSMI_CHECK' || exit 1 +import json +import sys +sys.path.insert(0, sys.argv[1]) +import amdsmi +version = amdsmi.amdsmi_get_lib_version() +assert version["major"] > 0, version +print(json.dumps({"amd_smi_library": version, "python_module": amdsmi.__version__})) +AMDSMI_CHECK + if [ -e /usr/local/bin/amd-smi ] && [ ! -L /usr/local/bin/amd-smi ]; then + echo "Refusing to replace a non-symlink at /usr/local/bin/amd-smi" >&2 + exit 1 + fi + mkdir -p /usr/local/bin || exit 1 + ln -sfnT "${cli_path}" /usr/local/bin/amd-smi || exit 1 + echo " - ${amdsmi_package} version ${amdsmi_version}; ${sysdeps_package} version ${sysdeps_version}" >> "${VHD_LOGS_FILEPATH}" || exit 1 + ) +} + installBcc() { echo "Installing BCC tools..." wait_for_apt_locks From 06b85e58850814a6e2b3b56ba617441efbf9ea5f Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 19 Sep 2026 08:00:04 +0000 Subject: [PATCH 2/7] fix(amd-gpu): support ShellSpec CI dependencies and jq version --- spec/shellspec.Dockerfile | 2 +- spec/vhdbuilder/packer/test/amd_gpu_content_spec.sh | 4 ++-- vhdbuilder/packer/test/linux-vhd-content-test.sh | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/spec/shellspec.Dockerfile b/spec/shellspec.Dockerfile index d6e31d080eb..0d72f05f43f 100644 --- a/spec/shellspec.Dockerfile +++ b/spec/shellspec.Dockerfile @@ -30,7 +30,7 @@ FROM aksdataplanedev.azurecr.io/shellspec/shellspec-debian:0.28.1 RUN sed -i -e 's/\(deb\|security\).debian.org/archive.debian.org/g' /etc/apt/sources.list && \ apt-get update && \ apt-get install -y --no-install-recommends \ - gawk jq curl dnsutils \ + gawk jq curl dnsutils python3 \ libcurl4 libdw1 libelf1 libssl1.1 zlib1g && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/spec/vhdbuilder/packer/test/amd_gpu_content_spec.sh b/spec/vhdbuilder/packer/test/amd_gpu_content_spec.sh index 6945e6ebeff..633a527a550 100644 --- a/spec/vhdbuilder/packer/test/amd_gpu_content_spec.sh +++ b/spec/vhdbuilder/packer/test/amd_gpu_content_spec.sh @@ -16,8 +16,8 @@ Describe 'AMD VHD content validation' SYSDEPS_PACKAGE=$(jq -r '.AMDGPUDiagnostics.sysdepsPackage' "${COMPONENTS_FILEPATH}") mkdir -p "${TEST_DIR}/marker" "${TEST_DIR}/modules/${KERNEL}/build" "${TEST_DIR}/sources/amdgpu-${DKMS_VERSION}" touch "${TEST_DIR}/modules/${KERNEL}/build/Makefile" "${TEST_DIR}/sources/amdgpu-${DKMS_VERSION}/dkms.conf" - jq -n --arg driver "${PACKAGE_VERSION}" --arg firmware "${FIRMWARE_VERSION}" --arg module "${MODULE_VERSION}" \ - '{schema_version: 1, package_version: $driver, firmware_package_version: $firmware, module_version: $module, + jq -n --arg driver "${PACKAGE_VERSION}" --arg firmware "${FIRMWARE_VERSION}" --arg module_version "${MODULE_VERSION}" \ + '{schema_version: 1, package_version: $driver, firmware_package_version: $firmware, module_version: $module_version, kernel_version: "earlier-bake-kernel"}' > "${TEST_DIR}/marker/driver.json" eval "$(sed -n '/^testAMDGPUDriver()/,/^}$/p' vhdbuilder/packer/test/linux-vhd-content-test.sh | sed -e "s|/opt/azure/amd-gpu|${TEST_DIR}/marker|g" -e "s|/lib/modules|${TEST_DIR}/modules|g" \ diff --git a/vhdbuilder/packer/test/linux-vhd-content-test.sh b/vhdbuilder/packer/test/linux-vhd-content-test.sh index ab6fdd640e8..21214b1b839 100644 --- a/vhdbuilder/packer/test/linux-vhd-content-test.sh +++ b/vhdbuilder/packer/test/linux-vhd-content-test.sh @@ -2753,9 +2753,9 @@ testAMDGPUDriver() { err "${test}" "Missing AMDGPU component metadata" return 1 fi - if ! jq -e --arg package "${package_version}" --arg firmware "${firmware_version}" --arg module "${module_version}" \ + if ! jq -e --arg package "${package_version}" --arg firmware "${firmware_version}" --arg module_version "${module_version}" \ '.schema_version == 1 and .package_version == $package and .firmware_package_version == $firmware and - .module_version == $module and (.kernel_version | type == "string" and length > 0)' "${marker}" >/dev/null; then + .module_version == $module_version and (.kernel_version | type == "string" and length > 0)' "${marker}" >/dev/null; then err "${test}" "Missing or inconsistent AMDGPU driver marker" return 1 fi From 3ce6d1735a874b87c744b078e23b25212749222d Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 19 Sep 2026 08:33:10 +0000 Subject: [PATCH 3/7] refactor(amd-gpu): isolate image build and preserve default paths --- .pipelines/.vsts-vhd-builder-amd.yaml | 81 ++++++ .pipelines/.vsts-vhd-builder-release.yaml | 29 -- .pipelines/.vsts-vhd-builder.yaml | 23 -- .../templates/.builder-release-template.yaml | 29 +- docs/amd-gpu-vhd.md | 27 +- e2e/config/vhd.go | 11 - e2e/config/vhd_amd.go | 16 ++ e2e/scenario/node_config.go | 1 - e2e/scenario/provision.go | 2 +- e2e/scenario/scenario_gpu_amd.go | 28 +- e2e/scenario/scenario_gpu_amd_test.go | 42 +-- e2e/scenario/types.go | 6 +- e2e/scenario/vmss.go | 26 +- e2e/scenario/vmss_test.go | 49 +--- parts/common/components.json | 23 -- .../linux/cloud-init/artifacts/cse_amd_gpu.sh | 66 +++++ .../linux/cloud-init/artifacts/cse_install.sh | 65 ----- parts/linux/cloud-init/artifacts/cse_main.sh | 1 + schemas/amd-gpu-components.cue | 33 +++ schemas/components.cue | 29 -- .../artifacts/cse_install_amd_gpu_spec.sh | 20 +- .../packer/amd_gpu_diagnostics_spec.sh | 14 +- spec/vhdbuilder/packer/amd_gpu_driver_spec.sh | 134 ++++++++- .../packer/amd_gpu_pipeline_spec.sh | 194 +++++++++++++ spec/vhdbuilder/packer/linux_sku_name_spec.sh | 24 +- .../packer/test/amd_gpu_content_spec.sh | 122 +++++++-- vhdbuilder/packer/amd-gpu-build-settings.sh | 60 +++++ vhdbuilder/packer/amd-gpu-components.json | 25 ++ vhdbuilder/packer/amd-gpu-packer-files.json | 17 ++ vhdbuilder/packer/cleanup-amd-gpu-build.sh | 45 ++++ vhdbuilder/packer/install-dependencies.sh | 37 +-- .../produce-packer-settings-functions.sh | 42 --- vhdbuilder/packer/produce-packer-settings.sh | 7 +- .../packer/test/amd-gpu-vhd-content-test.sh | 153 +++++++++++ .../packer/test/linux-vhd-content-test.sh | 144 +--------- vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh | 254 ++++++++++++++++++ .../linux/ubuntu/tool_installs_ubuntu.sh | 222 --------------- 37 files changed, 1344 insertions(+), 757 deletions(-) create mode 100644 .pipelines/.vsts-vhd-builder-amd.yaml create mode 100644 e2e/config/vhd_amd.go create mode 100644 parts/linux/cloud-init/artifacts/cse_amd_gpu.sh create mode 100644 schemas/amd-gpu-components.cue create mode 100644 spec/vhdbuilder/packer/amd_gpu_pipeline_spec.sh create mode 100644 vhdbuilder/packer/amd-gpu-build-settings.sh create mode 100644 vhdbuilder/packer/amd-gpu-components.json create mode 100644 vhdbuilder/packer/amd-gpu-packer-files.json create mode 100644 vhdbuilder/packer/cleanup-amd-gpu-build.sh create mode 100644 vhdbuilder/packer/test/amd-gpu-vhd-content-test.sh create mode 100644 vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh diff --git a/.pipelines/.vsts-vhd-builder-amd.yaml b/.pipelines/.vsts-vhd-builder-amd.yaml new file mode 100644 index 00000000000..baefd017d8c --- /dev/null +++ b/.pipelines/.vsts-vhd-builder-amd.yaml @@ -0,0 +1,81 @@ +# Register this manual pipeline with the existing nonproduction Linux VHD +# builder's pool, service connection and build-environment variables. +# Its single image job is independent of the shared PR and release pipelines. +name: $(Date:yyyyMMdd)$(Rev:.r)_AMD_$(BuildID) +trigger: none +pr: none + +parameters: + - name: replicationRegions + displayName: Capture replication targets (region=replicas, comma separated) + type: string + default: francecentral=1 + +pool: + name: $(POOL_NAME) + +variables: + - name: MODE + value: linuxVhdMode + - group: aks-vuln-to-kusto + - group: "AKS Node SIG UA Token (KV)" + - group: build_performance + - group: aks-vuln-to-kusto-tme + - name: BUILDER + value: packer + # Capture a test image and retain it for explicit MI300X qualification. + # Publishing/export and shared-gallery garbage collection are not part of this pipeline. + - name: GENERATE_PUBLISHING_INFO + value: 'False' + - name: RUN_PREFETCH_OPTIMIZATION + value: 'False' + - name: DRY_RUN + value: 'False' + - name: SKIP_CLEANUP + # Shared cleanup includes subscription-wide garbage collection. The final + # step below removes only this build's recorded and ownership-verified group. + value: 'True' + - name: REPLICATIONS + value: ${{ parameters.replicationRegions }} + +stages: + - stage: build_amd + displayName: Build dedicated AMD GPU VHD + jobs: + - job: build2404amdgpugen2containerd + timeoutInMinutes: 360 + steps: + - bash: | + echo '##vso[task.setvariable variable=OS_SKU]Ubuntu' + echo '##vso[task.setvariable variable=OS_VERSION]24.04' + echo '##vso[task.setvariable variable=IMG_PUBLISHER]Canonical' + echo '##vso[task.setvariable variable=IMG_OFFER]ubuntu-24_04-lts' + echo '##vso[task.setvariable variable=IMG_SKU]server' + echo '##vso[task.setvariable variable=IMG_VERSION]latest' + echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' + echo '##vso[task.setvariable variable=FEATURE_FLAGS]AMD_GPU' + echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' + echo '##vso[task.setvariable variable=ENABLE_FIPS]False' + echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' + echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]False' + displayName: Setup AMD Build Variables + - template: ./templates/.builder-release-template.yaml + parameters: + artifactName: 2404-amdgpu-gen2-containerd + - task: AzureCLI@2 + displayName: Clean up this AMD build's Packer resource group + condition: always() + inputs: + azureSubscription: $(VHD_ARM_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -euo pipefail + source vhdbuilder/packer/cleanup-amd-gpu-build.sh + cleanup_amd_gpu_build_resource_group \ + vhdbuilder/packer/settings.json \ + "${PACKER_RESOURCE_GROUP}" "${AMD_BUILD_ID}" + env: + PACKER_RESOURCE_GROUP: $(PKR_RG_NAME) + AMD_BUILD_ID: $(Build.BuildId) diff --git a/.pipelines/.vsts-vhd-builder-release.yaml b/.pipelines/.vsts-vhd-builder-release.yaml index 1e66877e542..a6e1c33f959 100644 --- a/.pipelines/.vsts-vhd-builder-release.yaml +++ b/.pipelines/.vsts-vhd-builder-release.yaml @@ -105,10 +105,6 @@ parameters: displayName: Build 2404 Gen2 Containerd type: boolean default: true - - name: build2404amdgpugen2containerd - displayName: Build 2404 AMD GPU Gen2 Containerd (preview) - type: boolean - default: false - name: build2404tlgen2containerd displayName: Build 2404 TL Gen2 Containerd type: boolean @@ -721,31 +717,6 @@ stages: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: 2404-gen2-containerd - - job: build2404amdgpugen2containerd - condition: eq('${{ parameters.build2404amdgpugen2containerd }}', true) - dependsOn: [ ] - timeoutInMinutes: 360 - steps: - - bash: | - echo '##vso[task.setvariable variable=OS_SKU]Ubuntu' - echo '##vso[task.setvariable variable=OS_VERSION]24.04' - echo '##vso[task.setvariable variable=IMG_PUBLISHER]Canonical' - echo '##vso[task.setvariable variable=IMG_OFFER]ubuntu-24_04-lts' - echo '##vso[task.setvariable variable=IMG_SKU]server' - echo '##vso[task.setvariable variable=IMG_VERSION]latest' - echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' - echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' - echo '##vso[task.setvariable variable=FEATURE_FLAGS]AMD_GPU' - echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' - echo '##vso[task.setvariable variable=ENABLE_FIPS]False' - echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' - echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]False' - displayName: Setup Build Variables - - template: ./templates/.builder-release-template.yaml - parameters: - useOverrides: ${{ parameters.useOverrides }} - overrideBranch: ${{ parameters.overrideBranch }} - artifactName: 2404-amdgpu-gen2-containerd - job: build2404tlgen2containerd condition: eq('${{ parameters.build2404tlgen2containerd }}', true) dependsOn: [ ] diff --git a/.pipelines/.vsts-vhd-builder.yaml b/.pipelines/.vsts-vhd-builder.yaml index 5e1e2e14c7d..a7e955095e3 100644 --- a/.pipelines/.vsts-vhd-builder.yaml +++ b/.pipelines/.vsts-vhd-builder.yaml @@ -144,29 +144,6 @@ stages: parameters: artifactName: 2404-gen2-containerd - - job: build2404amdgpugen2containerd - timeoutInMinutes: 360 - steps: - - bash: | - echo '##vso[task.setvariable variable=OS_SKU]Ubuntu' - echo '##vso[task.setvariable variable=OS_VERSION]24.04' - echo '##vso[task.setvariable variable=IMG_PUBLISHER]Canonical' - echo '##vso[task.setvariable variable=IMG_OFFER]ubuntu-24_04-lts' - echo '##vso[task.setvariable variable=IMG_SKU]server' - echo '##vso[task.setvariable variable=IMG_VERSION]latest' - echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' - # Build the kernel module on a CPU VM; MI300X validation uses the captured image. - echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' - echo '##vso[task.setvariable variable=FEATURE_FLAGS]AMD_GPU' - echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' - echo '##vso[task.setvariable variable=ENABLE_FIPS]False' - echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' - echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]False' - displayName: Setup Build Variables - - template: ./templates/.builder-release-template.yaml - parameters: - artifactName: 2404-amdgpu-gen2-containerd - - job: build2604minimalgen2containerd timeoutInMinutes: 360 steps: diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 40465d516da..16fd5a21c23 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -74,11 +74,30 @@ steps: displayName: Setup Cue - bash: | - set -e - source vhdbuilder/packer/produce-packer-settings-functions.sh - SKU_NAME=$(get_linux_sku_name) - echo "##vso[task.setvariable variable=SKU_NAME]$SKU_NAME" - echo "Set SKU_NAME to $SKU_NAME" + if [[ "${FEATURE_FLAGS}" == *AMD_GPU* ]]; then + "$(go env GOPATH)/bin/cue" vet -c ./schemas/amd-gpu-components.cue ./vhdbuilder/packer/amd-gpu-components.json || exit 1 + source vhdbuilder/packer/amd-gpu-build-settings.sh + SKU_NAME=$(get_amd_gpu_sku_name) || exit 1 + # Prepare only this AMD worker checkout after the template's checkout step. + prepare_amd_gpu_packer_template \ + vhdbuilder/packer/vhd-image-builder-base.json \ + vhdbuilder/packer/amd-gpu-packer-files.json || exit 1 + echo "##vso[task.setvariable variable=SKU_NAME]$SKU_NAME" + echo "Set SKU_NAME to $SKU_NAME" + else + SKU_NAME=${OS_VERSION} && \ + if grep -q "minimal" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}minimal"; fi && \ + if [[ "${HYPERV_GENERATION,,}" == "v2" ]]; then SKU_NAME="${SKU_NAME}gen2"; fi && \ + if [[ "${ARCHITECTURE,,}" == "arm64" ]]; then SKU_NAME="${SKU_NAME}arm64"; fi && \ + if grep -q "NVIDIA_GB" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}gb"; fi && \ + if [[ "${ENABLE_FIPS,,}" == "true" ]]; then SKU_NAME="${SKU_NAME}fips"; fi && \ + if grep -q "cvm" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}CVM"; fi && \ + if [[ "${ENABLE_TRUSTED_LAUNCH}" == "True" ]]; then SKU_NAME="${SKU_NAME}TL"; fi && \ + if [[ ${OS_SKU} != "CBLMariner" && ${OS_SKU} != "AzureLinux" && ${OS_SKU} != "AzureLinuxOSGuard" && ${OS_SKU} != "Flatcar" && ${OS_SKU} != "AzureContainerLinux" ]]; then SKU_NAME="${SKU_NAME}containerd"; fi && \ + SKU_NAME=$(echo ${SKU_NAME} | tr -d '.') && \ + echo "##vso[task.setvariable variable=SKU_NAME]$SKU_NAME" + echo "Set SKU_NAME to $SKU_NAME" + fi displayName: Set SKU Name - task: AzureCLI@2 diff --git a/docs/amd-gpu-vhd.md b/docs/amd-gpu-vhd.md index 04e06a7481b..89db8333524 100644 --- a/docs/amd-gpu-vhd.md +++ b/docs/amd-gpu-vhd.md @@ -11,7 +11,8 @@ VMs, ARM64, and combinations with NVIDIA image flags are rejected. The host contains the official AMDGPU DKMS driver, firmware, AMD SMI diagnostics, matching kernel headers, and build prerequisites. Exact AMD package versions, module version, repository, and signing-key fingerprint live in -`parts/common/components.json`. +`vhdbuilder/packer/amd-gpu-components.json`, validated separately by +`schemas/amd-gpu-components.cue`. The ordinary component manifest is unchanged. The build authenticates AMD repository metadata, installs only the selected driver, firmware, SMI and SMI sysdeps packages, builds the module for the image kernel, and removes its temporary repository and package-download state. It writes @@ -94,10 +95,19 @@ the loaded module version, `/dev/kfd`, and eight active KFD GPUs. Preallocated DRM render nodes are not counted as GPUs. Missing or incompatible drivers fail provisioning; there is no vendor download or compile fallback on node startup. -Build this flavor through the AMD job in `.pipelines/.vsts-vhd-builder.yaml`, -or enable `build2404amdgpugen2containerd` in the release pipeline. The release -parameter defaults to false. Existing CPU/NVIDIA image identities and -production image-selection defaults remain unchanged. +Build this flavor with the separate manual pipeline +`.pipelines/.vsts-vhd-builder-amd.yaml` (`trigger: none`, `pr: none`). Register it +with the existing nonproduction VHD builder pool, service connection, and +build-environment variables. It captures only `2404gen2amdgpucontainerd` and +replicates to France Central by default; it does not publish a production image. +The normal PR and release build matrices are unchanged and have no dependency +on the AMD pipeline. A failed AMD bake fails only that separate run. + +AMD installers, package metadata, content checks, and bootstrap validation live +in AMD-specific files. Shared scripts load them only behind `AMD_GPU` or +`AMD_GPU_NODE` guards. Node provisioning sources the validator baked into this +image, so normal VHDs do not execute or depend on AMD package installation. +Existing CPU/NVIDIA SKU naming and image-selection defaults remain unchanged. Use the captured AMD gallery image explicitly for testing. Its existing Ubuntu 24.04 Gen2 `Distro` describes OS behavior; its distinct gallery image name @@ -149,14 +159,19 @@ repository's normal E2E Azure configuration and its captured build metadata: cd e2e AGENTBAKER_E2E_ENABLE_MI300X=true ./e2e-local.sh \ --subscription-id "$SUBSCRIPTION_ID" \ + --vm-sku Standard_ND96isr_MI300X_v5 \ --vhd-metadata-file /absolute/path/to/vhd-build-metadata.json \ --parallel 1 --disable-scriptless \ + --ignore-missing-vhd=false --skip-capacity-errors=false \ + --tags '' --skip-tags '' \ Ubuntu2404_MI300X_AMDGPU ``` The metadata must contain `2404gen2amdgpucontainerd`, its real image-version resource ID, and the replication region. This scenario allocates MI300X -capacity and is skipped unless explicitly enabled. It validates the driver, +capacity and is skipped unless explicitly enabled. The existing `--vm-sku` +option sets the tested node size before capability queries; the scenario keeps +the shared AKS system pool on the ordinary CPU SKU. It validates the driver, deploys the digest-pinned AMD device plugin, requires eight advertised GPUs, and requires completed reference-checked training. A managed 256 GiB OS disk provides room to unpack the development workload image. The scenario does not diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index 217be73ec74..dd44c1e13be 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -154,17 +154,6 @@ var ( Gallery: &Config.GalleryLinux, } - // The dedicated AMD image is selected only by opt-in GPU scenarios. It uses - // the existing Ubuntu distro and does not change production SIG defaults. - VHDUbuntu2404Gen2AMDGPUContainerd = &Image{ - Name: "2404gen2amdgpucontainerd", - OS: OSUbuntu, - Arch: "amd64", - Distro: datamodel.AKSUbuntuContainerd2404Gen2, - Gallery: &Config.GalleryLinux, - OSDiskSizeGB: 256, // The on-demand PyTorch image is 20.46 GB compressed. - } - VHDUbuntu2404ArmContainerd = &Image{ Name: "2404gen2arm64containerd", OS: OSUbuntu, diff --git a/e2e/config/vhd_amd.go b/e2e/config/vhd_amd.go new file mode 100644 index 00000000000..bf4459cdc32 --- /dev/null +++ b/e2e/config/vhd_amd.go @@ -0,0 +1,16 @@ +package config + +import "github.com/Azure/agentbaker/pkg/agent/datamodel" + +var ( + // The dedicated AMD image is selected only by opt-in GPU scenarios. It uses + // the existing Ubuntu distro and does not change production SIG defaults. + VHDUbuntu2404Gen2AMDGPUContainerd = &Image{ + Name: "2404gen2amdgpucontainerd", + OS: OSUbuntu, + Arch: "amd64", + Distro: datamodel.AKSUbuntuContainerd2404Gen2, + Gallery: &Config.GalleryLinux, + OSDiskSizeGB: 256, // The on-demand PyTorch image is 20.46 GB compressed. + } +) diff --git a/e2e/scenario/node_config.go b/e2e/scenario/node_config.go index d897a905a8a..f8f53457056 100644 --- a/e2e/scenario/node_config.go +++ b/e2e/scenario/node_config.go @@ -282,7 +282,6 @@ func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) (*aksno VnetCniPluginsUrl: vnetCNIPluginURL, }, GpuConfig: &aksnodeconfigv1.GpuConfig{ - EnableAmdGpu: to.Ptr(nbc.EnableAMDGPU), ConfigGpuDriver: true, GpuDevicePlugin: false, GpuInstanceProfile: nbc.GPUInstanceProfile, diff --git a/e2e/scenario/provision.go b/e2e/scenario/provision.go index 35e5ccab109..6620562fc32 100644 --- a/e2e/scenario/provision.go +++ b/e2e/scenario/provision.go @@ -239,7 +239,7 @@ func runScenario(ctx context.Context, scenarioName string, s *Scenario) (runErr s.Runtime = &ScenarioRuntime{} } s.Runtime.Cluster = cluster - s.Runtime.VMSize = scenarioVMSize(s) + s.Runtime.VMSize = config.Config.VMSKU() s.Runtime.VMSSName = generateVMSSName(s) testKube, err := cluster.NewKubeclientForTest() diff --git a/e2e/scenario/scenario_gpu_amd.go b/e2e/scenario/scenario_gpu_amd.go index c11ecd72ce8..1c55261f53f 100644 --- a/e2e/scenario/scenario_gpu_amd.go +++ b/e2e/scenario/scenario_gpu_amd.go @@ -43,10 +43,11 @@ var _ = Register(newUbuntu2404MI300XAMDGPUScenario()) func newUbuntu2404MI300XAMDGPUScenario() *Scenario { return &Scenario{ - Name: "Ubuntu2404_MI300X_AMDGPU", - Description: "Opt-in Ubuntu 24.04 MI300X image with AMDGPU and AMD SMI: eight AMD GPUs, upstream device plugin, and containerized FP32 training checked against a CPU reference", - Tags: Tags{GPU: true}, - Location: "francecentral", + Name: "Ubuntu2404_MI300X_AMDGPU", + Description: "Opt-in Ubuntu 24.04 MI300X image with AMDGPU and AMD SMI: eight AMD GPUs, upstream device plugin, and containerized FP32 training checked against a CPU reference", + Tags: Tags{GPU: true}, + Location: "francecentral", + K8sSystemPoolSKU: config.DEFAULT_VMSKU, SkipIf: func(context.Context) string { if os.Getenv(amdMI300XOptIn) != "true" { return "set " + amdMI300XOptIn + "=true after publishing the dedicated VHD and confirming MI300X quota" @@ -54,9 +55,9 @@ func newUbuntu2404MI300XAMDGPUScenario() *Scenario { return "" }, Config: Config{ - Cluster: ClusterKubenet, - VHD: config.VHDUbuntu2404Gen2AMDGPUContainerd, - VMSize: amdMI300XVMSize, + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2404Gen2AMDGPUContainerd, + SkipNVMeOSDiskPlacement: true, BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.AgentPoolProfile.VMSize = amdMI300XVMSize nbc.EnableAMDGPU = true @@ -64,8 +65,15 @@ func newUbuntu2404MI300XAMDGPUScenario() *Scenario { nbc.ConfigGPUDriverIfNeeded = true nbc.EnableGPUDevicePluginIfNeeded = false }, + BootstrapConfigMutatorWithError: func(context.Context, *Cluster, *datamodel.NodeBootstrappingConfiguration) error { + if config.Config.VMSKU() != amdMI300XVMSize { + return fmt.Errorf("AMD scenario requires --vm-sku %s", amdMI300XVMSize) + } + return nil + }, AKSNodeConfigMutator: func(_ *Cluster, cfg *aksnodeconfigv1.Configuration) { cfg.VmSize = amdMI300XVMSize + cfg.GpuConfig.EnableAmdGpu = to.Ptr(true) }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.SKU.Name = to.Ptr(amdMI300XVMSize) @@ -101,7 +109,7 @@ func validateAMDMI300X(ctx context.Context, s *Scenario) error { } func validateAMDGPUHost(ctx context.Context, s *Scenario) error { - contents, err := os.ReadFile(repoPath("parts/common/components.json")) + contents, err := os.ReadFile(repoPath("vhdbuilder/packer/amd-gpu-components.json")) if err != nil { return err } @@ -126,12 +134,12 @@ func amdGPUHostCheckCommand(contents []byte) (string, error) { } for _, key := range []string{"packageVersion", "firmwarePackageVersion", "moduleVersion", "dkmsVersion"} { if manifest.AMDGPUDriver[key] == "" { - return "", fmt.Errorf("AMDGPUDriver.%s missing from components.json", key) + return "", fmt.Errorf("AMDGPUDriver.%s missing from amd-gpu-components.json", key) } } for _, key := range []string{"amdsmiPackage", "amdsmiVersion", "sysdepsPackage", "sysdepsVersion", "cliPath"} { if manifest.AMDGPUDiagnostics[key] == "" { - return "", fmt.Errorf("AMDGPUDiagnostics.%s missing from components.json", key) + return "", fmt.Errorf("AMDGPUDiagnostics.%s missing from amd-gpu-components.json", key) } } encoded, err := json.Marshal(manifest) diff --git a/e2e/scenario/scenario_gpu_amd_test.go b/e2e/scenario/scenario_gpu_amd_test.go index 1d6254cd788..022b7d4f886 100644 --- a/e2e/scenario/scenario_gpu_amd_test.go +++ b/e2e/scenario/scenario_gpu_amd_test.go @@ -20,7 +20,7 @@ import ( ) func TestAMDGPUHostValidationRequiresPinnedDiagnostics(t *testing.T) { - contents, err := os.ReadFile(repoPath("parts/common/components.json")) + contents, err := os.ReadFile(repoPath("vhdbuilder/packer/amd-gpu-components.json")) require.NoError(t, err) script, err := amdGPUHostCheckCommand(contents) require.NoError(t, err) @@ -68,8 +68,8 @@ func TestAMDMI300XScenarioIsOptInAndUsesDedicatedImage(t *testing.T) { t.Setenv(amdMI300XOptIn, "true") require.Empty(t, s.SkipIf(t.Context())) require.True(t, s.Tags.GPU) - require.Equal(t, amdMI300XVMSize, s.Config.VMSize) - require.Equal(t, amdMI300XVMSize, scenarioVMSize(s), "capability queries must use MI300X before bootstrap or VMSS mutation") + require.Equal(t, config.DEFAULT_VMSKU, s.K8sSystemPoolSKU, "the shared AKS system pool must stay on a CPU SKU") + require.True(t, s.SkipNVMeOSDiskPlacement, "managed OS disks have no ephemeral NVMe placement") require.Equal(t, "2404gen2amdgpucontainerd", s.VHD.Name) require.Equal(t, datamodel.AKSUbuntuContainerd2404Gen2, s.VHD.Distro) require.Equal(t, "2404gen2containerd", config.VHDUbuntu2404Gen2Containerd.Name, "generic image selection must stay unchanged") @@ -96,9 +96,10 @@ func TestAMDMI300XScenarioIsOptInAndUsesDedicatedImage(t *testing.T) { } s.VMConfigMutator(vmss) require.Equal(t, amdMI300XVMSize, *vmss.SKU.Name) - anc := &aksnodeconfigv1.Configuration{} + anc := &aksnodeconfigv1.Configuration{GpuConfig: &aksnodeconfigv1.GpuConfig{}} s.AKSNodeConfigMutator(nil, anc) require.Equal(t, amdMI300XVMSize, anc.VmSize) + require.True(t, anc.GpuConfig.GetEnableAmdGpu()) } func TestAMDMI300XScenarioUsesManagedOSDisk(t *testing.T) { @@ -138,7 +139,7 @@ func TestAMDMI300XScenarioUsesManagedOSDisk(t *testing.T) { }, } s.VMConfigMutator(vmss) - require.NoError(t, configureNVMeOSDiskPlacement(vmss), "a managed OS disk remains valid if the SKU reports NVMe support") + require.True(t, s.SkipNVMeOSDiskPlacement, "only this scenario opts out of ephemeral NVMe placement") require.Nil(t, osDisk.DiffDiskSettings, "the 256 GiB image must not depend on ephemeral ResourceDisk capacity") require.NotNil(t, osDisk.ManagedDisk) require.Equal(t, armcompute.StorageAccountTypesPremiumLRS, *osDisk.ManagedDisk.StorageAccountType) @@ -153,17 +154,15 @@ func TestAMDMI300XScenarioUsesManagedOSDisk(t *testing.T) { } } -func TestAMDGPUBootstrapFlagReachesANC(t *testing.T) { - for _, enabled := range []bool{false, true} { - nbc, err := baseTemplateLinux("francecentral", "1.34.0", "amd64") - require.NoError(t, err) - nbc.EnableAMDGPU = enabled - anc, err := nbcToAKSNodeConfigV1(nbc) - require.NoError(t, err) - require.NotNil(t, anc.GpuConfig.EnableAmdGpu) - require.Equal(t, enabled, anc.GpuConfig.GetEnableAmdGpu()) - require.True(t, anc.GpuConfig.ConfigGpuDriver) - } +func TestAMDScenarioEnablesANCFlag(t *testing.T) { + nbc, err := baseTemplateLinux("francecentral", "1.34.0", "amd64") + require.NoError(t, err) + anc, err := nbcToAKSNodeConfigV1(nbc) + require.NoError(t, err) + require.Nil(t, anc.GpuConfig.EnableAmdGpu, "default conversion stays unchanged") + newUbuntu2404MI300XAMDGPUScenario().AKSNodeConfigMutator(nil, anc) + require.True(t, anc.GpuConfig.GetEnableAmdGpu()) + require.True(t, anc.GpuConfig.ConfigGpuDriver) } func TestAMDGPUManifestsUseAssignedDevicesAndPinnedUserspace(t *testing.T) { @@ -262,3 +261,14 @@ func TestAMDGPUTrainingRequiresSuccessfulCompletion(t *testing.T) { require.False(t, done) require.ErrorContains(t, err, "reference mismatch") } + +func TestAMDMI300XRequiresExplicitRunnerSKU(t *testing.T) { + original := config.Config.DefaultVMSKU + t.Cleanup(func() { config.Config.DefaultVMSKU = original }) + s := newUbuntu2404MI300XAMDGPUScenario() + config.Config.DefaultVMSKU = config.DEFAULT_VMSKU + require.ErrorContains(t, s.BootstrapConfigMutatorWithError(t.Context(), nil, nil), "--vm-sku") + config.Config.DefaultVMSKU = amdMI300XVMSize + require.NoError(t, s.BootstrapConfigMutatorWithError(t.Context(), nil, nil)) + require.Equal(t, amdMI300XVMSize, scenarioVMSize(s), "existing runner SKU selection handles AMD before capability queries") +} diff --git a/e2e/scenario/types.go b/e2e/scenario/types.go index e8afc07a0be..ea1163cef7e 100644 --- a/e2e/scenario/types.go +++ b/e2e/scenario/types.go @@ -141,9 +141,6 @@ type Config struct { // VHD is the node image used by the scenario. VHD *config.Image - // VMSize overrides the default VM size before capability queries and VMSS model creation. - VMSize string - // BootstrapConfigMutator is a function which mutates the base NodeBootstrappingConfig according to the scenario's requirements BootstrapConfigMutator func(*Cluster, *datamodel.NodeBootstrappingConfiguration) @@ -202,6 +199,9 @@ type Config struct { // UseNVMe indicates whether to use NVMe-based disk placement/controller. This is required for certain VM sizes (e.g., v6 and v7 series) which only support NVMe disk controllers. UseNVMe bool + // SkipNVMeOSDiskPlacement leaves placement to scenarios using a managed OS disk. + SkipNVMeOSDiskPlacement bool + // EagerCSETimingExtraction when true causes CSE timing events to be extracted // immediately after SSH is established, before other validators run. // This prevents the Guest Agent from sweeping events before they can be read. diff --git a/e2e/scenario/vmss.go b/e2e/scenario/vmss.go index a379e8c65f2..a2b79ca0565 100644 --- a/e2e/scenario/vmss.go +++ b/e2e/scenario/vmss.go @@ -487,27 +487,18 @@ func createVMSSModel(ctx context.Context, s *Scenario) (armcompute.VirtualMachin return armcompute.VirtualMachineScaleSet{}, err } - if s.Config.UseNVMe { - if err := configureNVMeOSDiskPlacement(&model); err != nil { - return armcompute.VirtualMachineScaleSet{}, err + if s.Config.UseNVMe && !s.Config.SkipNVMeOSDiskPlacement { + if model.Properties == nil || model.Properties.VirtualMachineProfile == nil || + model.Properties.VirtualMachineProfile.StorageProfile == nil || + model.Properties.VirtualMachineProfile.StorageProfile.OSDisk == nil || + model.Properties.VirtualMachineProfile.StorageProfile.OSDisk.DiffDiskSettings == nil { + return armcompute.VirtualMachineScaleSet{}, fmt.Errorf("VMSS model is missing diff disk settings required for NVMe placement") } + model.Properties.VirtualMachineProfile.StorageProfile.OSDisk.DiffDiskSettings.Placement = to.Ptr(armcompute.DiffDiskPlacementNvmeDisk) } return model, nil } -func configureNVMeOSDiskPlacement(model *armcompute.VirtualMachineScaleSet) error { - if model == nil || model.Properties == nil || model.Properties.VirtualMachineProfile == nil || - model.Properties.VirtualMachineProfile.StorageProfile == nil || - model.Properties.VirtualMachineProfile.StorageProfile.OSDisk == nil { - return fmt.Errorf("VMSS model is missing OS disk settings required for NVMe placement") - } - // Managed OS disks have no ephemeral placement, even on a SKU supporting NVMe. - if diffDisk := model.Properties.VirtualMachineProfile.StorageProfile.OSDisk.DiffDiskSettings; diffDisk != nil { - diffDisk.Placement = to.Ptr(armcompute.DiffDiskPlacementNvmeDisk) - } - return nil -} - func usesScriptlessNBCCSECmd(s *Scenario) bool { if s == nil || s.Runtime == nil || s.Runtime.NBC == nil || s.VHD == nil { return false @@ -1679,9 +1670,6 @@ func scenarioVMSize(s *Scenario) string { if s.Runtime != nil && s.Runtime.VMSize != "" { return s.Runtime.VMSize } - if s.Config.VMSize != "" { - return s.Config.VMSize - } return config.Config.VMSKU() } diff --git a/e2e/scenario/vmss_test.go b/e2e/scenario/vmss_test.go index d0c5f8551c1..e11d4092797 100644 --- a/e2e/scenario/vmss_test.go +++ b/e2e/scenario/vmss_test.go @@ -395,53 +395,10 @@ func TestWriteScriptHotfixFixtureRejectsInvalidData(t *testing.T) { // ERR_OUTBOUND_CONN_FAIL in parts/linux/cloud-init/artifacts/cse_helpers.sh. If the // product error code changes, this test forces the harness mitigation to be updated. func TestGetBaseVMSSModelUsesScenarioVMSize(t *testing.T) { - for _, test := range []struct { - name string - s *Scenario - want string - }{ - {name: "default", s: &Scenario{}, want: config.Config.VMSKU()}, - {name: "configured before runtime initialization", s: &Scenario{Config: Config{VMSize: "Standard_ND96isr_MI300X_v5"}}, want: "Standard_ND96isr_MI300X_v5"}, - {name: "runtime compatibility fallback wins", s: &Scenario{Config: Config{VMSize: "Standard_ND96isr_MI300X_v5"}, Runtime: &ScenarioRuntime{VMSize: config.DEFAULT_VMSKU}}, want: config.DEFAULT_VMSKU}, - } { - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.want, scenarioVMSize(test.s)) - }) - } -} - -func TestConfigureNVMeOSDiskPlacement(t *testing.T) { - for _, managed := range []bool{false, true} { - t.Run(fmt.Sprintf("managed=%t", managed), func(t *testing.T) { - osDisk := &armcompute.VirtualMachineScaleSetOSDisk{DiskSizeGB: to.Ptr[int32](256)} - if managed { - osDisk.ManagedDisk = &armcompute.VirtualMachineScaleSetManagedDiskParameters{ - StorageAccountType: to.Ptr(armcompute.StorageAccountTypesPremiumLRS), - } - } else { - osDisk.DiffDiskSettings = &armcompute.DiffDiskSettings{ - Option: to.Ptr(armcompute.DiffDiskOptionsLocal), - Placement: to.Ptr(armcompute.DiffDiskPlacementResourceDisk), - } - } - vmss := &armcompute.VirtualMachineScaleSet{Properties: &armcompute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &armcompute.VirtualMachineScaleSetVMProfile{ - StorageProfile: &armcompute.VirtualMachineScaleSetStorageProfile{OSDisk: osDisk}, - }, - }} - require.NoError(t, configureNVMeOSDiskPlacement(vmss)) - require.Equal(t, int32(256), *osDisk.DiskSizeGB) - if managed { - require.Nil(t, osDisk.DiffDiskSettings) - require.Equal(t, armcompute.StorageAccountTypesPremiumLRS, *osDisk.ManagedDisk.StorageAccountType) - } else { - require.Equal(t, armcompute.DiffDiskPlacementNvmeDisk, *osDisk.DiffDiskSettings.Placement) - require.Equal(t, armcompute.DiffDiskOptionsLocal, *osDisk.DiffDiskSettings.Option) - } - }) + s := &Scenario{ + Runtime: &ScenarioRuntime{VMSize: config.DEFAULT_VMSKU}, } - require.Error(t, configureNVMeOSDiskPlacement(nil)) - require.Error(t, configureNVMeOSDiskPlacement(&armcompute.VirtualMachineScaleSet{})) + assert.Equal(t, s.Runtime.VMSize, scenarioVMSize(s)) } func TestCSEExitCodeOutboundConnFail(t *testing.T) { diff --git a/parts/common/components.json b/parts/common/components.json index 187d6ab2dbb..930f6d4cd8e 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -1,27 +1,4 @@ { - "AMDGPUDriver": { - "repositoryURL": "https://repo.radeon.com/amdgpu/31.50/ubuntu", - "signingKeyURL": "https://repo.radeon.com/rocm/rocm.gpg.key", - "signingKeyFingerprint": "CA8BB4727A47B4D09B4EE8969386B48A1A693C5C", - "distribution": "noble", - "component": "main", - "packageVersion": "1:7.1.3.31500000-2390945.24.04", - "firmwarePackageVersion": "1:31.50.0.0.31500000-2390945.24.04", - "moduleVersion": "7.1.3.31500000", - "dkmsVersion": "7.1.3-2390945.24.04" - }, - "AMDGPUDiagnostics": { - "repositoryURL": "https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/", - "signingKeyURL": "https://stable.repo.amd.com/rocm/gpg/packages.gpg", - "signingKeyFingerprint": "D0F004A0025A1145C7807FCD0701EAC4D5E02107", - "distribution": "stable", - "component": "main", - "amdsmiPackage": "amdrocm-amdsmi10.0", - "amdsmiVersion": "10.0.0-4", - "sysdepsPackage": "amdrocm-sysdeps10.0", - "sysdepsVersion": "10.0.0-4", - "cliPath": "/opt/rocm/core-10.0/bin/amd-smi" - }, "ContainerImages": [ { "downloadURL": "mcr.microsoft.com/windows/servercore:*", diff --git a/parts/linux/cloud-init/artifacts/cse_amd_gpu.sh b/parts/linux/cloud-init/artifacts/cse_amd_gpu.sh new file mode 100644 index 00000000000..57d1a0d5aa4 --- /dev/null +++ b/parts/linux/cloud-init/artifacts/cse_amd_gpu.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Validate the dedicated AMD image on each real node, including PIS nodes. There +# is deliberately no apt/DKMS installation fallback during node provisioning. +ensureAmdGpuDrivers() { + local vm_sku + vm_sku=$(get_compute_sku) || return $ERR_AMD_GPU_VALIDATE_FAIL + if [ "${OS}" != "${UBUNTU_OS_NAME}" ] || [ "${OS_VERSION}" != "24.04" ] || [ "$(uname -m)" != "x86_64" ] || [ "${GPU_NODE:-false}" = "true" ]; then + echo "AMD GPU requires the dedicated Ubuntu 24.04 amd64 image and exclusive AMD configuration" + return $ERR_AMD_GPU_UNSUPPORTED + fi + case "${vm_sku,,}" in + standard_nd96isr_mi300x_v5|standard_nd96is_mi300x_v5) ;; + *) echo "Unsupported AMD GPU VM SKU: ${vm_sku}"; return $ERR_AMD_GPU_UNSUPPORTED ;; + esac + + if ! validateAmdGpuDriver; then + echo "Baked AMD GPU driver validation failed; use a qualified AMD GPU VHD" + return $ERR_AMD_GPU_VALIDATE_FAIL + fi +} + +validateAmdGpuDriver() { + local marker=/opt/azure/amd-gpu/driver.json + local package_version firmware_version module_version module_path kernel_version loaded_version + [ -s "${marker}" ] || return 1 + jq -e '.schema_version == 1' "${marker}" >/dev/null || return 1 + package_version=$(jq -er '.package_version | strings | select(length > 0)' "${marker}") || return 1 + firmware_version=$(jq -er '.firmware_package_version | strings | select(length > 0)' "${marker}") || return 1 + module_version=$(jq -er '.module_version | strings | select(length > 0)' "${marker}") || return 1 + [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms)" = "install ok installed ${package_version}" ] || return 1 + [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms-firmware)" = "install ok installed ${firmware_version}" ] || return 1 + + kernel_version=$(uname -r) + module_path=$(modinfo -k "${kernel_version}" -F filename amdgpu) || return 1 + # Reject the inbox driver even if modprobe succeeds. DKMS must have built the + # pinned driver for the running kernel, which may differ from the bake kernel. + case "${module_path}" in + /lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*|/usr/lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*) ;; + *) return 1 ;; + esac + [ "$(modinfo -k "${kernel_version}" -F version amdgpu)" = "${module_version}" ] || return 1 + retrycmd_if_failure 12 5 30 modprobe amdgpu || return 1 + loaded_version=$(cat /sys/module/amdgpu/version) || return 1 + [ "${loaded_version}" = "${module_version}" ] || return 1 + retrycmd_if_failure 12 5 5 test -c /dev/kfd || return 1 + export -f validateAmdGpuDevices + retrycmd_if_failure 12 5 5 bash -c validateAmdGpuDevices || return 1 + echo "AMD GPU driver ${module_version} ready on ${kernel_version}; eight GPUs detected" +} + +validateAmdGpuDevices() { + local gpu_id_file gpu_id count=0 + # Count active KFD agents, not DRM render nodes: MI300X VFs can preallocate + # many more render nodes than the eight GPUs assigned to the VM. + for gpu_id_file in /sys/class/kfd/kfd/topology/nodes/*/gpu_id; do + [ -r "${gpu_id_file}" ] || continue + gpu_id=$(cat "${gpu_id_file}") || return 1 + case "${gpu_id}" in + ''|*[!0-9]*) return 1 ;; + 0) continue ;; + esac + count=$((count + 1)) + done + [ "${count}" -eq 8 ] +} diff --git a/parts/linux/cloud-init/artifacts/cse_install.sh b/parts/linux/cloud-init/artifacts/cse_install.sh index 710c508538f..f3331d66864 100755 --- a/parts/linux/cloud-init/artifacts/cse_install.sh +++ b/parts/linux/cloud-init/artifacts/cse_install.sh @@ -886,69 +886,4 @@ EOF systemctl restart nvidia-persistenced.service || exit 1 } -# Validate the dedicated AMD image on each real node, including PIS nodes. There -# is deliberately no apt/DKMS installation fallback during node provisioning. -ensureAmdGpuDrivers() { - local vm_sku - vm_sku=$(get_compute_sku) || return $ERR_AMD_GPU_VALIDATE_FAIL - if [ "${OS}" != "${UBUNTU_OS_NAME}" ] || [ "${OS_VERSION}" != "24.04" ] || [ "$(uname -m)" != "x86_64" ] || [ "${GPU_NODE:-false}" = "true" ]; then - echo "AMD GPU requires the dedicated Ubuntu 24.04 amd64 image and exclusive AMD configuration" - return $ERR_AMD_GPU_UNSUPPORTED - fi - case "${vm_sku,,}" in - standard_nd96isr_mi300x_v5|standard_nd96is_mi300x_v5) ;; - *) echo "Unsupported AMD GPU VM SKU: ${vm_sku}"; return $ERR_AMD_GPU_UNSUPPORTED ;; - esac - - if ! validateAmdGpuDriver; then - echo "Baked AMD GPU driver validation failed; use a qualified AMD GPU VHD" - return $ERR_AMD_GPU_VALIDATE_FAIL - fi -} - -validateAmdGpuDriver() { - local marker=/opt/azure/amd-gpu/driver.json - local package_version firmware_version module_version module_path kernel_version loaded_version - [ -s "${marker}" ] || return 1 - jq -e '.schema_version == 1' "${marker}" >/dev/null || return 1 - package_version=$(jq -er '.package_version | strings | select(length > 0)' "${marker}") || return 1 - firmware_version=$(jq -er '.firmware_package_version | strings | select(length > 0)' "${marker}") || return 1 - module_version=$(jq -er '.module_version | strings | select(length > 0)' "${marker}") || return 1 - [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms)" = "install ok installed ${package_version}" ] || return 1 - [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms-firmware)" = "install ok installed ${firmware_version}" ] || return 1 - - kernel_version=$(uname -r) - module_path=$(modinfo -k "${kernel_version}" -F filename amdgpu) || return 1 - # Reject the inbox driver even if modprobe succeeds. DKMS must have built the - # pinned driver for the running kernel, which may differ from the bake kernel. - case "${module_path}" in - /lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*|/usr/lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*) ;; - *) return 1 ;; - esac - [ "$(modinfo -k "${kernel_version}" -F version amdgpu)" = "${module_version}" ] || return 1 - retrycmd_if_failure 12 5 30 modprobe amdgpu || return 1 - loaded_version=$(cat /sys/module/amdgpu/version) || return 1 - [ "${loaded_version}" = "${module_version}" ] || return 1 - retrycmd_if_failure 12 5 5 test -c /dev/kfd || return 1 - export -f validateAmdGpuDevices - retrycmd_if_failure 12 5 5 bash -c validateAmdGpuDevices || return 1 - echo "AMD GPU driver ${module_version} ready on ${kernel_version}; eight GPUs detected" -} - -validateAmdGpuDevices() { - local gpu_id_file gpu_id count=0 - # Count active KFD agents, not DRM render nodes: MI300X VFs can preallocate - # many more render nodes than the eight GPUs assigned to the VM. - for gpu_id_file in /sys/class/kfd/kfd/topology/nodes/*/gpu_id; do - [ -r "${gpu_id_file}" ] || continue - gpu_id=$(cat "${gpu_id_file}") || return 1 - case "${gpu_id}" in - ''|*[!0-9]*) return 1 ;; - 0) continue ;; - esac - count=$((count + 1)) - done - [ "${count}" -eq 8 ] -} - #EOF diff --git a/parts/linux/cloud-init/artifacts/cse_main.sh b/parts/linux/cloud-init/artifacts/cse_main.sh index 0543f672e03..fbd7b1a105f 100755 --- a/parts/linux/cloud-init/artifacts/cse_main.sh +++ b/parts/linux/cloud-init/artifacts/cse_main.sh @@ -490,6 +490,7 @@ function nodePrep { # Hardware validation belongs in nodePrep so it also runs on PIS nodes. # The dedicated AMD image contains the driver; provisioning never downloads it. if [ "${AMD_GPU_NODE:-false}" = "true" ] && [ "${CONFIG_GPU_DRIVER_IF_NEEDED:-false}" = "true" ]; then + source /opt/azure/containers/amd-gpu-validate.sh || exit $ERR_AMD_GPU_VALIDATE_FAIL logs_to_events "AKS.CSE.ensureAmdGpuDrivers" ensureAmdGpuDrivers || exit $? fi diff --git a/schemas/amd-gpu-components.cue b/schemas/amd-gpu-components.cue new file mode 100644 index 00000000000..b0cb78a00b3 --- /dev/null +++ b/schemas/amd-gpu-components.cue @@ -0,0 +1,33 @@ +package amdgpu + +#AMDGPUDriver: { + repositoryURL: string & =~"^https://repo[.]radeon[.]com/amdgpu/[0-9.]+/ubuntu$" + signingKeyURL: string & =~"^https://repo[.]radeon[.]com/[^?[:space:]]+$" + signingKeyFingerprint: string & =~"^[A-F0-9]{40}$" + distribution: "noble" + component: "main" + packageVersion: string & =~"^[0-9]+:[0-9][0-9A-Za-z.+~-]+$" + firmwarePackageVersion: string & =~"^[0-9]+:[0-9][0-9A-Za-z.+~-]+$" + moduleVersion: string & =~"^[0-9][0-9.]+$" + dkmsVersion: string & =~"^[0-9][0-9A-Za-z.+~-]+$" +} + +#AMDGPUDiagnostics: { + repositoryURL: "https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/" + signingKeyURL: "https://stable.repo.amd.com/rocm/gpg/packages.gpg" + signingKeyFingerprint: string & =~"^[A-F0-9]{40}$" + distribution: "stable" + component: "main" + amdsmiPackage: string & =~"^amdrocm-amdsmi[0-9]+[.][0-9]+$" + amdsmiVersion: string & =~"^[0-9][0-9A-Za-z.+~-]+$" + sysdepsPackage: string & =~"^amdrocm-sysdeps[0-9]+[.][0-9]+$" + sysdepsVersion: string & =~"^[0-9][0-9A-Za-z.+~-]+$" + cliPath: string & =~"^/opt/rocm/core-[0-9]+[.][0-9]+/bin/amd-smi$" +} + +#Components: { + AMDGPUDriver: #AMDGPUDriver + AMDGPUDiagnostics: #AMDGPUDiagnostics +} + +#Components diff --git a/schemas/components.cue b/schemas/components.cue index 5136bea9209..89a9fe9a062 100644 --- a/schemas/components.cue +++ b/schemas/components.cue @@ -125,40 +125,11 @@ package components windowsVersions: [...#WindowsVersion] } -// The dedicated AMD image installs the kernel driver, firmware and diagnostics. Keeping -// this metadata outside Packages prevents generic images from caching it. -#AMDGPUDriver: { - repositoryURL: string & =~"^https://repo[.]radeon[.]com/amdgpu/[0-9.]+/ubuntu$" - signingKeyURL: string & =~"^https://repo[.]radeon[.]com/[^?[:space:]]+$" - signingKeyFingerprint: string & =~"^[A-F0-9]{40}$" - distribution: "noble" - component: "main" - packageVersion: string & =~"^[0-9]+:[0-9][0-9A-Za-z.+~-]+$" - firmwarePackageVersion: string & =~"^[0-9]+:[0-9][0-9A-Za-z.+~-]+$" - moduleVersion: string & =~"^[0-9][0-9.]+$" - dkmsVersion: string & =~"^[0-9][0-9A-Za-z.+~-]+$" -} - -#AMDGPUDiagnostics: { - repositoryURL: "https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/" - signingKeyURL: "https://stable.repo.amd.com/rocm/gpg/packages.gpg" - signingKeyFingerprint: string & =~"^[A-F0-9]{40}$" - distribution: "stable" - component: "main" - amdsmiPackage: string & =~"^amdrocm-amdsmi[0-9]+[.][0-9]+$" - amdsmiVersion: string & =~"^[0-9][0-9A-Za-z.+~-]+$" - sysdepsPackage: string & =~"^amdrocm-sysdeps[0-9]+[.][0-9]+$" - sysdepsVersion: string & =~"^[0-9][0-9A-Za-z.+~-]+$" - cliPath: string & =~"^/opt/rocm/core-[0-9]+[.][0-9]+/bin/amd-smi$" -} - #Components: { ContainerImages: #Images Packages: #Packages GPUContainerImages?: #GPUImages OCIArtifacts?: #OCIArtifacts - AMDGPUDriver?: #AMDGPUDriver - AMDGPUDiagnostics?: #AMDGPUDiagnostics } #Components diff --git a/spec/parts/linux/cloud-init/artifacts/cse_install_amd_gpu_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_install_amd_gpu_spec.sh index fb31656f28a..bdda86c64ef 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_install_amd_gpu_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_install_amd_gpu_spec.sh @@ -1,7 +1,7 @@ #!/bin/bash Describe 'dedicated AMD GPU image provisioning' - Include ./parts/linux/cloud-init/artifacts/cse_install.sh + Include ./parts/linux/cloud-init/artifacts/cse_amd_gpu.sh ERR_AMD_GPU_UNSUPPORTED=244 ERR_AMD_GPU_VALIDATE_FAIL=245 OS=UBUNTU @@ -27,21 +27,31 @@ Describe 'dedicated AMD GPU image provisioning' block && /^[[:space:]]*fi$/ { exit } ' parts/linux/cloud-init/artifacts/cse_main.sh)" } + source() { test "$1" = /opt/azure/containers/amd-gpu-validate.sh; } logs_to_events() { shift; "$@"; } ensureAmdGpuDrivers() { echo amd-driver-validated; } It 'runs hardware checks in nodePrep with both explicit flags' AMD_GPU_NODE=true CONFIG_GPU_DRIVER_IF_NEEDED=true - When call run_amd_node_prep + When run run_amd_node_prep The status should be success The output should equal amd-driver-validated End + It 'fails the AMD node if its dedicated image validator is missing' + AMD_GPU_NODE=true + CONFIG_GPU_DRIVER_IF_NEEDED=true + source() { return 1; } + When run run_amd_node_prep + The status should equal 245 + The output should equal '' + End + It 'honors driver configuration opt-out' AMD_GPU_NODE=true CONFIG_GPU_DRIVER_IF_NEEDED=false - When call run_amd_node_prep + When run run_amd_node_prep The status should be success The output should equal '' End @@ -49,7 +59,7 @@ Describe 'dedicated AMD GPU image provisioning' It 'does nothing on existing nodes without AMD enablement' unset AMD_GPU_NODE CONFIG_GPU_DRIVER_IF_NEEDED=true - When call run_amd_node_prep + When run run_amd_node_prep The status should be success The output should equal '' End @@ -105,7 +115,7 @@ Describe 'dedicated AMD GPU image provisioning' mkdir -p "${AMD_TEST_ROOT}/nodes/${i}" echo "${i}" > "${AMD_TEST_ROOT}/nodes/${i}/gpu_id" done - eval "$(sed -n '/^validateAmdGpuDriver()/,/^}/p; /^validateAmdGpuDevices()/,/^}/p' parts/linux/cloud-init/artifacts/cse_install.sh | + eval "$(sed -n '/^validateAmdGpuDriver()/,/^}/p; /^validateAmdGpuDevices()/,/^}/p' parts/linux/cloud-init/artifacts/cse_amd_gpu.sh | sed "s|/opt/azure/amd-gpu|${AMD_TEST_ROOT}|g; s|/sys/module/amdgpu|${AMD_TEST_ROOT}/module|g; s|/sys/class/kfd/kfd/topology/nodes|${AMD_TEST_ROOT}/nodes|g")" } cleanup() { rm -rf "${AMD_TEST_ROOT}"; } diff --git a/spec/vhdbuilder/packer/amd_gpu_diagnostics_spec.sh b/spec/vhdbuilder/packer/amd_gpu_diagnostics_spec.sh index 4878d3ba9b0..6703b60e090 100644 --- a/spec/vhdbuilder/packer/amd_gpu_diagnostics_spec.sh +++ b/spec/vhdbuilder/packer/amd_gpu_diagnostics_spec.sh @@ -7,20 +7,20 @@ Describe 'Minimal AMDGPU host diagnostics bake' TRACE="${TEST_DIR}/trace" : > "${TRACE}" OS=UBUNTU OS_VERSION=24.04 CPU_ARCH=amd64 HYPERV_GENERATION=v2 ENABLE_FIPS=false - COMPONENTS_FILEPATH="${TEST_DIR}/components.json" + AMD_COMPONENTS_FILEPATH="${TEST_DIR}/components.json" VHD_LOGS_FILEPATH="${TEST_DIR}/vhd.log" FAIL_STAGE="" PACKAGE_ARCH=amd64 mkdir -p "${TEST_DIR}/work" "${TEST_DIR}/bin" "${TEST_DIR}/rocm/core-10.0/bin" "${TEST_DIR}/rocm/core-10.0/share/amd_smi" jq --arg cli_path "${TEST_DIR}/rocm/core-10.0/bin/amd-smi" '.AMDGPUDiagnostics.cliPath = $cli_path' \ - parts/common/components.json > "${COMPONENTS_FILEPATH}" - AMDSMI_PACKAGE=$(jq -r '.AMDGPUDiagnostics.amdsmiPackage' "${COMPONENTS_FILEPATH}") - AMDSMI_VERSION=$(jq -r '.AMDGPUDiagnostics.amdsmiVersion' "${COMPONENTS_FILEPATH}") - SYSDEPS_PACKAGE=$(jq -r '.AMDGPUDiagnostics.sysdepsPackage' "${COMPONENTS_FILEPATH}") - SYSDEPS_VERSION=$(jq -r '.AMDGPUDiagnostics.sysdepsVersion' "${COMPONENTS_FILEPATH}") + vhdbuilder/packer/amd-gpu-components.json > "${AMD_COMPONENTS_FILEPATH}" + AMDSMI_PACKAGE=$(jq -r '.AMDGPUDiagnostics.amdsmiPackage' "${AMD_COMPONENTS_FILEPATH}") + AMDSMI_VERSION=$(jq -r '.AMDGPUDiagnostics.amdsmiVersion' "${AMD_COMPONENTS_FILEPATH}") + SYSDEPS_PACKAGE=$(jq -r '.AMDGPUDiagnostics.sysdepsPackage' "${AMD_COMPONENTS_FILEPATH}") + SYSDEPS_VERSION=$(jq -r '.AMDGPUDiagnostics.sysdepsVersion' "${AMD_COMPONENTS_FILEPATH}") # Run the production function, remapping only owned filesystem locations. # Network, package metadata and package installation are mocked below. - eval "$(sed -n '/^installAMDGPUDiagnostics()/,/^}$/p' vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh | + eval "$(sed -n '/^installAMDGPUDiagnostics()/,/^}$/p' vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh | sed -e "s|/opt/rocm|${TEST_DIR}/rocm|g" -e "s|/usr/local/bin|${TEST_DIR}/bin|g" \ -e "s|/tmp/amd-gpu-diagnostics|${TEST_DIR}/work/amd-gpu-diagnostics|g")" } diff --git a/spec/vhdbuilder/packer/amd_gpu_driver_spec.sh b/spec/vhdbuilder/packer/amd_gpu_driver_spec.sh index bf9f07ce3b8..f73a2955722 100644 --- a/spec/vhdbuilder/packer/amd_gpu_driver_spec.sh +++ b/spec/vhdbuilder/packer/amd_gpu_driver_spec.sh @@ -7,22 +7,22 @@ Describe 'CPU-only AMDGPU driver bake' TRACE="${TEST_DIR}/trace" : > "${TRACE}" OS=UBUNTU OS_VERSION=24.04 CPU_ARCH=amd64 HYPERV_GENERATION=v2 ENABLE_FIPS=false - COMPONENTS_FILEPATH="${PWD}/parts/common/components.json" + AMD_COMPONENTS_FILEPATH="${PWD}/vhdbuilder/packer/amd-gpu-components.json" VHD_LOGS_FILEPATH="${TEST_DIR}/vhd.log" FAIL_STAGE="" PACKAGE_ARCH=all KERNEL=6.8.0-test-azure - PACKAGE_VERSION=$(jq -r '.AMDGPUDriver.packageVersion' "${COMPONENTS_FILEPATH}") - FIRMWARE_VERSION=$(jq -r '.AMDGPUDriver.firmwarePackageVersion' "${COMPONENTS_FILEPATH}") - MODULE_VERSION=$(jq -r '.AMDGPUDriver.moduleVersion' "${COMPONENTS_FILEPATH}") - DKMS_VERSION=$(jq -r '.AMDGPUDriver.dkmsVersion' "${COMPONENTS_FILEPATH}") + PACKAGE_VERSION=$(jq -r '.AMDGPUDriver.packageVersion' "${AMD_COMPONENTS_FILEPATH}") + FIRMWARE_VERSION=$(jq -r '.AMDGPUDriver.firmwarePackageVersion' "${AMD_COMPONENTS_FILEPATH}") + MODULE_VERSION=$(jq -r '.AMDGPUDriver.moduleVersion' "${AMD_COMPONENTS_FILEPATH}") + DKMS_VERSION=$(jq -r '.AMDGPUDriver.dkmsVersion' "${AMD_COMPONENTS_FILEPATH}") mkdir -p "${TEST_DIR}/work" "${TEST_DIR}/marker" "${TEST_DIR}/modprobe" "${TEST_DIR}/vendor-modprobe" \ "${TEST_DIR}/modules/${KERNEL}/build" touch "${TEST_DIR}/modules/${KERNEL}/build/Makefile" printf 'blacklist amdgpu\nblacklist nouveau\n# retain this comment\n' > "${TEST_DIR}/modprobe/cloud.conf" # Remap owned filesystem locations only. Real sed exercises the cloud image # blacklist edit; apt, GPG, module commands and network are all mocked below. - eval "$(sed -n '/^installAMDGPUDriver()/,/^}$/p' vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh | + eval "$(sed -n '/^installAMDGPUDriver()/,/^}$/p' vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh | sed -e "s|/opt/azure/amd-gpu|${TEST_DIR}/marker|g" -e "s|/tmp/amd-gpu|${TEST_DIR}/work/amd-gpu|g" \ -e "s|/etc/modprobe.d|${TEST_DIR}/modprobe|g" -e "s|/usr/lib/modprobe.d|${TEST_DIR}/vendor-modprobe|g" \ -e "s|/lib/modules|${TEST_DIR}/modules|g")" @@ -241,7 +241,7 @@ Describe 'AMD-specific build dispatch' COMPONENTS_FILEPATH="${TEST_DIR}/components.json" jq '{Packages: ([.Packages[] | select(.name | test("nvidia|datacenter-gpu-manager|dcgm-exporter"))] + [{name: "retained-package"}])}' \ parts/common/components.json > "${COMPONENTS_FILEPATH}" - eval "$(sed -n '/^isAMDGPUBuild()/,/^}$/p' vhdbuilder/packer/install-dependencies.sh)" + eval "$(sed -n '/^isAMDGPUSkippedPackage()/,/^}$/p' vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh)" eval "$(sed -n '/^cachePackageAndBinaryComponents()/,/^}$/p' vhdbuilder/packer/install-dependencies.sh)" } cleanup_amd_dispatch() { rm -rf "${TEST_DIR}"; } @@ -262,6 +262,7 @@ Describe 'AMD-specific build dispatch' It 'continues processing NVIDIA packages on existing image SKUs' FEATURE_FLAGS=NVIDIA_CUDA_PREBAKE + unset -f isAMDGPUSkippedPackage When run cachePackageAndBinaryComponents The status should be success The output should include 'processing components.packages' @@ -271,3 +272,122 @@ Describe 'AMD-specific build dispatch' The contents of file "${TRACE}" should include 'retained-package' End End + +Describe 'Dedicated AMD image installation hook' + setup_amd_image() { + TEST_DIR=$(mktemp -d) + TRACE="${TEST_DIR}/trace" + : > "${TRACE}" + FEATURE_FLAGS=AMD_GPU OS=UBUNTU OS_VERSION=24.04 CPU_ARCH=amd64 HYPERV_GENERATION=v2 ENABLE_FIPS=false + SCRIPT_NAME=test-build FAIL_STAGE='' + AMD_COMPONENTS_FILEPATH="${TEST_DIR}/amd-gpu/components.json" + eval "$(sed -n '/^validateAMDGPUImageConfiguration()/,/^}$/p' vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh)" + eval "$(sed -n '/^installAMDGPUImage()/,/^}$/p' vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh | + sed 's/^ install / amd_test_install /')" + } + cleanup_amd_image() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_amd_image + AfterEach cleanup_amd_image + amd_test_install() { + case "$2" in + /home/packer/amd-gpu-components.json) + echo "metadata $*" >> "${TRACE}" + [ "${FAIL_STAGE}" != metadata ] + ;; + /home/packer/amd-gpu-validate.sh) + echo "validator $*" >> "${TRACE}" + [ "${FAIL_STAGE}" != validator ] + ;; + *) return 99 ;; + esac + } + installAMDGPUDriver() { echo driver >> "${TRACE}"; [ "${FAIL_STAGE}" != driver ]; } + installAMDGPUDiagnostics() { echo diagnostics >> "${TRACE}"; [ "${FAIL_STAGE}" != diagnostics ]; } + capture_benchmark() { echo "benchmark $1" >> "${TRACE}"; [ "${FAIL_STAGE}" != benchmark ]; } + + It 'copies only dedicated metadata and the baked validator around the AMD installation' + When run installAMDGPUImage + The status should be success + The output should eq '' + The stderr should eq '' + The contents of file "${TRACE}" should eq "metadata -Dm0644 /home/packer/amd-gpu-components.json ${AMD_COMPONENTS_FILEPATH} +driver +benchmark test-build_build_amd_gpu_kernel_module +diagnostics +benchmark test-build_install_amd_gpu_diagnostics +validator -Dm0755 /home/packer/amd-gpu-validate.sh /opt/azure/containers/amd-gpu-validate.sh" + End + + Describe 'incomplete image installation' + Parameters + metadata + driver + diagnostics + benchmark + validator + End + It 'propagates each installation failure' + FAIL_STAGE="$1" + When run installAMDGPUImage + The status should be failure + The output should eq '' + The stderr should eq '' + End + End + + Describe 'unsupported image configuration' + Parameters + NVIDIA_CUDA_PREBAKE UBUNTU 24.04 amd64 v2 false + AMD_GPU,NVIDIA_CUDA_PREBAKE UBUNTU 24.04 amd64 v2 false + AMD_GPU UBUNTU 22.04 amd64 v2 false + AMD_GPU MARINER 24.04 amd64 v2 false + AMD_GPU UBUNTU 24.04 arm64 v2 false + AMD_GPU UBUNTU 24.04 amd64 v1 false + AMD_GPU UBUNTU 24.04 amd64 v2 True + End + It 'fails before changing image content' + FEATURE_FLAGS="$1" OS="$2" OS_VERSION="$3" CPU_ARCH="$4" HYPERV_GENERATION="$5" ENABLE_FIPS="$6" + When run installAMDGPUImage + The status should be failure + The output should eq '' + The stderr should include 'requires the dedicated Ubuntu 24.04 amd64 Gen2 non-FIPS image' + The contents of file "${TRACE}" should eq '' + End + End +End + +Describe 'AMD installer loading does not affect existing images' + setup_shared_bake_hook() { + TEST_DIR=$(mktemp -d) + TRACE="${TEST_DIR}/trace" + : > "${TRACE}" + AMD_SUPPORT_FILE="${TEST_DIR}/amd_gpu.sh" + # Match the literal feature flag expression in the shared source hook. + # shellcheck disable=SC2016 + SHARED_HOOK=$(sed -n '/^case "${FEATURE_FLAGS:-}" in$/,/^esac$/p' vhdbuilder/packer/install-dependencies.sh | + sed "s|/home/packer/amd_gpu.sh|${AMD_SUPPORT_FILE}|g") + [ -n "${SHARED_HOOK}" ] + } + cleanup_shared_bake_hook() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_shared_bake_hook + AfterEach cleanup_shared_bake_hook + run_shared_bake_hook() { eval "${SHARED_HOOK}"; } + + Parameters + None missing + None broken + NVIDIA_CUDA_PREBAKE missing + NVIDIA_CUDA_PREBAKE broken + End + It 'does not load a missing or broken AMD support file on non-AMD images' + FEATURE_FLAGS="$1" + if [ "$2" = broken ]; then + printf 'echo AMD-file-was-sourced >> "%s"\nreturn 99\n' "${TRACE}" > "${AMD_SUPPORT_FILE}" + fi + When run run_shared_bake_hook + The status should be success + The output should eq '' + The stderr should eq '' + The contents of file "${TRACE}" should eq '' + End +End diff --git a/spec/vhdbuilder/packer/amd_gpu_pipeline_spec.sh b/spec/vhdbuilder/packer/amd_gpu_pipeline_spec.sh new file mode 100644 index 00000000000..213e5b15659 --- /dev/null +++ b/spec/vhdbuilder/packer/amd_gpu_pipeline_spec.sh @@ -0,0 +1,194 @@ +#!/bin/bash +# shellcheck disable=SC2034,SC2329 + +Describe 'AMD-only Packer template preparation' + Include './vhdbuilder/packer/amd-gpu-build-settings.sh' + + setup_template() { + TEST_DIR=$(mktemp -d) + TEMPLATE="${TEST_DIR}/template.json" + MAPPINGS="${PWD}/vhdbuilder/packer/amd-gpu-packer-files.json" + cp vhdbuilder/packer/vhd-image-builder-base.json "${TEMPLATE}" + OS_SKU=Ubuntu OS_VERSION=24.04 ARCHITECTURE=X86_64 HYPERV_GENERATION=V2 + FEATURE_FLAGS=AMD_GPU ENABLE_FIPS=False ENABLE_TRUSTED_LAUNCH=False TRUSTED_LAUNCH_SUPPORTED=False + SKU_NAME='' SIG_IMAGE_NAME='' + } + cleanup_template() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_template + AfterEach cleanup_template + + verify_upload_order() { + prepare_amd_gpu_packer_template "${TEMPLATE}" "${MAPPINGS}" || return 1 + jq -e --slurpfile mappings "${MAPPINGS}" --slurpfile original vhdbuilder/packer/vhd-image-builder-base.json ' + [.provisioners | to_entries[] | select(.value.type == "shell") | + select(any(.value.inline[]?; contains(" /home/packer/install-dependencies.sh"))) | .key][0] as $install | + .provisioners[$install - 3:$install] == $mappings[0] and + (.provisioners | map(. as $entry | select(all($mappings[0][]; . != $entry)))) == $original[0].provisioners and + del(.provisioners) == ($original[0] | del(.provisioners)) + ' "${TEMPLATE}" + } + + It 'uploads all three AMD files before installation and preserves the original template content' + When call verify_upload_order + The status should be success + The output should equal true + End + + verify_idempotency() { + prepare_amd_gpu_packer_template "${TEMPLATE}" "${MAPPINGS}" || return 1 + cp "${TEMPLATE}" "${TEST_DIR}/once.json" + prepare_amd_gpu_packer_template "${TEMPLATE}" "${MAPPINGS}" || return 1 + cmp "${TEMPLATE}" "${TEST_DIR}/once.json" + } + + It 'can prepare the worker checkout repeatedly without duplicate uploads' + When call verify_idempotency + The status should be success + The output should be blank + End + + break_template() { + case "$1" in + missing) + jq '.provisioners |= map(select(.type != "shell" or + (any(.inline[]?; contains(" /home/packer/install-dependencies.sh")) | not)))' "${TEMPLATE}" ;; + duplicate) + jq '.provisioners += [.provisioners[] | select(.type == "shell") | + select(any(.inline[]?; contains(" /home/packer/install-dependencies.sh")))]' "${TEMPLATE}" ;; + conflict) + jq '.provisioners += [{"type":"file", "source":"unexpected.sh", "destination":"/home/packer/amd_gpu.sh"}]' "${TEMPLATE}" ;; + esac > "${TEST_DIR}/broken.json" + mv "${TEST_DIR}/broken.json" "${TEMPLATE}" + } + + reject_without_truncation() { + local status=0 + cp "${TEMPLATE}" "${TEST_DIR}/before.json" + prepare_amd_gpu_packer_template "${TEMPLATE}" "${MAPPINGS}" || status=$? + cmp "${TEMPLATE}" "${TEST_DIR}/before.json" || return 99 + return "${status}" + } + + Describe 'invalid base templates' + Parameters + missing 'expected exactly one install-dependencies.sh provisioner' + duplicate 'expected exactly one install-dependencies.sh provisioner' + conflict 'conflicting AMD Packer file destination' + End + + It 'rejects an invalid template without replacing or truncating it' + break_template "$1" + When call reject_without_truncation + The status should equal 5 + The stderr should include "$2" + The output should be blank + End + End + + It 'rejects duplicate AMD destinations without changing the worker template' + jq '. + [.[0]]' "${MAPPINGS}" > "${TEST_DIR}/duplicates.json" + MAPPINGS="${TEST_DIR}/duplicates.json" + When call reject_without_truncation + The status should equal 5 + The stderr should include 'invalid AMD Packer file mappings' + The output should be blank + End + + It 'rejects preparation for an ordinary CPU image' + FEATURE_FLAGS=None + When call reject_without_truncation + The status should equal 1 + The stderr should include 'AMD_GPU requires Ubuntu 24.04 x86_64 Gen2' + The output should be blank + End +End + +Describe 'AMD build resource group cleanup' + Include './vhdbuilder/packer/cleanup-amd-gpu-build.sh' + + setup_cleanup() { + TEST_DIR=$(mktemp -d) + TRACE="${TEST_DIR}/trace" + SETTINGS="${TEST_DIR}/settings.json" + RESOURCE_GROUP=image-builder-recorded BUILD_ID=1234 + GROUP_JSON='{"name":"image-builder-recorded","id":"/subscriptions/subscription-build/resourceGroups/image-builder-recorded","tags":{"buildId":"1234","createdBy":"aks-vhd-pipeline"}}' + SHOW_STATUS=0 + printf '{"subscription_id":"subscription-build"}\n' > "${SETTINGS}" + : > "${TRACE}" + } + cleanup_test() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_cleanup + AfterEach cleanup_test + + az() { + printf '%s\n' "$*" >> "${TRACE}" + case "$1 $2" in + 'group show') printf '%s\n' "${GROUP_JSON}"; return "${SHOW_STATUS}" ;; + 'group delete') return 0 ;; + *) return 99 ;; + esac + } + cleanup_recorded_group() { + cleanup_amd_gpu_build_resource_group "${SETTINGS}" "${RESOURCE_GROUP}" "${BUILD_ID}" + } + + It 'deletes only the recorded group with matching build ownership in the recorded subscription' + When call cleanup_recorded_group + The status should be success + The output should include 'Deleting verified Packer resource group image-builder-recorded for build 1234' + The contents of file "${TRACE}" should equal 'group show --name image-builder-recorded --subscription subscription-build --output json +group delete --name image-builder-recorded --subscription subscription-build --yes --only-show-errors' + End + + Describe 'unverified resource ownership' + Parameters + build_id '.tags.buildId = "another-build"' + missing_tag 'del(.tags.buildId)' + owner '.tags.createdBy = "another-owner"' + name '.name = "another-resource-group"' + subscription '.id = "/subscriptions/another-subscription/resourceGroups/image-builder-recorded"' + End + + It 'does not delete a group without exact recorded ownership' + GROUP_JSON=$(jq "$2" <<< "${GROUP_JSON}") + When call cleanup_recorded_group + The status should be success + The output should include 'ownership does not match this build' + The contents of file "${TRACE}" should include 'group show ' + The contents of file "${TRACE}" should not include 'group delete ' + End + End + + Describe 'missing pipeline identity' + Parameters + RESOURCE_GROUP '' + RESOURCE_GROUP "\$(PKR_RG_NAME)" + BUILD_ID '' + BUILD_ID "\$(Build.BuildId)" + End + + It 'does not call Azure when the recorded group or build ID is unavailable' + printf -v "$1" '%s' "$2" + When call cleanup_recorded_group + The status should be success + The output should include 'skipping AMD build cleanup' + The contents of file "${TRACE}" should be blank + End + End + + It 'does not delete anything when the recorded subscription is missing' + printf '{}\n' > "${SETTINGS}" + When call cleanup_recorded_group + The status should be success + The output should include 'No recorded build subscription' + The contents of file "${TRACE}" should be blank + End + + It 'does not delete anything when the group cannot be read' + SHOW_STATUS=1 + When call cleanup_recorded_group + The status should be success + The output should include 'absent or could not be read' + The contents of file "${TRACE}" should not include 'group delete ' + End +End diff --git a/spec/vhdbuilder/packer/linux_sku_name_spec.sh b/spec/vhdbuilder/packer/linux_sku_name_spec.sh index 1ba3356e3d5..16bd2820c47 100644 --- a/spec/vhdbuilder/packer/linux_sku_name_spec.sh +++ b/spec/vhdbuilder/packer/linux_sku_name_spec.sh @@ -1,6 +1,7 @@ #!/bin/bash Describe 'Linux VHD flavor naming and AMD GPU build constraints' + Include './vhdbuilder/packer/amd-gpu-build-settings.sh' Include './vhdbuilder/packer/produce-packer-settings-functions.sh' setup_environment() { @@ -20,13 +21,13 @@ Describe 'Linux VHD flavor naming and AMD GPU build constraints' BeforeEach 'setup_environment' It 'uses a dedicated AMD GPU SKU instead of the shared Ubuntu image' - When call get_linux_sku_name + When call get_amd_gpu_sku_name The status should be success The output should equal '2404gen2amdgpucontainerd' End derive_sig_name() { - SKU_NAME=$(get_linux_sku_name) || return 1 + SKU_NAME=$(get_amd_gpu_sku_name) || return 1 ensure_sig_image_name_linux } @@ -53,11 +54,12 @@ Describe 'Linux VHD flavor naming and AMD GPU build constraints' FEATURE_FLAGS 'AMD_GPU,cvm' FEATURE_FLAGS 'AMD_GPU,minimal' FEATURE_FLAGS 'NOT_AMD_GPU' + FEATURE_FLAGS None End It 'rejects the unsupported AMD image configuration' printf -v "$1" '%s' "$2" - When call get_linux_sku_name + When call get_amd_gpu_sku_name The status should be failure The stderr should include 'AMD_GPU requires Ubuntu 24.04 x86_64 Gen2' The output should be blank @@ -94,7 +96,19 @@ Describe 'Linux VHD flavor naming and AMD GPU build constraints' The output should be blank End - # Preserve the existing public naming contract when extracting the pipeline logic. + # Run the actual pipeline step to verify that ordinary image names still use + # their existing algorithm without loading or invoking the AMD helper/schema. + derive_pipeline_sku_name() { + local script + script=$(awk ' + /^ - bash: \|$/ { script = ""; in_script = 1; next } + /^ displayName: Set SKU Name$/ { printf "%s", script; exit } + in_script && /^ / { sub(/^ /, ""); script = script $0 "\n" } + ' .pipelines/templates/.builder-release-template.yaml) + eval "${script}" >/dev/null || return 1 + printf '%s\n' "${SKU_NAME}" + } + Describe 'existing image flavors' Parameters Ubuntu 22.04 X86_64 V1 None False False 2204containerd @@ -119,7 +133,7 @@ Describe 'Linux VHD flavor naming and AMD GPU build constraints' FEATURE_FLAGS="$5" ENABLE_FIPS="$6" ENABLE_TRUSTED_LAUNCH="$7" - When call get_linux_sku_name + When call derive_pipeline_sku_name The status should be success The output should equal "$8" End diff --git a/spec/vhdbuilder/packer/test/amd_gpu_content_spec.sh b/spec/vhdbuilder/packer/test/amd_gpu_content_spec.sh index 633a527a550..380f3a2d8e9 100644 --- a/spec/vhdbuilder/packer/test/amd_gpu_content_spec.sh +++ b/spec/vhdbuilder/packer/test/amd_gpu_content_spec.sh @@ -5,21 +5,21 @@ Describe 'AMD VHD content validation' setup_amd_content() { TEST_DIR=$(mktemp -d) FEATURE_FLAGS=AMD_GPU OS_SKU=Ubuntu OS_VERSION=24.04 ENABLE_FIPS=false - COMPONENTS_FILEPATH="${PWD}/parts/common/components.json" + AMD_COMPONENTS_FILEPATH="${PWD}/vhdbuilder/packer/amd-gpu-components.json" KERNEL=6.8.0-test-azure FAIL_STAGE="" - PACKAGE_VERSION=$(jq -r '.AMDGPUDriver.packageVersion' "${COMPONENTS_FILEPATH}") - FIRMWARE_VERSION=$(jq -r '.AMDGPUDriver.firmwarePackageVersion' "${COMPONENTS_FILEPATH}") - MODULE_VERSION=$(jq -r '.AMDGPUDriver.moduleVersion' "${COMPONENTS_FILEPATH}") - DKMS_VERSION=$(jq -r '.AMDGPUDriver.dkmsVersion' "${COMPONENTS_FILEPATH}") - AMDSMI_PACKAGE=$(jq -r '.AMDGPUDiagnostics.amdsmiPackage' "${COMPONENTS_FILEPATH}") - SYSDEPS_PACKAGE=$(jq -r '.AMDGPUDiagnostics.sysdepsPackage' "${COMPONENTS_FILEPATH}") + PACKAGE_VERSION=$(jq -r '.AMDGPUDriver.packageVersion' "${AMD_COMPONENTS_FILEPATH}") + FIRMWARE_VERSION=$(jq -r '.AMDGPUDriver.firmwarePackageVersion' "${AMD_COMPONENTS_FILEPATH}") + MODULE_VERSION=$(jq -r '.AMDGPUDriver.moduleVersion' "${AMD_COMPONENTS_FILEPATH}") + DKMS_VERSION=$(jq -r '.AMDGPUDriver.dkmsVersion' "${AMD_COMPONENTS_FILEPATH}") + AMDSMI_PACKAGE=$(jq -r '.AMDGPUDiagnostics.amdsmiPackage' "${AMD_COMPONENTS_FILEPATH}") + SYSDEPS_PACKAGE=$(jq -r '.AMDGPUDiagnostics.sysdepsPackage' "${AMD_COMPONENTS_FILEPATH}") mkdir -p "${TEST_DIR}/marker" "${TEST_DIR}/modules/${KERNEL}/build" "${TEST_DIR}/sources/amdgpu-${DKMS_VERSION}" touch "${TEST_DIR}/modules/${KERNEL}/build/Makefile" "${TEST_DIR}/sources/amdgpu-${DKMS_VERSION}/dkms.conf" jq -n --arg driver "${PACKAGE_VERSION}" --arg firmware "${FIRMWARE_VERSION}" --arg module_version "${MODULE_VERSION}" \ '{schema_version: 1, package_version: $driver, firmware_package_version: $firmware, module_version: $module_version, kernel_version: "earlier-bake-kernel"}' > "${TEST_DIR}/marker/driver.json" - eval "$(sed -n '/^testAMDGPUDriver()/,/^}$/p' vhdbuilder/packer/test/linux-vhd-content-test.sh | + eval "$(sed -n '/^testAMDGPUDriver()/,/^}$/p' vhdbuilder/packer/test/amd-gpu-vhd-content-test.sh | sed -e "s|/opt/azure/amd-gpu|${TEST_DIR}/marker|g" -e "s|/lib/modules|${TEST_DIR}/modules|g" \ -e "s|/usr/src|${TEST_DIR}/sources|g")" } @@ -128,7 +128,7 @@ Describe 'AMD SMI VHD content validation' TEST_DIR=$(mktemp -d) FEATURE_FLAGS=AMD_GPU FAIL_STAGE="" - COMPONENTS_FILEPATH="${TEST_DIR}/components.json" + AMD_COMPONENTS_FILEPATH="${TEST_DIR}/components.json" CLI_PATH="${TEST_DIR}/rocm/core-10.0/bin/amd-smi" CLI_TARGET="${TEST_DIR}/rocm/core-10.0/libexec/amdsmi_cli/amdsmi_cli.py" MODULE_PATH="${TEST_DIR}/rocm/core-10.0/share/amd_smi/amdsmi/__init__.py" @@ -148,12 +148,12 @@ def amdsmi_init(): raise RuntimeError("GPU initialization must not run on the CPU bake machine") PY printf 'raise ImportError("unrelated PYTHONPATH must not affect the bake check")\n' > "${TEST_DIR}/environment/amdsmi.py" - jq --arg cli "${CLI_PATH}" '.AMDGPUDiagnostics.cliPath = $cli' parts/common/components.json > "${COMPONENTS_FILEPATH}" - AMDSMI_PACKAGE=$(jq -r '.AMDGPUDiagnostics.amdsmiPackage' "${COMPONENTS_FILEPATH}") - AMDSMI_VERSION=$(jq -r '.AMDGPUDiagnostics.amdsmiVersion' "${COMPONENTS_FILEPATH}") - SYSDEPS_PACKAGE=$(jq -r '.AMDGPUDiagnostics.sysdepsPackage' "${COMPONENTS_FILEPATH}") - SYSDEPS_VERSION=$(jq -r '.AMDGPUDiagnostics.sysdepsVersion' "${COMPONENTS_FILEPATH}") - eval "$(sed -n '/^testAMDGPUDiagnostics()/,/^}$/p' vhdbuilder/packer/test/linux-vhd-content-test.sh | + jq --arg cli "${CLI_PATH}" '.AMDGPUDiagnostics.cliPath = $cli' vhdbuilder/packer/amd-gpu-components.json > "${AMD_COMPONENTS_FILEPATH}" + AMDSMI_PACKAGE=$(jq -r '.AMDGPUDiagnostics.amdsmiPackage' "${AMD_COMPONENTS_FILEPATH}") + AMDSMI_VERSION=$(jq -r '.AMDGPUDiagnostics.amdsmiVersion' "${AMD_COMPONENTS_FILEPATH}") + SYSDEPS_PACKAGE=$(jq -r '.AMDGPUDiagnostics.sysdepsPackage' "${AMD_COMPONENTS_FILEPATH}") + SYSDEPS_VERSION=$(jq -r '.AMDGPUDiagnostics.sysdepsVersion' "${AMD_COMPONENTS_FILEPATH}") + eval "$(sed -n '/^testAMDGPUDiagnostics()/,/^}$/p' vhdbuilder/packer/test/amd-gpu-vhd-content-test.sh | sed -e "s|/usr/local/bin/amd-smi|${TEST_DIR}/bin/amd-smi|g" \ -e "s|/opt/rocm/|${TEST_DIR}/rocm/|g")" } @@ -241,10 +241,100 @@ PY End It 'reports missing diagnostics metadata' - printf '{}\n' > "${COMPONENTS_FILEPATH}" + printf '{}\n' > "${AMD_COMPONENTS_FILEPATH}" When run testAMDGPUDiagnostics The status should be failure The output should eq '' The stderr should include 'Missing AMDGPU diagnostics component metadata' End End + +Describe 'Dedicated AMD image content hook' + setup_amd_content_hook() { + TEST_DIR=$(mktemp -d) + TRACE="${TEST_DIR}/trace" + : > "${TRACE}" + touch "${TEST_DIR}/amd-gpu-validate.sh" + chmod +x "${TEST_DIR}/amd-gpu-validate.sh" + FAIL_STAGE='' + eval "$(sed -n '/^testAMDGPUImage()/,/^}$/p' vhdbuilder/packer/test/amd-gpu-vhd-content-test.sh | + sed "s|/opt/azure/containers/amd-gpu-validate.sh|${TEST_DIR}/amd-gpu-validate.sh|g")" + } + cleanup_amd_content_hook() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_amd_content_hook + AfterEach cleanup_amd_content_hook + err() { printf '%s: %s\n' "$1" "$2" >&2; } + testAMDGPUDriver() { echo driver >> "${TRACE}"; [ "${FAIL_STAGE}" != driver ]; } + testAMDGPUDiagnostics() { echo diagnostics >> "${TRACE}"; [ "${FAIL_STAGE}" != diagnostics ]; } + + It 'checks the driver, diagnostics and executable bootstrap validator' + When run testAMDGPUImage + The status should be success + The output should eq '' + The stderr should eq '' + The contents of file "${TRACE}" should eq "driver +diagnostics" + End + + It 'fails a missing bootstrap validator while still checking installed content' + rm "${TEST_DIR}/amd-gpu-validate.sh" + When run testAMDGPUImage + The status should be failure + The output should eq '' + The stderr should include 'bootstrap validator is missing or not executable' + The contents of file "${TRACE}" should eq "driver +diagnostics" + End + + Describe 'failed component checks' + Parameters + driver + diagnostics + End + It 'preserves failure even if the other component succeeds' + FAIL_STAGE="$1" + When run testAMDGPUImage + The status should be failure + The output should eq '' + The stderr should eq '' + The contents of file "${TRACE}" should eq "driver +diagnostics" + End + End +End + +Describe 'AMD content checks do not affect existing images' + setup_shared_content_hook() { + TEST_DIR=$(mktemp -d) + TRACE="${TEST_DIR}/trace" + : > "${TRACE}" + AMD_SUPPORT_FILE="${TEST_DIR}/amd-gpu-vhd-content-test.sh" + # Match the literal feature flag expression in the shared source hook. + # shellcheck disable=SC2016 + SHARED_HOOK=$(sed -n '/^if \[ "${FEATURE_FLAGS:-}" = "AMD_GPU" \]; then$/,/^fi$/p' vhdbuilder/packer/test/linux-vhd-content-test.sh | + sed "s|./AgentBaker/vhdbuilder/packer/test/amd-gpu-vhd-content-test.sh|${AMD_SUPPORT_FILE}|g") + [ -n "${SHARED_HOOK}" ] + } + cleanup_shared_content_hook() { rm -rf "${TEST_DIR}"; } + BeforeEach setup_shared_content_hook + AfterEach cleanup_shared_content_hook + run_shared_content_hook() { eval "${SHARED_HOOK}"; } + + Parameters + None missing + None broken + NVIDIA_CUDA_PREBAKE missing + NVIDIA_CUDA_PREBAKE broken + End + It 'does not load missing or broken AMD tests on non-AMD images' + FEATURE_FLAGS="$1" + if [ "$2" = broken ]; then + printf 'echo AMD-file-was-sourced >> "%s"\nreturn 99\n' "${TRACE}" > "${AMD_SUPPORT_FILE}" + fi + When run run_shared_content_hook + The status should be success + The output should eq '' + The stderr should eq '' + The contents of file "${TRACE}" should eq '' + End +End diff --git a/vhdbuilder/packer/amd-gpu-build-settings.sh b/vhdbuilder/packer/amd-gpu-build-settings.sh new file mode 100644 index 00000000000..6e4a4ee050e --- /dev/null +++ b/vhdbuilder/packer/amd-gpu-build-settings.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Loaded only by an explicit AMD build. Shared CPU/NVIDIA settings stay in their +# existing path and do not depend on this image's configuration or validation. +function validate_amd_gpu_build() { + if [ "${FEATURE_FLAGS:-}" != "AMD_GPU" ] || [ "${OS_SKU:-}" != "Ubuntu" ] || + [ "${OS_VERSION:-}" != "24.04" ] || [ "${ARCHITECTURE,,}" != "x86_64" ] || + [ "${HYPERV_GENERATION,,}" != "v2" ] || [ "${ENABLE_FIPS,,}" = "true" ] || + [ "${ENABLE_TRUSTED_LAUNCH,,}" = "true" ] || [ "${TRUSTED_LAUNCH_SUPPORTED,,}" = "true" ]; then + echo "AMD_GPU requires Ubuntu 24.04 x86_64 Gen2, without other feature flags, FIPS or Trusted Launch" >&2 + return 1 + fi + if [ -n "${SKU_NAME:-}" ] && [ "${SKU_NAME}" != "2404gen2amdgpucontainerd" ]; then + echo "AMD_GPU requires the dedicated SKU_NAME 2404gen2amdgpucontainerd" >&2 + return 1 + fi + if [ -n "${SIG_IMAGE_NAME:-}" ] && [ "${SIG_IMAGE_NAME}" != "2404gen2amdgpucontainerd" ]; then + echo "AMD_GPU requires the dedicated SIG_IMAGE_NAME 2404gen2amdgpucontainerd" >&2 + return 1 + fi +} + +function get_amd_gpu_sku_name() { + validate_amd_gpu_build || return 1 + printf '%s\n' '2404gen2amdgpucontainerd' +} + +# Only the manual AMD pipeline calls this function. The shared checked-in Packer +# template stays unchanged, and ordinary image builds never upload AMD files. +function prepare_amd_gpu_packer_template() { + local template="$1" mappings="$2" temporary status + validate_amd_gpu_build || return 1 + temporary=$(mktemp "${template}.amd.XXXXXX") || return 1 + jq --slurpfile mappings "${mappings}" ' + def install_boundary: + .type == "shell" and any(.inline[]?; + test("(^|[[:space:]])/home/packer/install-dependencies\\.sh([[:space:]]|$)")); + $mappings[0] as $uploads | + if ($mappings | length) != 1 or ($uploads | type) != "array" or + ($uploads | length) == 0 or + (all($uploads[]; .type == "file" and (.source | type) == "string" and + (.destination | type) == "string") | not) or + ([$uploads[].destination] | length) != ([$uploads[].destination] | unique | length) + then error("invalid AMD Packer file mappings") + elif any(.provisioners[]; . as $existing | + any($uploads[]; .destination == $existing.destination and . != $existing)) + then error("conflicting AMD Packer file destination") + else .provisioners |= ( + map(. as $existing | select(all($uploads[]; . != $existing))) | + [to_entries[] | select(.value | install_boundary) | .key] as $boundaries | + if ($boundaries | length) != 1 + then error("expected exactly one install-dependencies.sh provisioner") + else .[:$boundaries[0]] + $uploads + .[$boundaries[0]:] + end) + end + ' "${template}" >"${temporary}" && mv "${temporary}" "${template}" + status=$? + rm -f "${temporary}" + return "${status}" +} diff --git a/vhdbuilder/packer/amd-gpu-components.json b/vhdbuilder/packer/amd-gpu-components.json new file mode 100644 index 00000000000..5edd901a079 --- /dev/null +++ b/vhdbuilder/packer/amd-gpu-components.json @@ -0,0 +1,25 @@ +{ + "AMDGPUDriver": { + "repositoryURL": "https://repo.radeon.com/amdgpu/31.50/ubuntu", + "signingKeyURL": "https://repo.radeon.com/rocm/rocm.gpg.key", + "signingKeyFingerprint": "CA8BB4727A47B4D09B4EE8969386B48A1A693C5C", + "distribution": "noble", + "component": "main", + "packageVersion": "1:7.1.3.31500000-2390945.24.04", + "firmwarePackageVersion": "1:31.50.0.0.31500000-2390945.24.04", + "moduleVersion": "7.1.3.31500000", + "dkmsVersion": "7.1.3-2390945.24.04" + }, + "AMDGPUDiagnostics": { + "repositoryURL": "https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/", + "signingKeyURL": "https://stable.repo.amd.com/rocm/gpg/packages.gpg", + "signingKeyFingerprint": "D0F004A0025A1145C7807FCD0701EAC4D5E02107", + "distribution": "stable", + "component": "main", + "amdsmiPackage": "amdrocm-amdsmi10.0", + "amdsmiVersion": "10.0.0-4", + "sysdepsPackage": "amdrocm-sysdeps10.0", + "sysdepsVersion": "10.0.0-4", + "cliPath": "/opt/rocm/core-10.0/bin/amd-smi" + } +} diff --git a/vhdbuilder/packer/amd-gpu-packer-files.json b/vhdbuilder/packer/amd-gpu-packer-files.json new file mode 100644 index 00000000000..41cf7ae4ebd --- /dev/null +++ b/vhdbuilder/packer/amd-gpu-packer-files.json @@ -0,0 +1,17 @@ +[ + { + "type": "file", + "source": "vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh", + "destination": "/home/packer/amd_gpu.sh" + }, + { + "type": "file", + "source": "vhdbuilder/packer/amd-gpu-components.json", + "destination": "/home/packer/amd-gpu-components.json" + }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/cse_amd_gpu.sh", + "destination": "/home/packer/amd-gpu-validate.sh" + } +] diff --git a/vhdbuilder/packer/cleanup-amd-gpu-build.sh b/vhdbuilder/packer/cleanup-amd-gpu-build.sh new file mode 100644 index 00000000000..334335eaffa --- /dev/null +++ b/vhdbuilder/packer/cleanup-amd-gpu-build.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# The shared cleanup script also collects unrelated old images, galleries and +# resource groups. This manual pipeline cleans only its recorded Packer group. +function cleanup_amd_gpu_build_resource_group() { + local settings_file="$1" resource_group="$2" build_id="$3" + local subscription group + case "${resource_group}" in + '' | *"\$("*) + echo 'No recorded Packer resource group; skipping AMD build cleanup' + return 0 + ;; + esac + case "${build_id}" in + '' | *[!0-9]*) + echo 'No valid build ID; skipping AMD build cleanup' + return 0 + ;; + esac + subscription=$(jq -er '.subscription_id | select(type == "string" and length > 0)' "${settings_file}") || { + echo 'No recorded build subscription; skipping AMD build cleanup' + return 0 + } + case "${subscription}" in + *"\$("*) + echo 'Unresolved build subscription; skipping AMD build cleanup' + return 0 + ;; + esac + group=$(az group show --name "${resource_group}" --subscription "${subscription}" --output json) || { + echo 'Packer resource group is absent or could not be read; skipping AMD build cleanup' + return 0 + } + if ! jq -e --arg build_id "${build_id}" --arg resource_group "${resource_group}" --arg subscription "${subscription}" ' + .tags.buildId == $build_id and .tags.createdBy == "aks-vhd-pipeline" and + (.name | ascii_downcase) == ($resource_group | ascii_downcase) and + (.id | ascii_downcase) == + ("/subscriptions/" + $subscription + "/resourceGroups/" + $resource_group | ascii_downcase) + ' <<<"${group}" >/dev/null; then + echo 'Packer resource group ownership does not match this build; skipping AMD build cleanup' + return 0 + fi + echo "Deleting verified Packer resource group ${resource_group} for build ${build_id}" + az group delete --name "${resource_group}" --subscription "${subscription}" --yes --only-show-errors +} diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 6840418c4d2..14461471b49 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -42,18 +42,12 @@ LOCALDNS_BINARY_PATH="/opt/azure/containers/localdns/binary" PERFORMANCE_DATA_FILE=/opt/azure/vhd-build-performance-data.json GRID_COMPATIBILITY_DATA_FILE=/opt/azure/vhd-grid-compatibility-data.json -isAMDGPUBuild() { - case "${FEATURE_FLAGS:-}" in *AMD_GPU*) return 0 ;; *) return 1 ;; esac -} - -if isAMDGPUBuild; then - if [ "${FEATURE_FLAGS}" != "AMD_GPU" ] || [ "${OS}" != "${UBUNTU_OS_NAME}" ] || - [ "${OS_VERSION}" != "24.04" ] || [ "${CPU_ARCH}" != "amd64" ] || - [ "${HYPERV_GENERATION,,}" != "v2" ] || [ "${ENABLE_FIPS,,}" = "true" ]; then - echo "AMD_GPU requires the dedicated Ubuntu 24.04 amd64 Gen2 non-FIPS image" >&2 - exit 1 - fi -fi +case "${FEATURE_FLAGS:-}" in + *AMD_GPU*) + source /home/packer/amd_gpu.sh + validateAMDGPUImageConfiguration || exit 1 + ;; +esac string_replace() { echo ${1//\*/$2} @@ -518,13 +512,9 @@ cachePackageAndBinaryComponents() { while IFS= read -r p; do #getting metadata for each package name=$(echo "${p}" | jq .name -r) - if isAMDGPUBuild; then - case "${name}" in - nvidia-*|dra-driver-nvidia-*|datacenter-gpu-manager-*|dcgm-exporter) - echo "Skipping NVIDIA package ${name} on the AMD GPU image" - continue - ;; - esac + if [ "${FEATURE_FLAGS:-}" = "AMD_GPU" ] && isAMDGPUSkippedPackage "${name}"; then + echo "Skipping NVIDIA package ${name} on the AMD GPU image" + continue fi os=${OS} # TODO(mheberling): Remove this once kata uses standard containerd. This OS is referenced @@ -1232,7 +1222,7 @@ if isMarinerOrAzureLinux "$OS" && ! isAzureLinuxOSGuard "$OS" "$OS_VARIANT"; the activateNfConntrack elif [ "${OS}" = "${UBUNTU_OS_NAME}" ]; then updateAptWithMicrosoftPkg - if ! isAMDGPUBuild; then + if [ "${FEATURE_FLAGS:-}" != "AMD_GPU" ]; then updateAptWithNvidiaPkg fi fi @@ -1251,11 +1241,8 @@ ctr namespace create k8s.io # running them here (near-empty disk, BCC not yet started) avoids exhausting the 30GB packer build disk. # Running them after the container-image cache and/or concurrently with the BCC build fills the disk # (worse on 24.04), failing at the nvidia.ko link or the driver lib copy with "No space left on device". -if isAMDGPUBuild; then - installAMDGPUDriver || exit 1 - capture_benchmark "${SCRIPT_NAME}_build_amd_gpu_kernel_module" - installAMDGPUDiagnostics || exit 1 - capture_benchmark "${SCRIPT_NAME}_install_amd_gpu_diagnostics" +if [ "${FEATURE_FLAGS:-}" = "AMD_GPU" ]; then + installAMDGPUImage || exit 1 else cacheGPUContainerImageComponents buildNVIDIAKernelModule diff --git a/vhdbuilder/packer/produce-packer-settings-functions.sh b/vhdbuilder/packer/produce-packer-settings-functions.sh index e61f066dd0e..76572ca6531 100644 --- a/vhdbuilder/packer/produce-packer-settings-functions.sh +++ b/vhdbuilder/packer/produce-packer-settings-functions.sh @@ -1,47 +1,5 @@ #!/bin/bash -function validate_amd_gpu_build() { - case "${FEATURE_FLAGS:-}" in - *AMD_GPU*) ;; - *) return 0 ;; - esac - - # This is a separate driver-only image, not an addition to a shared NVIDIA VHD. - if [ "${FEATURE_FLAGS}" != "AMD_GPU" ] || [ "${OS_SKU}" != "Ubuntu" ] || - [ "${OS_VERSION}" != "24.04" ] || [ "${ARCHITECTURE,,}" != "x86_64" ] || - [ "${HYPERV_GENERATION,,}" != "v2" ] || [ "${ENABLE_FIPS,,}" = "true" ] || - [ "${ENABLE_TRUSTED_LAUNCH,,}" = "true" ] || [ "${TRUSTED_LAUNCH_SUPPORTED,,}" = "true" ]; then - echo "AMD_GPU requires Ubuntu 24.04 x86_64 Gen2, without other feature flags, FIPS or Trusted Launch" >&2 - return 1 - fi - if [ -n "${SKU_NAME:-}" ] && [ "${SKU_NAME}" != "2404gen2amdgpucontainerd" ]; then - echo "AMD_GPU requires the dedicated SKU_NAME 2404gen2amdgpucontainerd" >&2 - return 1 - fi - if [ -n "${SIG_IMAGE_NAME:-}" ] && [ "${SIG_IMAGE_NAME}" != "2404gen2amdgpucontainerd" ]; then - echo "AMD_GPU requires the dedicated SIG_IMAGE_NAME 2404gen2amdgpucontainerd" >&2 - return 1 - fi -} - -function get_linux_sku_name() { - validate_amd_gpu_build || return 1 - local sku_name="${OS_VERSION}" - if grep -q "minimal" <<<"$FEATURE_FLAGS"; then sku_name+="minimal"; fi - if [ "${HYPERV_GENERATION,,}" = "v2" ]; then sku_name+="gen2"; fi - if [ "${ARCHITECTURE,,}" = "arm64" ]; then sku_name+="arm64"; fi - if grep -q "NVIDIA_GB" <<<"$FEATURE_FLAGS"; then sku_name+="gb"; fi - if [ "${FEATURE_FLAGS}" = "AMD_GPU" ]; then sku_name+="amdgpu"; fi - if [ "${ENABLE_FIPS,,}" = "true" ]; then sku_name+="fips"; fi - if grep -q "cvm" <<<"$FEATURE_FLAGS"; then sku_name+="CVM"; fi - if [ "${ENABLE_TRUSTED_LAUNCH}" = "True" ]; then sku_name+="TL"; fi - case "${OS_SKU}" in - CBLMariner | AzureLinux | AzureLinuxOSGuard | Flatcar | AzureContainerLinux) ;; - *) sku_name+="containerd" ;; - esac - printf '%s\n' "${sku_name//./}" -} - function compute_msi_resource_strings() { # Populates the caller-declared array named by $1 (an explicit output parameter, via # nameref) with the UAMI resource string to attach to the VHD build VM, or leaves it diff --git a/vhdbuilder/packer/produce-packer-settings.sh b/vhdbuilder/packer/produce-packer-settings.sh index 9fa8036fa92..be6b2817eb6 100755 --- a/vhdbuilder/packer/produce-packer-settings.sh +++ b/vhdbuilder/packer/produce-packer-settings.sh @@ -5,7 +5,12 @@ SCRIPT_DIR=$(dirname "$0") source "$SCRIPT_DIR/produce-packer-settings-functions.sh" if [ "${MODE}" = "linuxVhdMode" ]; then - validate_amd_gpu_build + case "${FEATURE_FLAGS:-}" in + *AMD_GPU*) + source "$SCRIPT_DIR/amd-gpu-build-settings.sh" + validate_amd_gpu_build + ;; + esac fi CDIR=$(dirname "${BASH_SOURCE}") diff --git a/vhdbuilder/packer/test/amd-gpu-vhd-content-test.sh b/vhdbuilder/packer/test/amd-gpu-vhd-content-test.sh new file mode 100644 index 00000000000..ea8a35bfb7d --- /dev/null +++ b/vhdbuilder/packer/test/amd-gpu-vhd-content-test.sh @@ -0,0 +1,153 @@ +#!/bin/bash +# Sourced by linux-vhd-content-test.sh only for the dedicated AMD_GPU image. + +testAMDGPUImage() { + local status=0 + if [ ! -x /opt/azure/containers/amd-gpu-validate.sh ]; then + err testAMDGPUImage "Baked AMD GPU bootstrap validator is missing or not executable" + status=1 + fi + testAMDGPUDriver || status=1 + testAMDGPUDiagnostics || status=1 + return "${status}" +} + +# Hardware-independent validation: the AMD VHD is baked and content-tested on a +# CPU VM. Loading the module and checking KFD/GPU devices belongs to node CSE/E2E. +# shellcheck disable=SC2016 +testAMDGPUDriver() { + [ "${FEATURE_FLAGS:-}" = "AMD_GPU" ] || return 0 + local components_file="${AMD_COMPONENTS_FILEPATH:-/opt/azure/amd-gpu/components.json}" + local test=testAMDGPUDriver marker=/opt/azure/amd-gpu/driver.json + local metadata kernel_version module_path module_vermagic package_version firmware_version module_version dkms_version + local installed_packages modprobe_config package diagnostics_amdsmi diagnostics_sysdeps + if [ "${OS_SKU}" != "Ubuntu" ] || [ "${OS_VERSION}" != "24.04" ] || + [ "${ENABLE_FIPS,,}" = "true" ] || [ "$(uname -m)" != "x86_64" ]; then + err "${test}" "Unsupported AMD GPU image configuration" + return 1 + fi + if ! metadata=$(jq -ce '.AMDGPUDriver | select(type == "object")' "${components_file}") || + ! package_version=$(jq -er '.packageVersion | strings | select(length > 0)' <<< "${metadata}") || + ! firmware_version=$(jq -er '.firmwarePackageVersion | strings | select(length > 0)' <<< "${metadata}") || + ! module_version=$(jq -er '.moduleVersion | strings | select(length > 0)' <<< "${metadata}") || + ! dkms_version=$(jq -er '.dkmsVersion | strings | select(length > 0)' <<< "${metadata}"); then + err "${test}" "Missing AMDGPU component metadata" + return 1 + fi + if ! jq -e --arg package "${package_version}" --arg firmware "${firmware_version}" --arg module_version "${module_version}" \ + '.schema_version == 1 and .package_version == $package and .firmware_package_version == $firmware and + .module_version == $module_version and (.kernel_version | type == "string" and length > 0)' "${marker}" >/dev/null; then + err "${test}" "Missing or inconsistent AMDGPU driver marker" + return 1 + fi + if [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms)" != "install ok installed ${package_version}" ] || + [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms-firmware)" != "install ok installed ${firmware_version}" ]; then + err "${test}" "Installed AMDGPU driver/firmware differs from pinned versions" + return 1 + fi + kernel_version=$(uname -r) + if ! dkms status -m amdgpu -v "${dkms_version}" -k "${kernel_version}" | + grep -Fqx "amdgpu/${dkms_version}, ${kernel_version}, x86_64: installed"; then + err "${test}" "AMDGPU DKMS is not installed for the running kernel" + return 1 + fi + module_path=$(modinfo -k "${kernel_version}" -F filename amdgpu) + case "${module_path}" in + /lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*|/usr/lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*) ;; + *) err "${test}" "AMDGPU resolves to the inbox or a missing module"; return 1 ;; + esac + module_vermagic=$(modinfo -k "${kernel_version}" -F vermagic amdgpu) + if [ "$(modinfo -k "${kernel_version}" -F version amdgpu)" != "${module_version}" ] || + [ "${module_vermagic%% *}" != "${kernel_version}" ]; then + err "${test}" "AMDGPU module version or vermagic mismatch" + return 1 + fi + if [ ! -f "/lib/modules/${kernel_version}/build/Makefile" ] || [ ! -f "/usr/src/amdgpu-${dkms_version}/dkms.conf" ]; then + err "${test}" "Headers or AMDGPU sources required by future DKMS rebuilds are missing" + return 1 + fi + for package in build-essential dkms autoconf automake initramfs-tools "linux-headers-${kernel_version}" "linux-modules-extra-${kernel_version}"; do + if [ "$(dpkg-query -W -f='${Status}' "${package}")" != 'install ok installed' ]; then + err "${test}" "Missing retained build dependency ${package}" + return 1 + fi + done + if ! installed_packages=$(dpkg-query -W -f='${db:Status-Abbrev} ${binary:Package}\n'); then + err "${test}" "Unable to inspect installed packages" + return 1 + fi + if ! diagnostics_amdsmi=$(jq -er '.AMDGPUDiagnostics.amdsmiPackage | strings | select(length > 0)' "${components_file}") || + ! diagnostics_sysdeps=$(jq -er '.AMDGPUDiagnostics.sysdepsPackage | strings | select(length > 0)' "${components_file}"); then + err "${test}" "Missing AMDGPU diagnostics package allowlist" + return 1 + fi + if awk -v amdsmi="${diagnostics_amdsmi}" -v sysdeps="${diagnostics_sysdeps}" \ + '$1 == "ii" { sub(/:.*/, "", $2); if ($2 != amdsmi && $2 != sysdeps) print $2 }' <<< "${installed_packages}" | + grep -Eq '^(amdrocm|rocm|hip|hsa-rocr|rocblas|rocfft|rocrand|rocsolver|rocsparse|miopen|migraphx|amdgpu-(core|lib|install|pro)|lib.*-amdgpu-|nvidia-|libnvidia-|cuda-|datacenter-gpu-manager-|dcgm-exporter)'; then + err "${test}" "Unexpected GPU userspace or NVIDIA package on the AMD driver-and-diagnostics image" + return 1 + fi + if ! modprobe_config=$(modprobe -c) || + grep -Eq '^[[:space:]]*(blacklist|install)[[:space:]]+amdgpu([[:space:]]|$)' <<< "${modprobe_config}"; then + err "${test}" "AMDGPU is disabled in modprobe configuration" + return 1 + fi + echo "${test}: pinned driver and firmware, DKMS module and rebuild dependencies verified" +} + +# AMD SMI initializes GPU drivers even for --help. CPU bake validation checks +# contents and loads the library without amdsmi_init; device discovery runs in GPU E2E. +# shellcheck disable=SC2016 +testAMDGPUDiagnostics() { + [ "${FEATURE_FLAGS:-}" = "AMD_GPU" ] || return 0 + local components_file="${AMD_COMPONENTS_FILEPATH:-/opt/azure/amd-gpu/components.json}" + local test=testAMDGPUDiagnostics metadata amdsmi_package amdsmi_version sysdeps_package sysdeps_version cli_path library_output + local package tool tool_output + if ! metadata=$(jq -ce '.AMDGPUDiagnostics | select(type == "object")' "${components_file}") || + ! amdsmi_package=$(jq -er '.amdsmiPackage | strings | select(length > 0)' <<< "${metadata}") || + ! amdsmi_version=$(jq -er '.amdsmiVersion | strings | select(length > 0)' <<< "${metadata}") || + ! sysdeps_package=$(jq -er '.sysdepsPackage | strings | select(length > 0)' <<< "${metadata}") || + ! sysdeps_version=$(jq -er '.sysdepsVersion | strings | select(length > 0)' <<< "${metadata}") || + ! cli_path=$(jq -er '.cliPath | strings | select(startswith("/opt/rocm/"))' <<< "${metadata}"); then + err "${test}" "Missing AMDGPU diagnostics component metadata" + return 1 + fi + if [ "$(dpkg-query -W -f='${Status} ${Version}' "${amdsmi_package}")" != "install ok installed ${amdsmi_version}" ] || + [ "$(dpkg-query -W -f='${Status} ${Version}' "${sysdeps_package}")" != "install ok installed ${sysdeps_version}" ]; then + err "${test}" "Installed AMD SMI diagnostics differ from pinned packages" + return 1 + fi + for package in pciutils numactl; do + if [ "$(dpkg-query -W -f='${Status}' "${package}")" != 'install ok installed' ]; then + err "${test}" "Missing host diagnostics package ${package}" + return 1 + fi + done + for tool in lspci numactl; do + if ! tool_output=$("${tool}" --version 2>&1); then + err "${test}" "Host diagnostics command ${tool} failed: ${tool_output}" + return 1 + fi + done + if [ ! -x "${cli_path}" ] || [ ! -L /usr/local/bin/amd-smi ] || + [ "$(readlink -f /usr/local/bin/amd-smi)" != "$(readlink -f "${cli_path}")" ]; then + err "${test}" "AMD SMI executable or command symlink is missing or inconsistent" + return 1 + fi + if ! library_output=$(/usr/bin/python3 -I - "${cli_path}" <<'PY' +import json +from pathlib import Path +import sys +# The vendor bin/amd-smi symlink points into libexec; preserve its lexical bin path. +sys.path.insert(0, str(Path(sys.argv[1]).parents[1] / "share/amd_smi")) +import amdsmi +version = amdsmi.amdsmi_get_lib_version() +assert version["major"] > 0, version +print(json.dumps({"python_module": amdsmi.__version__, "library": version})) +PY + ); then + err "${test}" "AMD SMI library failed to load without GPU hardware: ${library_output}" + return 1 + fi + echo "${test}: pinned AMD SMI packages, command symlink, PCI/NUMA tools and hardware-independent library loading verified" +} diff --git a/vhdbuilder/packer/test/linux-vhd-content-test.sh b/vhdbuilder/packer/test/linux-vhd-content-test.sh index 21214b1b839..7637e9d20df 100644 --- a/vhdbuilder/packer/test/linux-vhd-content-test.sh +++ b/vhdbuilder/packer/test/linux-vhd-content-test.sh @@ -2732,144 +2732,6 @@ testContainerNetworkingPluginsInstalled() { return 0 } -# Hardware-independent validation: the AMD VHD is baked and content-tested on a -# CPU VM. Loading the module and checking KFD/GPU devices belongs to node CSE/E2E. -# shellcheck disable=SC2016 -testAMDGPUDriver() { - [ "${FEATURE_FLAGS:-}" = "AMD_GPU" ] || return 0 - local test=testAMDGPUDriver marker=/opt/azure/amd-gpu/driver.json - local metadata kernel_version module_path module_vermagic package_version firmware_version module_version dkms_version - local installed_packages modprobe_config package diagnostics_amdsmi diagnostics_sysdeps - if [ "${OS_SKU}" != "Ubuntu" ] || [ "${OS_VERSION}" != "24.04" ] || - [ "${ENABLE_FIPS,,}" = "true" ] || [ "$(uname -m)" != "x86_64" ]; then - err "${test}" "Unsupported AMD GPU image configuration" - return 1 - fi - if ! metadata=$(jq -ce '.AMDGPUDriver | select(type == "object")' "${COMPONENTS_FILEPATH}") || - ! package_version=$(jq -er '.packageVersion | strings | select(length > 0)' <<< "${metadata}") || - ! firmware_version=$(jq -er '.firmwarePackageVersion | strings | select(length > 0)' <<< "${metadata}") || - ! module_version=$(jq -er '.moduleVersion | strings | select(length > 0)' <<< "${metadata}") || - ! dkms_version=$(jq -er '.dkmsVersion | strings | select(length > 0)' <<< "${metadata}"); then - err "${test}" "Missing AMDGPU component metadata" - return 1 - fi - if ! jq -e --arg package "${package_version}" --arg firmware "${firmware_version}" --arg module_version "${module_version}" \ - '.schema_version == 1 and .package_version == $package and .firmware_package_version == $firmware and - .module_version == $module_version and (.kernel_version | type == "string" and length > 0)' "${marker}" >/dev/null; then - err "${test}" "Missing or inconsistent AMDGPU driver marker" - return 1 - fi - if [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms)" != "install ok installed ${package_version}" ] || - [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms-firmware)" != "install ok installed ${firmware_version}" ]; then - err "${test}" "Installed AMDGPU driver/firmware differs from pinned versions" - return 1 - fi - kernel_version=$(uname -r) - if ! dkms status -m amdgpu -v "${dkms_version}" -k "${kernel_version}" | - grep -Fqx "amdgpu/${dkms_version}, ${kernel_version}, x86_64: installed"; then - err "${test}" "AMDGPU DKMS is not installed for the running kernel" - return 1 - fi - module_path=$(modinfo -k "${kernel_version}" -F filename amdgpu) - case "${module_path}" in - /lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*|/usr/lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*) ;; - *) err "${test}" "AMDGPU resolves to the inbox or a missing module"; return 1 ;; - esac - module_vermagic=$(modinfo -k "${kernel_version}" -F vermagic amdgpu) - if [ "$(modinfo -k "${kernel_version}" -F version amdgpu)" != "${module_version}" ] || - [ "${module_vermagic%% *}" != "${kernel_version}" ]; then - err "${test}" "AMDGPU module version or vermagic mismatch" - return 1 - fi - if [ ! -f "/lib/modules/${kernel_version}/build/Makefile" ] || [ ! -f "/usr/src/amdgpu-${dkms_version}/dkms.conf" ]; then - err "${test}" "Headers or AMDGPU sources required by future DKMS rebuilds are missing" - return 1 - fi - for package in build-essential dkms autoconf automake initramfs-tools "linux-headers-${kernel_version}" "linux-modules-extra-${kernel_version}"; do - if [ "$(dpkg-query -W -f='${Status}' "${package}")" != 'install ok installed' ]; then - err "${test}" "Missing retained build dependency ${package}" - return 1 - fi - done - if ! installed_packages=$(dpkg-query -W -f='${db:Status-Abbrev} ${binary:Package}\n'); then - err "${test}" "Unable to inspect installed packages" - return 1 - fi - if ! diagnostics_amdsmi=$(jq -er '.AMDGPUDiagnostics.amdsmiPackage | strings | select(length > 0)' "${COMPONENTS_FILEPATH}") || - ! diagnostics_sysdeps=$(jq -er '.AMDGPUDiagnostics.sysdepsPackage | strings | select(length > 0)' "${COMPONENTS_FILEPATH}"); then - err "${test}" "Missing AMDGPU diagnostics package allowlist" - return 1 - fi - if awk -v amdsmi="${diagnostics_amdsmi}" -v sysdeps="${diagnostics_sysdeps}" \ - '$1 == "ii" { sub(/:.*/, "", $2); if ($2 != amdsmi && $2 != sysdeps) print $2 }' <<< "${installed_packages}" | - grep -Eq '^(amdrocm|rocm|hip|hsa-rocr|rocblas|rocfft|rocrand|rocsolver|rocsparse|miopen|migraphx|amdgpu-(core|lib|install|pro)|lib.*-amdgpu-|nvidia-|libnvidia-|cuda-|datacenter-gpu-manager-|dcgm-exporter)'; then - err "${test}" "Unexpected GPU userspace or NVIDIA package on the AMD driver-and-diagnostics image" - return 1 - fi - if ! modprobe_config=$(modprobe -c) || - grep -Eq '^[[:space:]]*(blacklist|install)[[:space:]]+amdgpu([[:space:]]|$)' <<< "${modprobe_config}"; then - err "${test}" "AMDGPU is disabled in modprobe configuration" - return 1 - fi - echo "${test}: pinned driver and firmware, DKMS module and rebuild dependencies verified" -} - -# AMD SMI initializes GPU drivers even for --help. CPU bake validation checks -# contents and loads the library without amdsmi_init; device discovery runs in GPU E2E. -# shellcheck disable=SC2016 -testAMDGPUDiagnostics() { - [ "${FEATURE_FLAGS:-}" = "AMD_GPU" ] || return 0 - local test=testAMDGPUDiagnostics metadata amdsmi_package amdsmi_version sysdeps_package sysdeps_version cli_path library_output - local package tool tool_output - if ! metadata=$(jq -ce '.AMDGPUDiagnostics | select(type == "object")' "${COMPONENTS_FILEPATH}") || - ! amdsmi_package=$(jq -er '.amdsmiPackage | strings | select(length > 0)' <<< "${metadata}") || - ! amdsmi_version=$(jq -er '.amdsmiVersion | strings | select(length > 0)' <<< "${metadata}") || - ! sysdeps_package=$(jq -er '.sysdepsPackage | strings | select(length > 0)' <<< "${metadata}") || - ! sysdeps_version=$(jq -er '.sysdepsVersion | strings | select(length > 0)' <<< "${metadata}") || - ! cli_path=$(jq -er '.cliPath | strings | select(startswith("/opt/rocm/"))' <<< "${metadata}"); then - err "${test}" "Missing AMDGPU diagnostics component metadata" - return 1 - fi - if [ "$(dpkg-query -W -f='${Status} ${Version}' "${amdsmi_package}")" != "install ok installed ${amdsmi_version}" ] || - [ "$(dpkg-query -W -f='${Status} ${Version}' "${sysdeps_package}")" != "install ok installed ${sysdeps_version}" ]; then - err "${test}" "Installed AMD SMI diagnostics differ from pinned packages" - return 1 - fi - for package in pciutils numactl; do - if [ "$(dpkg-query -W -f='${Status}' "${package}")" != 'install ok installed' ]; then - err "${test}" "Missing host diagnostics package ${package}" - return 1 - fi - done - for tool in lspci numactl; do - if ! tool_output=$("${tool}" --version 2>&1); then - err "${test}" "Host diagnostics command ${tool} failed: ${tool_output}" - return 1 - fi - done - if [ ! -x "${cli_path}" ] || [ ! -L /usr/local/bin/amd-smi ] || - [ "$(readlink -f /usr/local/bin/amd-smi)" != "$(readlink -f "${cli_path}")" ]; then - err "${test}" "AMD SMI executable or command symlink is missing or inconsistent" - return 1 - fi - if ! library_output=$(/usr/bin/python3 -I - "${cli_path}" <<'PY' -import json -from pathlib import Path -import sys -# The vendor bin/amd-smi symlink points into libexec; preserve its lexical bin path. -sys.path.insert(0, str(Path(sys.argv[1]).parents[1] / "share/amd_smi")) -import amdsmi -version = amdsmi.amdsmi_get_lib_version() -assert version["major"] > 0, version -print(json.dumps({"python_module": amdsmi.__version__, "library": version})) -PY - ); then - err "${test}" "AMD SMI library failed to load without GPU hardware: ${library_output}" - return 1 - fi - echo "${test}: pinned AMD SMI packages, command symlink, PCI/NUMA tools and hardware-independent library loading verified" -} - # As we call these tests, we need to bear in mind how the test results are processed by the # the caller in run-tests.sh. That code uses az vm run-command invoke to run this script # on a VM. It then looks at stderr to see if any errors were reported. Notably it doesn't @@ -2940,5 +2802,7 @@ testFileOwnership $OS_SKU testDiskQueueServiceIsActive testVulnerableKernelModulesDisabled $OS_SKU $OS_VERSION testArtifactStreamingPackagesCleanedUp -testAMDGPUDriver -testAMDGPUDiagnostics +if [ "${FEATURE_FLAGS:-}" = "AMD_GPU" ]; then + source ./AgentBaker/vhdbuilder/packer/test/amd-gpu-vhd-content-test.sh && + testAMDGPUImage || err testAMDGPUImage "AMD VHD content validation failed" +fi diff --git a/vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh b/vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh new file mode 100644 index 00000000000..77077f72f48 --- /dev/null +++ b/vhdbuilder/scripts/linux/ubuntu/amd_gpu.sh @@ -0,0 +1,254 @@ +#!/bin/bash +# Dedicated AMD_GPU Ubuntu VHD support. Sourced only by the AMD image build. + +validateAMDGPUImageConfiguration() { + local generation=${HYPERV_GENERATION:-} enable_fips=${ENABLE_FIPS:-false} + if [ "${FEATURE_FLAGS:-}" != "AMD_GPU" ] || [ "${OS:-}" != "UBUNTU" ] || + [ "${OS_VERSION:-}" != "24.04" ] || [ "${CPU_ARCH:-}" != "amd64" ] || + [ "${generation,,}" != "v2" ] || [ "${enable_fips,,}" = "true" ]; then + echo "AMD_GPU requires the dedicated Ubuntu 24.04 amd64 Gen2 non-FIPS image" >&2 + return 1 + fi +} + +isAMDGPUSkippedPackage() { + case "$1" in + nvidia-*|dra-driver-nvidia-*|datacenter-gpu-manager-*|dcgm-exporter) return 0 ;; + *) return 1 ;; + esac +} + +installAMDGPUImage() { + local components_file="${AMD_COMPONENTS_FILEPATH:-/opt/azure/amd-gpu/components.json}" + validateAMDGPUImageConfiguration || return 1 + install -Dm0644 /home/packer/amd-gpu-components.json "${components_file}" || return 1 + installAMDGPUDriver || return 1 + capture_benchmark "${SCRIPT_NAME}_build_amd_gpu_kernel_module" || return 1 + installAMDGPUDiagnostics || return 1 + capture_benchmark "${SCRIPT_NAME}_install_amd_gpu_diagnostics" || return 1 + install -Dm0755 /home/packer/amd-gpu-validate.sh /opt/azure/containers/amd-gpu-validate.sh || return 1 +} + +# Bake on a CPU VM: keep DKMS and its build dependencies for kernel servicing, +# but never load the driver or install the ROCm compute SDK on the host. +# The subshell owns its cleanup trap without replacing the caller's traps. +installAMDGPUDriver() { + ( + set -o pipefail + local components_file="${AMD_COMPONENTS_FILEPATH:-/opt/azure/amd-gpu/components.json}" + local metadata repository key_url fingerprint package_version firmware_version module_version dkms_version + local kernel_version work_dir marker_dir=/opt/azure/amd-gpu marker_tmp="" + local generation=${HYPERV_GENERATION:-} enable_fips=${ENABLE_FIPS:-false} + local actual_fingerprint archive package_name package_arch module_path module_vermagic modprobe_config conf + local driver_deb="" firmware_deb="" + local -a apt_options build_packages + if [ "${OS:-}" != "UBUNTU" ] || [ "${OS_VERSION:-}" != "24.04" ] || + [ "${CPU_ARCH:-}" != "amd64" ] || [ "${generation,,}" != "v2" ] || + [ "${enable_fips,,}" = "true" ]; then + echo "AMDGPU requires Ubuntu 24.04 amd64 Gen2 without FIPS" >&2 + exit 1 + fi + # A failed re-bake must not leave a success marker from an earlier attempt. + rm -f "${marker_dir}/driver.json" || exit 1 + metadata=$(jq -ce '.AMDGPUDriver | select(type == "object")' "${components_file}") || exit 1 + repository=$(jq -er '.repositoryURL | select(test("^https://repo[.]radeon[.]com/amdgpu/[0-9.]+/ubuntu$"))' <<< "${metadata}") || exit 1 + key_url=$(jq -er '.signingKeyURL | select(test("^https://repo[.]radeon[.]com/[^?[:space:]]+$"))' <<< "${metadata}") || exit 1 + fingerprint=$(jq -er '.signingKeyFingerprint | select(test("^[A-F0-9]{40}$"))' <<< "${metadata}") || exit 1 + jq -e '.distribution == "noble" and .component == "main"' <<< "${metadata}" >/dev/null || exit 1 + package_version=$(jq -er '.packageVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 + firmware_version=$(jq -er '.firmwarePackageVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 + module_version=$(jq -er '.moduleVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 + dkms_version=$(jq -er '.dkmsVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 + kernel_version=$(uname -r) || exit 1 + work_dir=$(mktemp -d /tmp/amd-gpu.XXXXXX) || exit 1 + trap 'rm -rf "${work_dir}"; if [ -n "${marker_tmp}" ]; then rm -f "${marker_tmp}"; fi' EXIT + build_packages=(ca-certificates curl gnupg build-essential dkms autoconf automake initramfs-tools + "linux-headers-${kernel_version}" "linux-modules-extra-${kernel_version}") + apt_get_update || exit 1 + apt_get_install 20 5 1800 "${build_packages[@]}" || exit 1 + test -f "/lib/modules/${kernel_version}/build/Makefile" || exit 1 + + mkdir -p "${work_dir}/gnupg" "${work_dir}/lists/partial" "${work_dir}/packages" || exit 1 + chmod 755 "${work_dir}" || exit 1 + chmod 700 "${work_dir}/gnupg" || exit 1 + retrycmd_if_failure 5 5 60 curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' \ + "${key_url}" -o "${work_dir}/key.asc" || exit 1 + # Compare primary fingerprints only; the legitimate key also has an encryption subkey. + actual_fingerprint=$(gpg --batch --homedir "${work_dir}/gnupg" --show-keys --with-colons "${work_dir}/key.asc" | + awk -F: '$1 == "pub" { primary = 1; next } primary && $1 == "fpr" { print $10; primary = 0 }') || exit 1 + if [ "${actual_fingerprint}" != "${fingerprint}" ]; then + echo "AMDGPU signing key fingerprint mismatch" >&2 + exit 1 + fi + gpg --batch --homedir "${work_dir}/gnupg" --dearmor --output "${work_dir}/key.gpg" "${work_dir}/key.asc" || exit 1 + chmod 644 "${work_dir}/key.gpg" || exit 1 + printf 'deb [arch=amd64 signed-by=%s/key.gpg] %s noble main\n' "${work_dir}" "${repository}" > "${work_dir}/amdgpu.list" || exit 1 + # AMD's signed metadata is isolated from normal APT state. Only these two + # exact packages come from AMD; Ubuntu supplies all remaining dependencies. + apt_options=(-o "Dir::Etc::sourcelist=${work_dir}/amdgpu.list" -o Dir::Etc::sourceparts=- + -o "Dir::State::lists=${work_dir}/lists" -o APT::Get::AllowUnauthenticated=false + -o Acquire::AllowInsecureRepositories=false -o Acquire::AllowDowngradeToInsecureRepositories=false) + retrycmd_if_failure 5 5 300 apt-get "${apt_options[@]}" update || exit 1 + cd "${work_dir}/packages" || exit 1 + retrycmd_if_failure 5 5 300 apt-get "${apt_options[@]}" download \ + "amdgpu-dkms=${package_version}" "amdgpu-dkms-firmware=${firmware_version}" || exit 1 + for archive in "${work_dir}/packages/"*.deb; do + package_name=$(dpkg-deb -f "${archive}" Package) || exit 1 + package_arch=$(dpkg-deb -f "${archive}" Architecture) || exit 1 + case "${package_arch}" in all|amd64) ;; *) echo "Unexpected AMDGPU package architecture" >&2; exit 1 ;; esac + case "${package_name}" in + amdgpu-dkms) + [ -z "${driver_deb}" ] && [ "$(dpkg-deb -f "${archive}" Version)" = "${package_version}" ] || exit 1 + driver_deb=${archive} + ;; + amdgpu-dkms-firmware) + [ -z "${firmware_deb}" ] && [ "$(dpkg-deb -f "${archive}" Version)" = "${firmware_version}" ] || exit 1 + firmware_deb=${archive} + ;; + *) echo "Unexpected AMDGPU package: ${package_name}" >&2; exit 1 ;; + esac + done + [ -n "${driver_deb}" ] && [ -n "${firmware_deb}" ] || exit 1 + apt_get_install 20 5 1800 "${driver_deb}" "${firmware_deb}" || exit 1 + # Keep the compiler, matching headers, firmware and DKMS sources through + # image cleanup. No vendor repository/key is persisted on the image. + apt-mark manual "${build_packages[@]}" amdgpu-dkms amdgpu-dkms-firmware || exit 1 + # shellcheck disable=SC2016 + [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms)" = "install ok installed ${package_version}" ] || exit 1 + # shellcheck disable=SC2016 + [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms-firmware)" = "install ok installed ${firmware_version}" ] || exit 1 + dkms install -m amdgpu -v "${dkms_version}" -k "${kernel_version}" || exit 1 + dkms status -m amdgpu -v "${dkms_version}" -k "${kernel_version}" | grep -Fqx "amdgpu/${dkms_version}, ${kernel_version}, x86_64: installed" || exit 1 + depmod -a "${kernel_version}" || exit 1 + module_path=$(modinfo -k "${kernel_version}" -F filename amdgpu) || exit 1 + case "${module_path}" in + /lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*|/usr/lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*) ;; + *) echo "AMDGPU resolves to an unexpected module: ${module_path}" >&2; exit 1 ;; + esac + [ "$(modinfo -k "${kernel_version}" -F version amdgpu)" = "${module_version}" ] || exit 1 + module_vermagic=$(modinfo -k "${kernel_version}" -F vermagic amdgpu) || exit 1 + [ "${module_vermagic%% *}" = "${kernel_version}" ] || exit 1 + + # Azure's CPU base image may blacklist the inbox GPU driver. Remove only + # the AMD entry, retaining unrelated cloud-image module deny rules. + for conf in /etc/modprobe.d/*.conf /usr/lib/modprobe.d/*.conf; do + [ -f "${conf}" ] || continue + sed -i -E '/^[[:space:]]*blacklist[[:space:]]+amdgpu([[:space:]]|$)/d' "${conf}" || exit 1 + done + modprobe_config=$(modprobe -c) || exit 1 + if grep -Eq '^[[:space:]]*(blacklist|install)[[:space:]]+amdgpu([[:space:]]|$)' <<< "${modprobe_config}"; then + echo "AMDGPU remains disabled in modprobe configuration" >&2 + exit 1 + fi + update-initramfs -u -k "${kernel_version}" || exit 1 + mkdir -p "${marker_dir}" || exit 1 + marker_tmp=$(mktemp "${marker_dir}/.driver.json.XXXXXX") || exit 1 + jq -n --arg package_version "${package_version}" --arg firmware_package_version "${firmware_version}" \ + --arg module_version "${module_version}" --arg kernel_version "${kernel_version}" \ + '{schema_version: 1, package_version: $package_version, firmware_package_version: $firmware_package_version, + module_version: $module_version, kernel_version: $kernel_version}' > "${marker_tmp}" || exit 1 + chmod 644 "${marker_tmp}" || exit 1 + echo " - amdgpu-dkms version ${package_version}; module ${module_version}; kernel ${kernel_version}" >> "${VHD_LOGS_FILEPATH}" || exit 1 + mv -f "${marker_tmp}" "${marker_dir}/driver.json" || exit 1 + ) +} + +# Host diagnostics are intentionally separate from the compute SDK. AMD SMI +# uses only its management library and sysdeps package; no pip installation. +installAMDGPUDiagnostics() { + ( + set -o pipefail + local components_file="${AMD_COMPONENTS_FILEPATH:-/opt/azure/amd-gpu/components.json}" + local metadata repository key_url fingerprint actual_fingerprint work_dir + local amdsmi_package amdsmi_version sysdeps_package sysdeps_version cli_path python_path + local archive package_name package_arch amdsmi_deb="" sysdeps_deb="" + local generation=${HYPERV_GENERATION:-} enable_fips=${ENABLE_FIPS:-false} + local -a apt_options ubuntu_packages + if [ "${OS:-}" != "UBUNTU" ] || [ "${OS_VERSION:-}" != "24.04" ] || + [ "${CPU_ARCH:-}" != "amd64" ] || [ "${generation,,}" != "v2" ] || + [ "${enable_fips,,}" = "true" ]; then + echo "AMD GPU diagnostics require Ubuntu 24.04 amd64 Gen2 without FIPS" >&2 + exit 1 + fi + metadata=$(jq -ce '.AMDGPUDiagnostics | select(type == "object")' "${components_file}") || exit 1 + repository=$(jq -er '.repositoryURL | select(. == "https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/")' <<< "${metadata}") || exit 1 + key_url=$(jq -er '.signingKeyURL | select(. == "https://stable.repo.amd.com/rocm/gpg/packages.gpg")' <<< "${metadata}") || exit 1 + fingerprint=$(jq -er '.signingKeyFingerprint | select(test("^[A-F0-9]{40}$"))' <<< "${metadata}") || exit 1 + jq -e '.distribution == "stable" and .component == "main"' <<< "${metadata}" >/dev/null || exit 1 + amdsmi_package=$(jq -er '.amdsmiPackage | select(test("^amdrocm-amdsmi[0-9]+[.][0-9]+$"))' <<< "${metadata}") || exit 1 + sysdeps_package=$(jq -er '.sysdepsPackage | select(test("^amdrocm-sysdeps[0-9]+[.][0-9]+$"))' <<< "${metadata}") || exit 1 + amdsmi_version=$(jq -er '.amdsmiVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 + sysdeps_version=$(jq -er '.sysdepsVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 + cli_path=$(jq -er '.cliPath | select(test("^/opt/rocm/core-[0-9]+[.][0-9]+/bin/amd-smi$"))' <<< "${metadata}") || exit 1 + python_path="${cli_path%/bin/amd-smi}/share/amd_smi" + work_dir=$(mktemp -d /tmp/amd-gpu-diagnostics.XXXXXX) || exit 1 + trap 'rm -rf "${work_dir}"' EXIT + # The vendor package omits the C++ runtime dependency from its metadata. + ubuntu_packages=(ca-certificates curl gnupg python3 libstdc++6 libgcc-s1 pciutils numactl) + apt_get_update || exit 1 + apt_get_install 20 5 600 "${ubuntu_packages[@]}" || exit 1 + mkdir -p "${work_dir}/gnupg" "${work_dir}/lists/partial" "${work_dir}/packages" || exit 1 + chmod 755 "${work_dir}" || exit 1 + chmod 700 "${work_dir}/gnupg" || exit 1 + retrycmd_if_failure 5 5 60 curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' \ + "${key_url}" -o "${work_dir}/key.asc" || exit 1 + actual_fingerprint=$(gpg --batch --homedir "${work_dir}/gnupg" --show-keys --with-colons "${work_dir}/key.asc" | + awk -F: '$1 == "pub" { primary = 1; next } primary && $1 == "fpr" { print $10; primary = 0 }') || exit 1 + if [ "${actual_fingerprint}" != "${fingerprint}" ]; then + echo "AMD GPU diagnostics signing key fingerprint mismatch" >&2 + exit 1 + fi + gpg --batch --homedir "${work_dir}/gnupg" --dearmor --output "${work_dir}/key.gpg" "${work_dir}/key.asc" || exit 1 + chmod 644 "${work_dir}/key.gpg" || exit 1 + printf 'deb [arch=amd64 signed-by=%s/key.gpg] %s stable main\n' "${work_dir}" "${repository}" > "${work_dir}/diagnostics.list" || exit 1 + apt_options=(-o "Dir::Etc::sourcelist=${work_dir}/diagnostics.list" -o Dir::Etc::sourceparts=- + -o "Dir::State::lists=${work_dir}/lists" -o APT::Get::AllowUnauthenticated=false + -o Acquire::AllowInsecureRepositories=false -o Acquire::AllowDowngradeToInsecureRepositories=false) + retrycmd_if_failure 5 5 300 apt-get "${apt_options[@]}" update || exit 1 + cd "${work_dir}/packages" || exit 1 + retrycmd_if_failure 5 5 300 apt-get "${apt_options[@]}" download \ + "${amdsmi_package}=${amdsmi_version}" "${sysdeps_package}=${sysdeps_version}" || exit 1 + for archive in "${work_dir}/packages/"*.deb; do + package_name=$(dpkg-deb -f "${archive}" Package) || exit 1 + package_arch=$(dpkg-deb -f "${archive}" Architecture) || exit 1 + [ "${package_arch}" = amd64 ] || { echo "Unexpected AMD diagnostics package architecture" >&2; exit 1; } + case "${package_name}" in + "${amdsmi_package}") + [ -z "${amdsmi_deb}" ] && [ "$(dpkg-deb -f "${archive}" Version)" = "${amdsmi_version}" ] || exit 1 + amdsmi_deb=${archive} + ;; + "${sysdeps_package}") + [ -z "${sysdeps_deb}" ] && [ "$(dpkg-deb -f "${archive}" Version)" = "${sysdeps_version}" ] || exit 1 + sysdeps_deb=${archive} + ;; + *) echo "Unexpected AMD diagnostics package: ${package_name}" >&2; exit 1 ;; + esac + done + [ -n "${amdsmi_deb}" ] && [ -n "${sysdeps_deb}" ] || exit 1 + apt_get_install 20 5 600 "${amdsmi_deb}" "${sysdeps_deb}" || exit 1 + apt-mark manual "${ubuntu_packages[@]}" "${amdsmi_package}" "${sysdeps_package}" || exit 1 + # shellcheck disable=SC2016 + [ "$(dpkg-query -W -f='${Status} ${Version}' "${amdsmi_package}")" = "install ok installed ${amdsmi_version}" ] || exit 1 + # shellcheck disable=SC2016 + [ "$(dpkg-query -W -f='${Status} ${Version}' "${sysdeps_package}")" = "install ok installed ${sysdeps_version}" ] || exit 1 + [ -x "${cli_path}" ] || exit 1 + # Even --help initializes hardware in AMD SMI. Verify the Python bindings + # and shared-library dependency closure without initialization on CPU bakes. + python3 -I - "${python_path}" <<'AMDSMI_CHECK' || exit 1 +import json +import sys +sys.path.insert(0, sys.argv[1]) +import amdsmi +version = amdsmi.amdsmi_get_lib_version() +assert version["major"] > 0, version +print(json.dumps({"amd_smi_library": version, "python_module": amdsmi.__version__})) +AMDSMI_CHECK + if [ -e /usr/local/bin/amd-smi ] && [ ! -L /usr/local/bin/amd-smi ]; then + echo "Refusing to replace a non-symlink at /usr/local/bin/amd-smi" >&2 + exit 1 + fi + mkdir -p /usr/local/bin || exit 1 + ln -sfnT "${cli_path}" /usr/local/bin/amd-smi || exit 1 + echo " - ${amdsmi_package} version ${amdsmi_version}; ${sysdeps_package} version ${sysdeps_version}" >> "${VHD_LOGS_FILEPATH}" || exit 1 + ) +} diff --git a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh index 3c54b6aedf4..4d6cba7f9e5 100755 --- a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh +++ b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh @@ -20,228 +20,6 @@ ERR_CHRONY_START_TIMEOUT=15 # Unable to start CHRONY echo "Sourcing tool_installs_ubuntu.sh" -# Bake on a CPU VM: keep DKMS and its build dependencies for kernel servicing, -# but never load the driver or install the ROCm compute SDK on the host. -# The subshell owns its cleanup trap without replacing the caller's traps. -installAMDGPUDriver() { - ( - set -o pipefail - local metadata repository key_url fingerprint package_version firmware_version module_version dkms_version - local kernel_version work_dir marker_dir=/opt/azure/amd-gpu marker_tmp="" - local generation=${HYPERV_GENERATION:-} enable_fips=${ENABLE_FIPS:-false} - local actual_fingerprint archive package_name package_arch module_path module_vermagic modprobe_config conf - local driver_deb="" firmware_deb="" - local -a apt_options build_packages - if [ "${OS:-}" != "UBUNTU" ] || [ "${OS_VERSION:-}" != "24.04" ] || - [ "${CPU_ARCH:-}" != "amd64" ] || [ "${generation,,}" != "v2" ] || - [ "${enable_fips,,}" = "true" ]; then - echo "AMDGPU requires Ubuntu 24.04 amd64 Gen2 without FIPS" >&2 - exit 1 - fi - # A failed re-bake must not leave a success marker from an earlier attempt. - rm -f "${marker_dir}/driver.json" || exit 1 - metadata=$(jq -ce '.AMDGPUDriver | select(type == "object")' "${COMPONENTS_FILEPATH}") || exit 1 - repository=$(jq -er '.repositoryURL | select(test("^https://repo[.]radeon[.]com/amdgpu/[0-9.]+/ubuntu$"))' <<< "${metadata}") || exit 1 - key_url=$(jq -er '.signingKeyURL | select(test("^https://repo[.]radeon[.]com/[^?[:space:]]+$"))' <<< "${metadata}") || exit 1 - fingerprint=$(jq -er '.signingKeyFingerprint | select(test("^[A-F0-9]{40}$"))' <<< "${metadata}") || exit 1 - jq -e '.distribution == "noble" and .component == "main"' <<< "${metadata}" >/dev/null || exit 1 - package_version=$(jq -er '.packageVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 - firmware_version=$(jq -er '.firmwarePackageVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 - module_version=$(jq -er '.moduleVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 - dkms_version=$(jq -er '.dkmsVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 - kernel_version=$(uname -r) || exit 1 - work_dir=$(mktemp -d /tmp/amd-gpu.XXXXXX) || exit 1 - trap 'rm -rf "${work_dir}"; if [ -n "${marker_tmp}" ]; then rm -f "${marker_tmp}"; fi' EXIT - build_packages=(ca-certificates curl gnupg build-essential dkms autoconf automake initramfs-tools - "linux-headers-${kernel_version}" "linux-modules-extra-${kernel_version}") - apt_get_update || exit 1 - apt_get_install 20 5 1800 "${build_packages[@]}" || exit 1 - test -f "/lib/modules/${kernel_version}/build/Makefile" || exit 1 - - mkdir -p "${work_dir}/gnupg" "${work_dir}/lists/partial" "${work_dir}/packages" || exit 1 - chmod 755 "${work_dir}" || exit 1 - chmod 700 "${work_dir}/gnupg" || exit 1 - retrycmd_if_failure 5 5 60 curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' \ - "${key_url}" -o "${work_dir}/key.asc" || exit 1 - # Compare primary fingerprints only; the legitimate key also has an encryption subkey. - actual_fingerprint=$(gpg --batch --homedir "${work_dir}/gnupg" --show-keys --with-colons "${work_dir}/key.asc" | - awk -F: '$1 == "pub" { primary = 1; next } primary && $1 == "fpr" { print $10; primary = 0 }') || exit 1 - if [ "${actual_fingerprint}" != "${fingerprint}" ]; then - echo "AMDGPU signing key fingerprint mismatch" >&2 - exit 1 - fi - gpg --batch --homedir "${work_dir}/gnupg" --dearmor --output "${work_dir}/key.gpg" "${work_dir}/key.asc" || exit 1 - chmod 644 "${work_dir}/key.gpg" || exit 1 - printf 'deb [arch=amd64 signed-by=%s/key.gpg] %s noble main\n' "${work_dir}" "${repository}" > "${work_dir}/amdgpu.list" || exit 1 - # AMD's signed metadata is isolated from normal APT state. Only these two - # exact packages come from AMD; Ubuntu supplies all remaining dependencies. - apt_options=(-o "Dir::Etc::sourcelist=${work_dir}/amdgpu.list" -o Dir::Etc::sourceparts=- - -o "Dir::State::lists=${work_dir}/lists" -o APT::Get::AllowUnauthenticated=false - -o Acquire::AllowInsecureRepositories=false -o Acquire::AllowDowngradeToInsecureRepositories=false) - retrycmd_if_failure 5 5 300 apt-get "${apt_options[@]}" update || exit 1 - cd "${work_dir}/packages" || exit 1 - retrycmd_if_failure 5 5 300 apt-get "${apt_options[@]}" download \ - "amdgpu-dkms=${package_version}" "amdgpu-dkms-firmware=${firmware_version}" || exit 1 - for archive in "${work_dir}/packages/"*.deb; do - package_name=$(dpkg-deb -f "${archive}" Package) || exit 1 - package_arch=$(dpkg-deb -f "${archive}" Architecture) || exit 1 - case "${package_arch}" in all|amd64) ;; *) echo "Unexpected AMDGPU package architecture" >&2; exit 1 ;; esac - case "${package_name}" in - amdgpu-dkms) - [ -z "${driver_deb}" ] && [ "$(dpkg-deb -f "${archive}" Version)" = "${package_version}" ] || exit 1 - driver_deb=${archive} - ;; - amdgpu-dkms-firmware) - [ -z "${firmware_deb}" ] && [ "$(dpkg-deb -f "${archive}" Version)" = "${firmware_version}" ] || exit 1 - firmware_deb=${archive} - ;; - *) echo "Unexpected AMDGPU package: ${package_name}" >&2; exit 1 ;; - esac - done - [ -n "${driver_deb}" ] && [ -n "${firmware_deb}" ] || exit 1 - apt_get_install 20 5 1800 "${driver_deb}" "${firmware_deb}" || exit 1 - # Keep the compiler, matching headers, firmware and DKMS sources through - # image cleanup. No vendor repository/key is persisted on the image. - apt-mark manual "${build_packages[@]}" amdgpu-dkms amdgpu-dkms-firmware || exit 1 - # shellcheck disable=SC2016 - [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms)" = "install ok installed ${package_version}" ] || exit 1 - # shellcheck disable=SC2016 - [ "$(dpkg-query -W -f='${Status} ${Version}' amdgpu-dkms-firmware)" = "install ok installed ${firmware_version}" ] || exit 1 - dkms install -m amdgpu -v "${dkms_version}" -k "${kernel_version}" || exit 1 - dkms status -m amdgpu -v "${dkms_version}" -k "${kernel_version}" | grep -Fqx "amdgpu/${dkms_version}, ${kernel_version}, x86_64: installed" || exit 1 - depmod -a "${kernel_version}" || exit 1 - module_path=$(modinfo -k "${kernel_version}" -F filename amdgpu) || exit 1 - case "${module_path}" in - /lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*|/usr/lib/modules/"${kernel_version}"/updates/dkms/amdgpu.ko*) ;; - *) echo "AMDGPU resolves to an unexpected module: ${module_path}" >&2; exit 1 ;; - esac - [ "$(modinfo -k "${kernel_version}" -F version amdgpu)" = "${module_version}" ] || exit 1 - module_vermagic=$(modinfo -k "${kernel_version}" -F vermagic amdgpu) || exit 1 - [ "${module_vermagic%% *}" = "${kernel_version}" ] || exit 1 - - # Azure's CPU base image may blacklist the inbox GPU driver. Remove only - # the AMD entry, retaining unrelated cloud-image module deny rules. - for conf in /etc/modprobe.d/*.conf /usr/lib/modprobe.d/*.conf; do - [ -f "${conf}" ] || continue - sed -i -E '/^[[:space:]]*blacklist[[:space:]]+amdgpu([[:space:]]|$)/d' "${conf}" || exit 1 - done - modprobe_config=$(modprobe -c) || exit 1 - if grep -Eq '^[[:space:]]*(blacklist|install)[[:space:]]+amdgpu([[:space:]]|$)' <<< "${modprobe_config}"; then - echo "AMDGPU remains disabled in modprobe configuration" >&2 - exit 1 - fi - update-initramfs -u -k "${kernel_version}" || exit 1 - mkdir -p "${marker_dir}" || exit 1 - marker_tmp=$(mktemp "${marker_dir}/.driver.json.XXXXXX") || exit 1 - jq -n --arg package_version "${package_version}" --arg firmware_package_version "${firmware_version}" \ - --arg module_version "${module_version}" --arg kernel_version "${kernel_version}" \ - '{schema_version: 1, package_version: $package_version, firmware_package_version: $firmware_package_version, - module_version: $module_version, kernel_version: $kernel_version}' > "${marker_tmp}" || exit 1 - chmod 644 "${marker_tmp}" || exit 1 - echo " - amdgpu-dkms version ${package_version}; module ${module_version}; kernel ${kernel_version}" >> "${VHD_LOGS_FILEPATH}" || exit 1 - mv -f "${marker_tmp}" "${marker_dir}/driver.json" || exit 1 - ) -} - -# Host diagnostics are intentionally separate from the compute SDK. AMD SMI -# uses only its management library and sysdeps package; no pip installation. -installAMDGPUDiagnostics() { - ( - set -o pipefail - local metadata repository key_url fingerprint actual_fingerprint work_dir - local amdsmi_package amdsmi_version sysdeps_package sysdeps_version cli_path python_path - local archive package_name package_arch amdsmi_deb="" sysdeps_deb="" - local generation=${HYPERV_GENERATION:-} enable_fips=${ENABLE_FIPS:-false} - local -a apt_options ubuntu_packages - if [ "${OS:-}" != "UBUNTU" ] || [ "${OS_VERSION:-}" != "24.04" ] || - [ "${CPU_ARCH:-}" != "amd64" ] || [ "${generation,,}" != "v2" ] || - [ "${enable_fips,,}" = "true" ]; then - echo "AMD GPU diagnostics require Ubuntu 24.04 amd64 Gen2 without FIPS" >&2 - exit 1 - fi - metadata=$(jq -ce '.AMDGPUDiagnostics | select(type == "object")' "${COMPONENTS_FILEPATH}") || exit 1 - repository=$(jq -er '.repositoryURL | select(. == "https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/")' <<< "${metadata}") || exit 1 - key_url=$(jq -er '.signingKeyURL | select(. == "https://stable.repo.amd.com/rocm/gpg/packages.gpg")' <<< "${metadata}") || exit 1 - fingerprint=$(jq -er '.signingKeyFingerprint | select(test("^[A-F0-9]{40}$"))' <<< "${metadata}") || exit 1 - jq -e '.distribution == "stable" and .component == "main"' <<< "${metadata}" >/dev/null || exit 1 - amdsmi_package=$(jq -er '.amdsmiPackage | select(test("^amdrocm-amdsmi[0-9]+[.][0-9]+$"))' <<< "${metadata}") || exit 1 - sysdeps_package=$(jq -er '.sysdepsPackage | select(test("^amdrocm-sysdeps[0-9]+[.][0-9]+$"))' <<< "${metadata}") || exit 1 - amdsmi_version=$(jq -er '.amdsmiVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 - sysdeps_version=$(jq -er '.sysdepsVersion | strings | select(length > 0)' <<< "${metadata}") || exit 1 - cli_path=$(jq -er '.cliPath | select(test("^/opt/rocm/core-[0-9]+[.][0-9]+/bin/amd-smi$"))' <<< "${metadata}") || exit 1 - python_path="${cli_path%/bin/amd-smi}/share/amd_smi" - work_dir=$(mktemp -d /tmp/amd-gpu-diagnostics.XXXXXX) || exit 1 - trap 'rm -rf "${work_dir}"' EXIT - # The vendor package omits the C++ runtime dependency from its metadata. - ubuntu_packages=(ca-certificates curl gnupg python3 libstdc++6 libgcc-s1 pciutils numactl) - apt_get_update || exit 1 - apt_get_install 20 5 600 "${ubuntu_packages[@]}" || exit 1 - mkdir -p "${work_dir}/gnupg" "${work_dir}/lists/partial" "${work_dir}/packages" || exit 1 - chmod 755 "${work_dir}" || exit 1 - chmod 700 "${work_dir}/gnupg" || exit 1 - retrycmd_if_failure 5 5 60 curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' \ - "${key_url}" -o "${work_dir}/key.asc" || exit 1 - actual_fingerprint=$(gpg --batch --homedir "${work_dir}/gnupg" --show-keys --with-colons "${work_dir}/key.asc" | - awk -F: '$1 == "pub" { primary = 1; next } primary && $1 == "fpr" { print $10; primary = 0 }') || exit 1 - if [ "${actual_fingerprint}" != "${fingerprint}" ]; then - echo "AMD GPU diagnostics signing key fingerprint mismatch" >&2 - exit 1 - fi - gpg --batch --homedir "${work_dir}/gnupg" --dearmor --output "${work_dir}/key.gpg" "${work_dir}/key.asc" || exit 1 - chmod 644 "${work_dir}/key.gpg" || exit 1 - printf 'deb [arch=amd64 signed-by=%s/key.gpg] %s stable main\n' "${work_dir}" "${repository}" > "${work_dir}/diagnostics.list" || exit 1 - apt_options=(-o "Dir::Etc::sourcelist=${work_dir}/diagnostics.list" -o Dir::Etc::sourceparts=- - -o "Dir::State::lists=${work_dir}/lists" -o APT::Get::AllowUnauthenticated=false - -o Acquire::AllowInsecureRepositories=false -o Acquire::AllowDowngradeToInsecureRepositories=false) - retrycmd_if_failure 5 5 300 apt-get "${apt_options[@]}" update || exit 1 - cd "${work_dir}/packages" || exit 1 - retrycmd_if_failure 5 5 300 apt-get "${apt_options[@]}" download \ - "${amdsmi_package}=${amdsmi_version}" "${sysdeps_package}=${sysdeps_version}" || exit 1 - for archive in "${work_dir}/packages/"*.deb; do - package_name=$(dpkg-deb -f "${archive}" Package) || exit 1 - package_arch=$(dpkg-deb -f "${archive}" Architecture) || exit 1 - [ "${package_arch}" = amd64 ] || { echo "Unexpected AMD diagnostics package architecture" >&2; exit 1; } - case "${package_name}" in - "${amdsmi_package}") - [ -z "${amdsmi_deb}" ] && [ "$(dpkg-deb -f "${archive}" Version)" = "${amdsmi_version}" ] || exit 1 - amdsmi_deb=${archive} - ;; - "${sysdeps_package}") - [ -z "${sysdeps_deb}" ] && [ "$(dpkg-deb -f "${archive}" Version)" = "${sysdeps_version}" ] || exit 1 - sysdeps_deb=${archive} - ;; - *) echo "Unexpected AMD diagnostics package: ${package_name}" >&2; exit 1 ;; - esac - done - [ -n "${amdsmi_deb}" ] && [ -n "${sysdeps_deb}" ] || exit 1 - apt_get_install 20 5 600 "${amdsmi_deb}" "${sysdeps_deb}" || exit 1 - apt-mark manual "${ubuntu_packages[@]}" "${amdsmi_package}" "${sysdeps_package}" || exit 1 - # shellcheck disable=SC2016 - [ "$(dpkg-query -W -f='${Status} ${Version}' "${amdsmi_package}")" = "install ok installed ${amdsmi_version}" ] || exit 1 - # shellcheck disable=SC2016 - [ "$(dpkg-query -W -f='${Status} ${Version}' "${sysdeps_package}")" = "install ok installed ${sysdeps_version}" ] || exit 1 - [ -x "${cli_path}" ] || exit 1 - # Even --help initializes hardware in AMD SMI. Verify the Python bindings - # and shared-library dependency closure without initialization on CPU bakes. - python3 -I - "${python_path}" <<'AMDSMI_CHECK' || exit 1 -import json -import sys -sys.path.insert(0, sys.argv[1]) -import amdsmi -version = amdsmi.amdsmi_get_lib_version() -assert version["major"] > 0, version -print(json.dumps({"amd_smi_library": version, "python_module": amdsmi.__version__})) -AMDSMI_CHECK - if [ -e /usr/local/bin/amd-smi ] && [ ! -L /usr/local/bin/amd-smi ]; then - echo "Refusing to replace a non-symlink at /usr/local/bin/amd-smi" >&2 - exit 1 - fi - mkdir -p /usr/local/bin || exit 1 - ln -sfnT "${cli_path}" /usr/local/bin/amd-smi || exit 1 - echo " - ${amdsmi_package} version ${amdsmi_version}; ${sysdeps_package} version ${sysdeps_version}" >> "${VHD_LOGS_FILEPATH}" || exit 1 - ) -} - installBcc() { echo "Installing BCC tools..." wait_for_apt_locks From 4de7f56daf84b848bf5321e7ead27c1644942fd8 Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 19 Sep 2026 10:40:12 +0000 Subject: [PATCH 4/7] fix: use baked scriptless provisioning and privileged AMD diagnostics in E2E --- docs/amd-gpu-vhd.md | 13 ++++++++----- e2e/scenario/scenario_gpu_amd.go | 3 ++- e2e/scenario/scenario_gpu_amd_test.go | 1 + 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/amd-gpu-vhd.md b/docs/amd-gpu-vhd.md index 89db8333524..a3a3ff05a01 100644 --- a/docs/amd-gpu-vhd.md +++ b/docs/amd-gpu-vhd.md @@ -161,7 +161,8 @@ AGENTBAKER_E2E_ENABLE_MI300X=true ./e2e-local.sh \ --subscription-id "$SUBSCRIPTION_ID" \ --vm-sku Standard_ND96isr_MI300X_v5 \ --vhd-metadata-file /absolute/path/to/vhd-build-metadata.json \ - --parallel 1 --disable-scriptless \ + --parallel 1 --disable-scriptless=false \ + --disable-scriptless-compilation=true \ --ignore-missing-vhd=false --skip-capacity-errors=false \ --tags '' --skip-tags '' \ Ubuntu2404_MI300X_AMDGPU @@ -177,10 +178,12 @@ and requires completed reference-checked training. A managed 256 GiB OS disk provides room to unpack the development workload image. The scenario does not yet automate reboot, serviced-kernel, or large all-to-all qualification. -The command above exercises legacy NBC provisioning. The current E2E runner's -scriptless path still executes an NBC command; it does not independently -qualify native ANC JSON provisioning. AMD ANC environment generation is -covered by parser tests, but native ANC boot requires separate qualification. +The command above uses scriptless NBC provisioning with the scripts and ANC +binary baked into the image. It avoids embedding the full legacy script payload +in Azure custom data, which can exceed Azure's size limit. This path still +executes an NBC command; it does not independently qualify native ANC JSON +provisioning. AMD ANC environment generation is covered by parser tests, but +native ANC boot requires separate qualification. AgentBaker's Ubuntu 24.04 build currently uses the 6.8 Azure LTS kernel policy. Require a successful build and boot of the captured image on MI300X, including diff --git a/e2e/scenario/scenario_gpu_amd.go b/e2e/scenario/scenario_gpu_amd.go index 1c55261f53f..d85753d9349 100644 --- a/e2e/scenario/scenario_gpu_amd.go +++ b/e2e/scenario/scenario_gpu_amd.go @@ -146,7 +146,8 @@ func amdGPUHostCheckCommand(contents []byte) (string, error) { if err != nil { return "", err } - return "python3 - '" + base64.StdEncoding.EncodeToString(encoded) + "' <<'AMDGPU_CHECK'\n" + amdGPUHostCheckScript + "\nAMDGPU_CHECK\n", nil + // AMD SMI needs KFD/render device access; the SSH user is not in the GPU groups. + return "sudo -n python3 - '" + base64.StdEncoding.EncodeToString(encoded) + "' <<'AMDGPU_CHECK'\n" + amdGPUHostCheckScript + "\nAMDGPU_CHECK\n", nil } func amdGPUTolerations() []corev1.Toleration { diff --git a/e2e/scenario/scenario_gpu_amd_test.go b/e2e/scenario/scenario_gpu_amd_test.go index 022b7d4f886..31d83a5f8cb 100644 --- a/e2e/scenario/scenario_gpu_amd_test.go +++ b/e2e/scenario/scenario_gpu_amd_test.go @@ -24,6 +24,7 @@ func TestAMDGPUHostValidationRequiresPinnedDiagnostics(t *testing.T) { require.NoError(t, err) script, err := amdGPUHostCheckCommand(contents) require.NoError(t, err) + require.True(t, strings.HasPrefix(script, "sudo -n python3 - '"), "host diagnostics require noninteractive access to GPU devices") encoded := strings.SplitN(script, "'", 3)[1] decoded, err := base64.StdEncoding.DecodeString(encoded) require.NoError(t, err) From b7af5742cba873fea01071003c60a27f77115bb0 Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 19 Sep 2026 11:10:47 +0000 Subject: [PATCH 5/7] docs: record fresh AMD VHD build and reboot qualification --- docs/amd-gpu-vhd.md | 95 ++++++++++++++++++++++++++++++--------------- 1 file changed, 63 insertions(+), 32 deletions(-) diff --git a/docs/amd-gpu-vhd.md b/docs/amd-gpu-vhd.md index a3a3ff05a01..e91a340e156 100644 --- a/docs/amd-gpu-vhd.md +++ b/docs/amd-gpu-vhd.md @@ -32,13 +32,14 @@ and C++ runtime dependencies. It does not install the ROCm compute runtime, SDK, or Python packages from pip. `pciutils` (`lspci`) and `numactl` are retained for PCI and NUMA diagnosis. -Read-only examples on a GPU node: +Read-only examples on a GPU node (the default SSH user needs `sudo` for AMD +SMI hardware access): ```bash -amd-smi version -amd-smi list --json -amd-smi static --json -amd-smi metric --json +sudo amd-smi version +sudo amd-smi list --json +sudo amd-smi static --json +sudo amd-smi metric --json lspci -nn numactl --hardware ``` @@ -129,32 +130,63 @@ the AKS resource provider's existing AMD driver-policy validation. ## Qualification -Local repository generation, Go tests, lint, and focused ShellSpec checks -passed. The actual installer also compiled the pinned module for -`6.8.0-1067-azure` in an isolated Ubuntu 24.04 CPU container. An idempotent -reinstall, APT autoremove, and the actual VHD driver content check passed; -the installed module and DKMS rebuild dependencies survived cleanup. The -container test supplied the target kernel to the installer; it did not boot -that kernel or build/capture a complete VHD. - -The AMD SMI addition also passed an actual minimal Ubuntu 24.04 installation, -repeat installation, APT autoremove, and the diagnostics content test with -`lspci`/`numactl`. Only the two selected AMD diagnostics packages were installed; -its Python binding and library loaded without GPU hardware or the compute SDK. -On the existing MI300X lab, `amd-smi` version, discovery, static information and -metrics succeeded. The new `/usr/local/bin/amd-smi` symlink was also tested with -a clean environment and detected eight GPUs. - -The existing MI300X lab used AMDGPU 31.50 with ROCm 10 containers on kernel -6.17.0-1022-azure. Its eight-GPU training and exact all-to-all integrity tests -passed. Those results establish this host/container combination; they do not -establish that a newly baked AgentBaker image has booted successfully. -The new training fixture also passed 40 steps on each of eight GPUs, comparing -outputs, losses, gradients, and updated parameters with a CPU reference. +On 2026-09-19, [dedicated AMD build 181874913](https://dev.azure.com/msazure/CloudNativeCompute/_build/results?buildId=181874913) +successfully captured and tested `2404gen2amdgpucontainerd/1.1789808022.4162` +and completed France Central replication. Its image inputs match PR commit +`3ce6d173`; validation commit `72e925a9` changes only the pipeline entrypoint. +The E2E runner includes the AMD test invocation fix in `4de7f56d`; that change +does not alter the captured VHD payload. + +A fresh MI300X node booted this exact image, joined AKS, and passed +`Ubuntu2404_MI300X_AMDGPU` with kernel `6.8.0-1067-azure` and AMDGPU module +`7.1.3.31500000`. AMD SMI and the device plugin discovered eight GPUs. Using +the [scenario's digest-pinned images](../e2e/scenario/scenario_gpu_amd.go), +PyTorch 2.13.0 / ROCm 10.0 completed 40 FP32 training steps per GPU, checking +outputs, loss, gradients, and updated parameters against a CPU reference. + +Additional checks passed before and after restarting that same instance. A +changed boot ID and node readiness were required before the second round: + +- Eight-rank all-to-all transfers checked every received element at + 64/80/128/256 MiB per peer: 42,631 measured calls across both rounds, 291.16 TB + received including 254.77 TB between GPUs, and zero mismatched elements. + Independent calculation verified complete timing samples and byte totals. +- A 49,016,832-parameter Transformer completed 100 BF16 DDP/AdamW steps across + eight GPUs in each round, processing 1,638,400 synthetic tokens per round. + Mean loss decreased from 6.519 to 0.000249; losses, gradients, and weights + remained finite, and all eight ranks produced the same full-model SHA-256. +- The 40-step CPU-reference training check passed again on all eight GPUs + after reboot. +- Host driver, AMD SMI, and runtime health checks passed before and after these + workloads; the host had no ROCm compute SDK installed. + +Remote-send bandwidth per GPU (decimal GB/s): + +| MiB per peer | Median before reboot | Median after reboot | Effective before | Effective after | +| --- | ---: | ---: | ---: | ---: | +| 64 | 278.7 | 278.8 | 269.9 | 272.6 | +| 80 | 281.9 | 282.0 | 281.2 | 277.7 | +| 128 | 286.5 | 286.5 | 286.3 | 286.3 | +| 256 | 291.1 | 291.4 | 289.6 | 290.5 | + +Rates use decimal GB/s and seven remote sends per GPU divided by the slowest +rank's call time, without send/receive double-counting. Effective rates include +all timed calls, including outliers; setup and data comparison are outside the +timed calls. These bounded tests do not establish peak throughput, model +quality, or integrity of untested transfers and network/storage paths. After capturing the image and replicating it to `francecentral`, use the repository's normal E2E Azure configuration and its captured build metadata: +The test nodes need outbound HTTPS access to the pinned images on Docker Hub. +The standard E2E firewall does not include this access. For an isolated AMD lab, +scope an additional rule to its GPU nodes for `registry-1.docker.io`, +`auth.docker.io`, and the image-layer CDN. The current image digests were served +by `production.cloudfront.docker.com`; recheck CDN destinations when updating +them. Remove temporary rules with the lab resources. A recurring AMD pipeline +can instead mirror the exact digests into an approved reachable registry and +verify the copied manifests and layers before use. + ```bash cd e2e AGENTBAKER_E2E_ENABLE_MI300X=true ./e2e-local.sh \ @@ -186,11 +218,10 @@ provisioning. AMD ANC environment generation is covered by parser tests, but native ANC boot requires separate qualification. AgentBaker's Ubuntu 24.04 build currently uses the 6.8 Azure LTS kernel policy. -Require a successful build and boot of the captured image on MI300X, including -node readiness, eight advertised GPUs, AI/reference checks, large all-to-all -integrity/bandwidth checks, reboot, and kernel-update/rebuild validation. Run -both legacy and ANC provisioning, and PIS validation before supporting PIS. -Record the image ID, kernel, driver, workload image digests, and results. +Native ANC JSON and legacy provisioning, PIS, and a serviced-kernel update with +DKMS rebuild and subsequent GPU workloads require separate qualification. +Record the image version, kernel, driver, +workload image digests, and results for each path. Production release additionally requires vendor driver security qualification. A successful workload run is not evidence that all driver security issues are From f3aea1c228fca232032577a1843175047bd56ab8 Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 19 Sep 2026 11:12:15 +0000 Subject: [PATCH 6/7] docs: keep internal build links out of public documentation --- docs/amd-gpu-vhd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/amd-gpu-vhd.md b/docs/amd-gpu-vhd.md index e91a340e156..69fa4fcb016 100644 --- a/docs/amd-gpu-vhd.md +++ b/docs/amd-gpu-vhd.md @@ -130,7 +130,7 @@ the AKS resource provider's existing AMD driver-policy validation. ## Qualification -On 2026-09-19, [dedicated AMD build 181874913](https://dev.azure.com/msazure/CloudNativeCompute/_build/results?buildId=181874913) +On 2026-09-19, dedicated AMD build `181874913` successfully captured and tested `2404gen2amdgpucontainerd/1.1789808022.4162` and completed France Central replication. Its image inputs match PR commit `3ce6d173`; validation commit `72e925a9` changes only the pipeline entrypoint. From 877bea0711414d4d98d74f312bdd6e62a9a88b06 Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sun, 20 Sep 2026 01:56:40 +0000 Subject: [PATCH 7/7] fix(amd-gpu): address dependency tracking and isolated build review --- .github/renovate.json | 100 ++++++++++++++++++ .github/workflows/validate-components.yml | 16 +++ .pipelines/.vsts-vhd-builder-amd.yaml | 2 + .../templates/.builder-release-template.yaml | 5 +- Makefile | 4 + e2e/scenario/scenario_gpu_amd.go | 5 +- e2e/scenario/scenario_gpu_amd_test.go | 2 + schemas/amd-gpu-components.cue | 26 ++++- 8 files changed, 156 insertions(+), 4 deletions(-) diff --git a/.github/renovate.json b/.github/renovate.json index 3d94540846d..1d140dfaa35 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -737,6 +737,30 @@ "rashmichandrashekar", "bragi92" ] + }, + { + "description": "AMD image updates always require coordinated pin review and fresh MI300X qualification", + "matchFileNames": [ + "vhdbuilder/packer/amd-gpu-components.json" + ], + "matchManagers": [ + "custom.regex" + ], + "enabled": true, + "automerge": false, + "ignoreUnstable": false, + "groupName": "amd-gpu-image", + "separateMajorMinor": false, + "separateMinorPatch": false, + "assignees": [ + "team:aks-node-lifecycle" + ], + "reviewers": [ + "team:aks-node-lifecycle" + ], + "prBodyNotes": [ + "Update AMD repository, driver, firmware, module and DKMS pins together; AMD CUE validation rejects mismatches. Rebuild the separate AMD VHD and qualify it on MI300X before merging." + ] } ], "customManagers": [ @@ -964,6 +988,78 @@ "matchStrings": [ "#\\s*renovate:\\s*datasource=(?\\S+)\\s+depName=(?\\S+)(?:\\s+versioning=(?\\S+))?(?:\\s+registryUrl=(?\\S+))?\\s*\\n[^\\n]*VERSION=\"(?[^\"]+)\"" ] + }, + { + "customType": "regex", + "description": "Track amdgpu-dkms in the isolated AMD VHD manifest", + "managerFilePatterns": [ + "/^vhdbuilder/packer/amd-gpu-components\\.json$/" + ], + "matchStrings": [ + "\"AMDGPUDriver\":\\s*\\{[^}]*?\"repositoryURL\":\\s*\"(?https://repo\\.radeon\\.com/amdgpu/[0-9.]+/ubuntu)\"[^}]*\\}", + "\"packageVersion\":\\s*\"(?[^\"]+)\"" + ], + "depNameTemplate": "amdgpu-dkms", + "datasourceTemplate": "deb", + "versioningTemplate": "deb", + "registryUrlTemplate": "{{{amdRepository}}}?suite=noble&components=main&binaryArch=amd64", + "matchStringsStrategy": "recursive" + }, + { + "customType": "regex", + "description": "Track amdgpu-dkms-firmware in the isolated AMD VHD manifest", + "managerFilePatterns": [ + "/^vhdbuilder/packer/amd-gpu-components\\.json$/" + ], + "matchStrings": [ + "\"AMDGPUDriver\":\\s*\\{[^}]*?\"repositoryURL\":\\s*\"(?https://repo\\.radeon\\.com/amdgpu/[0-9.]+/ubuntu)\"[^}]*\\}", + "\"firmwarePackageVersion\":\\s*\"(?[^\"]+)\"" + ], + "depNameTemplate": "amdgpu-dkms-firmware", + "datasourceTemplate": "deb", + "versioningTemplate": "deb", + "registryUrlTemplate": "{{{amdRepository}}}?suite=noble&components=main&binaryArch=amd64", + "matchStringsStrategy": "recursive" + }, + { + "customType": "regex", + "description": "Track AMD amdsmi in the isolated AMD VHD manifest", + "managerFilePatterns": [ + "/^vhdbuilder/packer/amd-gpu-components\\.json$/" + ], + "matchStrings": [ + "\"amdsmiPackage\":\\s*\"(?amdrocm-amdsmi[0-9]+\\.[0-9]+)\"[^}]*?\"amdsmiVersion\":\\s*\"(?[^\"]+)\"" + ], + "datasourceTemplate": "deb", + "versioningTemplate": "deb", + "registryUrlTemplate": "https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/?suite=stable&components=main&binaryArch=amd64" + }, + { + "customType": "regex", + "description": "Track AMD sysdeps in the isolated AMD VHD manifest", + "managerFilePatterns": [ + "/^vhdbuilder/packer/amd-gpu-components\\.json$/" + ], + "matchStrings": [ + "\"sysdepsPackage\":\\s*\"(?amdrocm-sysdeps[0-9]+\\.[0-9]+)\"[^}]*?\"sysdepsVersion\":\\s*\"(?[^\"]+)\"" + ], + "datasourceTemplate": "deb", + "versioningTemplate": "deb", + "registryUrlTemplate": "https://stable.repo.amd.com/rocm/core/packages/ubuntu2404/?suite=stable&components=main&binaryArch=amd64" + }, + { + "customType": "regex", + "description": "Detect new AMD driver release directories; coordinated pins require review", + "managerFilePatterns": [ + "/^vhdbuilder/packer/amd-gpu-components\\.json$/" + ], + "matchStrings": [ + "\"repositoryURL\":\\s*\"https://repo\\.radeon\\.com/amdgpu/(?[0-9.]+)/ubuntu\"" + ], + "depNameTemplate": "amdgpu-repository", + "datasourceTemplate": "custom.amdgpu-releases", + "versioningTemplate": "loose", + "extractVersionTemplate": "^(?[0-9]+(?:\\.[0-9]+)+)/$" } ], "customDatasources": { @@ -1022,6 +1118,10 @@ "transformTemplates": [ "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}" ] + }, + "amdgpu-releases": { + "defaultRegistryUrlTemplate": "https://repo.radeon.com/amdgpu/", + "format": "html" } } } diff --git a/.github/workflows/validate-components.yml b/.github/workflows/validate-components.yml index 3993267b126..f5eedaf00d0 100644 --- a/.github/workflows/validate-components.yml +++ b/.github/workflows/validate-components.yml @@ -2,6 +2,22 @@ name: Validate Components on: pull_request jobs: + amd-components: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: '1.26' + - name: Validate dedicated AMD manifest and coordinated package pins + run: | + go install cuelang.org/go/cmd/cue@v0.15.4 + "$(go env GOPATH)/bin/cue" vet -c ./schemas/amd-gpu-components.cue ./vhdbuilder/packer/amd-gpu-components.json + cue: runs-on: ubuntu-latest permissions: diff --git a/.pipelines/.vsts-vhd-builder-amd.yaml b/.pipelines/.vsts-vhd-builder-amd.yaml index baefd017d8c..7eddcf251b1 100644 --- a/.pipelines/.vsts-vhd-builder-amd.yaml +++ b/.pipelines/.vsts-vhd-builder-amd.yaml @@ -63,6 +63,8 @@ stages: - template: ./templates/.builder-release-template.yaml parameters: artifactName: 2404-amdgpu-gen2-containerd + # Retries overwrite the Packer output used for scoped cleanup. + packerBuildRetryCount: 0 - task: AzureCLI@2 displayName: Clean up this AMD build's Packer resource group condition: always() diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 16fd5a21c23..7a0d8365de0 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -8,6 +8,9 @@ parameters: - name: overrideBranch type: string default: master + - name: packerBuildRetryCount + type: number + default: 3 steps: - template: ./.template-override-components-json.yaml @@ -109,7 +112,7 @@ steps: make -f packer.mk run-packer condition: ne(variables['BUILDER'], 'imagecustomizer') displayName: Build VHD - retryCountOnTaskFailure: 3 + retryCountOnTaskFailure: ${{ parameters.packerBuildRetryCount }} env: OS_TYPE: Linux GIT_VERSION: $(Build.SourceVersion) diff --git a/Makefile b/Makefile index deb1212a203..247a225a95e 100644 --- a/Makefile +++ b/Makefile @@ -265,6 +265,10 @@ unit-tests: update-windows-base-versions: ./vhdbuilder/packer/windows/update_windows_base_versions.sh +.PHONY: validate-amd-components +validate-amd-components: + @./hack/tools/bin/cue vet -c ./schemas/amd-gpu-components.cue ./vhdbuilder/packer/amd-gpu-components.json + .PHONY: validate-components validate-components: @./hack/tools/bin/cue vet -c ./schemas/components.cue ./parts/common/components.json diff --git a/e2e/scenario/scenario_gpu_amd.go b/e2e/scenario/scenario_gpu_amd.go index d85753d9349..6bc7620eed5 100644 --- a/e2e/scenario/scenario_gpu_amd.go +++ b/e2e/scenario/scenario_gpu_amd.go @@ -172,7 +172,10 @@ func amdDevicePluginDaemonset(nodeName string, owner metav1.OwnerReference) *app Tolerations: amdGPUTolerations(), Containers: []corev1.Container{{ Name: "device-plugin", Image: amdDevicePluginImage, - SecurityContext: &corev1.SecurityContext{Privileged: to.Ptr(true)}, + SecurityContext: &corev1.SecurityContext{ + Privileged: to.Ptr(true), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m"), corev1.ResourceMemory: resource.MustParse("128Mi")}, Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("512Mi")}, diff --git a/e2e/scenario/scenario_gpu_amd_test.go b/e2e/scenario/scenario_gpu_amd_test.go index 31d83a5f8cb..6d8ab5378dc 100644 --- a/e2e/scenario/scenario_gpu_amd_test.go +++ b/e2e/scenario/scenario_gpu_amd_test.go @@ -177,6 +177,8 @@ func TestAMDGPUManifestsUseAssignedDevicesAndPinnedUserspace(t *testing.T) { plugin := ds.Spec.Template.Spec.Containers[0] require.Equal(t, "rocm/k8s-device-plugin@sha256:e4df5dc9a7fa34e2344852256dcc5762171a6d68f1f9a34026ce26786ae335e2", plugin.Image) require.True(t, *plugin.SecurityContext.Privileged) + require.NotNil(t, plugin.SecurityContext.Capabilities) + require.Equal(t, []corev1.Capability{"ALL"}, plugin.SecurityContext.Capabilities.Drop) require.Len(t, ds.Spec.Template.Spec.Volumes, 2) paths := map[string]bool{} for _, volume := range ds.Spec.Template.Spec.Volumes { diff --git a/schemas/amd-gpu-components.cue b/schemas/amd-gpu-components.cue index b0cb78a00b3..09e8d21f831 100644 --- a/schemas/amd-gpu-components.cue +++ b/schemas/amd-gpu-components.cue @@ -1,15 +1,30 @@ package amdgpu +import ( + "regexp" + "strings" +) + #AMDGPUDriver: { - repositoryURL: string & =~"^https://repo[.]radeon[.]com/amdgpu/[0-9.]+/ubuntu$" + repositoryURL: string & =~"^https://repo[.]radeon[.]com/amdgpu/[0-9]+([.][0-9]+){1,3}/ubuntu$" signingKeyURL: string & =~"^https://repo[.]radeon[.]com/[^?[:space:]]+$" signingKeyFingerprint: string & =~"^[A-F0-9]{40}$" distribution: "noble" component: "main" - packageVersion: string & =~"^[0-9]+:[0-9][0-9A-Za-z.+~-]+$" + packageVersion: string & =~"^[0-9]+:[0-9]+[.][0-9]+[.][0-9]+[.][0-9]+-[0-9]+[.]24[.]04$" firmwarePackageVersion: string & =~"^[0-9]+:[0-9][0-9A-Za-z.+~-]+$" moduleVersion: string & =~"^[0-9][0-9.]+$" dkmsVersion: string & =~"^[0-9][0-9A-Za-z.+~-]+$" + + // These fields describe one driver release, not independent update targets. + // Reject partial Renovate proposals until a reviewer supplies coherent pins. + _package: regexp.FindSubmatch("^([0-9]+):([0-9]+[.][0-9]+[.][0-9]+)[.]([0-9]+)-([0-9]+[.]24[.]04)$", packageVersion) + _release: strings.TrimSuffix(strings.TrimPrefix(repositoryURL, "https://repo.radeon.com/amdgpu/"), "/ubuntu") + _releaseParts: strings.Split(_release, ".") + _firmwareRelease: _release + strings.Repeat(".0", 4-len(_releaseParts)) + moduleVersion: "\(_package[2]).\(_package[3])" + dkmsVersion: "\(_package[2])-\(_package[4])" + firmwarePackageVersion: "\(_package[1]):\(_firmwareRelease).\(_package[3])-\(_package[4])" } #AMDGPUDiagnostics: { @@ -23,6 +38,13 @@ package amdgpu sysdepsPackage: string & =~"^amdrocm-sysdeps[0-9]+[.][0-9]+$" sysdepsVersion: string & =~"^[0-9][0-9A-Za-z.+~-]+$" cliPath: string & =~"^/opt/rocm/core-[0-9]+[.][0-9]+/bin/amd-smi$" + + // Keep package names, versions and the installed CLI in the same ROCm line. + _core: strings.TrimPrefix(amdsmiPackage, "amdrocm-amdsmi") + sysdepsPackage: "amdrocm-sysdeps\(_core)" + amdsmiVersion: =~"^\(regexp.QuoteMeta(_core))[.]" + sysdepsVersion: =~"^\(regexp.QuoteMeta(_core))[.]" + cliPath: "/opt/rocm/core-\(_core)/bin/amd-smi" } #Components: {