diff --git a/.pipelines/.vsts-vhd-builder-release.yaml b/.pipelines/.vsts-vhd-builder-release.yaml index a6e1c33f959..9ea1224dfc0 100644 --- a/.pipelines/.vsts-vhd-builder-release.yaml +++ b/.pipelines/.vsts-vhd-builder-release.yaml @@ -886,12 +886,15 @@ stages: echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]True' echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]True' echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER]mcr.microsoft.com/azurelinux/imagecustomizer:1.5@sha256:b1a46447f67d73dce1a76163d3f8dbcbe12988b65c51393621529c52af160729' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER_FALLBACK]ghcr.io/microsoft/imagecustomizer:1.5.0@sha256:c74e16c636e08486f517cd3b446e3577e52b24feea06aa245a8742a9f1c4a70a' displayName: Setup Build Variables - template: ./templates/.builder-release-template.yaml parameters: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: acl-tl-gen2 + enableCosiPublishing: true - job: buildaclarm64tlgen2 condition: eq('${{ parameters.buildaclarm64tlgen2 }}', true) dependsOn: [ ] @@ -912,12 +915,15 @@ stages: echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]True' echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]True' echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER]mcr.microsoft.com/azurelinux/imagecustomizer:1.5@sha256:b1a46447f67d73dce1a76163d3f8dbcbe12988b65c51393621529c52af160729' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER_FALLBACK]ghcr.io/microsoft/imagecustomizer:1.5.0@sha256:c74e16c636e08486f517cd3b446e3577e52b24feea06aa245a8742a9f1c4a70a' displayName: Setup Build Variables - template: ./templates/.builder-release-template.yaml parameters: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: acl-arm64-tl-gen2 + enableCosiPublishing: true - job: buildaclfipstlgen2 condition: eq('${{ parameters.buildaclfipstlgen2 }}', true) dependsOn: [ ] @@ -938,12 +944,15 @@ stages: echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]True' echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]True' echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER]mcr.microsoft.com/azurelinux/imagecustomizer:1.5@sha256:b1a46447f67d73dce1a76163d3f8dbcbe12988b65c51393621529c52af160729' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER_FALLBACK]ghcr.io/microsoft/imagecustomizer:1.5.0@sha256:c74e16c636e08486f517cd3b446e3577e52b24feea06aa245a8742a9f1c4a70a' displayName: Setup Build Variables - template: ./templates/.builder-release-template.yaml parameters: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: acl-fips-tl-gen2 + enableCosiPublishing: true - job: buildaclarm64fipstlgen2 condition: eq('${{ parameters.buildaclarm64fipstlgen2 }}', true) dependsOn: [ ] @@ -964,12 +973,81 @@ stages: echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]True' echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]True' echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER]mcr.microsoft.com/azurelinux/imagecustomizer:1.5@sha256:b1a46447f67d73dce1a76163d3f8dbcbe12988b65c51393621529c52af160729' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER_FALLBACK]ghcr.io/microsoft/imagecustomizer:1.5.0@sha256:c74e16c636e08486f517cd3b446e3577e52b24feea06aa245a8742a9f1c4a70a' displayName: Setup Build Variables - template: ./templates/.builder-release-template.yaml parameters: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: acl-arm64-fips-tl-gen2 + enableCosiPublishing: true + # Native-arm64 COSI conversion jobs. The arm64 ACL build jobs above stage the + # VHD in blob storage and skip COSI (ImageCustomizer's btrfs subvolume ioctls + # don't work under qemu emulation on the x86_64 build pool). These jobs run on + # the native arm64 pool to convert the staged VHD and publish the COSI. + - job: convertaclarm64tlgen2 + condition: and(succeeded(), eq(variables.GENERATE_PUBLISHING_INFO, 'True'), eq('${{ parameters.buildaclarm64tlgen2 }}', true)) + dependsOn: buildaclarm64tlgen2 + timeoutInMinutes: 120 + pool: + name: $(NODE_SIG_TME_POOL_ARM64_NAME) + variables: + CAPTURED_SIG_VERSION: $[ dependencies.buildaclarm64tlgen2.outputs['exportCapturedSigVersion.CAPTURED_SIG_VERSION'] ] + IMAGE_VERSION: $[ dependencies.buildaclarm64tlgen2.outputs['resolveCosiImageVersion.IMAGE_VERSION'] ] + VHD_SAS_URL: $[ dependencies.buildaclarm64tlgen2.outputs['exportVhdSas.VHD_SAS_URL'] ] + steps: + - bash: | + echo '##vso[task.setvariable variable=OS_SKU]AzureContainerLinux' + echo '##vso[task.setvariable variable=OS_VERSION]acl' + echo '##vso[task.setvariable variable=IMG_PUBLISHER]MicrosoftCBLMariner' + echo '##vso[task.setvariable variable=IMG_OFFER]azure-linux-3' + echo '##vso[task.setvariable variable=IMG_SKU]azure-linux-3-arm64-gen2-acl' + echo '##vso[task.setvariable variable=IMG_VERSION]3.20260616.01' + echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16pds_v6' + echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' + echo '##vso[task.setvariable variable=ARCHITECTURE]ARM64' + echo '##vso[task.setvariable variable=ENABLE_FIPS]false' + echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]True' + echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER]mcr.microsoft.com/azurelinux/imagecustomizer:1.5@sha256:b1a46447f67d73dce1a76163d3f8dbcbe12988b65c51393621529c52af160729' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER_FALLBACK]ghcr.io/microsoft/imagecustomizer:1.5.0@sha256:c74e16c636e08486f517cd3b446e3577e52b24feea06aa245a8742a9f1c4a70a' + displayName: Setup Build Variables + - template: ./templates/.builder-cosi-convert-template.yaml + parameters: + artifactName: acl-arm64-tl-gen2 + - job: convertaclarm64fipstlgen2 + condition: and(succeeded(), eq(variables.GENERATE_PUBLISHING_INFO, 'True'), eq('${{ parameters.buildaclarm64fipstlgen2 }}', true)) + dependsOn: buildaclarm64fipstlgen2 + timeoutInMinutes: 120 + pool: + name: $(NODE_SIG_TME_POOL_ARM64_NAME) + variables: + CAPTURED_SIG_VERSION: $[ dependencies.buildaclarm64fipstlgen2.outputs['exportCapturedSigVersion.CAPTURED_SIG_VERSION'] ] + IMAGE_VERSION: $[ dependencies.buildaclarm64fipstlgen2.outputs['resolveCosiImageVersion.IMAGE_VERSION'] ] + VHD_SAS_URL: $[ dependencies.buildaclarm64fipstlgen2.outputs['exportVhdSas.VHD_SAS_URL'] ] + steps: + - bash: | + echo '##vso[task.setvariable variable=OS_SKU]AzureContainerLinux' + echo '##vso[task.setvariable variable=OS_VERSION]acl' + echo '##vso[task.setvariable variable=IMG_PUBLISHER]MicrosoftCBLMariner' + echo '##vso[task.setvariable variable=IMG_OFFER]azure-linux-3' + echo '##vso[task.setvariable variable=IMG_SKU]azure-linux-3-arm64-gen2-acl' + echo '##vso[task.setvariable variable=IMG_VERSION]3.20260616.01' + echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16pds_v6' + echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' + echo '##vso[task.setvariable variable=ARCHITECTURE]ARM64' + echo '##vso[task.setvariable variable=ENABLE_FIPS]True' + echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]True' + echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER]mcr.microsoft.com/azurelinux/imagecustomizer:1.5@sha256:b1a46447f67d73dce1a76163d3f8dbcbe12988b65c51393621529c52af160729' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER_FALLBACK]ghcr.io/microsoft/imagecustomizer:1.5.0@sha256:c74e16c636e08486f517cd3b446e3577e52b24feea06aa245a8742a9f1c4a70a' + displayName: Setup Build Variables + - template: ./templates/.builder-cosi-convert-template.yaml + parameters: + artifactName: acl-arm64-fips-tl-gen2 - job: build2404arm64gbgen2containerd condition: eq('${{ parameters.build2404arm64gbgen2containerd }}', true) dependsOn: [ ] diff --git a/.pipelines/.vsts-vhd-builder.yaml b/.pipelines/.vsts-vhd-builder.yaml index a7e955095e3..d8d338e253c 100644 --- a/.pipelines/.vsts-vhd-builder.yaml +++ b/.pipelines/.vsts-vhd-builder.yaml @@ -257,6 +257,8 @@ stages: echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]True' echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]True' echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER]mcr.microsoft.com/azurelinux/imagecustomizer:1.5@sha256:b1a46447f67d73dce1a76163d3f8dbcbe12988b65c51393621529c52af160729' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER_FALLBACK]ghcr.io/microsoft/imagecustomizer:1.5.0@sha256:c74e16c636e08486f517cd3b446e3577e52b24feea06aa245a8742a9f1c4a70a' displayName: Setup Build Variables - template: ./templates/.builder-release-template.yaml parameters: @@ -280,6 +282,8 @@ stages: echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]True' echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]True' echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER]mcr.microsoft.com/azurelinux/imagecustomizer:1.5@sha256:b1a46447f67d73dce1a76163d3f8dbcbe12988b65c51393621529c52af160729' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER_FALLBACK]ghcr.io/microsoft/imagecustomizer:1.5.0@sha256:c74e16c636e08486f517cd3b446e3577e52b24feea06aa245a8742a9f1c4a70a' displayName: Setup Build Variables - template: ./templates/.builder-release-template.yaml parameters: @@ -303,6 +307,8 @@ stages: echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]True' echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]True' echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER]mcr.microsoft.com/azurelinux/imagecustomizer:1.5@sha256:b1a46447f67d73dce1a76163d3f8dbcbe12988b65c51393621529c52af160729' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER_FALLBACK]ghcr.io/microsoft/imagecustomizer:1.5.0@sha256:c74e16c636e08486f517cd3b446e3577e52b24feea06aa245a8742a9f1c4a70a' displayName: Setup Build Variables - template: ./templates/.builder-release-template.yaml parameters: @@ -326,6 +332,8 @@ stages: echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]True' echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]True' echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER]mcr.microsoft.com/azurelinux/imagecustomizer:1.5@sha256:b1a46447f67d73dce1a76163d3f8dbcbe12988b65c51393621529c52af160729' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER_FALLBACK]ghcr.io/microsoft/imagecustomizer:1.5.0@sha256:c74e16c636e08486f517cd3b446e3577e52b24feea06aa245a8742a9f1c4a70a' displayName: Setup Build Variables - template: ./templates/.builder-release-template.yaml parameters: diff --git a/.pipelines/templates/.builder-cosi-convert-template.yaml b/.pipelines/templates/.builder-cosi-convert-template.yaml new file mode 100644 index 00000000000..eb971fbe7c0 --- /dev/null +++ b/.pipelines/templates/.builder-cosi-convert-template.yaml @@ -0,0 +1,230 @@ +# Steps for the native-arm64 ACL COSI conversion job. +# +# ACL COSI conversion runs ImageCustomizer, which mounts the image's btrfs rootfs +# and enumerates subvolumes. Those btrfs ioctls are not supported under qemu-user +# emulation, so the arm64 conversion cannot run on the x86_64 build pool. Instead, +# the x86_64 build job stages the VHD in blob storage and skips COSI, and this +# template runs on the native arm64 pool to download that staged VHD, convert it, +# upload the COSI to PMC, publish the info artifact, and copy the VHD to immutable +# storage (mirroring what the build job does inline for x64 ACL). +# +# The consuming job must set the ACL build variables (OS_SKU, OS_VERSION, +# HYPERV_GENERATION, ARCHITECTURE, IMG_CUSTOMIZER_CONTAINER, etc.) and provide +# CAPTURED_SIG_VERSION (from the build job's exportCapturedSigVersion output). +# +# The arm64 pool has no git, so we skip the built-in `checkout: self` and hydrate +# the workspace from a source bundle the build job publishes. +parameters: + - name: artifactName + type: string + +steps: + - checkout: none + + # The arm64 pool image is minimal, so bootstrap every tool the convert flow + # needs: docker (privileged), az CLI, azcopy, make/tar/curl, and Go. Idempotent + # via `command -v` guards, so it's a no-op on any tool that's already present. + - bash: | + set -euo pipefail + echo "==== Bootstrapping tooling on $(uname -a) ====" + + if command -v apt-get >/dev/null 2>&1; then + PKG=apt + elif command -v tdnf >/dev/null 2>&1; then + PKG=tdnf + elif command -v dnf >/dev/null 2>&1; then + PKG=dnf + else + echo "##vso[task.logissue type=error]No supported package manager (apt/tdnf/dnf) on this agent" + exit 1 + fi + echo "Package manager: $PKG" + + sudo -n true >/dev/null 2>&1 || { + echo "##vso[task.logissue type=error]This agent user does not have passwordless sudo; cannot install tools" + exit 1 + } + + pkg_install() { + case "$PKG" in + apt) sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "$@" ;; + tdnf) sudo tdnf install -y "$@" ;; + dnf) sudo dnf install -y "$@" ;; + esac + } + + if [ "$PKG" = apt ]; then + sudo DEBIAN_FRONTEND=noninteractive apt-get update -y + fi + + # Baseline utilities used by scripts and downloads below. + pkg_install ca-certificates curl tar gnupg make jq + + # cosi-upload uses Microsoft's cgo-less OpenSSL backend (see packer.mk), + # so no build-time C toolchain or OpenSSL development packages are needed. + + # git isn't used by the convert scripts, but install it so any implicit + # agent-side cleanup checkouts don't tank the job. + if ! command -v git >/dev/null 2>&1; then + pkg_install git + fi + + # Docker (must support --privileged for ImageCustomizer's loopback mounts). + if ! command -v docker >/dev/null 2>&1; then + case "$PKG" in + apt) pkg_install docker.io ;; + tdnf|dnf) pkg_install moby-engine moby-cli || pkg_install docker ;; + esac + fi + sudo systemctl start docker 2>/dev/null || sudo service docker start 2>/dev/null || true + # Give the agent user access to the docker socket for the duration of the job. + sudo chown "$(id -u):$(id -g)" /var/run/docker.sock 2>/dev/null || true + docker version + + # Azure CLI. + if ! command -v az >/dev/null 2>&1; then + case "$PKG" in + apt) + # Install azure-cli from the pinned Microsoft apt repository so the + # install path is auditable and versionable (avoids piping a remote + # script into `sudo bash`). + pkg_install ca-certificates curl gnupg apt-transport-https lsb-release + sudo install -d -m 0755 /etc/apt/keyrings + tmp="$(mktemp -d)" + curl -fsSL https://packages.microsoft.com/keys/microsoft.asc -o "$tmp/microsoft.asc" + gpg --dearmor < "$tmp/microsoft.asc" > "$tmp/microsoft.gpg" + sudo install -m 0644 "$tmp/microsoft.gpg" /etc/apt/keyrings/microsoft.gpg + rm -rf "$tmp" + AZ_DIST="$(lsb_release -cs)" + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ ${AZ_DIST} main" \ + | sudo tee /etc/apt/sources.list.d/azure-cli.list >/dev/null + sudo apt-get update + pkg_install azure-cli + ;; + tdnf|dnf) pkg_install azure-cli ;; + esac + fi + az version + + # azcopy static binary. + if ! command -v azcopy >/dev/null 2>&1; then + # azureedge.net CDN was retired; azcopy releases are served from GitHub. + case "$(uname -m)" in + aarch64|arm64) + AZCOPY_URL=https://github.com/Azure/azure-storage-azcopy/releases/download/v10.22.1/azcopy_linux_arm64_10.22.1.tar.gz + AZCOPY_SHA256=4db9a4b48abc7775f1a5d6d928afc42361dcc57bbfcde23ac82e4c419a0dc8fc + ;; + *) + AZCOPY_URL=https://github.com/Azure/azure-storage-azcopy/releases/download/v10.22.1/azcopy_linux_amd64_10.22.1.tar.gz + AZCOPY_SHA256=7549424d56ab2d8b4033c84c2a9bb167dc2dcbb23998acd7fffb37bc1a71a267 + ;; + esac + tmp="$(mktemp -d)" + curl -fsSL "$AZCOPY_URL" -o "$tmp/azcopy.tgz" + echo "$AZCOPY_SHA256 $tmp/azcopy.tgz" | sha256sum --check --status + tar -xzf "$tmp/azcopy.tgz" -C "$tmp" + # extracted dir is like azcopy_linux_arm64_10.x.x/ + azcopy_bin="$(find "$tmp" -maxdepth 2 -type f -name azcopy | head -n1)" + if [ -z "$azcopy_bin" ] || [ ! -x "$azcopy_bin" ]; then + echo "##vso[task.logissue type=error]azcopy archive did not contain an executable binary" + exit 1 + fi + sudo install -m 0755 "$azcopy_bin" /usr/local/bin/azcopy + rm -rf "$tmp" + fi + azcopy --version + + # Azure Linux packages Microsoft Go; the preflight build rejects incompatible toolchains. + if ! command -v go >/dev/null 2>&1; then + if [ "$PKG" = apt ]; then + echo "Go not found yet; will install pinned version after source extraction" + else + pkg_install golang + fi + fi + command -v go >/dev/null 2>&1 && go version || true + + echo "==== Tooling bootstrap complete ====" + displayName: Bootstrap arm64 convert tooling + + - task: DownloadPipelineArtifact@2 + displayName: Download AgentBaker source bundle + inputs: + artifact: agentbaker-sources-${{ parameters.artifactName }} + path: $(Pipeline.Workspace)/agentbaker-sources + + - bash: | + set -euo pipefail + mkdir -p "$(Build.SourcesDirectory)" + tar -xzf "$(Pipeline.Workspace)/agentbaker-sources/agentbaker-sources.tar.gz" -C "$(Build.SourcesDirectory)" + ls "$(Build.SourcesDirectory)" + cd "$(Build.SourcesDirectory)" + if command -v apt-get >/dev/null 2>&1; then + ./hack/setup_golang.sh + fi + go version + + - bash: make -f packer.mk build-cosi-upload + displayName: Build COSI uploader with Microsoft system crypto + condition: and(succeeded(), eq(variables.GENERATE_PUBLISHING_INFO, 'True')) + + - bash: | + SKU_NAME=${OS_VERSION} && \ + if [[ "${HYPERV_GENERATION,,}" == "v2" ]]; then SKU_NAME="${SKU_NAME}gen2"; fi && \ + if [[ "${ARCHITECTURE,,}" == "arm64" ]]; then SKU_NAME="${SKU_NAME}arm64"; fi && \ + if grep -q "NVIDIA_GB" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}gb"; fi && \ + if [[ "${ENABLE_FIPS,,}" == "true" ]]; then SKU_NAME="${SKU_NAME}fips"; fi && \ + if grep -q "cvm" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}CVM"; fi && \ + if [[ "${IMG_SKU}" == *"minimal"* ]]; then SKU_NAME="${SKU_NAME}minimal"; fi && \ + if [[ "${ENABLE_TRUSTED_LAUNCH}" == "True" ]]; then SKU_NAME="${SKU_NAME}TL"; fi && \ + SKU_NAME=$(echo ${SKU_NAME} | tr -d '.') && \ + echo "##vso[task.setvariable variable=SKU_NAME]$SKU_NAME" + echo "Set SKU_NAME to $SKU_NAME" + displayName: Set SKU Name + + - bash: | + set -euo pipefail + if [ "${ENVIRONMENT,,}" = "tme" ]; then + container="${CLASSIC_BLOB_STAGING}" + else + container="${CLASSIC_BLOB}" + fi + echo "##vso[task.setvariable variable=DESTINATION_STORAGE_CONTAINER]${container}" + displayName: Set Destination Storage Container + + - task: AzureCLI@2 + inputs: + azureSubscription: $(VHD_ARM_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + make -f packer.mk convert-vhd-to-cosi + condition: and(succeeded(), eq(variables.GENERATE_PUBLISHING_INFO, 'True')) + displayName: Convert ACL VHD to COSI + env: + RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME) + OFFER_NAME: $(OS_SKU) + IMG_CUSTOMIZER_ALLOW_FALLBACK: "true" + AFD_DOWNLOAD_HOSTNAME: $(AFD_DOWNLOAD_HOSTNAME) + COSI_CONTAINER: $(COSI_CONTAINER) + VHD_SAS_URL: $(VHD_SAS_URL) + + - task: AzureCLI@2 + inputs: + azureSubscription: $(COSI_PMC_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + bash ./vhdbuilder/packer/imagecustomizer/scripts/upload-cosi-to-pmc.sh + condition: and(succeeded(), eq(variables.GENERATE_PUBLISHING_INFO, 'True')) + displayName: Upload COSI to PMC + env: + AFD_UPLOAD_ENDPOINT: $(AFD_UPLOAD_ENDPOINT) + COSI_CONTAINER: $(COSI_CONTAINER) + + - task: PublishPipelineArtifact@1 + condition: and(succeeded(), eq(variables.GENERATE_PUBLISHING_INFO, 'True')) + displayName: Publish COSI Publishing Info + inputs: + artifactName: 'cosi-publishing-info-${{ parameters.artifactName }}' + targetPath: 'cosi-publishing-info.json' diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index be55156ce30..5248821744e 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -8,6 +8,9 @@ parameters: - name: overrideBranch type: string default: master + - name: enableCosiPublishing + type: boolean + default: false steps: - template: ./.template-override-components-json.yaml @@ -15,6 +18,28 @@ steps: overrideBranch: ${{ parameters.overrideBranch }} useOverrides: ${{ parameters.useOverrides }} + # For arm64 ACL: publish a source bundle for the separate native-arm64 COSI + # convert job (the arm64 pool has no git, so its `checkout: self` is disabled + # and it downloads this bundle instead). Must run AFTER the checkout above but + # BEFORE Download CIS scanner (which drops binaries into vhdbuilder/ that + # would otherwise bloat the tarball). + - ${{ if parameters.enableCosiPublishing }}: + - bash: | + set -euo pipefail + cd "$(Build.SourcesDirectory)" + tar --exclude='./.git' \ + -czf "$(Build.ArtifactStagingDirectory)/agentbaker-sources.tar.gz" . + ls -lh "$(Build.ArtifactStagingDirectory)/agentbaker-sources.tar.gz" + condition: and(succeeded(), eq(variables.GENERATE_PUBLISHING_INFO, 'True'), eq(variables['OS_SKU'], 'AzureContainerLinux'), eq(variables['ARCHITECTURE'], 'ARM64')) + displayName: Bundle source for arm64 COSI convert + + - task: PublishPipelineArtifact@1 + condition: and(succeeded(), eq(variables.GENERATE_PUBLISHING_INFO, 'True'), eq(variables['OS_SKU'], 'AzureContainerLinux'), eq(variables['ARCHITECTURE'], 'ARM64')) + displayName: Publish source bundle for arm64 COSI convert + inputs: + artifactName: agentbaker-sources-${{ parameters.artifactName }} + targetPath: $(Build.ArtifactStagingDirectory)/agentbaker-sources.tar.gz + - task: UniversalPackages@0 displayName: Download CIS scanner inputs: @@ -63,7 +88,7 @@ steps: downloadPath: $(Build.SourcesDirectory) - bash: | - set -x + set -ex GOPATH="$(go env GOPATH)" echo "GOPATH is currently set to $GOPATH" export PATH="$PATH:$GOPATH/bin" @@ -72,6 +97,7 @@ steps: popd cue export ./schemas/manifest.cue > ./parts/linux/cloud-init/artifacts/manifest.json displayName: Setup Cue + retryCountOnTaskFailure: 3 - bash: | SKU_NAME=${OS_VERSION} && \ @@ -165,6 +191,21 @@ steps: condition: always() displayName: Set Post-Build Variables + - ${{ if parameters.enableCosiPublishing }}: + - bash: | + image_version="${IMAGE_VERSION:-$(date -u +%Y%m.%d.0)}" + echo "##vso[task.setvariable variable=IMAGE_VERSION]${image_version}" + echo "##vso[task.setvariable variable=IMAGE_VERSION;isOutput=true]${image_version}" + name: resolveCosiImageVersion + displayName: Resolve COSI Image Version + condition: and(succeeded(), eq(variables['OS_SKU'], 'AzureContainerLinux'), eq(variables.GENERATE_PUBLISHING_INFO, 'True')) + + - bash: | + echo "##vso[task.setvariable variable=CAPTURED_SIG_VERSION;isOutput=true]${CAPTURED_SIG_VERSION}" + name: exportCapturedSigVersion + displayName: Export Captured SIG Version for arm64 COSI convert + condition: and(succeeded(), eq(variables.GENERATE_PUBLISHING_INFO, 'True'), eq(variables['OS_SKU'], 'AzureContainerLinux'), eq(variables['ARCHITECTURE'], 'ARM64')) + - bash: | set -euo pipefail if [ "${BUILDER:-}" = "imagecustomizer" ]; then @@ -434,6 +475,63 @@ steps: env: RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME) + - ${{ if parameters.enableCosiPublishing }}: + - task: AzureCLI@2 + inputs: + azureSubscription: $(VHD_ARM_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + make -f packer.mk convert-vhd-to-cosi + # ARM64 conversion runs in the separate native-arm64 jobs. + condition: | + and( + succeeded(), + eq(variables.GENERATE_PUBLISHING_INFO, 'True'), + eq(variables['OS_SKU'], 'AzureContainerLinux'), + ne(variables['ARCHITECTURE'], 'ARM64') + ) + displayName: Convert ACL VHD to COSI + env: + RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME) + OFFER_NAME: $(OS_SKU) + IMG_CUSTOMIZER_ALLOW_FALLBACK: "true" + AFD_DOWNLOAD_HOSTNAME: $(AFD_DOWNLOAD_HOSTNAME) + COSI_CONTAINER: $(COSI_CONTAINER) + + - task: AzureCLI@2 + inputs: + azureSubscription: $(COSI_PMC_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + make -f packer.mk upload-cosi-to-pmc + condition: | + and( + succeeded(), + eq(variables.GENERATE_PUBLISHING_INFO, 'True'), + eq(variables['OS_SKU'], 'AzureContainerLinux'), + ne(variables['ARCHITECTURE'], 'ARM64') + ) + displayName: Upload COSI to PMC + env: + AFD_UPLOAD_ENDPOINT: $(AFD_UPLOAD_ENDPOINT) + COSI_CONTAINER: $(COSI_CONTAINER) + + - task: PublishPipelineArtifact@1 + condition: | + and( + succeeded(), + eq(variables.GENERATE_PUBLISHING_INFO, 'True'), + eq(variables['OS_SKU'], 'AzureContainerLinux'), + ne(variables['ARCHITECTURE'], 'ARM64') + ) + displayName: Publish COSI Publishing Info + inputs: + artifactName: 'cosi-publishing-info-${{ parameters.artifactName }}' + targetPath: 'cosi-publishing-info.json' + + - task: AzureCLI@2 inputs: azureSubscription: $(VHD_ARM_SERVICE_CONNECTION) @@ -447,11 +545,15 @@ steps: echo "Successfully copied to immutable container" # Remove old VHD from staging container after copy to immutable container is complete azcopy remove "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" --recursive=true + # arm64 ACL is handled by the dedicated step below (immutable copy + SAS + # handoff on this x64 pool) because its native-arm64 convert pool is + # Conditional-Access-blocked from acquiring storage tokens. condition: | and( succeeded(), eq(variables.ENVIRONMENT, 'tme'), eq(variables.GENERATE_PUBLISHING_INFO, 'True'), + not(and(eq('${{ parameters.enableCosiPublishing }}', true), eq(variables['OS_SKU'], 'AzureContainerLinux'), eq(variables['ARCHITECTURE'], 'ARM64'))), or( eq(variables.RUN_PREFETCH_OPTIMIZATION, 'False'), eq(variables.PREFETCH_COMPATIBLE, 'False') @@ -459,6 +561,29 @@ steps: ) displayName: Copy to Immutable Storage Container + # arm64 ACL: its native-arm64 convert pool is Conditional-Access-blocked from + # acquiring storage tokens, so copy the VHD to the immutable container and drop + # the staging copy here (this x64 pool has storage auth), then hand the convert + # job a short read-only SAS so it downloads the VHD with no storage login. + - ${{ if parameters.enableCosiPublishing }}: + - task: AzureCLI@2 + inputs: + azureSubscription: $(VHD_ARM_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + bash ./vhdbuilder/packer/imagecustomizer/scripts/export-vhd-sas.sh + name: exportVhdSas + condition: | + and( + succeeded(), + eq(variables.ENVIRONMENT, 'tme'), + eq(variables.GENERATE_PUBLISHING_INFO, 'True'), + eq(variables['OS_SKU'], 'AzureContainerLinux'), + eq(variables['ARCHITECTURE'], 'ARM64') + ) + displayName: Copy arm64 ACL VHD to Immutable + mint convert SAS + - task: DownloadPipelineArtifact@2 displayName: Download Build Performance Program condition: and(succeeded(), eq(variables.ENVIRONMENT, 'tme')) diff --git a/cmd/cosi-upload/main.go b/cmd/cosi-upload/main.go new file mode 100644 index 00000000000..078f51ca90a --- /dev/null +++ b/cmd/cosi-upload/main.go @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +// Command cosi-upload streams a staged COSI artifact to PMC's Azure Front Door +// upload endpoint using the Azure Blob SDK's chunked block-blob upload. Because +// the SDK splits the file into blocks (Put Block + Put Block List), it is not +// subject to the single Put Blob size limit that a plain PUT hits on large COSI +// images. Authentication is Microsoft Entra ID via the Azure CLI login supplied +// by the pipeline's AzureCLI@2 task; the AFD endpoint transparently forwards the +// bearer token to the blob origin. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" +) + +func main() { + endpoint := flag.String("endpoint", "", "AFD upload endpoint base URL, e.g. https:// (required)") + container := flag.String("container", "", "destination container name (required)") + blob := flag.String("blob", "", "destination blob name (required)") + file := flag.String("file", "", "path to the local COSI file to upload (required)") + flag.Parse() + + if *endpoint == "" || *container == "" || *blob == "" || *file == "" { + log.Fatal("--endpoint, --container, --blob and --file are all required") + } + + if err := run(context.Background(), *endpoint, *container, *blob, *file); err != nil { + log.Fatalf("upload COSI: %v", err) + } +} + +func run(ctx context.Context, endpoint, container, blob, filePath string) error { + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("create Azure CLI credential: %w", err) + } + + client, err := azblob.NewClient(endpoint, cred, nil) + if err != nil { + return fmt.Errorf("create blob client for %s: %w", endpoint, err) + } + + log.Printf("Uploading %s to %s/%s/%s", filePath, endpoint, container, blob) + if err := uploadFile(ctx, client, container, blob, filePath); err != nil { + return fmt.Errorf("upload %s -> %s/%s/%s: %w", filePath, endpoint, container, blob, err) + } + log.Printf("Successfully uploaded %s/%s/%s", endpoint, container, blob) + return nil +} + +func uploadFile(ctx context.Context, client *azblob.Client, container, blob, filePath string) error { + file, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("open %s: %w", filePath, err) + } + defer func() { _ = file.Close() }() + + _, err = client.UploadFile(ctx, container, blob, file, &azblob.UploadFileOptions{ + BlockSize: 16 * 1024 * 1024, // 16 MiB blocks + Concurrency: 8, + }) + return err +} diff --git a/cmd/cosi-upload/main_test.go b/cmd/cosi-upload/main_test.go new file mode 100644 index 00000000000..1b7a79cff06 --- /dev/null +++ b/cmd/cosi-upload/main_test.go @@ -0,0 +1,134 @@ +package main + +import ( + "context" + "encoding/xml" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" + "github.com/stretchr/testify/require" +) + +type testCredential struct{} + +func (testCredential) GetToken(context.Context, policy.TokenRequestOptions) (azcore.AccessToken, error) { + return azcore.AccessToken{Token: "test-token", ExpiresOn: time.Now().Add(time.Hour)}, nil +} + +func newUploadTestClient(test *testing.T, handler http.HandlerFunc) *azblob.Client { + test.Helper() + server := httptest.NewTLSServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.Header.Get("Authorization") != "Bearer test-token" { + test.Error("upload request is missing its bearer token") + response.WriteHeader(http.StatusUnauthorized) + return + } + handler(response, request) + })) + test.Cleanup(server.Close) + client, err := azblob.NewClient(server.URL, testCredential{}, &azblob.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: server.Client(), + Retry: policy.RetryOptions{MaxRetries: -1}, + }, + }) + require.NoError(test, err) + return client +} + +func TestUploadFileMultipart(test *testing.T) { + file, err := os.CreateTemp(test.TempDir(), "*.cosi") + require.NoError(test, err) + fileSize := int64(blockblob.MaxUploadBlobBytes + 1) + require.NoError(test, file.Truncate(fileSize)) + require.NoError(test, file.Close()) + + var mutex sync.Mutex + blocks := make(map[string]int64) + var committed struct { + Latest []string `xml:"Latest"` + } + client := newUploadTestClient(test, func(response http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodPut || request.URL.Path != "/cosi/acl-arm64.cosi" { + test.Errorf("unexpected upload request: %s %s", request.Method, request.URL.Path) + response.WriteHeader(http.StatusBadRequest) + return + } + switch request.URL.Query().Get("comp") { + case "block": + size, copyErr := io.Copy(io.Discard, request.Body) + if copyErr != nil { + test.Error(copyErr) + response.WriteHeader(http.StatusInternalServerError) + return + } + mutex.Lock() + blocks[request.URL.Query().Get("blockid")] = size + mutex.Unlock() + case "blocklist": + mutex.Lock() + decodeErr := xml.NewDecoder(request.Body).Decode(&committed) + mutex.Unlock() + if decodeErr != nil { + test.Error(decodeErr) + response.WriteHeader(http.StatusBadRequest) + return + } + default: + test.Error("large file did not use multipart upload") + response.WriteHeader(http.StatusBadRequest) + return + } + response.WriteHeader(http.StatusCreated) + }) + + require.NoError(test, uploadFile(context.Background(), client, "cosi", "acl-arm64.cosi", file.Name())) + mutex.Lock() + defer mutex.Unlock() + require.Len(test, blocks, 17) + require.Len(test, committed.Latest, len(blocks)) + var totalBytes int64 + for _, blockID := range committed.Latest { + size, exists := blocks[blockID] + require.True(test, exists, "committed an unstaged block") + require.Positive(test, size) + require.LessOrEqual(test, size, int64(16*1024*1024)) + totalBytes += size + delete(blocks, blockID) + } + require.Equal(test, fileSize, totalBytes) + require.Empty(test, blocks) +} + +func TestUploadFileMissingFile(test *testing.T) { + client := newUploadTestClient(test, func(http.ResponseWriter, *http.Request) { + test.Error("missing file must not send an upload request") + }) + err := uploadFile(context.Background(), client, "cosi", "missing.cosi", filepath.Join(test.TempDir(), "missing.cosi")) + require.ErrorIs(test, err, os.ErrNotExist) +} + +func TestUploadFileHTTPError(test *testing.T) { + filePath := filepath.Join(test.TempDir(), "test.cosi") + require.NoError(test, os.WriteFile(filePath, []byte("test COSI"), 0600)) + client := newUploadTestClient(test, func(response http.ResponseWriter, request *http.Request) { + _, _ = io.Copy(io.Discard, request.Body) + response.Header().Set("x-ms-error-code", "AuthorizationPermissionMismatch") + response.WriteHeader(http.StatusForbidden) + }) + err := uploadFile(context.Background(), client, "cosi", "test.cosi", filePath) + var responseError *azcore.ResponseError + require.ErrorAs(test, err, &responseError) + require.Equal(test, http.StatusForbidden, responseError.StatusCode) + require.Equal(test, "AuthorizationPermissionMismatch", responseError.ErrorCode) +} diff --git a/go.mod b/go.mod index ee334a598b2..d3cee501d74 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,9 @@ module github.com/Azure/agentbaker go 1.26.0 require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 github.com/Azure/go-autorest/autorest/to v0.4.1 github.com/Masterminds/semver/v3 v3.5.0 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df @@ -20,7 +23,9 @@ require ( ) require ( + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect github.com/aws/aws-sdk-go-v2 v1.38.2 // indirect github.com/clarketm/json v1.17.1 // indirect github.com/coreos/go-json v0.0.0-20230131223807-18775e0fb4fb // indirect @@ -30,15 +35,18 @@ require ( github.com/coreos/vcontext v0.0.0-20230201181013-d72178a18687 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/golang-jwt/jwt/v5 v5.2.3 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/kr/text v0.2.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/nxadm/tail v1.4.8 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/spf13/pflag v1.0.9 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.51.0 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.37.0 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect ) diff --git a/go.sum b/go.sum index f82287fd07f..1ff6b91a2f9 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,31 @@ +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 h1:Wc1ml6QlJs2BHQ/9Bqu1jiyggbsSjramq2oUmp5WeIo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1/go.mod h1:JdM5psgjfBf5fo2uWOZhflPWyDBZ/O/CNAH9CtsuZE4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZYLBR2kBz5C8Tg0fw5w5Y7meRXWI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest/to v0.4.1 h1:CxNHBqdzTr7rLtdrtb5CMjJcDut+WNGCVv7OmS5+lTc= github.com/Azure/go-autorest/autorest/to v0.4.1/go.mod h1:EtaofgU4zmtvn1zT2ARsjRFdq9vXx0YWtmElwL+GZ9M= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/aws/aws-sdk-go-v2 v1.38.2 h1:QUkLO1aTW0yqW95pVzZS0LGFanL71hJ0a49w4TJLMyM= github.com/aws/aws-sdk-go-v2 v1.38.2/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY= github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0= github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/clarketm/json v1.17.1 h1:U1IxjqJkJ7bRK4L6dyphmoO840P6bdhPdbbLySourqI= github.com/clarketm/json v1.17.1/go.mod h1:ynr2LRfb0fQU34l07csRNBTcivjySLLiY1YzQqKVfdo= github.com/coreos/butane v0.25.1 h1:Nm2WDRD7h3f6GUpazGlge1o417Z+eIC9bQlkpgVdNms= @@ -21,9 +39,10 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/coreos/vcontext v0.0.0-20230201181013-d72178a18687 h1:uSmlDgJGbUB0bwQBcZomBTottKwEDF5fF8UjSwKSzWM= github.com/coreos/vcontext v0.0.0-20230201181013-d72178a18687/go.mod h1:Salmysdw7DAVuobBW/LwsKKgpyCPHUhjyJoMJD+ZJiI= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flatcar/ignition/v2 v2.0.0-20250903113522-05b8a773288c h1:MDbKEbAGtJuTZQ6Axnp3xEgh1nsJNW12rIIr0qexSGs= @@ -34,6 +53,8 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/golang-jwt/jwt/v5 v5.2.3 h1:kkGXqQOBSDDWRhWNXTFpqGSCMyh/PLnqUvMGJPDJDs0= +github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -46,6 +67,8 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE= github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= @@ -53,13 +76,14 @@ github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWS github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -71,9 +95,13 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.41.0 h1:OwKp4pXNgVxf6sCplzYo794OFNuoL2q2SBMU5NSWOjA= github.com/onsi/gomega v1.41.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.8.0 h1:q3nRvjrlge/6UD7eTu/DSg2uYiU2mCL0G/uzBWqhicI= +github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -94,6 +122,8 @@ go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -115,6 +145,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/packer.mk b/packer.mk index 74288a9804e..223453b061b 100755 --- a/packer.mk +++ b/packer.mk @@ -104,6 +104,24 @@ generate-publishing-info: az-login convert-sig-to-classic-storage-account-blob: az-login @./vhdbuilder/packer/convert-sig-to-classic-storage-account-blob.sh +convert-vhd-to-cosi: az-login + @./vhdbuilder/packer/imagecustomizer/scripts/convert-vhd-to-cosi.sh "$${IMG_CUSTOMIZER_ALLOW_FALLBACK:-false}" + +# Go 1.27 enables cgo-less OpenSSL by default and rejects the old experiment. +build-cosi-upload: + @echo "Building cosi-upload binary" + @mkdir -p bin + @go_experiment=""; \ + if GOEXPERIMENT=ms_nocgo_opensslcrypto go env GOEXPERIMENT >/dev/null 2>&1; then \ + go_experiment=ms_nocgo_opensslcrypto; \ + fi; \ + GOEXPERIMENT="$$go_experiment" CGO_ENABLED=0 go build -o bin/cosi-upload ./cmd/cosi-upload + @go version -m bin/cosi-upload | grep -q 'microsoft_systemcrypto=1$$' || \ + { echo "cosi-upload must be built with Microsoft Go system crypto" >&2; exit 1; } + +upload-cosi-to-pmc: build-cosi-upload + @./vhdbuilder/packer/imagecustomizer/scripts/upload-cosi-to-pmc.sh + scanning-vhd: az-login @./vhdbuilder/packer/vhd-scanning.sh diff --git a/spec/packer_spec.sh b/spec/packer_spec.sh new file mode 100644 index 00000000000..d1e266320b3 --- /dev/null +++ b/spec/packer_spec.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +Describe 'build-cosi-upload' + setup() { + export MOCK_GO_VERSION=1.27 + export MOCK_SYSTEM_CRYPTO=1 + export MOCK_BUILD_STATUS=0 + + go() { + case "$1 $2" in + 'env GOEXPERIMENT') + [ "$MOCK_GO_VERSION" = 1.26 ] + ;; + 'build -o') + printf 'GOEXPERIMENT=%s CGO_ENABLED=%s\n' "${GOEXPERIMENT-}" "${CGO_ENABLED-}" + return "$MOCK_BUILD_STATUS" + ;; + 'version -m') + printf '\tbuild\tmicrosoft_systemcrypto=%s\n' "$MOCK_SYSTEM_CRYPTO" + ;; + *) return 1 ;; + esac + } + export -f go + } + BeforeEach 'setup' + + It 'uses the OpenSSL experiment on Go 1.26' + MOCK_GO_VERSION=1.26 + When run make --no-print-directory -f packer.mk SHELL=/bin/bash build-cosi-upload + The status should be success + The output should include 'GOEXPERIMENT=ms_nocgo_opensslcrypto CGO_ENABLED=0' + End + + It 'uses default system crypto on Go 1.27' + When run make --no-print-directory -f packer.mk SHELL=/bin/bash build-cosi-upload + The status should be success + The output should include 'GOEXPERIMENT= CGO_ENABLED=0' + End + + It 'rejects a binary without system crypto' + MOCK_SYSTEM_CRYPTO=0 + When run make --no-print-directory -f packer.mk SHELL=/bin/bash build-cosi-upload + The status should be failure + The output should include 'Building cosi-upload binary' + The error should include 'cosi-upload must be built with Microsoft Go system crypto' + End + + It 'propagates a compiler failure' + MOCK_BUILD_STATUS=17 + When run make --no-print-directory -f packer.mk SHELL=/bin/bash build-cosi-upload + The status should be failure + The output should include 'Building cosi-upload binary' + The error should include 'Error 17' + End +End \ No newline at end of file diff --git a/spec/shellspec.Dockerfile b/spec/shellspec.Dockerfile index a7d9313712e..f478f9cba98 100644 --- a/spec/shellspec.Dockerfile +++ b/spec/shellspec.Dockerfile @@ -30,7 +30,7 @@ FROM aksdataplanedev.azurecr.io/shellspec/shellspec-debian:0.28.1 RUN sed -i -e 's/\(deb\|security\).debian.org/archive.debian.org/g' /etc/apt/sources.list && \ apt-get update && \ apt-get install -y --no-install-recommends \ - gawk jq curl dnsutils openssl \ + gawk jq curl dnsutils make openssl \ libcurl4 libdw1 libelf1 libssl1.1 zlib1g && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/spec/vhdbuilder/packer/imagecustomizer/convert_vhd_to_cosi_spec.sh b/spec/vhdbuilder/packer/imagecustomizer/convert_vhd_to_cosi_spec.sh new file mode 100644 index 00000000000..7bf1bbb746f --- /dev/null +++ b/spec/vhdbuilder/packer/imagecustomizer/convert_vhd_to_cosi_spec.sh @@ -0,0 +1,395 @@ +#!/bin/bash + +# Tests for generate_cosi_package_version in +# vhdbuilder/packer/imagecustomizer/scripts/convert-vhd-to-cosi.sh. +# +# Nebraska validates COSI package versions as strict SemVer, which rejects +# leading zeros in numeric components (e.g. 202608.06.0). These tests pin the +# normalization: leading zeros stripped, -fips appended only for FIPS builds, +# and never duplicated. + +Describe 'generate_cosi_package_version' + setup() { + # Source only the functions (guarded by ${__SOURCED__:+return}), not the + # main conversion flow. + # shellcheck disable=SC1090 + __SOURCED__=1 . "./vhdbuilder/packer/imagecustomizer/scripts/convert-vhd-to-cosi.sh" + } + BeforeEach 'setup' + + Describe 'strips per-component leading zeros' + It 'normalizes day 01 (non-FIPS)' + When call generate_cosi_package_version "202608.01.0" "false" + The status should be success + The output should equal "202608.1.0" + End + + It 'normalizes day 06 (non-FIPS)' + When call generate_cosi_package_version "202608.06.0" "false" + The status should be success + The output should equal "202608.6.0" + End + + It 'leaves day 10 unchanged (non-FIPS)' + When call generate_cosi_package_version "202608.10.0" "false" + The status should be success + The output should equal "202608.10.0" + End + End + + Describe 'appends -fips only for FIPS builds' + It 'emits -fips when ENABLE_FIPS is true' + When call generate_cosi_package_version "202608.06.0" "true" + The status should be success + The output should equal "202608.6.0-fips" + End + + It 'omits -fips when ENABLE_FIPS is false' + When call generate_cosi_package_version "202608.06.0" "false" + The status should be success + The output should equal "202608.6.0" + End + + It 'treats ENABLE_FIPS case-insensitively (True)' + When call generate_cosi_package_version "202608.10.2" "True" + The status should be success + The output should equal "202608.10.2-fips" + End + + It 'does not duplicate -fips when the version already contains it' + When call generate_cosi_package_version "202608.06.0-fips" "true" + The status should be success + The output should equal "202608.6.0-fips" + End + End + + Describe 'rejects a core that is not exactly three numeric components' + It 'fails with too few components' + When call generate_cosi_package_version "202608.06" "false" + The status should be failure + The error should be present + End + + It 'fails with too many components' + When call generate_cosi_package_version "202608.06.0.1" "false" + The status should be failure + The error should be present + End + + It 'fails with a non-numeric component' + When call generate_cosi_package_version "202608.aug.0" "false" + The status should be failure + The error should be present + End + End +End + +Describe 'VHD SAS handoff' + setup_handoff() { + export DESTINATION_STORAGE_CONTAINER=https://storage.invalid/staging + export CAPTURED_SIG_VERSION=test + export STORAGE_ACCOUNT_NAME=mock + export VHD_CONTAINER_NAME=immutable + export MOCK_DESTINATION_EXISTS=false + export MOCK_SOURCE_EXISTS=true + export MOCK_EXISTS_STATUS=0 + export MOCK_COPY_STATUS=success + export MOCK_COMPLETE_AFTER_WAIT=0 + export MOCK_START_STATUS=0 + export MOCK_SHOW_STATUS=0 + export MOCK_SAS_STATUS=0 + + az() { + case "$*" in + 'storage blob exists --account-name '*) + printf '%s\n' "$MOCK_DESTINATION_EXISTS" + return "$MOCK_EXISTS_STATUS" + ;; + 'storage blob exists --blob-url '*) + printf '%s\n' "$MOCK_SOURCE_EXISTS" + ;; + 'storage blob copy start '*) + echo COPY_STARTED + return "$MOCK_START_STATUS" + ;; + 'storage blob show '*) + printf '%s\n' "$MOCK_COPY_STATUS" + return "$MOCK_SHOW_STATUS" + ;; + 'storage blob generate-sas '*) + echo MOCK_SAS + return "$MOCK_SAS_STATUS" + ;; + *) return 1 ;; + esac + } + azcopy() { echo SOURCE_REMOVED; } + sleep() { + [ "$1" = 15 ] || return 1 + echo COPY_WAITED + if [ "$MOCK_COMPLETE_AFTER_WAIT" = 1 ]; then + MOCK_COPY_STATUS=success + fi + } + export -f az azcopy sleep + } + BeforeEach 'setup_handoff' + + It 'waits for a successful copy before exporting the secret SAS and deleting the source' + MOCK_COPY_STATUS=pending + MOCK_COMPLETE_AFTER_WAIT=1 + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/export-vhd-sas.sh + The status should be success + The line 3 of output should equal COPY_WAITED + The line 4 of output should equal '##vso[task.setvariable variable=VHD_SAS_URL;isOutput=true;issecret=true]MOCK_SAS' + The line 6 of output should equal SOURCE_REMOVED + End + + It 'reuses a successful immutable copy on retry' + MOCK_DESTINATION_EXISTS=true + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/export-vhd-sas.sh + The status should be success + The output should include MOCK_SAS + The output should include SOURCE_REMOVED + The output should not include COPY_STARTED + The output should not include COPY_WAITED + End + + It 'waits for an existing pending copy instead of overwriting it' + MOCK_DESTINATION_EXISTS=true + MOCK_COPY_STATUS=pending + MOCK_COMPLETE_AFTER_WAIT=1 + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/export-vhd-sas.sh + The status should be success + The line 2 of output should equal COPY_WAITED + The line 3 of output should equal '##vso[task.setvariable variable=VHD_SAS_URL;isOutput=true;issecret=true]MOCK_SAS' + The output should include SOURCE_REMOVED + The output should not include COPY_STARTED + End + + It 'refreshes the SAS when the staging source is already removed' + MOCK_DESTINATION_EXISTS=true + MOCK_SOURCE_EXISTS=false + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/export-vhd-sas.sh + The status should be success + The output should include MOCK_SAS + The output should include 'is already removed' + The output should not include COPY_STARTED + The output should not include SOURCE_REMOVED + End + + It 'stops when checking the destination fails' + MOCK_EXISTS_STATUS=1 + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/export-vhd-sas.sh + The status should be failure + The output should be blank + End + + It 'rejects an unknown destination existence result' + MOCK_DESTINATION_EXISTS=unknown + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/export-vhd-sas.sh + The status should be failure + The output should include 'Unable to determine whether the immutable VHD exists' + The output should not include COPY_STARTED + The output should not include MOCK_SAS + The output should not include SOURCE_REMOVED + End + + It 'does not delete a source with an unknown existence result' + MOCK_SOURCE_EXISTS=unknown + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/export-vhd-sas.sh + The status should be failure + The output should include 'Unable to determine whether the staging VHD exists' + The output should not include SOURCE_REMOVED + End + + Describe 'unsuccessful copy states' + Parameters + failed + aborted + # shellcheck disable=SC2286 + '' + End + It 'preserves the source when the copy is not successful' + MOCK_COPY_STATUS="$1" + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/export-vhd-sas.sh + The status should be failure + The output should include 'VHD copy to immutable container finished with status' + The output should not include MOCK_SAS + The output should not include SOURCE_REMOVED + End + End + + It 'preserves the source when the copy stays pending until timeout' + MOCK_COPY_STATUS=pending + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/export-vhd-sas.sh + The status should be failure + The output should include 'Timed out waiting for the immutable VHD copy' + The output should not include MOCK_SAS + The output should not include SOURCE_REMOVED + End + + It 'stops when starting the copy fails' + MOCK_START_STATUS=1 + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/export-vhd-sas.sh + The status should be failure + The output should include COPY_STARTED + The output should not include COPY_WAITED + The output should not include SOURCE_REMOVED + End + + It 'preserves the source when reading the copy status fails' + MOCK_SHOW_STATUS=1 + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/export-vhd-sas.sh + The status should be failure + The output should include COPY_STARTED + The output should not include MOCK_SAS + The output should not include SOURCE_REMOVED + End + + It 'preserves the source when SAS minting fails' + MOCK_SAS_STATUS=1 + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/export-vhd-sas.sh + The status should be failure + The output should include COPY_STARTED + The output should not include MOCK_SAS + The output should not include SOURCE_REMOVED + End +End + +Describe 'COSI artifact names' + setup_artifacts() { + convert_script="$(pwd)/vhdbuilder/packer/imagecustomizer/scripts/convert-vhd-to-cosi.sh" + upload_script="$(pwd)/vhdbuilder/packer/imagecustomizer/scripts/upload-cosi-to-pmc.sh" + test_dir="$(mktemp -d)" + mkdir -p "$test_dir/bin" + ln -s /bin/echo "$test_dir/bin/cosi-upload" + export CAPTURED_SIG_VERSION=202609.21.0 + export IMAGE_VERSION=202609.21.0 + export DESTINATION_STORAGE_CONTAINER=https://storage.invalid/vhds + export IMG_CUSTOMIZER_CONTAINER=registry.invalid/imagecustomizer@sha256:0000000000000000000000000000000000000000000000000000000000000000 + export IMG_CUSTOMIZER_CONTAINER_FALLBACK=registry.invalid/fallback@sha256:1111111111111111111111111111111111111111111111111111111111111111 + export EXPECTED_IMG_CUSTOMIZER_REF="$IMG_CUSTOMIZER_CONTAINER" + export MOCK_PRIMARY_PULL_FAIL=0 + export AFD_DOWNLOAD_HOSTNAME=download.invalid + export AFD_UPLOAD_ENDPOINT=https://upload.invalid + export COSI_CONTAINER=cosi + + azcopy() { return 0; } + docker() { + if [ "$1" = pull ] && [ "$4" = "$IMG_CUSTOMIZER_CONTAINER" ] && [ "$MOCK_PRIMARY_PULL_FAIL" = 1 ]; then + return 1 + elif [ "$1" = run ]; then + [ "$9" = "$EXPECTED_IMG_CUSTOMIZER_REF" ] || return 1 + printf 'mock COSI\n' > "$PWD/cosi-convert/out/${CAPTURED_SIG_VERSION}.cosi" + fi + } + export -f azcopy docker + } + cleanup_artifacts() { rm -rf "$test_dir"; } + BeforeEach 'setup_artifacts' + AfterEach 'cleanup_artifacts' + + check_artifact_name() { + cd "$test_dir" || return 1 + bash "$convert_script" "${1:-false}" >/dev/null || return $? + jq -r .cosi_url cosi-publishing-info.json + bash "$upload_script" + } + + check_build_version() { + cd "$test_dir" || return 1 + date() { printf '%s\n' 202610.01.0; } + export -f date + bash "$convert_script" >/dev/null || return $? + jq -r .image_version cosi-publishing-info.json + } + + It 'keeps the build version when conversion starts on a later UTC date' + export SKU_NAME=aclgen2arm64fipsTL ARCHITECTURE=ARM64 ENABLE_FIPS=true + When call check_build_version + The status should be success + The output should equal '202609.21.0-fips' + End + + It 'rejects conversion without the build-resolved version' + export SKU_NAME=aclgen2arm64TL + unset IMAGE_VERSION + When run bash "$convert_script" + The status should be failure + The output should equal 'IMAGE_VERSION was not set!' + End + + It 'rejects a mutable primary image before downloading or converting' + export SKU_NAME=aclgen2arm64TL IMG_CUSTOMIZER_CONTAINER=mcr.microsoft.com/azurelinux/imagecustomizer:1.5 + When run bash "$convert_script" + The status should be failure + The output should be blank + The error should include 'ImageCustomizer reference must be pinned with a sha256 digest' + End + + It 'rejects an enabled mutable fallback before downloading or converting' + export SKU_NAME=aclgen2arm64TL IMG_CUSTOMIZER_CONTAINER_FALLBACK=ghcr.io/microsoft/imagecustomizer:1.5.0 + When run bash "$convert_script" true + The status should be failure + The output should be blank + The error should include 'ImageCustomizer reference must be pinned with a sha256 digest' + End + + It 'rejects an enabled fallback without an approved digest' + export SKU_NAME=aclgen2arm64TL + unset IMG_CUSTOMIZER_CONTAINER_FALLBACK + When run bash "$convert_script" true + The status should be failure + The output should be blank + The error should include 'ImageCustomizer reference must be pinned with a sha256 digest' + End + + It 'runs the pinned fallback when the primary pull fails' + export SKU_NAME=aclgen2arm64TL ARCHITECTURE=ARM64 ENABLE_FIPS=false + MOCK_PRIMARY_PULL_FAIL=1 + EXPECTED_IMG_CUSTOMIZER_REF="$IMG_CUSTOMIZER_CONTAINER_FALLBACK" + When call check_artifact_name true + The status should be success + The line 1 of output should equal "https://download.invalid/cosi/${SKU_NAME}-${CAPTURED_SIG_VERSION}.cosi" + The line 2 of output should include "--blob ${SKU_NAME}-${CAPTURED_SIG_VERSION}.cosi" + End + + It 'stops when the primary pull fails and fallback is disabled' + export SKU_NAME=aclgen2arm64TL + MOCK_PRIMARY_PULL_FAIL=1 + When call check_artifact_name false + The status should be failure + The output should be blank + End + + Describe 'variants with a shared capture version' + Parameters + aclgen2TL X86_64 false + aclgen2fipsTL X86_64 true + aclgen2arm64TL ARM64 false + aclgen2arm64fipsTL ARM64 true + End + It 'uses the SKU in both the metadata URL and upload destination' + export SKU_NAME="$1" ARCHITECTURE="$2" ENABLE_FIPS="$3" + When call check_artifact_name + The status should be success + The line 1 of output should equal "https://download.invalid/cosi/${SKU_NAME}-${CAPTURED_SIG_VERSION}.cosi" + The line 2 of output should include "--blob ${SKU_NAME}-${CAPTURED_SIG_VERSION}.cosi --file ${test_dir}/${SKU_NAME}-${CAPTURED_SIG_VERSION}.cosi" + End + End + + It 'rejects conversion without a SKU' + unset SKU_NAME + When run bash "$convert_script" + The status should be failure + The output should equal 'SKU_NAME was not set!' + End + + It 'rejects upload without a SKU' + unset SKU_NAME + When run bash "$upload_script" + The status should be failure + The output should equal 'SKU_NAME was not set!' + End +End diff --git a/vhdbuilder/packer/imagecustomizer/scripts/convert-vhd-to-cosi.sh b/vhdbuilder/packer/imagecustomizer/scripts/convert-vhd-to-cosi.sh new file mode 100755 index 00000000000..4b3f0157a88 --- /dev/null +++ b/vhdbuilder/packer/imagecustomizer/scripts/convert-vhd-to-cosi.sh @@ -0,0 +1,229 @@ +#!/bin/bash +set -euo pipefail + +# Converts an ACL VHD to COSI and stages it for the separate "Upload COSI to PMC" +# task (which runs under PMC's service connection). + +# Nebraska requires strict SemVer, so strip per-component leading zeros (e.g. +# 202608.06.0 -> 202608.6.0) and append -fips for FIPS builds. +generate_cosi_package_version() { + local raw_version="$1" + local enable_fips="$2" + + # Drop any existing -fips so it is never duplicated; ENABLE_FIPS re-adds it. + local core="$raw_version" + case "${raw_version,,}" in + *-fips) core="${raw_version%-*}" ;; + esac + + # Require exactly three numeric components (major.minor.patch). + local major minor patch extra + IFS='.' read -r major minor patch extra <<<"$core" + if [ -n "$extra" ]; then + echo "generate_cosi_package_version: '${core}' has more than three components" >&2 + return 1 + fi + local component + for component in "$major" "$minor" "$patch"; do + case "$component" in + ''|*[!0-9]*) + echo "generate_cosi_package_version: '${core}' must have exactly three numeric components" >&2 + return 1 + ;; + esac + done + + # 10# forces base-10 so values like 06 are not parsed as octal. + local version="$((10#$major)).$((10#$minor)).$((10#$patch))" + if [ "${enable_fips,,}" = "true" ]; then + version="${version}-fips" + fi + printf '%s\n' "$version" +} + +validate_imagecustomizer_reference() { + if ! printf '%s\n' "$1" | grep -Eq '^[^[:space:]@]+@sha256:[0-9a-f]{64}$'; then + echo "##vso[task.logissue type=error]ImageCustomizer reference must be pinned with a sha256 digest" >&2 + return 1 + fi +} + +# Source-guard: functions above are unit-tested; the flow below skips when sourced. +${__SOURCED__:+return} + +required_env_vars=( + "DESTINATION_STORAGE_CONTAINER" + "CAPTURED_SIG_VERSION" + "SKU_NAME" + "IMAGE_VERSION" + "IMG_CUSTOMIZER_CONTAINER" + "AFD_DOWNLOAD_HOSTNAME" + "COSI_CONTAINER" +) + +for v in "${required_env_vars[@]}" +do + if [ -z "${!v:-}" ]; then + echo "$v was not set!" + exit 1 + fi +done + +# Both primary and opt-in fallback images must be pinned before privileged execution. +ALLOW_GHCR_FALLBACK="${1:-false}" +validate_imagecustomizer_reference "$IMG_CUSTOMIZER_CONTAINER" +if [ "${ALLOW_GHCR_FALLBACK,,}" = "true" ]; then + validate_imagecustomizer_reference "${IMG_CUSTOMIZER_CONTAINER_FALLBACK:-}" +fi + +WORK_DIR="$(pwd)/cosi-convert" +mkdir -p "$WORK_DIR/build" "$WORK_DIR/out" + +cleanup() { + echo "Cleaning up working directory $WORK_DIR" + rm -rf "$WORK_DIR" +} +trap cleanup EXIT + +VHD_BLOB_URL="${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" +LOCAL_VHD="$WORK_DIR/${CAPTURED_SIG_VERSION}.vhd" +LOCAL_COSI="$WORK_DIR/out/${CAPTURED_SIG_VERSION}.cosi" + +COSI_NAME="${SKU_NAME}-${CAPTURED_SIG_VERSION}.cosi" +COSI_DOWNLOAD_URL="https://${AFD_DOWNLOAD_HOSTNAME}/${COSI_CONTAINER}/${COSI_NAME}" + +# VHD_SAS_URL is set by the build job for the arm64 convert, whose pool has no +# storage-token path; downloading via the SAS URL needs no Azure AD login. The +# x64 convert leaves it unset and authenticates azcopy via the Azure CLI login. +DOWNLOAD_SRC="${VHD_SAS_URL:-$VHD_BLOB_URL}" + +echo "Setting azcopy environment variables" +export AZCOPY_CONCURRENCY_VALUE="AUTO" +export AZCOPY_LOG_LOCATION="$WORK_DIR/azcopy-log-files/" +export AZCOPY_JOB_PLAN_LOCATION="$WORK_DIR/azcopy-job-plan-files/" +if [ -z "${VHD_SAS_URL:-}" ]; then + export AZCOPY_AUTO_LOGIN_TYPE="AZCLI" +fi +mkdir -p "${AZCOPY_LOG_LOCATION}" +mkdir -p "${AZCOPY_JOB_PLAN_LOCATION}" + +echo "Downloading VHD ${CAPTURED_SIG_VERSION}.vhd" +if azcopy copy "$DOWNLOAD_SRC" "$LOCAL_VHD" --recursive=true; then + echo "Downloaded VHD to ${LOCAL_VHD}" +else + azExitCode=$? + shopt -s nullglob + for f in "${AZCOPY_LOG_LOCATION}"/*.log; do + echo "Azcopy log file: $f" + echo "##vso[build.uploadlog]$f" + if grep -q '"level":"Error"' "$f"; then + echo "log file $f contains errors" + echo "##vso[task.logissue type=error]Azcopy log file $f contains errors" + cat "$f" + fi + done + shopt -u nullglob + echo "Failed to download VHD, exiting with code $azExitCode" + exit "$azExitCode" +fi + +# Run the arch-matched ImageCustomizer image so its objcopy can read the VHD's +# UKI kernel cmdline. ARCH_LOWER is reused for image_architecture below. This +# convert job always runs on an agent whose host arch matches ARCHITECTURE (arm64 +# ACL converts run on a native arm64 pool via a separate job); no emulation. +ARCH_LOWER="${ARCHITECTURE:-}" +if [ "${ARCH_LOWER,,}" = "arm64" ]; then + IMG_CUSTOMIZER_PLATFORM="linux/arm64" +else + IMG_CUSTOMIZER_PLATFORM="linux/amd64" +fi + +IMG_CUSTOMIZER_REF="${IMG_CUSTOMIZER_CONTAINER}" + +echo "Pulling ImageCustomizer image ${IMG_CUSTOMIZER_REF} for platform ${IMG_CUSTOMIZER_PLATFORM}" +if ! docker pull --platform "${IMG_CUSTOMIZER_PLATFORM}" "${IMG_CUSTOMIZER_REF}"; then + if [ "${ALLOW_GHCR_FALLBACK,,}" != "true" ]; then + echo "##vso[task.logissue type=error]Failed to pull ImageCustomizer image ${IMG_CUSTOMIZER_REF} and GHCR fallback is disabled" + exit 1 + fi + + if [ -z "${IMG_CUSTOMIZER_CONTAINER_FALLBACK:-}" ]; then + echo "##vso[task.logissue type=error]GHCR fallback is enabled but IMG_CUSTOMIZER_CONTAINER_FALLBACK is not set" + exit 1 + fi + + IMG_CUSTOMIZER_REF="${IMG_CUSTOMIZER_CONTAINER_FALLBACK}" + echo "MCR image unavailable; falling back to ${IMG_CUSTOMIZER_REF}" + if ! docker pull --platform "${IMG_CUSTOMIZER_PLATFORM}" "${IMG_CUSTOMIZER_REF}"; then + echo "##vso[task.logissue type=error]Failed to pull ImageCustomizer fallback image ${IMG_CUSTOMIZER_REF}" + exit 1 + fi +fi + +# No -v /dev:/dev: IC 1.5's entrypoint mounts its own devtmpfs on /dev, so bind- +# mounting the host's fails "devtmpfs already mounted on /dev" (exit 32) under +# QEMU. -v /dev:/dev is only for IC v1.4 and below; v1.5+ uses --privileged alone. +echo "Converting VHD to COSI using ImageCustomizer ${IMG_CUSTOMIZER_REF} (${IMG_CUSTOMIZER_PLATFORM})" +docker run \ + --platform "${IMG_CUSTOMIZER_PLATFORM}" \ + --rm \ + --interactive \ + --privileged=true \ + -v "$WORK_DIR:/convert" \ + "${IMG_CUSTOMIZER_REF}" \ + convert \ + --log-level "debug" \ + --build-dir /convert/build \ + --image-file "/convert/${CAPTURED_SIG_VERSION}.vhd" \ + --output-image-file "/convert/out/${CAPTURED_SIG_VERSION}.cosi" \ + --output-image-format cosi + +if [ ! -f "$LOCAL_COSI" ]; then + echo "##vso[task.logissue type=error]COSI file was not created at ${LOCAL_COSI}" + exit 1 +fi + +# Move out of WORK_DIR (removed by the cleanup trap) so the upload task can find it. +STAGED_COSI="$(pwd)/${COSI_NAME}" +mv "$LOCAL_COSI" "$STAGED_COSI" +echo "Staged COSI for upload at ${STAGED_COSI}" + +# cosi-publishing-info.json for aks-rp 'cosi register' (needs both sha256 and sha1). +COSI_SHA256=$(sha256sum "$STAGED_COSI" | awk '{print $1}') +COSI_SHA1=$(sha1sum "$STAGED_COSI" | awk '{print $1}') +COSI_SIZE=$(stat -c%s "$STAGED_COSI") + +# Normalize only the Nebraska-facing version; IMAGE_VERSION itself is unchanged. +if ! COSI_IMAGE_VERSION="$(generate_cosi_package_version "${IMAGE_VERSION}" "${ENABLE_FIPS:-false}")"; then + echo "##vso[task.logissue type=error]Failed to derive a strict-SemVer COSI package version from IMAGE_VERSION='${IMAGE_VERSION}' (ENABLE_FIPS='${ENABLE_FIPS:-false}')" + exit 1 +fi +echo "Normalized COSI package version: ${COSI_IMAGE_VERSION} (from IMAGE_VERSION=${IMAGE_VERSION}, ENABLE_FIPS=${ENABLE_FIPS:-false})" + +# ARCH_LOWER is derived once near the top of the script and reused here. +if [ "${ARCH_LOWER,,}" = "arm64" ]; then + IMAGE_ARCH="Arm64" +else + IMAGE_ARCH="x64" +fi + +# OS_NAME is always Linux for ACL COSI artifacts +OS_NAME="Linux" + +cat < cosi-publishing-info.json +{ + "cosi_url": "${COSI_DOWNLOAD_URL}", + "sha256": "${COSI_SHA256}", + "sha1": "${COSI_SHA1}", + "size_bytes": ${COSI_SIZE}, + "os_name": "$OS_NAME", + "sku_name": "${SKU_NAME:-}", + "offer_name": "${OFFER_NAME:-}", + "hyperv_generation": "${HYPERV_GENERATION:-}", + "image_architecture": "${IMAGE_ARCH}", + "image_version": "${COSI_IMAGE_VERSION}" +} +EOF + +echo "Generated cosi-publishing-info.json:" +cat cosi-publishing-info.json diff --git a/vhdbuilder/packer/imagecustomizer/scripts/export-vhd-sas.sh b/vhdbuilder/packer/imagecustomizer/scripts/export-vhd-sas.sh new file mode 100644 index 00000000000..7354318383f --- /dev/null +++ b/vhdbuilder/packer/imagecustomizer/scripts/export-vhd-sas.sh @@ -0,0 +1,60 @@ +#!/bin/bash +set -euo pipefail + +export AZCOPY_AUTO_LOGIN_TYPE="AZCLI" +export AZCOPY_CONCURRENCY_VALUE="AUTO" +vhd_name="${CAPTURED_SIG_VERSION}.vhd" +source_uri="${DESTINATION_STORAGE_CONTAINER}/${vhd_name}" + +destination_exists="$(az storage blob exists --account-name "$STORAGE_ACCOUNT_NAME" \ + --container-name "$VHD_CONTAINER_NAME" --name "$vhd_name" --auth-mode login \ + --query exists --output tsv)" +case "${destination_exists,,}" in + false) + echo "Copying ${source_uri} to immutable container ${VHD_CONTAINER_NAME}" + az storage blob copy start --account-name "$STORAGE_ACCOUNT_NAME" \ + --destination-blob "$vhd_name" --destination-container "$VHD_CONTAINER_NAME" \ + --source-uri "$source_uri" --auth-mode login + ;; + true) echo "Immutable VHD ${vhd_name} already exists; checking copy status" ;; + *) + echo "##vso[task.logissue type=error]Unable to determine whether the immutable VHD exists" + exit 1 + ;; +esac +for ((attempt = 0; attempt < 120; attempt++)); do + copy_status="$(az storage blob show --account-name "$STORAGE_ACCOUNT_NAME" \ + --container-name "$VHD_CONTAINER_NAME" --name "$vhd_name" --auth-mode login \ + --query properties.copy.status --output tsv)" + case "$copy_status" in + success) break ;; + pending) sleep 15 ;; + *) + echo "##vso[task.logissue type=error]VHD copy to immutable container finished with status ${copy_status}" + exit 1 + ;; + esac +done +if [ "$copy_status" != "success" ]; then + echo "##vso[task.logissue type=error]Timed out waiting for the immutable VHD copy" + exit 1 +fi + +expiry="$(date -u -d '+8 hours' '+%Y-%m-%dT%H:%MZ')" +sas_url="$(az storage blob generate-sas --account-name "$STORAGE_ACCOUNT_NAME" \ + --container-name "$VHD_CONTAINER_NAME" --name "$vhd_name" --permissions r \ + --expiry "$expiry" --https-only --as-user --auth-mode login --full-uri --output tsv)" +echo "##vso[task.setvariable variable=VHD_SAS_URL;isOutput=true;issecret=true]$sas_url" +source_exists="$(az storage blob exists --blob-url "$source_uri" --auth-mode login \ + --query exists --output tsv)" +case "${source_exists,,}" in + true) + echo "Removing staging copy of ${vhd_name}" + azcopy remove "$source_uri" --recursive=true + ;; + false) echo "Staging VHD ${vhd_name} is already removed" ;; + *) + echo "##vso[task.logissue type=error]Unable to determine whether the staging VHD exists" + exit 1 + ;; +esac \ No newline at end of file diff --git a/vhdbuilder/packer/imagecustomizer/scripts/upload-cosi-to-pmc.sh b/vhdbuilder/packer/imagecustomizer/scripts/upload-cosi-to-pmc.sh new file mode 100755 index 00000000000..10f191d8202 --- /dev/null +++ b/vhdbuilder/packer/imagecustomizer/scripts/upload-cosi-to-pmc.sh @@ -0,0 +1,44 @@ +#!/bin/bash +set -euo pipefail + +# Uploads the COSI staged by convert-vhd-to-cosi.sh to PMC's AFD upload endpoint +# under PMC's service connection. Delegates the transfer to the cmd/cosi-upload Go +# tool, which uses the Azure Blob SDK's chunked block-blob upload (Put Block + Put +# Block List) so it is not subject to the single Put Blob size limit, and +# authenticates via the AzureCLI@2 task's Microsoft Entra login. + +required_env_vars=( + "CAPTURED_SIG_VERSION" + "SKU_NAME" + "AFD_UPLOAD_ENDPOINT" + "COSI_CONTAINER" +) + +for v in "${required_env_vars[@]}" +do + if [ -z "${!v:-}" ]; then + echo "$v was not set!" + exit 1 + fi +done + +COSI_WORK_DIR="$(pwd)" +COSI_NAME="${SKU_NAME}-${CAPTURED_SIG_VERSION}.cosi" +STAGED_COSI="${COSI_WORK_DIR}/${COSI_NAME}" + +if [ ! -f "$STAGED_COSI" ]; then + echo "##vso[task.logissue type=error]Staged COSI not found at ${STAGED_COSI}; the convert-vhd-to-cosi step must run first" + exit 1 +fi + +UPLOADER="${COSI_WORK_DIR}/bin/cosi-upload" +if [ ! -x "$UPLOADER" ]; then + echo "##vso[task.logissue type=error]cosi-upload binary not found at ${UPLOADER}; build it with 'go build -o bin/cosi-upload ./cmd/cosi-upload'" + exit 1 +fi + +"$UPLOADER" \ + --endpoint "${AFD_UPLOAD_ENDPOINT%/}" \ + --container "${COSI_CONTAINER}" \ + --blob "${COSI_NAME}" \ + --file "${STAGED_COSI}"