diff --git a/.azuredevops/build-matrix.json b/.azuredevops/build-matrix.json new file mode 100644 index 0000000000..afcfe4827f --- /dev/null +++ b/.azuredevops/build-matrix.json @@ -0,0 +1,21 @@ +[ + {"host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "VS2022", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "1", "shipping": true, "package_suffix": ""}, + {"host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "VS2022", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": true, "package_suffix": ""}, + {"host": "windows", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "0", "shipping": true, "package_suffix": ""}, + + {"host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "1", "shipping": false, "package_suffix": "XDCV"}, + {"host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRCV"}, + {"host": "windows", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ADCV"}, + {"host": "windows", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ARCV"}, + + {"host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "1", "shipping": false, "package_suffix": "XDCW"}, + {"host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRCW"}, + {"host": "windows", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ADCW"}, + + {"host": "linux", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "1", "shipping": false, "package_suffix": "XDCL"}, + {"host": "linux", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRCL"}, + {"host": "linux", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ADCL"}, + {"host": "linux", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ARCL"}, + + {"host": "linux", "arch": "X64", "target": "RELEASE", "tool_chain": "GCC", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRG"} +] diff --git a/.azuredevops/pipelines/platform-ci.yml b/.azuredevops/pipelines/platform-ci.yml new file mode 100644 index 0000000000..feac93ef37 --- /dev/null +++ b/.azuredevops/pipelines/platform-ci.yml @@ -0,0 +1,13 @@ +name: $(SourceBranchName)_$(Date:yyyyMMdd_HHmmss)$(Rev:.r) + +trigger: + batch: false + branches: + include: + - official/* +pr: none + +extends: + template: /.azuredevops/templates/platform.yml + parameters: + oneBranchBase: v2/Microsoft.Official.yml diff --git a/.azuredevops/pipelines/platform-pr.yml b/.azuredevops/pipelines/platform-pr.yml new file mode 100644 index 0000000000..d77490f23d --- /dev/null +++ b/.azuredevops/pipelines/platform-pr.yml @@ -0,0 +1,6 @@ +trigger: none + +extends: + template: /.azuredevops/templates/platform.yml + parameters: + oneBranchBase: v2/Microsoft.NonOfficial.yml diff --git a/.azuredevops/pipelines/sync.yml b/.azuredevops/pipelines/sync.yml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.azuredevops/templates/build.yml b/.azuredevops/templates/build.yml new file mode 100644 index 0000000000..c12c4a0738 --- /dev/null +++ b/.azuredevops/templates/build.yml @@ -0,0 +1,194 @@ +parameters: + - name: matrixFile + type: string + - name: officialBuild + type: boolean + default: false + - name: hosts + type: object + default: + - windows + - linux + - name: windowsClangVersion + type: string + default: '22.1.1-47' + - name: windowsClangMajorVersion + type: string + default: '22' + +jobs: + - job: BuildMatrix + displayName: Select Build Matrix + pool: + type: windows + variables: + WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest + ob_outputDirectory: $(Build.ArtifactStagingDirectory)/BUILD_MATRIX + ob_nugetPublishing_enabled: false + ob_createvpack_enabled: false + steps: + - template: /.azuredevops/templates/windows-python.yml@self + + - ${{ each host in parameters.hosts }}: + - pwsh: | + $matrix = python ci/scripts/matrix.py --matrix "$env:MATRIX_FILE" --host "$env:BUILD_HOST" --format ado + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + Write-Host "##vso[task.setvariable variable=matrix;isOutput=true]$matrix" + name: ${{ host }} + displayName: Select ${{ host }} Builds + env: + MATRIX_FILE: ${{ parameters.matrixFile }} + BUILD_HOST: ${{ host }} + + - ${{ each host in parameters.hosts }}: + - job: BuildFirmware_${{ host }} + displayName: Firmware (${{ host }}) + dependsOn: BuildMatrix + pool: + type: ${{ host }} + strategy: + maxParallel: 4 + matrix: $[ dependencies.BuildMatrix.outputs['${{ host }}.matrix'] ] + variables: + WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest + LinuxContainerImage: onebranch.azurecr.io/linux/ubuntu-2404:latest + GIT_CONFIG_COUNT: "2" + GIT_CONFIG_KEY_0: core.autocrlf + GIT_CONFIG_VALUE_0: "false" + GIT_CONFIG_KEY_1: core.longpaths + GIT_CONFIG_VALUE_1: "true" + ob_outputDirectory: $(Build.ArtifactStagingDirectory)/BUILD/$(System.JobId) + ob_artifactBaseName: $(build_artifact_name) + ob_pipelineartifacts_enabled: true + ob_artifacts_main_condition: succeededOrFailed() + ob_nugetPublishing_enabled: false + ob_createvpack_enabled: false + WindowsClangRoot: $(Agent.TempDirectory)/windows-clang-${{ parameters.windowsClangVersion }} + steps: + - task: NuGetAuthenticate@1 + inputs: + nuGetServiceConnections: WindowsPartners_NugetFeedConnection + + - task: PipAuthenticate@1 + inputs: + artifactFeeds: $(System.TeamProject)/$(System.TeamProject)_PublicPackages + + - ${{ if eq(host, 'windows') }}: + - template: /.azuredevops/templates/windows-python.yml@self + + - pwsh: | + $vswhere = "${env:ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere.exe" + $installation = & $vswhere -latest -version '[17.0,18.0)' -products '*' -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + if ($LASTEXITCODE -ne 0 -or -not $installation) { throw 'Visual Studio 2022 was not found' } + $clangBin = Join-Path $installation 'VC/Tools/Llvm/x64/bin' + if (-not (Test-Path (Join-Path $clangBin 'clang.exe'))) { throw 'Visual Studio Clang was not found' } + Write-Host "##vso[task.setvariable variable=CLANG_BIN]$clangBin/" + displayName: Locate Visual Studio Clang + condition: and(succeeded(), eq(variables['tool_chain'], 'CLANGPDB'), eq(variables['compiler_source'], 'visual_studio')) + + - task: DownloadPackage@1 + displayName: Download Windows-org LLVM Tools + condition: and(succeeded(), eq(variables['compiler_source'], 'windows_org')) + inputs: + packageType: nuget + feed: ES-LLVM + definition: llvm.tools + version: ${{ parameters.windowsClangVersion }} + downloadPath: $(WindowsClangRoot)/bin + + - task: DownloadPackage@1 + displayName: Download Windows-org Clang Headers + condition: and(succeeded(), eq(variables['compiler_source'], 'windows_org')) + inputs: + packageType: nuget + feed: ES-LLVM + definition: clang.headers + version: ${{ parameters.windowsClangVersion }} + downloadPath: $(WindowsClangRoot)/lib/clang/${{ parameters.windowsClangMajorVersion }} + + - template: /.azuredevops/templates/repo-auth.yml@self + parameters: + mode: set + + - pwsh: | + $buildArgs = @('--arch', $env:BUILD_ARCH, '--target', $env:BUILD_TARGET, + '--tool-chain', $env:BUILD_TOOL_CHAIN, '--core', $env:BUILD_CORE, + '--host', 'windows', '--compiler-source', $env:BUILD_COMPILER_SOURCE, + '--legacy-debugger', $env:BUILD_LEGACY_DEBUGGER, '--source-origin', 'closed') + if ($env:BUILD_COMPILER_SOURCE -eq 'windows_org') { + $buildArgs += @('--clang-bin', "$env:WINDOWS_CLANG_ROOT/bin") + } + python ci/scripts/build.py @buildArgs + exit $LASTEXITCODE + displayName: Build Firmware + env: + BUILD_ARCH: $(arch) + BUILD_TARGET: $(target) + BUILD_TOOL_CHAIN: $(tool_chain) + BUILD_CORE: $(core) + BUILD_COMPILER_SOURCE: $(compiler_source) + BUILD_LEGACY_DEBUGGER: $(legacy_debugger) + WINDOWS_CLANG_ROOT: $(WindowsClangRoot) + OFFICIAL_BUILD: ${{ iif(parameters.officialBuild, '1', '0') }} + + - task: PublishSymbols@2 + displayName: Index Shipping Symbols + condition: and(succeeded(), eq(variables['shipping'], 'true')) + inputs: + SearchPattern: Build/Msvm$(arch)/$(target)_$(tool_chain)/PDB/*.pdb + IndexSources: true + PublishSymbols: false + + - template: /.azuredevops/templates/repo-auth.yml@self + parameters: + mode: unset + + - template: /.azuredevops/templates/stage-artifacts.yml@self + + - ${{ if eq(host, 'linux') }}: + - template: /.azuredevops/templates/repo-auth.yml@self + parameters: + mode: set + python: python3 + + - bash: | + set -euo pipefail + apt-get update + apt-get install -y python3-venv + case "$BUILD_TOOL_CHAIN" in + CLANGPDB) apt-get install -y clang lld llvm ;; + GCC) + case "$BUILD_ARCH" in + X64) triple=x86_64; packages=x86-64 ;; + AARCH64) triple=aarch64; packages=aarch64 ;; + *) exit 2 ;; + esac + apt-get install -y "gcc-$packages-linux-gnu" "binutils-$packages-linux-gnu" + export "GCC_${BUILD_ARCH}_PREFIX=/usr/bin/$triple-linux-gnu-" + ;; + *) exit 2 ;; + esac + python3 -m venv "$BUILD_VENV" + source "$BUILD_VENV/bin/activate" + python ci/scripts/build.py --arch "$BUILD_ARCH" --target "$BUILD_TARGET" \ + --tool-chain "$BUILD_TOOL_CHAIN" --core "$BUILD_CORE" --host linux \ + --compiler-source "$BUILD_COMPILER_SOURCE" --legacy-debugger "$BUILD_LEGACY_DEBUGGER" --source-origin closed + displayName: Prepare Linux Compiler and Build Firmware + env: + BUILD_ARCH: $(arch) + BUILD_TARGET: $(target) + BUILD_TOOL_CHAIN: $(tool_chain) + BUILD_CORE: $(core) + BUILD_COMPILER_SOURCE: $(compiler_source) + BUILD_LEGACY_DEBUGGER: $(legacy_debugger) + BUILD_VENV: $(Agent.TempDirectory)/firmware-venv + OFFICIAL_BUILD: ${{ iif(parameters.officialBuild, '1', '0') }} + + - template: /.azuredevops/templates/repo-auth.yml@self + parameters: + mode: unset + python: python3 + + - template: /.azuredevops/templates/stage-artifacts.yml@self + parameters: + python: python3 diff --git a/.azuredevops/templates/platform.yml b/.azuredevops/templates/platform.yml new file mode 100644 index 0000000000..4242f118ac --- /dev/null +++ b/.azuredevops/templates/platform.yml @@ -0,0 +1,47 @@ +parameters: + - name: oneBranchBase + type: string + +resources: + repositories: + - repository: templates_onebranch + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + +extends: + template: ${{ parameters.oneBranchBase }}@templates_onebranch + parameters: + platform: + name: windows_undocked + globalSdl: + tsa: + enabled: false + featureFlags: + WindowsHostVersion: 1ESWindows2022 + nugetPublishing: + feeds: + - name: OS/HyperVUefi + continueOnConflict: false + files_to_publish: '*.nupkg;!*.symbols.nupkg' + git: + submodules: false + stages: + - stage: SourceChecks + jobs: + - template: /.azuredevops/templates/source-checks.yml@self + - stage: Build + dependsOn: [] + jobs: + - template: /.azuredevops/templates/build.yml@self + parameters: + matrixFile: .azuredevops/build-matrix.json + officialBuild: ${{ eq(parameters.oneBranchBase, 'v2/Microsoft.Official.yml') }} + - stage: Packages + dependsOn: [SourceChecks, Build] + ${{ if eq(parameters.oneBranchBase, 'v2/Microsoft.Official.yml') }}: + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/official/'), ne(variables['Build.Reason'], 'PullRequest')) + jobs: + - template: /.azuredevops/templates/publish.yml@self + parameters: + officialBuild: ${{ eq(parameters.oneBranchBase, 'v2/Microsoft.Official.yml') }} diff --git a/.azuredevops/templates/publish.yml b/.azuredevops/templates/publish.yml new file mode 100644 index 0000000000..5512bd02ec --- /dev/null +++ b/.azuredevops/templates/publish.yml @@ -0,0 +1,106 @@ +parameters: + - name: officialBuild + type: boolean + default: false + +jobs: + - job: PackageMatrix + displayName: Select Shipping Packages + pool: + type: windows + variables: + WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest + ob_outputDirectory: $(Build.ArtifactStagingDirectory)/PACKAGE_MATRIX + ob_nugetPublishing_enabled: false + ob_createvpack_enabled: false + steps: + - template: /.azuredevops/templates/windows-python.yml@self + + - pwsh: | + $matrix = python -m ci.scripts.matrix --matrix .azuredevops/build-matrix.json --shipping-only --format ado + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $version = python -m ci.scripts.package_artifacts version + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $major, $minor = $version.Split('.') + Write-Host "##vso[task.setvariable variable=matrix;isOutput=true]$matrix" + Write-Host "##vso[task.setvariable variable=major;isOutput=true]$major" + Write-Host "##vso[task.setvariable variable=minor;isOutput=true]$minor" + name: shipping + displayName: Select Shipping Flavors and Release Version + + - job: PackageFirmware + displayName: Package Firmware + dependsOn: PackageMatrix + pool: + type: windows + strategy: + maxParallel: 3 + matrix: $[ dependencies.PackageMatrix.outputs['shipping.matrix'] ] + variables: + WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest + ReleaseMajor: $[ dependencies.PackageMatrix.outputs['shipping.major'] ] + ReleaseMinor: $[ dependencies.PackageMatrix.outputs['shipping.minor'] ] + ob_outputDirectory: $(Build.ArtifactStagingDirectory)/PACKAGES + ob_artifactBaseName: $(artifact_name) + ob_nugetPublishing_enabled: ${{ parameters.officialBuild }} + ob_createvpack_enabled: ${{ parameters.officialBuild }} + ${{ if eq(parameters.officialBuild, true) }}: + ob_createvpack_packagename: $(Build.Repository.Name).mscoreuefi.$(arch).$(target) + ${{ else }}: + ob_createvpack_packagename: $(Build.Repository.Name).mscoreuefi.PR.$(arch).$(target).Test + ob_createvpack_description: Hyper V UEFI Firmware + ob_createvpack_versionAs: parts + ob_createvpack_majorVer: $(ReleaseMajor) + ob_createvpack_minorVer: $(ReleaseMinor) + ob_createvpack_metadata: $(Build.SourceVersion) + ob_createvpack_failOnStdErr: true + ob_createvpack_verbose: true + ob_createvpack_vpackdirectory: $(ob_outputDirectory)/Build/Msvm$(arch)/$(target)_$(tool_chain)/Bin + steps: + - template: /.azuredevops/templates/windows-python.yml@self + + - task: DownloadPipelineArtifact@2 + displayName: Download Shipping Firmware + inputs: + buildType: current + artifactName: $(build_artifact_name) + targetPath: $(Agent.TempDirectory)/firmware-input + + - pwsh: | + $packageArgs = @('nuget', '--matrix', '.azuredevops/build-matrix.json', + '--build-id', $env:BUILD_ID, '--source', $env:ARTIFACT_INPUT, + '--output', $env:PACKAGE_MANIFEST, '--staging-output', $env:ARTIFACT_OUTPUT, + '--commit', $env:SOURCE_COMMIT, + '--repository', $env:SOURCE_REPOSITORY, '--branch', $env:SOURCE_BRANCH) + if ($env:OFFICIAL_BUILD -eq '1') { $packageArgs += '--official' } + python -m ci.scripts.package_artifacts @packageArgs + exit $LASTEXITCODE + displayName: Validate Firmware and Prepare NuGet Manifest + env: + BUILD_ID: $(build_id) + ARTIFACT_INPUT: $(Agent.TempDirectory)/firmware-input + ARTIFACT_OUTPUT: $(ob_outputDirectory) + PACKAGE_MANIFEST: $(Agent.TempDirectory)/firmware.nuspec + SOURCE_COMMIT: $(Build.SourceVersion) + SOURCE_REPOSITORY: $(Build.Repository.Name) + SOURCE_BRANCH: $(Build.SourceBranchName) + OFFICIAL_BUILD: ${{ iif(parameters.officialBuild, '1', '0') }} + + - task: NuGetCommand@2 + displayName: Pack NuGet + inputs: + command: pack + packagesToPack: $(Agent.TempDirectory)/firmware.nuspec + packDestination: $(ob_outputDirectory)/packages + packTimezone: utc + + - ${{ if eq(parameters.officialBuild, true) }}: + - task: PublishSymbols@2 + displayName: Publish Shipping Symbols + inputs: + SymbolsFolder: $(ob_outputDirectory)/firmware/PDB + SearchPattern: '*.pdb' + IndexSources: false + PublishSymbols: true + SymbolServerType: TeamServices + SymbolsProduct: Hyper-V UEFI diff --git a/.azuredevops/templates/repo-auth.yml b/.azuredevops/templates/repo-auth.yml new file mode 100644 index 0000000000..c38949c789 --- /dev/null +++ b/.azuredevops/templates/repo-auth.yml @@ -0,0 +1,18 @@ +parameters: + - name: mode + type: string + values: + - set + - unset + - name: python + type: string + default: python + +steps: + - script: ${{ parameters.python }} ci/scripts/repo_auth.py ${{ parameters.mode }} --url https://dev.azure.com/microsoft/OS/_git/hyperv.uefi.mu_basecore + displayName: ${{ parameters.mode }} ADO Repository Authentication + ${{ if eq(parameters.mode, 'unset') }}: + condition: always() + ${{ if eq(parameters.mode, 'set') }}: + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/.azuredevops/templates/source-checks.yml b/.azuredevops/templates/source-checks.yml new file mode 100644 index 0000000000..4e6db59061 --- /dev/null +++ b/.azuredevops/templates/source-checks.yml @@ -0,0 +1,57 @@ +jobs: + - job: SourceChecks + displayName: Stuart CI + pool: + type: windows + variables: + WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest + GIT_CONFIG_COUNT: "2" + GIT_CONFIG_KEY_0: core.autocrlf + GIT_CONFIG_VALUE_0: "false" + GIT_CONFIG_KEY_1: core.longpaths + GIT_CONFIG_VALUE_1: "true" + ob_outputDirectory: $(Build.ArtifactStagingDirectory)/SOURCE_CHECKS + ob_artifactBaseName: drop_Build_SourceChecks + ob_pipelineartifacts_enabled: true + ob_artifacts_main_condition: succeededOrFailed() + ob_nugetPublishing_enabled: false + ob_createvpack_enabled: false + steps: + - template: /.azuredevops/templates/windows-python.yml@self + + - task: NuGetAuthenticate@1 + inputs: + nuGetServiceConnections: WindowsPartners_NugetFeedConnection + + - task: PipAuthenticate@1 + inputs: + artifactFeeds: $(System.TeamProject)/$(System.TeamProject)_PublicPackages + + - template: /.azuredevops/templates/repo-auth.yml@self + parameters: + mode: set + + - script: python ci/scripts/source_checks.py --verbose + displayName: Run Source Checks + env: + GIT_TRACE: '1' + GIT_TRACE_CURL: '1' + GIT_TRACE_CURL_NO_DATA: '1' + GIT_TRACE_REDACT: '1' + + - task: PublishTestResults@2 + displayName: Publish Stuart Results + condition: succeededOrFailed() + inputs: + testResultsFormat: JUnit + testResultsFiles: Build/TestSuites.xml + testRunTitle: Stuart CI + failTaskOnFailedTests: true + + - template: /.azuredevops/templates/repo-auth.yml@self + parameters: + mode: unset + + - script: python -m ci.scripts.stage_artifacts --logs-only --ado-layout --output "$(ob_outputDirectory)" + displayName: Stage Source Check Logs + condition: always() diff --git a/.azuredevops/templates/stage-artifacts.yml b/.azuredevops/templates/stage-artifacts.yml new file mode 100644 index 0000000000..3b0b631799 --- /dev/null +++ b/.azuredevops/templates/stage-artifacts.yml @@ -0,0 +1,19 @@ +parameters: + - name: python + type: string + default: python + +steps: + - script: >- + ${{ parameters.python }} -m ci.scripts.stage_artifacts + --matrix .azuredevops/build-matrix.json --build-id "$(build_id)" + --output "$(ob_outputDirectory)" --ado-layout --build-succeeded + displayName: Stage Firmware Artifacts + condition: succeeded() + + - script: >- + ${{ parameters.python }} -m ci.scripts.stage_artifacts + --matrix .azuredevops/build-matrix.json --build-id "$(build_id)" + --output "$(ob_outputDirectory)" --ado-layout + displayName: Stage Failure Diagnostics + condition: not(succeeded()) diff --git a/.azuredevops/templates/windows-python.yml b/.azuredevops/templates/windows-python.yml new file mode 100644 index 0000000000..35677e6d62 --- /dev/null +++ b/.azuredevops/templates/windows-python.yml @@ -0,0 +1,18 @@ +steps: + - pwsh: | + $python = py -3.12 -c "import sys; print(sys.executable)" + if ($LASTEXITCODE -ne 0 -or -not $python) { + throw 'The image Python launcher must provide Python 3.12' + } + $python = $python.Trim() + if (-not (Test-Path -LiteralPath $python -PathType Leaf)) { + throw "Image Python was not found at $python" + } + $pythonHome = Split-Path -Parent $python + & $python -c "import sys; print(sys.executable); print(sys.version); sys.exit(0 if sys.version_info[:2] == (3, 12) else 1)" + if ($LASTEXITCODE -ne 0) { throw 'The image must provide Python 3.12' } + & $python -m pip --version + if ($LASTEXITCODE -ne 0) { throw 'The image Python must provide pip' } + Write-Host "##vso[task.prependpath]$pythonHome\Scripts" + Write-Host "##vso[task.prependpath]$pythonHome" + displayName: Use Image Python 3.12 diff --git a/.github/actions/build-platform/action.yml b/.github/actions/build-platform/action.yml deleted file mode 100644 index 3e65bf5de0..0000000000 --- a/.github/actions/build-platform/action.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Build MSVM Platform -description: Build MSVM firmware with Stuart - -inputs: - architecture: - description: Target architecture - required: true - c-core: - description: Whether to use the legacy C core - required: true - official-build: - description: Set to 1 to mark the build as official in the firmware version record - required: false - default: "" - platform-config: - description: Path to the Stuart platform configuration - required: false - default: MsvmPkg/PlatformBuild.py - target: - description: Build target - required: true - tool-chain: - description: EDK II tool chain tag - required: true - -runs: - using: composite - steps: - - name: Stuart setup - shell: pwsh - run: >- - stuart_setup - -c "${{ inputs.platform-config }}" - TOOL_CHAIN_TAG=${{ inputs.tool-chain }} - - - name: Stuart update - shell: pwsh - run: >- - stuart_update - -c "${{ inputs.platform-config }}" - TOOL_CHAIN_TAG=${{ inputs.tool-chain }} - - - name: Stuart build - shell: pwsh - env: - OFFICIAL_BUILD: ${{ inputs.official-build }} - run: >- - stuart_build - -c "${{ inputs.platform-config }}" - --verbose - TOOL_CHAIN_TAG=${{ inputs.tool-chain }} - TARGET=${{ inputs.target }} - BUILD_ARCH=${{ inputs.architecture }} - BLD_*_USE_LEGACY_C_CORE=${{ inputs.c-core }} diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml new file mode 100644 index 0000000000..3d5765b597 --- /dev/null +++ b/.github/actions/build/action.yml @@ -0,0 +1,60 @@ +name: Build Firmware +description: Prepare and build one firmware flavor through Stuart + +inputs: + arch: + description: Target architecture + required: true + target: + description: Build target + required: true + tool-chain: + description: EDK II toolchain tag + required: true + core: + description: DXE core (legacy or patina) + required: true + host: + description: Build host + required: true + compiler-source: + description: Compiler distribution + required: true + legacy-debugger: + description: Enable the legacy debugger (0 or 1) + required: true + +runs: + using: composite + steps: + - name: Locate Visual Studio Clang + if: inputs.tool-chain == 'CLANGPDB' && inputs.compiler-source == 'visual_studio' + shell: pwsh + run: | + $vswhere = "${env:ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere.exe" + $installation = & $vswhere -latest -version '[17.0,18.0)' -products '*' -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + if ($LASTEXITCODE -ne 0 -or -not $installation) { throw 'Visual Studio 2022 was not found' } + $clangBin = Join-Path $installation 'VC/Tools/Llvm/x64/bin' + if (-not (Test-Path (Join-Path $clangBin 'clang.exe'))) { throw 'Visual Studio Clang was not found' } + "CLANG_BIN=$clangBin/" >> $env:GITHUB_ENV + + - name: Build firmware + shell: pwsh + env: + BUILD_ARCH: ${{ inputs.arch }} + BUILD_TARGET: ${{ inputs.target }} + BUILD_TOOL_CHAIN: ${{ inputs.tool-chain }} + BUILD_CORE: ${{ inputs.core }} + BUILD_HOST: ${{ inputs.host }} + BUILD_COMPILER_SOURCE: ${{ inputs.compiler-source }} + BUILD_LEGACY_DEBUGGER: ${{ inputs.legacy-debugger }} + run: >- + python ci/scripts/build.py + --arch "$env:BUILD_ARCH" + --target "$env:BUILD_TARGET" + --tool-chain "$env:BUILD_TOOL_CHAIN" + --core "$env:BUILD_CORE" + --host "$env:BUILD_HOST" + --compiler-source "$env:BUILD_COMPILER_SOURCE" + --legacy-debugger "$env:BUILD_LEGACY_DEBUGGER" + --source-origin open diff --git a/.github/actions/openvmm-test/action.yml b/.github/actions/openvmm-test/action.yml deleted file mode 100644 index dc3d6d2da5..0000000000 --- a/.github/actions/openvmm-test/action.yml +++ /dev/null @@ -1,260 +0,0 @@ -name: Test MSVM Firmware with OpenVMM -description: Boot every X64 MSVM firmware artifact with OpenVMM - -inputs: - github-token: - description: GitHub token used to query and download OpenVMM artifacts - required: true - -runs: - using: composite - steps: - - name: Download all X64 firmware artifacts - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - pattern: firmware-*-X64-* - path: ${{ runner.temp }}/firmware - - - name: Verify X64 firmware matrix - shell: bash - run: | - mapfile -t firmware_files < <( - find "${{ runner.temp }}/firmware" -type f -name MSVM.fd | sort - ) - - if [[ "${#firmware_files[@]}" -eq 0 ]]; then - echo "::error::No X64 MSVM.fd files were downloaded from the completed build matrix" - exit 1 - fi - - - name: Resolve latest usable OpenVMM main run - id: openvmm - shell: bash - env: - GH_TOKEN: ${{ inputs.github-token }} - run: | - response="$(gh api \ - "repos/microsoft/openvmm/actions/workflows/openvmm-ci.yaml/runs?branch=main&event=push&status=success&per_page=20")" - - run_id="" - head_sha="" - candidates="$(jq -r '.workflow_runs[] | [.id, .head_sha] | @tsv' <<< "$response")" - while IFS=$'\t' read -r candidate_id candidate_sha; do - [[ -z "$candidate_id" ]] && continue - artifact_pages="$(gh api --paginate --slurp \ - "repos/microsoft/openvmm/actions/runs/$candidate_id/artifacts?per_page=100")" - artifacts="$(jq -c '[.[].artifacts[]]' <<< "$artifact_pages")" - - if jq -e ' - [ - "x64-linux-vmm-tests-archive", - "x64-linux-openvmm", - "x64-linux-musl-pipette" - ] as $required - | [.[] | select(.expired == false) | .name] as $available - | all($required[]; . as $name | $available | index($name) != null) - ' <<< "$artifacts" > /dev/null; then - run_id="$candidate_id" - head_sha="$candidate_sha" - break - fi - - artifact_status="$(jq -r ' - [ - "x64-linux-vmm-tests-archive", - "x64-linux-openvmm", - "x64-linux-musl-pipette" - ] as $required - | [ - $required[] as $name - | if any(.[]; .name == $name and .expired == false) then empty - elif any(.[]; .name == $name) then "\($name) (expired)" - else "\($name) (missing)" - end - ] - | join(", ") - ' <<< "$artifacts")" - echo "Skipping OpenVMM run $candidate_id: $artifact_status" - done <<< "$candidates" - - if [[ -z "$run_id" || -z "$head_sha" ]]; then - echo "::error::No successful OpenVMM main run has all required artifacts" - exit 1 - fi - - echo "run-id=$run_id" >> "$GITHUB_OUTPUT" - echo "head-sha=$head_sha" >> "$GITHUB_OUTPUT" - echo "Using OpenVMM run $run_id at $head_sha" - - - name: Download OpenVMM X64 test archive - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - repository: microsoft/openvmm - run-id: ${{ steps.openvmm.outputs.run-id }} - github-token: ${{ inputs.github-token }} - name: x64-linux-vmm-tests-archive - path: ${{ runner.temp }}/openvmm-bundle - - - name: Download OpenVMM X64 binary - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - repository: microsoft/openvmm - run-id: ${{ steps.openvmm.outputs.run-id }} - github-token: ${{ inputs.github-token }} - name: x64-linux-openvmm - path: ${{ runner.temp }}/openvmm-bundle - - - name: Download OpenVMM X64 pipette - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - repository: microsoft/openvmm - run-id: ${{ steps.openvmm.outputs.run-id }} - github-token: ${{ inputs.github-token }} - name: x64-linux-musl-pipette - path: ${{ runner.temp }}/openvmm-bundle - - - name: Checkout matching OpenVMM source metadata - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: microsoft/openvmm - ref: ${{ steps.openvmm.outputs.head-sha }} - path: openvmm-source - persist-credentials: false - - - name: Download standalone cargo-nextest - id: nextest - shell: bash - run: | - version="$(sed -n 's/^pub const NEXTEST: &str = "\([^"]*\)";/\1/p' \ - openvmm-source/flowey/flowey_lib_hvlite/src/_jobs/cfg_versions.rs)" - - if [[ -z "$version" ]]; then - echo "::error::Could not determine OpenVMM cargo-nextest version" - exit 1 - fi - - destination="${{ runner.temp }}/nextest" - mkdir -p "$destination" - curl --fail --location \ - "https://get.nexte.st/$version/x86_64-unknown-linux-gnu.tar.gz" \ - --output "${{ runner.temp }}/nextest.tar.gz" - tar -xf "${{ runner.temp }}/nextest.tar.gz" -C "$destination" - chmod +x "$destination/cargo-nextest" - echo "binary=$destination/cargo-nextest" >> "$GITHUB_OUTPUT" - - - name: Download Ubuntu 25.04 test image - shell: bash - run: | - image_dir="${{ runner.temp }}/openvmm-images" - image="$image_dir/ubuntu-25.04-server-cloudimg-amd64.vhd" - expected_size=3758211584 - - mkdir -p "$image_dir" - curl --fail --location --retry 5 --retry-all-errors \ - "https://hvlitetestvhds.blob.core.windows.net/vhds/ubuntu-25.04-server-cloudimg-amd64.vhd" \ - --output "$image" - - actual_size="$(stat --format=%s "$image")" - if [[ "$actual_size" -ne "$expected_size" ]]; then - echo "::error::Unexpected Ubuntu VHD size: expected $expected_size, found $actual_size" - exit 1 - fi - - - name: Prepare Linux virtualization - shell: bash - run: | - if [[ ! -c /dev/kvm ]]; then - echo "::error::The runner does not expose /dev/kvm" - exit 1 - fi - - sudo chmod a+rw /dev/kvm - echo 4096 | sudo tee \ - /sys/kernel/mm/hugepages/hugepages-2048kB/nr_overcommit_hugepages - - bundle="${{ runner.temp }}/openvmm-bundle" - echo "Downloaded OpenVMM bundle:" - find "$bundle" -maxdepth 4 -printf '%y %p\n' | sort - - normalize_bundle_file() { - local name="$1" - local relative_destination="$2" - local destination="$bundle/$relative_destination" - local matches=() - - mapfile -d '' matches < <(find "$bundle" -type f -name "$name" -print0) - if [[ "${#matches[@]}" -ne 1 ]]; then - echo "::error::Expected exactly one $name in the OpenVMM bundle; found ${#matches[@]}" - printf 'Candidate: %s\n' "${matches[@]}" - return 1 - fi - - if [[ "${matches[0]}" != "$destination" ]]; then - mkdir -p "$(dirname "$destination")" - cp "${matches[0]}" "$destination" - fi - } - - normalize_bundle_file openvmm x86_64-unknown-linux-gnu/openvmm - normalize_bundle_file pipette x86_64-unknown-linux-musl/pipette - normalize_bundle_file vmm_tests.tar.zst vmm_tests.tar.zst - chmod +x \ - "$bundle/x86_64-unknown-linux-gnu/openvmm" \ - "$bundle/x86_64-unknown-linux-musl/pipette" - - - name: Boot Ubuntu with every X64 firmware - working-directory: openvmm-source - shell: bash - env: - BUNDLE: ${{ runner.temp }}/openvmm-bundle - FIRMWARE_ROOT: ${{ runner.temp }}/firmware - NEXTEST: ${{ steps.nextest.outputs.binary }} - RESULTS_ROOT: ${{ runner.temp }}/openvmm-test-results - VMM_TEST_IMAGES: ${{ runner.temp }}/openvmm-images - OPENVMM_REQUIRE_2MB_HUGETLB: "1" - run: | - mapfile -t firmware_files < <( - find "$FIRMWARE_ROOT" -type f -name MSVM.fd | sort - ) - - destination="$BUNDLE/x86_64" - mkdir -p "$destination" "$RESULTS_ROOT" - overall_result=0 - - for firmware in "${firmware_files[@]}"; do - relative="${firmware#"$FIRMWARE_ROOT"/}" - variant="${relative%%/*}" - result_dir="$RESULTS_ROOT/$variant" - temp_dir="${{ runner.temp }}/openvmm-test-temp/$variant" - - mkdir -p "$result_dir" "$temp_dir" - cp "$firmware" "$destination/MSVM.fd" - - echo "::group::Testing $variant" - if VMM_TESTS_CONTENT_DIR="$BUNDLE" \ - TEST_OUTPUT_PATH="$result_dir" \ - TMPDIR="$temp_dir" \ - "$NEXTEST" nextest run \ - --archive-file "$BUNDLE/vmm_tests.tar.zst" \ - --workspace-remap "$PWD" \ - --config-file "$PWD/.config/nextest.toml" \ - --profile ci \ - --filter-expr \ - "test(openvmm_uefi_x64_ubuntu_2504_server_x64_boot)"; then - echo "$variant: passed" - else - echo "::error::$variant failed" - overall_result=1 - fi - echo "::endgroup::" - done - - exit "$overall_result" - - - name: Upload OpenVMM test results - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: openvmm-x64-firmware-test-results - if-no-files-found: warn - path: ${{ runner.temp }}/openvmm-test-results/ diff --git a/.github/actions/release/action.yml b/.github/actions/release/action.yml index 79372b7be1..428d60d64e 100644 --- a/.github/actions/release/action.yml +++ b/.github/actions/release/action.yml @@ -1,10 +1,7 @@ -name: Release MSVM Firmware -description: Package MSVM firmware artifacts and publish a GitHub release +name: Release Firmware +description: Validate and archive the shared staged outputs, then publish a GitHub release inputs: - base-version: - description: Major and minor release version - required: true github-token: description: GitHub token used to create the release required: true @@ -12,32 +9,38 @@ inputs: runs: using: composite steps: - - name: Download build artifacts + - name: Download firmware artifacts uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: pattern: firmware-* - path: artifacts + path: ${{ runner.temp }}/firmware-artifacts - name: Package release artifacts shell: bash - run: | - mkdir -p release - for directory in artifacts/*/; do - name="$(basename "$directory")" - tar -czf "release/${name}.tar.gz" -C "$directory" . - done + env: + RELEASE_COMMIT: ${{ github.sha }} + ARTIFACT_INPUT: ${{ runner.temp }}/firmware-artifacts + RELEASE_OUTPUT: ${{ runner.temp }}/firmware-release + run: >- + python -m ci.scripts.package_artifacts github + --matrix .github/build-matrix.json + --source "$ARTIFACT_INPUT" + --output "$RELEASE_OUTPUT" + --commit "$RELEASE_COMMIT" - name: Create GitHub release shell: bash env: - BASE_VERSION: ${{ inputs.base-version }} GH_TOKEN: ${{ inputs.github-token }} + RELEASE_REPOSITORY: ${{ github.repository }} + RELEASE_COMMIT: ${{ github.sha }} + RELEASE_OUTPUT: ${{ runner.temp }}/firmware-release run: | - patch=$(gh release list --repo "${{ github.repository }}" --limit 1000 --json tagName \ - --jq '[.[] | .tagName | select(startswith("v'"${BASE_VERSION}"'.")) | ltrimstr("v'"${BASE_VERSION}"'.") | tonumber] | max // -1 | . + 1') - tag="v${BASE_VERSION}.${patch}" - gh release create "$tag" \ - --repo "${{ github.repository }}" \ - --target "${{ github.sha }}" \ - --generate-notes \ - release/*.tar.gz + set -euo pipefail + base_version="$(python -m ci.scripts.package_artifacts version)" + tags="$(gh api --paginate "repos/$RELEASE_REPOSITORY/releases?per_page=100" --jq '.[].tag_name')" + patch="$(printf '%s\n' "$tags" | jq -Rs --arg prefix "v$base_version." \ + 'split("\n") | map(select(startswith($prefix)) | ltrimstr($prefix) | select(test("^[0-9]+$")) | tonumber) | (max // -1) + 1')" + gh release create "v$base_version.$patch" \ + --repo "$RELEASE_REPOSITORY" --target "$RELEASE_COMMIT" \ + --generate-notes "$RELEASE_OUTPUT"/*.tar.gz diff --git a/.github/actions/source-checks/action.yml b/.github/actions/source-checks/action.yml new file mode 100644 index 0000000000..c450288362 --- /dev/null +++ b/.github/actions/source-checks/action.yml @@ -0,0 +1,9 @@ +name: Run Stuart Source Checks +description: Prepare the workspace and run the mu_msvm Stuart CI plugins + +runs: + using: composite + steps: + - name: Run source checks + shell: pwsh + run: python ci/scripts/source_checks.py diff --git a/.github/actions/stage-artifacts/action.yml b/.github/actions/stage-artifacts/action.yml new file mode 100644 index 0000000000..9a77e32cff --- /dev/null +++ b/.github/actions/stage-artifacts/action.yml @@ -0,0 +1,48 @@ +name: Stage Firmware Artifacts +description: Stage shared firmware outputs and diagnostics, then upload GitHub artifacts + +inputs: + build-id: + description: Selected matrix build ID + required: true + artifact-name: + description: Firmware artifact name + required: true + logs-artifact-name: + description: Diagnostic artifact name + required: true + build-succeeded: + description: Whether the firmware build succeeded + required: true + +runs: + using: composite + steps: + - name: Stage artifacts + if: always() + shell: pwsh + env: + BUILD_ID: ${{ inputs.build-id }} + BUILD_SUCCEEDED: ${{ inputs.build-succeeded }} + ARTIFACT_OUTPUT: ${{ runner.temp }}/ci-artifacts + run: | + $stageArgs = @('--matrix', '.github/build-matrix.json', '--build-id', $env:BUILD_ID, '--output', $env:ARTIFACT_OUTPUT) + if ($env:BUILD_SUCCEEDED -eq 'true') { $stageArgs += '--build-succeeded' } + python -m ci.scripts.stage_artifacts @stageArgs + exit $LASTEXITCODE + + - name: Upload firmware + if: success() && inputs.build-succeeded == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ inputs.artifact-name }} + path: ${{ runner.temp }}/ci-artifacts/firmware/ + if-no-files-found: error + + - name: Upload diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ inputs.logs-artifact-name }} + path: ${{ runner.temp }}/ci-artifacts/logs/ + if-no-files-found: warn diff --git a/.github/build-matrix.json b/.github/build-matrix.json new file mode 100644 index 0000000000..758bacbf89 --- /dev/null +++ b/.github/build-matrix.json @@ -0,0 +1,12 @@ +[ + {"host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, + {"host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, + {"host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "VS2022", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, + {"host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "VS2022", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, + {"host": "windows", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, + {"host": "windows", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, + {"host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "patina", "legacy_debugger": "0"}, + {"host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "patina", "legacy_debugger": "0"}, + {"host": "windows", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "patina", "legacy_debugger": "0"}, + {"host": "windows", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "patina", "legacy_debugger": "0"} +] diff --git a/.github/workflows/platform-ci.yml b/.github/workflows/platform-ci.yml index 788057f82f..098fd806c7 100644 --- a/.github/workflows/platform-ci.yml +++ b/.github/workflows/platform-ci.yml @@ -5,158 +5,132 @@ on: branches: - main pull_request: - branches: - - "*" workflow_dispatch: -# Bump this when releasing a new major/minor version. The patch number -# auto-increments from the latest existing GitHub release with this prefix. -env: - RELEASE_BASE_VERSION: "26.0" +permissions: + contents: read jobs: - source-checks: - name: Source Checks - runs-on: windows-2022 + build-matrix: + name: Select Build Matrix + runs-on: ubuntu-24.04 + outputs: + matrix: ${{ steps.select.outputs.matrix }} steps: - - name: Disable automatic line ending conversion - shell: pwsh - run: git config --global core.autocrlf false - - - name: Enable long paths - shell: pwsh - run: git config --system core.longpaths true - - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false - - name: Set up Python 3.12 + - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - cache: pip - cache-dependency-path: pip-requirements.txt - - - name: Install Python dependencies - shell: pwsh - run: python -m pip install -r pip-requirements.txt - - name: Run source checks - id: source-checks - uses: microsoft/mu_devops/.github/actions/stuart-ci@131433bb4f29d68250154cc66096f04c19fe3ee9 # v18.0.7 - with: - packages: MsvmPkg - targets: NO-TARGET - architectures: X64,AARCH64 - toolchain: VS2022 - stuart-setup: true - stuart-ci-setup: false - stuart-args: >- - --disable-all - GuidCheck=run - LineEndingCheck=run - UncrustifyCheck=run - - - name: Upload source check logs - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: source-check-logs - path: ${{ steps.source-checks.outputs.log-path }} + - name: Select open-source builds + id: select + shell: bash + run: | + matrix="$(python ci/scripts/matrix.py --matrix .github/build-matrix.json --format github)" + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" build: - name: ${{ matrix.target }} ${{ matrix.arch }} ${{ matrix.tools }}${{ matrix.dxe-core == 'patina' && ' DXE=patina' || '' }} + name: ${{ matrix.display_name }} + needs: build-matrix runs-on: windows-2022 strategy: fail-fast: false - matrix: - target: [DEBUG, RELEASE] - arch: [X64, AARCH64] - tools: [VS2022, CLANGPDB] - dxe-core: [legacy, patina] - exclude: - - arch: AARCH64 - tools: VS2022 - - tools: VS2022 - dxe-core: patina - + matrix: ${{ fromJSON(needs.build-matrix.outputs.matrix) }} steps: - - name: Enable long paths + - name: Configure Git shell: pwsh - run: git config --system core.longpaths true + run: | + git config --global core.autocrlf false + git config --system core.longpaths true - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false - - name: Set up Python 3.12 + - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: pip cache-dependency-path: pip-requirements.txt - - name: Install Python dependencies - shell: pwsh - run: python -m pip install -r pip-requirements.txt - - - name: Build platform - uses: ./.github/actions/build-platform + - name: Build firmware + id: firmware + uses: ./.github/actions/build + env: + OFFICIAL_BUILD: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') && '1' || '0' }} with: - architecture: ${{ matrix.arch }} - c-core: ${{ matrix.dxe-core == 'legacy' && 'TRUE' || 'FALSE' }} - # Mark main-branch builds as official in the embedded firmware version record. - official-build: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') && '1' || '' }} + arch: ${{ matrix.arch }} target: ${{ matrix.target }} - tool-chain: ${{ matrix.tools }} + tool-chain: ${{ matrix.tool_chain }} + core: ${{ matrix.core }} + host: ${{ matrix.host }} + compiler-source: ${{ matrix.compiler_source }} + legacy-debugger: ${{ matrix.legacy_debugger }} - - name: Upload build artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: firmware-${{ matrix.target }}-${{ matrix.arch }}-${{ matrix.tools }}${{ matrix.dxe-core == 'patina' && '-patina' || '' }} - if-no-files-found: error - path: | - Build/Msvm${{ matrix.arch }}/${{ matrix.target }}_${{ matrix.tools }}/FV/MSVM.fd - Build/Msvm${{ matrix.arch }}/${{ matrix.target }}_${{ matrix.tools }}/MAP/ - Build/Msvm${{ matrix.arch }}/${{ matrix.target }}_${{ matrix.tools }}/PDB/ - - - name: Upload build logs + - name: Stage and upload artifacts if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: ./.github/actions/stage-artifacts with: - name: logs-${{ matrix.target }}-${{ matrix.arch }}-${{ matrix.tools }}${{ matrix.dxe-core == 'patina' && '-patina' || '' }} - if-no-files-found: ignore - path: | - Build/**/BUILDLOG*.txt - Build/**/BUILD_REPORT.TXT - Build/**/OVERRIDELOG.TXT - Build/SETUP*.txt - Build/UPDATE*.txt - - openvmm-test-x64: - name: OpenVMM Test X64 - needs: build - runs-on: ubuntu-24.04 - permissions: - actions: read - contents: read + build-id: ${{ matrix.id }} + artifact-name: ${{ matrix.artifact_name }} + logs-artifact-name: ${{ matrix.logs_artifact_name }} + build-succeeded: ${{ steps.firmware.outcome == 'success' }} + + source-checks: + name: Source Checks + runs-on: windows-2022 steps: + - name: Configure Git + shell: pwsh + run: | + git config --global core.autocrlf false + git config --system core.longpaths true + - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - persist-credentials: false + submodules: true - - name: Test X64 firmware - uses: ./.github/actions/openvmm-test + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - github-token: ${{ github.token }} + python-version: "3.12" + cache: pip + cache-dependency-path: pip-requirements.txt + + - name: Run source checks + uses: ./.github/actions/source-checks + + - name: Stage source-check logs + if: always() + shell: pwsh + env: + ARTIFACT_OUTPUT: ${{ runner.temp }}/source-check-artifacts + run: python -m ci.scripts.stage_artifacts --logs-only --output "$env:ARTIFACT_OUTPUT" + + - name: Upload source-check logs + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: source-check-logs + path: ${{ runner.temp }}/source-check-artifacts/logs/ + if-no-files-found: warn release: name: Release if: github.event_name == 'push' && github.ref == 'refs/heads/main' - needs: [source-checks, build, openvmm-test-x64] - runs-on: ubuntu-latest + needs: [source-checks, build] + runs-on: ubuntu-24.04 permissions: contents: write + actions: read concurrency: group: release cancel-in-progress: false @@ -166,8 +140,12 @@ jobs: with: persist-credentials: false + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - name: Release firmware uses: ./.github/actions/release with: - base-version: ${{ env.RELEASE_BASE_VERSION }} github-token: ${{ github.token }} diff --git a/MsvmPkg/BuildPlugins/FirmwareVersionBlob/FirmwareVersionBlob.py b/MsvmPkg/BuildPlugins/FirmwareVersionBlob/FirmwareVersionBlob.py index 142001f13e..4af9f455f7 100644 --- a/MsvmPkg/BuildPlugins/FirmwareVersionBlob/FirmwareVersionBlob.py +++ b/MsvmPkg/BuildPlugins/FirmwareVersionBlob/FirmwareVersionBlob.py @@ -25,12 +25,32 @@ # SPDX-License-Identifier: BSD-2-Clause-Patent ## import logging +import json import os import struct -import tomllib +from typing import Literal, TypedDict, cast + +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib from edk2toolext.environment import repo_resolver from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin +from git import Repo + + +class FirmwareVersionMetadata(TypedDict): + """Portable sidecar identity; independent of the host-facing binary ABI.""" + + schema_version: int + struct_version: int + interface_major: int + interface_minor: int + base_version: str + git_commit: str + flags: int + source_origin: Literal["unknown", "open", "closed"] class FirmwareVersionBlob(IUefiBuildPlugin): @@ -53,6 +73,7 @@ class FirmwareVersionBlob(IUefiBuildPlugin): # (BUILD_OUTPUT_BASE). The FDF references the same location via # $(OUTPUT_DIRECTORY)/$(TARGET)_$(TOOL_CHAIN_TAG). FW_VERSION_BLOB_SUBPATH = os.path.join("FwVersion", "FwVersionBlob.bin") + FW_VERSION_METADATA_SUBPATH = os.path.join("FwVersion", "FirmwareVersion.json") # Single source of truth for the version numbers, relative to the workspace. FW_VERSION_TOML_SUBPATH = os.path.join("MsvmPkg", "FirmwareVersion.toml") @@ -64,7 +85,7 @@ def _ReadVersionToml(self, workspace, env): minor = int(data["interface"]["minor"]) # The CI BASE_VERSION env override wins so the release workflow can own # the released value; the TOML provides the in-tree default. - release = env.GetValue("BASE_VERSION", "") or str(data["release"]["version"]) + release = env.GetValue("BASE_VERSION", os.environ.get("BASE_VERSION", "")) or str(data["release"]["version"]) return major, minor, release def _GetGitCommit(self, workspace): @@ -74,23 +95,49 @@ def _GetGitCommit(self, workspace): # untracked files; gitignored build outputs do not count). try: details = repo_resolver.repo_details(workspace) - return details["Head"]["HexSha"], bool(details["Dirty"]) except Exception as e: logging.warning(f"Could not determine git state for firmware version blob: {e}") return "unknown", False + if details["Dirty"]: + with Repo(workspace) as repo: + status = repo.git.status("--short", "--untracked-files=all", "--ignore-submodules=none") + logging.warning("Firmware version dirty-worktree flag is set. Git status (paths only):\n%s", status) + return details["Head"]["HexSha"], bool(details["Dirty"]) + + @staticmethod + def _EncodeRecordString(value: str, capacity: int, name: str) -> bytes: + """Encode an ASCII field while reserving its required NUL terminator. + + Raise ValueError before macros or files are emitted if packing would + replace characters, truncate the value, or produce an ambiguous string. + """ + try: + encoded = value.encode("ascii") + except UnicodeEncodeError as error: + raise ValueError(f"{name} must contain only ASCII characters") from error + if not encoded or b"\0" in encoded or len(encoded) >= capacity: + raise ValueError(f"{name} must be nonempty, NUL-free, and at most {capacity - 1} ASCII bytes") + return encoded def do_pre_build(self, thebuilder): workspace = thebuilder.GetWorkspaceRoot() major, minor, base_version = self._ReadVersionToml(workspace, thebuilder.env) commit, dirty = self._GetGitCommit(workspace) + if not (0 <= major <= 0xFFFF and 0 <= minor <= 0xFFFF): + raise ValueError("Firmware interface major/minor must fit unsigned 16-bit fields") + encoded_base = self._EncodeRecordString(base_version, self.FW_VERSION_BASE_VERSION_SIZE, "BASE_VERSION") + encoded_commit = self._EncodeRecordString(commit, self.FW_VERSION_GIT_COMMIT_SIZE, "GitCommit") + origin = thebuilder.env.GetValue("SOURCE_ORIGIN", os.environ.get("SOURCE_ORIGIN", "unknown")).strip().lower() + if origin not in ("unknown", "open", "closed"): + raise ValueError("SOURCE_ORIGIN must be unknown, open, or closed") flags = 0 if dirty: flags |= self.FW_VERSION_FLAG_DIRTY # Official builds are marked by the CI pipeline (e.g. building main, not # a PR) via the OFFICIAL_BUILD environment variable. Any non-empty value # other than "0"/"false" counts as official. - if thebuilder.env.GetValue("OFFICIAL_BUILD", "").strip().lower() not in ("", "0", "false"): + if thebuilder.env.GetValue("OFFICIAL_BUILD", os.environ.get("OFFICIAL_BUILD", "")).strip().lower() not in ("", "0", "false"): flags |= self.FW_VERSION_FLAG_OFFICIAL # (1) Expose the values to the firmware as FixedAtBuild PCDs. Setting @@ -111,8 +158,8 @@ def do_pre_build(self, thebuilder): flags, major, minor, - base_version.encode("ascii", "replace"), - commit.encode("ascii", "replace"), + encoded_base, + encoded_commit, ) # Emit into the per-build output directory so the artifact stays out of @@ -125,11 +172,27 @@ def do_pre_build(self, thebuilder): with open(out_path, "wb") as f: f.write(blob) + metadata: FirmwareVersionMetadata = { + "schema_version": 1, + "struct_version": self.FW_VERSION_STRUCT_VERSION, + "interface_major": major, + "interface_minor": minor, + "base_version": base_version, + "git_commit": commit, + "flags": flags, + "source_origin": cast(Literal["unknown", "open", "closed"], origin), + } + metadata_path = os.path.join(out_dir, self.FW_VERSION_METADATA_SUBPATH) + with open(metadata_path, "w", encoding="utf-8") as metadata_file: + json.dump(metadata, metadata_file, indent=2) + metadata_file.write("\n") + logging.info( f"Firmware version blob written: base={base_version} commit={commit}" f"{' (dirty)' if dirty else ''}" f"{' (official)' if flags & self.FW_VERSION_FLAG_OFFICIAL else ''}" f" interface={major}.{minor}" + f" origin={origin}" f" -> {out_path}" ) return 0 diff --git a/MsvmPkg/Docs/BUILDING.md b/MsvmPkg/Docs/BUILDING.md index 742b102ca4..3453ac9878 100644 --- a/MsvmPkg/Docs/BUILDING.md +++ b/MsvmPkg/Docs/BUILDING.md @@ -26,35 +26,295 @@ stuart_build -c .\MsvmPkg\PlatformBuild.py TARGET=RELEASE stuart_build -c .\MsvmPkg\PlatformBuild.py BUILD_ARCH=AARCH64 ``` -## Running CI Checks Locally +## Building One CI Flavor Locally -Run the source checks from the repository root before opening a pull request. Install the Python dependencies and download the -tools used by the checks on a new checkout or whenever their configuration changes: +Use your activated Python virtual environment with the selected compiler installed and configured. +From the repository root, run: ```powershell -python -m pip install -r pip-requirements.txt -stuart_update -c .\.pytool\CISettings.py +python .\ci\scripts\build.py --arch X64 --target DEBUG --tool-chain CLANGPDB --core legacy --source-origin open ``` -Run the same source checks enforced by GitHub Actions: +The script installs Python requirements, then runs `stuart_setup`, `stuart_update`, and `stuart_build` +with `MsvmPkg/PlatformBuild.py`. All three Stuart commands receive the selected flavor. +Architecture, target, toolchain, and core are required; use `--core patina` to select the Rust DXE core. +Stuart executables are resolved from the invoking Python environment, and execution stops on the first failure. + +`--host windows|linux` defaults to the current host. `--compiler-source` distinguishes `visual_studio`, +`windows_org`, and `distribution`; it defaults to Visual Studio on Windows and distribution tools on Linux. +`--legacy-debugger 1` requires closed-source X64 VS2022 or CLANGPDB with the legacy core. +CLANGPDB includes Windows Visual Studio Clang, Windows-org Clang, and Linux distribution Clang. +GitHub/open builds, GCC, and Patina builds must use `0`. ARM64 MSVC is rejected before preparation. +Actual execution requires the selected host; `--dry-run` can preview either host's commands. + +Windows-org Clang requires an explicit `--clang-bin` directory; it never falls back to Visual Studio Clang. +For example, with the internal compiler tools and matching resource headers already installed: ```powershell -stuart_ci_build ` - -c .\.pytool\CISettings.py ` - -p MsvmPkg ` - -t NO-TARGET ` - -a X64,AARCH64 ` - --disable-all ` - GuidCheck=run ` - LineEndingCheck=run ` - UncrustifyCheck=run +python .\ci\scripts\build.py --arch AARCH64 --target RELEASE --tool-chain CLANGPDB --core legacy --host windows --compiler-source windows_org --clang-bin C:/tools/windows-clang/bin --source-origin closed +``` + +The local runner does not download private compiler packages or install system compilers. ADO provisions +Windows-org LLVM tools and headers from the pinned internal feed; its Linux jobs install distribution tools +and create a Python virtual environment. Local developers provide the equivalent toolchain environment. + +Add `--dry-run` to print the commands without installing dependencies or starting a build. +This command builds one flavor only; it does not select an open/closed repository matrix or publish artifacts. + +### Selecting Repository Coverage + +Coverage is maintained separately in `.github/build-matrix.json` (open-source mu_msvm) and +`.azuredevops/build-matrix.json` (closed-source hyperv.uefi). Both use the shared validator and build script. +Rows have no `repositories` field: the caller explicitly selects a file with `--matrix` instead of `--repo`. +The path selects coverage, not source code, credentials, official-build status, or publishing permissions. +Relative paths resolve from the current directory; either file can be selected locally. + +```powershell +python .\ci\scripts\matrix.py --matrix .github/build-matrix.json +python .\ci\scripts\matrix.py --matrix .azuredevops/build-matrix.json +``` + +The open profile contains ten Windows-hosted builds: DEBUG and RELEASE for X64 VS2022 with the legacy core, +plus X64 and AARCH64 CLANGPDB with each of the legacy and Patina cores. +The closed profile restores 15 legacy-core builds from the preserved hyperv.uefi matrix: + +| Host | Compiler source | Coverage | +| --- | --- | --- | +| Windows | VS2022 | X64 DEBUG and RELEASE | +| Windows | Visual Studio Clang | X64 and AARCH64, DEBUG and RELEASE | +| Windows | Windows-org Clang | X64 and AARCH64, DEBUG and RELEASE | +| Linux | Distribution Clang | X64 and AARCH64, DEBUG and RELEASE | +| Linux | Distribution GCC | X64 RELEASE only | + +The four closed X64 DEBUG rows enable the legacy debugger: VS2022 and all three Clang compiler sources. +This matches the fetched ADO configuration. All 15 closed flavors compiled in PR +build 160110571; runtime debugging still requires validation. +All open rows and all GCC rows disable the legacy debugger. +ARM64 MSVC is unsupported. Disabled GCC combinations are not reintroduced. + +Persistent IDs are derived as `______`; +do not enter IDs in the JSON. Identical flavors may occur in both files, but may not occur twice within one file. +Edit the respective JSON to change coverage independently, without editing shared Python or provider YAML. +The selector rejects duplicate flavors and fails when the supplied file or requested host has no builds. + +`--format github` emits an `include` object; `--format ado` emits a short job-leg name to variables mapping. +ADO labels such as `DEBUG_X64_VS2022_Debugger` or `RELEASE_AARCH64_ClangWin` are independent +of the full `build_id` carried in the variables. Artifact names and package identities do not change. +These commands only print JSON. They do not execute builds, install tools, or publish anything. +GitHub's platform workflow supplies `.github/build-matrix.json`, feeds the result to its native `strategy.matrix`, and +calls the build action for each flavor. That action invokes the same `ci/scripts/build.py` used locally. +Builds depend only on matrix generation; source checks run independently. Each build stages firmware and +diagnostics through `python -m ci.scripts.stage_artifacts`; the provider adapter uploads them. + +ADO's shared platform template supplies `.azuredevops/build-matrix.json` through its build job template. A `BuildMatrix` +job exports separate Windows and Linux matrices using `--host`. Dependent `BuildFirmware_windows` and +`BuildFirmware_linux` jobs consume them through native `strategy.matrix`, with at most four parallel +builds per host group. Host pools are selected at template expansion time, not by runtime matrix variables. +Each job runs inside OneBranch. The build stage +runs independently of source checks, and each flavor invokes the same `ci/scripts/build.py` as GitHub. +NuGet and VPack publishing remain disabled in build jobs. A separate Windows packaging stage depends on +both source checks and the entire build stage; only the three shipping flavors are selected. + +ADO source checks and both firmware build hosts use `repo-auth.yml` to call the +shared `ci/scripts/repo_auth.py` before Stuart. The adapter supplies the job's +`System.AccessToken` and private repository URL; the helper writes a URL-scoped +authorization header to the dedicated container's global Git configuration. +An `always()` cleanup step removes it even after failures. This follows the +working closed pipeline's authentication mechanism without changing shared build +or source-check runners. GitHub does not invoke this adapter. The helper disables +Git tracing while handling credentials and must not be used against a developer's +global Git configuration. + +All ADO Windows jobs resolve the image's Python 3.12 interpreter with +`py -3.12 -c "import sys; print(sys.executable)"`. The shared template verifies +the version and pip, then prepends the resolved interpreter and Scripts directories +to PATH. It does not assume an installation directory, download Python, or fall +back to the default Python on PATH. Linux builds retain their distribution Python +and isolated virtual environment. + +OneBranch PR build 160110571 passed source checks and all 15 firmware builds, +validating the native matrices, runner tools, internal feeds, and repository +authentication. Official builds and later pipeline changes still need hosted validation. + +### Job Labels and Artifact Names + +Machine IDs remain stable and complete. They are not the human-facing job labels or package names. +Provider matrix output adds a `display_name` derived from the same flavor: + +- GitHub: `DEBUG X64 CLANGPDB (Legacy)` or `RELEASE AARCH64 CLANGPDB (Patina)`. +- ADO: the same label followed by host and compiler source, such as `/ Windows WinOrg` or `/ Linux Distro`. + Debugger-enabled rows also include `/ Legacy debugger` in the generated metadata. + +ADO job names cannot use the runtime matrix's `$(display_name)` in this OneBranch +pipeline: it appeared literally in the hosted run. Instead, job display names use +the compile-time `Firmware (windows)` or `Firmware (linux)` prefix; ADO appends +short matrix labels to distinguish each flavor while retaining the full `build_id` in the variables. The generated +`display_name` remains available as metadata, not as the job's UI name. + +GitHub matrix output also supplies `artifact_name` and `logs_artifact_name` for the upload steps. +They preserve the pre-refactor consumer names exactly: + +| Flavor | Firmware artifact | Log artifact | +| --- | --- | --- | +| X64 DEBUG CLANGPDB legacy | `firmware-DEBUG-X64-CLANGPDB` | `logs-DEBUG-X64-CLANGPDB` | +| AARCH64 RELEASE CLANGPDB Patina | `firmware-RELEASE-AARCH64-CLANGPDB-patina` | `logs-RELEASE-AARCH64-CLANGPDB-patina` | + +The pattern is `firmware---[-patina]` (and `logs-` for logs). +Legacy has no suffix. Keep the literal `AARCH64` and `CLANGPDB` in artifact names for existing consumers. +If future host/compiler variants collide under this pattern, selection fails instead of silently renaming artifacts. +Any expanded naming scheme must be coordinated with consumers first. + +ADO retains `drop_Build___[_]`, independent of the new display names. +Non-shipping builds upload these drops directly. Shipping builds first upload `input-` for +the packaging stage, which publishes the historical drop with the NuGet under `packages/`. +Historical `Firmware Binary File ...`, `Firmware Map Files ...`, `Firmware PDB Files ...`, +`Build Logs `, and `Build/Msvm/_/Bin/` paths are retained +alongside the shared staging layout. Source-check output retains `drop_Build_SourceChecks` and its +root-level logs, test results, and `MsvmPkg/` diagnostics. No consumer migration is required. + +### Artifact Staging and Publication + +The shared staging module runs after Stuart, rather than as a post-build plugin, so setup/update failures +also retain diagnostics. Existing Stuart plugins still own firmware version generation and PDB/MAP collection. +Successful staging requires nonempty firmware, version JSON/blob, MAP files, and (except GCC) PDB files. +Shipping flavors additionally require DSDT AML and `BiosInterface.h`. Missing required outputs fail staging; +they are not silently omitted from a successful artifact. + +The shared layout is: + +- `firmware/FV/MSVM.fd`, `firmware/MAP/`, `firmware/PDB/`, `firmware/FwVersion/`, and + `firmware/BuildInfo.json` (the selected matrix flavor). +- `logs/`, preserving paths relative to `Build/` to prevent same-name logs overwriting each other. + Failed builds additionally retain the selected flavor's full build tree under `logs/failed-build/`. +- ADO builds: `package/Bin/` with `MSVM.fd`, `Dsdt.aml`, `BiosInterface.h`, PDBs (except GCC), + version JSON/blob, and the flavor manifest. This is staged data, not permission to publish a package. + +GitHub uploads the contents of `firmware/`, preserving the existing `FV/MSVM.fd`, `PDB/`, and `MAP/` +paths. Logs upload even when building or staging fails. Source-check logs have their own artifact. +ADO includes the tree and compatibility folders at the drop root. Failed ADO builds also retain the +historical `all/` copy of the full `Build/` tree. Shipping build failures retain the `input-...` drop; +the final historical shipping drop is only produced after its packaging prerequisites pass. +The legacy flattened log view is retained, with warnings on basename collisions; the shared `logs/` +tree always retains every original path. +OneBranch owns uploads; the shared Python does not handle credentials or network publication. + +GitHub releases run only for pushes to `main`, after all builds and source checks succeed. They download +exactly the expected ten firmware artifacts, validate flavor, commit, source origin, version and clean/official +flags, then create the historical `firmware-*.tar.gz` assets. Release tags remain `v..`; +the prefix comes from `FirmwareVersion.toml`, and the serialized release job allocates the next patch. +**OpenVMM is temporarily omitted from the release gate**, as agreed while restoring CI. Restore it before +claiming runtime validation of releases. PRs and manual GitHub runs do not create releases. + +ADO indexes shipping PDBs in the build workspace while sources and submodules are present. The downstream +Windows packaging jobs download the staged outputs, validate their provenance, and pack NuGet artifacts. +PRs exercise packaging with `.PR.` / `.Test` IDs but cannot publish to feeds, VPack, or the symbol server. +Official publication additionally requires `refs/heads/official/*` and a non-PR run. OneBranch publishing +enable flags are compile-time booleans, never runtime matrix shipping variables. NuGet goes to the existing +`OS/HyperVUefi` feed; OneBranch retains VPack version allocation; symbols are uploaded without reindexing. +This change does not claim hosted Official validation. +The governed VPack task supports `push`, `pull`, and `list`, not a local pack-only operation. +PRs validate/stage the VPack payload and pack NuGet without external publication; VPack creation/push +is Official-only. Historical pipelines packed non-shipping `.Test` flavors as well; the current three-flavor +package scope is an intentional change, separate from retaining CI artifacts for all 15 builds. + +PR run 160943879 passed all firmware builds but package validation rejected the dirty-worktree flag. +Validation reports the exact mismatched fields, and the version plugin logs dirty paths when computing +the flag. The hosted cause still requires another run; the dirty guard remains enforced for both PR +and Official packaging rather than silently clearing or ignoring provenance. + +### Closed Package Identity + +Closed rows have explicit `shipping` and `package_suffix` metadata. Shipping variants are X64 DEBUG/RELEASE +VS2022 legacy and AARCH64 RELEASE Windows-org CLANGPDB legacy. These three keep their empty package suffix; +all other closed variants retain their historical test suffixes. Metadata describes eligibility, not permission +to publish, and does not mark a local or PR build official. + +Build IDs are not package names. The shared VPack/NuGet ID convention is: +`.mscoreuefi..<.suffix if present><.Test if PR or non-shipping>`. +For example, the official ARM64 shipping ID remains `hyperv.uefi.mscoreuefi.AARCH64.RELEASE`, while its PR +ID is `hyperv.uefi.mscoreuefi.PR.AARCH64.RELEASE.Test`. +NuGet versions and OneBranch VPack version allocation remain separate contracts. +Only shipping flavors are packaged now. Non-shipping Windows/Linux builds retain their historical suffix +metadata but produce CI artifacts/logs only, not `.Test` packages or symbol-server uploads. +`matrix.py --shipping-only` selects the three shipping rows; it does not authorize publishing. +NuGet versions retain `.-.`, while VPack uses the same TOML prefix and +OneBranch's allocated version. Neither changes the already-compiled firmware's embedded build identity. +NuGet preserves the old recursive paths: `Msvm/_/FV/MSVM.fd`, +`symbols/Msvm/_/PDB/`, and `map/Msvm/_/MAP/`. + +## Firmware Versioning + +Think of three different labels: + +| Label | Example | What it answers | +| --- | --- | --- | +| Interface version | `1.0` | Can this firmware communicate correctly with this VMM? | +| Embedded build identity | `26.0` + Git SHA + flags | Which source produced this firmware? | +| Package version | GitHub release / NuGet / VPack version | Which published download is this? | + +The first two come from `MsvmPkg/FirmwareVersion.toml` and the checkout. The last is assigned during publishing. +A package number assigned after compilation is not embedded in that firmware. + +Inside the firmware today: + +- **Interface version:** bump major for a breaking firmware/VMM contract change, minor for a compatible addition. +- **Release prefix:** `26.0`, not the final package number. +- **Git SHA:** the actual checkout built, including the merge commit for a PR merge checkout. +- **Dirty flag:** the source had uncommitted changes. +- **Official flag:** CI marked the build official. This does not mean signed or shipping. + +The plugin writes matching firmware PCDs and an 80-byte version-1 record in the DXE FV. That binary layout +is unchanged. Future packaging must retain a mapping from package version to this build identity and flavor. + +### Open Versus Closed Source + +The build also writes `FwVersion/FirmwareVersion.json` under its build output directory. This sidecar +contains the same identity values plus **`source_origin`**: `open`, `closed`, or `unknown`. +GitHub explicitly selects `open`; ADO explicitly selects `closed`. Origin describes the source repository, +not the compiler supplier, shipping eligibility, or official status. + +Locally, pass `--source-origin open` or `--source-origin closed` to `ci/scripts/build.py`. +Omitting it records `unknown`, rather than guessing from remotes or inheriting a stale shell setting. +For direct Stuart invocations, set `SOURCE_ORIGIN=open|closed|unknown` as a build or shell variable. +An explicit Stuart value wins over the shell; an unrecognized value is rejected before output is written. + +Origin is currently **sidecar metadata, not embedded in MSVM.fd**. Staging and packaging retain the JSON. +Identifying origin from the firmware file alone needs a separately reviewed +binary-record extension. Origin and the official flag are declarations, not security attestations. + +### Overrides and CI Policy + +`BASE_VERSION` uses an explicit Stuart value first, then the shell, then TOML. An empty Stuart value suppresses +the shell override and uses TOML. It must fit 15 ASCII bytes plus a NUL terminator; interface fields must fit UINT16. + +`OFFICIAL_BUILD` uses Stuart-over-shell precedence too. Empty, `0`, and case-insensitive `false` mean unofficial; +other nonempty values mean official. Local builds default to unofficial unless explicitly overridden. + +- GitHub sets `1` only for a push to `main`; PRs and manual dispatches get `0`. +- ADO sets `1` for the Official entry point and `0` for NonOfficial PR builds, on both build hosts. + +An official pipeline can produce test-only flavors; a shipping flavor built locally is still unofficial. +Dirty and official flags can both be set. Public/private mirrors have different SHAs: source origin tells +you which repository to look in, not how to translate commits between them. + +## Running CI Checks Locally + +Run the same source-check workflow used by platform CI from the repository root: + +```powershell +python .\ci\scripts\source_checks.py ``` -The `--disable-all` option disables the default plugin set; each `Check=run` argument enables a check used by this repository. -To run one check while iterating, specify only that check. For example: +The workflow installs the Python requirements, runs `stuart_setup` and `stuart_update`, and then runs the configured +`stuart_ci_build` checks. Pass `--verbose` to enable detailed Stuart setup logging. +ADO source checks enable this option and Git command/HTTP tracing, with credential +redaction enabled and HTTP payload logging disabled, to diagnose submodule access +failures. GitHub and local runs retain their default logging. + +To run one check while iterating, invoke Stuart directly. For example: ```powershell -stuart_ci_build -c .\.pytool\CISettings.py -p MsvmPkg -t NO-TARGET -a X64,AARCH64 --disable-all GuidCheck=run +stuart_ci_build -c .\.pytool\CISettings.py --disable-all GuidCheck=run ``` ### Fixing Uncrustify Failures @@ -64,9 +324,6 @@ Run Uncrustify in correction mode to update incorrectly formatted files in place ```powershell stuart_ci_build ` -c .\.pytool\CISettings.py ` - -p MsvmPkg ` - -t NO-TARGET ` - -a X64,AARCH64 ` --disable-all ` UncrustifyCheck=run ` UNCRUSTIFY_IN_PLACE=TRUE @@ -74,6 +331,24 @@ stuart_ci_build ` Review the resulting changes and rerun the full source-check command before committing them. +## Developing CI Python + +Run these checks from the repository root in your activated virtual environment: + +```powershell +python -m pip install -r ci/requirements-dev.txt +python -m mypy --config-file ci/mypy.ini +python -B -m unittest discover -s ci/tests -v +``` + +Strict type checking covers both CI scripts and tests. Keep parameter and return annotations complete; +document input/output contracts, side effects, and failure behavior in public API docstrings. +The matrix module validates JSON into typed flavor records before using it, keeping provider serialization +separate from the input definition. Runtime validation is still required: type annotations alone cannot +validate JSON or command-line input. Tests mock external commands and do not build or download anything. + +These are local development checks; they have not been added to the hosted workflow yet. + ## Specialized Builds ### Debug-Enabled Build @@ -113,7 +388,7 @@ Build artifacts follow this path structure: **Path Variables:** - `{root}` = Root directory of the UEFI project - `{architecture}` = X64, AARCH64, etc. -- `{flavor}` = DEBUG or RELEASE +- `{flavor}` = DEBUG or RELEASE - `{toolchain}` = VS2022, GCC, etc. **Examples:** diff --git a/ci/mypy.ini b/ci/mypy.ini new file mode 100644 index 0000000000..58827d9777 --- /dev/null +++ b/ci/mypy.ini @@ -0,0 +1,5 @@ +[mypy] +strict = True +explicit_package_bases = True +files = ci/scripts, ci/tests +cache_dir = Build/mypy-cache diff --git a/ci/requirements-dev.txt b/ci/requirements-dev.txt new file mode 100644 index 0000000000..0900565228 --- /dev/null +++ b/ci/requirements-dev.txt @@ -0,0 +1 @@ +mypy>=1.15,<2 diff --git a/ci/scripts/build.py b/ci/scripts/build.py new file mode 100644 index 0000000000..ae9f202422 --- /dev/null +++ b/ci/scripts/build.py @@ -0,0 +1,127 @@ +"""Prepare and build one firmware flavor using native Stuart commands. + +The invoking Python environment supplies pip and Stuart executables. Setup, +update, and compilation run from the repository root in that order. Compiler +installation, matrix scheduling, credentials, and publishing belong to callers. +""" + +from __future__ import annotations + +import argparse +import os +import shlex +import subprocess +import sys +import sysconfig +from pathlib import Path +from typing import Literal + +REPO_ROOT = Path(__file__).resolve().parents[2] +PLATFORM_SETTINGS = "MsvmPkg/PlatformBuild.py" + + +class BuildArguments(argparse.Namespace): + """Typed CLI inputs populated and choice-validated by argparse. + + Host and compiler source describe execution, not the target architecture. + An explicit Clang directory affects child processes only. Dry runs validate + the requested flavor but do not require its host or compiler to be present. + """ + + arch: Literal["X64", "AARCH64"] + target: Literal["DEBUG", "RELEASE"] + tool_chain: Literal["VS2022", "CLANGPDB", "GCC"] + core: Literal["legacy", "patina"] + host: Literal["windows", "linux"] + compiler_source: Literal["visual_studio", "windows_org", "distribution"] | None + legacy_debugger: Literal["0", "1"] + source_origin: Literal["unknown", "open", "closed"] + clang_bin: Path | None + dry_run: bool + + +def main(argv: list[str] | None = None) -> int: + """Run one explicit flavor, stopping at the first failed command. + + Args: + argv: CLI arguments, or None to read the process command line. + + Returns: + Zero on success or dry run; otherwise the first subprocess exit code. + + Raises: + SystemExit: Help (0) or an invalid/unsupported invocation (2). + OSError: A subprocess cannot be started, for example missing Stuart. + """ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--arch", required=True, choices=("X64", "AARCH64")) + parser.add_argument("--target", required=True, choices=("DEBUG", "RELEASE")) + parser.add_argument("--tool-chain", required=True, choices=("VS2022", "CLANGPDB", "GCC")) + parser.add_argument("--core", required=True, choices=("legacy", "patina")) + parser.add_argument("--host", choices=("windows", "linux"), default="windows" if os.name == "nt" else "linux") + parser.add_argument("--compiler-source", choices=("visual_studio", "windows_org", "distribution")) + parser.add_argument("--legacy-debugger", choices=("0", "1"), default="0") + parser.add_argument("--source-origin", choices=("unknown", "open", "closed"), default="unknown", + help="Source repository provenance for firmware metadata; never inferred from the host") + parser.add_argument("--clang-bin", type=Path, help="Explicit compiler bin directory; required for Windows-org Clang") + parser.add_argument("--dry-run", action="store_true", help="Print commands without executing them") + args = parser.parse_args(argv, namespace=BuildArguments()) + + compiler_source = args.compiler_source or ("visual_studio" if args.host == "windows" else "distribution") + if args.arch == "AARCH64" and args.tool_chain == "VS2022": + parser.error("ARM64 MSVC is not supported") + if args.legacy_debugger == "1" and ( + args.source_origin != "closed" or args.arch != "X64" or args.core != "legacy" + or args.tool_chain not in ("VS2022", "CLANGPDB") + ): + parser.error("Legacy debugger requires closed-source X64 VS2022 or CLANGPDB with the legacy core") + if (args.host, args.tool_chain, compiler_source) not in { + ("windows", "VS2022", "visual_studio"), ("windows", "CLANGPDB", "visual_studio"), + ("windows", "CLANGPDB", "windows_org"), ("linux", "CLANGPDB", "distribution"), + ("linux", "GCC", "distribution"), + }: + parser.error("Unsupported host/compiler combination") + if compiler_source == "windows_org" and args.clang_bin is None: + parser.error("Windows-org Clang requires --clang-bin; Visual Studio fallback is not allowed") + if not args.dry_run: + if args.host != ("windows" if os.name == "nt" else "linux"): + parser.error(f"This flavor requires a {args.host} host") + if args.clang_bin is not None and not (args.clang_bin / ("clang.exe" if args.host == "windows" else "clang")).is_file(): + parser.error("--clang-bin must contain the selected Clang executable") + + flavor: list[str] = [ + f"BUILD_ARCH={args.arch}", + f"TARGET={args.target}", + f"TOOL_CHAIN_TAG={args.tool_chain}", + f"BLD_*_USE_LEGACY_C_CORE={'TRUE' if args.core == 'legacy' else 'FALSE'}", + ] + flavor.append(f"BLD_*_LEGACY_DEBUGGER={args.legacy_debugger}") + environment: dict[str, str] = os.environ.copy() + environment["SOURCE_ORIGIN"] = args.source_origin + if args.clang_bin is not None: + environment["CLANG_BIN"] = f"{args.clang_bin.resolve().as_posix()}/" + print(f"CLANG_BIN={environment['CLANG_BIN']}", flush=True) + scripts_dir = Path(sysconfig.get_path("scripts")) + suffix = ".exe" if os.name == "nt" else "" + commands: list[list[str]] = [[sys.executable, "-m", "pip", "install", "-r", "pip-requirements.txt"]] + for tool in ("stuart_setup", "stuart_update", "stuart_build"): + commands.append([str(scripts_dir / (tool + suffix)), "-c", PLATFORM_SETTINGS, *flavor]) + commands[-1].extend([ + "BUILDREPORTING=TRUE", + "BUILDREPORT_TYPES=PCD DEPEX FLASH BUILD_FLAGS LIBRARY", + "LaunchLogOnSuccess=FALSE", + "LaunchLogOnError=FALSE", + ]) + + for command in commands: + print(f"==> {shlex.join(command)}", flush=True) + if not args.dry_run: + try: + subprocess.run(command, cwd=REPO_ROOT, check=True, env=environment) + except subprocess.CalledProcessError as error: + return error.returncode + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/scripts/matrix.py b/ci/scripts/matrix.py new file mode 100644 index 0000000000..cfdead1fe2 --- /dev/null +++ b/ci/scripts/matrix.py @@ -0,0 +1,266 @@ +"""Validate repository coverage and emit provider-specific matrix JSON. + +This module selects data only: it never builds firmware or authorizes publication. +All rows in the supplied file are validated before host filtering. +Generated IDs identify executions; package suffixes are a +separate consumer-facing contract. +""" + +from __future__ import annotations + +import argparse +import json +import re +from collections.abc import Mapping +from pathlib import Path +from typing import Literal, NotRequired, TypedDict, cast + +Host = Literal["windows", "linux"] +Architecture = Literal["X64", "AARCH64"] +Target = Literal["DEBUG", "RELEASE"] +ToolChain = Literal["VS2022", "CLANGPDB", "GCC"] +Core = Literal["legacy", "patina"] +CompilerSource = Literal["visual_studio", "windows_org", "distribution"] +Debugger = Literal["0", "1"] +OutputFormat = Literal["json", "github", "ado"] + + +class BuildFlavor(TypedDict): + """Execution inputs shared by build selection and provider callers.""" + + arch: Architecture + target: Target + tool_chain: ToolChain + core: Core + host: Host + compiler_source: CompilerSource + legacy_debugger: Debugger + + +class BuildDefinition(BuildFlavor): + """Validated JSON row; optional package metadata must supply both fields.""" + + shipping: NotRequired[bool] + package_suffix: NotRequired[str] + + +class SelectedBuild(BuildFlavor): + """Provider row with generated identity and optional closed package metadata. + + Shipping is serialized as text because ADO matrix values are variables. + It describes eligibility, not permission to publish an official package. + """ + + id: str + display_name: NotRequired[str] + artifact_name: NotRequired[str] + logs_artifact_name: NotRequired[str] + build_id: NotRequired[str] + build_artifact_name: NotRequired[str] + shipping: NotRequired[str] + package_suffix: NotRequired[str] + + +class MatrixArguments(argparse.Namespace): + """CLI values constrained by argparse choices before selection.""" + + matrix: Path + host: Host | None + format: OutputFormat + shipping_only: bool + +FLAVOR_CHOICES: dict[str, tuple[str, ...]] = { + "arch": ("X64", "AARCH64"), + "target": ("DEBUG", "RELEASE"), + "tool_chain": ("VS2022", "CLANGPDB", "GCC"), + "core": ("legacy", "patina"), +} +EXECUTION_CHOICES: dict[str, tuple[str, ...]] = { + "host": ("windows", "linux"), + "compiler_source": ("visual_studio", "windows_org", "distribution"), + "legacy_debugger": ("0", "1"), +} + + +def build_id(row: BuildFlavor) -> str: + """Derive a stable execution ID, independent of package names or repo mode.""" + return "_".join((row["host"], row["arch"], row["target"], row["tool_chain"], + row["compiler_source"], row["core"], row["legacy_debugger"])) + + +def ado_job_name(row: BuildFlavor) -> str: + """Use a short matrix leg label without changing the persistent build ID.""" + compiler: str = row["tool_chain"] + if compiler == "CLANGPDB": + compiler = {"visual_studio": "ClangVS", "windows_org": "ClangWin", + "distribution": "ClangLinux"}[row["compiler_source"]] + name = f"{row['target']}_{row['arch']}_{compiler}" + if row["core"] == "patina": + name += "_Patina" + if row["legacy_debugger"] == "1": + name += "_Debugger" + return name + + +def label_builds(rows: list[SelectedBuild], backend: OutputFormat) -> list[SelectedBuild]: + """Add provider presentation fields without changing execution/package IDs. + + GitHub artifact names preserve the pre-refactor consumer contract. Reject + collisions instead of silently renaming artifacts when new variants are + introduced. ADO artifact staging and package names are not derived from + these display labels. Plain JSON remains execution data only. + """ + labeled: list[SelectedBuild] = [] + artifact_names: set[str] = set() + for row in rows: + result: SelectedBuild = {**row} + if backend != "json": + core = "Legacy" if row["core"] == "legacy" else "Patina" + label = f"{row['target']} {row['arch']} {row['tool_chain']} ({core})" + if backend == "ado": + result["build_id"] = row["id"] + if "package_suffix" in row: + suffix = f"_{row['package_suffix']}" if row["package_suffix"] else "" + result["artifact_name"] = f"drop_Build_{row['target']}_{row['arch']}_{row['tool_chain']}{suffix}" + else: + result["artifact_name"] = f"firmware-{row['id']}" + result["build_artifact_name"] = ( + f"input-{row['id']}" if row.get("shipping") == "true" else result["artifact_name"] + ) + compiler = {"visual_studio": "VS", "windows_org": "WinOrg", "distribution": "Distro"}[row["compiler_source"]] + label += f" / {row['host'].title()} {compiler}" + if row["legacy_debugger"] == "1": + label += " / Legacy debugger" + result["display_name"] = label + if backend == "github": + suffix = "-patina" if row["core"] == "patina" else "" + stem = f"{row['target']}-{row['arch']}-{row['tool_chain']}{suffix}" + artifact_name = f"firmware-{stem}" + if artifact_name in artifact_names: + raise ValueError(f"Artifact name collision: {artifact_name}; choose an explicit consumer migration before adding this variant") + artifact_names.add(artifact_name) + result["artifact_name"] = artifact_name + result["logs_artifact_name"] = f"logs-{stem}" + labeled.append(result) + return labeled + + +def parse_definition(value: object) -> BuildDefinition: + """Validate one untrusted JSON row before admitting it to the typed model. + + Raises: + ValueError: Missing/unknown fields, invalid scalar types or choices, + unsupported compilers, or inconsistent package metadata. + """ + if not isinstance(value, dict): + raise ValueError("Each build must specify a complete execution flavor") + row = cast(dict[str, object], value) + required = {*FLAVOR_CHOICES, *EXECUTION_CHOICES} + if not required <= row.keys() or row.keys() - required - {"shipping", "package_suffix"}: + raise ValueError("Each build must specify a complete execution flavor") + for field, choices in {**FLAVOR_CHOICES, **EXECUTION_CHOICES}.items(): + if not isinstance(row[field], str) or row[field] not in choices: + raise ValueError(f"Invalid {field}: {row[field]!r}") + valid_compilers = { + ("windows", "VS2022", "visual_studio"), + ("windows", "CLANGPDB", "visual_studio"), + ("windows", "CLANGPDB", "windows_org"), + ("linux", "CLANGPDB", "distribution"), + ("linux", "GCC", "distribution"), + } + if (row["host"], row["tool_chain"], row["compiler_source"]) not in valid_compilers: + raise ValueError("Unsupported host/compiler combination") + if row["arch"] == "AARCH64" and row["tool_chain"] == "VS2022": + raise ValueError("ARM64 MSVC is not supported") + if row["legacy_debugger"] == "1" and ( + row["arch"] != "X64" or row["core"] != "legacy" or row["tool_chain"] not in ("VS2022", "CLANGPDB") + ): + raise ValueError("Legacy debugger requires X64 VS2022 or CLANGPDB with the legacy core") + if "shipping" in row or "package_suffix" in row: + suffix = row.get("package_suffix") + if type(row.get("shipping")) is not bool or not isinstance(suffix, str): + raise ValueError("Package metadata requires explicit shipping and package_suffix") + if not re.fullmatch(r"[A-Za-z0-9]*", suffix) or row["shipping"] != (suffix == ""): + raise ValueError("Invalid shipping/package suffix contract") + return cast(BuildDefinition, row) + + +def select_builds(path: Path, host: str | None = None) -> list[SelectedBuild]: + """Load the explicitly supplied matrix file, optionally restricted by host. + + Preserves declaration order and rejects duplicate execution/package IDs. + Relative paths resolve against the caller's working directory. No repository + or CI backend is inferred from the path; either file can be used locally. + Raises OSError for unreadable input and ValueError for invalid JSON, invalid + definitions, unknown filters, or an empty selection. No commands are run. + """ + if host is not None and host not in EXECUTION_CHOICES["host"]: + raise ValueError(f"Unknown host: {host}") + document: object = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(document, list): + raise ValueError("Build matrix must be a list") + selected: list[SelectedBuild] = [] + identifiers: set[str] = set() + package_names: set[tuple[Architecture, Target, str]] = set() + for value in cast(list[object], document): + row = parse_definition(value) + identifier = build_id(row) + if identifier in identifiers: + raise ValueError(f"Duplicate matrix identifier: {identifier}") + identifiers.add(identifier) + if "package_suffix" in row: + suffix = row["package_suffix"] + package_name = (row["arch"], row["target"], suffix) + if package_name in package_names: + raise ValueError(f"Duplicate closed package identity: {package_name}") + package_names.add(package_name) + if host is None or row["host"] == host: + selected_row: SelectedBuild = { + "id": identifier, + "arch": row["arch"], "target": row["target"], "tool_chain": row["tool_chain"], + "core": row["core"], "host": row["host"], "compiler_source": row["compiler_source"], + "legacy_debugger": row["legacy_debugger"], + } + if "shipping" in row: + selected_row["shipping"] = str(row["shipping"]).lower() + selected_row["package_suffix"] = row["package_suffix"] + selected.append(selected_row) + if not selected: + raise ValueError(f"No builds selected from {path} for host: {host or 'all'}") + return selected + + +def main(argv: list[str] | None = None) -> None: + """Print compact matrix JSON; argparse reports input failures with exit 2.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--matrix", required=True, type=Path, help="Path to the build coverage JSON file") + parser.add_argument("--host", choices=EXECUTION_CHOICES["host"]) + parser.add_argument("--format", choices=("json", "github", "ado"), default="json") + parser.add_argument("--shipping-only", action="store_true", help="Select only explicitly shipping package flavors") + args = parser.parse_args(argv, namespace=MatrixArguments()) + try: + rows = select_builds(args.matrix, host=args.host) + if args.shipping_only: + rows = [row for row in rows if row.get("shipping") == "true"] + if not rows: + raise ValueError("No shipping builds selected") + if args.format == "github" and any(row["legacy_debugger"] == "1" for row in rows): + raise ValueError("GitHub builds cannot enable the closed-source legacy debugger") + rows = label_builds(rows, args.format) + if args.format == "ado" and len({ado_job_name(row) for row in rows}) != len(rows): + raise ValueError("ADO job label collision; distinguish the new flavors before adding them") + except (OSError, ValueError) as error: + parser.error(str(error)) + result: object + if args.format == "github": + result = {"include": rows} + elif args.format == "ado": + result = {ado_job_name(row): {key: value for key, value in cast(Mapping[str, str], row).items() if key != "id"} + for row in rows} + else: + result = rows + print(json.dumps(result, separators=(",", ":"))) + + +if __name__ == "__main__": + main() diff --git a/ci/scripts/package_artifacts.py b/ci/scripts/package_artifacts.py new file mode 100644 index 0000000000..fda9cc70aa --- /dev/null +++ b/ci/scripts/package_artifacts.py @@ -0,0 +1,176 @@ +"""Validate staged firmware and prepare archives or a NuGet manifest, without publishing.""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import tarfile +import tomllib +import xml.etree.ElementTree as ET +from pathlib import Path + +from ci.scripts.matrix import SelectedBuild, label_builds, select_builds +from ci.scripts.stage_artifacts import ado_layout, select_flavor + + +def base_version() -> str: + with Path("MsvmPkg/FirmwareVersion.toml").open("rb") as stream: + version = str(tomllib.load(stream)["release"]["version"]) + if not re.fullmatch(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)", version): + raise ValueError("Release version must be major.minor") + return version + + +def validate_firmware(firmware: Path, flavor: SelectedBuild, commit: str, origin: str, official: bool) -> None: + if not re.fullmatch(r"[0-9a-fA-F]{40}", commit): + raise ValueError("Expected a full Git commit SHA") + info = json.loads((firmware / "BuildInfo.json").read_text(encoding="utf-8")) + if info != flavor: + raise ValueError(f"Artifact flavor does not match the selected matrix row: {firmware}") + metadata = json.loads((firmware / "FwVersion/FirmwareVersion.json").read_text(encoding="utf-8")) + mismatches = [] + for field, expected in (("git_commit", commit), ("source_origin", origin), ("base_version", base_version())): + if metadata.get(field) != expected: + mismatches.append(f"{field}: expected {expected!r}, got {metadata.get(field)!r}") + flags = metadata.get("flags") + if type(flags) is not int: + mismatches.append(f"flags must be an integer, got {flags!r}") + else: + if bool(flags & 2) != official: + mismatches.append(f"official flag: expected {official}, got {bool(flags & 2)}") + if flags & 1: + mismatches.append("dirty-worktree flag is set; inspect the firmware build's Git status diagnostics") + if mismatches: + raise ValueError(f"Artifact provenance/version does not match this build: {firmware}: {'; '.join(mismatches)}") + required = [firmware / "FV/MSVM.fd", firmware / "FwVersion/FwVersionBlob.bin"] + for file in required: + if not file.is_file() or file.stat().st_size == 0: + raise ValueError(f"Required artifact is missing or empty: {file}") + if not list((firmware / "MAP").glob("*.map")): + raise ValueError(f"Missing MAP artifacts: {firmware}") + if flavor["tool_chain"] != "GCC" and not list((firmware / "PDB").glob("*.pdb")): + raise ValueError(f"Missing PDB artifacts: {firmware}") + + +def release_archives(matrix: Path, source: Path, output: Path, commit: str) -> None: + rows = select_builds(matrix) + labeled = label_builds(rows, "github") + expected = {row["artifact_name"] for row in labeled} + if {path.name for path in source.iterdir()} != expected: + raise ValueError("Downloaded firmware artifacts do not exactly match the release matrix") + for row, label in zip(rows, labeled): + validate_firmware(source / label["artifact_name"], row, commit, "open", True) + output.mkdir(parents=True, exist_ok=False) + for row in labeled: + name = row["artifact_name"] + with tarfile.open(output / f"{name}.tar.gz", "w:gz") as archive: + for child in sorted((source / name).iterdir()): + archive.add(child, arcname=child.name) + + +def nuget_manifest( + matrix: Path, identifier: str, source: Path, output: Path, + commit: str, repository: str, branch: str, official: bool, + staging_output: Path | None = None, +) -> None: + flavor = select_flavor(matrix, identifier) + if flavor.get("shipping") != "true" or flavor["host"] != "windows": + raise ValueError("Only shipping Windows flavors may be packaged") + if not re.fullmatch(r"[A-Za-z0-9_.-]+", repository): + raise ValueError("Invalid package repository name") + if not re.fullmatch(r"[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*", branch): + raise ValueError("Branch name is not a valid NuGet prerelease identifier") + firmware = source / "firmware" + validate_firmware(firmware, flavor, commit, "closed", official) + package = source / "package/Bin" + for name in ("MSVM.fd", "Dsdt.aml", "BiosInterface.h", "FirmwareVersion.json", "FwVersionBlob.bin", "BuildInfo.json"): + if not (package / name).is_file() or (package / name).stat().st_size == 0: + raise ValueError(f"Missing package payload: {package / name}") + for source_file, package_file in ( + (firmware / "FV/MSVM.fd", package / "MSVM.fd"), + (firmware / "FwVersion/FirmwareVersion.json", package / "FirmwareVersion.json"), + (firmware / "FwVersion/FwVersionBlob.bin", package / "FwVersionBlob.bin"), + (firmware / "BuildInfo.json", package / "BuildInfo.json"), + ): + if source_file.read_bytes() != package_file.read_bytes(): + raise ValueError(f"Package payload differs from firmware artifact: {package_file}") + if staging_output is not None: + if staging_output.resolve().is_relative_to(source.resolve()) or source.resolve().is_relative_to(staging_output.resolve()): + raise ValueError("Publication staging must not overlap downloaded artifacts") + for directory in ("firmware", "logs", "package"): + shutil.copytree(source / directory, staging_output / directory) + ado_layout(staging_output, flavor) + firmware = staging_output / "firmware" + prefix = "" if official else "PR." + suffix = "" if official else ".Test" + identity = f"{repository}.mscoreuefi.{prefix}{flavor['arch']}.{flavor['target']}{suffix}" + root = ET.Element("package") + metadata = ET.SubElement(root, "metadata") + for key, value in { + "id": identity, "version": f"{base_version()}-{branch}.{commit}", + "authors": "Microsoft", "description": "Hyper-V UEFI firmware binaries.", + "owners": "Microsoft", + "projectUrl": "https://microsoft.visualstudio.com/DefaultCollection/OS/_git/hyperv.uefi/", + "copyright": "Copyright (C) Microsoft. All rights reserved.", + "tags": "Microsoft,Hyper-V,UEFI", + }.items(): + ET.SubElement(metadata, key).text = value + files = ET.SubElement(root, "files") + legacy_path = f"Msvm{flavor['arch']}/{flavor['target']}_{flavor['tool_chain']}" + for path, target in ( + ("FV/MSVM.fd", f"{legacy_path}/FV"), ("PDB/*.pdb", f"symbols/{legacy_path}/PDB"), + ("MAP/*.map", f"map/{legacy_path}/MAP"), + ("FwVersion/*", "FwVersion"), ("BuildInfo.json", ""), + ): + ET.SubElement(files, "file", src=str(firmware.resolve() / path), target=target) + output.parent.mkdir(parents=True, exist_ok=True) + ET.indent(root) + ET.ElementTree(root).write(output, encoding="utf-8", xml_declaration=True) + + +class Arguments(argparse.Namespace): + command: str + matrix: Path + source: Path + output: Path + commit: str + build_id: str + repository: str + branch: str + official: bool + staging_output: Path | None + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("version") + for command in ("github", "nuget"): + child = subparsers.add_parser(command) + child.add_argument("--matrix", required=True, type=Path) + child.add_argument("--source", required=True, type=Path) + child.add_argument("--output", required=True, type=Path) + child.add_argument("--commit", required=True) + if command == "nuget": + child.add_argument("--build-id", required=True) + child.add_argument("--repository", required=True) + child.add_argument("--branch", required=True) + child.add_argument("--official", action="store_true") + child.add_argument("--staging-output", type=Path, help="Stage a compatible OneBranch publication drop") + args = parser.parse_args(argv, namespace=Arguments()) + try: + if args.command == "version": + print(base_version()) + elif args.command == "github": + release_archives(args.matrix, args.source, args.output, args.commit) + else: + nuget_manifest(args.matrix, args.build_id, args.source, args.output, + args.commit, args.repository, args.branch, args.official, args.staging_output) + except (OSError, ValueError, KeyError) as error: + parser.error(str(error)) + + +if __name__ == "__main__": + main() diff --git a/ci/scripts/repo_auth.py b/ci/scripts/repo_auth.py new file mode 100644 index 0000000000..0aa59481b3 --- /dev/null +++ b/ci/scripts/repo_auth.py @@ -0,0 +1,48 @@ +"""Set or clear URL-scoped Git authentication in a dedicated CI container. + +ADO supplies SYSTEM_ACCESSTOKEN; shared Stuart runners remain provider-neutral. +Call unset in an always-running cleanup step, including after setup failures. +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from urllib.parse import urlsplit + + +def main(argv: list[str] | None = None) -> int: + """Configure Git, reporting failures without echoing credential arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("set", "unset")) + parser.add_argument("--url", required=True, help="HTTPS repository URL to authenticate") + args = parser.parse_args(argv) + url = urlsplit(args.url) + if url.scheme != "https" or not url.hostname or url.username or url.password or url.query or url.fragment: + parser.error("--url must be an HTTPS repository URL without credentials, query, or fragment") + key = f"http.{args.url}.extraheader" + command = ["git", "config", "--global"] + if args.mode == "set": + token = os.environ.get("SYSTEM_ACCESSTOKEN", "").strip() + if not token or token == "$(System.AccessToken)" or "\n" in token or "\r" in token: + parser.error("SYSTEM_ACCESSTOKEN must contain the pipeline job token") + command.extend(["--replace-all", key, f"AUTHORIZATION: bearer {token}"]) + else: + command.extend(["--unset-all", key]) + + environment = { + key: value for key, value in os.environ.items() + if not key.startswith("GIT_TRACE") and key != "GIT_CURL_VERBOSE" + } + result = subprocess.run(command, capture_output=True, env=environment) + if result.returncode != 0 and not (args.mode == "unset" and result.returncode == 5): + print(f"Git authentication {args.mode} failed (exit {result.returncode})", file=sys.stderr) + return result.returncode + print(f"Git authentication {args.mode} completed for {args.url}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/scripts/source_checks.py b/ci/scripts/source_checks.py new file mode 100644 index 0000000000..ad531b4f2b --- /dev/null +++ b/ci/scripts/source_checks.py @@ -0,0 +1,71 @@ +"""Prepare the workspace and run source checks using native Stuart commands. + +Shared by local developers and both provider adapters. Activate the intended +Python environment first: pip uses this interpreter and Stuart is found on PATH. +This entry point installs requirements and updates dependencies; for an inner +loop without preparation, invoke stuart_ci_build directly. +""" + +from __future__ import annotations + +import argparse +import shlex +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +CI_SETTINGS = ".pytool/CISettings.py" + + +def run(command: list[str]) -> None: + """Log and execute an argument vector at the repo root without a shell. + + Output is inherited by the caller. Raises CalledProcessError on a nonzero + exit or OSError when the executable cannot be started; neither is suppressed. + """ + print(f"==> {shlex.join(command)}", flush=True) + subprocess.run(command, cwd=REPO_ROOT, check=True) + + +def run_source_checks(verbose: bool = False) -> None: + """Install requirements, prepare CI dependencies, and run the selected checks. + + Uses CISettings.py, not platform build settings. Commands run sequentially; + the first failure propagates and prevents all subsequent commands. + Verbose mode enables Stuart setup diagnostics without logging pip credentials. + """ + commands: tuple[list[str], ...] = ( + [sys.executable, "-m", "pip", "install", "-r", "pip-requirements.txt"], + ["stuart_setup", "-c", CI_SETTINGS], + ["stuart_update", "-c", CI_SETTINGS], + [ + "stuart_ci_build", + "-c", + CI_SETTINGS, + "--disable-all", + "GuidCheck=run", + "LineEndingCheck=run", + "UncrustifyCheck=run", + ], + ) + if verbose: + commands[1].append("--verbose") + for command in commands: + run(command) + + +def main(argv: list[str] | None = None) -> None: + """Parse CLI options, then run source checks. + + None reads sys.argv. Argparse exits for help or invalid arguments; execution + failures propagate to the caller. This function does not create a venv. + """ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--verbose", action="store_true", help="Enable verbose Stuart setup logging") + args = parser.parse_args(argv) + run_source_checks(verbose=args.verbose) + + +if __name__ == "__main__": + main() diff --git a/ci/scripts/stage_artifacts.py b/ci/scripts/stage_artifacts.py new file mode 100644 index 0000000000..b5e677b189 --- /dev/null +++ b/ci/scripts/stage_artifacts.py @@ -0,0 +1,181 @@ +"""Stage portable firmware outputs and diagnostics; never upload or publish.""" + +from __future__ import annotations + +import argparse +import fnmatch +import json +import shutil +from pathlib import Path + +from ci.scripts.matrix import SelectedBuild, select_builds + +LOG_PATTERNS = ( + "buildlog*.txt", "buildlog*.md", "build_report.txt", "ci*.txt", "ci*.md", + "setup*.txt", "update*.txt", "prevallog.txt", "testsuites.xml", + "build_tools_report.*", "overridelog.txt", "basetools_build*.*", "fd_report.html", + "ci_buildlog.*", "setuplog.*", "update_log.*", +) + + +def select_flavor(matrix: Path, identifier: str) -> SelectedBuild: + for row in select_builds(matrix): + if row["id"] == identifier: + return row + raise ValueError(f"Build ID not present in {matrix}: {identifier}") + + +def copy_file(source: Path, destination: Path) -> None: + if not source.is_file() or source.stat().st_size == 0: + raise ValueError(f"Required artifact is missing or empty: {source}") + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + + +def copy_outputs(source: Path, destination: Path, suffix: str) -> None: + files = sorted(source.glob(f"*{suffix}")) + if not files: + raise ValueError(f"No {suffix} artifacts found in {source}") + for file in files: + copy_file(file, destination / file.name) + + +def stage_logs(workspace: Path, output: Path) -> None: + build = workspace / "Build" + destination = output / "logs" + destination.mkdir(parents=True, exist_ok=True) + count = 0 + for file in sorted(build.rglob("*")): + if file.is_file() and any(fnmatch.fnmatchcase(file.name.lower(), pattern) for pattern in LOG_PATTERNS): + target = destination / file.relative_to(build) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(file, target) + count += 1 + if not count: + print(f"WARNING: no build logs found under {build}", flush=True) + + +def ado_logs(output: Path, flavor: SelectedBuild) -> None: + name = f"{flavor['host'].title()} {flavor['arch']} {flavor['target']} {flavor['tool_chain']}" + if flavor["compiler_source"] == "windows_org": + name += " (WinOrg)" + destination = output / f"Build Logs {name}" + destination.mkdir(parents=True, exist_ok=True) + for file in sorted((output / "logs").rglob("*")): + if file.is_file(): + target = destination / file.name + if target.exists() and target.read_bytes() != file.read_bytes(): + print(f"WARNING: legacy log basename collision: {file.name}; full paths retained under logs/", flush=True) + shutil.copy2(file, target) + + +def source_check_logs(workspace: Path, output: Path, ado: bool) -> None: + stage_logs(workspace, output) + package = workspace / "Build/MsvmPkg" + if package.is_dir(): + shutil.copytree(package, output / "logs/MsvmPkg", dirs_exist_ok=True) + if ado: + shutil.copytree(package, output / "MsvmPkg", dirs_exist_ok=True) + if ado: + for file in (workspace / "Build").glob("*"): + if file.is_file() and any(fnmatch.fnmatchcase(file.name.lower(), pattern) for pattern in + ("ci_buildlog.*", "setuplog.*", "update_log.*", "testsuites.xml")): + shutil.copy2(file, output / file.name) + + +def ado_layout(output: Path, flavor: SelectedBuild) -> None: + """Keep historical drop paths alongside the provider-independent payload.""" + ado_logs(output, flavor) + suffix = f"{flavor['target']}_{flavor['arch']}" + copy_file(output / "firmware/FV/MSVM.fd", output / f"Firmware Binary File {suffix}/MSVM.fd") + shutil.copytree(output / "firmware/MAP", output / f"Firmware Map Files {suffix}") + if flavor["tool_chain"] != "GCC": + shutil.copytree(output / "firmware/PDB", output / f"Firmware PDB Files {suffix}") + build = Path("Build") / f"Msvm{flavor['arch']}" / f"{flavor['target']}_{flavor['tool_chain']}" + shutil.copytree(output / "package/Bin", output / build / "Bin") + + +def stage_build( + workspace: Path, output: Path, flavor: SelectedBuild, succeeded: bool, + ado: bool = False, +) -> None: + stage_logs(workspace, output) + build = workspace / "Build" / f"Msvm{flavor['arch']}" / f"{flavor['target']}_{flavor['tool_chain']}" + if not succeeded: + if ado: + ado_logs(output, flavor) + if (workspace / "Build").is_dir(): + shutil.copytree(workspace / "Build", output / "all", dirs_exist_ok=True) + else: + print(f"WARNING: build failed before Build directory was created: {workspace}", flush=True) + elif build.is_dir(): + shutil.copytree(build, output / "logs" / "failed-build") + else: + print(f"WARNING: build failed before output directory was created: {build}", flush=True) + return + + firmware = output / "firmware" + copy_file(build / "FV" / "MSVM.fd", firmware / "FV" / "MSVM.fd") + for name in ("FirmwareVersion.json", "FwVersionBlob.bin"): + copy_file(build / "FwVersion" / name, firmware / "FwVersion" / name) + copy_outputs(build / "MAP", firmware / "MAP", ".map") + if flavor["tool_chain"] != "GCC": + copy_outputs(build / "PDB", firmware / "PDB", ".pdb") + (firmware / "BuildInfo.json").write_text(json.dumps(flavor, indent=2) + "\n", encoding="utf-8") + + if flavor.get("shipping") == "true" or ado: + package = output / "package" / "Bin" + copy_file(build / "FV" / "MSVM.fd", package / "MSVM.fd") + copy_file(build / flavor["arch"] / "MsvmPkg/AcpiTables/AcpiTables/OUTPUT/Dsdt.aml", package / "Dsdt.aml") + copy_file(workspace / "MsvmPkg/Include/BiosInterface.h", package / "BiosInterface.h") + for name in ("FirmwareVersion.json", "FwVersionBlob.bin"): + copy_file(build / "FwVersion" / name, package / name) + copy_file(firmware / "BuildInfo.json", package / "BuildInfo.json") + if flavor["tool_chain"] != "GCC": + copy_outputs(build / "PDB", package / "PDB", ".pdb") + if ado: + ado_layout(output, flavor) + + +class Arguments(argparse.Namespace): + output: Path + matrix: Path | None + build_id: str | None + build_succeeded: bool + logs_only: bool + ado_layout: bool + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--matrix", type=Path) + parser.add_argument("--build-id") + parser.add_argument("--build-succeeded", action="store_true") + parser.add_argument("--logs-only", action="store_true") + parser.add_argument("--ado-layout", action="store_true", help="Also preserve historical OneBranch drop folders") + args = parser.parse_args(argv, namespace=Arguments()) + workspace = Path.cwd().resolve() + try: + output = args.output.resolve() + if output.is_relative_to(workspace / "Build") or workspace.is_relative_to(output): + raise ValueError("Artifact destination must be outside Build and must not contain the workspace") + if args.ado_layout: + if args.build_succeeded and any((output / name).exists() for name in ("firmware", "logs", "package", "all")): + raise ValueError(f"Artifact destination already contains staged output: {output}") + elif output.exists() and any(output.iterdir()): + raise ValueError(f"Artifact destination must be empty: {output}") + if args.logs_only: + if args.matrix or args.build_id or args.build_succeeded: + raise ValueError("--logs-only cannot be combined with build arguments") + source_check_logs(workspace, output, args.ado_layout) + else: + if args.matrix is None or args.build_id is None: + raise ValueError("Build staging requires --matrix and --build-id") + stage_build(workspace, output, select_flavor(args.matrix, args.build_id), args.build_succeeded, args.ado_layout) + except (OSError, ValueError) as error: + parser.error(str(error)) + + +if __name__ == "__main__": + main() diff --git a/ci/tests/test_ado_poc.py b/ci/tests/test_ado_poc.py new file mode 100644 index 0000000000..b58dfcefba --- /dev/null +++ b/ci/tests/test_ado_poc.py @@ -0,0 +1,93 @@ +"""Lock ADO adapter contracts without executing hosted tasks.""" + +import importlib +import unittest +from pathlib import Path +from typing import Any, cast + +ROOT = Path(__file__).resolve().parents[2] + + +def load_yaml(path: str) -> dict[str, Any]: + """Load an ADO template for contract assertions; never expand or execute it.""" + yaml = importlib.import_module("yaml") + return cast(dict[str, Any], yaml.safe_load((ROOT / path).read_text())) + + +class AdoPocTests(unittest.TestCase): + """Preserve provider-specific environment preparation.""" + + def test_build_job_labels_are_resolved_at_template_expansion(self) -> None: + job = load_yaml(".azuredevops/templates/build.yml")["jobs"][1]["${{ each host in parameters.hosts }}"][0] + self.assertEqual(job["displayName"], "Firmware (${{ host }})") + self.assertNotIn("$(", job["displayName"]) + self.assertEqual(job["strategy"]["matrix"], + "$[ dependencies.BuildMatrix.outputs['${{ host }}.matrix'] ]") + + def test_source_check_diagnostics_redact_git_credentials(self) -> None: + source = load_yaml(".azuredevops/templates/source-checks.yml")["jobs"][0] + step = next(step for step in source["steps"] if step.get("displayName") == "Run Source Checks") + self.assertEqual(step["script"], "python ci/scripts/source_checks.py --verbose") + for variable in ("GIT_TRACE", "GIT_TRACE_CURL", "GIT_TRACE_CURL_NO_DATA", "GIT_TRACE_REDACT"): + self.assertEqual(step["env"][variable], "1") + + def test_windows_jobs_use_image_python_without_downloading(self) -> None: + """All Windows callers select the image interpreter; Linux keeps its venv.""" + reference = {"template": "/.azuredevops/templates/windows-python.yml@self"} + template = load_yaml(".azuredevops/templates/build.yml") + self.assertEqual(template["jobs"][0]["steps"][0], reference) + source = load_yaml(".azuredevops/templates/source-checks.yml")["jobs"][0] + self.assertEqual(source["steps"][0], reference) + job = template["jobs"][1]["${{ each host in parameters.hosts }}"][0] + windows = next(branch["${{ if eq(host, 'windows') }}"] for branch in job["steps"] + if "${{ if eq(host, 'windows') }}" in branch) + linux = next(branch["${{ if eq(host, 'linux') }}"] for branch in job["steps"] + if "${{ if eq(host, 'linux') }}" in branch) + self.assertEqual(windows[0], reference) + self.assertNotIn(reference, linux) + self.assertIn('python3 -m venv "$BUILD_VENV"', next(step["bash"] for step in linux if "bash" in step)) + script = load_yaml(".azuredevops/templates/windows-python.yml")["steps"][0]["pwsh"] + self.assertIn('py -3.12 -c "import sys; print(sys.executable)"', script) + self.assertIn("if ($LASTEXITCODE -ne 0 -or -not $python)", script) + self.assertIn("$python = $python.Trim()", script) + self.assertIn("Test-Path -LiteralPath $python -PathType Leaf", script) + self.assertIn("$pythonHome = Split-Path -Parent $python", script) + self.assertNotIn("C:\\Program Files\\Python312", script) + self.assertIn("sys.version_info[:2] == (3, 12)", script) + self.assertIn('task.prependpath]$pythonHome\\Scripts', script) + self.assertIn('task.prependpath]$pythonHome"', script) + self.assertIn("-m pip --version", script) + self.assertNotIn("install", script) + for path in (ROOT / ".azuredevops/templates").glob("*.yml"): + self.assertNotIn("UsePythonVersion@", path.read_text(), str(path)) + + def test_repository_auth_wraps_source_checks_and_both_build_hosts(self) -> None: + reference = "/.azuredevops/templates/repo-auth.yml@self" + auth = load_yaml(".azuredevops/templates/repo-auth.yml")["steps"][0] + self.assertEqual(auth["${{ if eq(parameters.mode, 'unset') }}"]["condition"], "always()") + self.assertEqual(auth["${{ if eq(parameters.mode, 'set') }}"]["env"], + {"SYSTEM_ACCESSTOKEN": "$(System.AccessToken)"}) + self.assertIn("ci/scripts/repo_auth.py", auth["script"]) + source = load_yaml(".azuredevops/templates/source-checks.yml")["jobs"][0]["steps"] + job = load_yaml(".azuredevops/templates/build.yml")["jobs"][1]["${{ each host in parameters.hosts }}"][0] + groups = [(source, "ci/scripts/source_checks.py")] + for host in ("windows", "linux"): + key = "${{ if eq(host, '" + host + "') }}" + steps = next(branch[key] for branch in job["steps"] if key in branch) + groups.append((steps, "ci/scripts/build.py")) + for steps, runner in groups: + setup = next(i for i, step in enumerate(steps) + if step.get("template") == reference and step["parameters"]["mode"] == "set") + cleanup = next(i for i, step in enumerate(steps) + if step.get("template") == reference and step["parameters"]["mode"] == "unset") + invocation = next(i for i, step in enumerate(steps) + if runner in step.get("script", step.get("pwsh", step.get("bash", "")))) + self.assertLess(setup, invocation) + self.assertLess(invocation, cleanup) + self.assertNotIn("GIT_CONFIG_VALUE_2", steps[invocation].get("env", {})) + self.assertEqual(steps[setup]["parameters"].get("python", "python"), + steps[cleanup]["parameters"].get("python", "python")) + + +if __name__ == "__main__": + unittest.main() diff --git a/ci/tests/test_artifacts.py b/ci/tests/test_artifacts.py new file mode 100644 index 0000000000..ceabf70419 --- /dev/null +++ b/ci/tests/test_artifacts.py @@ -0,0 +1,339 @@ +"""Exercise real staging/archive payloads and provider publication gates.""" + +from __future__ import annotations + +import io +import json +import os +import shutil +import subprocess +import tarfile +import unittest +import xml.etree.ElementTree as ET +import zipfile +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from ci.scripts import matrix, package_artifacts, stage_artifacts +from ci.tests.test_ado_poc import load_yaml + +ROOT = Path(__file__).resolve().parents[2] +OPEN = ROOT / ".github/build-matrix.json" +CLOSED = ROOT / ".azuredevops/build-matrix.json" +COMMIT = "a" * 40 + + +def fixture(workspace: Path, row: matrix.SelectedBuild, origin: str, official: bool) -> Path: + build = workspace / "Build" / f"Msvm{row['arch']}" / f"{row['target']}_{row['tool_chain']}" + files = { + "FV/MSVM.fd": b"firmware", "MAP/Module.map": b"map", + "PDB/Module.pdb": b"symbols", "FwVersion/FwVersionBlob.bin": b"\0" * 80, + f"{row['arch']}/MsvmPkg/AcpiTables/AcpiTables/OUTPUT/Dsdt.aml": b"aml", + "FwVersion/FirmwareVersion.json": json.dumps({ + "schema_version": 1, "struct_version": 1, "base_version": package_artifacts.base_version(), + "git_commit": COMMIT, "flags": 2 if official else 0, "source_origin": origin, + }).encode(), + "BUILD_REPORT.TXT": b"report", "Nested/BUILD_REPORT.TXT": b"nested report", + } + for name, contents in files.items(): + path = build / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(contents) + header = workspace / "MsvmPkg/Include/BiosInterface.h" + header.parent.mkdir(parents=True, exist_ok=True) + header.write_text("header") + (workspace / "Build/SETUPLOG.txt").write_text("setup") + return build + + +class ArtifactTests(unittest.TestCase): + def test_ado_preserves_legacy_folders_for_all_15_flavors(self) -> None: + for row in matrix.select_builds(CLOSED): + with self.subTest(flavor=row["id"]), TemporaryDirectory() as temporary, redirect_stdout(io.StringIO()): + root = Path(temporary) + workspace, output = root / "workspace", root / "output" + fixture(workspace, row, "closed", False) + stage_artifacts.stage_build(workspace, output, row, True, ado=True) + suffix = f"{row['target']}_{row['arch']}" + self.assertEqual((output / f"Firmware Binary File {suffix}/MSVM.fd").read_bytes(), b"firmware") + self.assertTrue((output / f"Firmware Map Files {suffix}/Module.map").is_file()) + self.assertEqual((output / f"Firmware PDB Files {suffix}/Module.pdb").exists(), row["tool_chain"] != "GCC") + label = f"{row['host'].title()} {row['arch']} {row['target']} {row['tool_chain']}" + if row["compiler_source"] == "windows_org": + label += " (WinOrg)" + self.assertTrue((output / f"Build Logs {label}/BUILD_REPORT.TXT").is_file()) + self.assertEqual(len(list((output / "logs").rglob("BUILD_REPORT.TXT"))), 2) + legacy_bin = output / "Build" / f"Msvm{row['arch']}" / f"{row['target']}_{row['tool_chain']}" / "Bin" + for name in ("MSVM.fd", "Dsdt.aml", "BiosInterface.h", "FirmwareVersion.json", "FwVersionBlob.bin"): + self.assertTrue((legacy_bin / name).is_file(), name) + + def test_ado_failure_adds_legacy_all_tree_after_partial_staging(self) -> None: + with TemporaryDirectory() as temporary, redirect_stdout(io.StringIO()): + root = Path(temporary) + row = matrix.select_builds(CLOSED)[0] + build = fixture(root / "workspace", row, "closed", False) + (build / "MAP/Module.map").unlink() + arguments = ["--matrix", str(CLOSED), "--build-id", row["id"], "--ado-layout", + "--output", str(root / "output")] + with patch.object(Path, "cwd", return_value=root / "workspace"), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + stage_artifacts.main([*arguments, "--build-succeeded"]) + stage_artifacts.main(arguments) + self.assertTrue((root / "output/all/MsvmX64/DEBUG_VS2022/FV/MSVM.fd").is_file()) + self.assertTrue((root / "output/Build Logs Windows X64 DEBUG VS2022/SETUPLOG.txt").is_file()) + + def test_source_check_compatibility_keeps_package_diagnostics(self) -> None: + with TemporaryDirectory() as temporary: + root = Path(temporary) + build = root / "workspace/Build" + (build / "MsvmPkg/Uncrustify").mkdir(parents=True) + for name in ("CI_BUILDLOG.md", "SETUPLOG.txt", "UPDATE_LOG.txt", "TestSuites.xml", + "MsvmPkg/Uncrustify/output.diff"): + (build / name).write_text(name) + stage_artifacts.source_check_logs(root / "workspace", root / "output", True) + for name in ("CI_BUILDLOG.md", "SETUPLOG.txt", "UPDATE_LOG.txt", "TestSuites.xml", + "MsvmPkg/Uncrustify/output.diff"): + self.assertEqual((root / "output" / name).read_text(), name) + self.assertEqual((root / "output/logs" / name).read_text(), name) + + def test_all_25_flavors_stage_expected_payloads(self) -> None: + for coverage, origin in ((OPEN, "open"), (CLOSED, "closed")): + for row in matrix.select_builds(coverage): + with self.subTest(flavor=row["id"]), TemporaryDirectory() as temporary: + workspace = Path(temporary) / "workspace" + build = fixture(workspace, row, origin, False) + if row["tool_chain"] == "GCC": + (build / "PDB/Module.pdb").unlink() + output = Path(temporary) / "output" + stage_artifacts.stage_build(workspace, output, row, True) + firmware = output / "firmware" + self.assertEqual((firmware / "FV/MSVM.fd").read_bytes(), b"firmware") + self.assertEqual((firmware / "MAP/Module.map").read_bytes(), b"map") + self.assertEqual((firmware / "PDB/Module.pdb").exists(), row["tool_chain"] != "GCC") + self.assertEqual((firmware / "FwVersion/FirmwareVersion.json").read_bytes(), + (build / "FwVersion/FirmwareVersion.json").read_bytes()) + self.assertEqual(json.loads((firmware / "BuildInfo.json").read_text()), row) + self.assertEqual(len(list((output / "logs").rglob("BUILD_REPORT.TXT"))), 2) + self.assertTrue((output / "logs/SETUPLOG.txt").is_file()) + package = output / "package/Bin" + self.assertEqual(package.exists(), row.get("shipping") == "true") + if package.exists(): + self.assertEqual({file.relative_to(package).as_posix() for file in package.rglob("*") if file.is_file()}, + {"MSVM.fd", "Dsdt.aml", "BiosInterface.h", "PDB/Module.pdb", + "FirmwareVersion.json", "FwVersionBlob.bin", "BuildInfo.json"}) + + def test_success_rejects_missing_outputs_but_retains_logs(self) -> None: + row = matrix.select_builds(CLOSED)[0] + for missing in ("FV/MSVM.fd", "MAP/Module.map", "PDB/Module.pdb", + "FwVersion/FirmwareVersion.json", "FwVersion/FwVersionBlob.bin", + "X64/MsvmPkg/AcpiTables/AcpiTables/OUTPUT/Dsdt.aml"): + with self.subTest(missing=missing), TemporaryDirectory() as temporary: + workspace = Path(temporary) / "workspace" + build = fixture(workspace, row, "closed", False) + (build / missing).unlink() + output = Path(temporary) / "output" + with self.assertRaises(ValueError): + stage_artifacts.stage_build(workspace, output, row, True) + self.assertTrue((output / "logs/SETUPLOG.txt").is_file()) + + def test_failure_preserves_build_tree_without_releasable_firmware(self) -> None: + row = matrix.select_builds(OPEN)[0] + with TemporaryDirectory() as temporary: + workspace = Path(temporary) / "workspace" + fixture(workspace, row, "open", False) + output = Path(temporary) / "output" + stage_artifacts.stage_build(workspace, output, row, False) + self.assertFalse((output / "firmware").exists()) + self.assertTrue((output / "logs/failed-build/FV/MSVM.fd").is_file()) + + def test_early_failure_reports_missing_logs_and_outputs(self) -> None: + with TemporaryDirectory() as temporary, patch("builtins.print") as log: + stage_artifacts.stage_build(Path(temporary), Path(temporary) / "out", matrix.select_builds(OPEN)[0], False) + self.assertEqual(log.call_count, 2) + + def test_cli_rejects_unsafe_or_reused_output_and_unknown_flavor(self) -> None: + with TemporaryDirectory() as temporary: + workspace = Path(temporary) + (workspace / "used").mkdir() + (workspace / "used/file").write_text("keep") + for destination in (workspace, workspace / "Build/artifacts", workspace / "used"): + with patch.object(Path, "cwd", return_value=workspace), redirect_stderr(io.StringIO()), \ + self.assertRaises(SystemExit): + stage_artifacts.main(["--logs-only", "--output", str(destination)]) + with self.assertRaisesRegex(ValueError, "Build ID"): + stage_artifacts.select_flavor(OPEN, "missing") + + def test_release_archives_preserve_names_layout_and_sidecars(self) -> None: + rows = matrix.select_builds(OPEN) + with TemporaryDirectory() as temporary: + root = Path(temporary) + for row, label in zip(rows, matrix.label_builds(rows, "github")): + workspace = root / row["id"] + fixture(workspace, row, "open", True) + stage = root / "staged" / row["id"] + stage_artifacts.stage_build(workspace, stage, row, True) + destination = root / "downloaded" / label["artifact_name"] + shutil.copytree(stage / "firmware", destination) + package_artifacts.release_archives(OPEN, root / "downloaded", root / "release", COMMIT) + self.assertEqual({file.name for file in (root / "release").iterdir()}, + {f"{row['artifact_name']}.tar.gz" for row in matrix.label_builds(rows, "github")}) + for file in (root / "release").iterdir(): + with tarfile.open(file) as archive: + self.assertEqual({item.name for item in archive if item.isfile()}, + {"FV/MSVM.fd", "PDB/Module.pdb", "MAP/Module.map", + "FwVersion/FirmwareVersion.json", "FwVersion/FwVersionBlob.bin", "BuildInfo.json"}) + shutil.rmtree(root / "downloaded" / matrix.label_builds(rows, "github")[0]["artifact_name"]) + with self.assertRaisesRegex(ValueError, "exactly match"): + package_artifacts.release_archives(OPEN, root / "downloaded", root / "incomplete", COMMIT) + self.assertFalse((root / "incomplete").exists()) + + def test_nuget_identity_and_payload_for_all_shipping_flavors(self) -> None: + for row in matrix.select_builds(CLOSED): + if row["shipping"] != "true": + continue + for official in (True, False): + with self.subTest(flavor=row["id"], official=official), TemporaryDirectory() as temporary: + root = Path(temporary) + fixture(root / "workspace", row, "closed", official) + stage_artifacts.stage_build(root / "workspace", root / "staged", row, True) + output = root / "package.nuspec" + package_artifacts.nuget_manifest(CLOSED, row["id"], root / "staged", output, + COMMIT, "hyperv.uefi", "main" if official else "merge", official) + manifest = ET.parse(output) + prefix, suffix = ("", "") if official else ("PR.", ".Test") + self.assertEqual(manifest.findtext("metadata/id"), + f"hyperv.uefi.mscoreuefi.{prefix}{row['arch']}.{row['target']}{suffix}") + legacy = f"Msvm{row['arch']}/{row['target']}_{row['tool_chain']}" + self.assertEqual({file.get("target") for file in manifest.findall("files/file")}, + {"", f"{legacy}/FV", f"symbols/{legacy}/PDB", f"map/{legacy}/MAP", "FwVersion"}) + self.assertTrue(all(Path(file.attrib["src"]).is_absolute() for file in manifest.findall("files/file"))) + + def test_rejects_nonshipping_or_wrong_provenance_packages(self) -> None: + with TemporaryDirectory() as temporary: + root = Path(temporary) + row = matrix.select_builds(CLOSED)[0] + fixture(root / "workspace", row, "closed", True) + stage_artifacts.stage_build(root / "workspace", root / "staged", row, True) + firmware = root / "staged/firmware" + metadata_path = firmware / "FwVersion/FirmwareVersion.json" + metadata = json.loads(metadata_path.read_text()) + for change in ({"git_commit": "b" * 40}, {"source_origin": "open"}, {"base_version": "99.0"}, + {"flags": 0}, {"flags": 3}): + with self.subTest(change=change): + metadata_path.write_text(json.dumps({**metadata, **change})) + expected = "dirty-worktree flag" if change == {"flags": 3} else "provenance" + with self.assertRaisesRegex(ValueError, expected): + package_artifacts.validate_firmware(firmware, row, COMMIT, "closed", True) + metadata_path.write_text(json.dumps(metadata)) + for other in matrix.select_builds(CLOSED)[3:]: + with self.assertRaisesRegex(ValueError, "Only shipping"): + package_artifacts.nuget_manifest(CLOSED, other["id"], root / "staged", root / "bad.nuspec", + COMMIT, "hyperv.uefi", "main", True) + (root / "staged/package/Bin/MSVM.fd").write_bytes(b"wrong") + with self.assertRaisesRegex(ValueError, "differs"): + package_artifacts.nuget_manifest(CLOSED, row["id"], root / "staged", root / "bad.nuspec", + COMMIT, "hyperv.uefi", "main", True) + + @unittest.skipUnless(os.name == "nt" and shutil.which("nuget"), "Requires Windows NuGet CLI") + def test_real_nuget_pack_matches_legacy_payload_paths(self) -> None: + row = matrix.select_builds(CLOSED)[0] + nuget = shutil.which("nuget") + assert nuget is not None + with TemporaryDirectory() as temporary: + root = Path(temporary) + fixture(root / "workspace", row, "closed", True) + stage_artifacts.stage_build(root / "workspace", root / "staged", row, True) + manifest = root / "new.nuspec" + (root / "staged/build.manifest").write_text("Provider metadata must not be reused") + package_artifacts.nuget_manifest(CLOSED, row["id"], root / "staged", manifest, + COMMIT, "hyperv.uefi", "main", True, root / "publication") + self.assertFalse((root / "publication/build.manifest").exists()) + self.assertTrue((root / "publication/Firmware Binary File DEBUG_X64/MSVM.fd").is_file()) + self.assertTrue((root / "publication/Build/MsvmX64/DEBUG_VS2022/Bin/Dsdt.aml").is_file()) + legacy = ET.Element("package") + metadata = ET.SubElement(legacy, "metadata") + for key, value in {"id": "legacy", "version": "26.0", "authors": "Microsoft", "description": "Test"}.items(): + ET.SubElement(metadata, key).text = value + files = ET.SubElement(legacy, "files") + for src, target in (("FV/MSVM.fd", ""), ("PDB/*", "symbols"), ("MAP/*", "map")): + ET.SubElement(files, "file", src=str(root / "workspace/Build" / "**" / src), target=target) + legacy_path = root / "legacy.nuspec" + ET.ElementTree(legacy).write(legacy_path, encoding="utf-8", xml_declaration=True) + payloads: list[set[str]] = [] + for name, spec in (("legacy", legacy_path), ("publication/packages", manifest)): + result = subprocess.run([nuget, "pack", str(spec), "-OutputDirectory", str(root / name), + "-NoPackageAnalysis", "-NonInteractive"], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + with zipfile.ZipFile(next((root / name).glob("*.nupkg"))) as archive: + payloads.append({path for path in archive.namelist() + if path.endswith((".fd", ".map", ".pdb"))}) + self.assertEqual(payloads[0], payloads[1]) + legacy_prefix = "MsvmX64/DEBUG_VS2022" + self.assertEqual(payloads[1], {f"{legacy_prefix}/FV/MSVM.fd", + f"symbols/{legacy_prefix}/PDB/Module.pdb", + f"map/{legacy_prefix}/MAP/Module.map"}) + + +class PublicationContractTests(unittest.TestCase): + def test_build_jobs_upload_but_cannot_publish_packages(self) -> None: + job = load_yaml(".azuredevops/templates/build.yml")["jobs"][1]["${{ each host in parameters.hosts }}"][0] + variables = job["variables"] + self.assertFalse(variables["ob_nugetPublishing_enabled"]) + self.assertFalse(variables["ob_createvpack_enabled"]) + self.assertTrue(variables["ob_pipelineartifacts_enabled"]) + self.assertEqual(variables["ob_artifactBaseName"], "$(build_artifact_name)") + for host in ("windows", "linux"): + key = "${{ if eq(host, '" + host + "') }}" + steps = next(branch[key] for branch in job["steps"] if key in branch) + self.assertEqual(steps[-1]["template"], "/.azuredevops/templates/stage-artifacts.yml@self") + windows = next(branch["${{ if eq(host, 'windows') }}"] for branch in job["steps"] + if "${{ if eq(host, 'windows') }}" in branch) + index = next(step for step in windows if step.get("displayName") == "Index Shipping Symbols") + self.assertEqual(index["condition"], "and(succeeded(), eq(variables['shipping'], 'true'))") + self.assertTrue(index["inputs"]["IndexSources"]) + self.assertFalse(index["inputs"]["PublishSymbols"]) + + def test_packages_wait_for_all_checks_and_official_branch(self) -> None: + stages = load_yaml(".azuredevops/templates/platform.yml")["extends"]["parameters"]["stages"] + packages = next(stage for stage in stages if stage["stage"] == "Packages") + self.assertEqual(packages["dependsOn"], ["SourceChecks", "Build"]) + gate = packages["${{ if eq(parameters.oneBranchBase, 'v2/Microsoft.Official.yml') }}"]["condition"] + self.assertIn("succeeded()", gate) + self.assertIn("'refs/heads/official/'", gate) + self.assertIn("ne(variables['Build.Reason'], 'PullRequest')", gate) + template = load_yaml(".azuredevops/templates/publish.yml") + self.assertFalse(template["parameters"][0]["default"]) + self.assertIn("--shipping-only", template["jobs"][0]["steps"][1]["pwsh"]) + job = template["jobs"][1] + self.assertEqual(job["pool"]["type"], "windows") + self.assertEqual(job["variables"]["ob_artifactBaseName"], "$(artifact_name)") + download = next(step for step in job["steps"] if step.get("task") == "DownloadPipelineArtifact@2") + self.assertEqual(download["inputs"]["artifactName"], "$(build_artifact_name)") + source = load_yaml(".azuredevops/templates/source-checks.yml")["jobs"][0] + self.assertEqual(source["variables"]["ob_artifactBaseName"], "drop_Build_SourceChecks") + for variable in ("ob_createvpack_enabled", "ob_nugetPublishing_enabled"): + self.assertEqual(job["variables"][variable], "${{ parameters.officialBuild }}") + symbol = job["steps"][-1]["${{ if eq(parameters.officialBuild, true) }}"][0] + self.assertTrue(symbol["inputs"]["PublishSymbols"]) + self.assertFalse(symbol["inputs"]["IndexSources"]) + self.assertNotIn("continueOnError", symbol) + + def test_github_release_and_failure_artifact_gates(self) -> None: + workflow = load_yaml(".github/workflows/platform-ci.yml") + release = workflow["jobs"]["release"] + self.assertEqual(release["if"], "github.event_name == 'push' && github.ref == 'refs/heads/main'") + self.assertEqual(release["needs"], ["source-checks", "build"]) + self.assertEqual(release["permissions"]["contents"], "write") + self.assertFalse(release["concurrency"]["cancel-in-progress"]) + stage = workflow["jobs"]["build"]["steps"][-1] + self.assertEqual(stage["if"], "always()") + self.assertEqual(stage["with"]["build-succeeded"], "${{ steps.firmware.outcome == 'success' }}") + upload = load_yaml(".github/actions/stage-artifacts/action.yml")["runs"]["steps"] + self.assertEqual(upload[1]["with"]["if-no-files-found"], "error") + self.assertEqual(upload[2]["if"], "always()") + + +if __name__ == "__main__": + unittest.main() diff --git a/ci/tests/test_build.py b/ci/tests/test_build.py new file mode 100644 index 0000000000..4ee2cd2b25 --- /dev/null +++ b/ci/tests/test_build.py @@ -0,0 +1,158 @@ +"""Verify build orchestration without installing tools or compiling firmware.""" + +import io +import os +import subprocess +import sys +import sysconfig +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from unittest.mock import MagicMock, patch + +from ci.scripts import build + +SCRIPT_PATH = Path(__file__).parents[1] / "scripts" / "build.py" + + +class BuildTests(unittest.TestCase): + """Lock command ordering, flavor propagation, and fail-fast behavior.""" + ARGS = ["--arch", "X64", "--target", "DEBUG", "--tool-chain", "CLANGPDB", "--core", "legacy"] + + def setUp(self) -> None: + """Suppress command previews without executing external processes.""" + self.output = redirect_stdout(io.StringIO()) + self.output.__enter__() + self.addCleanup(self.output.__exit__, None, None, None) + + @patch("ci.scripts.build.subprocess.run") + def test_platform_preparation_precedes_build(self, run_mock: MagicMock) -> None: + self.assertEqual(build.main(self.ARGS), 0) + calls = run_mock.call_args_list + self.assertEqual(len(calls), 4) + self.assertEqual(calls[0].args[0], [sys.executable, "-m", "pip", "install", "-r", "pip-requirements.txt"]) + suffix = ".exe" if os.name == "nt" else "" + for invocation, tool in zip(calls[1:], ("stuart_setup", "stuart_update", "stuart_build")): + self.assertEqual(invocation.args[0][:7], [ + str(Path(sysconfig.get_path("scripts")) / (tool + suffix)), + "-c", "MsvmPkg/PlatformBuild.py", "BUILD_ARCH=X64", "TARGET=DEBUG", + "TOOL_CHAIN_TAG=CLANGPDB", "BLD_*_USE_LEGACY_C_CORE=TRUE", + ]) + for invocation in calls: + self.assertEqual(invocation.kwargs, {"cwd": SCRIPT_PATH.resolve().parents[2], "check": True, + "env": {**os.environ, "SOURCE_ORIGIN": "unknown"}}) + if invocation is not calls[0]: + self.assertIn("BLD_*_LEGACY_DEBUGGER=0", invocation.args[0]) + self.assertIn("BUILDREPORT_TYPES=PCD DEPEX FLASH BUILD_FLAGS LIBRARY", calls[-1].args[0]) + self.assertIn("LaunchLogOnSuccess=FALSE", calls[-1].args[0]) + self.assertIn("LaunchLogOnError=FALSE", calls[-1].args[0]) + + @patch("ci.scripts.build.subprocess.run") + def test_patina_release_flavor(self, run_mock: MagicMock) -> None: + build.main(["--arch", "AARCH64", "--target", "RELEASE", "--tool-chain", "CLANGPDB", "--core", "patina"]) + for invocation in run_mock.call_args_list[1:]: + self.assertIn("BLD_*_USE_LEGACY_C_CORE=FALSE", invocation.args[0]) + self.assertIn("BUILD_ARCH=AARCH64", invocation.args[0]) + self.assertIn("TARGET=RELEASE", invocation.args[0]) + + @patch("ci.scripts.build.subprocess.run") + def test_failure_stops_remaining_commands(self, run_mock: MagicMock) -> None: + for failed_step in range(4): + with self.subTest(failed_step=failed_step): + run_mock.reset_mock() + run_mock.side_effect = [None] * failed_step + [subprocess.CalledProcessError(7, "command")] + self.assertEqual(build.main(self.ARGS), 7) + self.assertEqual(run_mock.call_count, failed_step + 1) + + @patch("ci.scripts.build.subprocess.run") + def test_dry_run_has_no_side_effects(self, run_mock: MagicMock) -> None: + self.assertEqual(build.main([*self.ARGS, "--dry-run"]), 0) + run_mock.assert_not_called() + + @patch("ci.scripts.build.subprocess.run") + def test_debugger_passed_to_all_stuart_phases(self, run_mock: MagicMock) -> None: + build.main(["--arch", "X64", "--target", "DEBUG", "--tool-chain", "VS2022", "--core", "legacy", + "--source-origin", "closed", "--legacy-debugger", "1"]) + for invocation in run_mock.call_args_list[1:]: + self.assertIn("BLD_*_LEGACY_DEBUGGER=1", invocation.args[0]) + + @patch("ci.scripts.build.subprocess.run") + def test_debugger_rejected_for_open_unknown_gcc_and_patina(self, run_mock: MagicMock) -> None: + for origin, tool, core in (("open", "VS2022", "legacy"), ("unknown", "VS2022", "legacy"), + ("open", "CLANGPDB", "legacy"), ("unknown", "CLANGPDB", "legacy"), + ("closed", "GCC", "legacy"), ("closed", "VS2022", "patina")): + with self.subTest(origin=origin, tool=tool, core=core), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as error: + build.main(["--arch", "X64", "--target", "DEBUG", "--tool-chain", tool, "--core", core, + "--host", "linux" if tool == "GCC" else "windows", + "--source-origin", origin, "--legacy-debugger", "1", "--dry-run"]) + self.assertEqual(error.exception.code, 2) + run_mock.assert_not_called() + + @patch("ci.scripts.build.subprocess.run") + def test_closed_clang_debugger_variants_are_accepted(self, run_mock: MagicMock) -> None: + """Preserve all three Clang debugger combinations configured by ADO.""" + for host, source in (("windows", "visual_studio"), ("windows", "windows_org"), + ("linux", "distribution")): + with self.subTest(host=host, source=source): + args = [*self.ARGS, "--host", host, "--compiler-source", source, + "--source-origin", "closed", "--legacy-debugger", "1", "--dry-run"] + if source == "windows_org": + args.extend(["--clang-bin", "internal-llvm/bin"]) + output = io.StringIO() + with redirect_stdout(output): + self.assertEqual(build.main(args), 0) + self.assertEqual(output.getvalue().count("BLD_*_LEGACY_DEBUGGER=1"), 3) + run_mock.assert_not_called() + + @patch("ci.scripts.build.subprocess.run") + def test_source_origin_is_explicit_and_inherited_by_all_commands(self, run_mock: MagicMock) -> None: + for origin in ("open", "closed", "unknown"): + with self.subTest(origin=origin), patch.dict(os.environ, {"SOURCE_ORIGIN": "inherited"}): + run_mock.reset_mock() + self.assertEqual(build.main([*self.ARGS, "--source-origin", origin]), 0) + self.assertEqual(os.environ["SOURCE_ORIGIN"], "inherited") + for invocation in run_mock.call_args_list: + self.assertEqual(invocation.kwargs["env"]["SOURCE_ORIGIN"], origin) + + @patch("ci.scripts.build.subprocess.run") + def test_windows_org_cannot_fall_back_to_visual_studio(self, run_mock: MagicMock) -> None: + with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error: + build.main([*self.ARGS, "--host", "windows", "--compiler-source", "windows_org", "--dry-run"]) + self.assertEqual(error.exception.code, 2) + output = io.StringIO() + with redirect_stdout(output): + self.assertEqual(build.main([ + *self.ARGS, "--host", "windows", "--compiler-source", "windows_org", + "--clang-bin", "internal-llvm/bin", "--dry-run", + ]), 0) + self.assertIn("CLANG_BIN=", output.getvalue()) + run_mock.assert_not_called() + + @patch("ci.scripts.build.subprocess.run") + def test_arm64_msvc_is_rejected_before_setup(self, run_mock: MagicMock) -> None: + with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error: + build.main(["--arch", "AARCH64", "--target", "RELEASE", "--tool-chain", "VS2022", "--core", "legacy"]) + self.assertEqual(error.exception.code, 2) + run_mock.assert_not_called() + + @patch("ci.scripts.build.subprocess.run") + def test_explicit_clang_bin_overrides_child_environment_only(self, run_mock: MagicMock) -> None: + with patch.dict(os.environ, {"CLANG_BIN": "visual-studio/bin/"}), patch.object(Path, "is_file", return_value=True): + build.main([*self.ARGS, "--clang-bin", "internal-llvm/bin"]) + self.assertEqual(os.environ["CLANG_BIN"], "visual-studio/bin/") + for invocation in run_mock.call_args_list: + self.assertEqual(invocation.kwargs["env"]["CLANG_BIN"], Path("internal-llvm/bin").resolve().as_posix() + "/") + + @patch("ci.scripts.build.subprocess.run") + def test_invalid_or_missing_flavor_is_rejected(self, run_mock: MagicMock) -> None: + for args in ([], [*self.ARGS[:-1], "unknown"]): + with self.subTest(args=args), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as error: + build.main(args) + self.assertEqual(error.exception.code, 2) + run_mock.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/ci/tests/test_firmware_version.py b/ci/tests/test_firmware_version.py new file mode 100644 index 0000000000..a6c724bc79 --- /dev/null +++ b/ci/tests/test_firmware_version.py @@ -0,0 +1,252 @@ +"""Exercise the real firmware version plugin without compiling firmware. + +Port the internal repository's override regression cases to unittest. The +Stuart VarDict is loaded dynamically, like the plugin, because neither offers +a typed API. Both the binary ABI and the firmware macro values are checked. +""" + +import importlib +import importlib.util +import builtins +import json +import os +import struct +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from types import ModuleType, SimpleNamespace +from typing import Any +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] + + +def load_plugin() -> ModuleType: + """Load the production plugin without triggering Stuart discovery.""" + path = ROOT / "MsvmPkg/BuildPlugins/FirmwareVersionBlob/FirmwareVersionBlob.py" + spec = importlib.util.spec_from_file_location("firmware_version_blob", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class FirmwareVersionTests(unittest.TestCase): + """Build vars win over shell vars, including explicit empty/false values.""" + + def test_dirty_version_reports_paths_without_changing_identity(self) -> None: + module = load_plugin() + with TemporaryDirectory() as temporary: + repo = importlib.import_module("git").Repo.init(temporary) + try: + path = Path(temporary) / "tracked.txt" + path.write_text("original") + repo.index.add(["tracked.txt"]) + actor = importlib.import_module("git").Actor("CI test", "ci@example.invalid") + commit = repo.index.commit("Test baseline", author=actor, committer=actor).hexsha + plugin = module.FirmwareVersionBlob() + self.assertEqual(plugin._GetGitCommit(temporary), (commit, False)) + path.write_text("modified") + (Path(temporary) / "generated.txt").write_text("not logged contents") + with self.assertLogs(level="WARNING") as logs: + self.assertEqual(plugin._GetGitCommit(temporary), (commit, True)) + message = "\n".join(logs.output) + self.assertIn("tracked.txt", message) + self.assertIn("generated.txt", message) + self.assertNotIn("not logged contents", message) + finally: + repo.close() + + def test_source_origin_is_explicit_sidecar_metadata(self) -> None: + """Origins honor Stuart precedence and leave the v1 binary ABI unchanged.""" + module = load_plugin() + cases: list[tuple[str | None, str | None, str]] = [ + (None, None, "unknown"), ("open", None, "open"), ("closed", None, "closed"), + ("closed", "open", "open"), ("open", "closed", "closed"), + ("closed", "unknown", "unknown"), + ] + blobs: list[bytes] = [] + for shell_origin, build_origin, expected in cases: + with self.subTest(shell=shell_origin, build=build_origin), TemporaryDirectory() as output: + env = importlib.import_module("edk2toolext.environment.var_dict").VarDict() + env.SetValue("BUILD_OUTPUT_BASE", output, "test") + if build_origin is not None: + env.SetValue("SOURCE_ORIGIN", build_origin, "test") + environment = {"OFFICIAL_BUILD": "1"} + if shell_origin is not None: + environment["SOURCE_ORIGIN"] = shell_origin + plugin = module.FirmwareVersionBlob() + builder = SimpleNamespace(env=env, GetWorkspaceRoot=lambda: str(ROOT)) + with patch.dict(os.environ, environment, clear=True), \ + patch.object(plugin, "_GetGitCommit", return_value=("a" * 40, False)): + self.assertEqual(plugin.do_pre_build(builder), 0) + metadata = json.loads((Path(output) / plugin.FW_VERSION_METADATA_SUBPATH).read_text()) + self.assertEqual(metadata["source_origin"], expected) + self.assertEqual(metadata["schema_version"], 1) + self.assertEqual(metadata["flags"], 2) + blobs.append((Path(output) / plugin.FW_VERSION_BLOB_SUBPATH).read_bytes()) + self.assertTrue(all(blob == blobs[0] for blob in blobs)) + self.assertEqual(len(blobs[0]), 80) + + def test_invalid_origin_is_rejected_before_emission(self) -> None: + module = load_plugin() + for origin in ("", "github", "official"): + with self.subTest(origin=origin), TemporaryDirectory() as output: + env = importlib.import_module("edk2toolext.environment.var_dict").VarDict() + env.SetValue("BUILD_OUTPUT_BASE", output, "test") + env.SetValue("SOURCE_ORIGIN", origin, "test") + plugin = module.FirmwareVersionBlob() + builder = SimpleNamespace(env=env, GetWorkspaceRoot=lambda: str(ROOT)) + with patch.object(plugin, "_GetGitCommit", return_value=("a" * 40, False)), \ + self.assertRaisesRegex(ValueError, "SOURCE_ORIGIN"): + plugin.do_pre_build(builder) + self.assertIsNone(env.GetValue("BLD_*_MSVM_FW_FLAGS")) + self.assertFalse((Path(output) / plugin.FW_VERSION_BLOB_SUBPATH).exists()) + self.assertFalse((Path(output) / plugin.FW_VERSION_METADATA_SUBPATH).exists()) + + def test_record_limits_fail_before_emitting_macros_or_blob(self) -> None: + """Invalid overrides must not silently truncate or diverge from PCDs.""" + module = load_plugin() + cases = [ + (1, 0, "1" * 16, "a" * 40), (1, 0, "26.0\0bad", "a" * 40), + (1, 0, "26.\u00e9", "a" * 40), (1, 0, "26.0", "a" * 48), + (-1, 0, "26.0", "a" * 40), (1, 65536, "26.0", "a" * 40), + ] + for major, minor, base, commit in cases: + with self.subTest(major=major, minor=minor, base=base, commit=commit), TemporaryDirectory() as output: + env = importlib.import_module("edk2toolext.environment.var_dict").VarDict() + env.SetValue("BUILD_OUTPUT_BASE", output, "test") + builder = SimpleNamespace(env=env, GetWorkspaceRoot=lambda: str(ROOT)) + plugin = module.FirmwareVersionBlob() + with patch.object(plugin, "_ReadVersionToml", return_value=(major, minor, base)), \ + patch.object(plugin, "_GetGitCommit", return_value=(commit, False)), \ + self.assertRaises(ValueError): + plugin.do_pre_build(builder) + self.assertIsNone(env.GetValue("BLD_*_MSVM_FW_BASE_VERSION")) + self.assertFalse((Path(output) / plugin.FW_VERSION_BLOB_SUBPATH).exists()) + + def test_string_boundary_reserves_nul(self) -> None: + module = load_plugin() + encoded = module.FirmwareVersionBlob._EncodeRecordString("1" * 15, 16, "BASE_VERSION") + self.assertEqual(struct.pack("16s", encoded), b"1" * 15 + b"\0") + + def test_tomli_fallback_when_stdlib_toml_is_unavailable(self) -> None: + """Exercise fallback dispatch on this interpreter without installing tomli.""" + real_import = builtins.__import__ + toml = importlib.import_module("tomllib") + + def fallback_import(name: str, *args: Any, **kwargs: Any) -> Any: + if name == "tomllib": + raise ModuleNotFoundError(name) + if name == "tomli": + return toml + return real_import(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=fallback_import): + module = load_plugin() + self.assertIs(module.tomllib, toml) + + def test_provider_official_context_reaches_both_hosts(self) -> None: + """Official provenance comes from the pipeline, never shipping metadata.""" + yaml = importlib.import_module("yaml") + github = yaml.safe_load((ROOT / ".github/workflows/platform-ci.yml").read_text()) + build_step = next(step for step in github["jobs"]["build"]["steps"] if step.get("uses") == "./.github/actions/build") + self.assertEqual(build_step["env"]["OFFICIAL_BUILD"], + "${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') && '1' || '0' }}") + action = yaml.safe_load((ROOT / ".github/actions/build/action.yml").read_text()) + self.assertIn("--source-origin open", action["runs"]["steps"][-1]["run"]) + for entry, base in (("platform-ci.yml", "Official"), ("platform-pr.yml", "NonOfficial")): + pipeline = yaml.safe_load((ROOT / ".azuredevops/pipelines" / entry).read_text()) + self.assertEqual(pipeline["extends"]["parameters"]["oneBranchBase"], f"v2/Microsoft.{base}.yml") + platform = yaml.safe_load((ROOT / ".azuredevops/templates/platform.yml").read_text()) + stage = next(stage for stage in platform["extends"]["parameters"]["stages"] if stage["stage"] == "Build") + self.assertEqual(stage["jobs"][0]["parameters"]["officialBuild"], + "${{ eq(parameters.oneBranchBase, 'v2/Microsoft.Official.yml') }}") + template = yaml.safe_load((ROOT / ".azuredevops/templates/build.yml").read_text()) + parameter = next(item for item in template["parameters"] if item["name"] == "officialBuild") + self.assertIs(parameter["default"], False) + job = template["jobs"][1]["${{ each host in parameters.hosts }}"][0] + for branch in job["steps"]: + if "task" in branch: + continue + steps = next(iter(branch.values())) + build_step = next(step for step in steps if "ci/scripts/build.py" in step.get("pwsh", step.get("bash", ""))) + self.assertEqual(build_step["env"]["OFFICIAL_BUILD"], "${{ iif(parameters.officialBuild, '1', '0') }}") + command = build_step.get("pwsh", build_step.get("bash", "")) + self.assertTrue("'--source-origin', 'closed'" in command or "--source-origin closed" in command) + + def test_build_runner_preserves_version_environment(self) -> None: + """The orchestration layer passes overrides to Stuart unchanged.""" + from ci.scripts import build + + with patch.dict(os.environ, {"OFFICIAL_BUILD": "1", "BASE_VERSION": "99.2"}), \ + patch("ci.scripts.build.subprocess.run") as run_mock: + self.assertEqual(build.main([ + "--arch", "X64", "--target", "DEBUG", "--tool-chain", "CLANGPDB", "--core", "legacy", + ]), 0) + self.assertEqual(run_mock.call_count, 4) + for invocation in run_mock.call_args_list: + self.assertEqual(invocation.kwargs["env"]["OFFICIAL_BUILD"], "1") + self.assertEqual(invocation.kwargs["env"]["BASE_VERSION"], "99.2") + + def test_blob_macros_and_log_agree(self) -> None: + module = load_plugin() + cases: list[tuple[str | None, str | None, str | None, str | None, bool, str | None]] = [ + (None, None, None, None, False, None), + ("1", None, "99.2", None, True, "99.2"), + ("", None, "", None, False, None), + ("0", None, None, None, False, None), + (" FaLsE ", None, None, None, False, None), + ("1", "0", "99.2", "88.1", False, "88.1"), + ("1", "", "99.2", "", False, None), + ("0", "1", None, "88.1", True, "88.1"), + ] + for dirty in (False, True): + for shell_official, build_official, shell_base, build_base, official, release in cases: + with self.subTest(dirty=dirty, official=official, release=release), TemporaryDirectory() as output: + env = importlib.import_module("edk2toolext.environment.var_dict").VarDict() + env.SetValue("BUILD_OUTPUT_BASE", output, "test") + environment = {key: value for key, value in os.environ.items() + if key not in ("OFFICIAL_BUILD", "BASE_VERSION", "SOURCE_ORIGIN")} + for name, shell_value, build_value in ( + ("OFFICIAL_BUILD", shell_official, build_official), + ("BASE_VERSION", shell_base, build_base), + ): + if shell_value is not None: + environment[name] = shell_value + if build_value is not None: + env.SetValue(name, build_value, "test") + plugin = module.FirmwareVersionBlob() + with (ROOT / plugin.FW_VERSION_TOML_SUBPATH).open("rb") as manifest: + version = module.tomllib.load(manifest) + expected_release = release if release is not None else str(version["release"]["version"]) + flags = int(dirty) | (2 if official else 0) + commit = "a" * 40 + builder = SimpleNamespace(env=env, GetWorkspaceRoot=lambda: str(ROOT)) + with patch.dict(os.environ, environment, clear=True), \ + patch.object(plugin, "_GetGitCommit", return_value=(commit, dirty)), \ + self.assertLogs(level="INFO") as logs: + self.assertEqual(plugin.do_pre_build(builder), 0) + blob = (Path(output) / plugin.FW_VERSION_BLOB_SUBPATH).read_bytes() + record = struct.unpack(" None: + for path in MATRICES: + for row in matrix.select_builds(path): + with self.subTest(path=path, row=row), redirect_stdout(io.StringIO()): + args = [ + "--source-origin", "closed" if path == CLOSED_MATRIX else "open", + "--dry-run", "--arch", row["arch"], "--target", row["target"], + "--tool-chain", row["tool_chain"], "--core", row["core"], + "--host", row["host"], "--compiler-source", row["compiler_source"], + "--legacy-debugger", row["legacy_debugger"], + ] + if row["compiler_source"] == "windows_org": + args.extend(["--clang-bin", "internal-llvm/bin"]) + with patch("ci.scripts.build.subprocess.run") as run_mock: + self.assertEqual(build.main(args), 0) + run_mock.assert_not_called() + + def test_only_the_supplied_matrix_is_read(self) -> None: + """Coverage is selected by the caller, never inferred from the backend.""" + row = matrix.parse_definition(json.loads(OPEN_MATRIX.read_text())[0]) + with patch.object(Path, "read_text", autospec=True, return_value=json.dumps([row])) as read_mock: + path = Path("custom") / "coverage.json" + self.assertEqual([item["id"] for item in matrix.select_builds(path)], [matrix.build_id(row)]) + read_mock.assert_called_once_with(path, encoding="utf-8") + + def test_coverage_and_derived_identifiers(self) -> None: + rows = matrix.select_builds(OPEN_MATRIX) + expected = { + (arch, target, tool_chain, core) + for target in ("DEBUG", "RELEASE") + for arch, tool_chain, core in ( + ("X64", "VS2022", "legacy"), + ("X64", "CLANGPDB", "legacy"), + ("AARCH64", "CLANGPDB", "legacy"), + ("X64", "CLANGPDB", "patina"), + ("AARCH64", "CLANGPDB", "patina"), + ) + } + self.assertEqual({(row["arch"], row["target"], row["tool_chain"], row["core"]) for row in rows}, expected) + self.assertEqual(len(rows), 10) + self.assertTrue(all(row["legacy_debugger"] == "0" for row in rows)) + for row in rows: + self.assertEqual(row["id"], matrix.build_id(row)) + closed = matrix.select_builds(CLOSED_MATRIX) + self.assertEqual(len(closed), 15) + self.assertEqual(len({row["id"] for row in closed}), 15) + self.assertEqual(len(matrix.select_builds(CLOSED_MATRIX, host="windows")), 10) + self.assertEqual(len(matrix.select_builds(CLOSED_MATRIX, host="linux")), 5) + shipping = {(row["arch"], row["target"], row["tool_chain"], row["compiler_source"], row["core"]) + for row in closed if row["shipping"] == "true"} + self.assertEqual(shipping, { + ("X64", "DEBUG", "VS2022", "visual_studio", "legacy"), + ("X64", "RELEASE", "VS2022", "visual_studio", "legacy"), + ("AARCH64", "RELEASE", "CLANGPDB", "windows_org", "legacy"), + }) + for row in closed: + self.assertEqual(row["legacy_debugger"], "1" if row["arch"] == "X64" and row["target"] == "DEBUG" else "0") + self.assertEqual(row["shipping"] == "true", row["package_suffix"] == "") + self.assertEqual({row["package_suffix"] for row in closed if row["shipping"] == "false"}, + {"XDCV", "XRCV", "ADCV", "ARCV", "XDCW", "XRCW", "ADCW", "XDCL", "XRCL", "ADCL", "ARCL", "XRG"}) + + def test_provider_formats_preserve_same_flavors(self) -> None: + for path in MATRICES: + rows = matrix.select_builds(path) + for format_name in ("json", "github", "ado"): + if path == CLOSED_MATRIX and format_name == "github": + with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error: + matrix.main(["--matrix", str(path), "--format", format_name]) + self.assertEqual(error.exception.code, 2) + continue + with self.subTest(path=path, format=format_name): + output = io.StringIO() + with redirect_stdout(output): + matrix.main(["--matrix", str(path), "--format", format_name]) + result: object = json.loads(output.getvalue()) + expected: object = rows + if format_name == "github": + expected = {"include": matrix.label_builds(rows, "github")} + elif format_name == "ado": + expected = {matrix.ado_job_name(row): {key: value for key, value in row.items() if key != "id"} + for row in matrix.label_builds(rows, "ado")} + self.assertEqual(result, expected) + + def test_github_labels_preserve_historical_artifact_names(self) -> None: + rows = matrix.select_builds(OPEN_MATRIX) + labeled = matrix.label_builds(rows, "github") + for original, row in zip(rows, labeled): + suffix = "-patina" if row["core"] == "patina" else "" + stem = f"{row['target']}-{row['arch']}-{row['tool_chain']}{suffix}" + self.assertEqual(row["artifact_name"], f"firmware-{stem}") + self.assertEqual(row["logs_artifact_name"], f"logs-{stem}") + self.assertEqual(row["id"], original["id"]) + self.assertNotIn("display_name", original) + self.assertEqual(labeled[0]["display_name"], "DEBUG X64 CLANGPDB (Legacy)") + self.assertEqual(labeled[0]["artifact_name"], "firmware-DEBUG-X64-CLANGPDB") + self.assertEqual(labeled[-1]["display_name"], "RELEASE AARCH64 CLANGPDB (Patina)") + self.assertEqual(labeled[-1]["artifact_name"], "firmware-RELEASE-AARCH64-CLANGPDB-patina") + + def test_ado_job_names_are_short_and_separate_from_artifact_identity(self) -> None: + rows = matrix.select_builds(CLOSED_MATRIX) + names = [matrix.ado_job_name(row) for row in rows] + self.assertEqual(len(set(names)), 15) + self.assertTrue(all(len(name) <= 35 and not name.endswith(("_0", "_1")) for name in names)) + self.assertEqual(names[0], "DEBUG_X64_VS2022_Debugger") + self.assertEqual(names[2], "RELEASE_AARCH64_ClangWin") + self.assertEqual(names[-1], "RELEASE_X64_GCC") + output = io.StringIO() + with redirect_stdout(output): + matrix.main(["--matrix", str(CLOSED_MATRIX), "--format", "ado"]) + emitted = json.loads(output.getvalue()) + for row in rows: + leg = emitted[matrix.ado_job_name(row)] + self.assertEqual(leg["build_id"], row["id"]) + self.assertEqual(leg["artifact_name"], + next(item["artifact_name"] for item in matrix.label_builds(rows, "ado") if item["id"] == row["id"])) + + def test_github_artifact_collisions_require_explicit_migration(self) -> None: + row = matrix.select_builds(OPEN_MATRIX)[0] + other: matrix.SelectedBuild = {**row, "compiler_source": "windows_org"} + other["id"] = matrix.build_id(other) + with self.assertRaisesRegex(ValueError, "Artifact name collision"): + matrix.label_builds([row, other], "github") + + def test_ado_labels_distinguish_compilers_without_renaming_packages(self) -> None: + rows = matrix.select_builds(CLOSED_MATRIX) + labeled = matrix.label_builds(rows, "ado") + self.assertEqual(len({row["display_name"] for row in labeled}), 15) + for original, row in zip(rows, labeled): + self.assertEqual(row["id"], original["id"]) + self.assertEqual(row["package_suffix"], original["package_suffix"]) + self.assertEqual(row["shipping"], original["shipping"]) + suffix = f"_{original['package_suffix']}" if original["package_suffix"] else "" + self.assertEqual(row["artifact_name"], + f"drop_Build_{original['target']}_{original['arch']}_{original['tool_chain']}{suffix}") + self.assertEqual(row["build_artifact_name"], + f"input-{original['id']}" if original["shipping"] == "true" else row["artifact_name"]) + self.assertEqual(row["build_id"], original["id"]) + self.assertIn("/ Windows WinOrg", next(row["display_name"] for row in labeled + if row["compiler_source"] == "windows_org")) + self.assertEqual(len({row["artifact_name"] for row in labeled}), 15) + self.assertEqual(len({row["build_artifact_name"] for row in labeled}), 15) + + def test_shipping_matrix_selects_only_three_windows_flavors(self) -> None: + output = io.StringIO() + with redirect_stdout(output): + matrix.main(["--matrix", str(CLOSED_MATRIX), "--shipping-only", "--format", "ado"]) + rows = json.loads(output.getvalue()) + self.assertEqual(len(rows), 3) + self.assertTrue(all(row["shipping"] == "true" and row["host"] == "windows" for row in rows.values())) + with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + matrix.main(["--matrix", str(OPEN_MATRIX), "--shipping-only"]) + + def test_invalid_matrix_is_rejected(self) -> None: + row = matrix.parse_definition(json.loads(OPEN_MATRIX.read_text())[0]) + invalid_documents: list[object] = [ + {}, [], [row, row], [{**row, "id": "invalid-id"}], + [{**row, "repositories": ["auto"]}], [{**row, "arch": "unknown"}], + [{**row, "core": "unknown"}], [{**row, "unexpected": True}], + [{**row, "repositories": ["closed"]}], + [{**row, "arch": "AARCH64", "tool_chain": "VS2022"}], + [{**row, "host": "linux"}], + [{**row, "legacy_debugger": "1", "arch": "AARCH64"}], + [{**row, "legacy_debugger": "1", "core": "patina"}], + [{**row, "legacy_debugger": "1", "host": "linux", "tool_chain": "GCC", "compiler_source": "distribution"}], + [{**row, "shipping": True}], [{**row, "package_suffix": ""}], + ] + for document in invalid_documents: + with self.subTest(document=document), patch.object(Path, "read_text", return_value=json.dumps(document)): + with self.assertRaises(ValueError): + matrix.select_builds(OPEN_MATRIX) + + def test_wrong_json_types_are_rejected(self) -> None: + """Untrusted values must fail validation, not leak into typed records.""" + row = matrix.parse_definition(json.loads(OPEN_MATRIX.read_text())[0]) + invalid_rows: list[object] = [None, [], "flavor", 1] + invalid_scalars: tuple[object, ...] = (None, True, 1, [], {}) + invalid_repositories: tuple[object, ...] = (None, "open", [], [True], [[]]) + for field in (*matrix.FLAVOR_CHOICES, *matrix.EXECUTION_CHOICES): + invalid_rows.extend({**row, field: value} for value in invalid_scalars) + invalid_rows.extend({**row, "repositories": value} for value in invalid_repositories) + for invalid in invalid_rows: + with self.subTest(row=invalid), self.assertRaises(ValueError): + matrix.parse_definition(invalid) + closed = {**row, "shipping": False, "package_suffix": "XDCV"} + for field, value in (("shipping", "false"), ("shipping", 0), ("package_suffix", None)): + with self.subTest(field=field, value=value), self.assertRaises(ValueError): + matrix.parse_definition({**closed, field: value}) + + def test_package_collisions_are_rejected(self) -> None: + """Different executions cannot silently publish the same closed package.""" + raw: object = json.loads(CLOSED_MATRIX.read_text()) + definitions = [matrix.parse_definition(value) for value in cast(list[object], raw)] + closed = definitions[0] + other = {**closed, "legacy_debugger": "0"} + with patch.object(Path, "read_text", return_value=json.dumps([closed, other])): + with self.assertRaisesRegex(ValueError, "Duplicate closed package"): + matrix.select_builds(CLOSED_MATRIX) + + def test_matrix_path_is_required(self) -> None: + for args in ([], ["--repo", "open"]): + with self.subTest(args=args), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as error: + matrix.main(args) + self.assertEqual(error.exception.code, 2) + + def test_host_filter_rejects_empty_or_unknown_selection(self) -> None: + for host in ("linux", "unknown"): + with self.subTest(host=host), self.assertRaises(ValueError): + matrix.select_builds(OPEN_MATRIX, host=host) + + def test_split_files_have_no_repository_tags(self) -> None: + for path in MATRICES: + raw: object = json.loads(path.read_text()) + for value in cast(list[object], raw): + definition = matrix.parse_definition(value) + self.assertNotIn("repositories", definition) + self.assertEqual("shipping" in definition, path == CLOSED_MATRIX) + self.assertEqual("package_suffix" in definition, path == CLOSED_MATRIX) + + +if __name__ == "__main__": + unittest.main() diff --git a/ci/tests/test_repo_auth.py b/ci/tests/test_repo_auth.py new file mode 100644 index 0000000000..2daab210bb --- /dev/null +++ b/ci/tests/test_repo_auth.py @@ -0,0 +1,94 @@ +"""Test CI Git authentication without touching the developer's Git configuration.""" + +import io +import os +import subprocess +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from contextlib import redirect_stderr, redirect_stdout +from unittest.mock import MagicMock, patch + +from ci.scripts import repo_auth + +URL = "https://dev.azure.com/example/project/_git/basecore" + + +class RepoAuthTests(unittest.TestCase): + def test_real_git_reads_header_in_separate_process_and_cleanup_removes_it(self) -> None: + with TemporaryDirectory() as directory: + config = Path(directory) / "gitconfig" + environment = {key: value for key, value in os.environ.items() if not key.startswith("GIT_CONFIG")} + environment.update({ + "GIT_CONFIG_GLOBAL": str(config), + "GIT_CONFIG_NOSYSTEM": "1", + "SYSTEM_ACCESSTOKEN": "test-token", + }) + with patch.dict(os.environ, environment, clear=True), redirect_stdout(io.StringIO()): + self.assertEqual(repo_auth.main(["set", "--url", URL]), 0) + try: + result = subprocess.run( + ["git", "-C", directory, "config", "--get-urlmatch", "http.extraheader", URL], + capture_output=True, text=True, check=True, + ) + self.assertEqual(result.stdout.strip(), "AUTHORIZATION: bearer test-token") + finally: + self.assertEqual(repo_auth.main(["unset", "--url", URL]), 0) + self.assertEqual(repo_auth.main(["unset", "--url", URL]), 0) + result = subprocess.run( + ["git", "-C", directory, "config", "--get-urlmatch", "http.extraheader", URL], + capture_output=True, text=True, + ) + self.assertEqual(result.returncode, 1) + self.assertNotIn("test-token", config.read_text()) + + @patch("ci.scripts.repo_auth.subprocess.run") + def test_set_uses_global_url_scoped_header_without_logging_token(self, run: MagicMock) -> None: + run.return_value = subprocess.CompletedProcess([], 0) + output = io.StringIO() + with patch.dict(os.environ, {"SYSTEM_ACCESSTOKEN": "test-token", "GIT_TRACE": "1", + "GIT_TRACE2": "1", "GIT_CURL_VERBOSE": "1"}), redirect_stdout(output): + self.assertEqual(repo_auth.main(["set", "--url", URL]), 0) + self.assertEqual(run.call_args.args[0], [ + "git", "config", "--global", "--replace-all", f"http.{URL}.extraheader", + "AUTHORIZATION: bearer test-token", + ]) + environment = run.call_args.kwargs["env"] + self.assertFalse(any(key.startswith("GIT_TRACE") for key in environment)) + self.assertNotIn("GIT_CURL_VERBOSE", environment) + self.assertNotIn("test-token", output.getvalue()) + + @patch("ci.scripts.repo_auth.subprocess.run") + def test_cleanup_is_idempotent_and_needs_no_token(self, run: MagicMock) -> None: + for code in (0, 5): + run.return_value = subprocess.CompletedProcess([], code) + with patch.dict(os.environ, {}, clear=True), redirect_stdout(io.StringIO()): + self.assertEqual(repo_auth.main(["unset", "--url", URL]), 0) + self.assertEqual(run.call_args.args[0], [ + "git", "config", "--global", "--unset-all", f"http.{URL}.extraheader", + ]) + + @patch("ci.scripts.repo_auth.subprocess.run") + def test_git_errors_are_not_suppressed_or_echoed(self, run: MagicMock) -> None: + run.return_value = subprocess.CompletedProcess([], 3, stderr=b"test-token") + for mode in ("set", "unset"): + error = io.StringIO() + with patch.dict(os.environ, {"SYSTEM_ACCESSTOKEN": "test-token"}), redirect_stderr(error): + self.assertEqual(repo_auth.main([mode, "--url", URL]), 3) + self.assertIn("failed (exit 3)", error.getvalue()) + self.assertNotIn("test-token", error.getvalue()) + + @patch("ci.scripts.repo_auth.subprocess.run") + def test_missing_token_and_invalid_url_fail_before_git(self, run: MagicMock) -> None: + for token in ("", "$(System.AccessToken)", "bad\ntoken"): + with patch.dict(os.environ, {"SYSTEM_ACCESSTOKEN": token}), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + repo_auth.main(["set", "--url", URL]) + for url in ("http://example.com/repo", "https://user:pass@example.com/repo"): + with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + repo_auth.main(["unset", "--url", url]) + run.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/ci/tests/test_source_checks.py b/ci/tests/test_source_checks.py new file mode 100644 index 0000000000..8877b303b7 --- /dev/null +++ b/ci/tests/test_source_checks.py @@ -0,0 +1,55 @@ +"""Verify the shared source-check sequence without invoking pip or Stuart.""" + +import sys +import subprocess +import unittest +from unittest.mock import MagicMock, call, patch + +from ci.scripts import source_checks + + +class SourceCheckTests(unittest.TestCase): + """Ensure provider callers and local developers use the native lifecycle.""" + + @patch("ci.scripts.source_checks.subprocess.run") + def test_source_checks_use_native_stuart_commands(self, run_mock: MagicMock) -> None: + source_checks.main([]) + + expected_commands = [ + [sys.executable, "-m", "pip", "install", "-r", "pip-requirements.txt"], + ["stuart_setup", "-c", ".pytool/CISettings.py"], + ["stuart_update", "-c", ".pytool/CISettings.py"], + [ + "stuart_ci_build", + "-c", + ".pytool/CISettings.py", + "--disable-all", + "GuidCheck=run", + "LineEndingCheck=run", + "UncrustifyCheck=run", + ], + ] + self.assertEqual( + run_mock.call_args_list, + [call(command, cwd=source_checks.REPO_ROOT, check=True) for command in expected_commands], + ) + + @patch("ci.scripts.source_checks.subprocess.run") + def test_verbose_only_enables_stuart_setup_logging(self, run_mock: MagicMock) -> None: + source_checks.main(["--verbose"]) + commands = [invocation.args[0] for invocation in run_mock.call_args_list] + self.assertEqual(len(commands), 4) + self.assertEqual(commands[1], ["stuart_setup", "-c", ".pytool/CISettings.py", "--verbose"]) + for index in (0, 2, 3): + self.assertNotIn("--verbose", commands[index]) + + @patch("ci.scripts.source_checks.subprocess.run") + def test_verbose_setup_failure_stops_checks(self, run_mock: MagicMock) -> None: + run_mock.side_effect = [None, subprocess.CalledProcessError(1, "stuart_setup")] + with self.assertRaises(subprocess.CalledProcessError): + source_checks.main(["--verbose"]) + self.assertEqual(run_mock.call_count, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/pip-requirements.txt b/pip-requirements.txt index f4257b6701..13ab689982 100644 --- a/pip-requirements.txt +++ b/pip-requirements.txt @@ -1,3 +1,4 @@ edk2-pytool-library~=0.23.8 edk2-pytool-extensions~=0.30.3 -regex \ No newline at end of file +regex +tomli>=2.0.1; python_version < "3.11" \ No newline at end of file