[POC ONLY] Unify ADO and Github CI

.azuredevops/build-matrix.json

@@ -0,0 +1,21 @@
+[
+  {"host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "VS2022", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "1", "shipping": true, "package_suffix": ""},
+  {"host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "VS2022", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": true, "package_suffix": ""},
+  {"host": "windows", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "0", "shipping": true, "package_suffix": ""},
+
+  {"host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "1", "shipping": false, "package_suffix": "XDCV"},
+  {"host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRCV"},
+  {"host": "windows", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ADCV"},
+  {"host": "windows", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ARCV"},
+
+  {"host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "1", "shipping": false, "package_suffix": "XDCW"},
+  {"host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRCW"},
+  {"host": "windows", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ADCW"},
+
+  {"host": "linux", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "1", "shipping": false, "package_suffix": "XDCL"},
+  {"host": "linux", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRCL"},
+  {"host": "linux", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ADCL"},
+  {"host": "linux", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ARCL"},
+
+  {"host": "linux", "arch": "X64", "target": "RELEASE", "tool_chain": "GCC", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRG"}
+]

.azuredevops/pipelines/platform-ci.yml

@@ -0,0 +1,13 @@
+name: $(SourceBranchName)_$(Date:yyyyMMdd_HHmmss)$(Rev:.r)
+
+trigger:
+  batch: false
+  branches:
+    include:
+      - official/*
+pr: none
+
+extends:
+  template: /.azuredevops/templates/platform.yml
+  parameters:
+    oneBranchBase: v2/Microsoft.Official.yml

.azuredevops/pipelines/platform-pr.yml

@@ -0,0 +1,6 @@
+trigger: none
+
+extends:
+  template: /.azuredevops/templates/platform.yml
+  parameters:
+    oneBranchBase: v2/Microsoft.NonOfficial.yml

.azuredevops/templates/build.yml

@@ -0,0 +1,194 @@
+parameters:
+  - name: matrixFile
+    type: string
+  - name: officialBuild
+    type: boolean
+    default: false
+  - name: hosts
+    type: object
+    default:
+      - windows
+      - linux
+  - name: windowsClangVersion
+    type: string
+    default: '22.1.1-47'
+  - name: windowsClangMajorVersion
+    type: string
+    default: '22'
+
+jobs:
+  - job: BuildMatrix
+    displayName: Select Build Matrix
+    pool:
+      type: windows
+    variables:
+      WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest
+      ob_outputDirectory: $(Build.ArtifactStagingDirectory)/BUILD_MATRIX
+      ob_nugetPublishing_enabled: false
+      ob_createvpack_enabled: false
+    steps:
+      - template: /.azuredevops/templates/windows-python.yml@self
+
+      - ${{ each host in parameters.hosts }}:
+          - pwsh: |
+              $matrix = python ci/scripts/matrix.py --matrix "$env:MATRIX_FILE" --host "$env:BUILD_HOST" --format ado
+              if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+              Write-Host "##vso[task.setvariable variable=matrix;isOutput=true]$matrix"
+            name: ${{ host }}
+            displayName: Select ${{ host }} Builds
+            env:
+              MATRIX_FILE: ${{ parameters.matrixFile }}
+              BUILD_HOST: ${{ host }}
+
+  - ${{ each host in parameters.hosts }}:
+      - job: BuildFirmware_${{ host }}
+        displayName: Firmware (${{ host }})
+        dependsOn: BuildMatrix
+        pool:
+          type: ${{ host }}
+        strategy:
+          maxParallel: 4
+          matrix: $[ dependencies.BuildMatrix.outputs['${{ host }}.matrix'] ]
+        variables:
+          WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest
+          LinuxContainerImage: onebranch.azurecr.io/linux/ubuntu-2404:latest
+          GIT_CONFIG_COUNT: "2"
+          GIT_CONFIG_KEY_0: core.autocrlf
+          GIT_CONFIG_VALUE_0: "false"
+          GIT_CONFIG_KEY_1: core.longpaths
+          GIT_CONFIG_VALUE_1: "true"
+          ob_outputDirectory: $(Build.ArtifactStagingDirectory)/BUILD/$(System.JobId)
+          ob_artifactBaseName: $(build_artifact_name)
+          ob_pipelineartifacts_enabled: true
+          ob_artifacts_main_condition: succeededOrFailed()
+          ob_nugetPublishing_enabled: false
+          ob_createvpack_enabled: false
+          WindowsClangRoot: $(Agent.TempDirectory)/windows-clang-${{ parameters.windowsClangVersion }}
+        steps:
+          - task: NuGetAuthenticate@1
+            inputs:
+              nuGetServiceConnections: WindowsPartners_NugetFeedConnection
+
+          - task: PipAuthenticate@1
+            inputs:
+              artifactFeeds: $(System.TeamProject)/$(System.TeamProject)_PublicPackages
+
+          - ${{ if eq(host, 'windows') }}:
+              - template: /.azuredevops/templates/windows-python.yml@self
+
+              - pwsh: |
+                  $vswhere = "${env:ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere.exe"
+                  $installation = & $vswhere -latest -version '[17.0,18.0)' -products '*' -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath
+                  if ($LASTEXITCODE -ne 0 -or -not $installation) { throw 'Visual Studio 2022 was not found' }
+                  $clangBin = Join-Path $installation 'VC/Tools/Llvm/x64/bin'
+                  if (-not (Test-Path (Join-Path $clangBin 'clang.exe'))) { throw 'Visual Studio Clang was not found' }
+                  Write-Host "##vso[task.setvariable variable=CLANG_BIN]$clangBin/"
+                displayName: Locate Visual Studio Clang
+                condition: and(succeeded(), eq(variables['tool_chain'], 'CLANGPDB'), eq(variables['compiler_source'], 'visual_studio'))
+
+              - task: DownloadPackage@1
+                displayName: Download Windows-org LLVM Tools
+                condition: and(succeeded(), eq(variables['compiler_source'], 'windows_org'))
+                inputs:
+                  packageType: nuget
+                  feed: ES-LLVM
+                  definition: llvm.tools
+                  version: ${{ parameters.windowsClangVersion }}
+                  downloadPath: $(WindowsClangRoot)/bin
+
+              - task: DownloadPackage@1
+                displayName: Download Windows-org Clang Headers
+                condition: and(succeeded(), eq(variables['compiler_source'], 'windows_org'))
+                inputs:
+                  packageType: nuget
+                  feed: ES-LLVM
+                  definition: clang.headers
+                  version: ${{ parameters.windowsClangVersion }}
+                  downloadPath: $(WindowsClangRoot)/lib/clang/${{ parameters.windowsClangMajorVersion }}
+
+              - template: /.azuredevops/templates/repo-auth.yml@self
+                parameters:
+                  mode: set
+
+              - pwsh: |
+                  $buildArgs = @('--arch', $env:BUILD_ARCH, '--target', $env:BUILD_TARGET,
+                      '--tool-chain', $env:BUILD_TOOL_CHAIN, '--core', $env:BUILD_CORE,
+                      '--host', 'windows', '--compiler-source', $env:BUILD_COMPILER_SOURCE,
+                      '--legacy-debugger', $env:BUILD_LEGACY_DEBUGGER, '--source-origin', 'closed')
+                  if ($env:BUILD_COMPILER_SOURCE -eq 'windows_org') {
+                      $buildArgs += @('--clang-bin', "$env:WINDOWS_CLANG_ROOT/bin")
+                  }
+                  python ci/scripts/build.py @buildArgs
+                  exit $LASTEXITCODE
+                displayName: Build Firmware
+                env:
+                  BUILD_ARCH: $(arch)
+                  BUILD_TARGET: $(target)
+                  BUILD_TOOL_CHAIN: $(tool_chain)
+                  BUILD_CORE: $(core)
+                  BUILD_COMPILER_SOURCE: $(compiler_source)
+                  BUILD_LEGACY_DEBUGGER: $(legacy_debugger)
+                  WINDOWS_CLANG_ROOT: $(WindowsClangRoot)
+                  OFFICIAL_BUILD: ${{ iif(parameters.officialBuild, '1', '0') }}
+
+              - task: PublishSymbols@2
+                displayName: Index Shipping Symbols
+                condition: and(succeeded(), eq(variables['shipping'], 'true'))
+                inputs:
+                  SearchPattern: Build/Msvm$(arch)/$(target)_$(tool_chain)/PDB/*.pdb
+                  IndexSources: true
+                  PublishSymbols: false
+
+              - template: /.azuredevops/templates/repo-auth.yml@self
+                parameters:
+                  mode: unset
+
+              - template: /.azuredevops/templates/stage-artifacts.yml@self
+
+          - ${{ if eq(host, 'linux') }}:
+              - template: /.azuredevops/templates/repo-auth.yml@self
+                parameters:
+                  mode: set
+                  python: python3
+
+              - bash: |
+                  set -euo pipefail
+                  apt-get update
+                  apt-get install -y python3-venv
+                  case "$BUILD_TOOL_CHAIN" in
+                    CLANGPDB) apt-get install -y clang lld llvm ;;
+                    GCC)
+                      case "$BUILD_ARCH" in
+                        X64) triple=x86_64; packages=x86-64 ;;
+                        AARCH64) triple=aarch64; packages=aarch64 ;;
+                        *) exit 2 ;;
+                      esac
+                      apt-get install -y "gcc-$packages-linux-gnu" "binutils-$packages-linux-gnu"
+                      export "GCC_${BUILD_ARCH}_PREFIX=/usr/bin/$triple-linux-gnu-"
+                      ;;
+                    *) exit 2 ;;
+                  esac
+                  python3 -m venv "$BUILD_VENV"
+                  source "$BUILD_VENV/bin/activate"
+                  python ci/scripts/build.py --arch "$BUILD_ARCH" --target "$BUILD_TARGET" \
+                    --tool-chain "$BUILD_TOOL_CHAIN" --core "$BUILD_CORE" --host linux \
+                    --compiler-source "$BUILD_COMPILER_SOURCE" --legacy-debugger "$BUILD_LEGACY_DEBUGGER" --source-origin closed
+                displayName: Prepare Linux Compiler and Build Firmware
+                env:
+                  BUILD_ARCH: $(arch)
+                  BUILD_TARGET: $(target)
+                  BUILD_TOOL_CHAIN: $(tool_chain)
+                  BUILD_CORE: $(core)
+                  BUILD_COMPILER_SOURCE: $(compiler_source)
+                  BUILD_LEGACY_DEBUGGER: $(legacy_debugger)
+                  BUILD_VENV: $(Agent.TempDirectory)/firmware-venv
+                  OFFICIAL_BUILD: ${{ iif(parameters.officialBuild, '1', '0') }}
+
+              - template: /.azuredevops/templates/repo-auth.yml@self
+                parameters:
+                  mode: unset
+                  python: python3
+
+              - template: /.azuredevops/templates/stage-artifacts.yml@self
+                parameters:
+                  python: python3

.azuredevops/templates/platform.yml

@@ -0,0 +1,47 @@
+parameters:
+  - name: oneBranchBase
+    type: string
+
+resources:
+  repositories:
+    - repository: templates_onebranch
+      type: git
+      name: OneBranch.Pipelines/GovernedTemplates
+      ref: refs/heads/main
+
+extends:
+  template: ${{ parameters.oneBranchBase }}@templates_onebranch
+  parameters:
+    platform:
+      name: windows_undocked
+    globalSdl:
+      tsa:
+        enabled: false
+    featureFlags:
+      WindowsHostVersion: 1ESWindows2022
+    nugetPublishing:
+      feeds:
+        - name: OS/HyperVUefi
+          continueOnConflict: false
+          files_to_publish: '*.nupkg;!*.symbols.nupkg'
+    git:
+      submodules: false
+    stages:
+      - stage: SourceChecks
+        jobs:
+          - template: /.azuredevops/templates/source-checks.yml@self
+      - stage: Build
+        dependsOn: []
+        jobs:
+          - template: /.azuredevops/templates/build.yml@self
+            parameters:
+              matrixFile: .azuredevops/build-matrix.json
+              officialBuild: ${{ eq(parameters.oneBranchBase, 'v2/Microsoft.Official.yml') }}
+      - stage: Packages
+        dependsOn: [SourceChecks, Build]
+        ${{ if eq(parameters.oneBranchBase, 'v2/Microsoft.Official.yml') }}:
+          condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/official/'), ne(variables['Build.Reason'], 'PullRequest'))
+        jobs:
+          - template: /.azuredevops/templates/publish.yml@self
+            parameters:
+              officialBuild: ${{ eq(parameters.oneBranchBase, 'v2/Microsoft.Official.yml') }}

.azuredevops/templates/publish.yml

@@ -0,0 +1,106 @@
+parameters:
+  - name: officialBuild
+    type: boolean
+    default: false
+
+jobs:
+  - job: PackageMatrix
+    displayName: Select Shipping Packages
+    pool:
+      type: windows
+    variables:
+      WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest
+      ob_outputDirectory: $(Build.ArtifactStagingDirectory)/PACKAGE_MATRIX
+      ob_nugetPublishing_enabled: false
+      ob_createvpack_enabled: false
+    steps:
+      - template: /.azuredevops/templates/windows-python.yml@self
+
+      - pwsh: |
+          $matrix = python -m ci.scripts.matrix --matrix .azuredevops/build-matrix.json --shipping-only --format ado
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+          $version = python -m ci.scripts.package_artifacts version
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+          $major, $minor = $version.Split('.')
+          Write-Host "##vso[task.setvariable variable=matrix;isOutput=true]$matrix"
+          Write-Host "##vso[task.setvariable variable=major;isOutput=true]$major"
+          Write-Host "##vso[task.setvariable variable=minor;isOutput=true]$minor"
+        name: shipping
+        displayName: Select Shipping Flavors and Release Version
+
+  - job: PackageFirmware
+    displayName: Package Firmware
+    dependsOn: PackageMatrix
+    pool:
+      type: windows
+    strategy:
+      maxParallel: 3
+      matrix: $[ dependencies.PackageMatrix.outputs['shipping.matrix'] ]
+    variables:
+      WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest
+      ReleaseMajor: $[ dependencies.PackageMatrix.outputs['shipping.major'] ]
+      ReleaseMinor: $[ dependencies.PackageMatrix.outputs['shipping.minor'] ]
+      ob_outputDirectory: $(Build.ArtifactStagingDirectory)/PACKAGES
+      ob_artifactBaseName: $(artifact_name)
+      ob_nugetPublishing_enabled: ${{ parameters.officialBuild }}
+      ob_createvpack_enabled: ${{ parameters.officialBuild }}
+      ${{ if eq(parameters.officialBuild, true) }}:
+        ob_createvpack_packagename: $(Build.Repository.Name).mscoreuefi.$(arch).$(target)
+      ${{ else }}:
+        ob_createvpack_packagename: $(Build.Repository.Name).mscoreuefi.PR.$(arch).$(target).Test
+      ob_createvpack_description: Hyper V UEFI Firmware
+      ob_createvpack_versionAs: parts
+      ob_createvpack_majorVer: $(ReleaseMajor)
+      ob_createvpack_minorVer: $(ReleaseMinor)
+      ob_createvpack_metadata: $(Build.SourceVersion)
+      ob_createvpack_failOnStdErr: true
+      ob_createvpack_verbose: true
+      ob_createvpack_vpackdirectory: $(ob_outputDirectory)/Build/Msvm$(arch)/$(target)_$(tool_chain)/Bin
+    steps:
+      - template: /.azuredevops/templates/windows-python.yml@self
+
+      - task: DownloadPipelineArtifact@2
+        displayName: Download Shipping Firmware
+        inputs:
+          buildType: current
+          artifactName: $(build_artifact_name)
+          targetPath: $(Agent.TempDirectory)/firmware-input
+
+      - pwsh: |
+          $packageArgs = @('nuget', '--matrix', '.azuredevops/build-matrix.json',
+              '--build-id', $env:BUILD_ID, '--source', $env:ARTIFACT_INPUT,
+              '--output', $env:PACKAGE_MANIFEST, '--staging-output', $env:ARTIFACT_OUTPUT,
+              '--commit', $env:SOURCE_COMMIT,
+              '--repository', $env:SOURCE_REPOSITORY, '--branch', $env:SOURCE_BRANCH)
+          if ($env:OFFICIAL_BUILD -eq '1') { $packageArgs += '--official' }
+          python -m ci.scripts.package_artifacts @packageArgs
+          exit $LASTEXITCODE
+        displayName: Validate Firmware and Prepare NuGet Manifest
+        env:
+          BUILD_ID: $(build_id)
+          ARTIFACT_INPUT: $(Agent.TempDirectory)/firmware-input
+          ARTIFACT_OUTPUT: $(ob_outputDirectory)
+          PACKAGE_MANIFEST: $(Agent.TempDirectory)/firmware.nuspec
+          SOURCE_COMMIT: $(Build.SourceVersion)
+          SOURCE_REPOSITORY: $(Build.Repository.Name)
+          SOURCE_BRANCH: $(Build.SourceBranchName)
+          OFFICIAL_BUILD: ${{ iif(parameters.officialBuild, '1', '0') }}
+
+      - task: NuGetCommand@2
+        displayName: Pack NuGet
+        inputs:
+          command: pack
+          packagesToPack: $(Agent.TempDirectory)/firmware.nuspec
+          packDestination: $(ob_outputDirectory)/packages
+          packTimezone: utc
+
+      - ${{ if eq(parameters.officialBuild, true) }}:
+          - task: PublishSymbols@2
+            displayName: Publish Shipping Symbols
+            inputs:
+              SymbolsFolder: $(ob_outputDirectory)/firmware/PDB
+              SearchPattern: '*.pdb'
+              IndexSources: false
+              PublishSymbols: true
+              SymbolServerType: TeamServices
+              SymbolsProduct: Hyper-V UEFI