From 8e34d3f3b539c25bf5ca7229d45bb0e90b1c3ba1 Mon Sep 17 00:00:00 2001 From: Maheer Aeron Date: Thu, 17 Sep 2026 15:46:21 -0700 Subject: [PATCH 01/18] ci: preserve hyperv.uefi OneBranch reference --- .../pipelines/OneBranch/CI-Build.yaml | 23 ++ .../pipelines/OneBranch/PR-Build.yaml | 29 ++ .azuredevops/pipelines/OneBranch/common.yaml | 43 +++ .../pipelines/OneBranch/data/auth-urls.json | 6 + .../pipelines/OneBranch/data/matrix.yml | 65 ++++ .../pipelines/OneBranch/data/vars.yml | 24 ++ .azuredevops/pipelines/OneBranch/jobs.yml | 195 ++++++++++ .../pipelines/OneBranch/scripts/README.md | 134 +++++++ .../pipelines/OneBranch/scripts/build.py | 102 ++++++ .../pipelines/OneBranch/scripts/ci_common.py | 41 +++ .../OneBranch/scripts/get_vpack_name.py | 61 ++++ .../OneBranch/scripts/install_toolchain.py | 106 ++++++ .../OneBranch/scripts/run_local_ci.py | 96 +++++ .../OneBranch/scripts/set_repo_auth.py | 90 +++++ .../pipelines/OneBranch/scripts/setup.py | 60 ++++ .../OneBranch/scripts/source_checks.py | 48 +++ .../OneBranch/scripts/stage_artifacts.py | 156 ++++++++ .../scripts/tests/test_get_vpack_name.py | 111 ++++++ .../scripts/tests/test_source_checks.py | 120 +++++++ .../OneBranch/scripts/tests/verify_ci.py | 332 ++++++++++++++++++ .../pipelines/OneBranch/source-checks.yml | 59 ++++ .azuredevops/pipelines/Sync/README.md | 1 + 22 files changed, 1902 insertions(+) create mode 100644 .azuredevops/pipelines/OneBranch/CI-Build.yaml create mode 100644 .azuredevops/pipelines/OneBranch/PR-Build.yaml create mode 100644 .azuredevops/pipelines/OneBranch/common.yaml create mode 100644 .azuredevops/pipelines/OneBranch/data/auth-urls.json create mode 100644 .azuredevops/pipelines/OneBranch/data/matrix.yml create mode 100644 .azuredevops/pipelines/OneBranch/data/vars.yml create mode 100644 .azuredevops/pipelines/OneBranch/jobs.yml create mode 100644 .azuredevops/pipelines/OneBranch/scripts/README.md create mode 100644 .azuredevops/pipelines/OneBranch/scripts/build.py create mode 100644 .azuredevops/pipelines/OneBranch/scripts/ci_common.py create mode 100644 .azuredevops/pipelines/OneBranch/scripts/get_vpack_name.py create mode 100644 .azuredevops/pipelines/OneBranch/scripts/install_toolchain.py create mode 100644 .azuredevops/pipelines/OneBranch/scripts/run_local_ci.py create mode 100644 .azuredevops/pipelines/OneBranch/scripts/set_repo_auth.py create mode 100644 .azuredevops/pipelines/OneBranch/scripts/setup.py create mode 100644 .azuredevops/pipelines/OneBranch/scripts/source_checks.py create mode 100644 .azuredevops/pipelines/OneBranch/scripts/stage_artifacts.py create mode 100644 .azuredevops/pipelines/OneBranch/scripts/tests/test_get_vpack_name.py create mode 100644 .azuredevops/pipelines/OneBranch/scripts/tests/test_source_checks.py create mode 100644 .azuredevops/pipelines/OneBranch/scripts/tests/verify_ci.py create mode 100644 .azuredevops/pipelines/OneBranch/source-checks.yml create mode 100644 .azuredevops/pipelines/Sync/README.md diff --git a/.azuredevops/pipelines/OneBranch/CI-Build.yaml b/.azuredevops/pipelines/OneBranch/CI-Build.yaml new file mode 100644 index 0000000000..525d564fe8 --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/CI-Build.yaml @@ -0,0 +1,23 @@ + +name: $(SourceBranchName)_$(Date:yyyyMMdd_HHmmss)$(Rev:.r) + +# https://aka.ms/obpipelines/triggers +trigger: + batch: False + branches: + include: + - official/* +pr: none + +variables: +- template: data/vars.yml +# Because we have a template, we must use name/value to add more. +- name: PRdot + value: '' + +extends: + template: common.yaml + parameters: + base: v2/Microsoft.Official.yml # EXTEND MICROSOFT.OFFICIAL.YML + pipeline: Official + publishNuget: True diff --git a/.azuredevops/pipelines/OneBranch/PR-Build.yaml b/.azuredevops/pipelines/OneBranch/PR-Build.yaml new file mode 100644 index 0000000000..f008efd058 --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/PR-Build.yaml @@ -0,0 +1,29 @@ +# https://aka.ms/obpipelines/triggers +trigger: none + +parameters: + +# This controls only Nuget create, not Vpack creation. +# Due to upstream code/policy, vpacks cannot be created in the PR pipeline. +# The official "CI" pipeline, which creates vpacks, can be run against PRs with approval. +- name: PublishPRNuget + displayName: Publish Nuget Pkg for this PR for testing? + type: boolean + default: False + +variables: +- template: data/vars.yml +# Because we have a template, we must use name/value to add more. +- name: PRdot + value: 'PR.' + +extends: + template: common.yaml + parameters: + base: v2/Microsoft.NonOfficial.yml # https://aka.ms/obpipelines/templates + pipeline: PR + + # This controls only Nuget create, not Vpack creation. + # Due to upstream code/policy, vpacks cannot be created in the PR pipeline. + # The official "CI" pipeline, which creates vpacks, can be run against PRs with approval. + publishNuget: ${{parameters.PublishPRNuget}} diff --git a/.azuredevops/pipelines/OneBranch/common.yaml b/.azuredevops/pipelines/OneBranch/common.yaml new file mode 100644 index 0000000000..6ecd93d90f --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/common.yaml @@ -0,0 +1,43 @@ +parameters: +- name: base +- name: pipeline + type: string + values: + - Official + - PR +- name: publishNuget + type: boolean + default: False + +resources: + repositories: + - repository: templates_onebranch + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + +extends: + template: ${{parameters.base}}@templates_onebranch # https://aka.ms/obpipelines/templates + parameters: + platform: + name: windows_undocked + globalSdl: + tsa: + enabled: False + featureFlags: + WindowsHostVersion: 1ESWindows2022 + nugetPublishing: + feeds: + - name: OS/HyperVUefi + continueOnConflict: False + files_to_publish: '*.nupkg;!*.symbols.nupkg' + git: + submodules: False + stages: + - stage: Build + jobs: + - template: .azuredevops/pipelines/OneBranch/source-checks.yml@self + - template: .azuredevops/pipelines/OneBranch/data/matrix.yml@self + parameters: + pipeline: ${{parameters.pipeline}} + publishNuget: ${{parameters.publishNuget}} diff --git a/.azuredevops/pipelines/OneBranch/data/auth-urls.json b/.azuredevops/pipelines/OneBranch/data/auth-urls.json new file mode 100644 index 0000000000..2da9ce73ef --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/data/auth-urls.json @@ -0,0 +1,6 @@ +{ + "_comment": "Internal ADO repos needing System.AccessToken auth. Shared by set_repo_auth.py Set/Unset so the phases cannot drift.", + "urls": [ + "https://dev.azure.com/microsoft/OS/_git/hyperv.uefi.mu_basecore" + ] +} diff --git a/.azuredevops/pipelines/OneBranch/data/matrix.yml b/.azuredevops/pipelines/OneBranch/data/matrix.yml new file mode 100644 index 0000000000..02c6643639 --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/data/matrix.yml @@ -0,0 +1,65 @@ +# Matrix data + entry point for the Hyper-V UEFI CI build. +# +# Each row in Builds is one variant. Fields: +# Pool 'Windows' | 'Linux' +# Arch 'X64' | 'AARCH64' +# Target 'DEBUG' | 'RELEASE' +# ToolChainTag 'VS2022' | 'CLANGPDB' | 'GCC' +# ClangWindows true => Windows-org-built Clang (CLANGPDB + Pool=Windows only) +# GnuArch GNU triple for Linux GCC (e.g. 'x86-64', 'aarch64') +# Salt non-empty for test variants; encodes Arch+Target+Toolchain+Host +# (A/X, D/R, CV/CW/CL/G). +# Production VS2022 rows have NO Salt. +# Env extra env vars (typically CLANG_BIN). Trailing slash required. +# Extra extra PlatformBuild.py args. Every row explicitly selects its core. +# +# Shipped: the two VS2022 rows. All other rows are .Test variants. +# GCC is unshippable (no Codeview, 64K-aligned files); Clang/X64 not yet shippable. + +parameters: +- name: pipeline + type: string + values: [Official, PR] +- name: publishNuget + type: boolean + default: false + +- name: Builds + type: object + default: + # Production (shipped). No Salt. + - {Pool: Windows, Arch: X64, Target: DEBUG, ToolChainTag: VS2022, Extra: "BLD_*_USE_LEGACY_C_CORE=TRUE BLD_*_LEGACY_DEBUGGER=1"} + - {Pool: Windows, Arch: X64, Target: RELEASE, ToolChainTag: VS2022, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} + # Windows + Clang from Visual Studio. Salt: *CV. + - {Pool: Windows, Arch: X64, Target: DEBUG, ToolChainTag: CLANGPDB, Salt: XDCV, Env: {CLANG_BIN: $(CLANGVS)/bin/}, Extra: "BLD_*_USE_LEGACY_C_CORE=TRUE BLD_*_LEGACY_DEBUGGER=1"} + - {Pool: Windows, Arch: X64, Target: RELEASE, ToolChainTag: CLANGPDB, Salt: XRCV, Env: {CLANG_BIN: $(CLANGVS)/bin/}, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} + - {Pool: Windows, Arch: AARCH64, Target: DEBUG, ToolChainTag: CLANGPDB, Salt: ADCV, Env: {CLANG_BIN: $(CLANGVS)/bin/}, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} + - {Pool: Windows, Arch: AARCH64, Target: RELEASE, ToolChainTag: CLANGPDB, Salt: ARCV, Env: {CLANG_BIN: $(CLANGVS)/bin/}, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} + # Windows + Clang from Windows org. Salt: *CW. + - {Pool: Windows, Arch: X64, Target: DEBUG, ToolChainTag: CLANGPDB, ClangWindows: true, Salt: XDCW, Env: {CLANG_BIN: $(CLANGWIN_BIN)/}, Extra: "BLD_*_USE_LEGACY_C_CORE=TRUE BLD_*_LEGACY_DEBUGGER=1"} + - {Pool: Windows, Arch: X64, Target: RELEASE, ToolChainTag: CLANGPDB, ClangWindows: true, Salt: XRCW, Env: {CLANG_BIN: $(CLANGWIN_BIN)/}, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} + - {Pool: Windows, Arch: AARCH64, Target: DEBUG, ToolChainTag: CLANGPDB, ClangWindows: true, Salt: ADCW, Env: {CLANG_BIN: $(CLANGWIN_BIN)/}, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} + - {Pool: Windows, Arch: AARCH64, Target: RELEASE, ToolChainTag: CLANGPDB, ClangWindows: true, Env: {CLANG_BIN: $(CLANGWIN_BIN)/}, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} + # Linux + Clang. Salt: *CL. + - {Pool: Linux, Arch: X64, Target: DEBUG, ToolChainTag: CLANGPDB, Salt: XDCL, Extra: "BLD_*_USE_LEGACY_C_CORE=TRUE BLD_*_LEGACY_DEBUGGER=1"} + - {Pool: Linux, Arch: X64, Target: RELEASE, ToolChainTag: CLANGPDB, Salt: XRCL, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} + - {Pool: Linux, Arch: AARCH64, Target: DEBUG, ToolChainTag: CLANGPDB, Salt: ADCL, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} + - {Pool: Linux, Arch: AARCH64, Target: RELEASE, ToolChainTag: CLANGPDB, Salt: ARCL, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} + # Linux + GCC. Salt: *G. X64 DEBUG + AARCH64 disabled (mu_basecore PR #1710 / firmware size). + - {Pool: Linux, Arch: X64, Target: RELEASE, ToolChainTag: GCC, GnuArch: x86-64, Salt: XRG, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} + +- name: Nugets + type: object + default: + - {definition: llvm.tools, version: $(LlvmVersion), feed: $(LlvmFeed), downloadPath: $(PkgLlvm_Tools)} + - {definition: clang.headers, version: $(LlvmVersion), feed: $(LlvmFeed), downloadPath: $(PkgClang_Headers)} + +# Internal-repo auth URLs live in data/auth-urls.json so set_repo_auth.py owns both phases. + +jobs: +- template: ../jobs.yml + parameters: + pipeline: ${{parameters.pipeline}} + publishNuget: ${{parameters.publishNuget}} + Builds: ${{parameters.Builds}} + Nugets: ${{parameters.Nugets}} diff --git a/.azuredevops/pipelines/OneBranch/data/vars.yml b/.azuredevops/pipelines/OneBranch/data/vars.yml new file mode 100644 index 0000000000..2c1b7e54a0 --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/data/vars.yml @@ -0,0 +1,24 @@ +### Specify container image and other global variables in this section. ### +# https://aka.ms/obpipelines/variables + +variables: + Parameters.vPack.majorVer: "26" + Parameters.vPack.minorVer: "0" + + # Directory to download Nugets to, emulating CoreXT. + Pkg: $(System.ArtifactsDirectory)/pkg + + # Clang from VisualStudio2022 in the 1ES image. Windows hosted. + CLANGVS: C:/program files/Microsoft Visual Studio/2022/Enterprise/VC/Tools/Llvm/x64 + + # Clang built and supported by Windows org. Windows hosted. + # It is split into two Nugets. We keep them separate in the + # Nuget store and copy them together. + LlvmMajorVersion: 22 + LlvmFeed: ES-LLVM + LlvmVersion: $(LlvmMajorVersion).1.1-47 + CLANGWIN: $(Agent.TempDirectory)/llvm$(LlvmVersion) + CLANGWIN_BIN: $(CLANGWIN)/bin + CLANGWIN_INC: $(CLANGWIN)/lib/clang/$(LlvmMajorVersion) + PkgLlvm_Tools: $(Pkg)/llvm.tools.$(LlvmVersion) + PkgClang_Headers: $(Pkg)/clang.headers.$(LlvmVersion) diff --git a/.azuredevops/pipelines/OneBranch/jobs.yml b/.azuredevops/pipelines/OneBranch/jobs.yml new file mode 100644 index 0000000000..02cd2f1351 --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/jobs.yml @@ -0,0 +1,195 @@ +# ============================================================================= +# jobs.yml -- Per-variant build orchestration for Hyper-V UEFI CI. +# ============================================================================= +# +# Pure orchestration. Data lives in matrix.yml. Step bodies live in +# .azuredevops/pipelines/OneBranch/scripts/*.py (typed argparse scripts, runnable via +# run_local_ci.py without ADO). Every step here is a one-line task call. +# +# Per-job step flow: +# auth -> install_toolchain -> setup -> build +# -> stage_artifacts -> pack -> unset_repo_auth +# +# ============================================================================= + +parameters: +- name: pipeline + type: string +- name: publishNuget + type: boolean +- name: Builds + type: object +- name: Nugets + type: object + +jobs: +- ${{each Build in parameters.Builds}}: + - job: ${{Build.Target}}_${{Build.Arch}}_${{Build.ToolChainTag}}${{iif(ne(Build.Salt, ''), format('_{0}', Build.Salt), '')}} + dependsOn: SourceChecks + displayName: ${{Build.Pool}} ${{Build.Arch}} ${{Build.Target}} ${{Build.ToolChainTag}}${{iif(eq(coalesce(Build.ClangWindows, false), true), ' (WinOrg)', '')}} + pool: + type: ${{Build.Pool}} + + # ------------------------------------------------------------------------- + # Per-job variables. + # + # ob_* are read by the OneBranch base template at expansion time and must + # stay declared here (they cannot move to ##vso[task.setvariable]). + # Vpack name composition is mirrored and unit-tested in + # scripts/tests/test_get_vpack_name.py. + # ------------------------------------------------------------------------- + variables: + # --- Variant-specific env overrides (e.g. CLANG_BIN) --- + ${{each pair in Build.Env}}: + ${{pair.Key}}: ${{pair.Value}} + + # --- Toolchain / container environment --- + GCC_${{Build.Arch}}_PREFIX: /usr/bin/${{Build.GnuArch}}-linux-gnu- + GIT_TERMINAL_PROMPT: '0' + LinuxContainerImage: onebranch.azurecr.io/linux/ubuntu-2404:latest + WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest + + # --- Build paths --- + BuildDir: Build/Msvm${{Build.Arch}}/${{Build.Target}}_${{Build.ToolChainTag}} + PlatformPkg: MsvmPkg + ScriptsDir: .azuredevops/pipelines/OneBranch/scripts + + # --- Vpack name composition --- + DotSalt: ${{iif(eq(Build.Salt, ''), '', format('.{0}', Build.Salt))}} + DotTest: ${{iif(or(ne(Build.Salt, ''), eq(parameters.pipeline, 'PR')), '.Test', '')}} + + # --- OneBranch variables --- + ob_outputDirectory: $(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT + ob_nugetPublishing_enabled: ${{parameters.publishNuget}} + ob_createvpack_packagename: $(Build.Repository.Name).mscoreuefi.$(PRdot)${{Build.Arch}}.${{Build.Target}}$(DotSalt)$(DotTest) + ob_createvpack_description: Hyper V UEFI Firmware + ob_createvpack_enabled: true # Ignored in PR pipeline (upstream policy). + ob_createvpack_versionAs: parts + ob_createvpack_majorVer: $(Parameters.vPack.majorVer) + ob_createvpack_minorVer: $(Parameters.vPack.minorVer) + ob_createvpack_metadata: $(Build.SourceVersion) + ob_createvpack_failOnStdErr: true + ob_createvpack_verbose: true + ob_createvpack_vpackdirectory: $(ob_outputDirectory)/$(BuildDir)/Bin + + steps: + + # ========================================================================= + # Authentication + # ------------------------------------------------------------------------- + # NuGet feed auth, pip feed auth, and git http..extraheader for the + # internal ADO submodule(s) listed in auth-urls.json. + # ========================================================================= + + - task: NuGetAuthenticate@1 + displayName: Authenticate NuGet Feed + inputs: {nuGetServiceConnections: WindowsPartners_NugetFeedConnection} + + - task: PipAuthenticate@1 + displayName: Authenticate Pip Feed + inputs: {artifactFeeds: $(System.TeamProject)/$(System.TeamProject)_PublicPackages} + + - script: python $(ScriptsDir)/set_repo_auth.py set + displayName: Set ADO Repo Auth + env: {SYSTEM_ACCESSTOKEN: $(System.AccessToken)} + + # ========================================================================= + # Toolchain + # ------------------------------------------------------------------------- + # Windows-org Clang requires NuGet payloads (auth-typed download stays in + # YAML). install_toolchain.py then dispatches by Pool/ToolChainTag. + # ========================================================================= + + - ${{if eq(coalesce(Build.ClangWindows, false), true)}}: + - ${{each Nuget in parameters.Nugets}}: + - task: DownloadPackage@1 + displayName: Download ${{Nuget.definition}} + inputs: ${{Nuget}} + + - script: >- + python $(ScriptsDir)/install_toolchain.py + --pool ${{Build.Pool}} --tool-chain-tag ${{Build.ToolChainTag}} + --clang-windows ${{coalesce(Build.ClangWindows, false)}} + --gnu-arch "${{coalesce(Build.GnuArch, '')}}" + --clang-win-root "$(CLANGWIN)" --clang-bin "$(CLANGWIN_BIN)" --clang-inc "$(CLANGWIN_INC)" + --llvm-tools-source "$(PkgLlvm_Tools)" --clang-headers-source "$(PkgClang_Headers)" + displayName: Install Toolchain + + # ========================================================================= + # Build + # ------------------------------------------------------------------------- + # setup: pip install + Stuart --setup + Stuart --update. + # build: the single PlatformBuild.py invocation. Build logs are always + # dumped from the script's finally block (no separate log step). + # ========================================================================= + + - script: python $(ScriptsDir)/setup.py --platform-pkg $(PlatformPkg) + displayName: Setup + + - script: >- + python $(ScriptsDir)/build.py + --arch ${{Build.Arch}} --target ${{Build.Target}} --tool-chain-tag ${{Build.ToolChainTag}} + --extra "${{coalesce(Build.Extra, '')}}" --platform-pkg $(PlatformPkg) + displayName: Build + + # ========================================================================= + # Artifact staging + # ------------------------------------------------------------------------- + # stage_artifacts: flatten logs, copy fd/PDB/MAP, build vpack staging dir. + # copy_all_on_failure:dump the whole Build/ tree to ob_outputDirectory so we + # can debug a failed build offline. + # PublishSymbols: symbol-server upload (skipped for GCC -- no Codeview). + # ========================================================================= + + - script: >- + python $(ScriptsDir)/stage_artifacts.py + --arch ${{Build.Arch}} --target ${{Build.Target}} --tool-chain-tag ${{Build.ToolChainTag}} + --build-dir "$(BuildDir)" --out-dir "$(ob_outputDirectory)" --job-name "$(Agent.JobName)" + displayName: Stage Artifacts + condition: succeededOrFailed() + continueOnError: true + + - task: CopyFiles@2 + displayName: Copy Full Build Tree (on Failure) + condition: failed() + continueOnError: true + inputs: {SourceFolder: Build, Contents: '**', TargetFolder: $(ob_outputDirectory)/all} + + - ${{if ne(Build.ToolChainTag, 'GCC')}}: + - task: PublishSymbols@2 + displayName: Publish Symbols + continueOnError: true + inputs: + SearchPattern: $(BuildDir)/PDB/*.pdb + IndexSources: true + SymbolServerType: TeamServices + SymbolsProduct: Hyper-V UEFI + enablesymbolspublishing: true + + # ========================================================================= + # NuGet pack + # ------------------------------------------------------------------------- + # The OneBranch base template generates the vpack itself from + # ob_createvpack_vpackdirectory and increments the rightmost version digit. + # This step is the matching .nupkg for the NuGet feed. + # ========================================================================= + + - task: NuGetCommand@2 + displayName: Pack NuGet + inputs: + command: pack + packagesToPack: $(PlatformPkg)/HyperVUefi.NuSpec + packDestination: $(ob_outputDirectory)/packages + packTimezone: utc + buildProperties: id=$(ob_createvpack_packagename);version=$(Parameters.vPack.majorVer).$(Parameters.vPack.minorVer)-$(Build.SourceBranchName).$(Build.SourceVersion) + + # ========================================================================= + # Cleanup + # ------------------------------------------------------------------------- + # Always remove the extraheader auth so credentials are not left in + # git config on the agent. Mirrors set_repo_auth and shares its URL list. + # ========================================================================= + + - script: python $(ScriptsDir)/set_repo_auth.py unset + displayName: Unset ADO Repo Auth + condition: always() diff --git a/.azuredevops/pipelines/OneBranch/scripts/README.md b/.azuredevops/pipelines/OneBranch/scripts/README.md new file mode 100644 index 0000000000..ce2e2e147e --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/scripts/README.md @@ -0,0 +1,134 @@ +# OneBranch CI Scripts + +Python entrypoints used by `.azuredevops/pipelines/OneBranch/jobs.yml`. + +## Main scripts + +- `setup.py`: pip install + Stuart setup/update +- `install_toolchain.py`: pool/toolchain-specific tool install logic +- `build.py`: runs `PlatformBuild.py` for one variant and prints command/logs +- `stage_artifacts.py`: stages logs, firmware binaries, maps, and PDBs +- `set_repo_auth.py`: sets/unsets git extraheader auth for internal ADO repos +- `run_local_ci.py`: local developer entrypoint (setup + build + optional staging) +- `get_vpack_name.py`: reference implementation for vpack naming +- `source_checks.py`: updates CI dependencies and runs MsvmPkg source checks + +## Local usage + +Run one local build variant from repo root: + +```powershell +py .\.azuredevops\pipelines\OneBranch\scripts\run_local_ci.py --arch X64 --target DEBUG --tool-chain-tag VS2022 +``` + +Run local verification checks: + +```powershell +py .\.azuredevops\pipelines\OneBranch\scripts\tests\verify_ci.py +``` + +## Stuart Source Checks + +The initial onboarding ported mu_msvm commit `61216d796b1f873680c89b8e00141fb049c8efcd`. +Following the merged internal formatting change, enforcement adapts upstream +commit `acbd02cb02843ca6c3ebe5279665aff168876019` to OneBranch. +Shared Stuart settings also incorporate the final CI +changes in `8848e1d684f54b025ebe32707c0919e47a2c0c0e`, including Patina and +recursive submodule declarations. OneBranch temporarily uses non-recursive +source-check provisioning as described below. OpenVMM testing and the GitHub reusable-action +migration are not imported. Shared settings, check policy, BUILDING/DEVGUIDE +documentation, and `pip-requirements.txt` match that upstream revision. +OneBranch uses the same `pip-requirements.txt` as the shared documentation. +`.github/workflows/Platform-Build.yml` remains the earlier onboarding snapshot; +ADO executes OneBranch, not that historical GitHub workflow. +The mirrored workflow is not an authenticated GitHub build of this private +repository, whose MU_BASECORE submodule requires ADO access. + +Run from an existing development checkout with the stable Python requirements +installed and top-level submodules initialized: + +```powershell +python .\.azuredevops\pipelines\OneBranch\scripts\source_checks.py +``` + +Use `--setup` on a fresh checkout to install the stable Python requirements and +run `git -c core.autocrlf=false submodule update --init`. This initializes +MU_BASECORE, Common/MU, Feature/DEBUGGER, and Common/PATINA_EDK2 submodules at +their pinned revisions without recursively downloading nested submodules. +Authenticate to the private MU_BASECORE repository first. The script does not +provision compilers, Rust, or firmware binary dependencies. + +The intended setup command is `stuart_setup -c .pytool/CISettings.py`, keeping +submodule selection under the shared upstream settings. Its recursive checkout +failed in build 157900552 because the hosted agent could not connect to GitLab +to download libspdm's nested cmocka test dependency. Use top-level checkout for +these source-only checks until fresh hosted agents can resolve all nested +dependencies; the shared settings remain unchanged. + +For manual setup on a private development checkout, authenticate first, then run: + +```powershell +git config core.autocrlf false +git config core.longpaths true +python -m pip install -r pip-requirements.txt +git -c core.autocrlf=false submodule update --init +stuart_update -c .\.pytool\CISettings.py +``` + +To omit the internal bulk-formatting revision from local blame output: + +```powershell +git config blame.ignoreRevsFile .git-blame-ignore-revs +``` + +The shared OneBranch template includes one Windows `SourceChecks` job for both +PR and Official builds. All three selected checks are enforced. +It uses the existing NuGet, pip, and ADO repository authentication and always +removes the repository authentication afterward. Job-level `GIT_CONFIG_*` +variables disable automatic line-ending conversion and enable long paths for +Git processes, including checkout, without changing the agent's global Git configuration. The job uses +the container's Python, as existing OneBranch jobs do; GitHub pins Python 3.12. + +The settings select only the `cibuild` scope, `MsvmPkg`, `NO-TARGET`, and both +X64/AARCH64. All plugins are disabled except GuidCheck, LineEndingCheck, and +UncrustifyCheck. The workspace root is a package search root so internal sibling +packages participate in GUID comparisons; only MsvmPkg is selected for checks. +The upstream GUID exception list is unchanged. Source formatting and DSC +line-ending normalization landed in the preceding formatting PR. The package's +`.gitattributes` pins text files to CRLF on every platform. Package search paths +are workspace-relative and include Common/PATINA_EDK2, matching current upstream. +Including Patina in source-check setup does not enable Patina firmware builds. + +The pinned GuidCheck and LineEndingCheck plugins report failures directly. +UncrustifyCheck has `AuditOnly: false`, so formatting findings also fail the +local command and JUnit test rather than being reported as skipped. + +The Stuart invocation has no `continueOnError` override and test publication +uses `failTaskOnFailedTests: true`. Setup/update/check errors propagate as +nonzero exits and fail `SourceChecks`. Feed/repository authentication and +cleanup task failures are not suppressed. Each firmware matrix job depends on +`SourceChecks` succeeding, preventing both builds and their vpack/NuGet +publication after a failed check. This is the OneBranch counterpart to +upstream's release-job dependency; OneBranch packages inside each matrix job. + +The check job does not publish NuGet or vpack packages. It publishes +`Build/TestSuites.xml` to ADO Test Results and stages the CI/setup/update logs and +`Build/MsvmPkg` reports even when checks fail. Log staging is best-effort so a +missing `Build/` after a setup failure does not add another blocking error. + +At the pinned MU_BASECORE revision, Uncrustify is the SHA-256-pinned +`tianocore-uncrustify-release` 73.0.11 ZIP from GitHub, not a Project Mu NuGet +feed. Governed-container egress and Component Governance approval for the +CI-scoped binary dependencies still need confirmation; enabling enforcement +does not establish approval. No internal mirror is configured. + +See [Running CI Checks Locally](../../../../MsvmPkg/Docs/BUILDING.md#running-ci-checks-locally) +for individual check commands and in-place formatting. That shared guide retains +upstream's GitHub Actions wording for mirroring; this repository enforces the +same checks through OneBranch as described here. + +Local tests validate orchestration, failure propagation, and enforcement policy. +`verify_ci.py` validates YAML parsing and the Python scripts, but cannot expand +the governed OneBranch template. Queue an ADO PR build to validate checkout +environment propagation, private feed/submodule access, results publication, +and firmware jobs being skipped after a source-check failure on a hosted runner. diff --git a/.azuredevops/pipelines/OneBranch/scripts/build.py b/.azuredevops/pipelines/OneBranch/scripts/build.py new file mode 100644 index 0000000000..72f497c4d3 --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/scripts/build.py @@ -0,0 +1,102 @@ +""" +Runs the main Hyper-V UEFI build for one matrix variant. + +Invokes PlatformBuild.py with the canonical Stuart args. Always dumps +Build/ listing and Build/B*.txt log files in a finally block so a failed +job still produces actionable output in the pipeline log. +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +from ci_common import VALID_ARCHS, VALID_TARGETS, VALID_TOOLCHAINS, Arch, Target, ToolChainTag, die + +# Fixed Stuart args appended to every build invocation. +_FIXED_BUILD_ARGS: list[str] = [ + "BUILDREPORTING=TRUE", + "BUILDREPORT_TYPES=PCD DEPEX FLASH BUILD_FLAGS LIBRARY", + "LaunchLogOnSuccess=FALSE", + "LaunchLogOnError=FALSE", +] + + +def _dump_build_logs() -> None: + build_dir = Path("Build") + print("== Build/ listing ==") + if build_dir.exists(): + for entry in sorted(build_dir.iterdir()): + size = entry.stat().st_size if entry.is_file() else 0 + print(f" {entry.name:<45} {size:>10} bytes") + else: + print(" (Build directory missing)") + + print("== Build/B*.txt contents ==") + if build_dir.exists(): + for txt in sorted(build_dir.glob("B*.txt")): + print(f"---- {txt.name} ----") + print(txt.read_text(errors="replace")) + + +def run_build( + arch: Arch, + target: Target, + tool_chain_tag: ToolChainTag, + extra: str, + platform_pkg: str, +) -> None: + platform_build = Path(platform_pkg) / "PlatformBuild.py" + if not platform_build.exists(): + die(f"PlatformBuild.py not found at: {platform_build}") + + extra_tokens = extra.split() if extra.strip() else [] + cmd = [ + "stuart_build", "-c", str(platform_build), + *extra_tokens, + f"TOOL_CHAIN_TAG={tool_chain_tag}", + f"TARGET={target}", + f"BUILD_ARCH={arch}", + *_FIXED_BUILD_ARGS, + ] + print(f"== Command: {' '.join(cmd)} ==") + + try: + subprocess.run(cmd, check=True) + except subprocess.CalledProcessError as e: + sys.exit(e.returncode) + finally: + _dump_build_logs() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--arch", required=True, choices=VALID_ARCHS) + parser.add_argument("--target", required=True, choices=VALID_TARGETS) + parser.add_argument("--tool-chain-tag", required=True, choices=VALID_TOOLCHAINS) + parser.add_argument("--extra", default="", help="Extra PlatformBuild.py args (whitespace-separated)") + parser.add_argument( + "--platform-pkg", + default=os.environ.get("PlatformPkg", "MsvmPkg"), + help="Platform package path. Also read from $PlatformPkg env var.", + ) + args = parser.parse_args() + + platform_pkg = args.platform_pkg.strip() + if not platform_pkg: + die("--platform-pkg not specified and $PlatformPkg env var is not set.") + + run_build( + arch=args.arch, + target=args.target, + tool_chain_tag=args.tool_chain_tag, + extra=args.extra, + platform_pkg=platform_pkg, + ) + + +if __name__ == "__main__": + main() diff --git a/.azuredevops/pipelines/OneBranch/scripts/ci_common.py b/.azuredevops/pipelines/OneBranch/scripts/ci_common.py new file mode 100644 index 0000000000..24352851c5 --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/scripts/ci_common.py @@ -0,0 +1,41 @@ +"""Shared types and helpers for Hyper-V UEFI CI scripts.""" + +from __future__ import annotations + +import subprocess +import sys +from typing import Literal + +# --------------------------------------------------------------------------- +# Canonical value sets — used both as runtime choices in argparse and as +# type aliases for static analysis. +# --------------------------------------------------------------------------- + +Arch = Literal["X64", "AARCH64"] +Target = Literal["DEBUG", "RELEASE"] +ToolChainTag = Literal["VS2022", "CLANGPDB", "GCC"] +Pool = Literal["Windows", "Linux"] +Pipeline = Literal["Official", "PR"] + +VALID_ARCHS: tuple[str, ...] = ("X64", "AARCH64") +VALID_TARGETS: tuple[str, ...] = ("DEBUG", "RELEASE") +VALID_TOOLCHAINS: tuple[str, ...] = ("VS2022", "CLANGPDB", "GCC") +VALID_POOLS: tuple[str, ...] = ("Windows", "Linux") +VALID_PIPELINES: tuple[str, ...] = ("Official", "PR") + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def run(label: str, cmd: list[str]) -> None: + """Print *label*, execute *cmd*, and raise CalledProcessError on failure.""" + print(f"== {label} ==") + subprocess.run(cmd, check=True) + + +def die(msg: str) -> None: + """Print *msg* to stderr and exit 1.""" + print(f"ERROR: {msg}", file=sys.stderr) + sys.exit(1) diff --git a/.azuredevops/pipelines/OneBranch/scripts/get_vpack_name.py b/.azuredevops/pipelines/OneBranch/scripts/get_vpack_name.py new file mode 100644 index 0000000000..fab493c962 --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/scripts/get_vpack_name.py @@ -0,0 +1,61 @@ +""" +Computes the OneBranch vpack package name for a Hyper-V UEFI build variant. + +Naming rule: + .mscoreuefi.. + +Components: + pr_dot = 'PR.' for PR pipeline, '' for Official + dot_salt = '.' if salt non-empty, '' otherwise + dot_test = '.Test' if salt non-empty OR pipeline == PR, '' otherwise + +Production (shippable) variants have no salt and run in the Official pipeline — +they carry neither dot_salt nor dot_test. All other variants carry .Test. + +NOTE: This script does not influence the rightmost auto-increment digit added +by OneBranch; it only produces the package ID. The YAML pipeline composes the +same name inline (ob_createvpack_packagename); the pytest suite locks both +implementations so they cannot silently diverge. +""" + +from __future__ import annotations + +import argparse + +from ci_common import VALID_ARCHS, VALID_PIPELINES, VALID_TARGETS, Arch, Pipeline, Target + + +def get_vpack_name( + repo_name: str, + pipeline: Pipeline, + arch: Arch, + target: Target, + salt: str = "", +) -> str: + """Return the vpack package name for the given variant.""" + pr_dot = "PR." if pipeline == "PR" else "" + dot_salt = f".{salt}" if salt else "" + dot_test = ".Test" if (salt or pipeline == "PR") else "" + return f"{repo_name}.mscoreuefi.{pr_dot}{arch}.{target}{dot_salt}{dot_test}" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--repo-name", required=True, help="Build.Repository.Name in ADO (case-sensitive)") + parser.add_argument("--pipeline", required=True, choices=VALID_PIPELINES) + parser.add_argument("--arch", required=True, choices=VALID_ARCHS) + parser.add_argument("--target", required=True, choices=VALID_TARGETS) + parser.add_argument("--salt", default="", help="Matrix row salt (e.g. 'XDCV'). Empty for production rows.") + args = parser.parse_args() + + print(get_vpack_name( + repo_name=args.repo_name, + pipeline=args.pipeline, + arch=args.arch, + target=args.target, + salt=args.salt, + )) + + +if __name__ == "__main__": + main() diff --git a/.azuredevops/pipelines/OneBranch/scripts/install_toolchain.py b/.azuredevops/pipelines/OneBranch/scripts/install_toolchain.py new file mode 100644 index 0000000000..f1bd43c109 --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/scripts/install_toolchain.py @@ -0,0 +1,106 @@ +""" +Installs the build toolchain for a given Hyper-V UEFI CI variant. + +Dispatch table: + Linux + CLANGPDB -> apt-get install clang lld llvm + Linux + GCC -> apt-get install binutils- gcc- + Windows + VS2022 -> no-op (toolchain ships in the 1ES image) + Windows + CLANGPDB, clang_windows=false -> no-op (Clang from Visual Studio 2022) + Windows + CLANGPDB, clang_windows=true -> copy-install Windows-org Clang from + pre-downloaded NuGet payloads + +OneBranch Linux containers run as root; no sudo is needed. +""" + +from __future__ import annotations + +import argparse +import shutil +from pathlib import Path + +from ci_common import VALID_POOLS, VALID_TOOLCHAINS, Pool, ToolChainTag, die, run + + +def _install_linux(tool_chain_tag: ToolChainTag, gnu_arch: str) -> None: + run("apt-get update", ["apt-get", "update", "-y"]) + if tool_chain_tag == "CLANGPDB": + run("apt-get install clang lld llvm", ["apt-get", "install", "-y", "clang", "lld", "llvm"]) + elif tool_chain_tag == "GCC": + if not gnu_arch.strip(): + die("--gnu-arch is required for Linux + GCC.") + pkg_binutils = f"binutils-{gnu_arch}-linux-gnu" + pkg_gcc = f"gcc-{gnu_arch}-linux-gnu" + run(f"apt-get install {pkg_binutils} {pkg_gcc}", ["apt-get", "install", "-y", pkg_binutils, pkg_gcc]) + else: + die(f"Unsupported Linux toolchain: {tool_chain_tag}") + + +def _install_clang_windows( + clang_win_root: Path, + clang_bin: Path, + clang_inc: Path, + llvm_tools_source: Path, + clang_headers_source: Path, +) -> None: + required = { + "--clang-win-root": clang_win_root, + "--clang-bin": clang_bin, + "--clang-inc": clang_inc, + "--llvm-tools-source": llvm_tools_source, + "--clang-headers-source": clang_headers_source, + } + for flag, val in required.items(): + if not str(val).strip(): + die(f"{flag} is required when --clang-windows true.") + + if clang_win_root.exists(): + print(f"Removing existing CLANGWIN root: {clang_win_root}") + shutil.rmtree(clang_win_root) + + print(f"Installing LLVM tools: {llvm_tools_source} -> {clang_bin}") + clang_bin.mkdir(parents=True, exist_ok=True) + shutil.copytree(llvm_tools_source, clang_bin, dirs_exist_ok=True) + + print(f"Installing clang headers: {clang_headers_source} -> {clang_inc}") + clang_inc.mkdir(parents=True, exist_ok=True) + shutil.copytree(clang_headers_source, clang_inc, dirs_exist_ok=True) + + print("Windows-org Clang install complete.") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--pool", required=True, choices=VALID_POOLS) + parser.add_argument("--tool-chain-tag", required=True, choices=VALID_TOOLCHAINS) + parser.add_argument( + "--clang-windows", + type=str.lower, + default="false", + choices=["true", "false"], + help="'true' to install Windows-org Clang from NuGet payloads (string to match ADO template bool serialization)", + ) + parser.add_argument("--gnu-arch", default="", help="GNU triple arch for Linux GCC (e.g. 'x86-64', 'aarch64')") + parser.add_argument("--clang-win-root", type=Path, default=Path(""), help="Windows-org Clang root; removed before reinstall") + parser.add_argument("--clang-bin", type=Path, default=Path(""), help="Destination for LLVM tools") + parser.add_argument("--clang-inc", type=Path, default=Path(""), help="Destination for clang headers") + parser.add_argument("--llvm-tools-source", type=Path, default=Path(""), help="llvm.tools NuGet payload directory") + parser.add_argument("--clang-headers-source", type=Path, default=Path(""), help="clang.headers NuGet payload directory") + args = parser.parse_args() + + if args.pool == "Linux": + _install_linux(args.tool_chain_tag, args.gnu_arch) + else: + if args.clang_windows != "true": + print(f"Windows pool, {args.tool_chain_tag}: no toolchain install required (using image-provided tools).") + else: + _install_clang_windows( + clang_win_root=args.clang_win_root, + clang_bin=args.clang_bin, + clang_inc=args.clang_inc, + llvm_tools_source=args.llvm_tools_source, + clang_headers_source=args.clang_headers_source, + ) + + +if __name__ == "__main__": + main() diff --git a/.azuredevops/pipelines/OneBranch/scripts/run_local_ci.py b/.azuredevops/pipelines/OneBranch/scripts/run_local_ci.py new file mode 100644 index 0000000000..4527e5955f --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/scripts/run_local_ci.py @@ -0,0 +1,96 @@ +""" +Run one CI build variant locally, invoking the same scripts the pipeline uses. + +Mirrors jobs.yml for a single matrix row without an ADO runner. Run from +the repo root after installing toolchain and ensuring Python 3.10+ is active. + +Skipped vs. the pipeline: + set_repo_auth.py (assumes developer git credentials are configured) + NuGet download (assumes toolchain already installed) + publish_symbols (no symbol server upload locally) + pack_nuget (no .nupkg needed locally) + OneBranch vpack / artifact upload + +Usage: + python .azuredevops/pipelines/OneBranch/scripts/run_local_ci.py \\ + --arch X64 --target DEBUG --tool-chain-tag VS2022 +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +from pathlib import Path + +from ci_common import VALID_ARCHS, VALID_TARGETS, VALID_TOOLCHAINS + +_SCRIPTS_DIR = Path(__file__).parent + + +def _run_script(script: str, *args: str) -> None: + """Run a sibling script in the same Python interpreter.""" + cmd = ["python", str(_SCRIPTS_DIR / script), *args] + subprocess.run(cmd, check=True) + + +def main() -> None: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--arch", required=True, choices=VALID_ARCHS) + parser.add_argument("--target", required=True, choices=VALID_TARGETS) + parser.add_argument("--tool-chain-tag", required=True, choices=VALID_TOOLCHAINS) + parser.add_argument("--extra", default="", help="Extra PlatformBuild.py args") + parser.add_argument("--platform-pkg", default="MsvmPkg") + parser.add_argument( + "--out-dir", + type=Path, + default=Path.cwd() / "LocalCiOutput", + help="Artifact output directory (default: LocalCiOutput/)", + ) + parser.add_argument( + "--skip-stage", + action="store_true", + help="Skip artifact staging for a faster inner-loop build", + ) + args = parser.parse_args() + + os.environ["PlatformPkg"] = args.platform_pkg + + print("==== setup ====") + _run_script("setup.py", "--platform-pkg", args.platform_pkg) + + print("==== build ====") + build_args = [ + "--arch", args.arch, + "--target", args.target, + "--tool-chain-tag", args.tool_chain_tag, + "--platform-pkg", args.platform_pkg, + ] + if args.extra: + build_args += ["--extra", args.extra] + _run_script("build.py", *build_args) + + if not args.skip_stage: + build_dir = f"Build/Msvm{args.arch}/{args.target}_{args.tool_chain_tag}" + print(f"==== stage_artifacts (build_dir={build_dir}, out_dir={args.out_dir}) ====") + _run_script( + "stage_artifacts.py", + "--arch", args.arch, + "--target", args.target, + "--tool-chain-tag", args.tool_chain_tag, + "--build-dir", build_dir, + "--out-dir", str(args.out_dir), + "--job-name", "local", + ) + + extra_label = f" ({args.extra})" if args.extra else "" + print(f"\n==== Done: {args.target} {args.arch} {args.tool_chain_tag}{extra_label} ====") + if not args.skip_stage: + print(f"Output: {args.out_dir}") + + +if __name__ == "__main__": + main() diff --git a/.azuredevops/pipelines/OneBranch/scripts/set_repo_auth.py b/.azuredevops/pipelines/OneBranch/scripts/set_repo_auth.py new file mode 100644 index 0000000000..56e073979c --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/scripts/set_repo_auth.py @@ -0,0 +1,90 @@ +""" +Sets or unsets git http..extraheader auth for internal ADO repos. + +The URL list is read from data/auth-urls.json so the Set and Unset phases +share the same source and cannot drift. + +For 'set' mode, the OAuth bearer token is read from the SYSTEM_ACCESSTOKEN +environment variable so it does not appear in process arguments (where ADO +task logs may echo them verbatim). Inject it via the task's env: block: + + - script: python set_repo_auth.py set + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path + +_DEFAULT_URLS_JSON = Path(__file__).parent.parent / "data" / "auth-urls.json" + + +def _git(*args: str) -> int: + return subprocess.run(["git", *args]).returncode + + +def set_auth(urls: list[str], token: str) -> None: + for url in urls: + cfg_key = f"http.{url}.extraheader" + print(f"Set auth: {url}") + rc = _git("config", "--global", cfg_key, f"AUTHORIZATION: Bearer {token}") + if rc != 0: + print(f"ERROR: git config --global {cfg_key} failed (exit {rc})", file=sys.stderr) + sys.exit(rc) + + +def unset_auth(urls: list[str]) -> None: + for url in urls: + cfg_key = f"http.{url}.extraheader" + print(f"Unset auth: {url}") + rc = _git("config", "--global", "--unset", cfg_key) + # git config --unset exits 5 if the key is already absent; tolerate it + # so re-runs and partial-state cleanups don't fail the job. + if rc not in (0, 5): + print(f"ERROR: git config --global --unset {cfg_key} failed (exit {rc})", file=sys.stderr) + sys.exit(rc) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("mode", choices=["set", "unset"]) + parser.add_argument( + "--urls-json", + type=Path, + default=_DEFAULT_URLS_JSON, + help="Path to auth-urls.json (default: ../data/auth-urls.json relative to this script)", + ) + args = parser.parse_args() + + if not args.urls_json.exists(): + print(f"ERROR: auth-urls.json not found at: {args.urls_json}", file=sys.stderr) + sys.exit(1) + + data = json.loads(args.urls_json.read_text()) + urls: list[str] = data.get("urls", []) + if not urls: + print(f"ERROR: No 'urls' entries in {args.urls_json}", file=sys.stderr) + sys.exit(1) + + if args.mode == "set": + token = os.environ.get("SYSTEM_ACCESSTOKEN", "").strip() + if not token: + print( + "ERROR: SYSTEM_ACCESSTOKEN is not set. " + "Inject it via the task env: block (do not pass it as an argument).", + file=sys.stderr, + ) + sys.exit(1) + set_auth(urls, token) + else: + unset_auth(urls) + + +if __name__ == "__main__": + main() diff --git a/.azuredevops/pipelines/OneBranch/scripts/setup.py b/.azuredevops/pipelines/OneBranch/scripts/setup.py new file mode 100644 index 0000000000..338d3ef085 --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/scripts/setup.py @@ -0,0 +1,60 @@ +""" +Runs the pre-build setup phase: pip install, Stuart --setup, Stuart --update. + +Each invocation fails-fast so a partial environment is never silently +passed to the build step. +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +from pathlib import Path + +from ci_common import die, run + +_DEFAULT_PIP_REQUIREMENTS = "pip-requirements.txt" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + "--platform-pkg", + default=os.environ.get("PlatformPkg", ""), + help="Platform package path (e.g. 'MsvmPkg'). Also read from $PlatformPkg env var.", + ) + parser.add_argument( + "--pip-requirements", + default=_DEFAULT_PIP_REQUIREMENTS, + help=f"pip requirements file (default: {_DEFAULT_PIP_REQUIREMENTS})", + ) + args = parser.parse_args() + + platform_pkg = args.platform_pkg.strip() + if not platform_pkg: + die("--platform-pkg not specified and $PlatformPkg env var is not set.") + + pip_req = Path(args.pip_requirements) + if not pip_req.exists(): + die(f"pip requirements file not found: {pip_req}") + + platform_build = Path(platform_pkg) / "PlatformBuild.py" + if not platform_build.exists(): + die(f"PlatformBuild.py not found at: {platform_build}") + + try: + run("python --version", ["python", "--version"]) + run( + f"pip install -r {pip_req}", + ["python", "-m", "pip", "install", "--upgrade", "-r", str(pip_req), + "--force-reinstall", "--no-cache-dir"], + ) + run(f"stuart_setup -c {platform_build}", ["stuart_setup", "-c", str(platform_build)]) + run(f"stuart_update -c {platform_build}", ["stuart_update", "-c", str(platform_build)]) + except subprocess.CalledProcessError as exc: + die(f"Setup failed while running: {' '.join(exc.cmd)} (exit {exc.returncode})") + + +if __name__ == "__main__": + main() diff --git a/.azuredevops/pipelines/OneBranch/scripts/source_checks.py b/.azuredevops/pipelines/OneBranch/scripts/source_checks.py new file mode 100644 index 0000000000..eecf83c497 --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/scripts/source_checks.py @@ -0,0 +1,48 @@ +"""Run MsvmPkg Stuart source checks, optionally provisioning a fresh CI checkout.""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +from ci_common import run + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--setup", action="store_true", + help="Install Python dependencies and initialize top-level submodules before checking.", + ) + args = parser.parse_args() + os.chdir(Path(__file__).resolve().parents[4]) + + try: + if args.setup: + run("Install Python dependencies", [ + sys.executable, "-m", "pip", "install", "-r", "pip-requirements.txt", + ]) + # Prefer "stuart_setup -c .pytool/CISettings.py" so shared settings own setup. + # Its recursive checkout reached libspdm's GitLab-hosted cmocka dependency, + # which the OneBranch agent could not download (build 157900552). + # These source checks use top-level repositories; restore Stuart setup + # once fresh hosted agents can resolve all declared nested dependencies. + run("Initialize source-check submodules", [ + "git", "-c", "core.autocrlf=false", "submodule", "update", "--init", + ]) + run("Update source-check dependencies", ["stuart_update", "-c", ".pytool/CISettings.py"]) + run("Run source checks", [ + "stuart_ci_build", "-c", ".pytool/CISettings.py", + "-p", "MsvmPkg", "-t", "NO-TARGET", "-a", "X64,AARCH64", + "--disable-all", "GuidCheck=run", "LineEndingCheck=run", "UncrustifyCheck=run", + ]) + except subprocess.CalledProcessError as exc: + print(f"ERROR: Source checks failed: {' '.join(exc.cmd)} (exit {exc.returncode})", file=sys.stderr) + sys.exit(exc.returncode) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/.azuredevops/pipelines/OneBranch/scripts/stage_artifacts.py b/.azuredevops/pipelines/OneBranch/scripts/stage_artifacts.py new file mode 100644 index 0000000000..11170d553a --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/scripts/stage_artifacts.py @@ -0,0 +1,156 @@ +""" +Stages build outputs into the OneBranch ob_outputDirectory. + +Produces: + /Build Logs / -- flattened log set + /Firmware Binary File _/ -- MSVM.fd + /Firmware PDB Files _/ -- *.pdb (non-GCC only) + /Firmware Map Files _/ -- *.map + //Bin/ -- vpack staging dir: + MSVM.fd, Dsdt.aml, BiosInterface.h, PDB/ (non-GCC only) + +PDB and symbol files are skipped for GCC builds (no CodeView output). +""" + +from __future__ import annotations + +import argparse +import fnmatch +import os +import shutil +from pathlib import Path + +from ci_common import VALID_ARCHS, VALID_TARGETS, VALID_TOOLCHAINS, Arch, Target, ToolChainTag + +# Log files to collect from anywhere under Build/ (case-insensitive fnmatch). +_LOG_PATTERNS: tuple[str, ...] = ( + "BuildLog.txt", + "BUILDLOG_*.txt", + "BUILDLOG_*.md", + "BUILD_REPORT.txt", + "CI_*.txt", + "CI_*.md", + "CISETUP.txt", + "SETUPLOG.txt", + "UPDATE_LOG.txt", + "PREVALLOG.txt", + "TestSuites.xml", + "BUILD_TOOLS_REPORT.html", + "OVERRIDELOG.TXT", + "BASETOOLS_BUILD*.*", + "FD_REPORT.HTML", +) + + +def _is_log_file(name: str) -> bool: + return any(fnmatch.fnmatch(name.lower(), pat.lower()) for pat in _LOG_PATTERNS) + + +def _copy_file(src: Path, dst_dir: Path) -> None: + """Copy *src* file into *dst_dir*, warning if absent.""" + if not src.exists(): + print(f"WARNING: source missing (skipping): {src}") + return + shutil.copy2(src, dst_dir) + + +def _copy_tree(src: Path, dst_dir: Path) -> None: + """Copy directory *src* as a sub-tree of *dst_dir*, warning if absent.""" + if not src.exists(): + print(f"WARNING: source missing (skipping): {src}") + return + shutil.copytree(src, dst_dir / src.name, dirs_exist_ok=True) + + +def stage_artifacts( + arch: Arch, + target: Target, + tool_chain_tag: ToolChainTag, + build_dir: Path, + out_dir: Path, + job_name: str, +) -> None: + is_gcc = tool_chain_tag == "GCC" + arch_target = f"{target}_{arch}" + + # -- Build logs (flattened) -- + log_dest = out_dir / f"Build Logs {job_name}" + log_dest.mkdir(parents=True, exist_ok=True) + if build_dir.exists(): + # Logs are intentionally flattened by filename into one folder for ADO + # artifact browsing simplicity. If two files share a name, last copy wins. + for f in build_dir.rglob("*"): + if f.is_file() and _is_log_file(f.name): + try: + shutil.copy2(f, log_dest / f.name) + except OSError as exc: + print(f"WARNING: failed to stage log {f}: {exc}") + + # -- Firmware binary -- + fd_dest = out_dir / f"Firmware Binary File {arch_target}" + fd_dest.mkdir(parents=True, exist_ok=True) + _copy_file(build_dir / "FV" / "MSVM.fd", fd_dest) + + # -- PDB files (non-GCC only) -- + if not is_gcc: + pdb_dest = out_dir / f"Firmware PDB Files {arch_target}" + pdb_dest.mkdir(parents=True, exist_ok=True) + pdb_src = build_dir / "PDB" + if pdb_src.exists(): + shutil.copytree(pdb_src, pdb_dest, dirs_exist_ok=True) + else: + print(f"WARNING: PDB source missing (skipping): {pdb_src}") + + # -- MAP files -- + map_dest = out_dir / f"Firmware Map Files {arch_target}" + map_dest.mkdir(parents=True, exist_ok=True) + map_src = build_dir / "MAP" + if map_src.exists(): + shutil.copytree(map_src, map_dest, dirs_exist_ok=True) + else: + print(f"WARNING: MAP source missing (skipping): {map_src}") + + # -- Vpack staging: ob_createvpack_vpackdirectory = //Bin -- + vpack_dir = out_dir / build_dir / "Bin" + vpack_dir.mkdir(parents=True, exist_ok=True) + + _copy_file(build_dir / "FV" / "MSVM.fd", vpack_dir) + _copy_file(build_dir / arch / "MsvmPkg" / "AcpiTables" / "AcpiTables" / "OUTPUT" / "Dsdt.aml", vpack_dir) + _copy_file(Path("MsvmPkg") / "Include" / "BiosInterface.h", vpack_dir) + + if not is_gcc: + _copy_tree(build_dir / "PDB", vpack_dir) + + print("Artifact staging complete.") + print(f" Logs: {log_dest}") + print(f" Firmware: {fd_dest}") + print(f" Maps: {map_dest}") + print(f" Vpack: {vpack_dir}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--arch", required=True, choices=VALID_ARCHS) + parser.add_argument("--target", required=True, choices=VALID_TARGETS) + parser.add_argument("--tool-chain-tag", required=True, choices=VALID_TOOLCHAINS) + parser.add_argument("--build-dir", required=True, type=Path, help="Workspace-relative build output dir") + parser.add_argument("--out-dir", required=True, type=Path, help="Absolute path to ob_outputDirectory") + parser.add_argument( + "--job-name", + default=os.environ.get("AGENT_JOBNAME", "local"), + help="ADO job name for artifact folder. Falls back to 'local'.", + ) + args = parser.parse_args() + + stage_artifacts( + arch=args.arch, + target=args.target, + tool_chain_tag=args.tool_chain_tag, + build_dir=args.build_dir, + out_dir=args.out_dir, + job_name=args.job_name or "local", + ) + + +if __name__ == "__main__": + main() diff --git a/.azuredevops/pipelines/OneBranch/scripts/tests/test_get_vpack_name.py b/.azuredevops/pipelines/OneBranch/scripts/tests/test_get_vpack_name.py new file mode 100644 index 0000000000..6323128f5f --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/scripts/tests/test_get_vpack_name.py @@ -0,0 +1,111 @@ +""" +Pytest tests for get_vpack_name.py. + +These tests lock the case-sensitive vpack naming rules so they cannot +silently drift from the YAML's inline ob_createvpack_packagename composition. + +Run locally: + pytest .azuredevops/pipelines/OneBranch/scripts/tests/test_get_vpack_name.py +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from get_vpack_name import get_vpack_name # noqa: E402 + + +class TestProductionVS2022: + """Production rows: Official pipeline, VS2022, no salt — no .Test suffix.""" + + def test_official_x64_debug_no_test_suffix(self) -> None: + assert get_vpack_name("MsvmPkgX64", "Official", "X64", "DEBUG") == "MsvmPkgX64.mscoreuefi.X64.DEBUG" + + def test_official_x64_release_no_test_suffix(self) -> None: + assert get_vpack_name("MsvmPkgX64", "Official", "X64", "RELEASE") == "MsvmPkgX64.mscoreuefi.X64.RELEASE" + + +class TestPRPipeline: + """PR pipeline always produces .Test names (with and without salt).""" + + def test_pr_no_salt_has_pr_prefix_and_test(self) -> None: + assert ( + get_vpack_name("MsvmPkgX64", "PR", "X64", "RELEASE") + == "MsvmPkgX64.mscoreuefi.PR.X64.RELEASE.Test" + ) + + def test_pr_with_salt_exactly_one_test(self) -> None: + assert ( + get_vpack_name("MsvmPkgX64", "PR", "X64", "DEBUG", "XDCV") + == "MsvmPkgX64.mscoreuefi.PR.X64.DEBUG.XDCV.Test" + ) + + +class TestOfficialSaltedVariants: + """Official pipeline + salt -> .Test (non-shippable side-by-side variants).""" + + def test_official_salt_xdcv(self) -> None: + assert ( + get_vpack_name("MsvmPkgX64", "Official", "X64", "DEBUG", "XDCV") + == "MsvmPkgX64.mscoreuefi.X64.DEBUG.XDCV.Test" + ) + + def test_official_salt_arcv_aarch64(self) -> None: + assert ( + get_vpack_name("MsvmPkgX64", "Official", "AARCH64", "RELEASE", "ARCV") + == "MsvmPkgX64.mscoreuefi.AARCH64.RELEASE.ARCV.Test" + ) + + def test_official_salt_xrg_linux_gcc(self) -> None: + assert ( + get_vpack_name("MsvmPkgX64", "Official", "X64", "RELEASE", "XRG") + == "MsvmPkgX64.mscoreuefi.X64.RELEASE.XRG.Test" + ) + + def test_official_salt_adcw_aarch64_debug(self) -> None: + """AARCH64 ClangWindows DEBUG gets salt ADCW.""" + assert ( + get_vpack_name("MsvmPkgX64", "Official", "AARCH64", "DEBUG", "ADCW") + == "MsvmPkgX64.mscoreuefi.AARCH64.DEBUG.ADCW.Test" + ) + + def test_official_salt_arcw_aarch64_release(self) -> None: + """AARCH64 ClangWindows RELEASE gets salt ARCW.""" + assert ( + get_vpack_name("MsvmPkgX64", "Official", "AARCH64", "RELEASE", "ARCW") + == "MsvmPkgX64.mscoreuefi.AARCH64.RELEASE.ARCW.Test" + ) + + +class TestCaseSensitivity: + """Arch, target, salt, and repo name are passed through as-is.""" + + def test_preserves_aarch64_casing(self) -> None: + assert get_vpack_name("MsvmPkgX64", "Official", "AARCH64", "RELEASE") == "MsvmPkgX64.mscoreuefi.AARCH64.RELEASE" + + def test_preserves_repo_name_casing(self) -> None: + assert get_vpack_name("WeirdCaseRepo", "Official", "X64", "DEBUG") == "WeirdCaseRepo.mscoreuefi.X64.DEBUG" + + +class TestInvariants: + @pytest.mark.parametrize( + "pipeline,salt", + [ + ("Official", ""), + ("Official", "XDCV"), + ("PR", ""), + ("PR", "XDCV"), + ], + ) + def test_test_suffix_appears_at_most_once(self, pipeline: str, salt: str) -> None: + name = get_vpack_name("R", pipeline, "X64", "DEBUG", salt) # type: ignore[arg-type] + assert name.count(".Test") <= 1, f"Multiple .Test in: {name!r}" + + def test_pr_prefix_iff_pr_pipeline(self) -> None: + assert ".mscoreuefi.PR." in get_vpack_name("R", "PR", "X64", "DEBUG") + assert ".mscoreuefi.PR." not in get_vpack_name("R", "Official", "X64", "DEBUG") diff --git a/.azuredevops/pipelines/OneBranch/scripts/tests/test_source_checks.py b/.azuredevops/pipelines/OneBranch/scripts/tests/test_source_checks.py new file mode 100644 index 0000000000..39f77d2de5 --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/scripts/tests/test_source_checks.py @@ -0,0 +1,120 @@ +"""Verify source-check orchestration and the enforced pipeline contract.""" + +from __future__ import annotations + +import importlib.util +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +import source_checks + + +_ROOT = Path(__file__).resolve().parents[5] +_ONEBRANCH = _ROOT / ".azuredevops/pipelines/OneBranch" +_CHECK_COMMAND = [ + "stuart_ci_build", "-c", ".pytool/CISettings.py", + "-p", "MsvmPkg", "-t", "NO-TARGET", "-a", "X64,AARCH64", + "--disable-all", "GuidCheck=run", "LineEndingCheck=run", "UncrustifyCheck=run", +] + + +@pytest.mark.parametrize("setup", [False, True]) +def test_commands(monkeypatch: pytest.MonkeyPatch, setup: bool) -> None: + commands: list[list[str]] = [] + directories: list[Path] = [] + monkeypatch.setattr(sys, "argv", ["source_checks.py"] + (["--setup"] if setup else [])) + monkeypatch.setattr(source_checks, "run", lambda label, command: commands.append(command)) + monkeypatch.setattr(source_checks.os, "chdir", lambda path: directories.append(path)) + source_checks.main() + expected = [] + if setup: + expected.extend([ + [sys.executable, "-m", "pip", "install", "-r", "pip-requirements.txt"], + ["git", "-c", "core.autocrlf=false", "submodule", "update", "--init"], + ]) + expected.extend([["stuart_update", "-c", ".pytool/CISettings.py"], _CHECK_COMMAND]) + assert commands == expected + assert directories == [_ROOT] + + +@pytest.mark.parametrize("failed_step", range(4)) +def test_failure_stops_and_propagates(monkeypatch: pytest.MonkeyPatch, failed_step: int) -> None: + commands: list[list[str]] = [] + monkeypatch.setattr(sys, "argv", ["source_checks.py", "--setup"]) + monkeypatch.setattr(source_checks.os, "chdir", lambda path: None) + + def fail_selected_step(label: str, command: list[str]) -> None: + commands.append(command) + if len(commands) == failed_step + 1: + raise subprocess.CalledProcessError(7, command) + + monkeypatch.setattr(source_checks, "run", fail_selected_step) + with pytest.raises(SystemExit) as result: + source_checks.main() + assert result.value.code == 7 + assert len(commands) == failed_step + 1 + + +def test_pipeline_contract() -> None: + common = yaml.safe_load((_ONEBRANCH / "common.yaml").read_text()) + templates = common["extends"]["parameters"]["stages"][0]["jobs"] + assert templates[0] == { + "template": ".azuredevops/pipelines/OneBranch/source-checks.yml@self", + } + builds = yaml.safe_load((_ONEBRANCH / "jobs.yml").read_text()) + build_job = next(iter(builds["jobs"][0].values()))[0] + assert build_job["dependsOn"] == "SourceChecks" + assert build_job.get("condition", "succeeded()") == "succeeded()" + job = yaml.safe_load((_ONEBRANCH / "source-checks.yml").read_text())["jobs"][0] + assert job["job"] == "SourceChecks" + assert not job.get("continueOnError", False) + assert job["variables"]["GIT_CONFIG_COUNT"] == "2" + assert job["variables"]["GIT_CONFIG_KEY_0"] == "core.autocrlf" + assert job["variables"]["GIT_CONFIG_VALUE_0"] == "false" + assert job["variables"]["GIT_CONFIG_KEY_1"] == "core.longpaths" + assert job["variables"]["GIT_CONFIG_VALUE_1"] == "true" + assert job["variables"]["ob_createvpack_enabled"] is False + assert job["variables"]["ob_nugetPublishing_enabled"] is False + steps = job["steps"] + check = next(step for step in steps if step.get("script", "").endswith("source_checks.py --setup")) + assert not check.get("continueOnError", False) + results = next(step for step in steps if step.get("task") == "PublishTestResults@2") + assert results["condition"] == "succeededOrFailed()" + assert results["inputs"]["testResultsFiles"] == "Build/TestSuites.xml" + assert results["inputs"]["failTaskOnFailedTests"] is True + assert not results.get("continueOnError", False) + staging = next(step for step in steps if step.get("task") == "CopyFiles@2") + assert staging["condition"] == "succeededOrFailed()" + assert staging["continueOnError"] is True + assert "SETUPLOG.*" in staging["inputs"]["Contents"].splitlines() + assert steps[-1]["script"].endswith("set_repo_auth.py unset") + assert steps[-1]["condition"] == "always()" + + +def test_no_checks_are_audit_only() -> None: + policy = yaml.safe_load((_ROOT / "MsvmPkg/MsvmPkg.ci.yaml").read_text()) + assert policy["UncrustifyCheck"]["AuditOnly"] is False + assert not policy["GuidCheck"].get("AuditOnly", False) + assert not policy["LineEndingCheck"].get("AuditOnly", False) + + +def test_settings_use_upstream_workspace_relative_package_paths(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(_ONEBRANCH) + spec = importlib.util.spec_from_file_location("msvm_ci_settings", _ROOT / ".pytool/CISettings.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + settings = module.Settings() + assert Path(settings.GetWorkspaceRoot()) == _ROOT + submodules = ("MU_BASECORE", "Common/MU", "Feature/DEBUGGER", "Common/PATINA_EDK2") + assert settings.GetPackagesPath() == (".", *submodules) + assert isinstance(settings, module.SetupSettingsManager) + required = settings.GetRequiredSubmodules() + assert tuple(submodule.path for submodule in required) == submodules + assert all(submodule.recursive for submodule in required) \ No newline at end of file diff --git a/.azuredevops/pipelines/OneBranch/scripts/tests/verify_ci.py b/.azuredevops/pipelines/OneBranch/scripts/tests/verify_ci.py new file mode 100644 index 0000000000..7ac4f922e3 --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/scripts/tests/verify_ci.py @@ -0,0 +1,332 @@ +""" +Local verification sweep for the Hyper-V UEFI CI pipeline scripts. + +Runs every check that can be done without an ADO runner: + 1. YAML parse -- every *.yml / *.yaml under OneBranch/ + 2. Python syntax -- every *.py under scripts/ + 3. mypy -- type-check scripts/ (skipped if mypy not installed) + 4. pytest -- unit tests in tests/ + 5. CLI validation -- each main script rejects missing required args (exit 1 or 2) + 6. Vpack-name parity -- jobs.yml/CI-Build.yaml/PR-Build.yaml vpack composition + definitions + data/matrix.yml agree with get_vpack_name() + 7. F401 lint -- optional dead-import check via ruff (skipped if unavailable) + +Each section prints PASS / FAIL / SKIP. Exits 0 only if all sections pass. + +NOTE: This script does not validate the ADO pipeline itself. To verify the +full expanded pipeline, queue a PR build and inspect the YAML in the ADO UI. + +Usage: + python .azuredevops/pipelines/OneBranch/scripts/tests/verify_ci.py +""" + +from __future__ import annotations + +import py_compile +import subprocess +import sys +from pathlib import Path + +# --------------------------------------------------------------------------- +# Directory layout (relative to this file's position) +# --------------------------------------------------------------------------- + +_TESTS_DIR = Path(__file__).parent # .../scripts/tests/ +_SCRIPTS_DIR = _TESTS_DIR.parent # .../scripts/ +_ONEBRANCH_DIR = _SCRIPTS_DIR.parent # .../OneBranch/ +_REPO_ROOT = _ONEBRANCH_DIR.parent.parent.parent # workspace root + +sys.path.insert(0, str(_SCRIPTS_DIR)) + +# --------------------------------------------------------------------------- +# Section helpers +# --------------------------------------------------------------------------- + +_results: dict[str, bool] = {} + + +def _header(title: str) -> None: + print(f"\n{'=' * 60}") + print(f" {title}") + print("=" * 60) + + +# --------------------------------------------------------------------------- +# YAML parse +# --------------------------------------------------------------------------- + + +def _section_yaml_parse() -> bool: + _header("YAML parse") + try: + import yaml # noqa: PLC0415 + except ImportError: + print("SKIP: pyyaml not installed (run: pip install pyyaml).") + return True # non-blocking + + fail = 0 + for f in sorted(_ONEBRANCH_DIR.rglob("*.yml")) + sorted(_ONEBRANCH_DIR.rglob("*.yaml")): + try: + yaml.safe_load(f.read_text()) + print(f" PASS: {f.relative_to(_REPO_ROOT)}") + except Exception as exc: + print(f" FAIL: {f.relative_to(_REPO_ROOT)} -- {exc}") + fail += 1 + return fail == 0 + + +# --------------------------------------------------------------------------- +# Python syntax +# --------------------------------------------------------------------------- + + +def _section_python_syntax() -> bool: + _header("Python syntax (py_compile)") + fail = 0 + for f in sorted(_SCRIPTS_DIR.rglob("*.py")): + rel = f.relative_to(_REPO_ROOT) + try: + py_compile.compile(str(f), doraise=True) + print(f" PASS: {rel}") + except py_compile.PyCompileError as exc: + print(f" FAIL: {rel} -- {exc}") + fail += 1 + return fail == 0 + + +# --------------------------------------------------------------------------- +# mypy type check +# --------------------------------------------------------------------------- + + +def _section_mypy() -> bool: + _header("mypy type check") + try: + result = subprocess.run( + ["mypy", "--ignore-missing-imports", str(_SCRIPTS_DIR)], + capture_output=True, + text=True, + ) + except FileNotFoundError: + print(" SKIP: mypy not installed (run: pip install mypy)") + return True # non-blocking + if result.returncode == 0: + print(" PASS: 0 type errors") + return True + if "No module named mypy" in (result.stderr + result.stdout): + print(" SKIP: mypy not installed (run: pip install mypy)") + return True # non-blocking + print(" FAIL: mypy reported errors:") + for line in result.stdout.splitlines(): + print(f" {line}") + return False + + +# --------------------------------------------------------------------------- +# pytest unit tests +# --------------------------------------------------------------------------- + + +def _section_pytest() -> bool: + _header("pytest unit tests") + result = subprocess.run( + [sys.executable, "-m", "pytest", str(_TESTS_DIR), "-v", "--tb=short"], + cwd=_REPO_ROOT, + ) + return result.returncode == 0 + + +# --------------------------------------------------------------------------- +# CLI validation +# --------------------------------------------------------------------------- + + +def _section_cli_validation() -> bool: + _header("CLI validation (scripts must reject missing required args)") + cases = [ + "set_repo_auth.py", + "install_toolchain.py", + "build.py", + "stage_artifacts.py", + "get_vpack_name.py", + ] + fail = 0 + for script_name in cases: + result = subprocess.run( + [sys.executable, str(_SCRIPTS_DIR / script_name)], + capture_output=True, + ) + # argparse exits 2 for missing required args; sys.exit(1) exits 1. + if result.returncode in (1, 2): + print(f" PASS: {script_name} rejects missing required args (exit {result.returncode})") + else: + print(f" FAIL: {script_name} accepted empty args (exit {result.returncode})") + fail += 1 + return fail == 0 + + +# --------------------------------------------------------------------------- +# Vpack-name parity +# --------------------------------------------------------------------------- + + +def _section_vpack_parity() -> bool: + _header("Vpack-name parity (matrix.yml inline composition vs. get_vpack_name.py)") + try: + import yaml # noqa: PLC0415 + except ImportError: + print(" SKIP: pyyaml not installed.") + return True + + from get_vpack_name import get_vpack_name # noqa: PLC0415 + + jobs_text = (_ONEBRANCH_DIR / "jobs.yml").read_text() + ci_yaml = yaml.safe_load((_ONEBRANCH_DIR / "CI-Build.yaml").read_text()) + pr_yaml = yaml.safe_load((_ONEBRANCH_DIR / "PR-Build.yaml").read_text()) + matrix_path = _ONEBRANCH_DIR / "data" / "matrix.yml" + matrix = yaml.safe_load(matrix_path.read_text()) + builds_param = next(p for p in matrix["parameters"] if p["name"] == "Builds") + builds: list[dict] = builds_param["default"] + + repo_name = "MsvmPkgX64" + fail = 0 + total = 0 + + # Assert the YAML sources still use the expected composition primitives, + # so this check fails if the pipeline expression drifts from the Python impl. + expected_dot_salt = "${{iif(eq(Build.Salt, ''), '', format('.{0}', Build.Salt))}}" + expected_dot_test = "${{iif(or(ne(Build.Salt, ''), eq(parameters.pipeline, 'PR')), '.Test', '')}}" + expected_packagename = "$(Build.Repository.Name).mscoreuefi.$(PRdot)${{Build.Arch}}.${{Build.Target}}$(DotSalt)$(DotTest)" + for label, needle in ( + ("DotSalt expression", expected_dot_salt), + ("DotTest expression", expected_dot_test), + ("ob_createvpack_packagename", expected_packagename), + ): + if needle not in jobs_text: + print(f" FAIL: jobs.yml missing expected {label}: {needle}") + return False + + for row in builds: + extra = str(row.get("Extra", "")) + if "BLD_*_USE_LEGACY_C_CORE=TRUE" not in extra or "BLD_*_USE_LEGACY_C_CORE=FALSE" in extra: + print(f" FAIL: matrix row has incorrect core selection: {row}") + return False + + def _get_prdot(doc: dict) -> str: + for entry in doc.get("variables", []): + if isinstance(entry, dict) and entry.get("name") == "PRdot": + return str(entry.get("value", "")) + raise ValueError("PRdot variable not found") + + try: + prdot_by_pipeline = { + "Official": _get_prdot(ci_yaml), + "PR": _get_prdot(pr_yaml), + } + except ValueError as exc: + print(f" FAIL: {exc}") + return False + + if prdot_by_pipeline["Official"] != "": + print(f" FAIL: CI-Build.yaml PRdot expected '' but found {prdot_by_pipeline['Official']!r}") + return False + if prdot_by_pipeline["PR"] != "PR.": + print(f" FAIL: PR-Build.yaml PRdot expected 'PR.' but found {prdot_by_pipeline['PR']!r}") + return False + + for row in builds: + for pipeline in ("Official", "PR"): + total += 1 + salt = str(row.get("Salt", "")) + arch = row["Arch"] + target = row["Target"] + + # Recompose using values sourced from pipeline YAML definitions. + pr_dot = prdot_by_pipeline[pipeline] + dot_salt = f".{salt}" if salt else "" + dot_test = ".Test" if (salt or pipeline == "PR") else "" + yaml_name = f"{repo_name}.mscoreuefi.{pr_dot}{arch}.{target}{dot_salt}{dot_test}" + + spec_name = get_vpack_name(repo_name, pipeline, arch, target, salt) # type: ignore[arg-type] + + if yaml_name == spec_name: + print(f" PASS: {yaml_name}") + else: + print(f" FAIL: yaml_name={yaml_name!r} spec_name={spec_name!r}") + fail += 1 + + print(f"\n {total - fail}/{total} parity checks passed.") + return fail == 0 + + +# --------------------------------------------------------------------------- +# F401 lint (optional) +# --------------------------------------------------------------------------- + + +def _section_f401_lint() -> bool: + _header("F401 lint (ruff optional)") + try: + result = subprocess.run( + [sys.executable, "-m", "ruff", "check", "--select", "F401", str(_SCRIPTS_DIR)], + capture_output=True, + text=True, + ) + except FileNotFoundError: + print(" SKIP: python executable not found") + return True + + if result.returncode == 0: + print(" PASS: no unused imports") + return True + + stderr_stdout = (result.stderr + result.stdout).lower() + if "no module named ruff" in stderr_stdout: + print(" SKIP: ruff not installed (run: pip install ruff)") + return True + + print(" FAIL: ruff found issues:") + for line in result.stdout.splitlines(): + print(f" {line}") + return False + + +# --------------------------------------------------------------------------- +# main +# --------------------------------------------------------------------------- + + +def main() -> None: + sections = [ + ("YAML parse", _section_yaml_parse), + ("Python syntax", _section_python_syntax), + ("mypy", _section_mypy), + ("pytest", _section_pytest), + ("CLI validation", _section_cli_validation), + ("Vpack-name parity", _section_vpack_parity), + ("F401 lint", _section_f401_lint), + ] + + for title, fn in sections: + _results[title] = fn() + + print(f"\n{'=' * 60}") + print(" Overall") + print("=" * 60) + all_pass = True + for name, passed in _results.items(): + status = "PASS" if passed else "FAIL" + print(f" {status}: {name}") + if not passed: + all_pass = False + + if all_pass: + print("\nAll local verification sections PASSED.") + sys.exit(0) + else: + print("\nOne or more sections FAILED.") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/.azuredevops/pipelines/OneBranch/source-checks.yml b/.azuredevops/pipelines/OneBranch/source-checks.yml new file mode 100644 index 0000000000..8cb2412f18 --- /dev/null +++ b/.azuredevops/pipelines/OneBranch/source-checks.yml @@ -0,0 +1,59 @@ +jobs: +- job: SourceChecks + displayName: Stuart Source Checks + pool: + type: windows + variables: + WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest + ScriptsDir: .azuredevops/pipelines/OneBranch/scripts + GIT_TERMINAL_PROMPT: '0' + GIT_CONFIG_COUNT: '2' + GIT_CONFIG_KEY_0: core.autocrlf + GIT_CONFIG_VALUE_0: 'false' + GIT_CONFIG_KEY_1: core.longpaths + GIT_CONFIG_VALUE_1: 'true' + ob_outputDirectory: $(Build.ArtifactStagingDirectory)/SOURCE_CHECKS + ob_nugetPublishing_enabled: false + ob_createvpack_enabled: false + steps: + - task: NuGetAuthenticate@1 + displayName: Authenticate NuGet Feed + inputs: {nuGetServiceConnections: WindowsPartners_NugetFeedConnection} + + - task: PipAuthenticate@1 + displayName: Authenticate Pip Feed + inputs: {artifactFeeds: $(System.TeamProject)/$(System.TeamProject)_PublicPackages} + + - script: python $(ScriptsDir)/set_repo_auth.py set + displayName: Set ADO Repo Auth + env: {SYSTEM_ACCESSTOKEN: $(System.AccessToken)} + + - script: python $(ScriptsDir)/source_checks.py --setup + displayName: Run Stuart Source Checks + + - task: PublishTestResults@2 + displayName: Publish Source Check Results + condition: succeededOrFailed() + inputs: + testResultsFormat: JUnit + testResultsFiles: Build/TestSuites.xml + testRunTitle: Stuart Source Checks + failTaskOnFailedTests: true + + - task: CopyFiles@2 + displayName: Stage Source Check Logs + condition: succeededOrFailed() + continueOnError: true + inputs: + SourceFolder: Build + Contents: | + CI_BUILDLOG.* + SETUPLOG.* + UPDATE_LOG.* + TestSuites.xml + MsvmPkg/** + TargetFolder: $(ob_outputDirectory) + + - script: python $(ScriptsDir)/set_repo_auth.py unset + displayName: Unset ADO Repo Auth + condition: always() diff --git a/.azuredevops/pipelines/Sync/README.md b/.azuredevops/pipelines/Sync/README.md new file mode 100644 index 0000000000..5bcca460e0 --- /dev/null +++ b/.azuredevops/pipelines/Sync/README.md @@ -0,0 +1 @@ +# TODO: Introduce Sync-MuMsvm-HyperVUEFI pipeline \ No newline at end of file From 4434958c701470276fcd373256c6c516e9006065 Mon Sep 17 00:00:00 2001 From: Maheer Aeron Date: Thu, 17 Sep 2026 15:56:20 -0700 Subject: [PATCH 02/18] ci: establish clean Stuart CI baseline --- .azuredevops/pipelines/.gitkeep | 0 .../pipelines/OneBranch/CI-Build.yaml | 23 -- .../pipelines/OneBranch/PR-Build.yaml | 29 -- .azuredevops/pipelines/OneBranch/common.yaml | 43 --- .../pipelines/OneBranch/data/auth-urls.json | 6 - .../pipelines/OneBranch/data/matrix.yml | 65 ---- .../pipelines/OneBranch/data/vars.yml | 24 -- .azuredevops/pipelines/OneBranch/jobs.yml | 195 ---------- .../pipelines/OneBranch/scripts/README.md | 134 ------- .../pipelines/OneBranch/scripts/build.py | 102 ------ .../pipelines/OneBranch/scripts/ci_common.py | 41 --- .../OneBranch/scripts/get_vpack_name.py | 61 ---- .../OneBranch/scripts/install_toolchain.py | 106 ------ .../OneBranch/scripts/run_local_ci.py | 96 ----- .../OneBranch/scripts/set_repo_auth.py | 90 ----- .../pipelines/OneBranch/scripts/setup.py | 60 ---- .../OneBranch/scripts/source_checks.py | 48 --- .../OneBranch/scripts/stage_artifacts.py | 156 -------- .../scripts/tests/test_get_vpack_name.py | 111 ------ .../scripts/tests/test_source_checks.py | 120 ------- .../OneBranch/scripts/tests/verify_ci.py | 332 ------------------ .../pipelines/OneBranch/source-checks.yml | 59 ---- .azuredevops/pipelines/Sync/README.md | 1 - .github/actions/.gitkeep | 0 .github/actions/build-platform/action.yml | 54 --- .github/actions/openvmm-test/action.yml | 256 -------------- .github/actions/release/action.yml | 43 --- .github/dependabot.yml | 11 - .github/workflows/.gitkeep | 0 .github/workflows/platform-ci.yml | 173 --------- 30 files changed, 2439 deletions(-) create mode 100644 .azuredevops/pipelines/.gitkeep delete mode 100644 .azuredevops/pipelines/OneBranch/CI-Build.yaml delete mode 100644 .azuredevops/pipelines/OneBranch/PR-Build.yaml delete mode 100644 .azuredevops/pipelines/OneBranch/common.yaml delete mode 100644 .azuredevops/pipelines/OneBranch/data/auth-urls.json delete mode 100644 .azuredevops/pipelines/OneBranch/data/matrix.yml delete mode 100644 .azuredevops/pipelines/OneBranch/data/vars.yml delete mode 100644 .azuredevops/pipelines/OneBranch/jobs.yml delete mode 100644 .azuredevops/pipelines/OneBranch/scripts/README.md delete mode 100644 .azuredevops/pipelines/OneBranch/scripts/build.py delete mode 100644 .azuredevops/pipelines/OneBranch/scripts/ci_common.py delete mode 100644 .azuredevops/pipelines/OneBranch/scripts/get_vpack_name.py delete mode 100644 .azuredevops/pipelines/OneBranch/scripts/install_toolchain.py delete mode 100644 .azuredevops/pipelines/OneBranch/scripts/run_local_ci.py delete mode 100644 .azuredevops/pipelines/OneBranch/scripts/set_repo_auth.py delete mode 100644 .azuredevops/pipelines/OneBranch/scripts/setup.py delete mode 100644 .azuredevops/pipelines/OneBranch/scripts/source_checks.py delete mode 100644 .azuredevops/pipelines/OneBranch/scripts/stage_artifacts.py delete mode 100644 .azuredevops/pipelines/OneBranch/scripts/tests/test_get_vpack_name.py delete mode 100644 .azuredevops/pipelines/OneBranch/scripts/tests/test_source_checks.py delete mode 100644 .azuredevops/pipelines/OneBranch/scripts/tests/verify_ci.py delete mode 100644 .azuredevops/pipelines/OneBranch/source-checks.yml delete mode 100644 .azuredevops/pipelines/Sync/README.md create mode 100644 .github/actions/.gitkeep delete mode 100644 .github/actions/build-platform/action.yml delete mode 100644 .github/actions/openvmm-test/action.yml delete mode 100644 .github/actions/release/action.yml delete mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/.gitkeep delete mode 100644 .github/workflows/platform-ci.yml diff --git a/.azuredevops/pipelines/.gitkeep b/.azuredevops/pipelines/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.azuredevops/pipelines/OneBranch/CI-Build.yaml b/.azuredevops/pipelines/OneBranch/CI-Build.yaml deleted file mode 100644 index 525d564fe8..0000000000 --- a/.azuredevops/pipelines/OneBranch/CI-Build.yaml +++ /dev/null @@ -1,23 +0,0 @@ - -name: $(SourceBranchName)_$(Date:yyyyMMdd_HHmmss)$(Rev:.r) - -# https://aka.ms/obpipelines/triggers -trigger: - batch: False - branches: - include: - - official/* -pr: none - -variables: -- template: data/vars.yml -# Because we have a template, we must use name/value to add more. -- name: PRdot - value: '' - -extends: - template: common.yaml - parameters: - base: v2/Microsoft.Official.yml # EXTEND MICROSOFT.OFFICIAL.YML - pipeline: Official - publishNuget: True diff --git a/.azuredevops/pipelines/OneBranch/PR-Build.yaml b/.azuredevops/pipelines/OneBranch/PR-Build.yaml deleted file mode 100644 index f008efd058..0000000000 --- a/.azuredevops/pipelines/OneBranch/PR-Build.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# https://aka.ms/obpipelines/triggers -trigger: none - -parameters: - -# This controls only Nuget create, not Vpack creation. -# Due to upstream code/policy, vpacks cannot be created in the PR pipeline. -# The official "CI" pipeline, which creates vpacks, can be run against PRs with approval. -- name: PublishPRNuget - displayName: Publish Nuget Pkg for this PR for testing? - type: boolean - default: False - -variables: -- template: data/vars.yml -# Because we have a template, we must use name/value to add more. -- name: PRdot - value: 'PR.' - -extends: - template: common.yaml - parameters: - base: v2/Microsoft.NonOfficial.yml # https://aka.ms/obpipelines/templates - pipeline: PR - - # This controls only Nuget create, not Vpack creation. - # Due to upstream code/policy, vpacks cannot be created in the PR pipeline. - # The official "CI" pipeline, which creates vpacks, can be run against PRs with approval. - publishNuget: ${{parameters.PublishPRNuget}} diff --git a/.azuredevops/pipelines/OneBranch/common.yaml b/.azuredevops/pipelines/OneBranch/common.yaml deleted file mode 100644 index 6ecd93d90f..0000000000 --- a/.azuredevops/pipelines/OneBranch/common.yaml +++ /dev/null @@ -1,43 +0,0 @@ -parameters: -- name: base -- name: pipeline - type: string - values: - - Official - - PR -- name: publishNuget - type: boolean - default: False - -resources: - repositories: - - repository: templates_onebranch - type: git - name: OneBranch.Pipelines/GovernedTemplates - ref: refs/heads/main - -extends: - template: ${{parameters.base}}@templates_onebranch # https://aka.ms/obpipelines/templates - parameters: - platform: - name: windows_undocked - globalSdl: - tsa: - enabled: False - featureFlags: - WindowsHostVersion: 1ESWindows2022 - nugetPublishing: - feeds: - - name: OS/HyperVUefi - continueOnConflict: False - files_to_publish: '*.nupkg;!*.symbols.nupkg' - git: - submodules: False - stages: - - stage: Build - jobs: - - template: .azuredevops/pipelines/OneBranch/source-checks.yml@self - - template: .azuredevops/pipelines/OneBranch/data/matrix.yml@self - parameters: - pipeline: ${{parameters.pipeline}} - publishNuget: ${{parameters.publishNuget}} diff --git a/.azuredevops/pipelines/OneBranch/data/auth-urls.json b/.azuredevops/pipelines/OneBranch/data/auth-urls.json deleted file mode 100644 index 2da9ce73ef..0000000000 --- a/.azuredevops/pipelines/OneBranch/data/auth-urls.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "_comment": "Internal ADO repos needing System.AccessToken auth. Shared by set_repo_auth.py Set/Unset so the phases cannot drift.", - "urls": [ - "https://dev.azure.com/microsoft/OS/_git/hyperv.uefi.mu_basecore" - ] -} diff --git a/.azuredevops/pipelines/OneBranch/data/matrix.yml b/.azuredevops/pipelines/OneBranch/data/matrix.yml deleted file mode 100644 index 02c6643639..0000000000 --- a/.azuredevops/pipelines/OneBranch/data/matrix.yml +++ /dev/null @@ -1,65 +0,0 @@ -# Matrix data + entry point for the Hyper-V UEFI CI build. -# -# Each row in Builds is one variant. Fields: -# Pool 'Windows' | 'Linux' -# Arch 'X64' | 'AARCH64' -# Target 'DEBUG' | 'RELEASE' -# ToolChainTag 'VS2022' | 'CLANGPDB' | 'GCC' -# ClangWindows true => Windows-org-built Clang (CLANGPDB + Pool=Windows only) -# GnuArch GNU triple for Linux GCC (e.g. 'x86-64', 'aarch64') -# Salt non-empty for test variants; encodes Arch+Target+Toolchain+Host -# (A/X, D/R, CV/CW/CL/G). -# Production VS2022 rows have NO Salt. -# Env extra env vars (typically CLANG_BIN). Trailing slash required. -# Extra extra PlatformBuild.py args. Every row explicitly selects its core. -# -# Shipped: the two VS2022 rows. All other rows are .Test variants. -# GCC is unshippable (no Codeview, 64K-aligned files); Clang/X64 not yet shippable. - -parameters: -- name: pipeline - type: string - values: [Official, PR] -- name: publishNuget - type: boolean - default: false - -- name: Builds - type: object - default: - # Production (shipped). No Salt. - - {Pool: Windows, Arch: X64, Target: DEBUG, ToolChainTag: VS2022, Extra: "BLD_*_USE_LEGACY_C_CORE=TRUE BLD_*_LEGACY_DEBUGGER=1"} - - {Pool: Windows, Arch: X64, Target: RELEASE, ToolChainTag: VS2022, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} - # Windows + Clang from Visual Studio. Salt: *CV. - - {Pool: Windows, Arch: X64, Target: DEBUG, ToolChainTag: CLANGPDB, Salt: XDCV, Env: {CLANG_BIN: $(CLANGVS)/bin/}, Extra: "BLD_*_USE_LEGACY_C_CORE=TRUE BLD_*_LEGACY_DEBUGGER=1"} - - {Pool: Windows, Arch: X64, Target: RELEASE, ToolChainTag: CLANGPDB, Salt: XRCV, Env: {CLANG_BIN: $(CLANGVS)/bin/}, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} - - {Pool: Windows, Arch: AARCH64, Target: DEBUG, ToolChainTag: CLANGPDB, Salt: ADCV, Env: {CLANG_BIN: $(CLANGVS)/bin/}, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} - - {Pool: Windows, Arch: AARCH64, Target: RELEASE, ToolChainTag: CLANGPDB, Salt: ARCV, Env: {CLANG_BIN: $(CLANGVS)/bin/}, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} - # Windows + Clang from Windows org. Salt: *CW. - - {Pool: Windows, Arch: X64, Target: DEBUG, ToolChainTag: CLANGPDB, ClangWindows: true, Salt: XDCW, Env: {CLANG_BIN: $(CLANGWIN_BIN)/}, Extra: "BLD_*_USE_LEGACY_C_CORE=TRUE BLD_*_LEGACY_DEBUGGER=1"} - - {Pool: Windows, Arch: X64, Target: RELEASE, ToolChainTag: CLANGPDB, ClangWindows: true, Salt: XRCW, Env: {CLANG_BIN: $(CLANGWIN_BIN)/}, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} - - {Pool: Windows, Arch: AARCH64, Target: DEBUG, ToolChainTag: CLANGPDB, ClangWindows: true, Salt: ADCW, Env: {CLANG_BIN: $(CLANGWIN_BIN)/}, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} - - {Pool: Windows, Arch: AARCH64, Target: RELEASE, ToolChainTag: CLANGPDB, ClangWindows: true, Env: {CLANG_BIN: $(CLANGWIN_BIN)/}, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} - # Linux + Clang. Salt: *CL. - - {Pool: Linux, Arch: X64, Target: DEBUG, ToolChainTag: CLANGPDB, Salt: XDCL, Extra: "BLD_*_USE_LEGACY_C_CORE=TRUE BLD_*_LEGACY_DEBUGGER=1"} - - {Pool: Linux, Arch: X64, Target: RELEASE, ToolChainTag: CLANGPDB, Salt: XRCL, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} - - {Pool: Linux, Arch: AARCH64, Target: DEBUG, ToolChainTag: CLANGPDB, Salt: ADCL, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} - - {Pool: Linux, Arch: AARCH64, Target: RELEASE, ToolChainTag: CLANGPDB, Salt: ARCL, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} - # Linux + GCC. Salt: *G. X64 DEBUG + AARCH64 disabled (mu_basecore PR #1710 / firmware size). - - {Pool: Linux, Arch: X64, Target: RELEASE, ToolChainTag: GCC, GnuArch: x86-64, Salt: XRG, Extra: BLD_*_USE_LEGACY_C_CORE=TRUE} - -- name: Nugets - type: object - default: - - {definition: llvm.tools, version: $(LlvmVersion), feed: $(LlvmFeed), downloadPath: $(PkgLlvm_Tools)} - - {definition: clang.headers, version: $(LlvmVersion), feed: $(LlvmFeed), downloadPath: $(PkgClang_Headers)} - -# Internal-repo auth URLs live in data/auth-urls.json so set_repo_auth.py owns both phases. - -jobs: -- template: ../jobs.yml - parameters: - pipeline: ${{parameters.pipeline}} - publishNuget: ${{parameters.publishNuget}} - Builds: ${{parameters.Builds}} - Nugets: ${{parameters.Nugets}} diff --git a/.azuredevops/pipelines/OneBranch/data/vars.yml b/.azuredevops/pipelines/OneBranch/data/vars.yml deleted file mode 100644 index 2c1b7e54a0..0000000000 --- a/.azuredevops/pipelines/OneBranch/data/vars.yml +++ /dev/null @@ -1,24 +0,0 @@ -### Specify container image and other global variables in this section. ### -# https://aka.ms/obpipelines/variables - -variables: - Parameters.vPack.majorVer: "26" - Parameters.vPack.minorVer: "0" - - # Directory to download Nugets to, emulating CoreXT. - Pkg: $(System.ArtifactsDirectory)/pkg - - # Clang from VisualStudio2022 in the 1ES image. Windows hosted. - CLANGVS: C:/program files/Microsoft Visual Studio/2022/Enterprise/VC/Tools/Llvm/x64 - - # Clang built and supported by Windows org. Windows hosted. - # It is split into two Nugets. We keep them separate in the - # Nuget store and copy them together. - LlvmMajorVersion: 22 - LlvmFeed: ES-LLVM - LlvmVersion: $(LlvmMajorVersion).1.1-47 - CLANGWIN: $(Agent.TempDirectory)/llvm$(LlvmVersion) - CLANGWIN_BIN: $(CLANGWIN)/bin - CLANGWIN_INC: $(CLANGWIN)/lib/clang/$(LlvmMajorVersion) - PkgLlvm_Tools: $(Pkg)/llvm.tools.$(LlvmVersion) - PkgClang_Headers: $(Pkg)/clang.headers.$(LlvmVersion) diff --git a/.azuredevops/pipelines/OneBranch/jobs.yml b/.azuredevops/pipelines/OneBranch/jobs.yml deleted file mode 100644 index 02cd2f1351..0000000000 --- a/.azuredevops/pipelines/OneBranch/jobs.yml +++ /dev/null @@ -1,195 +0,0 @@ -# ============================================================================= -# jobs.yml -- Per-variant build orchestration for Hyper-V UEFI CI. -# ============================================================================= -# -# Pure orchestration. Data lives in matrix.yml. Step bodies live in -# .azuredevops/pipelines/OneBranch/scripts/*.py (typed argparse scripts, runnable via -# run_local_ci.py without ADO). Every step here is a one-line task call. -# -# Per-job step flow: -# auth -> install_toolchain -> setup -> build -# -> stage_artifacts -> pack -> unset_repo_auth -# -# ============================================================================= - -parameters: -- name: pipeline - type: string -- name: publishNuget - type: boolean -- name: Builds - type: object -- name: Nugets - type: object - -jobs: -- ${{each Build in parameters.Builds}}: - - job: ${{Build.Target}}_${{Build.Arch}}_${{Build.ToolChainTag}}${{iif(ne(Build.Salt, ''), format('_{0}', Build.Salt), '')}} - dependsOn: SourceChecks - displayName: ${{Build.Pool}} ${{Build.Arch}} ${{Build.Target}} ${{Build.ToolChainTag}}${{iif(eq(coalesce(Build.ClangWindows, false), true), ' (WinOrg)', '')}} - pool: - type: ${{Build.Pool}} - - # ------------------------------------------------------------------------- - # Per-job variables. - # - # ob_* are read by the OneBranch base template at expansion time and must - # stay declared here (they cannot move to ##vso[task.setvariable]). - # Vpack name composition is mirrored and unit-tested in - # scripts/tests/test_get_vpack_name.py. - # ------------------------------------------------------------------------- - variables: - # --- Variant-specific env overrides (e.g. CLANG_BIN) --- - ${{each pair in Build.Env}}: - ${{pair.Key}}: ${{pair.Value}} - - # --- Toolchain / container environment --- - GCC_${{Build.Arch}}_PREFIX: /usr/bin/${{Build.GnuArch}}-linux-gnu- - GIT_TERMINAL_PROMPT: '0' - LinuxContainerImage: onebranch.azurecr.io/linux/ubuntu-2404:latest - WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest - - # --- Build paths --- - BuildDir: Build/Msvm${{Build.Arch}}/${{Build.Target}}_${{Build.ToolChainTag}} - PlatformPkg: MsvmPkg - ScriptsDir: .azuredevops/pipelines/OneBranch/scripts - - # --- Vpack name composition --- - DotSalt: ${{iif(eq(Build.Salt, ''), '', format('.{0}', Build.Salt))}} - DotTest: ${{iif(or(ne(Build.Salt, ''), eq(parameters.pipeline, 'PR')), '.Test', '')}} - - # --- OneBranch variables --- - ob_outputDirectory: $(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT - ob_nugetPublishing_enabled: ${{parameters.publishNuget}} - ob_createvpack_packagename: $(Build.Repository.Name).mscoreuefi.$(PRdot)${{Build.Arch}}.${{Build.Target}}$(DotSalt)$(DotTest) - ob_createvpack_description: Hyper V UEFI Firmware - ob_createvpack_enabled: true # Ignored in PR pipeline (upstream policy). - ob_createvpack_versionAs: parts - ob_createvpack_majorVer: $(Parameters.vPack.majorVer) - ob_createvpack_minorVer: $(Parameters.vPack.minorVer) - ob_createvpack_metadata: $(Build.SourceVersion) - ob_createvpack_failOnStdErr: true - ob_createvpack_verbose: true - ob_createvpack_vpackdirectory: $(ob_outputDirectory)/$(BuildDir)/Bin - - steps: - - # ========================================================================= - # Authentication - # ------------------------------------------------------------------------- - # NuGet feed auth, pip feed auth, and git http..extraheader for the - # internal ADO submodule(s) listed in auth-urls.json. - # ========================================================================= - - - task: NuGetAuthenticate@1 - displayName: Authenticate NuGet Feed - inputs: {nuGetServiceConnections: WindowsPartners_NugetFeedConnection} - - - task: PipAuthenticate@1 - displayName: Authenticate Pip Feed - inputs: {artifactFeeds: $(System.TeamProject)/$(System.TeamProject)_PublicPackages} - - - script: python $(ScriptsDir)/set_repo_auth.py set - displayName: Set ADO Repo Auth - env: {SYSTEM_ACCESSTOKEN: $(System.AccessToken)} - - # ========================================================================= - # Toolchain - # ------------------------------------------------------------------------- - # Windows-org Clang requires NuGet payloads (auth-typed download stays in - # YAML). install_toolchain.py then dispatches by Pool/ToolChainTag. - # ========================================================================= - - - ${{if eq(coalesce(Build.ClangWindows, false), true)}}: - - ${{each Nuget in parameters.Nugets}}: - - task: DownloadPackage@1 - displayName: Download ${{Nuget.definition}} - inputs: ${{Nuget}} - - - script: >- - python $(ScriptsDir)/install_toolchain.py - --pool ${{Build.Pool}} --tool-chain-tag ${{Build.ToolChainTag}} - --clang-windows ${{coalesce(Build.ClangWindows, false)}} - --gnu-arch "${{coalesce(Build.GnuArch, '')}}" - --clang-win-root "$(CLANGWIN)" --clang-bin "$(CLANGWIN_BIN)" --clang-inc "$(CLANGWIN_INC)" - --llvm-tools-source "$(PkgLlvm_Tools)" --clang-headers-source "$(PkgClang_Headers)" - displayName: Install Toolchain - - # ========================================================================= - # Build - # ------------------------------------------------------------------------- - # setup: pip install + Stuart --setup + Stuart --update. - # build: the single PlatformBuild.py invocation. Build logs are always - # dumped from the script's finally block (no separate log step). - # ========================================================================= - - - script: python $(ScriptsDir)/setup.py --platform-pkg $(PlatformPkg) - displayName: Setup - - - script: >- - python $(ScriptsDir)/build.py - --arch ${{Build.Arch}} --target ${{Build.Target}} --tool-chain-tag ${{Build.ToolChainTag}} - --extra "${{coalesce(Build.Extra, '')}}" --platform-pkg $(PlatformPkg) - displayName: Build - - # ========================================================================= - # Artifact staging - # ------------------------------------------------------------------------- - # stage_artifacts: flatten logs, copy fd/PDB/MAP, build vpack staging dir. - # copy_all_on_failure:dump the whole Build/ tree to ob_outputDirectory so we - # can debug a failed build offline. - # PublishSymbols: symbol-server upload (skipped for GCC -- no Codeview). - # ========================================================================= - - - script: >- - python $(ScriptsDir)/stage_artifacts.py - --arch ${{Build.Arch}} --target ${{Build.Target}} --tool-chain-tag ${{Build.ToolChainTag}} - --build-dir "$(BuildDir)" --out-dir "$(ob_outputDirectory)" --job-name "$(Agent.JobName)" - displayName: Stage Artifacts - condition: succeededOrFailed() - continueOnError: true - - - task: CopyFiles@2 - displayName: Copy Full Build Tree (on Failure) - condition: failed() - continueOnError: true - inputs: {SourceFolder: Build, Contents: '**', TargetFolder: $(ob_outputDirectory)/all} - - - ${{if ne(Build.ToolChainTag, 'GCC')}}: - - task: PublishSymbols@2 - displayName: Publish Symbols - continueOnError: true - inputs: - SearchPattern: $(BuildDir)/PDB/*.pdb - IndexSources: true - SymbolServerType: TeamServices - SymbolsProduct: Hyper-V UEFI - enablesymbolspublishing: true - - # ========================================================================= - # NuGet pack - # ------------------------------------------------------------------------- - # The OneBranch base template generates the vpack itself from - # ob_createvpack_vpackdirectory and increments the rightmost version digit. - # This step is the matching .nupkg for the NuGet feed. - # ========================================================================= - - - task: NuGetCommand@2 - displayName: Pack NuGet - inputs: - command: pack - packagesToPack: $(PlatformPkg)/HyperVUefi.NuSpec - packDestination: $(ob_outputDirectory)/packages - packTimezone: utc - buildProperties: id=$(ob_createvpack_packagename);version=$(Parameters.vPack.majorVer).$(Parameters.vPack.minorVer)-$(Build.SourceBranchName).$(Build.SourceVersion) - - # ========================================================================= - # Cleanup - # ------------------------------------------------------------------------- - # Always remove the extraheader auth so credentials are not left in - # git config on the agent. Mirrors set_repo_auth and shares its URL list. - # ========================================================================= - - - script: python $(ScriptsDir)/set_repo_auth.py unset - displayName: Unset ADO Repo Auth - condition: always() diff --git a/.azuredevops/pipelines/OneBranch/scripts/README.md b/.azuredevops/pipelines/OneBranch/scripts/README.md deleted file mode 100644 index ce2e2e147e..0000000000 --- a/.azuredevops/pipelines/OneBranch/scripts/README.md +++ /dev/null @@ -1,134 +0,0 @@ -# OneBranch CI Scripts - -Python entrypoints used by `.azuredevops/pipelines/OneBranch/jobs.yml`. - -## Main scripts - -- `setup.py`: pip install + Stuart setup/update -- `install_toolchain.py`: pool/toolchain-specific tool install logic -- `build.py`: runs `PlatformBuild.py` for one variant and prints command/logs -- `stage_artifacts.py`: stages logs, firmware binaries, maps, and PDBs -- `set_repo_auth.py`: sets/unsets git extraheader auth for internal ADO repos -- `run_local_ci.py`: local developer entrypoint (setup + build + optional staging) -- `get_vpack_name.py`: reference implementation for vpack naming -- `source_checks.py`: updates CI dependencies and runs MsvmPkg source checks - -## Local usage - -Run one local build variant from repo root: - -```powershell -py .\.azuredevops\pipelines\OneBranch\scripts\run_local_ci.py --arch X64 --target DEBUG --tool-chain-tag VS2022 -``` - -Run local verification checks: - -```powershell -py .\.azuredevops\pipelines\OneBranch\scripts\tests\verify_ci.py -``` - -## Stuart Source Checks - -The initial onboarding ported mu_msvm commit `61216d796b1f873680c89b8e00141fb049c8efcd`. -Following the merged internal formatting change, enforcement adapts upstream -commit `acbd02cb02843ca6c3ebe5279665aff168876019` to OneBranch. -Shared Stuart settings also incorporate the final CI -changes in `8848e1d684f54b025ebe32707c0919e47a2c0c0e`, including Patina and -recursive submodule declarations. OneBranch temporarily uses non-recursive -source-check provisioning as described below. OpenVMM testing and the GitHub reusable-action -migration are not imported. Shared settings, check policy, BUILDING/DEVGUIDE -documentation, and `pip-requirements.txt` match that upstream revision. -OneBranch uses the same `pip-requirements.txt` as the shared documentation. -`.github/workflows/Platform-Build.yml` remains the earlier onboarding snapshot; -ADO executes OneBranch, not that historical GitHub workflow. -The mirrored workflow is not an authenticated GitHub build of this private -repository, whose MU_BASECORE submodule requires ADO access. - -Run from an existing development checkout with the stable Python requirements -installed and top-level submodules initialized: - -```powershell -python .\.azuredevops\pipelines\OneBranch\scripts\source_checks.py -``` - -Use `--setup` on a fresh checkout to install the stable Python requirements and -run `git -c core.autocrlf=false submodule update --init`. This initializes -MU_BASECORE, Common/MU, Feature/DEBUGGER, and Common/PATINA_EDK2 submodules at -their pinned revisions without recursively downloading nested submodules. -Authenticate to the private MU_BASECORE repository first. The script does not -provision compilers, Rust, or firmware binary dependencies. - -The intended setup command is `stuart_setup -c .pytool/CISettings.py`, keeping -submodule selection under the shared upstream settings. Its recursive checkout -failed in build 157900552 because the hosted agent could not connect to GitLab -to download libspdm's nested cmocka test dependency. Use top-level checkout for -these source-only checks until fresh hosted agents can resolve all nested -dependencies; the shared settings remain unchanged. - -For manual setup on a private development checkout, authenticate first, then run: - -```powershell -git config core.autocrlf false -git config core.longpaths true -python -m pip install -r pip-requirements.txt -git -c core.autocrlf=false submodule update --init -stuart_update -c .\.pytool\CISettings.py -``` - -To omit the internal bulk-formatting revision from local blame output: - -```powershell -git config blame.ignoreRevsFile .git-blame-ignore-revs -``` - -The shared OneBranch template includes one Windows `SourceChecks` job for both -PR and Official builds. All three selected checks are enforced. -It uses the existing NuGet, pip, and ADO repository authentication and always -removes the repository authentication afterward. Job-level `GIT_CONFIG_*` -variables disable automatic line-ending conversion and enable long paths for -Git processes, including checkout, without changing the agent's global Git configuration. The job uses -the container's Python, as existing OneBranch jobs do; GitHub pins Python 3.12. - -The settings select only the `cibuild` scope, `MsvmPkg`, `NO-TARGET`, and both -X64/AARCH64. All plugins are disabled except GuidCheck, LineEndingCheck, and -UncrustifyCheck. The workspace root is a package search root so internal sibling -packages participate in GUID comparisons; only MsvmPkg is selected for checks. -The upstream GUID exception list is unchanged. Source formatting and DSC -line-ending normalization landed in the preceding formatting PR. The package's -`.gitattributes` pins text files to CRLF on every platform. Package search paths -are workspace-relative and include Common/PATINA_EDK2, matching current upstream. -Including Patina in source-check setup does not enable Patina firmware builds. - -The pinned GuidCheck and LineEndingCheck plugins report failures directly. -UncrustifyCheck has `AuditOnly: false`, so formatting findings also fail the -local command and JUnit test rather than being reported as skipped. - -The Stuart invocation has no `continueOnError` override and test publication -uses `failTaskOnFailedTests: true`. Setup/update/check errors propagate as -nonzero exits and fail `SourceChecks`. Feed/repository authentication and -cleanup task failures are not suppressed. Each firmware matrix job depends on -`SourceChecks` succeeding, preventing both builds and their vpack/NuGet -publication after a failed check. This is the OneBranch counterpart to -upstream's release-job dependency; OneBranch packages inside each matrix job. - -The check job does not publish NuGet or vpack packages. It publishes -`Build/TestSuites.xml` to ADO Test Results and stages the CI/setup/update logs and -`Build/MsvmPkg` reports even when checks fail. Log staging is best-effort so a -missing `Build/` after a setup failure does not add another blocking error. - -At the pinned MU_BASECORE revision, Uncrustify is the SHA-256-pinned -`tianocore-uncrustify-release` 73.0.11 ZIP from GitHub, not a Project Mu NuGet -feed. Governed-container egress and Component Governance approval for the -CI-scoped binary dependencies still need confirmation; enabling enforcement -does not establish approval. No internal mirror is configured. - -See [Running CI Checks Locally](../../../../MsvmPkg/Docs/BUILDING.md#running-ci-checks-locally) -for individual check commands and in-place formatting. That shared guide retains -upstream's GitHub Actions wording for mirroring; this repository enforces the -same checks through OneBranch as described here. - -Local tests validate orchestration, failure propagation, and enforcement policy. -`verify_ci.py` validates YAML parsing and the Python scripts, but cannot expand -the governed OneBranch template. Queue an ADO PR build to validate checkout -environment propagation, private feed/submodule access, results publication, -and firmware jobs being skipped after a source-check failure on a hosted runner. diff --git a/.azuredevops/pipelines/OneBranch/scripts/build.py b/.azuredevops/pipelines/OneBranch/scripts/build.py deleted file mode 100644 index 72f497c4d3..0000000000 --- a/.azuredevops/pipelines/OneBranch/scripts/build.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -Runs the main Hyper-V UEFI build for one matrix variant. - -Invokes PlatformBuild.py with the canonical Stuart args. Always dumps -Build/ listing and Build/B*.txt log files in a finally block so a failed -job still produces actionable output in the pipeline log. -""" - -from __future__ import annotations - -import argparse -import os -import subprocess -import sys -from pathlib import Path - -from ci_common import VALID_ARCHS, VALID_TARGETS, VALID_TOOLCHAINS, Arch, Target, ToolChainTag, die - -# Fixed Stuart args appended to every build invocation. -_FIXED_BUILD_ARGS: list[str] = [ - "BUILDREPORTING=TRUE", - "BUILDREPORT_TYPES=PCD DEPEX FLASH BUILD_FLAGS LIBRARY", - "LaunchLogOnSuccess=FALSE", - "LaunchLogOnError=FALSE", -] - - -def _dump_build_logs() -> None: - build_dir = Path("Build") - print("== Build/ listing ==") - if build_dir.exists(): - for entry in sorted(build_dir.iterdir()): - size = entry.stat().st_size if entry.is_file() else 0 - print(f" {entry.name:<45} {size:>10} bytes") - else: - print(" (Build directory missing)") - - print("== Build/B*.txt contents ==") - if build_dir.exists(): - for txt in sorted(build_dir.glob("B*.txt")): - print(f"---- {txt.name} ----") - print(txt.read_text(errors="replace")) - - -def run_build( - arch: Arch, - target: Target, - tool_chain_tag: ToolChainTag, - extra: str, - platform_pkg: str, -) -> None: - platform_build = Path(platform_pkg) / "PlatformBuild.py" - if not platform_build.exists(): - die(f"PlatformBuild.py not found at: {platform_build}") - - extra_tokens = extra.split() if extra.strip() else [] - cmd = [ - "stuart_build", "-c", str(platform_build), - *extra_tokens, - f"TOOL_CHAIN_TAG={tool_chain_tag}", - f"TARGET={target}", - f"BUILD_ARCH={arch}", - *_FIXED_BUILD_ARGS, - ] - print(f"== Command: {' '.join(cmd)} ==") - - try: - subprocess.run(cmd, check=True) - except subprocess.CalledProcessError as e: - sys.exit(e.returncode) - finally: - _dump_build_logs() - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--arch", required=True, choices=VALID_ARCHS) - parser.add_argument("--target", required=True, choices=VALID_TARGETS) - parser.add_argument("--tool-chain-tag", required=True, choices=VALID_TOOLCHAINS) - parser.add_argument("--extra", default="", help="Extra PlatformBuild.py args (whitespace-separated)") - parser.add_argument( - "--platform-pkg", - default=os.environ.get("PlatformPkg", "MsvmPkg"), - help="Platform package path. Also read from $PlatformPkg env var.", - ) - args = parser.parse_args() - - platform_pkg = args.platform_pkg.strip() - if not platform_pkg: - die("--platform-pkg not specified and $PlatformPkg env var is not set.") - - run_build( - arch=args.arch, - target=args.target, - tool_chain_tag=args.tool_chain_tag, - extra=args.extra, - platform_pkg=platform_pkg, - ) - - -if __name__ == "__main__": - main() diff --git a/.azuredevops/pipelines/OneBranch/scripts/ci_common.py b/.azuredevops/pipelines/OneBranch/scripts/ci_common.py deleted file mode 100644 index 24352851c5..0000000000 --- a/.azuredevops/pipelines/OneBranch/scripts/ci_common.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Shared types and helpers for Hyper-V UEFI CI scripts.""" - -from __future__ import annotations - -import subprocess -import sys -from typing import Literal - -# --------------------------------------------------------------------------- -# Canonical value sets — used both as runtime choices in argparse and as -# type aliases for static analysis. -# --------------------------------------------------------------------------- - -Arch = Literal["X64", "AARCH64"] -Target = Literal["DEBUG", "RELEASE"] -ToolChainTag = Literal["VS2022", "CLANGPDB", "GCC"] -Pool = Literal["Windows", "Linux"] -Pipeline = Literal["Official", "PR"] - -VALID_ARCHS: tuple[str, ...] = ("X64", "AARCH64") -VALID_TARGETS: tuple[str, ...] = ("DEBUG", "RELEASE") -VALID_TOOLCHAINS: tuple[str, ...] = ("VS2022", "CLANGPDB", "GCC") -VALID_POOLS: tuple[str, ...] = ("Windows", "Linux") -VALID_PIPELINES: tuple[str, ...] = ("Official", "PR") - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def run(label: str, cmd: list[str]) -> None: - """Print *label*, execute *cmd*, and raise CalledProcessError on failure.""" - print(f"== {label} ==") - subprocess.run(cmd, check=True) - - -def die(msg: str) -> None: - """Print *msg* to stderr and exit 1.""" - print(f"ERROR: {msg}", file=sys.stderr) - sys.exit(1) diff --git a/.azuredevops/pipelines/OneBranch/scripts/get_vpack_name.py b/.azuredevops/pipelines/OneBranch/scripts/get_vpack_name.py deleted file mode 100644 index fab493c962..0000000000 --- a/.azuredevops/pipelines/OneBranch/scripts/get_vpack_name.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -Computes the OneBranch vpack package name for a Hyper-V UEFI build variant. - -Naming rule: - .mscoreuefi.. - -Components: - pr_dot = 'PR.' for PR pipeline, '' for Official - dot_salt = '.' if salt non-empty, '' otherwise - dot_test = '.Test' if salt non-empty OR pipeline == PR, '' otherwise - -Production (shippable) variants have no salt and run in the Official pipeline — -they carry neither dot_salt nor dot_test. All other variants carry .Test. - -NOTE: This script does not influence the rightmost auto-increment digit added -by OneBranch; it only produces the package ID. The YAML pipeline composes the -same name inline (ob_createvpack_packagename); the pytest suite locks both -implementations so they cannot silently diverge. -""" - -from __future__ import annotations - -import argparse - -from ci_common import VALID_ARCHS, VALID_PIPELINES, VALID_TARGETS, Arch, Pipeline, Target - - -def get_vpack_name( - repo_name: str, - pipeline: Pipeline, - arch: Arch, - target: Target, - salt: str = "", -) -> str: - """Return the vpack package name for the given variant.""" - pr_dot = "PR." if pipeline == "PR" else "" - dot_salt = f".{salt}" if salt else "" - dot_test = ".Test" if (salt or pipeline == "PR") else "" - return f"{repo_name}.mscoreuefi.{pr_dot}{arch}.{target}{dot_salt}{dot_test}" - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--repo-name", required=True, help="Build.Repository.Name in ADO (case-sensitive)") - parser.add_argument("--pipeline", required=True, choices=VALID_PIPELINES) - parser.add_argument("--arch", required=True, choices=VALID_ARCHS) - parser.add_argument("--target", required=True, choices=VALID_TARGETS) - parser.add_argument("--salt", default="", help="Matrix row salt (e.g. 'XDCV'). Empty for production rows.") - args = parser.parse_args() - - print(get_vpack_name( - repo_name=args.repo_name, - pipeline=args.pipeline, - arch=args.arch, - target=args.target, - salt=args.salt, - )) - - -if __name__ == "__main__": - main() diff --git a/.azuredevops/pipelines/OneBranch/scripts/install_toolchain.py b/.azuredevops/pipelines/OneBranch/scripts/install_toolchain.py deleted file mode 100644 index f1bd43c109..0000000000 --- a/.azuredevops/pipelines/OneBranch/scripts/install_toolchain.py +++ /dev/null @@ -1,106 +0,0 @@ -""" -Installs the build toolchain for a given Hyper-V UEFI CI variant. - -Dispatch table: - Linux + CLANGPDB -> apt-get install clang lld llvm - Linux + GCC -> apt-get install binutils- gcc- - Windows + VS2022 -> no-op (toolchain ships in the 1ES image) - Windows + CLANGPDB, clang_windows=false -> no-op (Clang from Visual Studio 2022) - Windows + CLANGPDB, clang_windows=true -> copy-install Windows-org Clang from - pre-downloaded NuGet payloads - -OneBranch Linux containers run as root; no sudo is needed. -""" - -from __future__ import annotations - -import argparse -import shutil -from pathlib import Path - -from ci_common import VALID_POOLS, VALID_TOOLCHAINS, Pool, ToolChainTag, die, run - - -def _install_linux(tool_chain_tag: ToolChainTag, gnu_arch: str) -> None: - run("apt-get update", ["apt-get", "update", "-y"]) - if tool_chain_tag == "CLANGPDB": - run("apt-get install clang lld llvm", ["apt-get", "install", "-y", "clang", "lld", "llvm"]) - elif tool_chain_tag == "GCC": - if not gnu_arch.strip(): - die("--gnu-arch is required for Linux + GCC.") - pkg_binutils = f"binutils-{gnu_arch}-linux-gnu" - pkg_gcc = f"gcc-{gnu_arch}-linux-gnu" - run(f"apt-get install {pkg_binutils} {pkg_gcc}", ["apt-get", "install", "-y", pkg_binutils, pkg_gcc]) - else: - die(f"Unsupported Linux toolchain: {tool_chain_tag}") - - -def _install_clang_windows( - clang_win_root: Path, - clang_bin: Path, - clang_inc: Path, - llvm_tools_source: Path, - clang_headers_source: Path, -) -> None: - required = { - "--clang-win-root": clang_win_root, - "--clang-bin": clang_bin, - "--clang-inc": clang_inc, - "--llvm-tools-source": llvm_tools_source, - "--clang-headers-source": clang_headers_source, - } - for flag, val in required.items(): - if not str(val).strip(): - die(f"{flag} is required when --clang-windows true.") - - if clang_win_root.exists(): - print(f"Removing existing CLANGWIN root: {clang_win_root}") - shutil.rmtree(clang_win_root) - - print(f"Installing LLVM tools: {llvm_tools_source} -> {clang_bin}") - clang_bin.mkdir(parents=True, exist_ok=True) - shutil.copytree(llvm_tools_source, clang_bin, dirs_exist_ok=True) - - print(f"Installing clang headers: {clang_headers_source} -> {clang_inc}") - clang_inc.mkdir(parents=True, exist_ok=True) - shutil.copytree(clang_headers_source, clang_inc, dirs_exist_ok=True) - - print("Windows-org Clang install complete.") - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--pool", required=True, choices=VALID_POOLS) - parser.add_argument("--tool-chain-tag", required=True, choices=VALID_TOOLCHAINS) - parser.add_argument( - "--clang-windows", - type=str.lower, - default="false", - choices=["true", "false"], - help="'true' to install Windows-org Clang from NuGet payloads (string to match ADO template bool serialization)", - ) - parser.add_argument("--gnu-arch", default="", help="GNU triple arch for Linux GCC (e.g. 'x86-64', 'aarch64')") - parser.add_argument("--clang-win-root", type=Path, default=Path(""), help="Windows-org Clang root; removed before reinstall") - parser.add_argument("--clang-bin", type=Path, default=Path(""), help="Destination for LLVM tools") - parser.add_argument("--clang-inc", type=Path, default=Path(""), help="Destination for clang headers") - parser.add_argument("--llvm-tools-source", type=Path, default=Path(""), help="llvm.tools NuGet payload directory") - parser.add_argument("--clang-headers-source", type=Path, default=Path(""), help="clang.headers NuGet payload directory") - args = parser.parse_args() - - if args.pool == "Linux": - _install_linux(args.tool_chain_tag, args.gnu_arch) - else: - if args.clang_windows != "true": - print(f"Windows pool, {args.tool_chain_tag}: no toolchain install required (using image-provided tools).") - else: - _install_clang_windows( - clang_win_root=args.clang_win_root, - clang_bin=args.clang_bin, - clang_inc=args.clang_inc, - llvm_tools_source=args.llvm_tools_source, - clang_headers_source=args.clang_headers_source, - ) - - -if __name__ == "__main__": - main() diff --git a/.azuredevops/pipelines/OneBranch/scripts/run_local_ci.py b/.azuredevops/pipelines/OneBranch/scripts/run_local_ci.py deleted file mode 100644 index 4527e5955f..0000000000 --- a/.azuredevops/pipelines/OneBranch/scripts/run_local_ci.py +++ /dev/null @@ -1,96 +0,0 @@ -""" -Run one CI build variant locally, invoking the same scripts the pipeline uses. - -Mirrors jobs.yml for a single matrix row without an ADO runner. Run from -the repo root after installing toolchain and ensuring Python 3.10+ is active. - -Skipped vs. the pipeline: - set_repo_auth.py (assumes developer git credentials are configured) - NuGet download (assumes toolchain already installed) - publish_symbols (no symbol server upload locally) - pack_nuget (no .nupkg needed locally) - OneBranch vpack / artifact upload - -Usage: - python .azuredevops/pipelines/OneBranch/scripts/run_local_ci.py \\ - --arch X64 --target DEBUG --tool-chain-tag VS2022 -""" - -from __future__ import annotations - -import argparse -import os -import subprocess -from pathlib import Path - -from ci_common import VALID_ARCHS, VALID_TARGETS, VALID_TOOLCHAINS - -_SCRIPTS_DIR = Path(__file__).parent - - -def _run_script(script: str, *args: str) -> None: - """Run a sibling script in the same Python interpreter.""" - cmd = ["python", str(_SCRIPTS_DIR / script), *args] - subprocess.run(cmd, check=True) - - -def main() -> None: - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument("--arch", required=True, choices=VALID_ARCHS) - parser.add_argument("--target", required=True, choices=VALID_TARGETS) - parser.add_argument("--tool-chain-tag", required=True, choices=VALID_TOOLCHAINS) - parser.add_argument("--extra", default="", help="Extra PlatformBuild.py args") - parser.add_argument("--platform-pkg", default="MsvmPkg") - parser.add_argument( - "--out-dir", - type=Path, - default=Path.cwd() / "LocalCiOutput", - help="Artifact output directory (default: LocalCiOutput/)", - ) - parser.add_argument( - "--skip-stage", - action="store_true", - help="Skip artifact staging for a faster inner-loop build", - ) - args = parser.parse_args() - - os.environ["PlatformPkg"] = args.platform_pkg - - print("==== setup ====") - _run_script("setup.py", "--platform-pkg", args.platform_pkg) - - print("==== build ====") - build_args = [ - "--arch", args.arch, - "--target", args.target, - "--tool-chain-tag", args.tool_chain_tag, - "--platform-pkg", args.platform_pkg, - ] - if args.extra: - build_args += ["--extra", args.extra] - _run_script("build.py", *build_args) - - if not args.skip_stage: - build_dir = f"Build/Msvm{args.arch}/{args.target}_{args.tool_chain_tag}" - print(f"==== stage_artifacts (build_dir={build_dir}, out_dir={args.out_dir}) ====") - _run_script( - "stage_artifacts.py", - "--arch", args.arch, - "--target", args.target, - "--tool-chain-tag", args.tool_chain_tag, - "--build-dir", build_dir, - "--out-dir", str(args.out_dir), - "--job-name", "local", - ) - - extra_label = f" ({args.extra})" if args.extra else "" - print(f"\n==== Done: {args.target} {args.arch} {args.tool_chain_tag}{extra_label} ====") - if not args.skip_stage: - print(f"Output: {args.out_dir}") - - -if __name__ == "__main__": - main() diff --git a/.azuredevops/pipelines/OneBranch/scripts/set_repo_auth.py b/.azuredevops/pipelines/OneBranch/scripts/set_repo_auth.py deleted file mode 100644 index 56e073979c..0000000000 --- a/.azuredevops/pipelines/OneBranch/scripts/set_repo_auth.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -Sets or unsets git http..extraheader auth for internal ADO repos. - -The URL list is read from data/auth-urls.json so the Set and Unset phases -share the same source and cannot drift. - -For 'set' mode, the OAuth bearer token is read from the SYSTEM_ACCESSTOKEN -environment variable so it does not appear in process arguments (where ADO -task logs may echo them verbatim). Inject it via the task's env: block: - - - script: python set_repo_auth.py set - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) -""" - -from __future__ import annotations - -import argparse -import json -import os -import subprocess -import sys -from pathlib import Path - -_DEFAULT_URLS_JSON = Path(__file__).parent.parent / "data" / "auth-urls.json" - - -def _git(*args: str) -> int: - return subprocess.run(["git", *args]).returncode - - -def set_auth(urls: list[str], token: str) -> None: - for url in urls: - cfg_key = f"http.{url}.extraheader" - print(f"Set auth: {url}") - rc = _git("config", "--global", cfg_key, f"AUTHORIZATION: Bearer {token}") - if rc != 0: - print(f"ERROR: git config --global {cfg_key} failed (exit {rc})", file=sys.stderr) - sys.exit(rc) - - -def unset_auth(urls: list[str]) -> None: - for url in urls: - cfg_key = f"http.{url}.extraheader" - print(f"Unset auth: {url}") - rc = _git("config", "--global", "--unset", cfg_key) - # git config --unset exits 5 if the key is already absent; tolerate it - # so re-runs and partial-state cleanups don't fail the job. - if rc not in (0, 5): - print(f"ERROR: git config --global --unset {cfg_key} failed (exit {rc})", file=sys.stderr) - sys.exit(rc) - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("mode", choices=["set", "unset"]) - parser.add_argument( - "--urls-json", - type=Path, - default=_DEFAULT_URLS_JSON, - help="Path to auth-urls.json (default: ../data/auth-urls.json relative to this script)", - ) - args = parser.parse_args() - - if not args.urls_json.exists(): - print(f"ERROR: auth-urls.json not found at: {args.urls_json}", file=sys.stderr) - sys.exit(1) - - data = json.loads(args.urls_json.read_text()) - urls: list[str] = data.get("urls", []) - if not urls: - print(f"ERROR: No 'urls' entries in {args.urls_json}", file=sys.stderr) - sys.exit(1) - - if args.mode == "set": - token = os.environ.get("SYSTEM_ACCESSTOKEN", "").strip() - if not token: - print( - "ERROR: SYSTEM_ACCESSTOKEN is not set. " - "Inject it via the task env: block (do not pass it as an argument).", - file=sys.stderr, - ) - sys.exit(1) - set_auth(urls, token) - else: - unset_auth(urls) - - -if __name__ == "__main__": - main() diff --git a/.azuredevops/pipelines/OneBranch/scripts/setup.py b/.azuredevops/pipelines/OneBranch/scripts/setup.py deleted file mode 100644 index 338d3ef085..0000000000 --- a/.azuredevops/pipelines/OneBranch/scripts/setup.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -Runs the pre-build setup phase: pip install, Stuart --setup, Stuart --update. - -Each invocation fails-fast so a partial environment is never silently -passed to the build step. -""" - -from __future__ import annotations - -import argparse -import os -import subprocess -from pathlib import Path - -from ci_common import die, run - -_DEFAULT_PIP_REQUIREMENTS = "pip-requirements.txt" - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument( - "--platform-pkg", - default=os.environ.get("PlatformPkg", ""), - help="Platform package path (e.g. 'MsvmPkg'). Also read from $PlatformPkg env var.", - ) - parser.add_argument( - "--pip-requirements", - default=_DEFAULT_PIP_REQUIREMENTS, - help=f"pip requirements file (default: {_DEFAULT_PIP_REQUIREMENTS})", - ) - args = parser.parse_args() - - platform_pkg = args.platform_pkg.strip() - if not platform_pkg: - die("--platform-pkg not specified and $PlatformPkg env var is not set.") - - pip_req = Path(args.pip_requirements) - if not pip_req.exists(): - die(f"pip requirements file not found: {pip_req}") - - platform_build = Path(platform_pkg) / "PlatformBuild.py" - if not platform_build.exists(): - die(f"PlatformBuild.py not found at: {platform_build}") - - try: - run("python --version", ["python", "--version"]) - run( - f"pip install -r {pip_req}", - ["python", "-m", "pip", "install", "--upgrade", "-r", str(pip_req), - "--force-reinstall", "--no-cache-dir"], - ) - run(f"stuart_setup -c {platform_build}", ["stuart_setup", "-c", str(platform_build)]) - run(f"stuart_update -c {platform_build}", ["stuart_update", "-c", str(platform_build)]) - except subprocess.CalledProcessError as exc: - die(f"Setup failed while running: {' '.join(exc.cmd)} (exit {exc.returncode})") - - -if __name__ == "__main__": - main() diff --git a/.azuredevops/pipelines/OneBranch/scripts/source_checks.py b/.azuredevops/pipelines/OneBranch/scripts/source_checks.py deleted file mode 100644 index eecf83c497..0000000000 --- a/.azuredevops/pipelines/OneBranch/scripts/source_checks.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Run MsvmPkg Stuart source checks, optionally provisioning a fresh CI checkout.""" - -from __future__ import annotations - -import argparse -import os -import subprocess -import sys -from pathlib import Path - -from ci_common import run - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--setup", action="store_true", - help="Install Python dependencies and initialize top-level submodules before checking.", - ) - args = parser.parse_args() - os.chdir(Path(__file__).resolve().parents[4]) - - try: - if args.setup: - run("Install Python dependencies", [ - sys.executable, "-m", "pip", "install", "-r", "pip-requirements.txt", - ]) - # Prefer "stuart_setup -c .pytool/CISettings.py" so shared settings own setup. - # Its recursive checkout reached libspdm's GitLab-hosted cmocka dependency, - # which the OneBranch agent could not download (build 157900552). - # These source checks use top-level repositories; restore Stuart setup - # once fresh hosted agents can resolve all declared nested dependencies. - run("Initialize source-check submodules", [ - "git", "-c", "core.autocrlf=false", "submodule", "update", "--init", - ]) - run("Update source-check dependencies", ["stuart_update", "-c", ".pytool/CISettings.py"]) - run("Run source checks", [ - "stuart_ci_build", "-c", ".pytool/CISettings.py", - "-p", "MsvmPkg", "-t", "NO-TARGET", "-a", "X64,AARCH64", - "--disable-all", "GuidCheck=run", "LineEndingCheck=run", "UncrustifyCheck=run", - ]) - except subprocess.CalledProcessError as exc: - print(f"ERROR: Source checks failed: {' '.join(exc.cmd)} (exit {exc.returncode})", file=sys.stderr) - sys.exit(exc.returncode) - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/.azuredevops/pipelines/OneBranch/scripts/stage_artifacts.py b/.azuredevops/pipelines/OneBranch/scripts/stage_artifacts.py deleted file mode 100644 index 11170d553a..0000000000 --- a/.azuredevops/pipelines/OneBranch/scripts/stage_artifacts.py +++ /dev/null @@ -1,156 +0,0 @@ -""" -Stages build outputs into the OneBranch ob_outputDirectory. - -Produces: - /Build Logs / -- flattened log set - /Firmware Binary File _/ -- MSVM.fd - /Firmware PDB Files _/ -- *.pdb (non-GCC only) - /Firmware Map Files _/ -- *.map - //Bin/ -- vpack staging dir: - MSVM.fd, Dsdt.aml, BiosInterface.h, PDB/ (non-GCC only) - -PDB and symbol files are skipped for GCC builds (no CodeView output). -""" - -from __future__ import annotations - -import argparse -import fnmatch -import os -import shutil -from pathlib import Path - -from ci_common import VALID_ARCHS, VALID_TARGETS, VALID_TOOLCHAINS, Arch, Target, ToolChainTag - -# Log files to collect from anywhere under Build/ (case-insensitive fnmatch). -_LOG_PATTERNS: tuple[str, ...] = ( - "BuildLog.txt", - "BUILDLOG_*.txt", - "BUILDLOG_*.md", - "BUILD_REPORT.txt", - "CI_*.txt", - "CI_*.md", - "CISETUP.txt", - "SETUPLOG.txt", - "UPDATE_LOG.txt", - "PREVALLOG.txt", - "TestSuites.xml", - "BUILD_TOOLS_REPORT.html", - "OVERRIDELOG.TXT", - "BASETOOLS_BUILD*.*", - "FD_REPORT.HTML", -) - - -def _is_log_file(name: str) -> bool: - return any(fnmatch.fnmatch(name.lower(), pat.lower()) for pat in _LOG_PATTERNS) - - -def _copy_file(src: Path, dst_dir: Path) -> None: - """Copy *src* file into *dst_dir*, warning if absent.""" - if not src.exists(): - print(f"WARNING: source missing (skipping): {src}") - return - shutil.copy2(src, dst_dir) - - -def _copy_tree(src: Path, dst_dir: Path) -> None: - """Copy directory *src* as a sub-tree of *dst_dir*, warning if absent.""" - if not src.exists(): - print(f"WARNING: source missing (skipping): {src}") - return - shutil.copytree(src, dst_dir / src.name, dirs_exist_ok=True) - - -def stage_artifacts( - arch: Arch, - target: Target, - tool_chain_tag: ToolChainTag, - build_dir: Path, - out_dir: Path, - job_name: str, -) -> None: - is_gcc = tool_chain_tag == "GCC" - arch_target = f"{target}_{arch}" - - # -- Build logs (flattened) -- - log_dest = out_dir / f"Build Logs {job_name}" - log_dest.mkdir(parents=True, exist_ok=True) - if build_dir.exists(): - # Logs are intentionally flattened by filename into one folder for ADO - # artifact browsing simplicity. If two files share a name, last copy wins. - for f in build_dir.rglob("*"): - if f.is_file() and _is_log_file(f.name): - try: - shutil.copy2(f, log_dest / f.name) - except OSError as exc: - print(f"WARNING: failed to stage log {f}: {exc}") - - # -- Firmware binary -- - fd_dest = out_dir / f"Firmware Binary File {arch_target}" - fd_dest.mkdir(parents=True, exist_ok=True) - _copy_file(build_dir / "FV" / "MSVM.fd", fd_dest) - - # -- PDB files (non-GCC only) -- - if not is_gcc: - pdb_dest = out_dir / f"Firmware PDB Files {arch_target}" - pdb_dest.mkdir(parents=True, exist_ok=True) - pdb_src = build_dir / "PDB" - if pdb_src.exists(): - shutil.copytree(pdb_src, pdb_dest, dirs_exist_ok=True) - else: - print(f"WARNING: PDB source missing (skipping): {pdb_src}") - - # -- MAP files -- - map_dest = out_dir / f"Firmware Map Files {arch_target}" - map_dest.mkdir(parents=True, exist_ok=True) - map_src = build_dir / "MAP" - if map_src.exists(): - shutil.copytree(map_src, map_dest, dirs_exist_ok=True) - else: - print(f"WARNING: MAP source missing (skipping): {map_src}") - - # -- Vpack staging: ob_createvpack_vpackdirectory = //Bin -- - vpack_dir = out_dir / build_dir / "Bin" - vpack_dir.mkdir(parents=True, exist_ok=True) - - _copy_file(build_dir / "FV" / "MSVM.fd", vpack_dir) - _copy_file(build_dir / arch / "MsvmPkg" / "AcpiTables" / "AcpiTables" / "OUTPUT" / "Dsdt.aml", vpack_dir) - _copy_file(Path("MsvmPkg") / "Include" / "BiosInterface.h", vpack_dir) - - if not is_gcc: - _copy_tree(build_dir / "PDB", vpack_dir) - - print("Artifact staging complete.") - print(f" Logs: {log_dest}") - print(f" Firmware: {fd_dest}") - print(f" Maps: {map_dest}") - print(f" Vpack: {vpack_dir}") - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--arch", required=True, choices=VALID_ARCHS) - parser.add_argument("--target", required=True, choices=VALID_TARGETS) - parser.add_argument("--tool-chain-tag", required=True, choices=VALID_TOOLCHAINS) - parser.add_argument("--build-dir", required=True, type=Path, help="Workspace-relative build output dir") - parser.add_argument("--out-dir", required=True, type=Path, help="Absolute path to ob_outputDirectory") - parser.add_argument( - "--job-name", - default=os.environ.get("AGENT_JOBNAME", "local"), - help="ADO job name for artifact folder. Falls back to 'local'.", - ) - args = parser.parse_args() - - stage_artifacts( - arch=args.arch, - target=args.target, - tool_chain_tag=args.tool_chain_tag, - build_dir=args.build_dir, - out_dir=args.out_dir, - job_name=args.job_name or "local", - ) - - -if __name__ == "__main__": - main() diff --git a/.azuredevops/pipelines/OneBranch/scripts/tests/test_get_vpack_name.py b/.azuredevops/pipelines/OneBranch/scripts/tests/test_get_vpack_name.py deleted file mode 100644 index 6323128f5f..0000000000 --- a/.azuredevops/pipelines/OneBranch/scripts/tests/test_get_vpack_name.py +++ /dev/null @@ -1,111 +0,0 @@ -""" -Pytest tests for get_vpack_name.py. - -These tests lock the case-sensitive vpack naming rules so they cannot -silently drift from the YAML's inline ob_createvpack_packagename composition. - -Run locally: - pytest .azuredevops/pipelines/OneBranch/scripts/tests/test_get_vpack_name.py -""" - -from __future__ import annotations - -import sys -from pathlib import Path - -import pytest - -sys.path.insert(0, str(Path(__file__).parent.parent)) - -from get_vpack_name import get_vpack_name # noqa: E402 - - -class TestProductionVS2022: - """Production rows: Official pipeline, VS2022, no salt — no .Test suffix.""" - - def test_official_x64_debug_no_test_suffix(self) -> None: - assert get_vpack_name("MsvmPkgX64", "Official", "X64", "DEBUG") == "MsvmPkgX64.mscoreuefi.X64.DEBUG" - - def test_official_x64_release_no_test_suffix(self) -> None: - assert get_vpack_name("MsvmPkgX64", "Official", "X64", "RELEASE") == "MsvmPkgX64.mscoreuefi.X64.RELEASE" - - -class TestPRPipeline: - """PR pipeline always produces .Test names (with and without salt).""" - - def test_pr_no_salt_has_pr_prefix_and_test(self) -> None: - assert ( - get_vpack_name("MsvmPkgX64", "PR", "X64", "RELEASE") - == "MsvmPkgX64.mscoreuefi.PR.X64.RELEASE.Test" - ) - - def test_pr_with_salt_exactly_one_test(self) -> None: - assert ( - get_vpack_name("MsvmPkgX64", "PR", "X64", "DEBUG", "XDCV") - == "MsvmPkgX64.mscoreuefi.PR.X64.DEBUG.XDCV.Test" - ) - - -class TestOfficialSaltedVariants: - """Official pipeline + salt -> .Test (non-shippable side-by-side variants).""" - - def test_official_salt_xdcv(self) -> None: - assert ( - get_vpack_name("MsvmPkgX64", "Official", "X64", "DEBUG", "XDCV") - == "MsvmPkgX64.mscoreuefi.X64.DEBUG.XDCV.Test" - ) - - def test_official_salt_arcv_aarch64(self) -> None: - assert ( - get_vpack_name("MsvmPkgX64", "Official", "AARCH64", "RELEASE", "ARCV") - == "MsvmPkgX64.mscoreuefi.AARCH64.RELEASE.ARCV.Test" - ) - - def test_official_salt_xrg_linux_gcc(self) -> None: - assert ( - get_vpack_name("MsvmPkgX64", "Official", "X64", "RELEASE", "XRG") - == "MsvmPkgX64.mscoreuefi.X64.RELEASE.XRG.Test" - ) - - def test_official_salt_adcw_aarch64_debug(self) -> None: - """AARCH64 ClangWindows DEBUG gets salt ADCW.""" - assert ( - get_vpack_name("MsvmPkgX64", "Official", "AARCH64", "DEBUG", "ADCW") - == "MsvmPkgX64.mscoreuefi.AARCH64.DEBUG.ADCW.Test" - ) - - def test_official_salt_arcw_aarch64_release(self) -> None: - """AARCH64 ClangWindows RELEASE gets salt ARCW.""" - assert ( - get_vpack_name("MsvmPkgX64", "Official", "AARCH64", "RELEASE", "ARCW") - == "MsvmPkgX64.mscoreuefi.AARCH64.RELEASE.ARCW.Test" - ) - - -class TestCaseSensitivity: - """Arch, target, salt, and repo name are passed through as-is.""" - - def test_preserves_aarch64_casing(self) -> None: - assert get_vpack_name("MsvmPkgX64", "Official", "AARCH64", "RELEASE") == "MsvmPkgX64.mscoreuefi.AARCH64.RELEASE" - - def test_preserves_repo_name_casing(self) -> None: - assert get_vpack_name("WeirdCaseRepo", "Official", "X64", "DEBUG") == "WeirdCaseRepo.mscoreuefi.X64.DEBUG" - - -class TestInvariants: - @pytest.mark.parametrize( - "pipeline,salt", - [ - ("Official", ""), - ("Official", "XDCV"), - ("PR", ""), - ("PR", "XDCV"), - ], - ) - def test_test_suffix_appears_at_most_once(self, pipeline: str, salt: str) -> None: - name = get_vpack_name("R", pipeline, "X64", "DEBUG", salt) # type: ignore[arg-type] - assert name.count(".Test") <= 1, f"Multiple .Test in: {name!r}" - - def test_pr_prefix_iff_pr_pipeline(self) -> None: - assert ".mscoreuefi.PR." in get_vpack_name("R", "PR", "X64", "DEBUG") - assert ".mscoreuefi.PR." not in get_vpack_name("R", "Official", "X64", "DEBUG") diff --git a/.azuredevops/pipelines/OneBranch/scripts/tests/test_source_checks.py b/.azuredevops/pipelines/OneBranch/scripts/tests/test_source_checks.py deleted file mode 100644 index 39f77d2de5..0000000000 --- a/.azuredevops/pipelines/OneBranch/scripts/tests/test_source_checks.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Verify source-check orchestration and the enforced pipeline contract.""" - -from __future__ import annotations - -import importlib.util -import subprocess -import sys -from pathlib import Path - -import pytest -import yaml - -sys.path.insert(0, str(Path(__file__).parent.parent)) - -import source_checks - - -_ROOT = Path(__file__).resolve().parents[5] -_ONEBRANCH = _ROOT / ".azuredevops/pipelines/OneBranch" -_CHECK_COMMAND = [ - "stuart_ci_build", "-c", ".pytool/CISettings.py", - "-p", "MsvmPkg", "-t", "NO-TARGET", "-a", "X64,AARCH64", - "--disable-all", "GuidCheck=run", "LineEndingCheck=run", "UncrustifyCheck=run", -] - - -@pytest.mark.parametrize("setup", [False, True]) -def test_commands(monkeypatch: pytest.MonkeyPatch, setup: bool) -> None: - commands: list[list[str]] = [] - directories: list[Path] = [] - monkeypatch.setattr(sys, "argv", ["source_checks.py"] + (["--setup"] if setup else [])) - monkeypatch.setattr(source_checks, "run", lambda label, command: commands.append(command)) - monkeypatch.setattr(source_checks.os, "chdir", lambda path: directories.append(path)) - source_checks.main() - expected = [] - if setup: - expected.extend([ - [sys.executable, "-m", "pip", "install", "-r", "pip-requirements.txt"], - ["git", "-c", "core.autocrlf=false", "submodule", "update", "--init"], - ]) - expected.extend([["stuart_update", "-c", ".pytool/CISettings.py"], _CHECK_COMMAND]) - assert commands == expected - assert directories == [_ROOT] - - -@pytest.mark.parametrize("failed_step", range(4)) -def test_failure_stops_and_propagates(monkeypatch: pytest.MonkeyPatch, failed_step: int) -> None: - commands: list[list[str]] = [] - monkeypatch.setattr(sys, "argv", ["source_checks.py", "--setup"]) - monkeypatch.setattr(source_checks.os, "chdir", lambda path: None) - - def fail_selected_step(label: str, command: list[str]) -> None: - commands.append(command) - if len(commands) == failed_step + 1: - raise subprocess.CalledProcessError(7, command) - - monkeypatch.setattr(source_checks, "run", fail_selected_step) - with pytest.raises(SystemExit) as result: - source_checks.main() - assert result.value.code == 7 - assert len(commands) == failed_step + 1 - - -def test_pipeline_contract() -> None: - common = yaml.safe_load((_ONEBRANCH / "common.yaml").read_text()) - templates = common["extends"]["parameters"]["stages"][0]["jobs"] - assert templates[0] == { - "template": ".azuredevops/pipelines/OneBranch/source-checks.yml@self", - } - builds = yaml.safe_load((_ONEBRANCH / "jobs.yml").read_text()) - build_job = next(iter(builds["jobs"][0].values()))[0] - assert build_job["dependsOn"] == "SourceChecks" - assert build_job.get("condition", "succeeded()") == "succeeded()" - job = yaml.safe_load((_ONEBRANCH / "source-checks.yml").read_text())["jobs"][0] - assert job["job"] == "SourceChecks" - assert not job.get("continueOnError", False) - assert job["variables"]["GIT_CONFIG_COUNT"] == "2" - assert job["variables"]["GIT_CONFIG_KEY_0"] == "core.autocrlf" - assert job["variables"]["GIT_CONFIG_VALUE_0"] == "false" - assert job["variables"]["GIT_CONFIG_KEY_1"] == "core.longpaths" - assert job["variables"]["GIT_CONFIG_VALUE_1"] == "true" - assert job["variables"]["ob_createvpack_enabled"] is False - assert job["variables"]["ob_nugetPublishing_enabled"] is False - steps = job["steps"] - check = next(step for step in steps if step.get("script", "").endswith("source_checks.py --setup")) - assert not check.get("continueOnError", False) - results = next(step for step in steps if step.get("task") == "PublishTestResults@2") - assert results["condition"] == "succeededOrFailed()" - assert results["inputs"]["testResultsFiles"] == "Build/TestSuites.xml" - assert results["inputs"]["failTaskOnFailedTests"] is True - assert not results.get("continueOnError", False) - staging = next(step for step in steps if step.get("task") == "CopyFiles@2") - assert staging["condition"] == "succeededOrFailed()" - assert staging["continueOnError"] is True - assert "SETUPLOG.*" in staging["inputs"]["Contents"].splitlines() - assert steps[-1]["script"].endswith("set_repo_auth.py unset") - assert steps[-1]["condition"] == "always()" - - -def test_no_checks_are_audit_only() -> None: - policy = yaml.safe_load((_ROOT / "MsvmPkg/MsvmPkg.ci.yaml").read_text()) - assert policy["UncrustifyCheck"]["AuditOnly"] is False - assert not policy["GuidCheck"].get("AuditOnly", False) - assert not policy["LineEndingCheck"].get("AuditOnly", False) - - -def test_settings_use_upstream_workspace_relative_package_paths(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(_ONEBRANCH) - spec = importlib.util.spec_from_file_location("msvm_ci_settings", _ROOT / ".pytool/CISettings.py") - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - settings = module.Settings() - assert Path(settings.GetWorkspaceRoot()) == _ROOT - submodules = ("MU_BASECORE", "Common/MU", "Feature/DEBUGGER", "Common/PATINA_EDK2") - assert settings.GetPackagesPath() == (".", *submodules) - assert isinstance(settings, module.SetupSettingsManager) - required = settings.GetRequiredSubmodules() - assert tuple(submodule.path for submodule in required) == submodules - assert all(submodule.recursive for submodule in required) \ No newline at end of file diff --git a/.azuredevops/pipelines/OneBranch/scripts/tests/verify_ci.py b/.azuredevops/pipelines/OneBranch/scripts/tests/verify_ci.py deleted file mode 100644 index 7ac4f922e3..0000000000 --- a/.azuredevops/pipelines/OneBranch/scripts/tests/verify_ci.py +++ /dev/null @@ -1,332 +0,0 @@ -""" -Local verification sweep for the Hyper-V UEFI CI pipeline scripts. - -Runs every check that can be done without an ADO runner: - 1. YAML parse -- every *.yml / *.yaml under OneBranch/ - 2. Python syntax -- every *.py under scripts/ - 3. mypy -- type-check scripts/ (skipped if mypy not installed) - 4. pytest -- unit tests in tests/ - 5. CLI validation -- each main script rejects missing required args (exit 1 or 2) - 6. Vpack-name parity -- jobs.yml/CI-Build.yaml/PR-Build.yaml vpack composition - definitions + data/matrix.yml agree with get_vpack_name() - 7. F401 lint -- optional dead-import check via ruff (skipped if unavailable) - -Each section prints PASS / FAIL / SKIP. Exits 0 only if all sections pass. - -NOTE: This script does not validate the ADO pipeline itself. To verify the -full expanded pipeline, queue a PR build and inspect the YAML in the ADO UI. - -Usage: - python .azuredevops/pipelines/OneBranch/scripts/tests/verify_ci.py -""" - -from __future__ import annotations - -import py_compile -import subprocess -import sys -from pathlib import Path - -# --------------------------------------------------------------------------- -# Directory layout (relative to this file's position) -# --------------------------------------------------------------------------- - -_TESTS_DIR = Path(__file__).parent # .../scripts/tests/ -_SCRIPTS_DIR = _TESTS_DIR.parent # .../scripts/ -_ONEBRANCH_DIR = _SCRIPTS_DIR.parent # .../OneBranch/ -_REPO_ROOT = _ONEBRANCH_DIR.parent.parent.parent # workspace root - -sys.path.insert(0, str(_SCRIPTS_DIR)) - -# --------------------------------------------------------------------------- -# Section helpers -# --------------------------------------------------------------------------- - -_results: dict[str, bool] = {} - - -def _header(title: str) -> None: - print(f"\n{'=' * 60}") - print(f" {title}") - print("=" * 60) - - -# --------------------------------------------------------------------------- -# YAML parse -# --------------------------------------------------------------------------- - - -def _section_yaml_parse() -> bool: - _header("YAML parse") - try: - import yaml # noqa: PLC0415 - except ImportError: - print("SKIP: pyyaml not installed (run: pip install pyyaml).") - return True # non-blocking - - fail = 0 - for f in sorted(_ONEBRANCH_DIR.rglob("*.yml")) + sorted(_ONEBRANCH_DIR.rglob("*.yaml")): - try: - yaml.safe_load(f.read_text()) - print(f" PASS: {f.relative_to(_REPO_ROOT)}") - except Exception as exc: - print(f" FAIL: {f.relative_to(_REPO_ROOT)} -- {exc}") - fail += 1 - return fail == 0 - - -# --------------------------------------------------------------------------- -# Python syntax -# --------------------------------------------------------------------------- - - -def _section_python_syntax() -> bool: - _header("Python syntax (py_compile)") - fail = 0 - for f in sorted(_SCRIPTS_DIR.rglob("*.py")): - rel = f.relative_to(_REPO_ROOT) - try: - py_compile.compile(str(f), doraise=True) - print(f" PASS: {rel}") - except py_compile.PyCompileError as exc: - print(f" FAIL: {rel} -- {exc}") - fail += 1 - return fail == 0 - - -# --------------------------------------------------------------------------- -# mypy type check -# --------------------------------------------------------------------------- - - -def _section_mypy() -> bool: - _header("mypy type check") - try: - result = subprocess.run( - ["mypy", "--ignore-missing-imports", str(_SCRIPTS_DIR)], - capture_output=True, - text=True, - ) - except FileNotFoundError: - print(" SKIP: mypy not installed (run: pip install mypy)") - return True # non-blocking - if result.returncode == 0: - print(" PASS: 0 type errors") - return True - if "No module named mypy" in (result.stderr + result.stdout): - print(" SKIP: mypy not installed (run: pip install mypy)") - return True # non-blocking - print(" FAIL: mypy reported errors:") - for line in result.stdout.splitlines(): - print(f" {line}") - return False - - -# --------------------------------------------------------------------------- -# pytest unit tests -# --------------------------------------------------------------------------- - - -def _section_pytest() -> bool: - _header("pytest unit tests") - result = subprocess.run( - [sys.executable, "-m", "pytest", str(_TESTS_DIR), "-v", "--tb=short"], - cwd=_REPO_ROOT, - ) - return result.returncode == 0 - - -# --------------------------------------------------------------------------- -# CLI validation -# --------------------------------------------------------------------------- - - -def _section_cli_validation() -> bool: - _header("CLI validation (scripts must reject missing required args)") - cases = [ - "set_repo_auth.py", - "install_toolchain.py", - "build.py", - "stage_artifacts.py", - "get_vpack_name.py", - ] - fail = 0 - for script_name in cases: - result = subprocess.run( - [sys.executable, str(_SCRIPTS_DIR / script_name)], - capture_output=True, - ) - # argparse exits 2 for missing required args; sys.exit(1) exits 1. - if result.returncode in (1, 2): - print(f" PASS: {script_name} rejects missing required args (exit {result.returncode})") - else: - print(f" FAIL: {script_name} accepted empty args (exit {result.returncode})") - fail += 1 - return fail == 0 - - -# --------------------------------------------------------------------------- -# Vpack-name parity -# --------------------------------------------------------------------------- - - -def _section_vpack_parity() -> bool: - _header("Vpack-name parity (matrix.yml inline composition vs. get_vpack_name.py)") - try: - import yaml # noqa: PLC0415 - except ImportError: - print(" SKIP: pyyaml not installed.") - return True - - from get_vpack_name import get_vpack_name # noqa: PLC0415 - - jobs_text = (_ONEBRANCH_DIR / "jobs.yml").read_text() - ci_yaml = yaml.safe_load((_ONEBRANCH_DIR / "CI-Build.yaml").read_text()) - pr_yaml = yaml.safe_load((_ONEBRANCH_DIR / "PR-Build.yaml").read_text()) - matrix_path = _ONEBRANCH_DIR / "data" / "matrix.yml" - matrix = yaml.safe_load(matrix_path.read_text()) - builds_param = next(p for p in matrix["parameters"] if p["name"] == "Builds") - builds: list[dict] = builds_param["default"] - - repo_name = "MsvmPkgX64" - fail = 0 - total = 0 - - # Assert the YAML sources still use the expected composition primitives, - # so this check fails if the pipeline expression drifts from the Python impl. - expected_dot_salt = "${{iif(eq(Build.Salt, ''), '', format('.{0}', Build.Salt))}}" - expected_dot_test = "${{iif(or(ne(Build.Salt, ''), eq(parameters.pipeline, 'PR')), '.Test', '')}}" - expected_packagename = "$(Build.Repository.Name).mscoreuefi.$(PRdot)${{Build.Arch}}.${{Build.Target}}$(DotSalt)$(DotTest)" - for label, needle in ( - ("DotSalt expression", expected_dot_salt), - ("DotTest expression", expected_dot_test), - ("ob_createvpack_packagename", expected_packagename), - ): - if needle not in jobs_text: - print(f" FAIL: jobs.yml missing expected {label}: {needle}") - return False - - for row in builds: - extra = str(row.get("Extra", "")) - if "BLD_*_USE_LEGACY_C_CORE=TRUE" not in extra or "BLD_*_USE_LEGACY_C_CORE=FALSE" in extra: - print(f" FAIL: matrix row has incorrect core selection: {row}") - return False - - def _get_prdot(doc: dict) -> str: - for entry in doc.get("variables", []): - if isinstance(entry, dict) and entry.get("name") == "PRdot": - return str(entry.get("value", "")) - raise ValueError("PRdot variable not found") - - try: - prdot_by_pipeline = { - "Official": _get_prdot(ci_yaml), - "PR": _get_prdot(pr_yaml), - } - except ValueError as exc: - print(f" FAIL: {exc}") - return False - - if prdot_by_pipeline["Official"] != "": - print(f" FAIL: CI-Build.yaml PRdot expected '' but found {prdot_by_pipeline['Official']!r}") - return False - if prdot_by_pipeline["PR"] != "PR.": - print(f" FAIL: PR-Build.yaml PRdot expected 'PR.' but found {prdot_by_pipeline['PR']!r}") - return False - - for row in builds: - for pipeline in ("Official", "PR"): - total += 1 - salt = str(row.get("Salt", "")) - arch = row["Arch"] - target = row["Target"] - - # Recompose using values sourced from pipeline YAML definitions. - pr_dot = prdot_by_pipeline[pipeline] - dot_salt = f".{salt}" if salt else "" - dot_test = ".Test" if (salt or pipeline == "PR") else "" - yaml_name = f"{repo_name}.mscoreuefi.{pr_dot}{arch}.{target}{dot_salt}{dot_test}" - - spec_name = get_vpack_name(repo_name, pipeline, arch, target, salt) # type: ignore[arg-type] - - if yaml_name == spec_name: - print(f" PASS: {yaml_name}") - else: - print(f" FAIL: yaml_name={yaml_name!r} spec_name={spec_name!r}") - fail += 1 - - print(f"\n {total - fail}/{total} parity checks passed.") - return fail == 0 - - -# --------------------------------------------------------------------------- -# F401 lint (optional) -# --------------------------------------------------------------------------- - - -def _section_f401_lint() -> bool: - _header("F401 lint (ruff optional)") - try: - result = subprocess.run( - [sys.executable, "-m", "ruff", "check", "--select", "F401", str(_SCRIPTS_DIR)], - capture_output=True, - text=True, - ) - except FileNotFoundError: - print(" SKIP: python executable not found") - return True - - if result.returncode == 0: - print(" PASS: no unused imports") - return True - - stderr_stdout = (result.stderr + result.stdout).lower() - if "no module named ruff" in stderr_stdout: - print(" SKIP: ruff not installed (run: pip install ruff)") - return True - - print(" FAIL: ruff found issues:") - for line in result.stdout.splitlines(): - print(f" {line}") - return False - - -# --------------------------------------------------------------------------- -# main -# --------------------------------------------------------------------------- - - -def main() -> None: - sections = [ - ("YAML parse", _section_yaml_parse), - ("Python syntax", _section_python_syntax), - ("mypy", _section_mypy), - ("pytest", _section_pytest), - ("CLI validation", _section_cli_validation), - ("Vpack-name parity", _section_vpack_parity), - ("F401 lint", _section_f401_lint), - ] - - for title, fn in sections: - _results[title] = fn() - - print(f"\n{'=' * 60}") - print(" Overall") - print("=" * 60) - all_pass = True - for name, passed in _results.items(): - status = "PASS" if passed else "FAIL" - print(f" {status}: {name}") - if not passed: - all_pass = False - - if all_pass: - print("\nAll local verification sections PASSED.") - sys.exit(0) - else: - print("\nOne or more sections FAILED.") - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/.azuredevops/pipelines/OneBranch/source-checks.yml b/.azuredevops/pipelines/OneBranch/source-checks.yml deleted file mode 100644 index 8cb2412f18..0000000000 --- a/.azuredevops/pipelines/OneBranch/source-checks.yml +++ /dev/null @@ -1,59 +0,0 @@ -jobs: -- job: SourceChecks - displayName: Stuart Source Checks - pool: - type: windows - variables: - WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest - ScriptsDir: .azuredevops/pipelines/OneBranch/scripts - GIT_TERMINAL_PROMPT: '0' - GIT_CONFIG_COUNT: '2' - GIT_CONFIG_KEY_0: core.autocrlf - GIT_CONFIG_VALUE_0: 'false' - GIT_CONFIG_KEY_1: core.longpaths - GIT_CONFIG_VALUE_1: 'true' - ob_outputDirectory: $(Build.ArtifactStagingDirectory)/SOURCE_CHECKS - ob_nugetPublishing_enabled: false - ob_createvpack_enabled: false - steps: - - task: NuGetAuthenticate@1 - displayName: Authenticate NuGet Feed - inputs: {nuGetServiceConnections: WindowsPartners_NugetFeedConnection} - - - task: PipAuthenticate@1 - displayName: Authenticate Pip Feed - inputs: {artifactFeeds: $(System.TeamProject)/$(System.TeamProject)_PublicPackages} - - - script: python $(ScriptsDir)/set_repo_auth.py set - displayName: Set ADO Repo Auth - env: {SYSTEM_ACCESSTOKEN: $(System.AccessToken)} - - - script: python $(ScriptsDir)/source_checks.py --setup - displayName: Run Stuart Source Checks - - - task: PublishTestResults@2 - displayName: Publish Source Check Results - condition: succeededOrFailed() - inputs: - testResultsFormat: JUnit - testResultsFiles: Build/TestSuites.xml - testRunTitle: Stuart Source Checks - failTaskOnFailedTests: true - - - task: CopyFiles@2 - displayName: Stage Source Check Logs - condition: succeededOrFailed() - continueOnError: true - inputs: - SourceFolder: Build - Contents: | - CI_BUILDLOG.* - SETUPLOG.* - UPDATE_LOG.* - TestSuites.xml - MsvmPkg/** - TargetFolder: $(ob_outputDirectory) - - - script: python $(ScriptsDir)/set_repo_auth.py unset - displayName: Unset ADO Repo Auth - condition: always() diff --git a/.azuredevops/pipelines/Sync/README.md b/.azuredevops/pipelines/Sync/README.md deleted file mode 100644 index 5bcca460e0..0000000000 --- a/.azuredevops/pipelines/Sync/README.md +++ /dev/null @@ -1 +0,0 @@ -# TODO: Introduce Sync-MuMsvm-HyperVUEFI pipeline \ No newline at end of file diff --git a/.github/actions/.gitkeep b/.github/actions/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/actions/build-platform/action.yml b/.github/actions/build-platform/action.yml deleted file mode 100644 index 3e65bf5de0..0000000000 --- a/.github/actions/build-platform/action.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Build MSVM Platform -description: Build MSVM firmware with Stuart - -inputs: - architecture: - description: Target architecture - required: true - c-core: - description: Whether to use the legacy C core - required: true - official-build: - description: Set to 1 to mark the build as official in the firmware version record - required: false - default: "" - platform-config: - description: Path to the Stuart platform configuration - required: false - default: MsvmPkg/PlatformBuild.py - target: - description: Build target - required: true - tool-chain: - description: EDK II tool chain tag - required: true - -runs: - using: composite - steps: - - name: Stuart setup - shell: pwsh - run: >- - stuart_setup - -c "${{ inputs.platform-config }}" - TOOL_CHAIN_TAG=${{ inputs.tool-chain }} - - - name: Stuart update - shell: pwsh - run: >- - stuart_update - -c "${{ inputs.platform-config }}" - TOOL_CHAIN_TAG=${{ inputs.tool-chain }} - - - name: Stuart build - shell: pwsh - env: - OFFICIAL_BUILD: ${{ inputs.official-build }} - run: >- - stuart_build - -c "${{ inputs.platform-config }}" - --verbose - TOOL_CHAIN_TAG=${{ inputs.tool-chain }} - TARGET=${{ inputs.target }} - BUILD_ARCH=${{ inputs.architecture }} - BLD_*_USE_LEGACY_C_CORE=${{ inputs.c-core }} diff --git a/.github/actions/openvmm-test/action.yml b/.github/actions/openvmm-test/action.yml deleted file mode 100644 index e1f1c5af48..0000000000 --- a/.github/actions/openvmm-test/action.yml +++ /dev/null @@ -1,256 +0,0 @@ -name: Test MSVM Firmware with OpenVMM -description: Boot every X64 MSVM firmware artifact with OpenVMM - -inputs: - github-token: - description: GitHub token used to query and download OpenVMM artifacts - required: true - -runs: - using: composite - steps: - - name: Download all X64 firmware artifacts - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - pattern: firmware-*-X64-* - path: ${{ runner.temp }}/firmware - - - name: Verify X64 firmware matrix - shell: bash - run: | - mapfile -t firmware_files < <( - find "${{ runner.temp }}/firmware" -type f -name MSVM.fd | sort - ) - - if [[ "${#firmware_files[@]}" -eq 0 ]]; then - echo "::error::No X64 MSVM.fd files were downloaded from the completed build matrix" - exit 1 - fi - - - name: Resolve latest usable OpenVMM main run - id: openvmm - shell: bash - env: - GH_TOKEN: ${{ inputs.github-token }} - run: | - response="$(gh api \ - "repos/microsoft/openvmm/actions/workflows/openvmm-ci.yaml/runs?branch=main&event=push&status=success&per_page=20")" - - run_id="" - head_sha="" - candidates="$(jq -r '.workflow_runs[] | [.id, .head_sha] | @tsv' <<< "$response")" - while IFS=$'\t' read -r candidate_id candidate_sha; do - [[ -z "$candidate_id" ]] && continue - artifact_pages="$(gh api --paginate --slurp \ - "repos/microsoft/openvmm/actions/runs/$candidate_id/artifacts?per_page=100")" - artifacts="$(jq -c '[.[].artifacts[]]' <<< "$artifact_pages")" - - if jq -e ' - [ - "x64-linux-vmm-tests-archive", - "x64-linux-openvmm", - "x64-linux-musl-pipette" - ] as $required - | [.[] | select(.expired == false) | .name] as $available - | all($required[]; . as $name | $available | index($name) != null) - ' <<< "$artifacts" > /dev/null; then - run_id="$candidate_id" - head_sha="$candidate_sha" - break - fi - - artifact_status="$(jq -r ' - [ - "x64-linux-vmm-tests-archive", - "x64-linux-openvmm", - "x64-linux-musl-pipette" - ] as $required - | [ - $required[] as $name - | if any(.[]; .name == $name and .expired == false) then empty - elif any(.[]; .name == $name) then "\($name) (expired)" - else "\($name) (missing)" - end - ] - | join(", ") - ' <<< "$artifacts")" - echo "Skipping OpenVMM run $candidate_id: $artifact_status" - done <<< "$candidates" - - if [[ -z "$run_id" || -z "$head_sha" ]]; then - echo "::error::No successful OpenVMM main run has all required artifacts" - exit 1 - fi - - echo "run-id=$run_id" >> "$GITHUB_OUTPUT" - echo "head-sha=$head_sha" >> "$GITHUB_OUTPUT" - echo "Using OpenVMM run $run_id at $head_sha" - - - name: Download OpenVMM X64 test archive - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - repository: microsoft/openvmm - run-id: ${{ steps.openvmm.outputs.run-id }} - github-token: ${{ inputs.github-token }} - name: x64-linux-vmm-tests-archive - path: ${{ runner.temp }}/openvmm-bundle - - - name: Download OpenVMM X64 binary - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - repository: microsoft/openvmm - run-id: ${{ steps.openvmm.outputs.run-id }} - github-token: ${{ inputs.github-token }} - name: x64-linux-openvmm - path: ${{ runner.temp }}/openvmm-bundle - - - name: Download OpenVMM X64 pipette - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - repository: microsoft/openvmm - run-id: ${{ steps.openvmm.outputs.run-id }} - github-token: ${{ inputs.github-token }} - name: x64-linux-musl-pipette - path: ${{ runner.temp }}/openvmm-bundle - - - name: Checkout matching OpenVMM source metadata - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: microsoft/openvmm - ref: ${{ steps.openvmm.outputs.head-sha }} - path: openvmm-source - persist-credentials: false - - - name: Download standalone cargo-nextest - id: nextest - shell: bash - run: | - version="$(sed -n 's/^pub const NEXTEST: &str = "\([^"]*\)";/\1/p' \ - openvmm-source/flowey/flowey_lib_hvlite/src/_jobs/cfg_versions.rs)" - - if [[ -z "$version" ]]; then - echo "::error::Could not determine OpenVMM cargo-nextest version" - exit 1 - fi - - destination="${{ runner.temp }}/nextest" - mkdir -p "$destination" - curl --fail --location \ - "https://get.nexte.st/$version/x86_64-unknown-linux-gnu.tar.gz" \ - --output "${{ runner.temp }}/nextest.tar.gz" - tar -xf "${{ runner.temp }}/nextest.tar.gz" -C "$destination" - chmod +x "$destination/cargo-nextest" - echo "binary=$destination/cargo-nextest" >> "$GITHUB_OUTPUT" - - - name: Download Ubuntu 25.04 test image - shell: bash - run: | - image_dir="${{ runner.temp }}/openvmm-images" - image="$image_dir/ubuntu-25.04-server-cloudimg-amd64.vhd" - expected_size=3758211584 - - mkdir -p "$image_dir" - curl --fail --location --retry 5 --retry-all-errors \ - "https://hvlitetestvhds.blob.core.windows.net/vhds/ubuntu-25.04-server-cloudimg-amd64.vhd" \ - --output "$image" - - actual_size="$(stat --format=%s "$image")" - if [[ "$actual_size" -ne "$expected_size" ]]; then - echo "::error::Unexpected Ubuntu VHD size: expected $expected_size, found $actual_size" - exit 1 - fi - - - name: Prepare Linux virtualization - shell: bash - run: | - if [[ ! -c /dev/kvm ]]; then - echo "::error::The runner does not expose /dev/kvm" - exit 1 - fi - - sudo chmod a+rw /dev/kvm - echo 4096 | sudo tee \ - /sys/kernel/mm/hugepages/hugepages-2048kB/nr_overcommit_hugepages - - bundle="${{ runner.temp }}/openvmm-bundle" - echo "Downloaded OpenVMM bundle:" - find "$bundle" -maxdepth 4 -printf '%y %p\n' | sort - - normalize_bundle_file() { - local name="$1" - local destination="$bundle/$name" - local matches=() - - mapfile -d '' matches < <(find "$bundle" -type f -name "$name" -print0) - if [[ "${#matches[@]}" -ne 1 ]]; then - echo "::error::Expected exactly one $name in the OpenVMM bundle; found ${#matches[@]}" - printf 'Candidate: %s\n' "${matches[@]}" - return 1 - fi - - if [[ "${matches[0]}" != "$destination" ]]; then - cp "${matches[0]}" "$destination" - fi - } - - normalize_bundle_file openvmm - normalize_bundle_file pipette - normalize_bundle_file vmm_tests.tar.zst - chmod +x "$bundle/openvmm" "$bundle/pipette" - - - name: Boot Ubuntu with every X64 firmware - working-directory: openvmm-source - shell: bash - env: - BUNDLE: ${{ runner.temp }}/openvmm-bundle - FIRMWARE_ROOT: ${{ runner.temp }}/firmware - NEXTEST: ${{ steps.nextest.outputs.binary }} - RESULTS_ROOT: ${{ runner.temp }}/openvmm-test-results - VMM_TEST_IMAGES: ${{ runner.temp }}/openvmm-images - OPENVMM_REQUIRE_2MB_HUGETLB: "1" - run: | - mapfile -t firmware_files < <( - find "$FIRMWARE_ROOT" -type f -name MSVM.fd | sort - ) - - destination="$BUNDLE/hyperv.uefi.mscoreuefi.x64.RELEASE/MsvmX64/RELEASE_VS2022/FV" - mkdir -p "$destination" "$RESULTS_ROOT" - overall_result=0 - - for firmware in "${firmware_files[@]}"; do - relative="${firmware#"$FIRMWARE_ROOT"/}" - variant="${relative%%/*}" - result_dir="$RESULTS_ROOT/$variant" - temp_dir="${{ runner.temp }}/openvmm-test-temp/$variant" - - mkdir -p "$result_dir" "$temp_dir" - cp "$firmware" "$destination/MSVM.fd" - - echo "::group::Testing $variant" - if VMM_TESTS_CONTENT_DIR="$BUNDLE" \ - TEST_OUTPUT_PATH="$result_dir" \ - TMPDIR="$temp_dir" \ - "$NEXTEST" nextest run \ - --archive-file "$BUNDLE/vmm_tests.tar.zst" \ - --workspace-remap "$PWD" \ - --config-file "$PWD/.config/nextest.toml" \ - --profile ci \ - --filter-expr \ - "test(openvmm_uefi_x64_ubuntu_2504_server_x64_boot)"; then - echo "$variant: passed" - else - echo "::error::$variant failed" - overall_result=1 - fi - echo "::endgroup::" - done - - exit "$overall_result" - - - name: Upload OpenVMM test results - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: openvmm-x64-firmware-test-results - if-no-files-found: warn - path: ${{ runner.temp }}/openvmm-test-results/ diff --git a/.github/actions/release/action.yml b/.github/actions/release/action.yml deleted file mode 100644 index 79372b7be1..0000000000 --- a/.github/actions/release/action.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Release MSVM Firmware -description: Package MSVM firmware artifacts and publish a GitHub release - -inputs: - base-version: - description: Major and minor release version - required: true - github-token: - description: GitHub token used to create the release - required: true - -runs: - using: composite - steps: - - name: Download build artifacts - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - pattern: firmware-* - path: artifacts - - - name: Package release artifacts - shell: bash - run: | - mkdir -p release - for directory in artifacts/*/; do - name="$(basename "$directory")" - tar -czf "release/${name}.tar.gz" -C "$directory" . - done - - - name: Create GitHub release - shell: bash - env: - BASE_VERSION: ${{ inputs.base-version }} - GH_TOKEN: ${{ inputs.github-token }} - run: | - patch=$(gh release list --repo "${{ github.repository }}" --limit 1000 --json tagName \ - --jq '[.[] | .tagName | select(startswith("v'"${BASE_VERSION}"'.")) | ltrimstr("v'"${BASE_VERSION}"'.") | tonumber] | max // -1 | . + 1') - tag="v${BASE_VERSION}.${patch}" - gh release create "$tag" \ - --repo "${{ github.repository }}" \ - --target "${{ github.sha }}" \ - --generate-notes \ - release/*.tar.gz diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 2c48305b7e..0000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "github-actions" - directory: "/" - groups: - github-actions: - patterns: ["*"] - schedule: - interval: "weekly" - cooldown: - default-days: 7 diff --git a/.github/workflows/.gitkeep b/.github/workflows/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/workflows/platform-ci.yml b/.github/workflows/platform-ci.yml deleted file mode 100644 index 788057f82f..0000000000 --- a/.github/workflows/platform-ci.yml +++ /dev/null @@ -1,173 +0,0 @@ -name: Platform CI - -on: - push: - branches: - - main - pull_request: - branches: - - "*" - workflow_dispatch: - -# Bump this when releasing a new major/minor version. The patch number -# auto-increments from the latest existing GitHub release with this prefix. -env: - RELEASE_BASE_VERSION: "26.0" - -jobs: - source-checks: - name: Source Checks - runs-on: windows-2022 - steps: - - name: Disable automatic line ending conversion - shell: pwsh - run: git config --global core.autocrlf false - - - name: Enable long paths - shell: pwsh - run: git config --system core.longpaths true - - - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Set up Python 3.12 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - cache: pip - cache-dependency-path: pip-requirements.txt - - - name: Install Python dependencies - shell: pwsh - run: python -m pip install -r pip-requirements.txt - - - name: Run source checks - id: source-checks - uses: microsoft/mu_devops/.github/actions/stuart-ci@131433bb4f29d68250154cc66096f04c19fe3ee9 # v18.0.7 - with: - packages: MsvmPkg - targets: NO-TARGET - architectures: X64,AARCH64 - toolchain: VS2022 - stuart-setup: true - stuart-ci-setup: false - stuart-args: >- - --disable-all - GuidCheck=run - LineEndingCheck=run - UncrustifyCheck=run - - - name: Upload source check logs - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: source-check-logs - path: ${{ steps.source-checks.outputs.log-path }} - - build: - name: ${{ matrix.target }} ${{ matrix.arch }} ${{ matrix.tools }}${{ matrix.dxe-core == 'patina' && ' DXE=patina' || '' }} - runs-on: windows-2022 - strategy: - fail-fast: false - matrix: - target: [DEBUG, RELEASE] - arch: [X64, AARCH64] - tools: [VS2022, CLANGPDB] - dxe-core: [legacy, patina] - exclude: - - arch: AARCH64 - tools: VS2022 - - tools: VS2022 - dxe-core: patina - - steps: - - name: Enable long paths - shell: pwsh - run: git config --system core.longpaths true - - - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Set up Python 3.12 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - cache: pip - cache-dependency-path: pip-requirements.txt - - - name: Install Python dependencies - shell: pwsh - run: python -m pip install -r pip-requirements.txt - - - name: Build platform - uses: ./.github/actions/build-platform - with: - architecture: ${{ matrix.arch }} - c-core: ${{ matrix.dxe-core == 'legacy' && 'TRUE' || 'FALSE' }} - # Mark main-branch builds as official in the embedded firmware version record. - official-build: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') && '1' || '' }} - target: ${{ matrix.target }} - tool-chain: ${{ matrix.tools }} - - - name: Upload build artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: firmware-${{ matrix.target }}-${{ matrix.arch }}-${{ matrix.tools }}${{ matrix.dxe-core == 'patina' && '-patina' || '' }} - if-no-files-found: error - path: | - Build/Msvm${{ matrix.arch }}/${{ matrix.target }}_${{ matrix.tools }}/FV/MSVM.fd - Build/Msvm${{ matrix.arch }}/${{ matrix.target }}_${{ matrix.tools }}/MAP/ - Build/Msvm${{ matrix.arch }}/${{ matrix.target }}_${{ matrix.tools }}/PDB/ - - - name: Upload build logs - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: logs-${{ matrix.target }}-${{ matrix.arch }}-${{ matrix.tools }}${{ matrix.dxe-core == 'patina' && '-patina' || '' }} - if-no-files-found: ignore - path: | - Build/**/BUILDLOG*.txt - Build/**/BUILD_REPORT.TXT - Build/**/OVERRIDELOG.TXT - Build/SETUP*.txt - Build/UPDATE*.txt - - openvmm-test-x64: - name: OpenVMM Test X64 - needs: build - runs-on: ubuntu-24.04 - permissions: - actions: read - contents: read - steps: - - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - - name: Test X64 firmware - uses: ./.github/actions/openvmm-test - with: - github-token: ${{ github.token }} - - release: - name: Release - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - needs: [source-checks, build, openvmm-test-x64] - runs-on: ubuntu-latest - permissions: - contents: write - concurrency: - group: release - cancel-in-progress: false - steps: - - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - - name: Release firmware - uses: ./.github/actions/release - with: - base-version: ${{ env.RELEASE_BASE_VERSION }} - github-token: ${{ github.token }} From 6802ba2d496699a7a16e40da8b6bc0649bb7e501 Mon Sep 17 00:00:00 2001 From: Maheer Aeron Date: Thu, 17 Sep 2026 18:12:59 -0700 Subject: [PATCH 03/18] ci: add Stuart source check pipelines --- .azuredevops/pipelines/platform-ci.yml | 13 +++++++ .azuredevops/pipelines/platform-pr.yml | 6 +++ .azuredevops/pipelines/{.gitkeep => sync.yml} | 0 .azuredevops/templates/platform.yml | 24 ++++++++++++ .azuredevops/templates/source-checks.yml | 39 +++++++++++++++++++ .github/actions/.gitkeep | 0 .github/actions/source-checks/action.yml | 19 +++++++++ .github/workflows/.gitkeep | 0 .github/workflows/platform-ci.yml | 38 ++++++++++++++++++ MsvmPkg/Docs/BUILDING.md | 12 ++---- 10 files changed, 142 insertions(+), 9 deletions(-) create mode 100644 .azuredevops/pipelines/platform-ci.yml create mode 100644 .azuredevops/pipelines/platform-pr.yml rename .azuredevops/pipelines/{.gitkeep => sync.yml} (100%) create mode 100644 .azuredevops/templates/platform.yml create mode 100644 .azuredevops/templates/source-checks.yml delete mode 100644 .github/actions/.gitkeep create mode 100644 .github/actions/source-checks/action.yml delete mode 100644 .github/workflows/.gitkeep create mode 100644 .github/workflows/platform-ci.yml diff --git a/.azuredevops/pipelines/platform-ci.yml b/.azuredevops/pipelines/platform-ci.yml new file mode 100644 index 0000000000..feac93ef37 --- /dev/null +++ b/.azuredevops/pipelines/platform-ci.yml @@ -0,0 +1,13 @@ +name: $(SourceBranchName)_$(Date:yyyyMMdd_HHmmss)$(Rev:.r) + +trigger: + batch: false + branches: + include: + - official/* +pr: none + +extends: + template: /.azuredevops/templates/platform.yml + parameters: + oneBranchBase: v2/Microsoft.Official.yml diff --git a/.azuredevops/pipelines/platform-pr.yml b/.azuredevops/pipelines/platform-pr.yml new file mode 100644 index 0000000000..d77490f23d --- /dev/null +++ b/.azuredevops/pipelines/platform-pr.yml @@ -0,0 +1,6 @@ +trigger: none + +extends: + template: /.azuredevops/templates/platform.yml + parameters: + oneBranchBase: v2/Microsoft.NonOfficial.yml diff --git a/.azuredevops/pipelines/.gitkeep b/.azuredevops/pipelines/sync.yml similarity index 100% rename from .azuredevops/pipelines/.gitkeep rename to .azuredevops/pipelines/sync.yml diff --git a/.azuredevops/templates/platform.yml b/.azuredevops/templates/platform.yml new file mode 100644 index 0000000000..3b4fb261d1 --- /dev/null +++ b/.azuredevops/templates/platform.yml @@ -0,0 +1,24 @@ +parameters: + - name: oneBranchBase + type: string + +resources: + repositories: + - repository: templates_onebranch + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + +extends: + template: ${{ parameters.oneBranchBase }}@templates_onebranch + parameters: + platform: + name: windows_undocked + featureFlags: + WindowsHostVersion: 1ESWindows2022 + git: + submodules: true + stages: + - stage: SourceChecks + jobs: + - template: /.azuredevops/templates/source-checks.yml@self diff --git a/.azuredevops/templates/source-checks.yml b/.azuredevops/templates/source-checks.yml new file mode 100644 index 0000000000..cf1b848be8 --- /dev/null +++ b/.azuredevops/templates/source-checks.yml @@ -0,0 +1,39 @@ +jobs: + - job: SourceChecks + displayName: Stuart CI + pool: + type: windows + variables: + WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest + GIT_CONFIG_COUNT: "2" + GIT_CONFIG_KEY_0: core.autocrlf + GIT_CONFIG_VALUE_0: "false" + GIT_CONFIG_KEY_1: core.longpaths + GIT_CONFIG_VALUE_1: "true" + ob_outputDirectory: $(Build.ArtifactStagingDirectory)/SOURCE_CHECKS + ob_nugetPublishing_enabled: false + ob_createvpack_enabled: false + steps: + - script: python -m pip install -r pip-requirements.txt + displayName: Install Python Dependencies + + - script: stuart_update -c .pytool/CISettings.py + displayName: Update Stuart Dependencies + + - script: >- + stuart_ci_build + -c .pytool/CISettings.py + --disable-all + GuidCheck=run + LineEndingCheck=run + UncrustifyCheck=run + displayName: Run Stuart CI + + - task: PublishTestResults@2 + displayName: Publish Stuart Results + condition: succeededOrFailed() + inputs: + testResultsFormat: JUnit + testResultsFiles: Build/TestSuites.xml + testRunTitle: Stuart CI + failTaskOnFailedTests: true diff --git a/.github/actions/.gitkeep b/.github/actions/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.github/actions/source-checks/action.yml b/.github/actions/source-checks/action.yml new file mode 100644 index 0000000000..f0e5bc2617 --- /dev/null +++ b/.github/actions/source-checks/action.yml @@ -0,0 +1,19 @@ +name: Run Stuart Source Checks +description: Run the mu_msvm Stuart CI plugins + +runs: + using: composite + steps: + - name: Update Stuart dependencies + shell: pwsh + run: stuart_update -c .pytool/CISettings.py + + - name: Run Stuart CI + shell: pwsh + run: >- + stuart_ci_build + -c .pytool/CISettings.py + --disable-all + GuidCheck=run + LineEndingCheck=run + UncrustifyCheck=run diff --git a/.github/workflows/.gitkeep b/.github/workflows/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.github/workflows/platform-ci.yml b/.github/workflows/platform-ci.yml new file mode 100644 index 0000000000..a0999802af --- /dev/null +++ b/.github/workflows/platform-ci.yml @@ -0,0 +1,38 @@ +name: Platform CI + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +jobs: + source-checks: + name: Source Checks + runs-on: windows-2022 + steps: + - name: Configure Git + shell: pwsh + run: | + git config --global core.autocrlf false + git config --system core.longpaths true + + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: true + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pip-requirements.txt + + - name: Install Python dependencies + shell: pwsh + run: python -m pip install -r pip-requirements.txt + + - name: Run source checks + uses: ./.github/actions/source-checks diff --git a/MsvmPkg/Docs/BUILDING.md b/MsvmPkg/Docs/BUILDING.md index 742b102ca4..c36149312b 100644 --- a/MsvmPkg/Docs/BUILDING.md +++ b/MsvmPkg/Docs/BUILDING.md @@ -36,14 +36,11 @@ python -m pip install -r pip-requirements.txt stuart_update -c .\.pytool\CISettings.py ``` -Run the same source checks enforced by GitHub Actions: +Run the same source checks enforced by platform CI: ```powershell stuart_ci_build ` -c .\.pytool\CISettings.py ` - -p MsvmPkg ` - -t NO-TARGET ` - -a X64,AARCH64 ` --disable-all ` GuidCheck=run ` LineEndingCheck=run ` @@ -54,7 +51,7 @@ The `--disable-all` option disables the default plugin set; each `Check=run` arg To run one check while iterating, specify only that check. For example: ```powershell -stuart_ci_build -c .\.pytool\CISettings.py -p MsvmPkg -t NO-TARGET -a X64,AARCH64 --disable-all GuidCheck=run +stuart_ci_build -c .\.pytool\CISettings.py --disable-all GuidCheck=run ``` ### Fixing Uncrustify Failures @@ -64,9 +61,6 @@ Run Uncrustify in correction mode to update incorrectly formatted files in place ```powershell stuart_ci_build ` -c .\.pytool\CISettings.py ` - -p MsvmPkg ` - -t NO-TARGET ` - -a X64,AARCH64 ` --disable-all ` UncrustifyCheck=run ` UNCRUSTIFY_IN_PLACE=TRUE @@ -113,7 +107,7 @@ Build artifacts follow this path structure: **Path Variables:** - `{root}` = Root directory of the UEFI project - `{architecture}` = X64, AARCH64, etc. -- `{flavor}` = DEBUG or RELEASE +- `{flavor}` = DEBUG or RELEASE - `{toolchain}` = VS2022, GCC, etc. **Examples:** From aba9ad1c67f9badd46da8f392416daf0eccf1996 Mon Sep 17 00:00:00 2001 From: Maheer Aeron Date: Thu, 17 Sep 2026 18:17:18 -0700 Subject: [PATCH 04/18] ci: share Stuart workspace setup --- .azuredevops/templates/source-checks.yml | 6 +----- .azuredevops/templates/stuart-setup.yml | 9 +++++++++ .github/actions/source-checks/action.yml | 4 ---- .github/actions/stuart-setup/action.yml | 17 +++++++++++++++++ .github/workflows/platform-ci.yml | 5 ++--- MsvmPkg/Docs/BUILDING.md | 1 + 6 files changed, 30 insertions(+), 12 deletions(-) create mode 100644 .azuredevops/templates/stuart-setup.yml create mode 100644 .github/actions/stuart-setup/action.yml diff --git a/.azuredevops/templates/source-checks.yml b/.azuredevops/templates/source-checks.yml index cf1b848be8..6e0efc5009 100644 --- a/.azuredevops/templates/source-checks.yml +++ b/.azuredevops/templates/source-checks.yml @@ -14,11 +14,7 @@ jobs: ob_nugetPublishing_enabled: false ob_createvpack_enabled: false steps: - - script: python -m pip install -r pip-requirements.txt - displayName: Install Python Dependencies - - - script: stuart_update -c .pytool/CISettings.py - displayName: Update Stuart Dependencies + - template: /.azuredevops/templates/stuart-setup.yml@self - script: >- stuart_ci_build diff --git a/.azuredevops/templates/stuart-setup.yml b/.azuredevops/templates/stuart-setup.yml new file mode 100644 index 0000000000..f585e20906 --- /dev/null +++ b/.azuredevops/templates/stuart-setup.yml @@ -0,0 +1,9 @@ +steps: + - script: python -m pip install -r pip-requirements.txt + displayName: Install Python Dependencies + + - script: stuart_setup -c .pytool/CISettings.py + displayName: Set Up Stuart Dependencies + + - script: stuart_update -c .pytool/CISettings.py + displayName: Update Stuart Dependencies diff --git a/.github/actions/source-checks/action.yml b/.github/actions/source-checks/action.yml index f0e5bc2617..7a13bb7326 100644 --- a/.github/actions/source-checks/action.yml +++ b/.github/actions/source-checks/action.yml @@ -4,10 +4,6 @@ description: Run the mu_msvm Stuart CI plugins runs: using: composite steps: - - name: Update Stuart dependencies - shell: pwsh - run: stuart_update -c .pytool/CISettings.py - - name: Run Stuart CI shell: pwsh run: >- diff --git a/.github/actions/stuart-setup/action.yml b/.github/actions/stuart-setup/action.yml new file mode 100644 index 0000000000..6b3b207bbe --- /dev/null +++ b/.github/actions/stuart-setup/action.yml @@ -0,0 +1,17 @@ +name: Set Up Stuart +description: Install Python requirements and prepare Stuart dependencies + +runs: + using: composite + steps: + - name: Install Python dependencies + shell: pwsh + run: python -m pip install -r pip-requirements.txt + + - name: Set up Stuart dependencies + shell: pwsh + run: stuart_setup -c .pytool/CISettings.py + + - name: Update Stuart dependencies + shell: pwsh + run: stuart_update -c .pytool/CISettings.py diff --git a/.github/workflows/platform-ci.yml b/.github/workflows/platform-ci.yml index a0999802af..f3b76a5845 100644 --- a/.github/workflows/platform-ci.yml +++ b/.github/workflows/platform-ci.yml @@ -30,9 +30,8 @@ jobs: cache: pip cache-dependency-path: pip-requirements.txt - - name: Install Python dependencies - shell: pwsh - run: python -m pip install -r pip-requirements.txt + - name: Set up Stuart + uses: ./.github/actions/stuart-setup - name: Run source checks uses: ./.github/actions/source-checks diff --git a/MsvmPkg/Docs/BUILDING.md b/MsvmPkg/Docs/BUILDING.md index c36149312b..06b3077b83 100644 --- a/MsvmPkg/Docs/BUILDING.md +++ b/MsvmPkg/Docs/BUILDING.md @@ -33,6 +33,7 @@ tools used by the checks on a new checkout or whenever their configuration chang ```powershell python -m pip install -r pip-requirements.txt +stuart_setup -c .\.pytool\CISettings.py stuart_update -c .\.pytool\CISettings.py ``` From a5f304dbc1accd597635a01103c8202852fc2342 Mon Sep 17 00:00:00 2001 From: Maheer Aeron Date: Thu, 17 Sep 2026 18:45:33 -0700 Subject: [PATCH 05/18] ci: centralize source checks in shared script --- .azuredevops/templates/source-checks.yml | 12 ++----- .azuredevops/templates/stuart-setup.yml | 9 ----- .github/actions/source-checks/action.yml | 12 ++----- .github/actions/stuart-setup/action.yml | 17 --------- .github/workflows/platform-ci.yml | 3 -- MsvmPkg/Docs/BUILDING.md | 22 +++--------- ci/scripts/source_checks.py | 46 ++++++++++++++++++++++++ ci/tests/test_source_checks.py | 40 +++++++++++++++++++++ 8 files changed, 95 insertions(+), 66 deletions(-) delete mode 100644 .azuredevops/templates/stuart-setup.yml delete mode 100644 .github/actions/stuart-setup/action.yml create mode 100644 ci/scripts/source_checks.py create mode 100644 ci/tests/test_source_checks.py diff --git a/.azuredevops/templates/source-checks.yml b/.azuredevops/templates/source-checks.yml index 6e0efc5009..58ff08e935 100644 --- a/.azuredevops/templates/source-checks.yml +++ b/.azuredevops/templates/source-checks.yml @@ -14,16 +14,8 @@ jobs: ob_nugetPublishing_enabled: false ob_createvpack_enabled: false steps: - - template: /.azuredevops/templates/stuart-setup.yml@self - - - script: >- - stuart_ci_build - -c .pytool/CISettings.py - --disable-all - GuidCheck=run - LineEndingCheck=run - UncrustifyCheck=run - displayName: Run Stuart CI + - script: python ci/scripts/source_checks.py + displayName: Run Source Checks - task: PublishTestResults@2 displayName: Publish Stuart Results diff --git a/.azuredevops/templates/stuart-setup.yml b/.azuredevops/templates/stuart-setup.yml deleted file mode 100644 index f585e20906..0000000000 --- a/.azuredevops/templates/stuart-setup.yml +++ /dev/null @@ -1,9 +0,0 @@ -steps: - - script: python -m pip install -r pip-requirements.txt - displayName: Install Python Dependencies - - - script: stuart_setup -c .pytool/CISettings.py - displayName: Set Up Stuart Dependencies - - - script: stuart_update -c .pytool/CISettings.py - displayName: Update Stuart Dependencies diff --git a/.github/actions/source-checks/action.yml b/.github/actions/source-checks/action.yml index 7a13bb7326..c450288362 100644 --- a/.github/actions/source-checks/action.yml +++ b/.github/actions/source-checks/action.yml @@ -1,15 +1,9 @@ name: Run Stuart Source Checks -description: Run the mu_msvm Stuart CI plugins +description: Prepare the workspace and run the mu_msvm Stuart CI plugins runs: using: composite steps: - - name: Run Stuart CI + - name: Run source checks shell: pwsh - run: >- - stuart_ci_build - -c .pytool/CISettings.py - --disable-all - GuidCheck=run - LineEndingCheck=run - UncrustifyCheck=run + run: python ci/scripts/source_checks.py diff --git a/.github/actions/stuart-setup/action.yml b/.github/actions/stuart-setup/action.yml deleted file mode 100644 index 6b3b207bbe..0000000000 --- a/.github/actions/stuart-setup/action.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Set Up Stuart -description: Install Python requirements and prepare Stuart dependencies - -runs: - using: composite - steps: - - name: Install Python dependencies - shell: pwsh - run: python -m pip install -r pip-requirements.txt - - - name: Set up Stuart dependencies - shell: pwsh - run: stuart_setup -c .pytool/CISettings.py - - - name: Update Stuart dependencies - shell: pwsh - run: stuart_update -c .pytool/CISettings.py diff --git a/.github/workflows/platform-ci.yml b/.github/workflows/platform-ci.yml index f3b76a5845..af9e43c040 100644 --- a/.github/workflows/platform-ci.yml +++ b/.github/workflows/platform-ci.yml @@ -30,8 +30,5 @@ jobs: cache: pip cache-dependency-path: pip-requirements.txt - - name: Set up Stuart - uses: ./.github/actions/stuart-setup - - name: Run source checks uses: ./.github/actions/source-checks diff --git a/MsvmPkg/Docs/BUILDING.md b/MsvmPkg/Docs/BUILDING.md index 06b3077b83..f5711a762d 100644 --- a/MsvmPkg/Docs/BUILDING.md +++ b/MsvmPkg/Docs/BUILDING.md @@ -28,28 +28,14 @@ stuart_build -c .\MsvmPkg\PlatformBuild.py BUILD_ARCH=AARCH64 ## Running CI Checks Locally -Run the source checks from the repository root before opening a pull request. Install the Python dependencies and download the -tools used by the checks on a new checkout or whenever their configuration changes: +Run the same source-check workflow used by platform CI from the repository root: ```powershell -python -m pip install -r pip-requirements.txt -stuart_setup -c .\.pytool\CISettings.py -stuart_update -c .\.pytool\CISettings.py +python .\ci\scripts\source_checks.py ``` -Run the same source checks enforced by platform CI: - -```powershell -stuart_ci_build ` - -c .\.pytool\CISettings.py ` - --disable-all ` - GuidCheck=run ` - LineEndingCheck=run ` - UncrustifyCheck=run -``` - -The `--disable-all` option disables the default plugin set; each `Check=run` argument enables a check used by this repository. -To run one check while iterating, specify only that check. For example: +The workflow installs the Python requirements, runs `stuart_setup` and `stuart_update`, and then runs the configured +`stuart_ci_build` checks. To run one check while iterating, invoke Stuart directly. For example: ```powershell stuart_ci_build -c .\.pytool\CISettings.py --disable-all GuidCheck=run diff --git a/ci/scripts/source_checks.py b/ci/scripts/source_checks.py new file mode 100644 index 0000000000..c752f574ff --- /dev/null +++ b/ci/scripts/source_checks.py @@ -0,0 +1,46 @@ +"""Prepare the workspace and run source checks using native Stuart commands.""" + +from __future__ import annotations + +import argparse +import shlex +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +CI_SETTINGS = ".pytool/CISettings.py" + + +def run(command: list[str]) -> None: + print(f"==> {shlex.join(command)}", flush=True) + subprocess.run(command, cwd=REPO_ROOT, check=True) + + +def run_source_checks() -> None: + commands = ( + [sys.executable, "-m", "pip", "install", "-r", "pip-requirements.txt"], + ["stuart_setup", "-c", CI_SETTINGS], + ["stuart_update", "-c", CI_SETTINGS], + [ + "stuart_ci_build", + "-c", + CI_SETTINGS, + "--disable-all", + "GuidCheck=run", + "LineEndingCheck=run", + "UncrustifyCheck=run", + ], + ) + for command in commands: + run(command) + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.parse_args(argv) + run_source_checks() + + +if __name__ == "__main__": + main() diff --git a/ci/tests/test_source_checks.py b/ci/tests/test_source_checks.py new file mode 100644 index 0000000000..3b3fe72ca0 --- /dev/null +++ b/ci/tests/test_source_checks.py @@ -0,0 +1,40 @@ +import importlib.util +import sys +import unittest +from pathlib import Path +from unittest.mock import call, patch + +SCRIPT_PATH = Path(__file__).parents[1] / "scripts" / "source_checks.py" +SPEC = importlib.util.spec_from_file_location("source_checks", SCRIPT_PATH) +assert SPEC is not None and SPEC.loader is not None +source_checks = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(source_checks) + + +class SourceCheckTests(unittest.TestCase): + @patch.object(source_checks.subprocess, "run") + def test_source_checks_use_native_stuart_commands(self, run_mock): + source_checks.main([]) + + expected_commands = [ + [sys.executable, "-m", "pip", "install", "-r", "pip-requirements.txt"], + ["stuart_setup", "-c", ".pytool/CISettings.py"], + ["stuart_update", "-c", ".pytool/CISettings.py"], + [ + "stuart_ci_build", + "-c", + ".pytool/CISettings.py", + "--disable-all", + "GuidCheck=run", + "LineEndingCheck=run", + "UncrustifyCheck=run", + ], + ] + self.assertEqual( + run_mock.call_args_list, + [call(command, cwd=source_checks.REPO_ROOT, check=True) for command in expected_commands], + ) + + +if __name__ == "__main__": + unittest.main() From b8312ab9c94108171bdd7bbfdf1e11982edd5fee Mon Sep 17 00:00:00 2001 From: Maheer Aeron Date: Thu, 17 Sep 2026 18:49:40 -0700 Subject: [PATCH 06/18] ci: add single-flavor Stuart build runner --- MsvmPkg/Docs/BUILDING.md | 17 +++++++++ ci/scripts/build.py | 55 ++++++++++++++++++++++++++++ ci/tests/test_build.py | 79 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 151 insertions(+) create mode 100644 ci/scripts/build.py create mode 100644 ci/tests/test_build.py diff --git a/MsvmPkg/Docs/BUILDING.md b/MsvmPkg/Docs/BUILDING.md index f5711a762d..8151dbd71d 100644 --- a/MsvmPkg/Docs/BUILDING.md +++ b/MsvmPkg/Docs/BUILDING.md @@ -26,6 +26,23 @@ stuart_build -c .\MsvmPkg\PlatformBuild.py TARGET=RELEASE stuart_build -c .\MsvmPkg\PlatformBuild.py BUILD_ARCH=AARCH64 ``` +## Building One CI Flavor Locally + +Use your activated Python virtual environment with the selected compiler installed and configured. +From the repository root, run: + +```powershell +python .\ci\scripts\build.py --arch X64 --target DEBUG --tool-chain CLANGPDB --core legacy +``` + +The script installs Python requirements, then runs `stuart_setup`, `stuart_update`, and `stuart_build` +with `MsvmPkg/PlatformBuild.py`. All three Stuart commands receive the selected flavor. +Architecture, target, toolchain, and core are required; use `--core patina` to select the Rust DXE core. +Stuart executables are resolved from the invoking Python environment, and execution stops on the first failure. + +Add `--dry-run` to print the commands without installing dependencies or starting a build. +This command builds one flavor only; it does not select an open/closed repository matrix or publish artifacts. + ## Running CI Checks Locally Run the same source-check workflow used by platform CI from the repository root: diff --git a/ci/scripts/build.py b/ci/scripts/build.py new file mode 100644 index 0000000000..7dfa60b18e --- /dev/null +++ b/ci/scripts/build.py @@ -0,0 +1,55 @@ +"""Prepare and build one firmware flavor using native Stuart commands.""" + +from __future__ import annotations + +import argparse +import os +import shlex +import subprocess +import sys +import sysconfig +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +PLATFORM_SETTINGS = "MsvmPkg/PlatformBuild.py" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--arch", required=True, choices=("X64", "AARCH64")) + parser.add_argument("--target", required=True, choices=("DEBUG", "RELEASE")) + parser.add_argument("--tool-chain", required=True, choices=("VS2022", "CLANGPDB", "GCC")) + parser.add_argument("--core", required=True, choices=("legacy", "patina")) + parser.add_argument("--dry-run", action="store_true", help="Print commands without executing them") + args = parser.parse_args(argv) + + flavor = [ + f"BUILD_ARCH={args.arch}", + f"TARGET={args.target}", + f"TOOL_CHAIN_TAG={args.tool_chain}", + f"BLD_*_USE_LEGACY_C_CORE={'TRUE' if args.core == 'legacy' else 'FALSE'}", + ] + scripts_dir = Path(sysconfig.get_path("scripts")) + suffix = ".exe" if os.name == "nt" else "" + commands = [[sys.executable, "-m", "pip", "install", "-r", "pip-requirements.txt"]] + for tool in ("stuart_setup", "stuart_update", "stuart_build"): + commands.append([str(scripts_dir / (tool + suffix)), "-c", PLATFORM_SETTINGS, *flavor]) + commands[-1].extend([ + "BUILDREPORTING=TRUE", + "BUILDREPORT_TYPES=PCD DEPEX FLASH BUILD_FLAGS LIBRARY", + "LaunchLogOnSuccess=FALSE", + "LaunchLogOnError=FALSE", + ]) + + for command in commands: + print(f"==> {shlex.join(command)}", flush=True) + if not args.dry_run: + try: + subprocess.run(command, cwd=REPO_ROOT, check=True) + except subprocess.CalledProcessError as error: + return error.returncode + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/tests/test_build.py b/ci/tests/test_build.py new file mode 100644 index 0000000000..a1bd65cf19 --- /dev/null +++ b/ci/tests/test_build.py @@ -0,0 +1,79 @@ +import importlib.util +import io +import os +import subprocess +import sys +import sysconfig +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from unittest.mock import patch + +SCRIPT_PATH = Path(__file__).parents[1] / "scripts" / "build.py" +SPEC = importlib.util.spec_from_file_location("build", SCRIPT_PATH) +assert SPEC is not None and SPEC.loader is not None +build = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(build) + + +class BuildTests(unittest.TestCase): + ARGS = ["--arch", "X64", "--target", "DEBUG", "--tool-chain", "CLANGPDB", "--core", "legacy"] + + def setUp(self): + self.output = redirect_stdout(io.StringIO()) + self.output.__enter__() + self.addCleanup(self.output.__exit__, None, None, None) + + @patch.object(build.subprocess, "run") + def test_platform_preparation_precedes_build(self, run_mock): + self.assertEqual(build.main(self.ARGS), 0) + calls = run_mock.call_args_list + self.assertEqual(len(calls), 4) + self.assertEqual(calls[0].args[0], [sys.executable, "-m", "pip", "install", "-r", "pip-requirements.txt"]) + suffix = ".exe" if os.name == "nt" else "" + for invocation, tool in zip(calls[1:], ("stuart_setup", "stuart_update", "stuart_build")): + self.assertEqual(invocation.args[0][:7], [ + str(Path(sysconfig.get_path("scripts")) / (tool + suffix)), + "-c", "MsvmPkg/PlatformBuild.py", "BUILD_ARCH=X64", "TARGET=DEBUG", + "TOOL_CHAIN_TAG=CLANGPDB", "BLD_*_USE_LEGACY_C_CORE=TRUE", + ]) + for invocation in calls: + self.assertEqual(invocation.kwargs, {"cwd": SCRIPT_PATH.resolve().parents[2], "check": True}) + self.assertIn("BUILDREPORT_TYPES=PCD DEPEX FLASH BUILD_FLAGS LIBRARY", calls[-1].args[0]) + self.assertIn("LaunchLogOnSuccess=FALSE", calls[-1].args[0]) + self.assertIn("LaunchLogOnError=FALSE", calls[-1].args[0]) + + @patch.object(build.subprocess, "run") + def test_patina_release_flavor(self, run_mock): + build.main(["--arch", "AARCH64", "--target", "RELEASE", "--tool-chain", "CLANGPDB", "--core", "patina"]) + for invocation in run_mock.call_args_list[1:]: + self.assertIn("BLD_*_USE_LEGACY_C_CORE=FALSE", invocation.args[0]) + self.assertIn("BUILD_ARCH=AARCH64", invocation.args[0]) + self.assertIn("TARGET=RELEASE", invocation.args[0]) + + @patch.object(build.subprocess, "run") + def test_failure_stops_remaining_commands(self, run_mock): + for failed_step in range(4): + with self.subTest(failed_step=failed_step): + run_mock.reset_mock() + run_mock.side_effect = [None] * failed_step + [subprocess.CalledProcessError(7, "command")] + self.assertEqual(build.main(self.ARGS), 7) + self.assertEqual(run_mock.call_count, failed_step + 1) + + @patch.object(build.subprocess, "run") + def test_dry_run_has_no_side_effects(self, run_mock): + self.assertEqual(build.main([*self.ARGS, "--dry-run"]), 0) + run_mock.assert_not_called() + + @patch.object(build.subprocess, "run") + def test_invalid_or_missing_flavor_is_rejected(self, run_mock): + for args in ([], [*self.ARGS[:-1], "unknown"]): + with self.subTest(args=args), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as error: + build.main(args) + self.assertEqual(error.exception.code, 2) + run_mock.assert_not_called() + + +if __name__ == "__main__": + unittest.main() From 34593f91225fe0e189451325ea5774c6827139fb Mon Sep 17 00:00:00 2001 From: Maheer Aeron Date: Thu, 17 Sep 2026 18:56:42 -0700 Subject: [PATCH 07/18] ci: share build matrix and wire GitHub builds --- .github/actions/build/action.yml | 44 +++++++++++++ .github/workflows/platform-ci.yml | 57 +++++++++++++++++ MsvmPkg/Docs/BUILDING.md | 30 +++++++++ ci/build-matrix.json | 18 ++++++ ci/scripts/matrix.py | 66 ++++++++++++++++++++ ci/tests/test_matrix.py | 100 ++++++++++++++++++++++++++++++ 6 files changed, 315 insertions(+) create mode 100644 .github/actions/build/action.yml create mode 100644 ci/build-matrix.json create mode 100644 ci/scripts/matrix.py create mode 100644 ci/tests/test_matrix.py diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml new file mode 100644 index 0000000000..ad6cce689c --- /dev/null +++ b/.github/actions/build/action.yml @@ -0,0 +1,44 @@ +name: Build Firmware +description: Prepare and build one firmware flavor through Stuart + +inputs: + arch: + description: Target architecture + required: true + target: + description: Build target + required: true + tool-chain: + description: EDK II toolchain tag + required: true + core: + description: DXE core (legacy or patina) + required: true + +runs: + using: composite + steps: + - name: Locate Visual Studio Clang + if: inputs.tool-chain == 'CLANGPDB' + shell: pwsh + run: | + $vswhere = "${env:ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere.exe" + $installation = & $vswhere -latest -version '[17.0,18.0)' -products '*' -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + if ($LASTEXITCODE -ne 0 -or -not $installation) { throw 'Visual Studio 2022 was not found' } + $clangBin = Join-Path $installation 'VC/Tools/Llvm/x64/bin' + if (-not (Test-Path (Join-Path $clangBin 'clang.exe'))) { throw 'Visual Studio Clang was not found' } + "CLANG_BIN=$clangBin/" >> $env:GITHUB_ENV + + - name: Build firmware + shell: pwsh + env: + BUILD_ARCH: ${{ inputs.arch }} + BUILD_TARGET: ${{ inputs.target }} + BUILD_TOOL_CHAIN: ${{ inputs.tool-chain }} + BUILD_CORE: ${{ inputs.core }} + run: >- + python ci/scripts/build.py + --arch "$env:BUILD_ARCH" + --target "$env:BUILD_TARGET" + --tool-chain "$env:BUILD_TOOL_CHAIN" + --core "$env:BUILD_CORE" diff --git a/.github/workflows/platform-ci.yml b/.github/workflows/platform-ci.yml index af9e43c040..5151c250f4 100644 --- a/.github/workflows/platform-ci.yml +++ b/.github/workflows/platform-ci.yml @@ -8,6 +8,63 @@ on: workflow_dispatch: jobs: + build-matrix: + name: Select Build Matrix + runs-on: ubuntu-24.04 + outputs: + matrix: ${{ steps.select.outputs.matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Select open-source builds + id: select + shell: bash + run: | + matrix="$(python ci/scripts/matrix.py --repo open --format github)" + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + + build: + name: Build ${{ matrix.id }} + needs: build-matrix + runs-on: windows-2022 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.build-matrix.outputs.matrix) }} + steps: + - name: Configure Git + shell: pwsh + run: | + git config --global core.autocrlf false + git config --system core.longpaths true + + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pip-requirements.txt + + - name: Build firmware + uses: ./.github/actions/build + with: + arch: ${{ matrix.arch }} + target: ${{ matrix.target }} + tool-chain: ${{ matrix.tool_chain }} + core: ${{ matrix.core }} + source-checks: name: Source Checks runs-on: windows-2022 diff --git a/MsvmPkg/Docs/BUILDING.md b/MsvmPkg/Docs/BUILDING.md index 8151dbd71d..8560be773d 100644 --- a/MsvmPkg/Docs/BUILDING.md +++ b/MsvmPkg/Docs/BUILDING.md @@ -43,6 +43,36 @@ Stuart executables are resolved from the invoking Python environment, and execut Add `--dry-run` to print the commands without installing dependencies or starting a build. This command builds one flavor only; it does not select an open/closed repository matrix or publish artifacts. +### Selecting Repository Coverage + +`ci/build-matrix.json` is the shared build list. Each row explicitly lists the repository modes that use it: +`open` for mu_msvm and `closed` for hyperv.uefi. Repository mode selects coverage, not source code, +credentials, official-build status, or publishing permissions. + +```powershell +python .\ci\scripts\matrix.py --repo open +python .\ci\scripts\matrix.py --repo closed +``` + +The open profile contains ten Windows-hosted builds: DEBUG and RELEASE for X64 VS2022 with the legacy core, +plus X64 and AARCH64 CLANGPDB with each of the legacy and Patina cores. +The closed profile still contains only X64 DEBUG CLANGPDB with the legacy core; it is an experimental +baseline, not a replacement for the full hyperv.uefi coverage policy. + +IDs are derived as `___`; do not enter IDs in the JSON. +Additional coverage can be added by editing the JSON, without duplicating flavor definitions in provider YAML. +The selector rejects duplicate flavors and fails when a requested repository has no builds. + +`--format github` emits an `include` object; `--format ado` emits a job-leg name to variables mapping. +These commands only print JSON. They do not execute builds, install tools, or publish anything. +GitHub's platform workflow selects `open` coverage, feeds the result to its native `strategy.matrix`, and +calls the build action for each flavor. That action invokes the same `ci/scripts/build.py` used locally. +Builds depend only on matrix generation; source checks run independently. Build artifact publication is not +part of this step yet. + +ADO build matrix integration remains unwired. Its JSON output format does not establish support for runtime +matrix expansion in the governed OneBranch templates; that requires validation before enabling ADO builds. + ## Running CI Checks Locally Run the same source-check workflow used by platform CI from the repository root: diff --git a/ci/build-matrix.json b/ci/build-matrix.json new file mode 100644 index 0000000000..6fe6d8d00f --- /dev/null +++ b/ci/build-matrix.json @@ -0,0 +1,18 @@ +[ + { + "repositories": ["open", "closed"], + "arch": "X64", + "target": "DEBUG", + "tool_chain": "CLANGPDB", + "core": "legacy" + }, + {"repositories": ["open"], "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "core": "legacy"}, + {"repositories": ["open"], "arch": "X64", "target": "DEBUG", "tool_chain": "VS2022", "core": "legacy"}, + {"repositories": ["open"], "arch": "X64", "target": "RELEASE", "tool_chain": "VS2022", "core": "legacy"}, + {"repositories": ["open"], "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "core": "legacy"}, + {"repositories": ["open"], "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "core": "legacy"}, + {"repositories": ["open"], "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "core": "patina"}, + {"repositories": ["open"], "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "core": "patina"}, + {"repositories": ["open"], "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "core": "patina"}, + {"repositories": ["open"], "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "core": "patina"} +] diff --git a/ci/scripts/matrix.py b/ci/scripts/matrix.py new file mode 100644 index 0000000000..f9c5153f42 --- /dev/null +++ b/ci/scripts/matrix.py @@ -0,0 +1,66 @@ +"""Select explicit repository build coverage and emit CI-provider matrix data.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +MATRIX_PATH = Path(__file__).resolve().parents[1] / "build-matrix.json" +REPOSITORIES = ("open", "closed") +FLAVOR_CHOICES = { + "arch": ("X64", "AARCH64"), + "target": ("DEBUG", "RELEASE"), + "tool_chain": ("VS2022", "CLANGPDB", "GCC"), + "core": ("legacy", "patina"), +} + + +def select_builds(repository: str, path: Path = MATRIX_PATH) -> list[dict]: + if repository not in REPOSITORIES: + raise ValueError(f"Unknown repository mode: {repository}") + rows = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(rows, list): + raise ValueError("Build matrix must be a list") + selected = [] + identifiers = set() + for row in rows: + if not isinstance(row, dict) or set(row) != {"repositories", *FLAVOR_CHOICES}: + raise ValueError("Each build must specify repositories, arch, target, tool_chain, and core") + for field, choices in FLAVOR_CHOICES.items(): + if row[field] not in choices: + raise ValueError(f"Invalid {field}: {row[field]!r}") + identifier = "_".join(row[field] for field in FLAVOR_CHOICES) + if identifier in identifiers: + raise ValueError(f"Duplicate matrix identifier: {identifier}") + identifiers.add(identifier) + repositories = row["repositories"] + if not isinstance(repositories, list) or not repositories or any(mode not in REPOSITORIES for mode in repositories): + raise ValueError(f"Invalid repositories for {identifier}") + if repository in repositories: + selected.append({"id": identifier, **{field: row[field] for field in FLAVOR_CHOICES}}) + if not selected: + raise ValueError(f"No builds configured for repository mode: {repository}") + return selected + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", required=True, choices=REPOSITORIES) + parser.add_argument("--format", choices=("json", "github", "ado"), default="json") + args = parser.parse_args(argv) + try: + rows = select_builds(args.repo) + except (OSError, ValueError) as error: + parser.error(str(error)) + if args.format == "github": + result = {"include": rows} + elif args.format == "ado": + result = {row["id"]: {key: value for key, value in row.items() if key != "id"} for row in rows} + else: + result = rows + print(json.dumps(result, separators=(",", ":"))) + + +if __name__ == "__main__": + main() diff --git a/ci/tests/test_matrix.py b/ci/tests/test_matrix.py new file mode 100644 index 0000000000..821c5c8853 --- /dev/null +++ b/ci/tests/test_matrix.py @@ -0,0 +1,100 @@ +import importlib.util +import io +import json +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from unittest.mock import patch + +SCRIPTS = Path(__file__).parents[1] / "scripts" + + +def load_script(name): + spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +matrix = load_script("matrix") +build = load_script("build") + + +class MatrixTests(unittest.TestCase): + def test_each_repository_row_is_accepted_by_build_runner(self): + for repository in matrix.REPOSITORIES: + for row in matrix.select_builds(repository): + with self.subTest(repository=repository, row=row), redirect_stdout(io.StringIO()): + args = ["--dry-run"] + for field in matrix.FLAVOR_CHOICES: + args.extend(["--" + field.replace("_", "-"), row[field]]) + with patch.object(build.subprocess, "run") as run_mock: + self.assertEqual(build.main(args), 0) + run_mock.assert_not_called() + + def test_profiles_are_filtered_explicitly(self): + row = json.loads(matrix.MATRIX_PATH.read_text())[0] + open_row = {**row, "target": "DEBUG", "repositories": ["open"]} + closed_row = {**row, "target": "RELEASE", "repositories": ["closed"]} + with patch.object(Path, "read_text", return_value=json.dumps([open_row, closed_row])): + self.assertEqual([item["id"] for item in matrix.select_builds("open")], ["X64_DEBUG_CLANGPDB_legacy"]) + self.assertEqual([item["id"] for item in matrix.select_builds("closed")], ["X64_RELEASE_CLANGPDB_legacy"]) + + def test_coverage_and_derived_identifiers(self): + rows = matrix.select_builds("open") + expected = { + (arch, target, tool_chain, core) + for target in ("DEBUG", "RELEASE") + for arch, tool_chain, core in ( + ("X64", "VS2022", "legacy"), + ("X64", "CLANGPDB", "legacy"), + ("AARCH64", "CLANGPDB", "legacy"), + ("X64", "CLANGPDB", "patina"), + ("AARCH64", "CLANGPDB", "patina"), + ) + } + self.assertEqual({tuple(row[field] for field in matrix.FLAVOR_CHOICES) for row in rows}, expected) + self.assertEqual(len(rows), 10) + for row in rows: + self.assertEqual(row["id"], "_".join(row[field] for field in matrix.FLAVOR_CHOICES)) + self.assertEqual([row["id"] for row in matrix.select_builds("closed")], ["X64_DEBUG_CLANGPDB_legacy"]) + + def test_provider_formats_preserve_same_flavors(self): + for repository in matrix.REPOSITORIES: + rows = matrix.select_builds(repository) + for format_name in ("json", "github", "ado"): + with self.subTest(repository=repository, format=format_name): + output = io.StringIO() + with redirect_stdout(output): + matrix.main(["--repo", repository, "--format", format_name]) + result = json.loads(output.getvalue()) + if format_name == "github": + result = result["include"] + elif format_name == "ado": + result = [{"id": identifier, **flavor} for identifier, flavor in result.items()] + self.assertEqual(result, rows) + + def test_invalid_matrix_is_rejected(self): + row = json.loads(matrix.MATRIX_PATH.read_text())[0] + invalid_documents = [ + {}, [], [row, row], [{**row, "id": "invalid-id"}], + [{**row, "repositories": ["auto"]}], [{**row, "arch": "unknown"}], + [{**row, "core": "unknown"}], [{**row, "unexpected": True}], + [{**row, "repositories": ["closed"]}], + ] + for document in invalid_documents: + with self.subTest(document=document), patch.object(Path, "read_text", return_value=json.dumps(document)): + with self.assertRaises(ValueError): + matrix.select_builds("open") + + def test_repository_mode_is_required(self): + for args in ([], ["--repo", "auto"]): + with self.subTest(args=args), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as error: + matrix.main(args) + self.assertEqual(error.exception.code, 2) + + +if __name__ == "__main__": + unittest.main() From 270cdd9d2e39b228cdeefbf9ac4167d4d8ab63f1 Mon Sep 17 00:00:00 2001 From: Maheer Aeron Date: Fri, 18 Sep 2026 02:59:48 -0700 Subject: [PATCH 08/18] ci: restore closed build coverage and wire OneBranch matrix --- .azuredevops/templates/build.yml | 151 ++++++++++++++++++++++++++++ .azuredevops/templates/platform.yml | 6 ++ .github/actions/build/action.yml | 17 +++- .github/workflows/platform-ci.yml | 3 + MsvmPkg/Docs/BUILDING.md | 65 ++++++++++-- ci/build-matrix.json | 41 +++++--- ci/scripts/build.py | 29 +++++- ci/scripts/matrix.py | 68 +++++++++++-- ci/tests/test_build.py | 37 ++++++- ci/tests/test_matrix.py | 33 ++++-- 10 files changed, 408 insertions(+), 42 deletions(-) create mode 100644 .azuredevops/templates/build.yml diff --git a/.azuredevops/templates/build.yml b/.azuredevops/templates/build.yml new file mode 100644 index 0000000000..7f56bb87fe --- /dev/null +++ b/.azuredevops/templates/build.yml @@ -0,0 +1,151 @@ +parameters: + - name: repositoryMode + type: string + values: + - open + - closed + - name: hosts + type: object + default: + - windows + - linux + - name: windowsClangVersion + type: string + default: '22.1.1-47' + - name: windowsClangMajorVersion + type: string + default: '22' + +jobs: + - job: BuildMatrix + displayName: Select Build Matrix + pool: + type: windows + variables: + WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest + ob_outputDirectory: $(Build.ArtifactStagingDirectory)/BUILD_MATRIX + ob_nugetPublishing_enabled: false + ob_createvpack_enabled: false + steps: + - ${{ each host in parameters.hosts }}: + - pwsh: | + $matrix = python ci/scripts/matrix.py --repo "$env:REPOSITORY_MODE" --host "$env:BUILD_HOST" --format ado + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + Write-Host "##vso[task.setvariable variable=matrix;isOutput=true]$matrix" + name: ${{ host }} + displayName: Select ${{ host }} Builds + env: + REPOSITORY_MODE: ${{ parameters.repositoryMode }} + BUILD_HOST: ${{ host }} + + - ${{ each host in parameters.hosts }}: + - job: BuildFirmware_${{ host }} + displayName: Build Firmware (${{ host }}) + dependsOn: BuildMatrix + pool: + type: ${{ host }} + strategy: + maxParallel: 4 + matrix: $[ dependencies.BuildMatrix.outputs['${{ host }}.matrix'] ] + variables: + WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest + LinuxContainerImage: onebranch.azurecr.io/linux/ubuntu-2404:latest + GIT_CONFIG_COUNT: "2" + GIT_CONFIG_KEY_0: core.autocrlf + GIT_CONFIG_VALUE_0: "false" + GIT_CONFIG_KEY_1: core.longpaths + GIT_CONFIG_VALUE_1: "true" + ob_outputDirectory: $(Build.ArtifactStagingDirectory)/BUILD/$(System.JobId) + ob_nugetPublishing_enabled: false + ob_createvpack_enabled: false + WindowsClangRoot: $(Agent.TempDirectory)/windows-clang-${{ parameters.windowsClangVersion }} + steps: + - ${{ if eq(host, 'windows') }}: + - pwsh: | + $vswhere = "${env:ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere.exe" + $installation = & $vswhere -latest -version '[17.0,18.0)' -products '*' -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + if ($LASTEXITCODE -ne 0 -or -not $installation) { throw 'Visual Studio 2022 was not found' } + $clangBin = Join-Path $installation 'VC/Tools/Llvm/x64/bin' + if (-not (Test-Path (Join-Path $clangBin 'clang.exe'))) { throw 'Visual Studio Clang was not found' } + Write-Host "##vso[task.setvariable variable=CLANG_BIN]$clangBin/" + displayName: Locate Visual Studio Clang + condition: and(succeeded(), eq(variables['tool_chain'], 'CLANGPDB'), eq(variables['compiler_source'], 'visual_studio')) + + - task: NuGetAuthenticate@1 + displayName: Authenticate Internal Compiler Feed + condition: and(succeeded(), eq(variables['compiler_source'], 'windows_org')) + inputs: + nuGetServiceConnections: WindowsPartners_NugetFeedConnection + + - task: DownloadPackage@1 + displayName: Download Windows-org LLVM Tools + condition: and(succeeded(), eq(variables['compiler_source'], 'windows_org')) + inputs: + packageType: nuget + feed: ES-LLVM + definition: llvm.tools + version: ${{ parameters.windowsClangVersion }} + downloadPath: $(WindowsClangRoot)/bin + + - task: DownloadPackage@1 + displayName: Download Windows-org Clang Headers + condition: and(succeeded(), eq(variables['compiler_source'], 'windows_org')) + inputs: + packageType: nuget + feed: ES-LLVM + definition: clang.headers + version: ${{ parameters.windowsClangVersion }} + downloadPath: $(WindowsClangRoot)/lib/clang/${{ parameters.windowsClangMajorVersion }} + + - pwsh: | + $buildArgs = @('--arch', $env:BUILD_ARCH, '--target', $env:BUILD_TARGET, + '--tool-chain', $env:BUILD_TOOL_CHAIN, '--core', $env:BUILD_CORE, + '--host', 'windows', '--compiler-source', $env:BUILD_COMPILER_SOURCE, + '--legacy-debugger', $env:BUILD_LEGACY_DEBUGGER) + if ($env:BUILD_COMPILER_SOURCE -eq 'windows_org') { + $buildArgs += @('--clang-bin', "$env:WINDOWS_CLANG_ROOT/bin") + } + python ci/scripts/build.py @buildArgs + exit $LASTEXITCODE + displayName: Build Firmware + env: + BUILD_ARCH: $(arch) + BUILD_TARGET: $(target) + BUILD_TOOL_CHAIN: $(tool_chain) + BUILD_CORE: $(core) + BUILD_COMPILER_SOURCE: $(compiler_source) + BUILD_LEGACY_DEBUGGER: $(legacy_debugger) + WINDOWS_CLANG_ROOT: $(WindowsClangRoot) + + - ${{ if eq(host, 'linux') }}: + - bash: | + set -euo pipefail + apt-get update + apt-get install -y python3-venv + case "$BUILD_TOOL_CHAIN" in + CLANGPDB) apt-get install -y clang lld llvm ;; + GCC) + case "$BUILD_ARCH" in + X64) triple=x86_64; packages=x86-64 ;; + AARCH64) triple=aarch64; packages=aarch64 ;; + *) exit 2 ;; + esac + apt-get install -y "gcc-$packages-linux-gnu" "binutils-$packages-linux-gnu" + export "GCC_${BUILD_ARCH}_PREFIX=/usr/bin/$triple-linux-gnu-" + ;; + *) exit 2 ;; + esac + python3 -m venv "$BUILD_VENV" + source "$BUILD_VENV/bin/activate" + python ci/scripts/build.py --arch "$BUILD_ARCH" --target "$BUILD_TARGET" \ + --tool-chain "$BUILD_TOOL_CHAIN" --core "$BUILD_CORE" --host linux \ + --compiler-source "$BUILD_COMPILER_SOURCE" --legacy-debugger "$BUILD_LEGACY_DEBUGGER" + displayName: Prepare Linux Compiler and Build Firmware + env: + BUILD_ARCH: $(arch) + BUILD_TARGET: $(target) + BUILD_TOOL_CHAIN: $(tool_chain) + BUILD_CORE: $(core) + BUILD_COMPILER_SOURCE: $(compiler_source) + BUILD_LEGACY_DEBUGGER: $(legacy_debugger) + BUILD_VENV: $(Agent.TempDirectory)/firmware-venv diff --git a/.azuredevops/templates/platform.yml b/.azuredevops/templates/platform.yml index 3b4fb261d1..5055e632d4 100644 --- a/.azuredevops/templates/platform.yml +++ b/.azuredevops/templates/platform.yml @@ -22,3 +22,9 @@ extends: - stage: SourceChecks jobs: - template: /.azuredevops/templates/source-checks.yml@self + - stage: Build + dependsOn: [] + jobs: + - template: /.azuredevops/templates/build.yml@self + parameters: + repositoryMode: closed diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index ad6cce689c..05da4a7602 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -14,12 +14,21 @@ inputs: core: description: DXE core (legacy or patina) required: true + host: + description: Build host + required: true + compiler-source: + description: Compiler distribution + required: true + legacy-debugger: + description: Enable the legacy debugger (0 or 1) + required: true runs: using: composite steps: - name: Locate Visual Studio Clang - if: inputs.tool-chain == 'CLANGPDB' + if: inputs.tool-chain == 'CLANGPDB' && inputs.compiler-source == 'visual_studio' shell: pwsh run: | $vswhere = "${env:ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere.exe" @@ -36,9 +45,15 @@ runs: BUILD_TARGET: ${{ inputs.target }} BUILD_TOOL_CHAIN: ${{ inputs.tool-chain }} BUILD_CORE: ${{ inputs.core }} + BUILD_HOST: ${{ inputs.host }} + BUILD_COMPILER_SOURCE: ${{ inputs.compiler-source }} + BUILD_LEGACY_DEBUGGER: ${{ inputs.legacy-debugger }} run: >- python ci/scripts/build.py --arch "$env:BUILD_ARCH" --target "$env:BUILD_TARGET" --tool-chain "$env:BUILD_TOOL_CHAIN" --core "$env:BUILD_CORE" + --host "$env:BUILD_HOST" + --compiler-source "$env:BUILD_COMPILER_SOURCE" + --legacy-debugger "$env:BUILD_LEGACY_DEBUGGER" diff --git a/.github/workflows/platform-ci.yml b/.github/workflows/platform-ci.yml index 5151c250f4..433b6afacd 100644 --- a/.github/workflows/platform-ci.yml +++ b/.github/workflows/platform-ci.yml @@ -64,6 +64,9 @@ jobs: target: ${{ matrix.target }} tool-chain: ${{ matrix.tool_chain }} core: ${{ matrix.core }} + host: ${{ matrix.host }} + compiler-source: ${{ matrix.compiler_source }} + legacy-debugger: ${{ matrix.legacy_debugger }} source-checks: name: Source Checks diff --git a/MsvmPkg/Docs/BUILDING.md b/MsvmPkg/Docs/BUILDING.md index 8560be773d..7bf7ed0f81 100644 --- a/MsvmPkg/Docs/BUILDING.md +++ b/MsvmPkg/Docs/BUILDING.md @@ -40,6 +40,22 @@ with `MsvmPkg/PlatformBuild.py`. All three Stuart commands receive the selected Architecture, target, toolchain, and core are required; use `--core patina` to select the Rust DXE core. Stuart executables are resolved from the invoking Python environment, and execution stops on the first failure. +`--host windows|linux` defaults to the current host. `--compiler-source` distinguishes `visual_studio`, +`windows_org`, and `distribution`; it defaults to Visual Studio on Windows and distribution tools on Linux. +Use `--legacy-debugger 1` for the legacy-debugger variants. ARM64 MSVC is rejected before preparation. +Actual execution requires the selected host; `--dry-run` can preview either host's commands. + +Windows-org Clang requires an explicit `--clang-bin` directory; it never falls back to Visual Studio Clang. +For example, with the internal compiler tools and matching resource headers already installed: + +```powershell +python .\ci\scripts\build.py --arch AARCH64 --target RELEASE --tool-chain CLANGPDB --core legacy --host windows --compiler-source windows_org --clang-bin C:/tools/windows-clang/bin +``` + +The local runner does not download private compiler packages or install system compilers. ADO provisions +Windows-org LLVM tools and headers from the pinned internal feed; its Linux jobs install distribution tools +and create a Python virtual environment. Local developers provide the equivalent toolchain environment. + Add `--dry-run` to print the commands without installing dependencies or starting a build. This command builds one flavor only; it does not select an open/closed repository matrix or publish artifacts. @@ -56,10 +72,22 @@ python .\ci\scripts\matrix.py --repo closed The open profile contains ten Windows-hosted builds: DEBUG and RELEASE for X64 VS2022 with the legacy core, plus X64 and AARCH64 CLANGPDB with each of the legacy and Patina cores. -The closed profile still contains only X64 DEBUG CLANGPDB with the legacy core; it is an experimental -baseline, not a replacement for the full hyperv.uefi coverage policy. - -IDs are derived as `___`; do not enter IDs in the JSON. +The closed profile restores 15 legacy-core builds from the preserved hyperv.uefi matrix: + +| Host | Compiler source | Coverage | +| --- | --- | --- | +| Windows | VS2022 | X64 DEBUG and RELEASE | +| Windows | Visual Studio Clang | X64 and AARCH64, DEBUG and RELEASE | +| Windows | Windows-org Clang | X64 and AARCH64, DEBUG and RELEASE | +| Linux | Distribution Clang | X64 and AARCH64, DEBUG and RELEASE | +| Linux | Distribution GCC | X64 RELEASE only | + +Closed X64 DEBUG builds enable the legacy debugger. Open builds retain their existing debugger selection. +ARM64 MSVC is unsupported. Disabled GCC combinations are not reintroduced. + +IDs are derived as `______`; +do not enter IDs in the JSON. Identical flavors may belong to different repository profiles, but may not +occur twice within one profile. Additional coverage can be added by editing the JSON, without duplicating flavor definitions in provider YAML. The selector rejects duplicate flavors and fails when a requested repository has no builds. @@ -70,8 +98,33 @@ calls the build action for each flavor. That action invokes the same `ci/scripts Builds depend only on matrix generation; source checks run independently. Build artifact publication is not part of this step yet. -ADO build matrix integration remains unwired. Its JSON output format does not establish support for runtime -matrix expansion in the governed OneBranch templates; that requires validation before enabling ADO builds. +ADO's shared platform template selects `closed` coverage through its build job template. A `BuildMatrix` +job exports separate Windows and Linux matrices using `--host`. Dependent `BuildFirmware_windows` and +`BuildFirmware_linux` jobs consume them through native `strategy.matrix`, with at most four parallel +builds per host group. Host pools are selected at template expansion time, not by runtime matrix variables. +Each job runs inside OneBranch. The build stage +runs independently of source checks, and each flavor invokes the same `ci/scripts/build.py` as GitHub. +NuGet and VPack publishing remain disabled for these jobs. + +The inspected OneBranch Windows job template forwards job strategies and dependencies. Local validation +checks our output binding and flavor arguments, but a hosted OneBranch run is still required to verify +the full governed-template expansion, runner tools, internal feed credentials, and repository access. + +### Closed Package Identity + +Closed rows have explicit `shipping` and `package_suffix` metadata. Shipping variants are X64 DEBUG/RELEASE +VS2022 legacy and AARCH64 RELEASE Windows-org CLANGPDB legacy. These three keep their empty package suffix; +all other closed variants retain their historical test suffixes. Metadata describes eligibility, not permission +to publish, and does not mark a local or PR build official. + +Build IDs are not package names. Future packaging must preserve the shared VPack/NuGet ID convention: +`.mscoreuefi..<.suffix if present><.Test if PR or non-shipping>`. +For example, the official ARM64 shipping ID remains `hyperv.uefi.mscoreuefi.AARCH64.RELEASE`, while its PR +ID is `hyperv.uefi.mscoreuefi.PR.AARCH64.RELEASE.Test`. +NuGet versions and OneBranch VPack version allocation remain separate contracts. +No package creation or publication is enabled here. In particular, runtime build matrices do not establish +that per-flavor OneBranch publishing policy can be expanded at runtime; validate or introduce a separate +compile-time publishing stage before enabling it. ## Running CI Checks Locally diff --git a/ci/build-matrix.json b/ci/build-matrix.json index 6fe6d8d00f..b59c220dac 100644 --- a/ci/build-matrix.json +++ b/ci/build-matrix.json @@ -1,18 +1,27 @@ [ - { - "repositories": ["open", "closed"], - "arch": "X64", - "target": "DEBUG", - "tool_chain": "CLANGPDB", - "core": "legacy" - }, - {"repositories": ["open"], "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "core": "legacy"}, - {"repositories": ["open"], "arch": "X64", "target": "DEBUG", "tool_chain": "VS2022", "core": "legacy"}, - {"repositories": ["open"], "arch": "X64", "target": "RELEASE", "tool_chain": "VS2022", "core": "legacy"}, - {"repositories": ["open"], "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "core": "legacy"}, - {"repositories": ["open"], "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "core": "legacy"}, - {"repositories": ["open"], "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "core": "patina"}, - {"repositories": ["open"], "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "core": "patina"}, - {"repositories": ["open"], "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "core": "patina"}, - {"repositories": ["open"], "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "core": "patina"} + {"repositories": ["open"], "host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, + {"repositories": ["open"], "host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, + {"repositories": ["open"], "host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "VS2022", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, + {"repositories": ["open"], "host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "VS2022", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, + {"repositories": ["open"], "host": "windows", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, + {"repositories": ["open"], "host": "windows", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, + {"repositories": ["open"], "host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "patina", "legacy_debugger": "0"}, + {"repositories": ["open"], "host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "patina", "legacy_debugger": "0"}, + {"repositories": ["open"], "host": "windows", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "patina", "legacy_debugger": "0"}, + {"repositories": ["open"], "host": "windows", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "patina", "legacy_debugger": "0"}, + {"repositories": ["closed"], "host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "VS2022", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "1", "shipping": true, "package_suffix": ""}, + {"repositories": ["closed"], "host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "VS2022", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": true, "package_suffix": ""}, + {"repositories": ["closed"], "host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "1", "shipping": false, "package_suffix": "XDCV"}, + {"repositories": ["closed"], "host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRCV"}, + {"repositories": ["closed"], "host": "windows", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ADCV"}, + {"repositories": ["closed"], "host": "windows", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ARCV"}, + {"repositories": ["closed"], "host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "1", "shipping": false, "package_suffix": "XDCW"}, + {"repositories": ["closed"], "host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRCW"}, + {"repositories": ["closed"], "host": "windows", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ADCW"}, + {"repositories": ["closed"], "host": "windows", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "0", "shipping": true, "package_suffix": ""}, + {"repositories": ["closed"], "host": "linux", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "1", "shipping": false, "package_suffix": "XDCL"}, + {"repositories": ["closed"], "host": "linux", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRCL"}, + {"repositories": ["closed"], "host": "linux", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ADCL"}, + {"repositories": ["closed"], "host": "linux", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ARCL"}, + {"repositories": ["closed"], "host": "linux", "arch": "X64", "target": "RELEASE", "tool_chain": "GCC", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRG"} ] diff --git a/ci/scripts/build.py b/ci/scripts/build.py index 7dfa60b18e..5e9ade916f 100644 --- a/ci/scripts/build.py +++ b/ci/scripts/build.py @@ -20,15 +20,42 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--target", required=True, choices=("DEBUG", "RELEASE")) parser.add_argument("--tool-chain", required=True, choices=("VS2022", "CLANGPDB", "GCC")) parser.add_argument("--core", required=True, choices=("legacy", "patina")) + parser.add_argument("--host", choices=("windows", "linux"), default="windows" if os.name == "nt" else "linux") + parser.add_argument("--compiler-source", choices=("visual_studio", "windows_org", "distribution")) + parser.add_argument("--legacy-debugger", choices=("0", "1"), default="0") + parser.add_argument("--clang-bin", type=Path, help="Explicit compiler bin directory; required for Windows-org Clang") parser.add_argument("--dry-run", action="store_true", help="Print commands without executing them") args = parser.parse_args(argv) + compiler_source = args.compiler_source or ("visual_studio" if args.host == "windows" else "distribution") + if args.arch == "AARCH64" and args.tool_chain == "VS2022": + parser.error("ARM64 MSVC is not supported") + if (args.host, args.tool_chain, compiler_source) not in { + ("windows", "VS2022", "visual_studio"), ("windows", "CLANGPDB", "visual_studio"), + ("windows", "CLANGPDB", "windows_org"), ("linux", "CLANGPDB", "distribution"), + ("linux", "GCC", "distribution"), + }: + parser.error("Unsupported host/compiler combination") + if compiler_source == "windows_org" and args.clang_bin is None: + parser.error("Windows-org Clang requires --clang-bin; Visual Studio fallback is not allowed") + if not args.dry_run: + if args.host != ("windows" if os.name == "nt" else "linux"): + parser.error(f"This flavor requires a {args.host} host") + if args.clang_bin is not None and not (args.clang_bin / ("clang.exe" if args.host == "windows" else "clang")).is_file(): + parser.error("--clang-bin must contain the selected Clang executable") + flavor = [ f"BUILD_ARCH={args.arch}", f"TARGET={args.target}", f"TOOL_CHAIN_TAG={args.tool_chain}", f"BLD_*_USE_LEGACY_C_CORE={'TRUE' if args.core == 'legacy' else 'FALSE'}", ] + if args.legacy_debugger == "1": + flavor.append("BLD_*_LEGACY_DEBUGGER=1") + environment = os.environ.copy() + if args.clang_bin is not None: + environment["CLANG_BIN"] = f"{args.clang_bin.resolve().as_posix()}/" + print(f"CLANG_BIN={environment['CLANG_BIN']}", flush=True) scripts_dir = Path(sysconfig.get_path("scripts")) suffix = ".exe" if os.name == "nt" else "" commands = [[sys.executable, "-m", "pip", "install", "-r", "pip-requirements.txt"]] @@ -45,7 +72,7 @@ def main(argv: list[str] | None = None) -> int: print(f"==> {shlex.join(command)}", flush=True) if not args.dry_run: try: - subprocess.run(command, cwd=REPO_ROOT, check=True) + subprocess.run(command, cwd=REPO_ROOT, check=True, env=environment) except subprocess.CalledProcessError as error: return error.returncode return 0 diff --git a/ci/scripts/matrix.py b/ci/scripts/matrix.py index f9c5153f42..aef36c6e01 100644 --- a/ci/scripts/matrix.py +++ b/ci/scripts/matrix.py @@ -4,6 +4,7 @@ import argparse import json +import re from pathlib import Path MATRIX_PATH = Path(__file__).resolve().parents[1] / "build-matrix.json" @@ -14,31 +15,75 @@ "tool_chain": ("VS2022", "CLANGPDB", "GCC"), "core": ("legacy", "patina"), } +EXECUTION_CHOICES = { + "host": ("windows", "linux"), + "compiler_source": ("visual_studio", "windows_org", "distribution"), + "legacy_debugger": ("0", "1"), +} + + +def build_id(row: dict) -> str: + return "_".join(row[field] for field in ( + "host", "arch", "target", "tool_chain", "compiler_source", "core", "legacy_debugger", + )) -def select_builds(repository: str, path: Path = MATRIX_PATH) -> list[dict]: +def select_builds(repository: str, path: Path = MATRIX_PATH, host: str | None = None) -> list[dict]: if repository not in REPOSITORIES: raise ValueError(f"Unknown repository mode: {repository}") + if host is not None and host not in EXECUTION_CHOICES["host"]: + raise ValueError(f"Unknown host: {host}") rows = json.loads(path.read_text(encoding="utf-8")) if not isinstance(rows, list): raise ValueError("Build matrix must be a list") selected = [] identifiers = set() + package_names = set() for row in rows: - if not isinstance(row, dict) or set(row) != {"repositories", *FLAVOR_CHOICES}: - raise ValueError("Each build must specify repositories, arch, target, tool_chain, and core") - for field, choices in FLAVOR_CHOICES.items(): + required = {"repositories", *FLAVOR_CHOICES, *EXECUTION_CHOICES} + if not isinstance(row, dict) or not required <= row.keys() or row.keys() - required - {"shipping", "package_suffix"}: + raise ValueError("Each build must specify repository coverage and a complete execution flavor") + for field, choices in {**FLAVOR_CHOICES, **EXECUTION_CHOICES}.items(): if row[field] not in choices: raise ValueError(f"Invalid {field}: {row[field]!r}") - identifier = "_".join(row[field] for field in FLAVOR_CHOICES) - if identifier in identifiers: - raise ValueError(f"Duplicate matrix identifier: {identifier}") - identifiers.add(identifier) + identifier = build_id(row) + valid_compilers = { + ("windows", "VS2022", "visual_studio"), + ("windows", "CLANGPDB", "visual_studio"), + ("windows", "CLANGPDB", "windows_org"), + ("linux", "CLANGPDB", "distribution"), + ("linux", "GCC", "distribution"), + } + if (row["host"], row["tool_chain"], row["compiler_source"]) not in valid_compilers: + raise ValueError(f"Unsupported host/compiler combination: {identifier}") + if row["arch"] == "AARCH64" and row["tool_chain"] == "VS2022": + raise ValueError("ARM64 MSVC is not supported") repositories = row["repositories"] if not isinstance(repositories, list) or not repositories or any(mode not in REPOSITORIES for mode in repositories): raise ValueError(f"Invalid repositories for {identifier}") - if repository in repositories: - selected.append({"id": identifier, **{field: row[field] for field in FLAVOR_CHOICES}}) + for mode in repositories: + if (mode, identifier) in identifiers: + raise ValueError(f"Duplicate matrix identifier for {mode}: {identifier}") + identifiers.add((mode, identifier)) + if "closed" in repositories: + if type(row.get("shipping")) is not bool or not isinstance(row.get("package_suffix"), str): + raise ValueError(f"Closed builds require explicit shipping and package_suffix: {identifier}") + suffix = row["package_suffix"] + if not re.fullmatch(r"[A-Za-z0-9]*", suffix) or row["shipping"] != (suffix == ""): + raise ValueError(f"Invalid shipping/package suffix contract: {identifier}") + package_name = (row["arch"], row["target"], suffix) + if package_name in package_names: + raise ValueError(f"Duplicate closed package identity: {package_name}") + package_names.add(package_name) + elif "shipping" in row or "package_suffix" in row: + raise ValueError("Closed package metadata must belong to closed repository coverage") + if repository in repositories and (host is None or row["host"] == host): + selected.append({ + "id": identifier, + **{field: row[field] for field in (*FLAVOR_CHOICES, *EXECUTION_CHOICES)}, + **({"shipping": str(row["shipping"]).lower(), "package_suffix": row["package_suffix"]} + if repository == "closed" else {}), + }) if not selected: raise ValueError(f"No builds configured for repository mode: {repository}") return selected @@ -47,10 +92,11 @@ def select_builds(repository: str, path: Path = MATRIX_PATH) -> list[dict]: def main(argv: list[str] | None = None) -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--repo", required=True, choices=REPOSITORIES) + parser.add_argument("--host", choices=EXECUTION_CHOICES["host"]) parser.add_argument("--format", choices=("json", "github", "ado"), default="json") args = parser.parse_args(argv) try: - rows = select_builds(args.repo) + rows = select_builds(args.repo, host=args.host) except (OSError, ValueError) as error: parser.error(str(error)) if args.format == "github": diff --git a/ci/tests/test_build.py b/ci/tests/test_build.py index a1bd65cf19..0278b09ae2 100644 --- a/ci/tests/test_build.py +++ b/ci/tests/test_build.py @@ -38,7 +38,7 @@ def test_platform_preparation_precedes_build(self, run_mock): "TOOL_CHAIN_TAG=CLANGPDB", "BLD_*_USE_LEGACY_C_CORE=TRUE", ]) for invocation in calls: - self.assertEqual(invocation.kwargs, {"cwd": SCRIPT_PATH.resolve().parents[2], "check": True}) + self.assertEqual(invocation.kwargs, {"cwd": SCRIPT_PATH.resolve().parents[2], "check": True, "env": dict(os.environ)}) self.assertIn("BUILDREPORT_TYPES=PCD DEPEX FLASH BUILD_FLAGS LIBRARY", calls[-1].args[0]) self.assertIn("LaunchLogOnSuccess=FALSE", calls[-1].args[0]) self.assertIn("LaunchLogOnError=FALSE", calls[-1].args[0]) @@ -65,6 +65,41 @@ def test_dry_run_has_no_side_effects(self, run_mock): self.assertEqual(build.main([*self.ARGS, "--dry-run"]), 0) run_mock.assert_not_called() + @patch.object(build.subprocess, "run") + def test_debugger_passed_to_all_stuart_phases(self, run_mock): + build.main([*self.ARGS, "--legacy-debugger", "1"]) + for invocation in run_mock.call_args_list[1:]: + self.assertIn("BLD_*_LEGACY_DEBUGGER=1", invocation.args[0]) + + @patch.object(build.subprocess, "run") + def test_windows_org_cannot_fall_back_to_visual_studio(self, run_mock): + with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error: + build.main([*self.ARGS, "--host", "windows", "--compiler-source", "windows_org", "--dry-run"]) + self.assertEqual(error.exception.code, 2) + output = io.StringIO() + with redirect_stdout(output): + self.assertEqual(build.main([ + *self.ARGS, "--host", "windows", "--compiler-source", "windows_org", + "--clang-bin", "internal-llvm/bin", "--dry-run", + ]), 0) + self.assertIn("CLANG_BIN=", output.getvalue()) + run_mock.assert_not_called() + + @patch.object(build.subprocess, "run") + def test_arm64_msvc_is_rejected_before_setup(self, run_mock): + with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error: + build.main(["--arch", "AARCH64", "--target", "RELEASE", "--tool-chain", "VS2022", "--core", "legacy"]) + self.assertEqual(error.exception.code, 2) + run_mock.assert_not_called() + + @patch.object(build.subprocess, "run") + def test_explicit_clang_bin_overrides_child_environment_only(self, run_mock): + with patch.dict(os.environ, {"CLANG_BIN": "visual-studio/bin/"}), patch.object(Path, "is_file", return_value=True): + build.main([*self.ARGS, "--clang-bin", "internal-llvm/bin"]) + self.assertEqual(os.environ["CLANG_BIN"], "visual-studio/bin/") + for invocation in run_mock.call_args_list: + self.assertEqual(invocation.kwargs["env"]["CLANG_BIN"], Path("internal-llvm/bin").resolve().as_posix() + "/") + @patch.object(build.subprocess, "run") def test_invalid_or_missing_flavor_is_rejected(self, run_mock): for args in ([], [*self.ARGS[:-1], "unknown"]): diff --git a/ci/tests/test_matrix.py b/ci/tests/test_matrix.py index 821c5c8853..dc6fe16393 100644 --- a/ci/tests/test_matrix.py +++ b/ci/tests/test_matrix.py @@ -27,8 +27,10 @@ def test_each_repository_row_is_accepted_by_build_runner(self): for row in matrix.select_builds(repository): with self.subTest(repository=repository, row=row), redirect_stdout(io.StringIO()): args = ["--dry-run"] - for field in matrix.FLAVOR_CHOICES: + for field in (*matrix.FLAVOR_CHOICES, *matrix.EXECUTION_CHOICES): args.extend(["--" + field.replace("_", "-"), row[field]]) + if row["compiler_source"] == "windows_org": + args.extend(["--clang-bin", "internal-llvm/bin"]) with patch.object(build.subprocess, "run") as run_mock: self.assertEqual(build.main(args), 0) run_mock.assert_not_called() @@ -36,10 +38,10 @@ def test_each_repository_row_is_accepted_by_build_runner(self): def test_profiles_are_filtered_explicitly(self): row = json.loads(matrix.MATRIX_PATH.read_text())[0] open_row = {**row, "target": "DEBUG", "repositories": ["open"]} - closed_row = {**row, "target": "RELEASE", "repositories": ["closed"]} + closed_row = {**row, "target": "RELEASE", "repositories": ["closed"], "shipping": False, "package_suffix": "XRCV"} with patch.object(Path, "read_text", return_value=json.dumps([open_row, closed_row])): - self.assertEqual([item["id"] for item in matrix.select_builds("open")], ["X64_DEBUG_CLANGPDB_legacy"]) - self.assertEqual([item["id"] for item in matrix.select_builds("closed")], ["X64_RELEASE_CLANGPDB_legacy"]) + self.assertEqual([item["id"] for item in matrix.select_builds("open")], [matrix.build_id(open_row)]) + self.assertEqual([item["id"] for item in matrix.select_builds("closed")], [matrix.build_id(closed_row)]) def test_coverage_and_derived_identifiers(self): rows = matrix.select_builds("open") @@ -57,8 +59,24 @@ def test_coverage_and_derived_identifiers(self): self.assertEqual({tuple(row[field] for field in matrix.FLAVOR_CHOICES) for row in rows}, expected) self.assertEqual(len(rows), 10) for row in rows: - self.assertEqual(row["id"], "_".join(row[field] for field in matrix.FLAVOR_CHOICES)) - self.assertEqual([row["id"] for row in matrix.select_builds("closed")], ["X64_DEBUG_CLANGPDB_legacy"]) + self.assertEqual(row["id"], matrix.build_id(row)) + closed = matrix.select_builds("closed") + self.assertEqual(len(closed), 15) + self.assertEqual(len({row["id"] for row in closed}), 15) + self.assertEqual(len(matrix.select_builds("closed", host="windows")), 10) + self.assertEqual(len(matrix.select_builds("closed", host="linux")), 5) + shipping = {(row["arch"], row["target"], row["tool_chain"], row["compiler_source"], row["core"]) + for row in closed if row["shipping"] == "true"} + self.assertEqual(shipping, { + ("X64", "DEBUG", "VS2022", "visual_studio", "legacy"), + ("X64", "RELEASE", "VS2022", "visual_studio", "legacy"), + ("AARCH64", "RELEASE", "CLANGPDB", "windows_org", "legacy"), + }) + for row in closed: + self.assertEqual(row["legacy_debugger"], "1" if row["arch"] == "X64" and row["target"] == "DEBUG" else "0") + self.assertEqual(row["shipping"] == "true", row["package_suffix"] == "") + self.assertEqual({row["package_suffix"] for row in closed if row["shipping"] == "false"}, + {"XDCV", "XRCV", "ADCV", "ARCV", "XDCW", "XRCW", "ADCW", "XDCL", "XRCL", "ADCL", "ARCL", "XRG"}) def test_provider_formats_preserve_same_flavors(self): for repository in matrix.REPOSITORIES: @@ -82,6 +100,9 @@ def test_invalid_matrix_is_rejected(self): [{**row, "repositories": ["auto"]}], [{**row, "arch": "unknown"}], [{**row, "core": "unknown"}], [{**row, "unexpected": True}], [{**row, "repositories": ["closed"]}], + [{**row, "arch": "AARCH64", "tool_chain": "VS2022"}], + [{**row, "host": "linux"}], + [{**row, "shipping": True, "package_suffix": ""}], ] for document in invalid_documents: with self.subTest(document=document), patch.object(Path, "read_text", return_value=json.dumps(document)): From 931329629d87e03f643f6b722811af7191b2e5ac Mon Sep 17 00:00:00 2001 From: Maheer Aeron Date: Fri, 18 Sep 2026 11:44:52 -0700 Subject: [PATCH 09/18] ci: strongly type and document Python automation --- MsvmPkg/Docs/BUILDING.md | 18 ++++ ci/mypy.ini | 5 + ci/requirements-dev.txt | 1 + ci/scripts/build.py | 47 ++++++++- ci/scripts/matrix.py | 185 ++++++++++++++++++++++++--------- ci/scripts/source_checks.py | 25 ++++- ci/tests/test_build.py | 51 ++++----- ci/tests/test_matrix.py | 91 ++++++++++------ ci/tests/test_source_checks.py | 18 ++-- 9 files changed, 318 insertions(+), 123 deletions(-) create mode 100644 ci/mypy.ini create mode 100644 ci/requirements-dev.txt diff --git a/MsvmPkg/Docs/BUILDING.md b/MsvmPkg/Docs/BUILDING.md index 7bf7ed0f81..1221f5f45f 100644 --- a/MsvmPkg/Docs/BUILDING.md +++ b/MsvmPkg/Docs/BUILDING.md @@ -155,6 +155,24 @@ stuart_ci_build ` Review the resulting changes and rerun the full source-check command before committing them. +## Developing CI Python + +Run these checks from the repository root in your activated virtual environment: + +```powershell +python -m pip install -r ci/requirements-dev.txt +python -m mypy --config-file ci/mypy.ini +python -B -m unittest discover -s ci/tests -v +``` + +Strict type checking covers both CI scripts and tests. Keep parameter and return annotations complete; +document input/output contracts, side effects, and failure behavior in public API docstrings. +The matrix module validates JSON into typed flavor records before using it, keeping provider serialization +separate from the input definition. Runtime validation is still required: type annotations alone cannot +validate JSON or command-line input. Tests mock external commands and do not build or download anything. + +These are local development checks; they have not been added to the hosted workflow yet. + ## Specialized Builds ### Debug-Enabled Build diff --git a/ci/mypy.ini b/ci/mypy.ini new file mode 100644 index 0000000000..58827d9777 --- /dev/null +++ b/ci/mypy.ini @@ -0,0 +1,5 @@ +[mypy] +strict = True +explicit_package_bases = True +files = ci/scripts, ci/tests +cache_dir = Build/mypy-cache diff --git a/ci/requirements-dev.txt b/ci/requirements-dev.txt new file mode 100644 index 0000000000..0900565228 --- /dev/null +++ b/ci/requirements-dev.txt @@ -0,0 +1 @@ +mypy>=1.15,<2 diff --git a/ci/scripts/build.py b/ci/scripts/build.py index 5e9ade916f..3f13a622b2 100644 --- a/ci/scripts/build.py +++ b/ci/scripts/build.py @@ -1,4 +1,9 @@ -"""Prepare and build one firmware flavor using native Stuart commands.""" +"""Prepare and build one firmware flavor using native Stuart commands. + +The invoking Python environment supplies pip and Stuart executables. Setup, +update, and compilation run from the repository root in that order. Compiler +installation, matrix scheduling, credentials, and publishing belong to callers. +""" from __future__ import annotations @@ -9,12 +14,44 @@ import sys import sysconfig from pathlib import Path +from typing import Literal REPO_ROOT = Path(__file__).resolve().parents[2] PLATFORM_SETTINGS = "MsvmPkg/PlatformBuild.py" +class BuildArguments(argparse.Namespace): + """Typed CLI inputs populated and choice-validated by argparse. + + Host and compiler source describe execution, not the target architecture. + An explicit Clang directory affects child processes only. Dry runs validate + the requested flavor but do not require its host or compiler to be present. + """ + + arch: Literal["X64", "AARCH64"] + target: Literal["DEBUG", "RELEASE"] + tool_chain: Literal["VS2022", "CLANGPDB", "GCC"] + core: Literal["legacy", "patina"] + host: Literal["windows", "linux"] + compiler_source: Literal["visual_studio", "windows_org", "distribution"] | None + legacy_debugger: Literal["0", "1"] + clang_bin: Path | None + dry_run: bool + + def main(argv: list[str] | None = None) -> int: + """Run one explicit flavor, stopping at the first failed command. + + Args: + argv: CLI arguments, or None to read the process command line. + + Returns: + Zero on success or dry run; otherwise the first subprocess exit code. + + Raises: + SystemExit: Help (0) or an invalid/unsupported invocation (2). + OSError: A subprocess cannot be started, for example missing Stuart. + """ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--arch", required=True, choices=("X64", "AARCH64")) parser.add_argument("--target", required=True, choices=("DEBUG", "RELEASE")) @@ -25,7 +62,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--legacy-debugger", choices=("0", "1"), default="0") parser.add_argument("--clang-bin", type=Path, help="Explicit compiler bin directory; required for Windows-org Clang") parser.add_argument("--dry-run", action="store_true", help="Print commands without executing them") - args = parser.parse_args(argv) + args = parser.parse_args(argv, namespace=BuildArguments()) compiler_source = args.compiler_source or ("visual_studio" if args.host == "windows" else "distribution") if args.arch == "AARCH64" and args.tool_chain == "VS2022": @@ -44,7 +81,7 @@ def main(argv: list[str] | None = None) -> int: if args.clang_bin is not None and not (args.clang_bin / ("clang.exe" if args.host == "windows" else "clang")).is_file(): parser.error("--clang-bin must contain the selected Clang executable") - flavor = [ + flavor: list[str] = [ f"BUILD_ARCH={args.arch}", f"TARGET={args.target}", f"TOOL_CHAIN_TAG={args.tool_chain}", @@ -52,13 +89,13 @@ def main(argv: list[str] | None = None) -> int: ] if args.legacy_debugger == "1": flavor.append("BLD_*_LEGACY_DEBUGGER=1") - environment = os.environ.copy() + environment: dict[str, str] = os.environ.copy() if args.clang_bin is not None: environment["CLANG_BIN"] = f"{args.clang_bin.resolve().as_posix()}/" print(f"CLANG_BIN={environment['CLANG_BIN']}", flush=True) scripts_dir = Path(sysconfig.get_path("scripts")) suffix = ".exe" if os.name == "nt" else "" - commands = [[sys.executable, "-m", "pip", "install", "-r", "pip-requirements.txt"]] + commands: list[list[str]] = [[sys.executable, "-m", "pip", "install", "-r", "pip-requirements.txt"]] for tool in ("stuart_setup", "stuart_update", "stuart_build"): commands.append([str(scripts_dir / (tool + suffix)), "-c", PLATFORM_SETTINGS, *flavor]) commands[-1].extend([ diff --git a/ci/scripts/matrix.py b/ci/scripts/matrix.py index aef36c6e01..baadfb5ef1 100644 --- a/ci/scripts/matrix.py +++ b/ci/scripts/matrix.py @@ -1,108 +1,199 @@ -"""Select explicit repository build coverage and emit CI-provider matrix data.""" +"""Validate repository coverage and emit provider-specific matrix JSON. + +This module selects data only: it never builds firmware or authorizes publication. +All rows are validated before filtering, including rows outside the requested +repository/host. Generated IDs identify executions; package suffixes are a +separate consumer-facing contract. +""" from __future__ import annotations import argparse import json import re +from collections.abc import Mapping from pathlib import Path +from typing import Literal, NotRequired, TypedDict, cast + +Repository = Literal["open", "closed"] +Host = Literal["windows", "linux"] +Architecture = Literal["X64", "AARCH64"] +Target = Literal["DEBUG", "RELEASE"] +ToolChain = Literal["VS2022", "CLANGPDB", "GCC"] +Core = Literal["legacy", "patina"] +CompilerSource = Literal["visual_studio", "windows_org", "distribution"] +Debugger = Literal["0", "1"] +OutputFormat = Literal["json", "github", "ado"] + + +class BuildFlavor(TypedDict): + """Execution inputs shared by build selection and provider callers.""" + + arch: Architecture + target: Target + tool_chain: ToolChain + core: Core + host: Host + compiler_source: CompilerSource + legacy_debugger: Debugger + + +class BuildDefinition(BuildFlavor): + """Validated JSON row; closed coverage requires both package fields.""" + + repositories: list[Repository] + shipping: NotRequired[bool] + package_suffix: NotRequired[str] + + +class SelectedBuild(BuildFlavor): + """Provider row with generated identity and optional closed package metadata. + + Shipping is serialized as text because ADO matrix values are variables. + It describes eligibility, not permission to publish an official package. + """ + + id: str + shipping: NotRequired[str] + package_suffix: NotRequired[str] + + +class MatrixArguments(argparse.Namespace): + """CLI values constrained by argparse choices before selection.""" + + repo: Repository + host: Host | None + format: OutputFormat MATRIX_PATH = Path(__file__).resolve().parents[1] / "build-matrix.json" -REPOSITORIES = ("open", "closed") -FLAVOR_CHOICES = { +REPOSITORIES: tuple[Repository, ...] = ("open", "closed") +FLAVOR_CHOICES: dict[str, tuple[str, ...]] = { "arch": ("X64", "AARCH64"), "target": ("DEBUG", "RELEASE"), "tool_chain": ("VS2022", "CLANGPDB", "GCC"), "core": ("legacy", "patina"), } -EXECUTION_CHOICES = { +EXECUTION_CHOICES: dict[str, tuple[str, ...]] = { "host": ("windows", "linux"), "compiler_source": ("visual_studio", "windows_org", "distribution"), "legacy_debugger": ("0", "1"), } -def build_id(row: dict) -> str: - return "_".join(row[field] for field in ( - "host", "arch", "target", "tool_chain", "compiler_source", "core", "legacy_debugger", - )) +def build_id(row: BuildFlavor) -> str: + """Derive a stable execution ID, independent of package names or repo mode.""" + return "_".join((row["host"], row["arch"], row["target"], row["tool_chain"], + row["compiler_source"], row["core"], row["legacy_debugger"])) + + +def parse_definition(value: object) -> BuildDefinition: + """Validate one untrusted JSON row before admitting it to the typed model. + + Raises: + ValueError: Missing/unknown fields, invalid scalar types or choices, + unsupported compilers, or inconsistent package metadata. + """ + if not isinstance(value, dict): + raise ValueError("Each build must specify repository coverage and a complete execution flavor") + row = cast(dict[str, object], value) + required = {"repositories", *FLAVOR_CHOICES, *EXECUTION_CHOICES} + if not required <= row.keys() or row.keys() - required - {"shipping", "package_suffix"}: + raise ValueError("Each build must specify repository coverage and a complete execution flavor") + for field, choices in {**FLAVOR_CHOICES, **EXECUTION_CHOICES}.items(): + if not isinstance(row[field], str) or row[field] not in choices: + raise ValueError(f"Invalid {field}: {row[field]!r}") + repositories = row["repositories"] + if not isinstance(repositories, list) or not repositories: + raise ValueError("Invalid repositories") + modes = cast(list[object], repositories) + if any(not isinstance(mode, str) or mode not in REPOSITORIES for mode in modes): + raise ValueError("Invalid repositories") + valid_compilers = { + ("windows", "VS2022", "visual_studio"), + ("windows", "CLANGPDB", "visual_studio"), + ("windows", "CLANGPDB", "windows_org"), + ("linux", "CLANGPDB", "distribution"), + ("linux", "GCC", "distribution"), + } + if (row["host"], row["tool_chain"], row["compiler_source"]) not in valid_compilers: + raise ValueError("Unsupported host/compiler combination") + if row["arch"] == "AARCH64" and row["tool_chain"] == "VS2022": + raise ValueError("ARM64 MSVC is not supported") + if "closed" in modes: + suffix = row.get("package_suffix") + if type(row.get("shipping")) is not bool or not isinstance(suffix, str): + raise ValueError("Closed builds require explicit shipping and package_suffix") + if not re.fullmatch(r"[A-Za-z0-9]*", suffix) or row["shipping"] != (suffix == ""): + raise ValueError("Invalid shipping/package suffix contract") + elif "shipping" in row or "package_suffix" in row: + raise ValueError("Closed package metadata must belong to closed repository coverage") + return cast(BuildDefinition, row) + +def select_builds(repository: str, path: Path = MATRIX_PATH, host: str | None = None) -> list[SelectedBuild]: + """Load validated builds for a repository, optionally restricted by host. -def select_builds(repository: str, path: Path = MATRIX_PATH, host: str | None = None) -> list[dict]: + Preserves declaration order and rejects duplicate execution/package IDs. + Raises OSError for unreadable input and ValueError for invalid JSON, invalid + definitions, unknown filters, or an empty selection. No commands are run. + """ if repository not in REPOSITORIES: raise ValueError(f"Unknown repository mode: {repository}") if host is not None and host not in EXECUTION_CHOICES["host"]: raise ValueError(f"Unknown host: {host}") - rows = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(rows, list): + document: object = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(document, list): raise ValueError("Build matrix must be a list") - selected = [] - identifiers = set() - package_names = set() - for row in rows: - required = {"repositories", *FLAVOR_CHOICES, *EXECUTION_CHOICES} - if not isinstance(row, dict) or not required <= row.keys() or row.keys() - required - {"shipping", "package_suffix"}: - raise ValueError("Each build must specify repository coverage and a complete execution flavor") - for field, choices in {**FLAVOR_CHOICES, **EXECUTION_CHOICES}.items(): - if row[field] not in choices: - raise ValueError(f"Invalid {field}: {row[field]!r}") + selected: list[SelectedBuild] = [] + identifiers: set[tuple[Repository, str]] = set() + package_names: set[tuple[Architecture, Target, str]] = set() + for value in cast(list[object], document): + row = parse_definition(value) identifier = build_id(row) - valid_compilers = { - ("windows", "VS2022", "visual_studio"), - ("windows", "CLANGPDB", "visual_studio"), - ("windows", "CLANGPDB", "windows_org"), - ("linux", "CLANGPDB", "distribution"), - ("linux", "GCC", "distribution"), - } - if (row["host"], row["tool_chain"], row["compiler_source"]) not in valid_compilers: - raise ValueError(f"Unsupported host/compiler combination: {identifier}") - if row["arch"] == "AARCH64" and row["tool_chain"] == "VS2022": - raise ValueError("ARM64 MSVC is not supported") repositories = row["repositories"] - if not isinstance(repositories, list) or not repositories or any(mode not in REPOSITORIES for mode in repositories): - raise ValueError(f"Invalid repositories for {identifier}") for mode in repositories: if (mode, identifier) in identifiers: raise ValueError(f"Duplicate matrix identifier for {mode}: {identifier}") identifiers.add((mode, identifier)) if "closed" in repositories: - if type(row.get("shipping")) is not bool or not isinstance(row.get("package_suffix"), str): - raise ValueError(f"Closed builds require explicit shipping and package_suffix: {identifier}") suffix = row["package_suffix"] - if not re.fullmatch(r"[A-Za-z0-9]*", suffix) or row["shipping"] != (suffix == ""): - raise ValueError(f"Invalid shipping/package suffix contract: {identifier}") package_name = (row["arch"], row["target"], suffix) if package_name in package_names: raise ValueError(f"Duplicate closed package identity: {package_name}") package_names.add(package_name) - elif "shipping" in row or "package_suffix" in row: - raise ValueError("Closed package metadata must belong to closed repository coverage") if repository in repositories and (host is None or row["host"] == host): - selected.append({ + selected_row: SelectedBuild = { "id": identifier, - **{field: row[field] for field in (*FLAVOR_CHOICES, *EXECUTION_CHOICES)}, - **({"shipping": str(row["shipping"]).lower(), "package_suffix": row["package_suffix"]} - if repository == "closed" else {}), - }) + "arch": row["arch"], "target": row["target"], "tool_chain": row["tool_chain"], + "core": row["core"], "host": row["host"], "compiler_source": row["compiler_source"], + "legacy_debugger": row["legacy_debugger"], + } + if repository == "closed": + selected_row["shipping"] = str(row["shipping"]).lower() + selected_row["package_suffix"] = row["package_suffix"] + selected.append(selected_row) if not selected: raise ValueError(f"No builds configured for repository mode: {repository}") return selected def main(argv: list[str] | None = None) -> None: + """Print compact matrix JSON; argparse reports input failures with exit 2.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--repo", required=True, choices=REPOSITORIES) parser.add_argument("--host", choices=EXECUTION_CHOICES["host"]) parser.add_argument("--format", choices=("json", "github", "ado"), default="json") - args = parser.parse_args(argv) + args = parser.parse_args(argv, namespace=MatrixArguments()) try: rows = select_builds(args.repo, host=args.host) except (OSError, ValueError) as error: parser.error(str(error)) + result: object if args.format == "github": result = {"include": rows} elif args.format == "ado": - result = {row["id"]: {key: value for key, value in row.items() if key != "id"} for row in rows} + result = {row["id"]: {key: value for key, value in cast(Mapping[str, str], row).items() if key != "id"} + for row in rows} else: result = rows print(json.dumps(result, separators=(",", ":"))) diff --git a/ci/scripts/source_checks.py b/ci/scripts/source_checks.py index c752f574ff..7d6c47c47e 100644 --- a/ci/scripts/source_checks.py +++ b/ci/scripts/source_checks.py @@ -1,4 +1,10 @@ -"""Prepare the workspace and run source checks using native Stuart commands.""" +"""Prepare the workspace and run source checks using native Stuart commands. + +Shared by local developers and both provider adapters. Activate the intended +Python environment first: pip uses this interpreter and Stuart is found on PATH. +This entry point installs requirements and updates dependencies; for an inner +loop without preparation, invoke stuart_ci_build directly. +""" from __future__ import annotations @@ -13,12 +19,22 @@ def run(command: list[str]) -> None: + """Log and execute an argument vector at the repo root without a shell. + + Output is inherited by the caller. Raises CalledProcessError on a nonzero + exit or OSError when the executable cannot be started; neither is suppressed. + """ print(f"==> {shlex.join(command)}", flush=True) subprocess.run(command, cwd=REPO_ROOT, check=True) def run_source_checks() -> None: - commands = ( + """Install requirements, prepare CI dependencies, and run the selected checks. + + Uses CISettings.py, not platform build settings. Commands run sequentially; + the first failure propagates and prevents all subsequent commands. + """ + commands: tuple[list[str], ...] = ( [sys.executable, "-m", "pip", "install", "-r", "pip-requirements.txt"], ["stuart_setup", "-c", CI_SETTINGS], ["stuart_update", "-c", CI_SETTINGS], @@ -37,6 +53,11 @@ def run_source_checks() -> None: def main(argv: list[str] | None = None) -> None: + """Parse the no-argument CLI (including --help), then run source checks. + + None reads sys.argv. Argparse exits for help or invalid arguments; execution + failures propagate to the caller. This function does not create a venv. + """ parser = argparse.ArgumentParser(description=__doc__) parser.parse_args(argv) run_source_checks() diff --git a/ci/tests/test_build.py b/ci/tests/test_build.py index 0278b09ae2..a63dfa5333 100644 --- a/ci/tests/test_build.py +++ b/ci/tests/test_build.py @@ -1,4 +1,5 @@ -import importlib.util +"""Verify build orchestration without installing tools or compiling firmware.""" + import io import os import subprocess @@ -7,25 +8,25 @@ import unittest from contextlib import redirect_stderr, redirect_stdout from pathlib import Path -from unittest.mock import patch +from unittest.mock import MagicMock, patch + +from ci.scripts import build SCRIPT_PATH = Path(__file__).parents[1] / "scripts" / "build.py" -SPEC = importlib.util.spec_from_file_location("build", SCRIPT_PATH) -assert SPEC is not None and SPEC.loader is not None -build = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(build) class BuildTests(unittest.TestCase): + """Lock command ordering, flavor propagation, and fail-fast behavior.""" ARGS = ["--arch", "X64", "--target", "DEBUG", "--tool-chain", "CLANGPDB", "--core", "legacy"] - def setUp(self): + def setUp(self) -> None: + """Suppress command previews without executing external processes.""" self.output = redirect_stdout(io.StringIO()) self.output.__enter__() self.addCleanup(self.output.__exit__, None, None, None) - @patch.object(build.subprocess, "run") - def test_platform_preparation_precedes_build(self, run_mock): + @patch("ci.scripts.build.subprocess.run") + def test_platform_preparation_precedes_build(self, run_mock: MagicMock) -> None: self.assertEqual(build.main(self.ARGS), 0) calls = run_mock.call_args_list self.assertEqual(len(calls), 4) @@ -43,16 +44,16 @@ def test_platform_preparation_precedes_build(self, run_mock): self.assertIn("LaunchLogOnSuccess=FALSE", calls[-1].args[0]) self.assertIn("LaunchLogOnError=FALSE", calls[-1].args[0]) - @patch.object(build.subprocess, "run") - def test_patina_release_flavor(self, run_mock): + @patch("ci.scripts.build.subprocess.run") + def test_patina_release_flavor(self, run_mock: MagicMock) -> None: build.main(["--arch", "AARCH64", "--target", "RELEASE", "--tool-chain", "CLANGPDB", "--core", "patina"]) for invocation in run_mock.call_args_list[1:]: self.assertIn("BLD_*_USE_LEGACY_C_CORE=FALSE", invocation.args[0]) self.assertIn("BUILD_ARCH=AARCH64", invocation.args[0]) self.assertIn("TARGET=RELEASE", invocation.args[0]) - @patch.object(build.subprocess, "run") - def test_failure_stops_remaining_commands(self, run_mock): + @patch("ci.scripts.build.subprocess.run") + def test_failure_stops_remaining_commands(self, run_mock: MagicMock) -> None: for failed_step in range(4): with self.subTest(failed_step=failed_step): run_mock.reset_mock() @@ -60,19 +61,19 @@ def test_failure_stops_remaining_commands(self, run_mock): self.assertEqual(build.main(self.ARGS), 7) self.assertEqual(run_mock.call_count, failed_step + 1) - @patch.object(build.subprocess, "run") - def test_dry_run_has_no_side_effects(self, run_mock): + @patch("ci.scripts.build.subprocess.run") + def test_dry_run_has_no_side_effects(self, run_mock: MagicMock) -> None: self.assertEqual(build.main([*self.ARGS, "--dry-run"]), 0) run_mock.assert_not_called() - @patch.object(build.subprocess, "run") - def test_debugger_passed_to_all_stuart_phases(self, run_mock): + @patch("ci.scripts.build.subprocess.run") + def test_debugger_passed_to_all_stuart_phases(self, run_mock: MagicMock) -> None: build.main([*self.ARGS, "--legacy-debugger", "1"]) for invocation in run_mock.call_args_list[1:]: self.assertIn("BLD_*_LEGACY_DEBUGGER=1", invocation.args[0]) - @patch.object(build.subprocess, "run") - def test_windows_org_cannot_fall_back_to_visual_studio(self, run_mock): + @patch("ci.scripts.build.subprocess.run") + def test_windows_org_cannot_fall_back_to_visual_studio(self, run_mock: MagicMock) -> None: with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error: build.main([*self.ARGS, "--host", "windows", "--compiler-source", "windows_org", "--dry-run"]) self.assertEqual(error.exception.code, 2) @@ -85,23 +86,23 @@ def test_windows_org_cannot_fall_back_to_visual_studio(self, run_mock): self.assertIn("CLANG_BIN=", output.getvalue()) run_mock.assert_not_called() - @patch.object(build.subprocess, "run") - def test_arm64_msvc_is_rejected_before_setup(self, run_mock): + @patch("ci.scripts.build.subprocess.run") + def test_arm64_msvc_is_rejected_before_setup(self, run_mock: MagicMock) -> None: with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error: build.main(["--arch", "AARCH64", "--target", "RELEASE", "--tool-chain", "VS2022", "--core", "legacy"]) self.assertEqual(error.exception.code, 2) run_mock.assert_not_called() - @patch.object(build.subprocess, "run") - def test_explicit_clang_bin_overrides_child_environment_only(self, run_mock): + @patch("ci.scripts.build.subprocess.run") + def test_explicit_clang_bin_overrides_child_environment_only(self, run_mock: MagicMock) -> None: with patch.dict(os.environ, {"CLANG_BIN": "visual-studio/bin/"}), patch.object(Path, "is_file", return_value=True): build.main([*self.ARGS, "--clang-bin", "internal-llvm/bin"]) self.assertEqual(os.environ["CLANG_BIN"], "visual-studio/bin/") for invocation in run_mock.call_args_list: self.assertEqual(invocation.kwargs["env"]["CLANG_BIN"], Path("internal-llvm/bin").resolve().as_posix() + "/") - @patch.object(build.subprocess, "run") - def test_invalid_or_missing_flavor_is_rejected(self, run_mock): + @patch("ci.scripts.build.subprocess.run") + def test_invalid_or_missing_flavor_is_rejected(self, run_mock: MagicMock) -> None: for args in ([], [*self.ARGS[:-1], "unknown"]): with self.subTest(args=args), redirect_stderr(io.StringIO()): with self.assertRaises(SystemExit) as error: diff --git a/ci/tests/test_matrix.py b/ci/tests/test_matrix.py index dc6fe16393..10daf88edf 100644 --- a/ci/tests/test_matrix.py +++ b/ci/tests/test_matrix.py @@ -1,49 +1,44 @@ -import importlib.util +"""Test repository coverage, serialization, and invalid input boundaries.""" + import io import json import unittest from contextlib import redirect_stderr, redirect_stdout from pathlib import Path +from typing import cast from unittest.mock import patch -SCRIPTS = Path(__file__).parents[1] / "scripts" - - -def load_script(name): - spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -matrix = load_script("matrix") -build = load_script("build") +from ci.scripts import build, matrix class MatrixTests(unittest.TestCase): - def test_each_repository_row_is_accepted_by_build_runner(self): + """Preserve execution and package contracts for both CI providers.""" + + def test_each_repository_row_is_accepted_by_build_runner(self) -> None: for repository in matrix.REPOSITORIES: for row in matrix.select_builds(repository): with self.subTest(repository=repository, row=row), redirect_stdout(io.StringIO()): - args = ["--dry-run"] - for field in (*matrix.FLAVOR_CHOICES, *matrix.EXECUTION_CHOICES): - args.extend(["--" + field.replace("_", "-"), row[field]]) + args = [ + "--dry-run", "--arch", row["arch"], "--target", row["target"], + "--tool-chain", row["tool_chain"], "--core", row["core"], + "--host", row["host"], "--compiler-source", row["compiler_source"], + "--legacy-debugger", row["legacy_debugger"], + ] if row["compiler_source"] == "windows_org": args.extend(["--clang-bin", "internal-llvm/bin"]) - with patch.object(build.subprocess, "run") as run_mock: + with patch("ci.scripts.build.subprocess.run") as run_mock: self.assertEqual(build.main(args), 0) run_mock.assert_not_called() - def test_profiles_are_filtered_explicitly(self): - row = json.loads(matrix.MATRIX_PATH.read_text())[0] - open_row = {**row, "target": "DEBUG", "repositories": ["open"]} - closed_row = {**row, "target": "RELEASE", "repositories": ["closed"], "shipping": False, "package_suffix": "XRCV"} + def test_profiles_are_filtered_explicitly(self) -> None: + row = matrix.parse_definition(json.loads(matrix.MATRIX_PATH.read_text())[0]) + open_row: matrix.BuildDefinition = {**row, "target": "DEBUG", "repositories": ["open"]} + closed_row: matrix.BuildDefinition = {**row, "target": "RELEASE", "repositories": ["closed"], "shipping": False, "package_suffix": "XRCV"} with patch.object(Path, "read_text", return_value=json.dumps([open_row, closed_row])): self.assertEqual([item["id"] for item in matrix.select_builds("open")], [matrix.build_id(open_row)]) self.assertEqual([item["id"] for item in matrix.select_builds("closed")], [matrix.build_id(closed_row)]) - def test_coverage_and_derived_identifiers(self): + def test_coverage_and_derived_identifiers(self) -> None: rows = matrix.select_builds("open") expected = { (arch, target, tool_chain, core) @@ -56,7 +51,7 @@ def test_coverage_and_derived_identifiers(self): ("AARCH64", "CLANGPDB", "patina"), ) } - self.assertEqual({tuple(row[field] for field in matrix.FLAVOR_CHOICES) for row in rows}, expected) + self.assertEqual({(row["arch"], row["target"], row["tool_chain"], row["core"]) for row in rows}, expected) self.assertEqual(len(rows), 10) for row in rows: self.assertEqual(row["id"], matrix.build_id(row)) @@ -78,7 +73,7 @@ def test_coverage_and_derived_identifiers(self): self.assertEqual({row["package_suffix"] for row in closed if row["shipping"] == "false"}, {"XDCV", "XRCV", "ADCV", "ARCV", "XDCW", "XRCW", "ADCW", "XDCL", "XRCL", "ADCL", "ARCL", "XRG"}) - def test_provider_formats_preserve_same_flavors(self): + def test_provider_formats_preserve_same_flavors(self) -> None: for repository in matrix.REPOSITORIES: rows = matrix.select_builds(repository) for format_name in ("json", "github", "ado"): @@ -86,16 +81,17 @@ def test_provider_formats_preserve_same_flavors(self): output = io.StringIO() with redirect_stdout(output): matrix.main(["--repo", repository, "--format", format_name]) - result = json.loads(output.getvalue()) + result: object = json.loads(output.getvalue()) + expected: object = rows if format_name == "github": - result = result["include"] + expected = {"include": rows} elif format_name == "ado": - result = [{"id": identifier, **flavor} for identifier, flavor in result.items()] - self.assertEqual(result, rows) + expected = {row["id"]: {key: value for key, value in row.items() if key != "id"} for row in rows} + self.assertEqual(result, expected) - def test_invalid_matrix_is_rejected(self): - row = json.loads(matrix.MATRIX_PATH.read_text())[0] - invalid_documents = [ + def test_invalid_matrix_is_rejected(self) -> None: + row = matrix.parse_definition(json.loads(matrix.MATRIX_PATH.read_text())[0]) + invalid_documents: list[object] = [ {}, [], [row, row], [{**row, "id": "invalid-id"}], [{**row, "repositories": ["auto"]}], [{**row, "arch": "unknown"}], [{**row, "core": "unknown"}], [{**row, "unexpected": True}], @@ -109,7 +105,34 @@ def test_invalid_matrix_is_rejected(self): with self.assertRaises(ValueError): matrix.select_builds("open") - def test_repository_mode_is_required(self): + def test_wrong_json_types_are_rejected(self) -> None: + """Untrusted values must fail validation, not leak into typed records.""" + row = matrix.parse_definition(json.loads(matrix.MATRIX_PATH.read_text())[0]) + invalid_rows: list[object] = [None, [], "flavor", 1] + invalid_scalars: tuple[object, ...] = (None, True, 1, [], {}) + invalid_repositories: tuple[object, ...] = (None, "open", [], [True], [[]]) + for field in (*matrix.FLAVOR_CHOICES, *matrix.EXECUTION_CHOICES): + invalid_rows.extend({**row, field: value} for value in invalid_scalars) + invalid_rows.extend({**row, "repositories": value} for value in invalid_repositories) + for invalid in invalid_rows: + with self.subTest(row=invalid), self.assertRaises(ValueError): + matrix.parse_definition(invalid) + closed = {**row, "repositories": ["closed"], "shipping": False, "package_suffix": "XDCV"} + for field, value in (("shipping", "false"), ("shipping", 0), ("package_suffix", None)): + with self.subTest(field=field, value=value), self.assertRaises(ValueError): + matrix.parse_definition({**closed, field: value}) + + def test_package_collisions_are_rejected(self) -> None: + """Different executions cannot silently publish the same closed package.""" + raw: object = json.loads(matrix.MATRIX_PATH.read_text()) + definitions = [matrix.parse_definition(value) for value in cast(list[object], raw)] + closed = next(row for row in definitions if "closed" in row["repositories"]) + other = {**closed, "legacy_debugger": "0"} + with patch.object(Path, "read_text", return_value=json.dumps([closed, other])): + with self.assertRaisesRegex(ValueError, "Duplicate closed package"): + matrix.select_builds("closed") + + def test_repository_mode_is_required(self) -> None: for args in ([], ["--repo", "auto"]): with self.subTest(args=args), redirect_stderr(io.StringIO()): with self.assertRaises(SystemExit) as error: diff --git a/ci/tests/test_source_checks.py b/ci/tests/test_source_checks.py index 3b3fe72ca0..ddeb32a878 100644 --- a/ci/tests/test_source_checks.py +++ b/ci/tests/test_source_checks.py @@ -1,19 +1,17 @@ -import importlib.util +"""Verify the shared source-check sequence without invoking pip or Stuart.""" + import sys import unittest -from pathlib import Path -from unittest.mock import call, patch +from unittest.mock import MagicMock, call, patch -SCRIPT_PATH = Path(__file__).parents[1] / "scripts" / "source_checks.py" -SPEC = importlib.util.spec_from_file_location("source_checks", SCRIPT_PATH) -assert SPEC is not None and SPEC.loader is not None -source_checks = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(source_checks) +from ci.scripts import source_checks class SourceCheckTests(unittest.TestCase): - @patch.object(source_checks.subprocess, "run") - def test_source_checks_use_native_stuart_commands(self, run_mock): + """Ensure provider callers and local developers use the native lifecycle.""" + + @patch("ci.scripts.source_checks.subprocess.run") + def test_source_checks_use_native_stuart_commands(self, run_mock: MagicMock) -> None: source_checks.main([]) expected_commands = [ From e474adcf5460f1dd6119c01b0f9e713099f37a85 Mon Sep 17 00:00:00 2001 From: Maheer Aeron Date: Wed, 23 Sep 2026 14:31:19 -0700 Subject: [PATCH 10/18] ci: split backend build matrices and group shipping flavors --- .azuredevops/build-matrix.json | 21 ++++++++ .azuredevops/templates/build.yml | 9 ++-- .azuredevops/templates/platform.yml | 2 +- .github/build-matrix.json | 12 +++++ .github/workflows/platform-ci.yml | 2 +- MsvmPkg/Docs/BUILDING.md | 23 ++++----- ci/build-matrix.json | 27 ----------- ci/scripts/matrix.py | 60 +++++++++-------------- ci/tests/test_matrix.py | 75 ++++++++++++++++++----------- 9 files changed, 120 insertions(+), 111 deletions(-) create mode 100644 .azuredevops/build-matrix.json create mode 100644 .github/build-matrix.json delete mode 100644 ci/build-matrix.json diff --git a/.azuredevops/build-matrix.json b/.azuredevops/build-matrix.json new file mode 100644 index 0000000000..afcfe4827f --- /dev/null +++ b/.azuredevops/build-matrix.json @@ -0,0 +1,21 @@ +[ + {"host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "VS2022", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "1", "shipping": true, "package_suffix": ""}, + {"host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "VS2022", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": true, "package_suffix": ""}, + {"host": "windows", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "0", "shipping": true, "package_suffix": ""}, + + {"host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "1", "shipping": false, "package_suffix": "XDCV"}, + {"host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRCV"}, + {"host": "windows", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ADCV"}, + {"host": "windows", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ARCV"}, + + {"host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "1", "shipping": false, "package_suffix": "XDCW"}, + {"host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRCW"}, + {"host": "windows", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ADCW"}, + + {"host": "linux", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "1", "shipping": false, "package_suffix": "XDCL"}, + {"host": "linux", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRCL"}, + {"host": "linux", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ADCL"}, + {"host": "linux", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ARCL"}, + + {"host": "linux", "arch": "X64", "target": "RELEASE", "tool_chain": "GCC", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRG"} +] diff --git a/.azuredevops/templates/build.yml b/.azuredevops/templates/build.yml index 7f56bb87fe..d439e63bc3 100644 --- a/.azuredevops/templates/build.yml +++ b/.azuredevops/templates/build.yml @@ -1,9 +1,6 @@ parameters: - - name: repositoryMode + - name: matrixFile type: string - values: - - open - - closed - name: hosts type: object default: @@ -29,13 +26,13 @@ jobs: steps: - ${{ each host in parameters.hosts }}: - pwsh: | - $matrix = python ci/scripts/matrix.py --repo "$env:REPOSITORY_MODE" --host "$env:BUILD_HOST" --format ado + $matrix = python ci/scripts/matrix.py --matrix "$env:MATRIX_FILE" --host "$env:BUILD_HOST" --format ado if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } Write-Host "##vso[task.setvariable variable=matrix;isOutput=true]$matrix" name: ${{ host }} displayName: Select ${{ host }} Builds env: - REPOSITORY_MODE: ${{ parameters.repositoryMode }} + MATRIX_FILE: ${{ parameters.matrixFile }} BUILD_HOST: ${{ host }} - ${{ each host in parameters.hosts }}: diff --git a/.azuredevops/templates/platform.yml b/.azuredevops/templates/platform.yml index 5055e632d4..7abcdc51a1 100644 --- a/.azuredevops/templates/platform.yml +++ b/.azuredevops/templates/platform.yml @@ -27,4 +27,4 @@ extends: jobs: - template: /.azuredevops/templates/build.yml@self parameters: - repositoryMode: closed + matrixFile: .azuredevops/build-matrix.json diff --git a/.github/build-matrix.json b/.github/build-matrix.json new file mode 100644 index 0000000000..758bacbf89 --- /dev/null +++ b/.github/build-matrix.json @@ -0,0 +1,12 @@ +[ + {"host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, + {"host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, + {"host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "VS2022", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, + {"host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "VS2022", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, + {"host": "windows", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, + {"host": "windows", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, + {"host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "patina", "legacy_debugger": "0"}, + {"host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "patina", "legacy_debugger": "0"}, + {"host": "windows", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "patina", "legacy_debugger": "0"}, + {"host": "windows", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "patina", "legacy_debugger": "0"} +] diff --git a/.github/workflows/platform-ci.yml b/.github/workflows/platform-ci.yml index 433b6afacd..b071ebe21f 100644 --- a/.github/workflows/platform-ci.yml +++ b/.github/workflows/platform-ci.yml @@ -28,7 +28,7 @@ jobs: id: select shell: bash run: | - matrix="$(python ci/scripts/matrix.py --repo open --format github)" + matrix="$(python ci/scripts/matrix.py --matrix .github/build-matrix.json --format github)" echo "matrix=$matrix" >> "$GITHUB_OUTPUT" build: diff --git a/MsvmPkg/Docs/BUILDING.md b/MsvmPkg/Docs/BUILDING.md index 1221f5f45f..5351b0b2ab 100644 --- a/MsvmPkg/Docs/BUILDING.md +++ b/MsvmPkg/Docs/BUILDING.md @@ -61,13 +61,15 @@ This command builds one flavor only; it does not select an open/closed repositor ### Selecting Repository Coverage -`ci/build-matrix.json` is the shared build list. Each row explicitly lists the repository modes that use it: -`open` for mu_msvm and `closed` for hyperv.uefi. Repository mode selects coverage, not source code, -credentials, official-build status, or publishing permissions. +Coverage is maintained separately in `.github/build-matrix.json` (open-source mu_msvm) and +`.azuredevops/build-matrix.json` (closed-source hyperv.uefi). Both use the shared validator and build script. +Rows have no `repositories` field: the caller explicitly selects a file with `--matrix` instead of `--repo`. +The path selects coverage, not source code, credentials, official-build status, or publishing permissions. +Relative paths resolve from the current directory; either file can be selected locally. ```powershell -python .\ci\scripts\matrix.py --repo open -python .\ci\scripts\matrix.py --repo closed +python .\ci\scripts\matrix.py --matrix .github/build-matrix.json +python .\ci\scripts\matrix.py --matrix .azuredevops/build-matrix.json ``` The open profile contains ten Windows-hosted builds: DEBUG and RELEASE for X64 VS2022 with the legacy core, @@ -86,19 +88,18 @@ Closed X64 DEBUG builds enable the legacy debugger. Open builds retain their exi ARM64 MSVC is unsupported. Disabled GCC combinations are not reintroduced. IDs are derived as `______`; -do not enter IDs in the JSON. Identical flavors may belong to different repository profiles, but may not -occur twice within one profile. -Additional coverage can be added by editing the JSON, without duplicating flavor definitions in provider YAML. -The selector rejects duplicate flavors and fails when a requested repository has no builds. +do not enter IDs in the JSON. Identical flavors may occur in both files, but may not occur twice within one file. +Edit the respective JSON to change coverage independently, without editing shared Python or provider YAML. +The selector rejects duplicate flavors and fails when the supplied file or requested host has no builds. `--format github` emits an `include` object; `--format ado` emits a job-leg name to variables mapping. These commands only print JSON. They do not execute builds, install tools, or publish anything. -GitHub's platform workflow selects `open` coverage, feeds the result to its native `strategy.matrix`, and +GitHub's platform workflow supplies `.github/build-matrix.json`, feeds the result to its native `strategy.matrix`, and calls the build action for each flavor. That action invokes the same `ci/scripts/build.py` used locally. Builds depend only on matrix generation; source checks run independently. Build artifact publication is not part of this step yet. -ADO's shared platform template selects `closed` coverage through its build job template. A `BuildMatrix` +ADO's shared platform template supplies `.azuredevops/build-matrix.json` through its build job template. A `BuildMatrix` job exports separate Windows and Linux matrices using `--host`. Dependent `BuildFirmware_windows` and `BuildFirmware_linux` jobs consume them through native `strategy.matrix`, with at most four parallel builds per host group. Host pools are selected at template expansion time, not by runtime matrix variables. diff --git a/ci/build-matrix.json b/ci/build-matrix.json deleted file mode 100644 index b59c220dac..0000000000 --- a/ci/build-matrix.json +++ /dev/null @@ -1,27 +0,0 @@ -[ - {"repositories": ["open"], "host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, - {"repositories": ["open"], "host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, - {"repositories": ["open"], "host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "VS2022", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, - {"repositories": ["open"], "host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "VS2022", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, - {"repositories": ["open"], "host": "windows", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, - {"repositories": ["open"], "host": "windows", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0"}, - {"repositories": ["open"], "host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "patina", "legacy_debugger": "0"}, - {"repositories": ["open"], "host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "patina", "legacy_debugger": "0"}, - {"repositories": ["open"], "host": "windows", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "patina", "legacy_debugger": "0"}, - {"repositories": ["open"], "host": "windows", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "patina", "legacy_debugger": "0"}, - {"repositories": ["closed"], "host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "VS2022", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "1", "shipping": true, "package_suffix": ""}, - {"repositories": ["closed"], "host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "VS2022", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": true, "package_suffix": ""}, - {"repositories": ["closed"], "host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "1", "shipping": false, "package_suffix": "XDCV"}, - {"repositories": ["closed"], "host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRCV"}, - {"repositories": ["closed"], "host": "windows", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ADCV"}, - {"repositories": ["closed"], "host": "windows", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "visual_studio", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ARCV"}, - {"repositories": ["closed"], "host": "windows", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "1", "shipping": false, "package_suffix": "XDCW"}, - {"repositories": ["closed"], "host": "windows", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRCW"}, - {"repositories": ["closed"], "host": "windows", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ADCW"}, - {"repositories": ["closed"], "host": "windows", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "windows_org", "core": "legacy", "legacy_debugger": "0", "shipping": true, "package_suffix": ""}, - {"repositories": ["closed"], "host": "linux", "arch": "X64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "1", "shipping": false, "package_suffix": "XDCL"}, - {"repositories": ["closed"], "host": "linux", "arch": "X64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRCL"}, - {"repositories": ["closed"], "host": "linux", "arch": "AARCH64", "target": "DEBUG", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ADCL"}, - {"repositories": ["closed"], "host": "linux", "arch": "AARCH64", "target": "RELEASE", "tool_chain": "CLANGPDB", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "ARCL"}, - {"repositories": ["closed"], "host": "linux", "arch": "X64", "target": "RELEASE", "tool_chain": "GCC", "compiler_source": "distribution", "core": "legacy", "legacy_debugger": "0", "shipping": false, "package_suffix": "XRG"} -] diff --git a/ci/scripts/matrix.py b/ci/scripts/matrix.py index baadfb5ef1..8f925e0321 100644 --- a/ci/scripts/matrix.py +++ b/ci/scripts/matrix.py @@ -1,8 +1,8 @@ """Validate repository coverage and emit provider-specific matrix JSON. This module selects data only: it never builds firmware or authorizes publication. -All rows are validated before filtering, including rows outside the requested -repository/host. Generated IDs identify executions; package suffixes are a +All rows in the supplied file are validated before host filtering. +Generated IDs identify executions; package suffixes are a separate consumer-facing contract. """ @@ -15,7 +15,6 @@ from pathlib import Path from typing import Literal, NotRequired, TypedDict, cast -Repository = Literal["open", "closed"] Host = Literal["windows", "linux"] Architecture = Literal["X64", "AARCH64"] Target = Literal["DEBUG", "RELEASE"] @@ -39,9 +38,8 @@ class BuildFlavor(TypedDict): class BuildDefinition(BuildFlavor): - """Validated JSON row; closed coverage requires both package fields.""" + """Validated JSON row; optional package metadata must supply both fields.""" - repositories: list[Repository] shipping: NotRequired[bool] package_suffix: NotRequired[str] @@ -61,12 +59,10 @@ class SelectedBuild(BuildFlavor): class MatrixArguments(argparse.Namespace): """CLI values constrained by argparse choices before selection.""" - repo: Repository + matrix: Path host: Host | None format: OutputFormat -MATRIX_PATH = Path(__file__).resolve().parents[1] / "build-matrix.json" -REPOSITORIES: tuple[Repository, ...] = ("open", "closed") FLAVOR_CHOICES: dict[str, tuple[str, ...]] = { "arch": ("X64", "AARCH64"), "target": ("DEBUG", "RELEASE"), @@ -94,20 +90,14 @@ def parse_definition(value: object) -> BuildDefinition: unsupported compilers, or inconsistent package metadata. """ if not isinstance(value, dict): - raise ValueError("Each build must specify repository coverage and a complete execution flavor") + raise ValueError("Each build must specify a complete execution flavor") row = cast(dict[str, object], value) - required = {"repositories", *FLAVOR_CHOICES, *EXECUTION_CHOICES} + required = {*FLAVOR_CHOICES, *EXECUTION_CHOICES} if not required <= row.keys() or row.keys() - required - {"shipping", "package_suffix"}: - raise ValueError("Each build must specify repository coverage and a complete execution flavor") + raise ValueError("Each build must specify a complete execution flavor") for field, choices in {**FLAVOR_CHOICES, **EXECUTION_CHOICES}.items(): if not isinstance(row[field], str) or row[field] not in choices: raise ValueError(f"Invalid {field}: {row[field]!r}") - repositories = row["repositories"] - if not isinstance(repositories, list) or not repositories: - raise ValueError("Invalid repositories") - modes = cast(list[object], repositories) - if any(not isinstance(mode, str) or mode not in REPOSITORIES for mode in modes): - raise ValueError("Invalid repositories") valid_compilers = { ("windows", "VS2022", "visual_studio"), ("windows", "CLANGPDB", "visual_studio"), @@ -119,73 +109,69 @@ def parse_definition(value: object) -> BuildDefinition: raise ValueError("Unsupported host/compiler combination") if row["arch"] == "AARCH64" and row["tool_chain"] == "VS2022": raise ValueError("ARM64 MSVC is not supported") - if "closed" in modes: + if "shipping" in row or "package_suffix" in row: suffix = row.get("package_suffix") if type(row.get("shipping")) is not bool or not isinstance(suffix, str): - raise ValueError("Closed builds require explicit shipping and package_suffix") + raise ValueError("Package metadata requires explicit shipping and package_suffix") if not re.fullmatch(r"[A-Za-z0-9]*", suffix) or row["shipping"] != (suffix == ""): raise ValueError("Invalid shipping/package suffix contract") - elif "shipping" in row or "package_suffix" in row: - raise ValueError("Closed package metadata must belong to closed repository coverage") return cast(BuildDefinition, row) -def select_builds(repository: str, path: Path = MATRIX_PATH, host: str | None = None) -> list[SelectedBuild]: - """Load validated builds for a repository, optionally restricted by host. +def select_builds(path: Path, host: str | None = None) -> list[SelectedBuild]: + """Load the explicitly supplied matrix file, optionally restricted by host. Preserves declaration order and rejects duplicate execution/package IDs. + Relative paths resolve against the caller's working directory. No repository + or CI backend is inferred from the path; either file can be used locally. Raises OSError for unreadable input and ValueError for invalid JSON, invalid definitions, unknown filters, or an empty selection. No commands are run. """ - if repository not in REPOSITORIES: - raise ValueError(f"Unknown repository mode: {repository}") if host is not None and host not in EXECUTION_CHOICES["host"]: raise ValueError(f"Unknown host: {host}") document: object = json.loads(path.read_text(encoding="utf-8")) if not isinstance(document, list): raise ValueError("Build matrix must be a list") selected: list[SelectedBuild] = [] - identifiers: set[tuple[Repository, str]] = set() + identifiers: set[str] = set() package_names: set[tuple[Architecture, Target, str]] = set() for value in cast(list[object], document): row = parse_definition(value) identifier = build_id(row) - repositories = row["repositories"] - for mode in repositories: - if (mode, identifier) in identifiers: - raise ValueError(f"Duplicate matrix identifier for {mode}: {identifier}") - identifiers.add((mode, identifier)) - if "closed" in repositories: + if identifier in identifiers: + raise ValueError(f"Duplicate matrix identifier: {identifier}") + identifiers.add(identifier) + if "package_suffix" in row: suffix = row["package_suffix"] package_name = (row["arch"], row["target"], suffix) if package_name in package_names: raise ValueError(f"Duplicate closed package identity: {package_name}") package_names.add(package_name) - if repository in repositories and (host is None or row["host"] == host): + if host is None or row["host"] == host: selected_row: SelectedBuild = { "id": identifier, "arch": row["arch"], "target": row["target"], "tool_chain": row["tool_chain"], "core": row["core"], "host": row["host"], "compiler_source": row["compiler_source"], "legacy_debugger": row["legacy_debugger"], } - if repository == "closed": + if "shipping" in row: selected_row["shipping"] = str(row["shipping"]).lower() selected_row["package_suffix"] = row["package_suffix"] selected.append(selected_row) if not selected: - raise ValueError(f"No builds configured for repository mode: {repository}") + raise ValueError(f"No builds selected from {path} for host: {host or 'all'}") return selected def main(argv: list[str] | None = None) -> None: """Print compact matrix JSON; argparse reports input failures with exit 2.""" parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--repo", required=True, choices=REPOSITORIES) + parser.add_argument("--matrix", required=True, type=Path, help="Path to the build coverage JSON file") parser.add_argument("--host", choices=EXECUTION_CHOICES["host"]) parser.add_argument("--format", choices=("json", "github", "ado"), default="json") args = parser.parse_args(argv, namespace=MatrixArguments()) try: - rows = select_builds(args.repo, host=args.host) + rows = select_builds(args.matrix, host=args.host) except (OSError, ValueError) as error: parser.error(str(error)) result: object diff --git a/ci/tests/test_matrix.py b/ci/tests/test_matrix.py index 10daf88edf..cb86b65b1a 100644 --- a/ci/tests/test_matrix.py +++ b/ci/tests/test_matrix.py @@ -10,14 +10,19 @@ from ci.scripts import build, matrix +REPO_ROOT = Path(__file__).resolve().parents[2] +OPEN_MATRIX = REPO_ROOT / ".github" / "build-matrix.json" +CLOSED_MATRIX = REPO_ROOT / ".azuredevops" / "build-matrix.json" +MATRICES = (OPEN_MATRIX, CLOSED_MATRIX) + class MatrixTests(unittest.TestCase): """Preserve execution and package contracts for both CI providers.""" def test_each_repository_row_is_accepted_by_build_runner(self) -> None: - for repository in matrix.REPOSITORIES: - for row in matrix.select_builds(repository): - with self.subTest(repository=repository, row=row), redirect_stdout(io.StringIO()): + for path in MATRICES: + for row in matrix.select_builds(path): + with self.subTest(path=path, row=row), redirect_stdout(io.StringIO()): args = [ "--dry-run", "--arch", row["arch"], "--target", row["target"], "--tool-chain", row["tool_chain"], "--core", row["core"], @@ -30,16 +35,16 @@ def test_each_repository_row_is_accepted_by_build_runner(self) -> None: self.assertEqual(build.main(args), 0) run_mock.assert_not_called() - def test_profiles_are_filtered_explicitly(self) -> None: - row = matrix.parse_definition(json.loads(matrix.MATRIX_PATH.read_text())[0]) - open_row: matrix.BuildDefinition = {**row, "target": "DEBUG", "repositories": ["open"]} - closed_row: matrix.BuildDefinition = {**row, "target": "RELEASE", "repositories": ["closed"], "shipping": False, "package_suffix": "XRCV"} - with patch.object(Path, "read_text", return_value=json.dumps([open_row, closed_row])): - self.assertEqual([item["id"] for item in matrix.select_builds("open")], [matrix.build_id(open_row)]) - self.assertEqual([item["id"] for item in matrix.select_builds("closed")], [matrix.build_id(closed_row)]) + def test_only_the_supplied_matrix_is_read(self) -> None: + """Coverage is selected by the caller, never inferred from the backend.""" + row = matrix.parse_definition(json.loads(OPEN_MATRIX.read_text())[0]) + with patch.object(Path, "read_text", autospec=True, return_value=json.dumps([row])) as read_mock: + path = Path("custom") / "coverage.json" + self.assertEqual([item["id"] for item in matrix.select_builds(path)], [matrix.build_id(row)]) + read_mock.assert_called_once_with(path, encoding="utf-8") def test_coverage_and_derived_identifiers(self) -> None: - rows = matrix.select_builds("open") + rows = matrix.select_builds(OPEN_MATRIX) expected = { (arch, target, tool_chain, core) for target in ("DEBUG", "RELEASE") @@ -55,11 +60,11 @@ def test_coverage_and_derived_identifiers(self) -> None: self.assertEqual(len(rows), 10) for row in rows: self.assertEqual(row["id"], matrix.build_id(row)) - closed = matrix.select_builds("closed") + closed = matrix.select_builds(CLOSED_MATRIX) self.assertEqual(len(closed), 15) self.assertEqual(len({row["id"] for row in closed}), 15) - self.assertEqual(len(matrix.select_builds("closed", host="windows")), 10) - self.assertEqual(len(matrix.select_builds("closed", host="linux")), 5) + self.assertEqual(len(matrix.select_builds(CLOSED_MATRIX, host="windows")), 10) + self.assertEqual(len(matrix.select_builds(CLOSED_MATRIX, host="linux")), 5) shipping = {(row["arch"], row["target"], row["tool_chain"], row["compiler_source"], row["core"]) for row in closed if row["shipping"] == "true"} self.assertEqual(shipping, { @@ -74,13 +79,13 @@ def test_coverage_and_derived_identifiers(self) -> None: {"XDCV", "XRCV", "ADCV", "ARCV", "XDCW", "XRCW", "ADCW", "XDCL", "XRCL", "ADCL", "ARCL", "XRG"}) def test_provider_formats_preserve_same_flavors(self) -> None: - for repository in matrix.REPOSITORIES: - rows = matrix.select_builds(repository) + for path in MATRICES: + rows = matrix.select_builds(path) for format_name in ("json", "github", "ado"): - with self.subTest(repository=repository, format=format_name): + with self.subTest(path=path, format=format_name): output = io.StringIO() with redirect_stdout(output): - matrix.main(["--repo", repository, "--format", format_name]) + matrix.main(["--matrix", str(path), "--format", format_name]) result: object = json.loads(output.getvalue()) expected: object = rows if format_name == "github": @@ -90,7 +95,7 @@ def test_provider_formats_preserve_same_flavors(self) -> None: self.assertEqual(result, expected) def test_invalid_matrix_is_rejected(self) -> None: - row = matrix.parse_definition(json.loads(matrix.MATRIX_PATH.read_text())[0]) + row = matrix.parse_definition(json.loads(OPEN_MATRIX.read_text())[0]) invalid_documents: list[object] = [ {}, [], [row, row], [{**row, "id": "invalid-id"}], [{**row, "repositories": ["auto"]}], [{**row, "arch": "unknown"}], @@ -98,16 +103,16 @@ def test_invalid_matrix_is_rejected(self) -> None: [{**row, "repositories": ["closed"]}], [{**row, "arch": "AARCH64", "tool_chain": "VS2022"}], [{**row, "host": "linux"}], - [{**row, "shipping": True, "package_suffix": ""}], + [{**row, "shipping": True}], [{**row, "package_suffix": ""}], ] for document in invalid_documents: with self.subTest(document=document), patch.object(Path, "read_text", return_value=json.dumps(document)): with self.assertRaises(ValueError): - matrix.select_builds("open") + matrix.select_builds(OPEN_MATRIX) def test_wrong_json_types_are_rejected(self) -> None: """Untrusted values must fail validation, not leak into typed records.""" - row = matrix.parse_definition(json.loads(matrix.MATRIX_PATH.read_text())[0]) + row = matrix.parse_definition(json.loads(OPEN_MATRIX.read_text())[0]) invalid_rows: list[object] = [None, [], "flavor", 1] invalid_scalars: tuple[object, ...] = (None, True, 1, [], {}) invalid_repositories: tuple[object, ...] = (None, "open", [], [True], [[]]) @@ -117,28 +122,42 @@ def test_wrong_json_types_are_rejected(self) -> None: for invalid in invalid_rows: with self.subTest(row=invalid), self.assertRaises(ValueError): matrix.parse_definition(invalid) - closed = {**row, "repositories": ["closed"], "shipping": False, "package_suffix": "XDCV"} + closed = {**row, "shipping": False, "package_suffix": "XDCV"} for field, value in (("shipping", "false"), ("shipping", 0), ("package_suffix", None)): with self.subTest(field=field, value=value), self.assertRaises(ValueError): matrix.parse_definition({**closed, field: value}) def test_package_collisions_are_rejected(self) -> None: """Different executions cannot silently publish the same closed package.""" - raw: object = json.loads(matrix.MATRIX_PATH.read_text()) + raw: object = json.loads(CLOSED_MATRIX.read_text()) definitions = [matrix.parse_definition(value) for value in cast(list[object], raw)] - closed = next(row for row in definitions if "closed" in row["repositories"]) + closed = definitions[0] other = {**closed, "legacy_debugger": "0"} with patch.object(Path, "read_text", return_value=json.dumps([closed, other])): with self.assertRaisesRegex(ValueError, "Duplicate closed package"): - matrix.select_builds("closed") + matrix.select_builds(CLOSED_MATRIX) - def test_repository_mode_is_required(self) -> None: - for args in ([], ["--repo", "auto"]): + def test_matrix_path_is_required(self) -> None: + for args in ([], ["--repo", "open"]): with self.subTest(args=args), redirect_stderr(io.StringIO()): with self.assertRaises(SystemExit) as error: matrix.main(args) self.assertEqual(error.exception.code, 2) + def test_host_filter_rejects_empty_or_unknown_selection(self) -> None: + for host in ("linux", "unknown"): + with self.subTest(host=host), self.assertRaises(ValueError): + matrix.select_builds(OPEN_MATRIX, host=host) + + def test_split_files_have_no_repository_tags(self) -> None: + for path in MATRICES: + raw: object = json.loads(path.read_text()) + for value in cast(list[object], raw): + definition = matrix.parse_definition(value) + self.assertNotIn("repositories", definition) + self.assertEqual("shipping" in definition, path == CLOSED_MATRIX) + self.assertEqual("package_suffix" in definition, path == CLOSED_MATRIX) + if __name__ == "__main__": unittest.main() From ca2190ee0b373805e1e33f8e2c831f062d3e6f1b Mon Sep 17 00:00:00 2001 From: Maheer Aeron Date: Wed, 23 Sep 2026 14:35:06 -0700 Subject: [PATCH 11/18] ci: restore Dependabot configuration --- .github/dependabot.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..2c48305b7e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + groups: + github-actions: + patterns: ["*"] + schedule: + interval: "weekly" + cooldown: + default-days: 7 From 0c6540f21f960e4c12a6eef1161df462439c3d40 Mon Sep 17 00:00:00 2001 From: Maheer Aeron Date: Wed, 23 Sep 2026 14:58:39 -0700 Subject: [PATCH 12/18] ci: sync firmware versioning and record source provenance --- .azuredevops/templates/build.yml | 9 +- .azuredevops/templates/platform.yml | 1 + .github/actions/build/action.yml | 1 + .github/workflows/platform-ci.yml | 2 + .../FirmwareVersionBlob.py | 68 +++++- MsvmPkg/Docs/BUILDING.md | 67 +++++- ci/scripts/build.py | 12 +- ci/scripts/matrix.py | 6 + ci/tests/test_build.py | 47 +++- ci/tests/test_firmware_version.py | 227 ++++++++++++++++++ ci/tests/test_matrix.py | 10 + pip-requirements.txt | 3 +- 12 files changed, 437 insertions(+), 16 deletions(-) create mode 100644 ci/tests/test_firmware_version.py diff --git a/.azuredevops/templates/build.yml b/.azuredevops/templates/build.yml index d439e63bc3..e8ec55372c 100644 --- a/.azuredevops/templates/build.yml +++ b/.azuredevops/templates/build.yml @@ -1,6 +1,9 @@ parameters: - name: matrixFile type: string + - name: officialBuild + type: boolean + default: false - name: hosts type: object default: @@ -98,7 +101,7 @@ jobs: $buildArgs = @('--arch', $env:BUILD_ARCH, '--target', $env:BUILD_TARGET, '--tool-chain', $env:BUILD_TOOL_CHAIN, '--core', $env:BUILD_CORE, '--host', 'windows', '--compiler-source', $env:BUILD_COMPILER_SOURCE, - '--legacy-debugger', $env:BUILD_LEGACY_DEBUGGER) + '--legacy-debugger', $env:BUILD_LEGACY_DEBUGGER, '--source-origin', 'closed') if ($env:BUILD_COMPILER_SOURCE -eq 'windows_org') { $buildArgs += @('--clang-bin', "$env:WINDOWS_CLANG_ROOT/bin") } @@ -113,6 +116,7 @@ jobs: BUILD_COMPILER_SOURCE: $(compiler_source) BUILD_LEGACY_DEBUGGER: $(legacy_debugger) WINDOWS_CLANG_ROOT: $(WindowsClangRoot) + OFFICIAL_BUILD: ${{ iif(parameters.officialBuild, '1', '0') }} - ${{ if eq(host, 'linux') }}: - bash: | @@ -136,7 +140,7 @@ jobs: source "$BUILD_VENV/bin/activate" python ci/scripts/build.py --arch "$BUILD_ARCH" --target "$BUILD_TARGET" \ --tool-chain "$BUILD_TOOL_CHAIN" --core "$BUILD_CORE" --host linux \ - --compiler-source "$BUILD_COMPILER_SOURCE" --legacy-debugger "$BUILD_LEGACY_DEBUGGER" + --compiler-source "$BUILD_COMPILER_SOURCE" --legacy-debugger "$BUILD_LEGACY_DEBUGGER" --source-origin closed displayName: Prepare Linux Compiler and Build Firmware env: BUILD_ARCH: $(arch) @@ -146,3 +150,4 @@ jobs: BUILD_COMPILER_SOURCE: $(compiler_source) BUILD_LEGACY_DEBUGGER: $(legacy_debugger) BUILD_VENV: $(Agent.TempDirectory)/firmware-venv + OFFICIAL_BUILD: ${{ iif(parameters.officialBuild, '1', '0') }} diff --git a/.azuredevops/templates/platform.yml b/.azuredevops/templates/platform.yml index 7abcdc51a1..3a5d5649d6 100644 --- a/.azuredevops/templates/platform.yml +++ b/.azuredevops/templates/platform.yml @@ -28,3 +28,4 @@ extends: - template: /.azuredevops/templates/build.yml@self parameters: matrixFile: .azuredevops/build-matrix.json + officialBuild: ${{ eq(parameters.oneBranchBase, 'v2/Microsoft.Official.yml') }} diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 05da4a7602..3d5765b597 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -57,3 +57,4 @@ runs: --host "$env:BUILD_HOST" --compiler-source "$env:BUILD_COMPILER_SOURCE" --legacy-debugger "$env:BUILD_LEGACY_DEBUGGER" + --source-origin open diff --git a/.github/workflows/platform-ci.yml b/.github/workflows/platform-ci.yml index b071ebe21f..35e1807f10 100644 --- a/.github/workflows/platform-ci.yml +++ b/.github/workflows/platform-ci.yml @@ -59,6 +59,8 @@ jobs: - name: Build firmware uses: ./.github/actions/build + env: + OFFICIAL_BUILD: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') && '1' || '0' }} with: arch: ${{ matrix.arch }} target: ${{ matrix.target }} diff --git a/MsvmPkg/BuildPlugins/FirmwareVersionBlob/FirmwareVersionBlob.py b/MsvmPkg/BuildPlugins/FirmwareVersionBlob/FirmwareVersionBlob.py index 142001f13e..7c123bc246 100644 --- a/MsvmPkg/BuildPlugins/FirmwareVersionBlob/FirmwareVersionBlob.py +++ b/MsvmPkg/BuildPlugins/FirmwareVersionBlob/FirmwareVersionBlob.py @@ -25,14 +25,33 @@ # SPDX-License-Identifier: BSD-2-Clause-Patent ## import logging +import json import os import struct -import tomllib +from typing import Literal, TypedDict, cast + +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib from edk2toolext.environment import repo_resolver from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin +class FirmwareVersionMetadata(TypedDict): + """Portable sidecar identity; independent of the host-facing binary ABI.""" + + schema_version: int + struct_version: int + interface_major: int + interface_minor: int + base_version: str + git_commit: str + flags: int + source_origin: Literal["unknown", "open", "closed"] + + class FirmwareVersionBlob(IUefiBuildPlugin): # # Layout of MSVM_FIRMWARE_VERSION_INFO (see @@ -53,6 +72,7 @@ class FirmwareVersionBlob(IUefiBuildPlugin): # (BUILD_OUTPUT_BASE). The FDF references the same location via # $(OUTPUT_DIRECTORY)/$(TARGET)_$(TOOL_CHAIN_TAG). FW_VERSION_BLOB_SUBPATH = os.path.join("FwVersion", "FwVersionBlob.bin") + FW_VERSION_METADATA_SUBPATH = os.path.join("FwVersion", "FirmwareVersion.json") # Single source of truth for the version numbers, relative to the workspace. FW_VERSION_TOML_SUBPATH = os.path.join("MsvmPkg", "FirmwareVersion.toml") @@ -64,7 +84,7 @@ def _ReadVersionToml(self, workspace, env): minor = int(data["interface"]["minor"]) # The CI BASE_VERSION env override wins so the release workflow can own # the released value; the TOML provides the in-tree default. - release = env.GetValue("BASE_VERSION", "") or str(data["release"]["version"]) + release = env.GetValue("BASE_VERSION", os.environ.get("BASE_VERSION", "")) or str(data["release"]["version"]) return major, minor, release def _GetGitCommit(self, workspace): @@ -79,18 +99,40 @@ def _GetGitCommit(self, workspace): logging.warning(f"Could not determine git state for firmware version blob: {e}") return "unknown", False + @staticmethod + def _EncodeRecordString(value: str, capacity: int, name: str) -> bytes: + """Encode an ASCII field while reserving its required NUL terminator. + + Raise ValueError before macros or files are emitted if packing would + replace characters, truncate the value, or produce an ambiguous string. + """ + try: + encoded = value.encode("ascii") + except UnicodeEncodeError as error: + raise ValueError(f"{name} must contain only ASCII characters") from error + if not encoded or b"\0" in encoded or len(encoded) >= capacity: + raise ValueError(f"{name} must be nonempty, NUL-free, and at most {capacity - 1} ASCII bytes") + return encoded + def do_pre_build(self, thebuilder): workspace = thebuilder.GetWorkspaceRoot() major, minor, base_version = self._ReadVersionToml(workspace, thebuilder.env) commit, dirty = self._GetGitCommit(workspace) + if not (0 <= major <= 0xFFFF and 0 <= minor <= 0xFFFF): + raise ValueError("Firmware interface major/minor must fit unsigned 16-bit fields") + encoded_base = self._EncodeRecordString(base_version, self.FW_VERSION_BASE_VERSION_SIZE, "BASE_VERSION") + encoded_commit = self._EncodeRecordString(commit, self.FW_VERSION_GIT_COMMIT_SIZE, "GitCommit") + origin = thebuilder.env.GetValue("SOURCE_ORIGIN", os.environ.get("SOURCE_ORIGIN", "unknown")).strip().lower() + if origin not in ("unknown", "open", "closed"): + raise ValueError("SOURCE_ORIGIN must be unknown, open, or closed") flags = 0 if dirty: flags |= self.FW_VERSION_FLAG_DIRTY # Official builds are marked by the CI pipeline (e.g. building main, not # a PR) via the OFFICIAL_BUILD environment variable. Any non-empty value # other than "0"/"false" counts as official. - if thebuilder.env.GetValue("OFFICIAL_BUILD", "").strip().lower() not in ("", "0", "false"): + if thebuilder.env.GetValue("OFFICIAL_BUILD", os.environ.get("OFFICIAL_BUILD", "")).strip().lower() not in ("", "0", "false"): flags |= self.FW_VERSION_FLAG_OFFICIAL # (1) Expose the values to the firmware as FixedAtBuild PCDs. Setting @@ -111,8 +153,8 @@ def do_pre_build(self, thebuilder): flags, major, minor, - base_version.encode("ascii", "replace"), - commit.encode("ascii", "replace"), + encoded_base, + encoded_commit, ) # Emit into the per-build output directory so the artifact stays out of @@ -125,11 +167,27 @@ def do_pre_build(self, thebuilder): with open(out_path, "wb") as f: f.write(blob) + metadata: FirmwareVersionMetadata = { + "schema_version": 1, + "struct_version": self.FW_VERSION_STRUCT_VERSION, + "interface_major": major, + "interface_minor": minor, + "base_version": base_version, + "git_commit": commit, + "flags": flags, + "source_origin": cast(Literal["unknown", "open", "closed"], origin), + } + metadata_path = os.path.join(out_dir, self.FW_VERSION_METADATA_SUBPATH) + with open(metadata_path, "w", encoding="utf-8") as metadata_file: + json.dump(metadata, metadata_file, indent=2) + metadata_file.write("\n") + logging.info( f"Firmware version blob written: base={base_version} commit={commit}" f"{' (dirty)' if dirty else ''}" f"{' (official)' if flags & self.FW_VERSION_FLAG_OFFICIAL else ''}" f" interface={major}.{minor}" + f" origin={origin}" f" -> {out_path}" ) return 0 diff --git a/MsvmPkg/Docs/BUILDING.md b/MsvmPkg/Docs/BUILDING.md index 5351b0b2ab..cf65e74d64 100644 --- a/MsvmPkg/Docs/BUILDING.md +++ b/MsvmPkg/Docs/BUILDING.md @@ -32,7 +32,7 @@ Use your activated Python virtual environment with the selected compiler install From the repository root, run: ```powershell -python .\ci\scripts\build.py --arch X64 --target DEBUG --tool-chain CLANGPDB --core legacy +python .\ci\scripts\build.py --arch X64 --target DEBUG --tool-chain CLANGPDB --core legacy --source-origin open ``` The script installs Python requirements, then runs `stuart_setup`, `stuart_update`, and `stuart_build` @@ -42,14 +42,16 @@ Stuart executables are resolved from the invoking Python environment, and execut `--host windows|linux` defaults to the current host. `--compiler-source` distinguishes `visual_studio`, `windows_org`, and `distribution`; it defaults to Visual Studio on Windows and distribution tools on Linux. -Use `--legacy-debugger 1` for the legacy-debugger variants. ARM64 MSVC is rejected before preparation. +`--legacy-debugger 1` requires closed-source X64 VS2022 or CLANGPDB with the legacy core. +CLANGPDB includes Windows Visual Studio Clang, Windows-org Clang, and Linux distribution Clang. +GitHub/open builds, GCC, and Patina builds must use `0`. ARM64 MSVC is rejected before preparation. Actual execution requires the selected host; `--dry-run` can preview either host's commands. Windows-org Clang requires an explicit `--clang-bin` directory; it never falls back to Visual Studio Clang. For example, with the internal compiler tools and matching resource headers already installed: ```powershell -python .\ci\scripts\build.py --arch AARCH64 --target RELEASE --tool-chain CLANGPDB --core legacy --host windows --compiler-source windows_org --clang-bin C:/tools/windows-clang/bin +python .\ci\scripts\build.py --arch AARCH64 --target RELEASE --tool-chain CLANGPDB --core legacy --host windows --compiler-source windows_org --clang-bin C:/tools/windows-clang/bin --source-origin closed ``` The local runner does not download private compiler packages or install system compilers. ADO provisions @@ -84,7 +86,9 @@ The closed profile restores 15 legacy-core builds from the preserved hyperv.uefi | Linux | Distribution Clang | X64 and AARCH64, DEBUG and RELEASE | | Linux | Distribution GCC | X64 RELEASE only | -Closed X64 DEBUG builds enable the legacy debugger. Open builds retain their existing debugger selection. +The four closed X64 DEBUG rows enable the legacy debugger: VS2022 and all three Clang compiler sources. +This matches the fetched ADO configuration; successful compilation and runtime debugging still require validation. +All open rows and all GCC rows disable the legacy debugger. ARM64 MSVC is unsupported. Disabled GCC combinations are not reintroduced. IDs are derived as `______`; @@ -127,6 +131,61 @@ No package creation or publication is enabled here. In particular, runtime build that per-flavor OneBranch publishing policy can be expanded at runtime; validate or introduce a separate compile-time publishing stage before enabling it. +## Firmware Versioning + +Think of three different labels: + +| Label | Example | What it answers | +| --- | --- | --- | +| Interface version | `1.0` | Can this firmware communicate correctly with this VMM? | +| Embedded build identity | `26.0` + Git SHA + flags | Which source produced this firmware? | +| Package version | GitHub release / NuGet / VPack version | Which published download is this? | + +The first two come from `MsvmPkg/FirmwareVersion.toml` and the checkout. The last is assigned during publishing. +A package number assigned after compilation is not embedded in that firmware. + +Inside the firmware today: + +- **Interface version:** bump major for a breaking firmware/VMM contract change, minor for a compatible addition. +- **Release prefix:** `26.0`, not the final package number. +- **Git SHA:** the actual checkout built, including the merge commit for a PR merge checkout. +- **Dirty flag:** the source had uncommitted changes. +- **Official flag:** CI marked the build official. This does not mean signed or shipping. + +The plugin writes matching firmware PCDs and an 80-byte version-1 record in the DXE FV. That binary layout +is unchanged. Future packaging must retain a mapping from package version to this build identity and flavor. + +### Open Versus Closed Source + +The build also writes `FwVersion/FirmwareVersion.json` under its build output directory. This sidecar +contains the same identity values plus **`source_origin`**: `open`, `closed`, or `unknown`. +GitHub explicitly selects `open`; ADO explicitly selects `closed`. Origin describes the source repository, +not the compiler supplier, shipping eligibility, or official status. + +Locally, pass `--source-origin open` or `--source-origin closed` to `ci/scripts/build.py`. +Omitting it records `unknown`, rather than guessing from remotes or inheriting a stale shell setting. +For direct Stuart invocations, set `SOURCE_ORIGIN=open|closed|unknown` as a build or shell variable. +An explicit Stuart value wins over the shell; an unrecognized value is rejected before output is written. + +Origin is currently **sidecar metadata, not embedded in MSVM.fd**. Keep the JSON with future build artifacts; +artifact upload is not wired yet. Identifying origin from the firmware file alone needs a separately reviewed +binary-record extension. Origin and the official flag are declarations, not security attestations. + +### Overrides and CI Policy + +`BASE_VERSION` uses an explicit Stuart value first, then the shell, then TOML. An empty Stuart value suppresses +the shell override and uses TOML. It must fit 15 ASCII bytes plus a NUL terminator; interface fields must fit UINT16. + +`OFFICIAL_BUILD` uses Stuart-over-shell precedence too. Empty, `0`, and case-insensitive `false` mean unofficial; +other nonempty values mean official. Local builds default to unofficial unless explicitly overridden. + +- GitHub sets `1` only for a push to `main`; PRs and manual dispatches get `0`. +- ADO sets `1` for the Official entry point and `0` for NonOfficial PR builds, on both build hosts. + +An official pipeline can produce test-only flavors; a shipping flavor built locally is still unofficial. +Dirty and official flags can both be set. Public/private mirrors have different SHAs: source origin tells +you which repository to look in, not how to translate commits between them. + ## Running CI Checks Locally Run the same source-check workflow used by platform CI from the repository root: diff --git a/ci/scripts/build.py b/ci/scripts/build.py index 3f13a622b2..ae9f202422 100644 --- a/ci/scripts/build.py +++ b/ci/scripts/build.py @@ -35,6 +35,7 @@ class BuildArguments(argparse.Namespace): host: Literal["windows", "linux"] compiler_source: Literal["visual_studio", "windows_org", "distribution"] | None legacy_debugger: Literal["0", "1"] + source_origin: Literal["unknown", "open", "closed"] clang_bin: Path | None dry_run: bool @@ -60,6 +61,8 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--host", choices=("windows", "linux"), default="windows" if os.name == "nt" else "linux") parser.add_argument("--compiler-source", choices=("visual_studio", "windows_org", "distribution")) parser.add_argument("--legacy-debugger", choices=("0", "1"), default="0") + parser.add_argument("--source-origin", choices=("unknown", "open", "closed"), default="unknown", + help="Source repository provenance for firmware metadata; never inferred from the host") parser.add_argument("--clang-bin", type=Path, help="Explicit compiler bin directory; required for Windows-org Clang") parser.add_argument("--dry-run", action="store_true", help="Print commands without executing them") args = parser.parse_args(argv, namespace=BuildArguments()) @@ -67,6 +70,11 @@ def main(argv: list[str] | None = None) -> int: compiler_source = args.compiler_source or ("visual_studio" if args.host == "windows" else "distribution") if args.arch == "AARCH64" and args.tool_chain == "VS2022": parser.error("ARM64 MSVC is not supported") + if args.legacy_debugger == "1" and ( + args.source_origin != "closed" or args.arch != "X64" or args.core != "legacy" + or args.tool_chain not in ("VS2022", "CLANGPDB") + ): + parser.error("Legacy debugger requires closed-source X64 VS2022 or CLANGPDB with the legacy core") if (args.host, args.tool_chain, compiler_source) not in { ("windows", "VS2022", "visual_studio"), ("windows", "CLANGPDB", "visual_studio"), ("windows", "CLANGPDB", "windows_org"), ("linux", "CLANGPDB", "distribution"), @@ -87,9 +95,9 @@ def main(argv: list[str] | None = None) -> int: f"TOOL_CHAIN_TAG={args.tool_chain}", f"BLD_*_USE_LEGACY_C_CORE={'TRUE' if args.core == 'legacy' else 'FALSE'}", ] - if args.legacy_debugger == "1": - flavor.append("BLD_*_LEGACY_DEBUGGER=1") + flavor.append(f"BLD_*_LEGACY_DEBUGGER={args.legacy_debugger}") environment: dict[str, str] = os.environ.copy() + environment["SOURCE_ORIGIN"] = args.source_origin if args.clang_bin is not None: environment["CLANG_BIN"] = f"{args.clang_bin.resolve().as_posix()}/" print(f"CLANG_BIN={environment['CLANG_BIN']}", flush=True) diff --git a/ci/scripts/matrix.py b/ci/scripts/matrix.py index 8f925e0321..de7f1d8ede 100644 --- a/ci/scripts/matrix.py +++ b/ci/scripts/matrix.py @@ -109,6 +109,10 @@ def parse_definition(value: object) -> BuildDefinition: raise ValueError("Unsupported host/compiler combination") if row["arch"] == "AARCH64" and row["tool_chain"] == "VS2022": raise ValueError("ARM64 MSVC is not supported") + if row["legacy_debugger"] == "1" and ( + row["arch"] != "X64" or row["core"] != "legacy" or row["tool_chain"] not in ("VS2022", "CLANGPDB") + ): + raise ValueError("Legacy debugger requires X64 VS2022 or CLANGPDB with the legacy core") if "shipping" in row or "package_suffix" in row: suffix = row.get("package_suffix") if type(row.get("shipping")) is not bool or not isinstance(suffix, str): @@ -172,6 +176,8 @@ def main(argv: list[str] | None = None) -> None: args = parser.parse_args(argv, namespace=MatrixArguments()) try: rows = select_builds(args.matrix, host=args.host) + if args.format == "github" and any(row["legacy_debugger"] == "1" for row in rows): + raise ValueError("GitHub builds cannot enable the closed-source legacy debugger") except (OSError, ValueError) as error: parser.error(str(error)) result: object diff --git a/ci/tests/test_build.py b/ci/tests/test_build.py index a63dfa5333..4ee2cd2b25 100644 --- a/ci/tests/test_build.py +++ b/ci/tests/test_build.py @@ -39,7 +39,10 @@ def test_platform_preparation_precedes_build(self, run_mock: MagicMock) -> None: "TOOL_CHAIN_TAG=CLANGPDB", "BLD_*_USE_LEGACY_C_CORE=TRUE", ]) for invocation in calls: - self.assertEqual(invocation.kwargs, {"cwd": SCRIPT_PATH.resolve().parents[2], "check": True, "env": dict(os.environ)}) + self.assertEqual(invocation.kwargs, {"cwd": SCRIPT_PATH.resolve().parents[2], "check": True, + "env": {**os.environ, "SOURCE_ORIGIN": "unknown"}}) + if invocation is not calls[0]: + self.assertIn("BLD_*_LEGACY_DEBUGGER=0", invocation.args[0]) self.assertIn("BUILDREPORT_TYPES=PCD DEPEX FLASH BUILD_FLAGS LIBRARY", calls[-1].args[0]) self.assertIn("LaunchLogOnSuccess=FALSE", calls[-1].args[0]) self.assertIn("LaunchLogOnError=FALSE", calls[-1].args[0]) @@ -68,10 +71,50 @@ def test_dry_run_has_no_side_effects(self, run_mock: MagicMock) -> None: @patch("ci.scripts.build.subprocess.run") def test_debugger_passed_to_all_stuart_phases(self, run_mock: MagicMock) -> None: - build.main([*self.ARGS, "--legacy-debugger", "1"]) + build.main(["--arch", "X64", "--target", "DEBUG", "--tool-chain", "VS2022", "--core", "legacy", + "--source-origin", "closed", "--legacy-debugger", "1"]) for invocation in run_mock.call_args_list[1:]: self.assertIn("BLD_*_LEGACY_DEBUGGER=1", invocation.args[0]) + @patch("ci.scripts.build.subprocess.run") + def test_debugger_rejected_for_open_unknown_gcc_and_patina(self, run_mock: MagicMock) -> None: + for origin, tool, core in (("open", "VS2022", "legacy"), ("unknown", "VS2022", "legacy"), + ("open", "CLANGPDB", "legacy"), ("unknown", "CLANGPDB", "legacy"), + ("closed", "GCC", "legacy"), ("closed", "VS2022", "patina")): + with self.subTest(origin=origin, tool=tool, core=core), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as error: + build.main(["--arch", "X64", "--target", "DEBUG", "--tool-chain", tool, "--core", core, + "--host", "linux" if tool == "GCC" else "windows", + "--source-origin", origin, "--legacy-debugger", "1", "--dry-run"]) + self.assertEqual(error.exception.code, 2) + run_mock.assert_not_called() + + @patch("ci.scripts.build.subprocess.run") + def test_closed_clang_debugger_variants_are_accepted(self, run_mock: MagicMock) -> None: + """Preserve all three Clang debugger combinations configured by ADO.""" + for host, source in (("windows", "visual_studio"), ("windows", "windows_org"), + ("linux", "distribution")): + with self.subTest(host=host, source=source): + args = [*self.ARGS, "--host", host, "--compiler-source", source, + "--source-origin", "closed", "--legacy-debugger", "1", "--dry-run"] + if source == "windows_org": + args.extend(["--clang-bin", "internal-llvm/bin"]) + output = io.StringIO() + with redirect_stdout(output): + self.assertEqual(build.main(args), 0) + self.assertEqual(output.getvalue().count("BLD_*_LEGACY_DEBUGGER=1"), 3) + run_mock.assert_not_called() + + @patch("ci.scripts.build.subprocess.run") + def test_source_origin_is_explicit_and_inherited_by_all_commands(self, run_mock: MagicMock) -> None: + for origin in ("open", "closed", "unknown"): + with self.subTest(origin=origin), patch.dict(os.environ, {"SOURCE_ORIGIN": "inherited"}): + run_mock.reset_mock() + self.assertEqual(build.main([*self.ARGS, "--source-origin", origin]), 0) + self.assertEqual(os.environ["SOURCE_ORIGIN"], "inherited") + for invocation in run_mock.call_args_list: + self.assertEqual(invocation.kwargs["env"]["SOURCE_ORIGIN"], origin) + @patch("ci.scripts.build.subprocess.run") def test_windows_org_cannot_fall_back_to_visual_studio(self, run_mock: MagicMock) -> None: with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error: diff --git a/ci/tests/test_firmware_version.py b/ci/tests/test_firmware_version.py new file mode 100644 index 0000000000..dc2d17389e --- /dev/null +++ b/ci/tests/test_firmware_version.py @@ -0,0 +1,227 @@ +"""Exercise the real firmware version plugin without compiling firmware. + +Port the internal repository's override regression cases to unittest. The +Stuart VarDict is loaded dynamically, like the plugin, because neither offers +a typed API. Both the binary ABI and the firmware macro values are checked. +""" + +import importlib +import importlib.util +import builtins +import json +import os +import struct +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from types import ModuleType, SimpleNamespace +from typing import Any +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] + + +def load_plugin() -> ModuleType: + """Load the production plugin without triggering Stuart discovery.""" + path = ROOT / "MsvmPkg/BuildPlugins/FirmwareVersionBlob/FirmwareVersionBlob.py" + spec = importlib.util.spec_from_file_location("firmware_version_blob", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class FirmwareVersionTests(unittest.TestCase): + """Build vars win over shell vars, including explicit empty/false values.""" + + def test_source_origin_is_explicit_sidecar_metadata(self) -> None: + """Origins honor Stuart precedence and leave the v1 binary ABI unchanged.""" + module = load_plugin() + cases: list[tuple[str | None, str | None, str]] = [ + (None, None, "unknown"), ("open", None, "open"), ("closed", None, "closed"), + ("closed", "open", "open"), ("open", "closed", "closed"), + ("closed", "unknown", "unknown"), + ] + blobs: list[bytes] = [] + for shell_origin, build_origin, expected in cases: + with self.subTest(shell=shell_origin, build=build_origin), TemporaryDirectory() as output: + env = importlib.import_module("edk2toolext.environment.var_dict").VarDict() + env.SetValue("BUILD_OUTPUT_BASE", output, "test") + if build_origin is not None: + env.SetValue("SOURCE_ORIGIN", build_origin, "test") + environment = {"OFFICIAL_BUILD": "1"} + if shell_origin is not None: + environment["SOURCE_ORIGIN"] = shell_origin + plugin = module.FirmwareVersionBlob() + builder = SimpleNamespace(env=env, GetWorkspaceRoot=lambda: str(ROOT)) + with patch.dict(os.environ, environment, clear=True), \ + patch.object(plugin, "_GetGitCommit", return_value=("a" * 40, False)): + self.assertEqual(plugin.do_pre_build(builder), 0) + metadata = json.loads((Path(output) / plugin.FW_VERSION_METADATA_SUBPATH).read_text()) + self.assertEqual(metadata["source_origin"], expected) + self.assertEqual(metadata["schema_version"], 1) + self.assertEqual(metadata["flags"], 2) + blobs.append((Path(output) / plugin.FW_VERSION_BLOB_SUBPATH).read_bytes()) + self.assertTrue(all(blob == blobs[0] for blob in blobs)) + self.assertEqual(len(blobs[0]), 80) + + def test_invalid_origin_is_rejected_before_emission(self) -> None: + module = load_plugin() + for origin in ("", "github", "official"): + with self.subTest(origin=origin), TemporaryDirectory() as output: + env = importlib.import_module("edk2toolext.environment.var_dict").VarDict() + env.SetValue("BUILD_OUTPUT_BASE", output, "test") + env.SetValue("SOURCE_ORIGIN", origin, "test") + plugin = module.FirmwareVersionBlob() + builder = SimpleNamespace(env=env, GetWorkspaceRoot=lambda: str(ROOT)) + with patch.object(plugin, "_GetGitCommit", return_value=("a" * 40, False)), \ + self.assertRaisesRegex(ValueError, "SOURCE_ORIGIN"): + plugin.do_pre_build(builder) + self.assertIsNone(env.GetValue("BLD_*_MSVM_FW_FLAGS")) + self.assertFalse((Path(output) / plugin.FW_VERSION_BLOB_SUBPATH).exists()) + self.assertFalse((Path(output) / plugin.FW_VERSION_METADATA_SUBPATH).exists()) + + def test_record_limits_fail_before_emitting_macros_or_blob(self) -> None: + """Invalid overrides must not silently truncate or diverge from PCDs.""" + module = load_plugin() + cases = [ + (1, 0, "1" * 16, "a" * 40), (1, 0, "26.0\0bad", "a" * 40), + (1, 0, "26.\u00e9", "a" * 40), (1, 0, "26.0", "a" * 48), + (-1, 0, "26.0", "a" * 40), (1, 65536, "26.0", "a" * 40), + ] + for major, minor, base, commit in cases: + with self.subTest(major=major, minor=minor, base=base, commit=commit), TemporaryDirectory() as output: + env = importlib.import_module("edk2toolext.environment.var_dict").VarDict() + env.SetValue("BUILD_OUTPUT_BASE", output, "test") + builder = SimpleNamespace(env=env, GetWorkspaceRoot=lambda: str(ROOT)) + plugin = module.FirmwareVersionBlob() + with patch.object(plugin, "_ReadVersionToml", return_value=(major, minor, base)), \ + patch.object(plugin, "_GetGitCommit", return_value=(commit, False)), \ + self.assertRaises(ValueError): + plugin.do_pre_build(builder) + self.assertIsNone(env.GetValue("BLD_*_MSVM_FW_BASE_VERSION")) + self.assertFalse((Path(output) / plugin.FW_VERSION_BLOB_SUBPATH).exists()) + + def test_string_boundary_reserves_nul(self) -> None: + module = load_plugin() + encoded = module.FirmwareVersionBlob._EncodeRecordString("1" * 15, 16, "BASE_VERSION") + self.assertEqual(struct.pack("16s", encoded), b"1" * 15 + b"\0") + + def test_tomli_fallback_when_stdlib_toml_is_unavailable(self) -> None: + """Exercise fallback dispatch on this interpreter without installing tomli.""" + real_import = builtins.__import__ + toml = importlib.import_module("tomllib") + + def fallback_import(name: str, *args: Any, **kwargs: Any) -> Any: + if name == "tomllib": + raise ModuleNotFoundError(name) + if name == "tomli": + return toml + return real_import(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=fallback_import): + module = load_plugin() + self.assertIs(module.tomllib, toml) + + def test_provider_official_context_reaches_both_hosts(self) -> None: + """Official provenance comes from the pipeline, never shipping metadata.""" + yaml = importlib.import_module("yaml") + github = yaml.safe_load((ROOT / ".github/workflows/platform-ci.yml").read_text()) + build_step = next(step for step in github["jobs"]["build"]["steps"] if step.get("uses") == "./.github/actions/build") + self.assertEqual(build_step["env"]["OFFICIAL_BUILD"], + "${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') && '1' || '0' }}") + action = yaml.safe_load((ROOT / ".github/actions/build/action.yml").read_text()) + self.assertIn("--source-origin open", action["runs"]["steps"][-1]["run"]) + for entry, base in (("platform-ci.yml", "Official"), ("platform-pr.yml", "NonOfficial")): + pipeline = yaml.safe_load((ROOT / ".azuredevops/pipelines" / entry).read_text()) + self.assertEqual(pipeline["extends"]["parameters"]["oneBranchBase"], f"v2/Microsoft.{base}.yml") + platform = yaml.safe_load((ROOT / ".azuredevops/templates/platform.yml").read_text()) + stage = next(stage for stage in platform["extends"]["parameters"]["stages"] if stage["stage"] == "Build") + self.assertEqual(stage["jobs"][0]["parameters"]["officialBuild"], + "${{ eq(parameters.oneBranchBase, 'v2/Microsoft.Official.yml') }}") + template = yaml.safe_load((ROOT / ".azuredevops/templates/build.yml").read_text()) + parameter = next(item for item in template["parameters"] if item["name"] == "officialBuild") + self.assertIs(parameter["default"], False) + job = template["jobs"][1]["${{ each host in parameters.hosts }}"][0] + for branch in job["steps"]: + steps = next(iter(branch.values())) + build_step = next(step for step in steps if "ci/scripts/build.py" in step.get("pwsh", step.get("bash", ""))) + self.assertEqual(build_step["env"]["OFFICIAL_BUILD"], "${{ iif(parameters.officialBuild, '1', '0') }}") + command = build_step.get("pwsh", build_step.get("bash", "")) + self.assertTrue("'--source-origin', 'closed'" in command or "--source-origin closed" in command) + + def test_build_runner_preserves_version_environment(self) -> None: + """The orchestration layer passes overrides to Stuart unchanged.""" + from ci.scripts import build + + with patch.dict(os.environ, {"OFFICIAL_BUILD": "1", "BASE_VERSION": "99.2"}), \ + patch("ci.scripts.build.subprocess.run") as run_mock: + self.assertEqual(build.main([ + "--arch", "X64", "--target", "DEBUG", "--tool-chain", "CLANGPDB", "--core", "legacy", + ]), 0) + self.assertEqual(run_mock.call_count, 4) + for invocation in run_mock.call_args_list: + self.assertEqual(invocation.kwargs["env"]["OFFICIAL_BUILD"], "1") + self.assertEqual(invocation.kwargs["env"]["BASE_VERSION"], "99.2") + + def test_blob_macros_and_log_agree(self) -> None: + module = load_plugin() + cases: list[tuple[str | None, str | None, str | None, str | None, bool, str | None]] = [ + (None, None, None, None, False, None), + ("1", None, "99.2", None, True, "99.2"), + ("", None, "", None, False, None), + ("0", None, None, None, False, None), + (" FaLsE ", None, None, None, False, None), + ("1", "0", "99.2", "88.1", False, "88.1"), + ("1", "", "99.2", "", False, None), + ("0", "1", None, "88.1", True, "88.1"), + ] + for dirty in (False, True): + for shell_official, build_official, shell_base, build_base, official, release in cases: + with self.subTest(dirty=dirty, official=official, release=release), TemporaryDirectory() as output: + env = importlib.import_module("edk2toolext.environment.var_dict").VarDict() + env.SetValue("BUILD_OUTPUT_BASE", output, "test") + environment = {key: value for key, value in os.environ.items() + if key not in ("OFFICIAL_BUILD", "BASE_VERSION", "SOURCE_ORIGIN")} + for name, shell_value, build_value in ( + ("OFFICIAL_BUILD", shell_official, build_official), + ("BASE_VERSION", shell_base, build_base), + ): + if shell_value is not None: + environment[name] = shell_value + if build_value is not None: + env.SetValue(name, build_value, "test") + plugin = module.FirmwareVersionBlob() + with (ROOT / plugin.FW_VERSION_TOML_SUBPATH).open("rb") as manifest: + version = module.tomllib.load(manifest) + expected_release = release if release is not None else str(version["release"]["version"]) + flags = int(dirty) | (2 if official else 0) + commit = "a" * 40 + builder = SimpleNamespace(env=env, GetWorkspaceRoot=lambda: str(ROOT)) + with patch.dict(os.environ, environment, clear=True), \ + patch.object(plugin, "_GetGitCommit", return_value=(commit, dirty)), \ + self.assertLogs(level="INFO") as logs: + self.assertEqual(plugin.do_pre_build(builder), 0) + blob = (Path(output) / plugin.FW_VERSION_BLOB_SUBPATH).read_bytes() + record = struct.unpack(" None: for row in matrix.select_builds(path): with self.subTest(path=path, row=row), redirect_stdout(io.StringIO()): args = [ + "--source-origin", "closed" if path == CLOSED_MATRIX else "open", "--dry-run", "--arch", row["arch"], "--target", row["target"], "--tool-chain", row["tool_chain"], "--core", row["core"], "--host", row["host"], "--compiler-source", row["compiler_source"], @@ -58,6 +59,7 @@ def test_coverage_and_derived_identifiers(self) -> None: } self.assertEqual({(row["arch"], row["target"], row["tool_chain"], row["core"]) for row in rows}, expected) self.assertEqual(len(rows), 10) + self.assertTrue(all(row["legacy_debugger"] == "0" for row in rows)) for row in rows: self.assertEqual(row["id"], matrix.build_id(row)) closed = matrix.select_builds(CLOSED_MATRIX) @@ -82,6 +84,11 @@ def test_provider_formats_preserve_same_flavors(self) -> None: for path in MATRICES: rows = matrix.select_builds(path) for format_name in ("json", "github", "ado"): + if path == CLOSED_MATRIX and format_name == "github": + with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error: + matrix.main(["--matrix", str(path), "--format", format_name]) + self.assertEqual(error.exception.code, 2) + continue with self.subTest(path=path, format=format_name): output = io.StringIO() with redirect_stdout(output): @@ -103,6 +110,9 @@ def test_invalid_matrix_is_rejected(self) -> None: [{**row, "repositories": ["closed"]}], [{**row, "arch": "AARCH64", "tool_chain": "VS2022"}], [{**row, "host": "linux"}], + [{**row, "legacy_debugger": "1", "arch": "AARCH64"}], + [{**row, "legacy_debugger": "1", "core": "patina"}], + [{**row, "legacy_debugger": "1", "host": "linux", "tool_chain": "GCC", "compiler_source": "distribution"}], [{**row, "shipping": True}], [{**row, "package_suffix": ""}], ] for document in invalid_documents: diff --git a/pip-requirements.txt b/pip-requirements.txt index f4257b6701..13ab689982 100644 --- a/pip-requirements.txt +++ b/pip-requirements.txt @@ -1,3 +1,4 @@ edk2-pytool-library~=0.23.8 edk2-pytool-extensions~=0.30.3 -regex \ No newline at end of file +regex +tomli>=2.0.1; python_version < "3.11" \ No newline at end of file From c439e2eaf1a41ba47fc49c3ec84e5ee052e1ff21 Mon Sep 17 00:00:00 2001 From: Maheer Aeron Date: Wed, 23 Sep 2026 15:09:13 -0700 Subject: [PATCH 13/18] ci: add readable build labels and preserve artifact names --- .azuredevops/templates/build.yml | 2 +- .github/workflows/platform-ci.yml | 2 +- MsvmPkg/Docs/BUILDING.md | 31 ++++++++++++++++++++++++ ci/scripts/matrix.py | 38 ++++++++++++++++++++++++++++++ ci/tests/test_matrix.py | 39 +++++++++++++++++++++++++++++-- 5 files changed, 108 insertions(+), 4 deletions(-) diff --git a/.azuredevops/templates/build.yml b/.azuredevops/templates/build.yml index e8ec55372c..1d4c3a511a 100644 --- a/.azuredevops/templates/build.yml +++ b/.azuredevops/templates/build.yml @@ -40,7 +40,7 @@ jobs: - ${{ each host in parameters.hosts }}: - job: BuildFirmware_${{ host }} - displayName: Build Firmware (${{ host }}) + displayName: $(display_name) dependsOn: BuildMatrix pool: type: ${{ host }} diff --git a/.github/workflows/platform-ci.yml b/.github/workflows/platform-ci.yml index 35e1807f10..86057febf3 100644 --- a/.github/workflows/platform-ci.yml +++ b/.github/workflows/platform-ci.yml @@ -32,7 +32,7 @@ jobs: echo "matrix=$matrix" >> "$GITHUB_OUTPUT" build: - name: Build ${{ matrix.id }} + name: ${{ matrix.display_name }} needs: build-matrix runs-on: windows-2022 strategy: diff --git a/MsvmPkg/Docs/BUILDING.md b/MsvmPkg/Docs/BUILDING.md index cf65e74d64..a57a8b693d 100644 --- a/MsvmPkg/Docs/BUILDING.md +++ b/MsvmPkg/Docs/BUILDING.md @@ -115,6 +115,37 @@ The inspected OneBranch Windows job template forwards job strategies and depende checks our output binding and flavor arguments, but a hosted OneBranch run is still required to verify the full governed-template expansion, runner tools, internal feed credentials, and repository access. +### Job Labels and Artifact Names + +Machine IDs remain stable and complete. They are not the human-facing job labels or package names. +Provider matrix output adds a `display_name` derived from the same flavor: + +- GitHub: `DEBUG X64 CLANGPDB (Legacy)` or `RELEASE AARCH64 CLANGPDB (Patina)`. +- ADO: the same label followed by host and compiler source, such as `/ Windows WinOrg` or `/ Linux Distro`. + Debugger-enabled rows also show `/ Legacy debugger`. ADO may additionally show its matrix leg key in the UI. + +GitHub matrix output also supplies `artifact_name` and `logs_artifact_name` for the future upload steps. +They preserve the pre-refactor consumer names exactly: + +| Flavor | Firmware artifact | Log artifact | +| --- | --- | --- | +| X64 DEBUG CLANGPDB legacy | `firmware-DEBUG-X64-CLANGPDB` | `logs-DEBUG-X64-CLANGPDB` | +| AARCH64 RELEASE CLANGPDB Patina | `firmware-RELEASE-AARCH64-CLANGPDB-patina` | `logs-RELEASE-AARCH64-CLANGPDB-patina` | + +The pattern is `firmware---[-patina]` (and `logs-` for logs). +Legacy has no suffix. Keep the literal `AARCH64` and `CLANGPDB` in artifact names for existing consumers. +If future host/compiler variants collide under this pattern, selection fails instead of silently renaming artifacts. +Any expanded naming scheme must be coordinated with consumers first. + +ADO naming is a separate compatibility contract: historical staged folders were +`Firmware Binary File _`, `Firmware Map Files _`, +`Firmware PDB Files _`, and `Build Logs ` inside a per-job OneBranch artifact. +Those folder names alone are not globally unique. Changing job identity or log-folder labels can affect consumers +of the enclosing artifact and must be checked when staging/upload is restored. No ADO artifact names are inferred +from the new display label. Package names and suffixes below remain unchanged. + +This change defines labels and names only; artifact upload and staging are not enabled yet. + ### Closed Package Identity Closed rows have explicit `shipping` and `package_suffix` metadata. Shipping variants are X64 DEBUG/RELEASE diff --git a/ci/scripts/matrix.py b/ci/scripts/matrix.py index de7f1d8ede..d334d9402d 100644 --- a/ci/scripts/matrix.py +++ b/ci/scripts/matrix.py @@ -52,6 +52,9 @@ class SelectedBuild(BuildFlavor): """ id: str + display_name: NotRequired[str] + artifact_name: NotRequired[str] + logs_artifact_name: NotRequired[str] shipping: NotRequired[str] package_suffix: NotRequired[str] @@ -82,6 +85,40 @@ def build_id(row: BuildFlavor) -> str: row["compiler_source"], row["core"], row["legacy_debugger"])) +def label_builds(rows: list[SelectedBuild], backend: OutputFormat) -> list[SelectedBuild]: + """Add provider presentation fields without changing execution/package IDs. + + GitHub artifact names preserve the pre-refactor consumer contract. Reject + collisions instead of silently renaming artifacts when new variants are + introduced. ADO artifact staging and package names are not derived from + these display labels. Plain JSON remains execution data only. + """ + labeled: list[SelectedBuild] = [] + artifact_names: set[str] = set() + for row in rows: + result: SelectedBuild = {**row} + if backend != "json": + core = "Legacy" if row["core"] == "legacy" else "Patina" + label = f"{row['target']} {row['arch']} {row['tool_chain']} ({core})" + if backend == "ado": + compiler = {"visual_studio": "VS", "windows_org": "WinOrg", "distribution": "Distro"}[row["compiler_source"]] + label += f" / {row['host'].title()} {compiler}" + if row["legacy_debugger"] == "1": + label += " / Legacy debugger" + result["display_name"] = label + if backend == "github": + suffix = "-patina" if row["core"] == "patina" else "" + stem = f"{row['target']}-{row['arch']}-{row['tool_chain']}{suffix}" + artifact_name = f"firmware-{stem}" + if artifact_name in artifact_names: + raise ValueError(f"Artifact name collision: {artifact_name}; choose an explicit consumer migration before adding this variant") + artifact_names.add(artifact_name) + result["artifact_name"] = artifact_name + result["logs_artifact_name"] = f"logs-{stem}" + labeled.append(result) + return labeled + + def parse_definition(value: object) -> BuildDefinition: """Validate one untrusted JSON row before admitting it to the typed model. @@ -178,6 +215,7 @@ def main(argv: list[str] | None = None) -> None: rows = select_builds(args.matrix, host=args.host) if args.format == "github" and any(row["legacy_debugger"] == "1" for row in rows): raise ValueError("GitHub builds cannot enable the closed-source legacy debugger") + rows = label_builds(rows, args.format) except (OSError, ValueError) as error: parser.error(str(error)) result: object diff --git a/ci/tests/test_matrix.py b/ci/tests/test_matrix.py index d66c86d6c6..359a11f92e 100644 --- a/ci/tests/test_matrix.py +++ b/ci/tests/test_matrix.py @@ -96,11 +96,46 @@ def test_provider_formats_preserve_same_flavors(self) -> None: result: object = json.loads(output.getvalue()) expected: object = rows if format_name == "github": - expected = {"include": rows} + expected = {"include": matrix.label_builds(rows, "github")} elif format_name == "ado": - expected = {row["id"]: {key: value for key, value in row.items() if key != "id"} for row in rows} + expected = {row["id"]: {key: value for key, value in row.items() if key != "id"} + for row in matrix.label_builds(rows, "ado")} self.assertEqual(result, expected) + def test_github_labels_preserve_historical_artifact_names(self) -> None: + rows = matrix.select_builds(OPEN_MATRIX) + labeled = matrix.label_builds(rows, "github") + for original, row in zip(rows, labeled): + suffix = "-patina" if row["core"] == "patina" else "" + stem = f"{row['target']}-{row['arch']}-{row['tool_chain']}{suffix}" + self.assertEqual(row["artifact_name"], f"firmware-{stem}") + self.assertEqual(row["logs_artifact_name"], f"logs-{stem}") + self.assertEqual(row["id"], original["id"]) + self.assertNotIn("display_name", original) + self.assertEqual(labeled[0]["display_name"], "DEBUG X64 CLANGPDB (Legacy)") + self.assertEqual(labeled[0]["artifact_name"], "firmware-DEBUG-X64-CLANGPDB") + self.assertEqual(labeled[-1]["display_name"], "RELEASE AARCH64 CLANGPDB (Patina)") + self.assertEqual(labeled[-1]["artifact_name"], "firmware-RELEASE-AARCH64-CLANGPDB-patina") + + def test_github_artifact_collisions_require_explicit_migration(self) -> None: + row = matrix.select_builds(OPEN_MATRIX)[0] + other: matrix.SelectedBuild = {**row, "compiler_source": "windows_org"} + other["id"] = matrix.build_id(other) + with self.assertRaisesRegex(ValueError, "Artifact name collision"): + matrix.label_builds([row, other], "github") + + def test_ado_labels_distinguish_compilers_without_renaming_packages(self) -> None: + rows = matrix.select_builds(CLOSED_MATRIX) + labeled = matrix.label_builds(rows, "ado") + self.assertEqual(len({row["display_name"] for row in labeled}), 15) + for original, row in zip(rows, labeled): + self.assertEqual(row["id"], original["id"]) + self.assertEqual(row["package_suffix"], original["package_suffix"]) + self.assertEqual(row["shipping"], original["shipping"]) + self.assertNotIn("artifact_name", row) + self.assertIn("/ Windows WinOrg", next(row["display_name"] for row in labeled + if row["compiler_source"] == "windows_org")) + def test_invalid_matrix_is_rejected(self) -> None: row = matrix.parse_definition(json.loads(OPEN_MATRIX.read_text())[0]) invalid_documents: list[object] = [ From cc5542b4dd5ba2b8f034ed183d9653b40cdcf708 Mon Sep 17 00:00:00 2001 From: maheeraeron Date: Fri, 2 Oct 2026 20:03:38 +0000 Subject: [PATCH 14/18] temp commit --- .azuredevops/templates/build.yml | 4 ++ .azuredevops/templates/source-checks.yml | 9 +++- .azuredevops/templates/windows-python.yml | 18 +++++++ MsvmPkg/Docs/BUILDING.md | 14 +++++- ci/scripts/source_checks.py | 12 +++-- ci/tests/test_ado_poc.py | 59 +++++++++++++++++++++++ ci/tests/test_source_checks.py | 17 +++++++ 7 files changed, 127 insertions(+), 6 deletions(-) create mode 100644 .azuredevops/templates/windows-python.yml create mode 100644 ci/tests/test_ado_poc.py diff --git a/.azuredevops/templates/build.yml b/.azuredevops/templates/build.yml index 1d4c3a511a..4eb599a2e1 100644 --- a/.azuredevops/templates/build.yml +++ b/.azuredevops/templates/build.yml @@ -27,6 +27,8 @@ jobs: ob_nugetPublishing_enabled: false ob_createvpack_enabled: false steps: + - template: /.azuredevops/templates/windows-python.yml@self + - ${{ each host in parameters.hosts }}: - pwsh: | $matrix = python ci/scripts/matrix.py --matrix "$env:MATRIX_FILE" --host "$env:BUILD_HOST" --format ado @@ -61,6 +63,8 @@ jobs: WindowsClangRoot: $(Agent.TempDirectory)/windows-clang-${{ parameters.windowsClangVersion }} steps: - ${{ if eq(host, 'windows') }}: + - template: /.azuredevops/templates/windows-python.yml@self + - pwsh: | $vswhere = "${env:ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere.exe" $installation = & $vswhere -latest -version '[17.0,18.0)' -products '*' -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath diff --git a/.azuredevops/templates/source-checks.yml b/.azuredevops/templates/source-checks.yml index 58ff08e935..bd38bf7d38 100644 --- a/.azuredevops/templates/source-checks.yml +++ b/.azuredevops/templates/source-checks.yml @@ -14,8 +14,15 @@ jobs: ob_nugetPublishing_enabled: false ob_createvpack_enabled: false steps: - - script: python ci/scripts/source_checks.py + - template: /.azuredevops/templates/windows-python.yml@self + + - script: python ci/scripts/source_checks.py --verbose displayName: Run Source Checks + env: + GIT_TRACE: '1' + GIT_TRACE_CURL: '1' + GIT_TRACE_CURL_NO_DATA: '1' + GIT_TRACE_REDACT: '1' - task: PublishTestResults@2 displayName: Publish Stuart Results diff --git a/.azuredevops/templates/windows-python.yml b/.azuredevops/templates/windows-python.yml new file mode 100644 index 0000000000..35677e6d62 --- /dev/null +++ b/.azuredevops/templates/windows-python.yml @@ -0,0 +1,18 @@ +steps: + - pwsh: | + $python = py -3.12 -c "import sys; print(sys.executable)" + if ($LASTEXITCODE -ne 0 -or -not $python) { + throw 'The image Python launcher must provide Python 3.12' + } + $python = $python.Trim() + if (-not (Test-Path -LiteralPath $python -PathType Leaf)) { + throw "Image Python was not found at $python" + } + $pythonHome = Split-Path -Parent $python + & $python -c "import sys; print(sys.executable); print(sys.version); sys.exit(0 if sys.version_info[:2] == (3, 12) else 1)" + if ($LASTEXITCODE -ne 0) { throw 'The image must provide Python 3.12' } + & $python -m pip --version + if ($LASTEXITCODE -ne 0) { throw 'The image Python must provide pip' } + Write-Host "##vso[task.prependpath]$pythonHome\Scripts" + Write-Host "##vso[task.prependpath]$pythonHome" + displayName: Use Image Python 3.12 diff --git a/MsvmPkg/Docs/BUILDING.md b/MsvmPkg/Docs/BUILDING.md index a57a8b693d..33406507ac 100644 --- a/MsvmPkg/Docs/BUILDING.md +++ b/MsvmPkg/Docs/BUILDING.md @@ -111,6 +111,13 @@ Each job runs inside OneBranch. The build stage runs independently of source checks, and each flavor invokes the same `ci/scripts/build.py` as GitHub. NuGet and VPack publishing remain disabled for these jobs. +All ADO Windows jobs resolve the image's Python 3.12 interpreter with +`py -3.12 -c "import sys; print(sys.executable)"`. The shared template verifies +the version and pip, then prepends the resolved interpreter and Scripts directories +to PATH. It does not assume an installation directory, download Python, or fall +back to the default Python on PATH. Linux builds retain their distribution Python +and isolated virtual environment. + The inspected OneBranch Windows job template forwards job strategies and dependencies. Local validation checks our output binding and flavor arguments, but a hosted OneBranch run is still required to verify the full governed-template expansion, runner tools, internal feed credentials, and repository access. @@ -226,7 +233,12 @@ python .\ci\scripts\source_checks.py ``` The workflow installs the Python requirements, runs `stuart_setup` and `stuart_update`, and then runs the configured -`stuart_ci_build` checks. To run one check while iterating, invoke Stuart directly. For example: +`stuart_ci_build` checks. Pass `--verbose` to enable detailed Stuart setup logging. +ADO source checks enable this option and Git command/HTTP tracing, with credential +redaction enabled and HTTP payload logging disabled, to diagnose submodule access +failures. GitHub and local runs retain their default logging. + +To run one check while iterating, invoke Stuart directly. For example: ```powershell stuart_ci_build -c .\.pytool\CISettings.py --disable-all GuidCheck=run diff --git a/ci/scripts/source_checks.py b/ci/scripts/source_checks.py index 7d6c47c47e..ad531b4f2b 100644 --- a/ci/scripts/source_checks.py +++ b/ci/scripts/source_checks.py @@ -28,11 +28,12 @@ def run(command: list[str]) -> None: subprocess.run(command, cwd=REPO_ROOT, check=True) -def run_source_checks() -> None: +def run_source_checks(verbose: bool = False) -> None: """Install requirements, prepare CI dependencies, and run the selected checks. Uses CISettings.py, not platform build settings. Commands run sequentially; the first failure propagates and prevents all subsequent commands. + Verbose mode enables Stuart setup diagnostics without logging pip credentials. """ commands: tuple[list[str], ...] = ( [sys.executable, "-m", "pip", "install", "-r", "pip-requirements.txt"], @@ -48,19 +49,22 @@ def run_source_checks() -> None: "UncrustifyCheck=run", ], ) + if verbose: + commands[1].append("--verbose") for command in commands: run(command) def main(argv: list[str] | None = None) -> None: - """Parse the no-argument CLI (including --help), then run source checks. + """Parse CLI options, then run source checks. None reads sys.argv. Argparse exits for help or invalid arguments; execution failures propagate to the caller. This function does not create a venv. """ parser = argparse.ArgumentParser(description=__doc__) - parser.parse_args(argv) - run_source_checks() + parser.add_argument("--verbose", action="store_true", help="Enable verbose Stuart setup logging") + args = parser.parse_args(argv) + run_source_checks(verbose=args.verbose) if __name__ == "__main__": diff --git a/ci/tests/test_ado_poc.py b/ci/tests/test_ado_poc.py new file mode 100644 index 0000000000..6f132f41d9 --- /dev/null +++ b/ci/tests/test_ado_poc.py @@ -0,0 +1,59 @@ +"""Lock ADO adapter contracts without executing hosted tasks.""" + +import importlib +import unittest +from pathlib import Path +from typing import Any, cast + +ROOT = Path(__file__).resolve().parents[2] + + +def load_yaml(path: str) -> dict[str, Any]: + """Load an ADO template for contract assertions; never expand or execute it.""" + yaml = importlib.import_module("yaml") + return cast(dict[str, Any], yaml.safe_load((ROOT / path).read_text())) + + +class AdoPocTests(unittest.TestCase): + """Preserve provider-specific environment preparation.""" + + def test_source_check_diagnostics_redact_git_credentials(self) -> None: + source = load_yaml(".azuredevops/templates/source-checks.yml")["jobs"][0] + step = next(step for step in source["steps"] if step.get("displayName") == "Run Source Checks") + self.assertEqual(step["script"], "python ci/scripts/source_checks.py --verbose") + for variable in ("GIT_TRACE", "GIT_TRACE_CURL", "GIT_TRACE_CURL_NO_DATA", "GIT_TRACE_REDACT"): + self.assertEqual(step["env"][variable], "1") + + def test_windows_jobs_use_image_python_without_downloading(self) -> None: + """All Windows callers select the image interpreter; Linux keeps its venv.""" + reference = {"template": "/.azuredevops/templates/windows-python.yml@self"} + template = load_yaml(".azuredevops/templates/build.yml") + self.assertEqual(template["jobs"][0]["steps"][0], reference) + source = load_yaml(".azuredevops/templates/source-checks.yml")["jobs"][0] + self.assertEqual(source["steps"][0], reference) + job = template["jobs"][1]["${{ each host in parameters.hosts }}"][0] + windows = next(branch["${{ if eq(host, 'windows') }}"] for branch in job["steps"] + if "${{ if eq(host, 'windows') }}" in branch) + linux = next(branch["${{ if eq(host, 'linux') }}"] for branch in job["steps"] + if "${{ if eq(host, 'linux') }}" in branch) + self.assertEqual(windows[0], reference) + self.assertNotIn(reference, linux) + self.assertIn('python3 -m venv "$BUILD_VENV"', linux[0]["bash"]) + script = load_yaml(".azuredevops/templates/windows-python.yml")["steps"][0]["pwsh"] + self.assertIn('py -3.12 -c "import sys; print(sys.executable)"', script) + self.assertIn("if ($LASTEXITCODE -ne 0 -or -not $python)", script) + self.assertIn("$python = $python.Trim()", script) + self.assertIn("Test-Path -LiteralPath $python -PathType Leaf", script) + self.assertIn("$pythonHome = Split-Path -Parent $python", script) + self.assertNotIn("C:\\Program Files\\Python312", script) + self.assertIn("sys.version_info[:2] == (3, 12)", script) + self.assertIn('task.prependpath]$pythonHome\\Scripts', script) + self.assertIn('task.prependpath]$pythonHome"', script) + self.assertIn("-m pip --version", script) + self.assertNotIn("install", script) + for path in (ROOT / ".azuredevops/templates").glob("*.yml"): + self.assertNotIn("UsePythonVersion@", path.read_text(), str(path)) + + +if __name__ == "__main__": + unittest.main() diff --git a/ci/tests/test_source_checks.py b/ci/tests/test_source_checks.py index ddeb32a878..8877b303b7 100644 --- a/ci/tests/test_source_checks.py +++ b/ci/tests/test_source_checks.py @@ -1,6 +1,7 @@ """Verify the shared source-check sequence without invoking pip or Stuart.""" import sys +import subprocess import unittest from unittest.mock import MagicMock, call, patch @@ -33,6 +34,22 @@ def test_source_checks_use_native_stuart_commands(self, run_mock: MagicMock) -> [call(command, cwd=source_checks.REPO_ROOT, check=True) for command in expected_commands], ) + @patch("ci.scripts.source_checks.subprocess.run") + def test_verbose_only_enables_stuart_setup_logging(self, run_mock: MagicMock) -> None: + source_checks.main(["--verbose"]) + commands = [invocation.args[0] for invocation in run_mock.call_args_list] + self.assertEqual(len(commands), 4) + self.assertEqual(commands[1], ["stuart_setup", "-c", ".pytool/CISettings.py", "--verbose"]) + for index in (0, 2, 3): + self.assertNotIn("--verbose", commands[index]) + + @patch("ci.scripts.source_checks.subprocess.run") + def test_verbose_setup_failure_stops_checks(self, run_mock: MagicMock) -> None: + run_mock.side_effect = [None, subprocess.CalledProcessError(1, "stuart_setup")] + with self.assertRaises(subprocess.CalledProcessError): + source_checks.main(["--verbose"]) + self.assertEqual(run_mock.call_count, 2) + if __name__ == "__main__": unittest.main() From f3e156c286a549e05b6fce24151ad6e256255cd0 Mon Sep 17 00:00:00 2001 From: maheeraeron Date: Thu, 15 Oct 2026 23:31:52 +0000 Subject: [PATCH 15/18] temp commit --- .azuredevops/templates/build.yml | 18 +++++++++++++++ .azuredevops/templates/source-checks.yml | 8 +++++++ MsvmPkg/Docs/BUILDING.md | 10 ++++++++ ci/tests/test_ado_poc.py | 29 +++++++++++++++++++++++- 4 files changed, 64 insertions(+), 1 deletion(-) diff --git a/.azuredevops/templates/build.yml b/.azuredevops/templates/build.yml index 4eb599a2e1..ebf378c748 100644 --- a/.azuredevops/templates/build.yml +++ b/.azuredevops/templates/build.yml @@ -101,6 +101,10 @@ jobs: version: ${{ parameters.windowsClangVersion }} downloadPath: $(WindowsClangRoot)/lib/clang/${{ parameters.windowsClangMajorVersion }} + - template: /.azuredevops/templates/repo-auth.yml@self + parameters: + mode: set + - pwsh: | $buildArgs = @('--arch', $env:BUILD_ARCH, '--target', $env:BUILD_TARGET, '--tool-chain', $env:BUILD_TOOL_CHAIN, '--core', $env:BUILD_CORE, @@ -122,7 +126,16 @@ jobs: WINDOWS_CLANG_ROOT: $(WindowsClangRoot) OFFICIAL_BUILD: ${{ iif(parameters.officialBuild, '1', '0') }} + - template: /.azuredevops/templates/repo-auth.yml@self + parameters: + mode: unset + - ${{ if eq(host, 'linux') }}: + - template: /.azuredevops/templates/repo-auth.yml@self + parameters: + mode: set + python: python3 + - bash: | set -euo pipefail apt-get update @@ -155,3 +168,8 @@ jobs: BUILD_LEGACY_DEBUGGER: $(legacy_debugger) BUILD_VENV: $(Agent.TempDirectory)/firmware-venv OFFICIAL_BUILD: ${{ iif(parameters.officialBuild, '1', '0') }} + + - template: /.azuredevops/templates/repo-auth.yml@self + parameters: + mode: unset + python: python3 diff --git a/.azuredevops/templates/source-checks.yml b/.azuredevops/templates/source-checks.yml index bd38bf7d38..39a41c9829 100644 --- a/.azuredevops/templates/source-checks.yml +++ b/.azuredevops/templates/source-checks.yml @@ -16,6 +16,10 @@ jobs: steps: - template: /.azuredevops/templates/windows-python.yml@self + - template: /.azuredevops/templates/repo-auth.yml@self + parameters: + mode: set + - script: python ci/scripts/source_checks.py --verbose displayName: Run Source Checks env: @@ -32,3 +36,7 @@ jobs: testResultsFiles: Build/TestSuites.xml testRunTitle: Stuart CI failTaskOnFailedTests: true + + - template: /.azuredevops/templates/repo-auth.yml@self + parameters: + mode: unset diff --git a/MsvmPkg/Docs/BUILDING.md b/MsvmPkg/Docs/BUILDING.md index 33406507ac..4a7de56c8b 100644 --- a/MsvmPkg/Docs/BUILDING.md +++ b/MsvmPkg/Docs/BUILDING.md @@ -111,6 +111,16 @@ Each job runs inside OneBranch. The build stage runs independently of source checks, and each flavor invokes the same `ci/scripts/build.py` as GitHub. NuGet and VPack publishing remain disabled for these jobs. +ADO source checks and both firmware build hosts use `repo-auth.yml` to call the +shared `ci/scripts/repo_auth.py` before Stuart. The adapter supplies the job's +`System.AccessToken` and private repository URL; the helper writes a URL-scoped +authorization header to the dedicated container's global Git configuration. +An `always()` cleanup step removes it even after failures. This follows the +working closed pipeline's authentication mechanism without changing shared build +or source-check runners. GitHub does not invoke this adapter. The helper disables +Git tracing while handling credentials and must not be used against a developer's +global Git configuration. + All ADO Windows jobs resolve the image's Python 3.12 interpreter with `py -3.12 -c "import sys; print(sys.executable)"`. The shared template verifies the version and pip, then prepends the resolved interpreter and Scripts directories diff --git a/ci/tests/test_ado_poc.py b/ci/tests/test_ado_poc.py index 6f132f41d9..086daa21da 100644 --- a/ci/tests/test_ado_poc.py +++ b/ci/tests/test_ado_poc.py @@ -38,7 +38,7 @@ def test_windows_jobs_use_image_python_without_downloading(self) -> None: if "${{ if eq(host, 'linux') }}" in branch) self.assertEqual(windows[0], reference) self.assertNotIn(reference, linux) - self.assertIn('python3 -m venv "$BUILD_VENV"', linux[0]["bash"]) + self.assertIn('python3 -m venv "$BUILD_VENV"', next(step["bash"] for step in linux if "bash" in step)) script = load_yaml(".azuredevops/templates/windows-python.yml")["steps"][0]["pwsh"] self.assertIn('py -3.12 -c "import sys; print(sys.executable)"', script) self.assertIn("if ($LASTEXITCODE -ne 0 -or -not $python)", script) @@ -54,6 +54,33 @@ def test_windows_jobs_use_image_python_without_downloading(self) -> None: for path in (ROOT / ".azuredevops/templates").glob("*.yml"): self.assertNotIn("UsePythonVersion@", path.read_text(), str(path)) + def test_repository_auth_wraps_source_checks_and_both_build_hosts(self) -> None: + reference = "/.azuredevops/templates/repo-auth.yml@self" + auth = load_yaml(".azuredevops/templates/repo-auth.yml")["steps"][0] + self.assertEqual(auth["${{ if eq(parameters.mode, 'unset') }}"]["condition"], "always()") + self.assertEqual(auth["${{ if eq(parameters.mode, 'set') }}"]["env"], + {"SYSTEM_ACCESSTOKEN": "$(System.AccessToken)"}) + self.assertIn("ci/scripts/repo_auth.py", auth["script"]) + source = load_yaml(".azuredevops/templates/source-checks.yml")["jobs"][0]["steps"] + job = load_yaml(".azuredevops/templates/build.yml")["jobs"][1]["${{ each host in parameters.hosts }}"][0] + groups = [(source, "ci/scripts/source_checks.py")] + for host in ("windows", "linux"): + key = "${{ if eq(host, '" + host + "') }}" + steps = next(branch[key] for branch in job["steps"] if key in branch) + groups.append((steps, "ci/scripts/build.py")) + for steps, runner in groups: + setup = next(i for i, step in enumerate(steps) + if step.get("template") == reference and step["parameters"]["mode"] == "set") + cleanup = next(i for i, step in enumerate(steps) + if step.get("template") == reference and step["parameters"]["mode"] == "unset") + invocation = next(i for i, step in enumerate(steps) + if runner in step.get("script", step.get("pwsh", step.get("bash", "")))) + self.assertLess(setup, invocation) + self.assertLess(invocation, cleanup) + self.assertNotIn("GIT_CONFIG_VALUE_2", steps[invocation].get("env", {})) + self.assertEqual(steps[setup]["parameters"].get("python", "python"), + steps[cleanup]["parameters"].get("python", "python")) + if __name__ == "__main__": unittest.main() From 441eb70ccebb6d8bc1e0ff46874f6a2862980b34 Mon Sep 17 00:00:00 2001 From: maheeraeron Date: Mon, 19 Oct 2026 06:38:52 +0000 Subject: [PATCH 16/18] ci: synchronize ADO adapters and fix job display names Restore shared authentication files omitted from the temporary commit and align OneBranch adapters with the validated closed pipeline. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .azuredevops/templates/build.yml | 16 ++-- .azuredevops/templates/platform.yml | 5 +- .azuredevops/templates/repo-auth.yml | 18 +++++ .azuredevops/templates/source-checks.yml | 8 ++ MsvmPkg/Docs/BUILDING.md | 17 +++-- ci/scripts/repo_auth.py | 48 ++++++++++++ ci/tests/test_ado_poc.py | 7 ++ ci/tests/test_firmware_version.py | 2 + ci/tests/test_repo_auth.py | 94 ++++++++++++++++++++++++ 9 files changed, 202 insertions(+), 13 deletions(-) create mode 100644 .azuredevops/templates/repo-auth.yml create mode 100644 ci/scripts/repo_auth.py create mode 100644 ci/tests/test_repo_auth.py diff --git a/.azuredevops/templates/build.yml b/.azuredevops/templates/build.yml index ebf378c748..3ec91a0ca0 100644 --- a/.azuredevops/templates/build.yml +++ b/.azuredevops/templates/build.yml @@ -42,7 +42,7 @@ jobs: - ${{ each host in parameters.hosts }}: - job: BuildFirmware_${{ host }} - displayName: $(display_name) + displayName: Firmware (${{ host }}) dependsOn: BuildMatrix pool: type: ${{ host }} @@ -62,6 +62,14 @@ jobs: ob_createvpack_enabled: false WindowsClangRoot: $(Agent.TempDirectory)/windows-clang-${{ parameters.windowsClangVersion }} steps: + - task: NuGetAuthenticate@1 + inputs: + nuGetServiceConnections: WindowsPartners_NugetFeedConnection + + - task: PipAuthenticate@1 + inputs: + artifactFeeds: $(System.TeamProject)/$(System.TeamProject)_PublicPackages + - ${{ if eq(host, 'windows') }}: - template: /.azuredevops/templates/windows-python.yml@self @@ -75,12 +83,6 @@ jobs: displayName: Locate Visual Studio Clang condition: and(succeeded(), eq(variables['tool_chain'], 'CLANGPDB'), eq(variables['compiler_source'], 'visual_studio')) - - task: NuGetAuthenticate@1 - displayName: Authenticate Internal Compiler Feed - condition: and(succeeded(), eq(variables['compiler_source'], 'windows_org')) - inputs: - nuGetServiceConnections: WindowsPartners_NugetFeedConnection - - task: DownloadPackage@1 displayName: Download Windows-org LLVM Tools condition: and(succeeded(), eq(variables['compiler_source'], 'windows_org')) diff --git a/.azuredevops/templates/platform.yml b/.azuredevops/templates/platform.yml index 3a5d5649d6..f292235c10 100644 --- a/.azuredevops/templates/platform.yml +++ b/.azuredevops/templates/platform.yml @@ -14,10 +14,13 @@ extends: parameters: platform: name: windows_undocked + globalSdl: + tsa: + enabled: false featureFlags: WindowsHostVersion: 1ESWindows2022 git: - submodules: true + submodules: false stages: - stage: SourceChecks jobs: diff --git a/.azuredevops/templates/repo-auth.yml b/.azuredevops/templates/repo-auth.yml new file mode 100644 index 0000000000..c38949c789 --- /dev/null +++ b/.azuredevops/templates/repo-auth.yml @@ -0,0 +1,18 @@ +parameters: + - name: mode + type: string + values: + - set + - unset + - name: python + type: string + default: python + +steps: + - script: ${{ parameters.python }} ci/scripts/repo_auth.py ${{ parameters.mode }} --url https://dev.azure.com/microsoft/OS/_git/hyperv.uefi.mu_basecore + displayName: ${{ parameters.mode }} ADO Repository Authentication + ${{ if eq(parameters.mode, 'unset') }}: + condition: always() + ${{ if eq(parameters.mode, 'set') }}: + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/.azuredevops/templates/source-checks.yml b/.azuredevops/templates/source-checks.yml index 39a41c9829..e6728c16c5 100644 --- a/.azuredevops/templates/source-checks.yml +++ b/.azuredevops/templates/source-checks.yml @@ -16,6 +16,14 @@ jobs: steps: - template: /.azuredevops/templates/windows-python.yml@self + - task: NuGetAuthenticate@1 + inputs: + nuGetServiceConnections: WindowsPartners_NugetFeedConnection + + - task: PipAuthenticate@1 + inputs: + artifactFeeds: $(System.TeamProject)/$(System.TeamProject)_PublicPackages + - template: /.azuredevops/templates/repo-auth.yml@self parameters: mode: set diff --git a/MsvmPkg/Docs/BUILDING.md b/MsvmPkg/Docs/BUILDING.md index 4a7de56c8b..45cbe65936 100644 --- a/MsvmPkg/Docs/BUILDING.md +++ b/MsvmPkg/Docs/BUILDING.md @@ -87,7 +87,8 @@ The closed profile restores 15 legacy-core builds from the preserved hyperv.uefi | Linux | Distribution GCC | X64 RELEASE only | The four closed X64 DEBUG rows enable the legacy debugger: VS2022 and all three Clang compiler sources. -This matches the fetched ADO configuration; successful compilation and runtime debugging still require validation. +This matches the fetched ADO configuration. All 15 closed flavors compiled in PR +build 160110571; runtime debugging still requires validation. All open rows and all GCC rows disable the legacy debugger. ARM64 MSVC is unsupported. Disabled GCC combinations are not reintroduced. @@ -128,9 +129,9 @@ to PATH. It does not assume an installation directory, download Python, or fall back to the default Python on PATH. Linux builds retain their distribution Python and isolated virtual environment. -The inspected OneBranch Windows job template forwards job strategies and dependencies. Local validation -checks our output binding and flavor arguments, but a hosted OneBranch run is still required to verify -the full governed-template expansion, runner tools, internal feed credentials, and repository access. +OneBranch PR build 160110571 passed source checks and all 15 firmware builds, +validating the native matrices, runner tools, internal feeds, and repository +authentication. Official builds and later pipeline changes still need hosted validation. ### Job Labels and Artifact Names @@ -139,7 +140,13 @@ Provider matrix output adds a `display_name` derived from the same flavor: - GitHub: `DEBUG X64 CLANGPDB (Legacy)` or `RELEASE AARCH64 CLANGPDB (Patina)`. - ADO: the same label followed by host and compiler source, such as `/ Windows WinOrg` or `/ Linux Distro`. - Debugger-enabled rows also show `/ Legacy debugger`. ADO may additionally show its matrix leg key in the UI. + Debugger-enabled rows also include `/ Legacy debugger` in the generated metadata. + +ADO job names cannot use the runtime matrix's `$(display_name)` in this OneBranch +pipeline: it appeared literally in the hosted run. Instead, job display names use +the compile-time `Firmware (windows)` or `Firmware (linux)` prefix; ADO appends +the unchanged matrix leg ID to distinguish each flavor. The generated +`display_name` remains available as metadata, not as the job's UI name. GitHub matrix output also supplies `artifact_name` and `logs_artifact_name` for the future upload steps. They preserve the pre-refactor consumer names exactly: diff --git a/ci/scripts/repo_auth.py b/ci/scripts/repo_auth.py new file mode 100644 index 0000000000..0aa59481b3 --- /dev/null +++ b/ci/scripts/repo_auth.py @@ -0,0 +1,48 @@ +"""Set or clear URL-scoped Git authentication in a dedicated CI container. + +ADO supplies SYSTEM_ACCESSTOKEN; shared Stuart runners remain provider-neutral. +Call unset in an always-running cleanup step, including after setup failures. +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from urllib.parse import urlsplit + + +def main(argv: list[str] | None = None) -> int: + """Configure Git, reporting failures without echoing credential arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("set", "unset")) + parser.add_argument("--url", required=True, help="HTTPS repository URL to authenticate") + args = parser.parse_args(argv) + url = urlsplit(args.url) + if url.scheme != "https" or not url.hostname or url.username or url.password or url.query or url.fragment: + parser.error("--url must be an HTTPS repository URL without credentials, query, or fragment") + key = f"http.{args.url}.extraheader" + command = ["git", "config", "--global"] + if args.mode == "set": + token = os.environ.get("SYSTEM_ACCESSTOKEN", "").strip() + if not token or token == "$(System.AccessToken)" or "\n" in token or "\r" in token: + parser.error("SYSTEM_ACCESSTOKEN must contain the pipeline job token") + command.extend(["--replace-all", key, f"AUTHORIZATION: bearer {token}"]) + else: + command.extend(["--unset-all", key]) + + environment = { + key: value for key, value in os.environ.items() + if not key.startswith("GIT_TRACE") and key != "GIT_CURL_VERBOSE" + } + result = subprocess.run(command, capture_output=True, env=environment) + if result.returncode != 0 and not (args.mode == "unset" and result.returncode == 5): + print(f"Git authentication {args.mode} failed (exit {result.returncode})", file=sys.stderr) + return result.returncode + print(f"Git authentication {args.mode} completed for {args.url}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/tests/test_ado_poc.py b/ci/tests/test_ado_poc.py index 086daa21da..b58dfcefba 100644 --- a/ci/tests/test_ado_poc.py +++ b/ci/tests/test_ado_poc.py @@ -17,6 +17,13 @@ def load_yaml(path: str) -> dict[str, Any]: class AdoPocTests(unittest.TestCase): """Preserve provider-specific environment preparation.""" + def test_build_job_labels_are_resolved_at_template_expansion(self) -> None: + job = load_yaml(".azuredevops/templates/build.yml")["jobs"][1]["${{ each host in parameters.hosts }}"][0] + self.assertEqual(job["displayName"], "Firmware (${{ host }})") + self.assertNotIn("$(", job["displayName"]) + self.assertEqual(job["strategy"]["matrix"], + "$[ dependencies.BuildMatrix.outputs['${{ host }}.matrix'] ]") + def test_source_check_diagnostics_redact_git_credentials(self) -> None: source = load_yaml(".azuredevops/templates/source-checks.yml")["jobs"][0] step = next(step for step in source["steps"] if step.get("displayName") == "Run Source Checks") diff --git a/ci/tests/test_firmware_version.py b/ci/tests/test_firmware_version.py index dc2d17389e..39b48296ed 100644 --- a/ci/tests/test_firmware_version.py +++ b/ci/tests/test_firmware_version.py @@ -144,6 +144,8 @@ def test_provider_official_context_reaches_both_hosts(self) -> None: self.assertIs(parameter["default"], False) job = template["jobs"][1]["${{ each host in parameters.hosts }}"][0] for branch in job["steps"]: + if "task" in branch: + continue steps = next(iter(branch.values())) build_step = next(step for step in steps if "ci/scripts/build.py" in step.get("pwsh", step.get("bash", ""))) self.assertEqual(build_step["env"]["OFFICIAL_BUILD"], "${{ iif(parameters.officialBuild, '1', '0') }}") diff --git a/ci/tests/test_repo_auth.py b/ci/tests/test_repo_auth.py new file mode 100644 index 0000000000..2daab210bb --- /dev/null +++ b/ci/tests/test_repo_auth.py @@ -0,0 +1,94 @@ +"""Test CI Git authentication without touching the developer's Git configuration.""" + +import io +import os +import subprocess +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from contextlib import redirect_stderr, redirect_stdout +from unittest.mock import MagicMock, patch + +from ci.scripts import repo_auth + +URL = "https://dev.azure.com/example/project/_git/basecore" + + +class RepoAuthTests(unittest.TestCase): + def test_real_git_reads_header_in_separate_process_and_cleanup_removes_it(self) -> None: + with TemporaryDirectory() as directory: + config = Path(directory) / "gitconfig" + environment = {key: value for key, value in os.environ.items() if not key.startswith("GIT_CONFIG")} + environment.update({ + "GIT_CONFIG_GLOBAL": str(config), + "GIT_CONFIG_NOSYSTEM": "1", + "SYSTEM_ACCESSTOKEN": "test-token", + }) + with patch.dict(os.environ, environment, clear=True), redirect_stdout(io.StringIO()): + self.assertEqual(repo_auth.main(["set", "--url", URL]), 0) + try: + result = subprocess.run( + ["git", "-C", directory, "config", "--get-urlmatch", "http.extraheader", URL], + capture_output=True, text=True, check=True, + ) + self.assertEqual(result.stdout.strip(), "AUTHORIZATION: bearer test-token") + finally: + self.assertEqual(repo_auth.main(["unset", "--url", URL]), 0) + self.assertEqual(repo_auth.main(["unset", "--url", URL]), 0) + result = subprocess.run( + ["git", "-C", directory, "config", "--get-urlmatch", "http.extraheader", URL], + capture_output=True, text=True, + ) + self.assertEqual(result.returncode, 1) + self.assertNotIn("test-token", config.read_text()) + + @patch("ci.scripts.repo_auth.subprocess.run") + def test_set_uses_global_url_scoped_header_without_logging_token(self, run: MagicMock) -> None: + run.return_value = subprocess.CompletedProcess([], 0) + output = io.StringIO() + with patch.dict(os.environ, {"SYSTEM_ACCESSTOKEN": "test-token", "GIT_TRACE": "1", + "GIT_TRACE2": "1", "GIT_CURL_VERBOSE": "1"}), redirect_stdout(output): + self.assertEqual(repo_auth.main(["set", "--url", URL]), 0) + self.assertEqual(run.call_args.args[0], [ + "git", "config", "--global", "--replace-all", f"http.{URL}.extraheader", + "AUTHORIZATION: bearer test-token", + ]) + environment = run.call_args.kwargs["env"] + self.assertFalse(any(key.startswith("GIT_TRACE") for key in environment)) + self.assertNotIn("GIT_CURL_VERBOSE", environment) + self.assertNotIn("test-token", output.getvalue()) + + @patch("ci.scripts.repo_auth.subprocess.run") + def test_cleanup_is_idempotent_and_needs_no_token(self, run: MagicMock) -> None: + for code in (0, 5): + run.return_value = subprocess.CompletedProcess([], code) + with patch.dict(os.environ, {}, clear=True), redirect_stdout(io.StringIO()): + self.assertEqual(repo_auth.main(["unset", "--url", URL]), 0) + self.assertEqual(run.call_args.args[0], [ + "git", "config", "--global", "--unset-all", f"http.{URL}.extraheader", + ]) + + @patch("ci.scripts.repo_auth.subprocess.run") + def test_git_errors_are_not_suppressed_or_echoed(self, run: MagicMock) -> None: + run.return_value = subprocess.CompletedProcess([], 3, stderr=b"test-token") + for mode in ("set", "unset"): + error = io.StringIO() + with patch.dict(os.environ, {"SYSTEM_ACCESSTOKEN": "test-token"}), redirect_stderr(error): + self.assertEqual(repo_auth.main([mode, "--url", URL]), 3) + self.assertIn("failed (exit 3)", error.getvalue()) + self.assertNotIn("test-token", error.getvalue()) + + @patch("ci.scripts.repo_auth.subprocess.run") + def test_missing_token_and_invalid_url_fail_before_git(self, run: MagicMock) -> None: + for token in ("", "$(System.AccessToken)", "bad\ntoken"): + with patch.dict(os.environ, {"SYSTEM_ACCESSTOKEN": token}), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + repo_auth.main(["set", "--url", URL]) + for url in ("http://example.com/repo", "https://user:pass@example.com/repo"): + with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + repo_auth.main(["unset", "--url", url]) + run.assert_not_called() + + +if __name__ == "__main__": + unittest.main() From ad44b11780391b2e458ddba06ebb8e96c1b9c6b2 Mon Sep 17 00:00:00 2001 From: maheeraeron Date: Mon, 19 Oct 2026 07:06:29 +0000 Subject: [PATCH 17/18] ci: restore shared artifact staging and gated publishing Preserve GitHub and OneBranch artifact contracts, including legacy NuGet payload paths. Package only shipping Windows flavors after all checks and builds; exercise PR packing without external publication. Restore main GitHub releases while OpenVMM remains deferred. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .azuredevops/templates/build.yml | 17 ++ .azuredevops/templates/platform.yml | 13 + .azuredevops/templates/publish.yml | 106 +++++++ .azuredevops/templates/source-checks.yml | 7 + .azuredevops/templates/stage-artifacts.yml | 19 ++ .github/actions/release/action.yml | 46 +++ .github/actions/stage-artifacts/action.yml | 48 +++ .github/workflows/platform-ci.yml | 55 ++++ MsvmPkg/Docs/BUILDING.md | 81 +++-- ci/scripts/matrix.py | 17 ++ ci/scripts/package_artifacts.py | 167 ++++++++++ ci/scripts/stage_artifacts.py | 181 +++++++++++ ci/tests/test_artifacts.py | 338 +++++++++++++++++++++ ci/tests/test_matrix.py | 19 +- 14 files changed, 1095 insertions(+), 19 deletions(-) create mode 100644 .azuredevops/templates/publish.yml create mode 100644 .azuredevops/templates/stage-artifacts.yml create mode 100644 .github/actions/release/action.yml create mode 100644 .github/actions/stage-artifacts/action.yml create mode 100644 ci/scripts/package_artifacts.py create mode 100644 ci/scripts/stage_artifacts.py create mode 100644 ci/tests/test_artifacts.py diff --git a/.azuredevops/templates/build.yml b/.azuredevops/templates/build.yml index 3ec91a0ca0..c12c4a0738 100644 --- a/.azuredevops/templates/build.yml +++ b/.azuredevops/templates/build.yml @@ -58,6 +58,9 @@ jobs: GIT_CONFIG_KEY_1: core.longpaths GIT_CONFIG_VALUE_1: "true" ob_outputDirectory: $(Build.ArtifactStagingDirectory)/BUILD/$(System.JobId) + ob_artifactBaseName: $(build_artifact_name) + ob_pipelineartifacts_enabled: true + ob_artifacts_main_condition: succeededOrFailed() ob_nugetPublishing_enabled: false ob_createvpack_enabled: false WindowsClangRoot: $(Agent.TempDirectory)/windows-clang-${{ parameters.windowsClangVersion }} @@ -128,10 +131,20 @@ jobs: WINDOWS_CLANG_ROOT: $(WindowsClangRoot) OFFICIAL_BUILD: ${{ iif(parameters.officialBuild, '1', '0') }} + - task: PublishSymbols@2 + displayName: Index Shipping Symbols + condition: and(succeeded(), eq(variables['shipping'], 'true')) + inputs: + SearchPattern: Build/Msvm$(arch)/$(target)_$(tool_chain)/PDB/*.pdb + IndexSources: true + PublishSymbols: false + - template: /.azuredevops/templates/repo-auth.yml@self parameters: mode: unset + - template: /.azuredevops/templates/stage-artifacts.yml@self + - ${{ if eq(host, 'linux') }}: - template: /.azuredevops/templates/repo-auth.yml@self parameters: @@ -175,3 +188,7 @@ jobs: parameters: mode: unset python: python3 + + - template: /.azuredevops/templates/stage-artifacts.yml@self + parameters: + python: python3 diff --git a/.azuredevops/templates/platform.yml b/.azuredevops/templates/platform.yml index f292235c10..4242f118ac 100644 --- a/.azuredevops/templates/platform.yml +++ b/.azuredevops/templates/platform.yml @@ -19,6 +19,11 @@ extends: enabled: false featureFlags: WindowsHostVersion: 1ESWindows2022 + nugetPublishing: + feeds: + - name: OS/HyperVUefi + continueOnConflict: false + files_to_publish: '*.nupkg;!*.symbols.nupkg' git: submodules: false stages: @@ -32,3 +37,11 @@ extends: parameters: matrixFile: .azuredevops/build-matrix.json officialBuild: ${{ eq(parameters.oneBranchBase, 'v2/Microsoft.Official.yml') }} + - stage: Packages + dependsOn: [SourceChecks, Build] + ${{ if eq(parameters.oneBranchBase, 'v2/Microsoft.Official.yml') }}: + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/official/'), ne(variables['Build.Reason'], 'PullRequest')) + jobs: + - template: /.azuredevops/templates/publish.yml@self + parameters: + officialBuild: ${{ eq(parameters.oneBranchBase, 'v2/Microsoft.Official.yml') }} diff --git a/.azuredevops/templates/publish.yml b/.azuredevops/templates/publish.yml new file mode 100644 index 0000000000..5512bd02ec --- /dev/null +++ b/.azuredevops/templates/publish.yml @@ -0,0 +1,106 @@ +parameters: + - name: officialBuild + type: boolean + default: false + +jobs: + - job: PackageMatrix + displayName: Select Shipping Packages + pool: + type: windows + variables: + WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest + ob_outputDirectory: $(Build.ArtifactStagingDirectory)/PACKAGE_MATRIX + ob_nugetPublishing_enabled: false + ob_createvpack_enabled: false + steps: + - template: /.azuredevops/templates/windows-python.yml@self + + - pwsh: | + $matrix = python -m ci.scripts.matrix --matrix .azuredevops/build-matrix.json --shipping-only --format ado + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $version = python -m ci.scripts.package_artifacts version + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $major, $minor = $version.Split('.') + Write-Host "##vso[task.setvariable variable=matrix;isOutput=true]$matrix" + Write-Host "##vso[task.setvariable variable=major;isOutput=true]$major" + Write-Host "##vso[task.setvariable variable=minor;isOutput=true]$minor" + name: shipping + displayName: Select Shipping Flavors and Release Version + + - job: PackageFirmware + displayName: Package Firmware + dependsOn: PackageMatrix + pool: + type: windows + strategy: + maxParallel: 3 + matrix: $[ dependencies.PackageMatrix.outputs['shipping.matrix'] ] + variables: + WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest + ReleaseMajor: $[ dependencies.PackageMatrix.outputs['shipping.major'] ] + ReleaseMinor: $[ dependencies.PackageMatrix.outputs['shipping.minor'] ] + ob_outputDirectory: $(Build.ArtifactStagingDirectory)/PACKAGES + ob_artifactBaseName: $(artifact_name) + ob_nugetPublishing_enabled: ${{ parameters.officialBuild }} + ob_createvpack_enabled: ${{ parameters.officialBuild }} + ${{ if eq(parameters.officialBuild, true) }}: + ob_createvpack_packagename: $(Build.Repository.Name).mscoreuefi.$(arch).$(target) + ${{ else }}: + ob_createvpack_packagename: $(Build.Repository.Name).mscoreuefi.PR.$(arch).$(target).Test + ob_createvpack_description: Hyper V UEFI Firmware + ob_createvpack_versionAs: parts + ob_createvpack_majorVer: $(ReleaseMajor) + ob_createvpack_minorVer: $(ReleaseMinor) + ob_createvpack_metadata: $(Build.SourceVersion) + ob_createvpack_failOnStdErr: true + ob_createvpack_verbose: true + ob_createvpack_vpackdirectory: $(ob_outputDirectory)/Build/Msvm$(arch)/$(target)_$(tool_chain)/Bin + steps: + - template: /.azuredevops/templates/windows-python.yml@self + + - task: DownloadPipelineArtifact@2 + displayName: Download Shipping Firmware + inputs: + buildType: current + artifactName: $(build_artifact_name) + targetPath: $(Agent.TempDirectory)/firmware-input + + - pwsh: | + $packageArgs = @('nuget', '--matrix', '.azuredevops/build-matrix.json', + '--build-id', $env:BUILD_ID, '--source', $env:ARTIFACT_INPUT, + '--output', $env:PACKAGE_MANIFEST, '--staging-output', $env:ARTIFACT_OUTPUT, + '--commit', $env:SOURCE_COMMIT, + '--repository', $env:SOURCE_REPOSITORY, '--branch', $env:SOURCE_BRANCH) + if ($env:OFFICIAL_BUILD -eq '1') { $packageArgs += '--official' } + python -m ci.scripts.package_artifacts @packageArgs + exit $LASTEXITCODE + displayName: Validate Firmware and Prepare NuGet Manifest + env: + BUILD_ID: $(build_id) + ARTIFACT_INPUT: $(Agent.TempDirectory)/firmware-input + ARTIFACT_OUTPUT: $(ob_outputDirectory) + PACKAGE_MANIFEST: $(Agent.TempDirectory)/firmware.nuspec + SOURCE_COMMIT: $(Build.SourceVersion) + SOURCE_REPOSITORY: $(Build.Repository.Name) + SOURCE_BRANCH: $(Build.SourceBranchName) + OFFICIAL_BUILD: ${{ iif(parameters.officialBuild, '1', '0') }} + + - task: NuGetCommand@2 + displayName: Pack NuGet + inputs: + command: pack + packagesToPack: $(Agent.TempDirectory)/firmware.nuspec + packDestination: $(ob_outputDirectory)/packages + packTimezone: utc + + - ${{ if eq(parameters.officialBuild, true) }}: + - task: PublishSymbols@2 + displayName: Publish Shipping Symbols + inputs: + SymbolsFolder: $(ob_outputDirectory)/firmware/PDB + SearchPattern: '*.pdb' + IndexSources: false + PublishSymbols: true + SymbolServerType: TeamServices + SymbolsProduct: Hyper-V UEFI diff --git a/.azuredevops/templates/source-checks.yml b/.azuredevops/templates/source-checks.yml index e6728c16c5..4e6db59061 100644 --- a/.azuredevops/templates/source-checks.yml +++ b/.azuredevops/templates/source-checks.yml @@ -11,6 +11,9 @@ jobs: GIT_CONFIG_KEY_1: core.longpaths GIT_CONFIG_VALUE_1: "true" ob_outputDirectory: $(Build.ArtifactStagingDirectory)/SOURCE_CHECKS + ob_artifactBaseName: drop_Build_SourceChecks + ob_pipelineartifacts_enabled: true + ob_artifacts_main_condition: succeededOrFailed() ob_nugetPublishing_enabled: false ob_createvpack_enabled: false steps: @@ -48,3 +51,7 @@ jobs: - template: /.azuredevops/templates/repo-auth.yml@self parameters: mode: unset + + - script: python -m ci.scripts.stage_artifacts --logs-only --ado-layout --output "$(ob_outputDirectory)" + displayName: Stage Source Check Logs + condition: always() diff --git a/.azuredevops/templates/stage-artifacts.yml b/.azuredevops/templates/stage-artifacts.yml new file mode 100644 index 0000000000..3b0b631799 --- /dev/null +++ b/.azuredevops/templates/stage-artifacts.yml @@ -0,0 +1,19 @@ +parameters: + - name: python + type: string + default: python + +steps: + - script: >- + ${{ parameters.python }} -m ci.scripts.stage_artifacts + --matrix .azuredevops/build-matrix.json --build-id "$(build_id)" + --output "$(ob_outputDirectory)" --ado-layout --build-succeeded + displayName: Stage Firmware Artifacts + condition: succeeded() + + - script: >- + ${{ parameters.python }} -m ci.scripts.stage_artifacts + --matrix .azuredevops/build-matrix.json --build-id "$(build_id)" + --output "$(ob_outputDirectory)" --ado-layout + displayName: Stage Failure Diagnostics + condition: not(succeeded()) diff --git a/.github/actions/release/action.yml b/.github/actions/release/action.yml new file mode 100644 index 0000000000..428d60d64e --- /dev/null +++ b/.github/actions/release/action.yml @@ -0,0 +1,46 @@ +name: Release Firmware +description: Validate and archive the shared staged outputs, then publish a GitHub release + +inputs: + github-token: + description: GitHub token used to create the release + required: true + +runs: + using: composite + steps: + - name: Download firmware artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: firmware-* + path: ${{ runner.temp }}/firmware-artifacts + + - name: Package release artifacts + shell: bash + env: + RELEASE_COMMIT: ${{ github.sha }} + ARTIFACT_INPUT: ${{ runner.temp }}/firmware-artifacts + RELEASE_OUTPUT: ${{ runner.temp }}/firmware-release + run: >- + python -m ci.scripts.package_artifacts github + --matrix .github/build-matrix.json + --source "$ARTIFACT_INPUT" + --output "$RELEASE_OUTPUT" + --commit "$RELEASE_COMMIT" + + - name: Create GitHub release + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + RELEASE_REPOSITORY: ${{ github.repository }} + RELEASE_COMMIT: ${{ github.sha }} + RELEASE_OUTPUT: ${{ runner.temp }}/firmware-release + run: | + set -euo pipefail + base_version="$(python -m ci.scripts.package_artifacts version)" + tags="$(gh api --paginate "repos/$RELEASE_REPOSITORY/releases?per_page=100" --jq '.[].tag_name')" + patch="$(printf '%s\n' "$tags" | jq -Rs --arg prefix "v$base_version." \ + 'split("\n") | map(select(startswith($prefix)) | ltrimstr($prefix) | select(test("^[0-9]+$")) | tonumber) | (max // -1) + 1')" + gh release create "v$base_version.$patch" \ + --repo "$RELEASE_REPOSITORY" --target "$RELEASE_COMMIT" \ + --generate-notes "$RELEASE_OUTPUT"/*.tar.gz diff --git a/.github/actions/stage-artifacts/action.yml b/.github/actions/stage-artifacts/action.yml new file mode 100644 index 0000000000..9a77e32cff --- /dev/null +++ b/.github/actions/stage-artifacts/action.yml @@ -0,0 +1,48 @@ +name: Stage Firmware Artifacts +description: Stage shared firmware outputs and diagnostics, then upload GitHub artifacts + +inputs: + build-id: + description: Selected matrix build ID + required: true + artifact-name: + description: Firmware artifact name + required: true + logs-artifact-name: + description: Diagnostic artifact name + required: true + build-succeeded: + description: Whether the firmware build succeeded + required: true + +runs: + using: composite + steps: + - name: Stage artifacts + if: always() + shell: pwsh + env: + BUILD_ID: ${{ inputs.build-id }} + BUILD_SUCCEEDED: ${{ inputs.build-succeeded }} + ARTIFACT_OUTPUT: ${{ runner.temp }}/ci-artifacts + run: | + $stageArgs = @('--matrix', '.github/build-matrix.json', '--build-id', $env:BUILD_ID, '--output', $env:ARTIFACT_OUTPUT) + if ($env:BUILD_SUCCEEDED -eq 'true') { $stageArgs += '--build-succeeded' } + python -m ci.scripts.stage_artifacts @stageArgs + exit $LASTEXITCODE + + - name: Upload firmware + if: success() && inputs.build-succeeded == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ inputs.artifact-name }} + path: ${{ runner.temp }}/ci-artifacts/firmware/ + if-no-files-found: error + + - name: Upload diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ inputs.logs-artifact-name }} + path: ${{ runner.temp }}/ci-artifacts/logs/ + if-no-files-found: warn diff --git a/.github/workflows/platform-ci.yml b/.github/workflows/platform-ci.yml index 86057febf3..098fd806c7 100644 --- a/.github/workflows/platform-ci.yml +++ b/.github/workflows/platform-ci.yml @@ -7,6 +7,9 @@ on: pull_request: workflow_dispatch: +permissions: + contents: read + jobs: build-matrix: name: Select Build Matrix @@ -58,6 +61,7 @@ jobs: cache-dependency-path: pip-requirements.txt - name: Build firmware + id: firmware uses: ./.github/actions/build env: OFFICIAL_BUILD: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') && '1' || '0' }} @@ -70,6 +74,15 @@ jobs: compiler-source: ${{ matrix.compiler_source }} legacy-debugger: ${{ matrix.legacy_debugger }} + - name: Stage and upload artifacts + if: always() + uses: ./.github/actions/stage-artifacts + with: + build-id: ${{ matrix.id }} + artifact-name: ${{ matrix.artifact_name }} + logs-artifact-name: ${{ matrix.logs_artifact_name }} + build-succeeded: ${{ steps.firmware.outcome == 'success' }} + source-checks: name: Source Checks runs-on: windows-2022 @@ -94,3 +107,45 @@ jobs: - name: Run source checks uses: ./.github/actions/source-checks + + - name: Stage source-check logs + if: always() + shell: pwsh + env: + ARTIFACT_OUTPUT: ${{ runner.temp }}/source-check-artifacts + run: python -m ci.scripts.stage_artifacts --logs-only --output "$env:ARTIFACT_OUTPUT" + + - name: Upload source-check logs + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: source-check-logs + path: ${{ runner.temp }}/source-check-artifacts/logs/ + if-no-files-found: warn + + release: + name: Release + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: [source-checks, build] + runs-on: ubuntu-24.04 + permissions: + contents: write + actions: read + concurrency: + group: release + cancel-in-progress: false + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Release firmware + uses: ./.github/actions/release + with: + github-token: ${{ github.token }} diff --git a/MsvmPkg/Docs/BUILDING.md b/MsvmPkg/Docs/BUILDING.md index 45cbe65936..d6ac990082 100644 --- a/MsvmPkg/Docs/BUILDING.md +++ b/MsvmPkg/Docs/BUILDING.md @@ -101,8 +101,8 @@ The selector rejects duplicate flavors and fails when the supplied file or reque These commands only print JSON. They do not execute builds, install tools, or publish anything. GitHub's platform workflow supplies `.github/build-matrix.json`, feeds the result to its native `strategy.matrix`, and calls the build action for each flavor. That action invokes the same `ci/scripts/build.py` used locally. -Builds depend only on matrix generation; source checks run independently. Build artifact publication is not -part of this step yet. +Builds depend only on matrix generation; source checks run independently. Each build stages firmware and +diagnostics through `python -m ci.scripts.stage_artifacts`; the provider adapter uploads them. ADO's shared platform template supplies `.azuredevops/build-matrix.json` through its build job template. A `BuildMatrix` job exports separate Windows and Linux matrices using `--host`. Dependent `BuildFirmware_windows` and @@ -110,7 +110,8 @@ job exports separate Windows and Linux matrices using `--host`. Dependent `Build builds per host group. Host pools are selected at template expansion time, not by runtime matrix variables. Each job runs inside OneBranch. The build stage runs independently of source checks, and each flavor invokes the same `ci/scripts/build.py` as GitHub. -NuGet and VPack publishing remain disabled for these jobs. +NuGet and VPack publishing remain disabled in build jobs. A separate Windows packaging stage depends on +both source checks and the entire build stage; only the three shipping flavors are selected. ADO source checks and both firmware build hosts use `repo-auth.yml` to call the shared `ci/scripts/repo_auth.py` before Stuart. The adapter supplies the job's @@ -148,7 +149,7 @@ the compile-time `Firmware (windows)` or `Firmware (linux)` prefix; ADO appends the unchanged matrix leg ID to distinguish each flavor. The generated `display_name` remains available as metadata, not as the job's UI name. -GitHub matrix output also supplies `artifact_name` and `logs_artifact_name` for the future upload steps. +GitHub matrix output also supplies `artifact_name` and `logs_artifact_name` for the upload steps. They preserve the pre-refactor consumer names exactly: | Flavor | Firmware artifact | Log artifact | @@ -161,14 +162,54 @@ Legacy has no suffix. Keep the literal `AARCH64` and `CLANGPDB` in artifact name If future host/compiler variants collide under this pattern, selection fails instead of silently renaming artifacts. Any expanded naming scheme must be coordinated with consumers first. -ADO naming is a separate compatibility contract: historical staged folders were -`Firmware Binary File _`, `Firmware Map Files _`, -`Firmware PDB Files _`, and `Build Logs ` inside a per-job OneBranch artifact. -Those folder names alone are not globally unique. Changing job identity or log-folder labels can affect consumers -of the enclosing artifact and must be checked when staging/upload is restored. No ADO artifact names are inferred -from the new display label. Package names and suffixes below remain unchanged. - -This change defines labels and names only; artifact upload and staging are not enabled yet. +ADO retains `drop_Build___[_]`, independent of the new display names. +Non-shipping builds upload these drops directly. Shipping builds first upload `input-` for +the packaging stage, which publishes the historical drop with the NuGet under `packages/`. +Historical `Firmware Binary File ...`, `Firmware Map Files ...`, `Firmware PDB Files ...`, +`Build Logs `, and `Build/Msvm/_/Bin/` paths are retained +alongside the shared staging layout. Source-check output retains `drop_Build_SourceChecks` and its +root-level logs, test results, and `MsvmPkg/` diagnostics. No consumer migration is required. + +### Artifact Staging and Publication + +The shared staging module runs after Stuart, rather than as a post-build plugin, so setup/update failures +also retain diagnostics. Existing Stuart plugins still own firmware version generation and PDB/MAP collection. +Successful staging requires nonempty firmware, version JSON/blob, MAP files, and (except GCC) PDB files. +Shipping flavors additionally require DSDT AML and `BiosInterface.h`. Missing required outputs fail staging; +they are not silently omitted from a successful artifact. + +The shared layout is: + +- `firmware/FV/MSVM.fd`, `firmware/MAP/`, `firmware/PDB/`, `firmware/FwVersion/`, and + `firmware/BuildInfo.json` (the selected matrix flavor). +- `logs/`, preserving paths relative to `Build/` to prevent same-name logs overwriting each other. + Failed builds additionally retain the selected flavor's full build tree under `logs/failed-build/`. +- ADO builds: `package/Bin/` with `MSVM.fd`, `Dsdt.aml`, `BiosInterface.h`, PDBs (except GCC), + version JSON/blob, and the flavor manifest. This is staged data, not permission to publish a package. + +GitHub uploads the contents of `firmware/`, preserving the existing `FV/MSVM.fd`, `PDB/`, and `MAP/` +paths. Logs upload even when building or staging fails. Source-check logs have their own artifact. +ADO includes the tree and compatibility folders at the drop root. Failed ADO builds also retain the +historical `all/` copy of the full `Build/` tree. Shipping build failures retain the `input-...` drop; +the final historical shipping drop is only produced after its packaging prerequisites pass. +The legacy flattened log view is retained, with warnings on basename collisions; the shared `logs/` +tree always retains every original path. +OneBranch owns uploads; the shared Python does not handle credentials or network publication. + +GitHub releases run only for pushes to `main`, after all builds and source checks succeed. They download +exactly the expected ten firmware artifacts, validate flavor, commit, source origin, version and clean/official +flags, then create the historical `firmware-*.tar.gz` assets. Release tags remain `v..`; +the prefix comes from `FirmwareVersion.toml`, and the serialized release job allocates the next patch. +**OpenVMM is temporarily omitted from the release gate**, as agreed while restoring CI. Restore it before +claiming runtime validation of releases. PRs and manual GitHub runs do not create releases. + +ADO indexes shipping PDBs in the build workspace while sources and submodules are present. The downstream +Windows packaging jobs download the staged outputs, validate their provenance, and pack NuGet artifacts. +PRs exercise packaging with `.PR.` / `.Test` IDs but cannot publish to feeds, VPack, or the symbol server. +Official publication additionally requires `refs/heads/official/*` and a non-PR run. OneBranch publishing +enable flags are compile-time booleans, never runtime matrix shipping variables. NuGet goes to the existing +`OS/HyperVUefi` feed; OneBranch retains VPack version allocation; symbols are uploaded without reindexing. +This change does not claim hosted Official validation. ### Closed Package Identity @@ -177,14 +218,18 @@ VS2022 legacy and AARCH64 RELEASE Windows-org CLANGPDB legacy. These three keep all other closed variants retain their historical test suffixes. Metadata describes eligibility, not permission to publish, and does not mark a local or PR build official. -Build IDs are not package names. Future packaging must preserve the shared VPack/NuGet ID convention: +Build IDs are not package names. The shared VPack/NuGet ID convention is: `.mscoreuefi..<.suffix if present><.Test if PR or non-shipping>`. For example, the official ARM64 shipping ID remains `hyperv.uefi.mscoreuefi.AARCH64.RELEASE`, while its PR ID is `hyperv.uefi.mscoreuefi.PR.AARCH64.RELEASE.Test`. NuGet versions and OneBranch VPack version allocation remain separate contracts. -No package creation or publication is enabled here. In particular, runtime build matrices do not establish -that per-flavor OneBranch publishing policy can be expanded at runtime; validate or introduce a separate -compile-time publishing stage before enabling it. +Only shipping flavors are packaged now. Non-shipping Windows/Linux builds retain their historical suffix +metadata but produce CI artifacts/logs only, not `.Test` packages or symbol-server uploads. +`matrix.py --shipping-only` selects the three shipping rows; it does not authorize publishing. +NuGet versions retain `.-.`, while VPack uses the same TOML prefix and +OneBranch's allocated version. Neither changes the already-compiled firmware's embedded build identity. +NuGet preserves the old recursive paths: `Msvm/_/FV/MSVM.fd`, +`symbols/Msvm/_/PDB/`, and `map/Msvm/_/MAP/`. ## Firmware Versioning @@ -222,8 +267,8 @@ Omitting it records `unknown`, rather than guessing from remotes or inheriting a For direct Stuart invocations, set `SOURCE_ORIGIN=open|closed|unknown` as a build or shell variable. An explicit Stuart value wins over the shell; an unrecognized value is rejected before output is written. -Origin is currently **sidecar metadata, not embedded in MSVM.fd**. Keep the JSON with future build artifacts; -artifact upload is not wired yet. Identifying origin from the firmware file alone needs a separately reviewed +Origin is currently **sidecar metadata, not embedded in MSVM.fd**. Staging and packaging retain the JSON. +Identifying origin from the firmware file alone needs a separately reviewed binary-record extension. Origin and the official flag are declarations, not security attestations. ### Overrides and CI Policy diff --git a/ci/scripts/matrix.py b/ci/scripts/matrix.py index d334d9402d..3551c67672 100644 --- a/ci/scripts/matrix.py +++ b/ci/scripts/matrix.py @@ -55,6 +55,8 @@ class SelectedBuild(BuildFlavor): display_name: NotRequired[str] artifact_name: NotRequired[str] logs_artifact_name: NotRequired[str] + build_id: NotRequired[str] + build_artifact_name: NotRequired[str] shipping: NotRequired[str] package_suffix: NotRequired[str] @@ -65,6 +67,7 @@ class MatrixArguments(argparse.Namespace): matrix: Path host: Host | None format: OutputFormat + shipping_only: bool FLAVOR_CHOICES: dict[str, tuple[str, ...]] = { "arch": ("X64", "AARCH64"), @@ -101,6 +104,15 @@ def label_builds(rows: list[SelectedBuild], backend: OutputFormat) -> list[Selec core = "Legacy" if row["core"] == "legacy" else "Patina" label = f"{row['target']} {row['arch']} {row['tool_chain']} ({core})" if backend == "ado": + result["build_id"] = row["id"] + if "package_suffix" in row: + suffix = f"_{row['package_suffix']}" if row["package_suffix"] else "" + result["artifact_name"] = f"drop_Build_{row['target']}_{row['arch']}_{row['tool_chain']}{suffix}" + else: + result["artifact_name"] = f"firmware-{row['id']}" + result["build_artifact_name"] = ( + f"input-{row['id']}" if row.get("shipping") == "true" else result["artifact_name"] + ) compiler = {"visual_studio": "VS", "windows_org": "WinOrg", "distribution": "Distro"}[row["compiler_source"]] label += f" / {row['host'].title()} {compiler}" if row["legacy_debugger"] == "1": @@ -210,9 +222,14 @@ def main(argv: list[str] | None = None) -> None: parser.add_argument("--matrix", required=True, type=Path, help="Path to the build coverage JSON file") parser.add_argument("--host", choices=EXECUTION_CHOICES["host"]) parser.add_argument("--format", choices=("json", "github", "ado"), default="json") + parser.add_argument("--shipping-only", action="store_true", help="Select only explicitly shipping package flavors") args = parser.parse_args(argv, namespace=MatrixArguments()) try: rows = select_builds(args.matrix, host=args.host) + if args.shipping_only: + rows = [row for row in rows if row.get("shipping") == "true"] + if not rows: + raise ValueError("No shipping builds selected") if args.format == "github" and any(row["legacy_debugger"] == "1" for row in rows): raise ValueError("GitHub builds cannot enable the closed-source legacy debugger") rows = label_builds(rows, args.format) diff --git a/ci/scripts/package_artifacts.py b/ci/scripts/package_artifacts.py new file mode 100644 index 0000000000..a4d31144a5 --- /dev/null +++ b/ci/scripts/package_artifacts.py @@ -0,0 +1,167 @@ +"""Validate staged firmware and prepare archives or a NuGet manifest, without publishing.""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import tarfile +import tomllib +import xml.etree.ElementTree as ET +from pathlib import Path + +from ci.scripts.matrix import SelectedBuild, label_builds, select_builds +from ci.scripts.stage_artifacts import ado_layout, select_flavor + + +def base_version() -> str: + with Path("MsvmPkg/FirmwareVersion.toml").open("rb") as stream: + version = str(tomllib.load(stream)["release"]["version"]) + if not re.fullmatch(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)", version): + raise ValueError("Release version must be major.minor") + return version + + +def validate_firmware(firmware: Path, flavor: SelectedBuild, commit: str, origin: str, official: bool) -> None: + if not re.fullmatch(r"[0-9a-fA-F]{40}", commit): + raise ValueError("Expected a full Git commit SHA") + info = json.loads((firmware / "BuildInfo.json").read_text(encoding="utf-8")) + if info != flavor: + raise ValueError(f"Artifact flavor does not match the selected matrix row: {firmware}") + metadata = json.loads((firmware / "FwVersion/FirmwareVersion.json").read_text(encoding="utf-8")) + if (metadata.get("git_commit") != commit or metadata.get("source_origin") != origin + or metadata.get("base_version") != base_version() + or type(metadata.get("flags")) is not int + or bool(metadata["flags"] & 2) != official or metadata["flags"] & 1): + raise ValueError(f"Artifact provenance/version does not match this build: {firmware}") + required = [firmware / "FV/MSVM.fd", firmware / "FwVersion/FwVersionBlob.bin"] + for file in required: + if not file.is_file() or file.stat().st_size == 0: + raise ValueError(f"Required artifact is missing or empty: {file}") + if not list((firmware / "MAP").glob("*.map")): + raise ValueError(f"Missing MAP artifacts: {firmware}") + if flavor["tool_chain"] != "GCC" and not list((firmware / "PDB").glob("*.pdb")): + raise ValueError(f"Missing PDB artifacts: {firmware}") + + +def release_archives(matrix: Path, source: Path, output: Path, commit: str) -> None: + rows = select_builds(matrix) + labeled = label_builds(rows, "github") + expected = {row["artifact_name"] for row in labeled} + if {path.name for path in source.iterdir()} != expected: + raise ValueError("Downloaded firmware artifacts do not exactly match the release matrix") + for row, label in zip(rows, labeled): + validate_firmware(source / label["artifact_name"], row, commit, "open", True) + output.mkdir(parents=True, exist_ok=False) + for row in labeled: + name = row["artifact_name"] + with tarfile.open(output / f"{name}.tar.gz", "w:gz") as archive: + for child in sorted((source / name).iterdir()): + archive.add(child, arcname=child.name) + + +def nuget_manifest( + matrix: Path, identifier: str, source: Path, output: Path, + commit: str, repository: str, branch: str, official: bool, + staging_output: Path | None = None, +) -> None: + flavor = select_flavor(matrix, identifier) + if flavor.get("shipping") != "true" or flavor["host"] != "windows": + raise ValueError("Only shipping Windows flavors may be packaged") + if not re.fullmatch(r"[A-Za-z0-9_.-]+", repository): + raise ValueError("Invalid package repository name") + if not re.fullmatch(r"[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*", branch): + raise ValueError("Branch name is not a valid NuGet prerelease identifier") + firmware = source / "firmware" + validate_firmware(firmware, flavor, commit, "closed", official) + package = source / "package/Bin" + for name in ("MSVM.fd", "Dsdt.aml", "BiosInterface.h", "FirmwareVersion.json", "FwVersionBlob.bin", "BuildInfo.json"): + if not (package / name).is_file() or (package / name).stat().st_size == 0: + raise ValueError(f"Missing package payload: {package / name}") + for source_file, package_file in ( + (firmware / "FV/MSVM.fd", package / "MSVM.fd"), + (firmware / "FwVersion/FirmwareVersion.json", package / "FirmwareVersion.json"), + (firmware / "FwVersion/FwVersionBlob.bin", package / "FwVersionBlob.bin"), + (firmware / "BuildInfo.json", package / "BuildInfo.json"), + ): + if source_file.read_bytes() != package_file.read_bytes(): + raise ValueError(f"Package payload differs from firmware artifact: {package_file}") + if staging_output is not None: + if staging_output.resolve().is_relative_to(source.resolve()) or source.resolve().is_relative_to(staging_output.resolve()): + raise ValueError("Publication staging must not overlap downloaded artifacts") + for directory in ("firmware", "logs", "package"): + shutil.copytree(source / directory, staging_output / directory) + ado_layout(staging_output, flavor) + firmware = staging_output / "firmware" + prefix = "" if official else "PR." + suffix = "" if official else ".Test" + identity = f"{repository}.mscoreuefi.{prefix}{flavor['arch']}.{flavor['target']}{suffix}" + root = ET.Element("package") + metadata = ET.SubElement(root, "metadata") + for key, value in { + "id": identity, "version": f"{base_version()}-{branch}.{commit}", + "authors": "Microsoft", "description": "Hyper-V UEFI firmware binaries.", + "owners": "Microsoft", + "projectUrl": "https://microsoft.visualstudio.com/DefaultCollection/OS/_git/hyperv.uefi/", + "copyright": "Copyright (C) Microsoft. All rights reserved.", + "tags": "Microsoft,Hyper-V,UEFI", + }.items(): + ET.SubElement(metadata, key).text = value + files = ET.SubElement(root, "files") + legacy_path = f"Msvm{flavor['arch']}/{flavor['target']}_{flavor['tool_chain']}" + for path, target in ( + ("FV/MSVM.fd", f"{legacy_path}/FV"), ("PDB/*.pdb", f"symbols/{legacy_path}/PDB"), + ("MAP/*.map", f"map/{legacy_path}/MAP"), + ("FwVersion/*", "FwVersion"), ("BuildInfo.json", ""), + ): + ET.SubElement(files, "file", src=str(firmware.resolve() / path), target=target) + output.parent.mkdir(parents=True, exist_ok=True) + ET.indent(root) + ET.ElementTree(root).write(output, encoding="utf-8", xml_declaration=True) + + +class Arguments(argparse.Namespace): + command: str + matrix: Path + source: Path + output: Path + commit: str + build_id: str + repository: str + branch: str + official: bool + staging_output: Path | None + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("version") + for command in ("github", "nuget"): + child = subparsers.add_parser(command) + child.add_argument("--matrix", required=True, type=Path) + child.add_argument("--source", required=True, type=Path) + child.add_argument("--output", required=True, type=Path) + child.add_argument("--commit", required=True) + if command == "nuget": + child.add_argument("--build-id", required=True) + child.add_argument("--repository", required=True) + child.add_argument("--branch", required=True) + child.add_argument("--official", action="store_true") + child.add_argument("--staging-output", type=Path, help="Stage a compatible OneBranch publication drop") + args = parser.parse_args(argv, namespace=Arguments()) + try: + if args.command == "version": + print(base_version()) + elif args.command == "github": + release_archives(args.matrix, args.source, args.output, args.commit) + else: + nuget_manifest(args.matrix, args.build_id, args.source, args.output, + args.commit, args.repository, args.branch, args.official, args.staging_output) + except (OSError, ValueError, KeyError) as error: + parser.error(str(error)) + + +if __name__ == "__main__": + main() diff --git a/ci/scripts/stage_artifacts.py b/ci/scripts/stage_artifacts.py new file mode 100644 index 0000000000..b5e677b189 --- /dev/null +++ b/ci/scripts/stage_artifacts.py @@ -0,0 +1,181 @@ +"""Stage portable firmware outputs and diagnostics; never upload or publish.""" + +from __future__ import annotations + +import argparse +import fnmatch +import json +import shutil +from pathlib import Path + +from ci.scripts.matrix import SelectedBuild, select_builds + +LOG_PATTERNS = ( + "buildlog*.txt", "buildlog*.md", "build_report.txt", "ci*.txt", "ci*.md", + "setup*.txt", "update*.txt", "prevallog.txt", "testsuites.xml", + "build_tools_report.*", "overridelog.txt", "basetools_build*.*", "fd_report.html", + "ci_buildlog.*", "setuplog.*", "update_log.*", +) + + +def select_flavor(matrix: Path, identifier: str) -> SelectedBuild: + for row in select_builds(matrix): + if row["id"] == identifier: + return row + raise ValueError(f"Build ID not present in {matrix}: {identifier}") + + +def copy_file(source: Path, destination: Path) -> None: + if not source.is_file() or source.stat().st_size == 0: + raise ValueError(f"Required artifact is missing or empty: {source}") + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + + +def copy_outputs(source: Path, destination: Path, suffix: str) -> None: + files = sorted(source.glob(f"*{suffix}")) + if not files: + raise ValueError(f"No {suffix} artifacts found in {source}") + for file in files: + copy_file(file, destination / file.name) + + +def stage_logs(workspace: Path, output: Path) -> None: + build = workspace / "Build" + destination = output / "logs" + destination.mkdir(parents=True, exist_ok=True) + count = 0 + for file in sorted(build.rglob("*")): + if file.is_file() and any(fnmatch.fnmatchcase(file.name.lower(), pattern) for pattern in LOG_PATTERNS): + target = destination / file.relative_to(build) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(file, target) + count += 1 + if not count: + print(f"WARNING: no build logs found under {build}", flush=True) + + +def ado_logs(output: Path, flavor: SelectedBuild) -> None: + name = f"{flavor['host'].title()} {flavor['arch']} {flavor['target']} {flavor['tool_chain']}" + if flavor["compiler_source"] == "windows_org": + name += " (WinOrg)" + destination = output / f"Build Logs {name}" + destination.mkdir(parents=True, exist_ok=True) + for file in sorted((output / "logs").rglob("*")): + if file.is_file(): + target = destination / file.name + if target.exists() and target.read_bytes() != file.read_bytes(): + print(f"WARNING: legacy log basename collision: {file.name}; full paths retained under logs/", flush=True) + shutil.copy2(file, target) + + +def source_check_logs(workspace: Path, output: Path, ado: bool) -> None: + stage_logs(workspace, output) + package = workspace / "Build/MsvmPkg" + if package.is_dir(): + shutil.copytree(package, output / "logs/MsvmPkg", dirs_exist_ok=True) + if ado: + shutil.copytree(package, output / "MsvmPkg", dirs_exist_ok=True) + if ado: + for file in (workspace / "Build").glob("*"): + if file.is_file() and any(fnmatch.fnmatchcase(file.name.lower(), pattern) for pattern in + ("ci_buildlog.*", "setuplog.*", "update_log.*", "testsuites.xml")): + shutil.copy2(file, output / file.name) + + +def ado_layout(output: Path, flavor: SelectedBuild) -> None: + """Keep historical drop paths alongside the provider-independent payload.""" + ado_logs(output, flavor) + suffix = f"{flavor['target']}_{flavor['arch']}" + copy_file(output / "firmware/FV/MSVM.fd", output / f"Firmware Binary File {suffix}/MSVM.fd") + shutil.copytree(output / "firmware/MAP", output / f"Firmware Map Files {suffix}") + if flavor["tool_chain"] != "GCC": + shutil.copytree(output / "firmware/PDB", output / f"Firmware PDB Files {suffix}") + build = Path("Build") / f"Msvm{flavor['arch']}" / f"{flavor['target']}_{flavor['tool_chain']}" + shutil.copytree(output / "package/Bin", output / build / "Bin") + + +def stage_build( + workspace: Path, output: Path, flavor: SelectedBuild, succeeded: bool, + ado: bool = False, +) -> None: + stage_logs(workspace, output) + build = workspace / "Build" / f"Msvm{flavor['arch']}" / f"{flavor['target']}_{flavor['tool_chain']}" + if not succeeded: + if ado: + ado_logs(output, flavor) + if (workspace / "Build").is_dir(): + shutil.copytree(workspace / "Build", output / "all", dirs_exist_ok=True) + else: + print(f"WARNING: build failed before Build directory was created: {workspace}", flush=True) + elif build.is_dir(): + shutil.copytree(build, output / "logs" / "failed-build") + else: + print(f"WARNING: build failed before output directory was created: {build}", flush=True) + return + + firmware = output / "firmware" + copy_file(build / "FV" / "MSVM.fd", firmware / "FV" / "MSVM.fd") + for name in ("FirmwareVersion.json", "FwVersionBlob.bin"): + copy_file(build / "FwVersion" / name, firmware / "FwVersion" / name) + copy_outputs(build / "MAP", firmware / "MAP", ".map") + if flavor["tool_chain"] != "GCC": + copy_outputs(build / "PDB", firmware / "PDB", ".pdb") + (firmware / "BuildInfo.json").write_text(json.dumps(flavor, indent=2) + "\n", encoding="utf-8") + + if flavor.get("shipping") == "true" or ado: + package = output / "package" / "Bin" + copy_file(build / "FV" / "MSVM.fd", package / "MSVM.fd") + copy_file(build / flavor["arch"] / "MsvmPkg/AcpiTables/AcpiTables/OUTPUT/Dsdt.aml", package / "Dsdt.aml") + copy_file(workspace / "MsvmPkg/Include/BiosInterface.h", package / "BiosInterface.h") + for name in ("FirmwareVersion.json", "FwVersionBlob.bin"): + copy_file(build / "FwVersion" / name, package / name) + copy_file(firmware / "BuildInfo.json", package / "BuildInfo.json") + if flavor["tool_chain"] != "GCC": + copy_outputs(build / "PDB", package / "PDB", ".pdb") + if ado: + ado_layout(output, flavor) + + +class Arguments(argparse.Namespace): + output: Path + matrix: Path | None + build_id: str | None + build_succeeded: bool + logs_only: bool + ado_layout: bool + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--matrix", type=Path) + parser.add_argument("--build-id") + parser.add_argument("--build-succeeded", action="store_true") + parser.add_argument("--logs-only", action="store_true") + parser.add_argument("--ado-layout", action="store_true", help="Also preserve historical OneBranch drop folders") + args = parser.parse_args(argv, namespace=Arguments()) + workspace = Path.cwd().resolve() + try: + output = args.output.resolve() + if output.is_relative_to(workspace / "Build") or workspace.is_relative_to(output): + raise ValueError("Artifact destination must be outside Build and must not contain the workspace") + if args.ado_layout: + if args.build_succeeded and any((output / name).exists() for name in ("firmware", "logs", "package", "all")): + raise ValueError(f"Artifact destination already contains staged output: {output}") + elif output.exists() and any(output.iterdir()): + raise ValueError(f"Artifact destination must be empty: {output}") + if args.logs_only: + if args.matrix or args.build_id or args.build_succeeded: + raise ValueError("--logs-only cannot be combined with build arguments") + source_check_logs(workspace, output, args.ado_layout) + else: + if args.matrix is None or args.build_id is None: + raise ValueError("Build staging requires --matrix and --build-id") + stage_build(workspace, output, select_flavor(args.matrix, args.build_id), args.build_succeeded, args.ado_layout) + except (OSError, ValueError) as error: + parser.error(str(error)) + + +if __name__ == "__main__": + main() diff --git a/ci/tests/test_artifacts.py b/ci/tests/test_artifacts.py new file mode 100644 index 0000000000..bbef615b7e --- /dev/null +++ b/ci/tests/test_artifacts.py @@ -0,0 +1,338 @@ +"""Exercise real staging/archive payloads and provider publication gates.""" + +from __future__ import annotations + +import io +import json +import os +import shutil +import subprocess +import tarfile +import unittest +import xml.etree.ElementTree as ET +import zipfile +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from ci.scripts import matrix, package_artifacts, stage_artifacts +from ci.tests.test_ado_poc import load_yaml + +ROOT = Path(__file__).resolve().parents[2] +OPEN = ROOT / ".github/build-matrix.json" +CLOSED = ROOT / ".azuredevops/build-matrix.json" +COMMIT = "a" * 40 + + +def fixture(workspace: Path, row: matrix.SelectedBuild, origin: str, official: bool) -> Path: + build = workspace / "Build" / f"Msvm{row['arch']}" / f"{row['target']}_{row['tool_chain']}" + files = { + "FV/MSVM.fd": b"firmware", "MAP/Module.map": b"map", + "PDB/Module.pdb": b"symbols", "FwVersion/FwVersionBlob.bin": b"\0" * 80, + f"{row['arch']}/MsvmPkg/AcpiTables/AcpiTables/OUTPUT/Dsdt.aml": b"aml", + "FwVersion/FirmwareVersion.json": json.dumps({ + "schema_version": 1, "struct_version": 1, "base_version": package_artifacts.base_version(), + "git_commit": COMMIT, "flags": 2 if official else 0, "source_origin": origin, + }).encode(), + "BUILD_REPORT.TXT": b"report", "Nested/BUILD_REPORT.TXT": b"nested report", + } + for name, contents in files.items(): + path = build / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(contents) + header = workspace / "MsvmPkg/Include/BiosInterface.h" + header.parent.mkdir(parents=True, exist_ok=True) + header.write_text("header") + (workspace / "Build/SETUPLOG.txt").write_text("setup") + return build + + +class ArtifactTests(unittest.TestCase): + def test_ado_preserves_legacy_folders_for_all_15_flavors(self) -> None: + for row in matrix.select_builds(CLOSED): + with self.subTest(flavor=row["id"]), TemporaryDirectory() as temporary, redirect_stdout(io.StringIO()): + root = Path(temporary) + workspace, output = root / "workspace", root / "output" + fixture(workspace, row, "closed", False) + stage_artifacts.stage_build(workspace, output, row, True, ado=True) + suffix = f"{row['target']}_{row['arch']}" + self.assertEqual((output / f"Firmware Binary File {suffix}/MSVM.fd").read_bytes(), b"firmware") + self.assertTrue((output / f"Firmware Map Files {suffix}/Module.map").is_file()) + self.assertEqual((output / f"Firmware PDB Files {suffix}/Module.pdb").exists(), row["tool_chain"] != "GCC") + label = f"{row['host'].title()} {row['arch']} {row['target']} {row['tool_chain']}" + if row["compiler_source"] == "windows_org": + label += " (WinOrg)" + self.assertTrue((output / f"Build Logs {label}/BUILD_REPORT.TXT").is_file()) + self.assertEqual(len(list((output / "logs").rglob("BUILD_REPORT.TXT"))), 2) + legacy_bin = output / "Build" / f"Msvm{row['arch']}" / f"{row['target']}_{row['tool_chain']}" / "Bin" + for name in ("MSVM.fd", "Dsdt.aml", "BiosInterface.h", "FirmwareVersion.json", "FwVersionBlob.bin"): + self.assertTrue((legacy_bin / name).is_file(), name) + + def test_ado_failure_adds_legacy_all_tree_after_partial_staging(self) -> None: + with TemporaryDirectory() as temporary, redirect_stdout(io.StringIO()): + root = Path(temporary) + row = matrix.select_builds(CLOSED)[0] + build = fixture(root / "workspace", row, "closed", False) + (build / "MAP/Module.map").unlink() + arguments = ["--matrix", str(CLOSED), "--build-id", row["id"], "--ado-layout", + "--output", str(root / "output")] + with patch.object(Path, "cwd", return_value=root / "workspace"), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + stage_artifacts.main([*arguments, "--build-succeeded"]) + stage_artifacts.main(arguments) + self.assertTrue((root / "output/all/MsvmX64/DEBUG_VS2022/FV/MSVM.fd").is_file()) + self.assertTrue((root / "output/Build Logs Windows X64 DEBUG VS2022/SETUPLOG.txt").is_file()) + + def test_source_check_compatibility_keeps_package_diagnostics(self) -> None: + with TemporaryDirectory() as temporary: + root = Path(temporary) + build = root / "workspace/Build" + (build / "MsvmPkg/Uncrustify").mkdir(parents=True) + for name in ("CI_BUILDLOG.md", "SETUPLOG.txt", "UPDATE_LOG.txt", "TestSuites.xml", + "MsvmPkg/Uncrustify/output.diff"): + (build / name).write_text(name) + stage_artifacts.source_check_logs(root / "workspace", root / "output", True) + for name in ("CI_BUILDLOG.md", "SETUPLOG.txt", "UPDATE_LOG.txt", "TestSuites.xml", + "MsvmPkg/Uncrustify/output.diff"): + self.assertEqual((root / "output" / name).read_text(), name) + self.assertEqual((root / "output/logs" / name).read_text(), name) + + def test_all_25_flavors_stage_expected_payloads(self) -> None: + for coverage, origin in ((OPEN, "open"), (CLOSED, "closed")): + for row in matrix.select_builds(coverage): + with self.subTest(flavor=row["id"]), TemporaryDirectory() as temporary: + workspace = Path(temporary) / "workspace" + build = fixture(workspace, row, origin, False) + if row["tool_chain"] == "GCC": + (build / "PDB/Module.pdb").unlink() + output = Path(temporary) / "output" + stage_artifacts.stage_build(workspace, output, row, True) + firmware = output / "firmware" + self.assertEqual((firmware / "FV/MSVM.fd").read_bytes(), b"firmware") + self.assertEqual((firmware / "MAP/Module.map").read_bytes(), b"map") + self.assertEqual((firmware / "PDB/Module.pdb").exists(), row["tool_chain"] != "GCC") + self.assertEqual((firmware / "FwVersion/FirmwareVersion.json").read_bytes(), + (build / "FwVersion/FirmwareVersion.json").read_bytes()) + self.assertEqual(json.loads((firmware / "BuildInfo.json").read_text()), row) + self.assertEqual(len(list((output / "logs").rglob("BUILD_REPORT.TXT"))), 2) + self.assertTrue((output / "logs/SETUPLOG.txt").is_file()) + package = output / "package/Bin" + self.assertEqual(package.exists(), row.get("shipping") == "true") + if package.exists(): + self.assertEqual({file.relative_to(package).as_posix() for file in package.rglob("*") if file.is_file()}, + {"MSVM.fd", "Dsdt.aml", "BiosInterface.h", "PDB/Module.pdb", + "FirmwareVersion.json", "FwVersionBlob.bin", "BuildInfo.json"}) + + def test_success_rejects_missing_outputs_but_retains_logs(self) -> None: + row = matrix.select_builds(CLOSED)[0] + for missing in ("FV/MSVM.fd", "MAP/Module.map", "PDB/Module.pdb", + "FwVersion/FirmwareVersion.json", "FwVersion/FwVersionBlob.bin", + "X64/MsvmPkg/AcpiTables/AcpiTables/OUTPUT/Dsdt.aml"): + with self.subTest(missing=missing), TemporaryDirectory() as temporary: + workspace = Path(temporary) / "workspace" + build = fixture(workspace, row, "closed", False) + (build / missing).unlink() + output = Path(temporary) / "output" + with self.assertRaises(ValueError): + stage_artifacts.stage_build(workspace, output, row, True) + self.assertTrue((output / "logs/SETUPLOG.txt").is_file()) + + def test_failure_preserves_build_tree_without_releasable_firmware(self) -> None: + row = matrix.select_builds(OPEN)[0] + with TemporaryDirectory() as temporary: + workspace = Path(temporary) / "workspace" + fixture(workspace, row, "open", False) + output = Path(temporary) / "output" + stage_artifacts.stage_build(workspace, output, row, False) + self.assertFalse((output / "firmware").exists()) + self.assertTrue((output / "logs/failed-build/FV/MSVM.fd").is_file()) + + def test_early_failure_reports_missing_logs_and_outputs(self) -> None: + with TemporaryDirectory() as temporary, patch("builtins.print") as log: + stage_artifacts.stage_build(Path(temporary), Path(temporary) / "out", matrix.select_builds(OPEN)[0], False) + self.assertEqual(log.call_count, 2) + + def test_cli_rejects_unsafe_or_reused_output_and_unknown_flavor(self) -> None: + with TemporaryDirectory() as temporary: + workspace = Path(temporary) + (workspace / "used").mkdir() + (workspace / "used/file").write_text("keep") + for destination in (workspace, workspace / "Build/artifacts", workspace / "used"): + with patch.object(Path, "cwd", return_value=workspace), redirect_stderr(io.StringIO()), \ + self.assertRaises(SystemExit): + stage_artifacts.main(["--logs-only", "--output", str(destination)]) + with self.assertRaisesRegex(ValueError, "Build ID"): + stage_artifacts.select_flavor(OPEN, "missing") + + def test_release_archives_preserve_names_layout_and_sidecars(self) -> None: + rows = matrix.select_builds(OPEN) + with TemporaryDirectory() as temporary: + root = Path(temporary) + for row, label in zip(rows, matrix.label_builds(rows, "github")): + workspace = root / row["id"] + fixture(workspace, row, "open", True) + stage = root / "staged" / row["id"] + stage_artifacts.stage_build(workspace, stage, row, True) + destination = root / "downloaded" / label["artifact_name"] + shutil.copytree(stage / "firmware", destination) + package_artifacts.release_archives(OPEN, root / "downloaded", root / "release", COMMIT) + self.assertEqual({file.name for file in (root / "release").iterdir()}, + {f"{row['artifact_name']}.tar.gz" for row in matrix.label_builds(rows, "github")}) + for file in (root / "release").iterdir(): + with tarfile.open(file) as archive: + self.assertEqual({item.name for item in archive if item.isfile()}, + {"FV/MSVM.fd", "PDB/Module.pdb", "MAP/Module.map", + "FwVersion/FirmwareVersion.json", "FwVersion/FwVersionBlob.bin", "BuildInfo.json"}) + shutil.rmtree(root / "downloaded" / matrix.label_builds(rows, "github")[0]["artifact_name"]) + with self.assertRaisesRegex(ValueError, "exactly match"): + package_artifacts.release_archives(OPEN, root / "downloaded", root / "incomplete", COMMIT) + self.assertFalse((root / "incomplete").exists()) + + def test_nuget_identity_and_payload_for_all_shipping_flavors(self) -> None: + for row in matrix.select_builds(CLOSED): + if row["shipping"] != "true": + continue + for official in (True, False): + with self.subTest(flavor=row["id"], official=official), TemporaryDirectory() as temporary: + root = Path(temporary) + fixture(root / "workspace", row, "closed", official) + stage_artifacts.stage_build(root / "workspace", root / "staged", row, True) + output = root / "package.nuspec" + package_artifacts.nuget_manifest(CLOSED, row["id"], root / "staged", output, + COMMIT, "hyperv.uefi", "main" if official else "merge", official) + manifest = ET.parse(output) + prefix, suffix = ("", "") if official else ("PR.", ".Test") + self.assertEqual(manifest.findtext("metadata/id"), + f"hyperv.uefi.mscoreuefi.{prefix}{row['arch']}.{row['target']}{suffix}") + legacy = f"Msvm{row['arch']}/{row['target']}_{row['tool_chain']}" + self.assertEqual({file.get("target") for file in manifest.findall("files/file")}, + {"", f"{legacy}/FV", f"symbols/{legacy}/PDB", f"map/{legacy}/MAP", "FwVersion"}) + self.assertTrue(all(Path(file.attrib["src"]).is_absolute() for file in manifest.findall("files/file"))) + + def test_rejects_nonshipping_or_wrong_provenance_packages(self) -> None: + with TemporaryDirectory() as temporary: + root = Path(temporary) + row = matrix.select_builds(CLOSED)[0] + fixture(root / "workspace", row, "closed", True) + stage_artifacts.stage_build(root / "workspace", root / "staged", row, True) + firmware = root / "staged/firmware" + metadata_path = firmware / "FwVersion/FirmwareVersion.json" + metadata = json.loads(metadata_path.read_text()) + for change in ({"git_commit": "b" * 40}, {"source_origin": "open"}, {"base_version": "99.0"}, + {"flags": 0}, {"flags": 3}): + with self.subTest(change=change): + metadata_path.write_text(json.dumps({**metadata, **change})) + with self.assertRaisesRegex(ValueError, "provenance"): + package_artifacts.validate_firmware(firmware, row, COMMIT, "closed", True) + metadata_path.write_text(json.dumps(metadata)) + for other in matrix.select_builds(CLOSED)[3:]: + with self.assertRaisesRegex(ValueError, "Only shipping"): + package_artifacts.nuget_manifest(CLOSED, other["id"], root / "staged", root / "bad.nuspec", + COMMIT, "hyperv.uefi", "main", True) + (root / "staged/package/Bin/MSVM.fd").write_bytes(b"wrong") + with self.assertRaisesRegex(ValueError, "differs"): + package_artifacts.nuget_manifest(CLOSED, row["id"], root / "staged", root / "bad.nuspec", + COMMIT, "hyperv.uefi", "main", True) + + @unittest.skipUnless(os.name == "nt" and shutil.which("nuget"), "Requires Windows NuGet CLI") + def test_real_nuget_pack_matches_legacy_payload_paths(self) -> None: + row = matrix.select_builds(CLOSED)[0] + nuget = shutil.which("nuget") + assert nuget is not None + with TemporaryDirectory() as temporary: + root = Path(temporary) + fixture(root / "workspace", row, "closed", True) + stage_artifacts.stage_build(root / "workspace", root / "staged", row, True) + manifest = root / "new.nuspec" + (root / "staged/build.manifest").write_text("Provider metadata must not be reused") + package_artifacts.nuget_manifest(CLOSED, row["id"], root / "staged", manifest, + COMMIT, "hyperv.uefi", "main", True, root / "publication") + self.assertFalse((root / "publication/build.manifest").exists()) + self.assertTrue((root / "publication/Firmware Binary File DEBUG_X64/MSVM.fd").is_file()) + self.assertTrue((root / "publication/Build/MsvmX64/DEBUG_VS2022/Bin/Dsdt.aml").is_file()) + legacy = ET.Element("package") + metadata = ET.SubElement(legacy, "metadata") + for key, value in {"id": "legacy", "version": "26.0", "authors": "Microsoft", "description": "Test"}.items(): + ET.SubElement(metadata, key).text = value + files = ET.SubElement(legacy, "files") + for src, target in (("FV/MSVM.fd", ""), ("PDB/*", "symbols"), ("MAP/*", "map")): + ET.SubElement(files, "file", src=str(root / "workspace/Build" / "**" / src), target=target) + legacy_path = root / "legacy.nuspec" + ET.ElementTree(legacy).write(legacy_path, encoding="utf-8", xml_declaration=True) + payloads: list[set[str]] = [] + for name, spec in (("legacy", legacy_path), ("publication/packages", manifest)): + result = subprocess.run([nuget, "pack", str(spec), "-OutputDirectory", str(root / name), + "-NoPackageAnalysis", "-NonInteractive"], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + with zipfile.ZipFile(next((root / name).glob("*.nupkg"))) as archive: + payloads.append({path for path in archive.namelist() + if path.endswith((".fd", ".map", ".pdb"))}) + self.assertEqual(payloads[0], payloads[1]) + legacy_prefix = "MsvmX64/DEBUG_VS2022" + self.assertEqual(payloads[1], {f"{legacy_prefix}/FV/MSVM.fd", + f"symbols/{legacy_prefix}/PDB/Module.pdb", + f"map/{legacy_prefix}/MAP/Module.map"}) + + +class PublicationContractTests(unittest.TestCase): + def test_build_jobs_upload_but_cannot_publish_packages(self) -> None: + job = load_yaml(".azuredevops/templates/build.yml")["jobs"][1]["${{ each host in parameters.hosts }}"][0] + variables = job["variables"] + self.assertFalse(variables["ob_nugetPublishing_enabled"]) + self.assertFalse(variables["ob_createvpack_enabled"]) + self.assertTrue(variables["ob_pipelineartifacts_enabled"]) + self.assertEqual(variables["ob_artifactBaseName"], "$(build_artifact_name)") + for host in ("windows", "linux"): + key = "${{ if eq(host, '" + host + "') }}" + steps = next(branch[key] for branch in job["steps"] if key in branch) + self.assertEqual(steps[-1]["template"], "/.azuredevops/templates/stage-artifacts.yml@self") + windows = next(branch["${{ if eq(host, 'windows') }}"] for branch in job["steps"] + if "${{ if eq(host, 'windows') }}" in branch) + index = next(step for step in windows if step.get("displayName") == "Index Shipping Symbols") + self.assertEqual(index["condition"], "and(succeeded(), eq(variables['shipping'], 'true'))") + self.assertTrue(index["inputs"]["IndexSources"]) + self.assertFalse(index["inputs"]["PublishSymbols"]) + + def test_packages_wait_for_all_checks_and_official_branch(self) -> None: + stages = load_yaml(".azuredevops/templates/platform.yml")["extends"]["parameters"]["stages"] + packages = next(stage for stage in stages if stage["stage"] == "Packages") + self.assertEqual(packages["dependsOn"], ["SourceChecks", "Build"]) + gate = packages["${{ if eq(parameters.oneBranchBase, 'v2/Microsoft.Official.yml') }}"]["condition"] + self.assertIn("succeeded()", gate) + self.assertIn("'refs/heads/official/'", gate) + self.assertIn("ne(variables['Build.Reason'], 'PullRequest')", gate) + template = load_yaml(".azuredevops/templates/publish.yml") + self.assertFalse(template["parameters"][0]["default"]) + self.assertIn("--shipping-only", template["jobs"][0]["steps"][1]["pwsh"]) + job = template["jobs"][1] + self.assertEqual(job["pool"]["type"], "windows") + self.assertEqual(job["variables"]["ob_artifactBaseName"], "$(artifact_name)") + download = next(step for step in job["steps"] if step.get("task") == "DownloadPipelineArtifact@2") + self.assertEqual(download["inputs"]["artifactName"], "$(build_artifact_name)") + source = load_yaml(".azuredevops/templates/source-checks.yml")["jobs"][0] + self.assertEqual(source["variables"]["ob_artifactBaseName"], "drop_Build_SourceChecks") + for variable in ("ob_createvpack_enabled", "ob_nugetPublishing_enabled"): + self.assertEqual(job["variables"][variable], "${{ parameters.officialBuild }}") + symbol = job["steps"][-1]["${{ if eq(parameters.officialBuild, true) }}"][0] + self.assertTrue(symbol["inputs"]["PublishSymbols"]) + self.assertFalse(symbol["inputs"]["IndexSources"]) + self.assertNotIn("continueOnError", symbol) + + def test_github_release_and_failure_artifact_gates(self) -> None: + workflow = load_yaml(".github/workflows/platform-ci.yml") + release = workflow["jobs"]["release"] + self.assertEqual(release["if"], "github.event_name == 'push' && github.ref == 'refs/heads/main'") + self.assertEqual(release["needs"], ["source-checks", "build"]) + self.assertEqual(release["permissions"]["contents"], "write") + self.assertFalse(release["concurrency"]["cancel-in-progress"]) + stage = workflow["jobs"]["build"]["steps"][-1] + self.assertEqual(stage["if"], "always()") + self.assertEqual(stage["with"]["build-succeeded"], "${{ steps.firmware.outcome == 'success' }}") + upload = load_yaml(".github/actions/stage-artifacts/action.yml")["runs"]["steps"] + self.assertEqual(upload[1]["with"]["if-no-files-found"], "error") + self.assertEqual(upload[2]["if"], "always()") + + +if __name__ == "__main__": + unittest.main() diff --git a/ci/tests/test_matrix.py b/ci/tests/test_matrix.py index 359a11f92e..5e4eaeb0e7 100644 --- a/ci/tests/test_matrix.py +++ b/ci/tests/test_matrix.py @@ -132,9 +132,26 @@ def test_ado_labels_distinguish_compilers_without_renaming_packages(self) -> Non self.assertEqual(row["id"], original["id"]) self.assertEqual(row["package_suffix"], original["package_suffix"]) self.assertEqual(row["shipping"], original["shipping"]) - self.assertNotIn("artifact_name", row) + suffix = f"_{original['package_suffix']}" if original["package_suffix"] else "" + self.assertEqual(row["artifact_name"], + f"drop_Build_{original['target']}_{original['arch']}_{original['tool_chain']}{suffix}") + self.assertEqual(row["build_artifact_name"], + f"input-{original['id']}" if original["shipping"] == "true" else row["artifact_name"]) + self.assertEqual(row["build_id"], original["id"]) self.assertIn("/ Windows WinOrg", next(row["display_name"] for row in labeled if row["compiler_source"] == "windows_org")) + self.assertEqual(len({row["artifact_name"] for row in labeled}), 15) + self.assertEqual(len({row["build_artifact_name"] for row in labeled}), 15) + + def test_shipping_matrix_selects_only_three_windows_flavors(self) -> None: + output = io.StringIO() + with redirect_stdout(output): + matrix.main(["--matrix", str(CLOSED_MATRIX), "--shipping-only", "--format", "ado"]) + rows = json.loads(output.getvalue()) + self.assertEqual(len(rows), 3) + self.assertTrue(all(row["shipping"] == "true" and row["host"] == "windows" for row in rows.values())) + with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + matrix.main(["--matrix", str(OPEN_MATRIX), "--shipping-only"]) def test_invalid_matrix_is_rejected(self) -> None: row = matrix.parse_definition(json.loads(OPEN_MATRIX.read_text())[0]) From d423794caf60e6d8cc0d585d91411a1f6fdb1a38 Mon Sep 17 00:00:00 2001 From: maheeraeron Date: Tue, 20 Oct 2026 22:03:03 +0000 Subject: [PATCH 18/18] ci: shorten ADO matrix labels and explain dirty package failures Keep artifact IDs stable and publication gates strict. Report provenance mismatches precisely and log dirty paths when computing the firmware version flag. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../FirmwareVersionBlob.py | 7 +++++- MsvmPkg/Docs/BUILDING.md | 17 +++++++++++--- ci/scripts/matrix.py | 18 ++++++++++++++- ci/scripts/package_artifacts.py | 19 +++++++++++---- ci/tests/test_artifacts.py | 3 ++- ci/tests/test_firmware_version.py | 23 +++++++++++++++++++ ci/tests/test_matrix.py | 20 +++++++++++++++- 7 files changed, 95 insertions(+), 12 deletions(-) diff --git a/MsvmPkg/BuildPlugins/FirmwareVersionBlob/FirmwareVersionBlob.py b/MsvmPkg/BuildPlugins/FirmwareVersionBlob/FirmwareVersionBlob.py index 7c123bc246..4af9f455f7 100644 --- a/MsvmPkg/BuildPlugins/FirmwareVersionBlob/FirmwareVersionBlob.py +++ b/MsvmPkg/BuildPlugins/FirmwareVersionBlob/FirmwareVersionBlob.py @@ -37,6 +37,7 @@ from edk2toolext.environment import repo_resolver from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin +from git import Repo class FirmwareVersionMetadata(TypedDict): @@ -94,10 +95,14 @@ def _GetGitCommit(self, workspace): # untracked files; gitignored build outputs do not count). try: details = repo_resolver.repo_details(workspace) - return details["Head"]["HexSha"], bool(details["Dirty"]) except Exception as e: logging.warning(f"Could not determine git state for firmware version blob: {e}") return "unknown", False + if details["Dirty"]: + with Repo(workspace) as repo: + status = repo.git.status("--short", "--untracked-files=all", "--ignore-submodules=none") + logging.warning("Firmware version dirty-worktree flag is set. Git status (paths only):\n%s", status) + return details["Head"]["HexSha"], bool(details["Dirty"]) @staticmethod def _EncodeRecordString(value: str, capacity: int, name: str) -> bytes: diff --git a/MsvmPkg/Docs/BUILDING.md b/MsvmPkg/Docs/BUILDING.md index d6ac990082..3453ac9878 100644 --- a/MsvmPkg/Docs/BUILDING.md +++ b/MsvmPkg/Docs/BUILDING.md @@ -92,12 +92,14 @@ build 160110571; runtime debugging still requires validation. All open rows and all GCC rows disable the legacy debugger. ARM64 MSVC is unsupported. Disabled GCC combinations are not reintroduced. -IDs are derived as `______`; +Persistent IDs are derived as `______`; do not enter IDs in the JSON. Identical flavors may occur in both files, but may not occur twice within one file. Edit the respective JSON to change coverage independently, without editing shared Python or provider YAML. The selector rejects duplicate flavors and fails when the supplied file or requested host has no builds. -`--format github` emits an `include` object; `--format ado` emits a job-leg name to variables mapping. +`--format github` emits an `include` object; `--format ado` emits a short job-leg name to variables mapping. +ADO labels such as `DEBUG_X64_VS2022_Debugger` or `RELEASE_AARCH64_ClangWin` are independent +of the full `build_id` carried in the variables. Artifact names and package identities do not change. These commands only print JSON. They do not execute builds, install tools, or publish anything. GitHub's platform workflow supplies `.github/build-matrix.json`, feeds the result to its native `strategy.matrix`, and calls the build action for each flavor. That action invokes the same `ci/scripts/build.py` used locally. @@ -146,7 +148,7 @@ Provider matrix output adds a `display_name` derived from the same flavor: ADO job names cannot use the runtime matrix's `$(display_name)` in this OneBranch pipeline: it appeared literally in the hosted run. Instead, job display names use the compile-time `Firmware (windows)` or `Firmware (linux)` prefix; ADO appends -the unchanged matrix leg ID to distinguish each flavor. The generated +short matrix labels to distinguish each flavor while retaining the full `build_id` in the variables. The generated `display_name` remains available as metadata, not as the job's UI name. GitHub matrix output also supplies `artifact_name` and `logs_artifact_name` for the upload steps. @@ -210,6 +212,15 @@ Official publication additionally requires `refs/heads/official/*` and a non-PR enable flags are compile-time booleans, never runtime matrix shipping variables. NuGet goes to the existing `OS/HyperVUefi` feed; OneBranch retains VPack version allocation; symbols are uploaded without reindexing. This change does not claim hosted Official validation. +The governed VPack task supports `push`, `pull`, and `list`, not a local pack-only operation. +PRs validate/stage the VPack payload and pack NuGet without external publication; VPack creation/push +is Official-only. Historical pipelines packed non-shipping `.Test` flavors as well; the current three-flavor +package scope is an intentional change, separate from retaining CI artifacts for all 15 builds. + +PR run 160943879 passed all firmware builds but package validation rejected the dirty-worktree flag. +Validation reports the exact mismatched fields, and the version plugin logs dirty paths when computing +the flag. The hosted cause still requires another run; the dirty guard remains enforced for both PR +and Official packaging rather than silently clearing or ignoring provenance. ### Closed Package Identity diff --git a/ci/scripts/matrix.py b/ci/scripts/matrix.py index 3551c67672..cfdead1fe2 100644 --- a/ci/scripts/matrix.py +++ b/ci/scripts/matrix.py @@ -88,6 +88,20 @@ def build_id(row: BuildFlavor) -> str: row["compiler_source"], row["core"], row["legacy_debugger"])) +def ado_job_name(row: BuildFlavor) -> str: + """Use a short matrix leg label without changing the persistent build ID.""" + compiler: str = row["tool_chain"] + if compiler == "CLANGPDB": + compiler = {"visual_studio": "ClangVS", "windows_org": "ClangWin", + "distribution": "ClangLinux"}[row["compiler_source"]] + name = f"{row['target']}_{row['arch']}_{compiler}" + if row["core"] == "patina": + name += "_Patina" + if row["legacy_debugger"] == "1": + name += "_Debugger" + return name + + def label_builds(rows: list[SelectedBuild], backend: OutputFormat) -> list[SelectedBuild]: """Add provider presentation fields without changing execution/package IDs. @@ -233,13 +247,15 @@ def main(argv: list[str] | None = None) -> None: if args.format == "github" and any(row["legacy_debugger"] == "1" for row in rows): raise ValueError("GitHub builds cannot enable the closed-source legacy debugger") rows = label_builds(rows, args.format) + if args.format == "ado" and len({ado_job_name(row) for row in rows}) != len(rows): + raise ValueError("ADO job label collision; distinguish the new flavors before adding them") except (OSError, ValueError) as error: parser.error(str(error)) result: object if args.format == "github": result = {"include": rows} elif args.format == "ado": - result = {row["id"]: {key: value for key, value in cast(Mapping[str, str], row).items() if key != "id"} + result = {ado_job_name(row): {key: value for key, value in cast(Mapping[str, str], row).items() if key != "id"} for row in rows} else: result = rows diff --git a/ci/scripts/package_artifacts.py b/ci/scripts/package_artifacts.py index a4d31144a5..fda9cc70aa 100644 --- a/ci/scripts/package_artifacts.py +++ b/ci/scripts/package_artifacts.py @@ -30,11 +30,20 @@ def validate_firmware(firmware: Path, flavor: SelectedBuild, commit: str, origin if info != flavor: raise ValueError(f"Artifact flavor does not match the selected matrix row: {firmware}") metadata = json.loads((firmware / "FwVersion/FirmwareVersion.json").read_text(encoding="utf-8")) - if (metadata.get("git_commit") != commit or metadata.get("source_origin") != origin - or metadata.get("base_version") != base_version() - or type(metadata.get("flags")) is not int - or bool(metadata["flags"] & 2) != official or metadata["flags"] & 1): - raise ValueError(f"Artifact provenance/version does not match this build: {firmware}") + mismatches = [] + for field, expected in (("git_commit", commit), ("source_origin", origin), ("base_version", base_version())): + if metadata.get(field) != expected: + mismatches.append(f"{field}: expected {expected!r}, got {metadata.get(field)!r}") + flags = metadata.get("flags") + if type(flags) is not int: + mismatches.append(f"flags must be an integer, got {flags!r}") + else: + if bool(flags & 2) != official: + mismatches.append(f"official flag: expected {official}, got {bool(flags & 2)}") + if flags & 1: + mismatches.append("dirty-worktree flag is set; inspect the firmware build's Git status diagnostics") + if mismatches: + raise ValueError(f"Artifact provenance/version does not match this build: {firmware}: {'; '.join(mismatches)}") required = [firmware / "FV/MSVM.fd", firmware / "FwVersion/FwVersionBlob.bin"] for file in required: if not file.is_file() or file.stat().st_size == 0: diff --git a/ci/tests/test_artifacts.py b/ci/tests/test_artifacts.py index bbef615b7e..ceabf70419 100644 --- a/ci/tests/test_artifacts.py +++ b/ci/tests/test_artifacts.py @@ -223,7 +223,8 @@ def test_rejects_nonshipping_or_wrong_provenance_packages(self) -> None: {"flags": 0}, {"flags": 3}): with self.subTest(change=change): metadata_path.write_text(json.dumps({**metadata, **change})) - with self.assertRaisesRegex(ValueError, "provenance"): + expected = "dirty-worktree flag" if change == {"flags": 3} else "provenance" + with self.assertRaisesRegex(ValueError, expected): package_artifacts.validate_firmware(firmware, row, COMMIT, "closed", True) metadata_path.write_text(json.dumps(metadata)) for other in matrix.select_builds(CLOSED)[3:]: diff --git a/ci/tests/test_firmware_version.py b/ci/tests/test_firmware_version.py index 39b48296ed..a6c724bc79 100644 --- a/ci/tests/test_firmware_version.py +++ b/ci/tests/test_firmware_version.py @@ -34,6 +34,29 @@ def load_plugin() -> ModuleType: class FirmwareVersionTests(unittest.TestCase): """Build vars win over shell vars, including explicit empty/false values.""" + def test_dirty_version_reports_paths_without_changing_identity(self) -> None: + module = load_plugin() + with TemporaryDirectory() as temporary: + repo = importlib.import_module("git").Repo.init(temporary) + try: + path = Path(temporary) / "tracked.txt" + path.write_text("original") + repo.index.add(["tracked.txt"]) + actor = importlib.import_module("git").Actor("CI test", "ci@example.invalid") + commit = repo.index.commit("Test baseline", author=actor, committer=actor).hexsha + plugin = module.FirmwareVersionBlob() + self.assertEqual(plugin._GetGitCommit(temporary), (commit, False)) + path.write_text("modified") + (Path(temporary) / "generated.txt").write_text("not logged contents") + with self.assertLogs(level="WARNING") as logs: + self.assertEqual(plugin._GetGitCommit(temporary), (commit, True)) + message = "\n".join(logs.output) + self.assertIn("tracked.txt", message) + self.assertIn("generated.txt", message) + self.assertNotIn("not logged contents", message) + finally: + repo.close() + def test_source_origin_is_explicit_sidecar_metadata(self) -> None: """Origins honor Stuart precedence and leave the v1 binary ABI unchanged.""" module = load_plugin() diff --git a/ci/tests/test_matrix.py b/ci/tests/test_matrix.py index 5e4eaeb0e7..6a356c4760 100644 --- a/ci/tests/test_matrix.py +++ b/ci/tests/test_matrix.py @@ -98,7 +98,7 @@ def test_provider_formats_preserve_same_flavors(self) -> None: if format_name == "github": expected = {"include": matrix.label_builds(rows, "github")} elif format_name == "ado": - expected = {row["id"]: {key: value for key, value in row.items() if key != "id"} + expected = {matrix.ado_job_name(row): {key: value for key, value in row.items() if key != "id"} for row in matrix.label_builds(rows, "ado")} self.assertEqual(result, expected) @@ -117,6 +117,24 @@ def test_github_labels_preserve_historical_artifact_names(self) -> None: self.assertEqual(labeled[-1]["display_name"], "RELEASE AARCH64 CLANGPDB (Patina)") self.assertEqual(labeled[-1]["artifact_name"], "firmware-RELEASE-AARCH64-CLANGPDB-patina") + def test_ado_job_names_are_short_and_separate_from_artifact_identity(self) -> None: + rows = matrix.select_builds(CLOSED_MATRIX) + names = [matrix.ado_job_name(row) for row in rows] + self.assertEqual(len(set(names)), 15) + self.assertTrue(all(len(name) <= 35 and not name.endswith(("_0", "_1")) for name in names)) + self.assertEqual(names[0], "DEBUG_X64_VS2022_Debugger") + self.assertEqual(names[2], "RELEASE_AARCH64_ClangWin") + self.assertEqual(names[-1], "RELEASE_X64_GCC") + output = io.StringIO() + with redirect_stdout(output): + matrix.main(["--matrix", str(CLOSED_MATRIX), "--format", "ado"]) + emitted = json.loads(output.getvalue()) + for row in rows: + leg = emitted[matrix.ado_job_name(row)] + self.assertEqual(leg["build_id"], row["id"]) + self.assertEqual(leg["artifact_name"], + next(item["artifact_name"] for item in matrix.label_builds(rows, "ado") if item["id"] == row["id"])) + def test_github_artifact_collisions_require_explicit_migration(self) -> None: row = matrix.select_builds(OPEN_MATRIX)[0] other: matrix.SelectedBuild = {**row, "compiler_source": "windows_org"}