From 79a633523dfe23d940ff93320a829b51b50fb6d7 Mon Sep 17 00:00:00 2001 From: Jens Geyer Date: Fri, 2 Oct 2026 00:34:27 +0200 Subject: [PATCH] THRIFT-6400: Package the voted compiler in the .NET tool Client: build The .NET tool workflow built thrift.exe from the release tag and packed that build. A new build of the same source is not the same bytes, so Apache.Thrift.Compiler did not carry the thrift-.exe the release vote covered: the compiler in the 0.25.0 package has SHA-256 3d3f2e8a..., the voted one 9e1cb466.... THRIFT-6397 made the Windows installer package the voted executable, and the .NET tool now does the same. When the GitHub release is published, a new voted-compiler job runs the tests of get-voted-compiler.ps1 and then uses it to fetch thrift-.exe from dist/release, checking its checksums and its signature against KEYS. The package carries that file and the LICENSE and NOTICE of the release tag. If the fetch or a check fails, nothing is packed or published: a release does not fall back to building its own compiler. A manual run with the new compiler_url input packs the executable at that URL, for example a release candidate's, and never publishes. Pull requests, and manual runs without compiler_url, still build the compiler from source. test-dotnet-tool.ps1 gets -ExpectedSha256, which the workflow uses to check that the thrift.exe in the package is the file it packed. The run summary names the compiler's source and hash. The README shown on nuget.org, build/windows/README.md and doc/ReleaseManagement.md describe it. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/dotnet-tool.yml | 182 +++++++++++++++++- build/windows/README.md | 23 ++- build/windows/dotnet-tool/README.md | 9 +- .../windows/dotnet-tool/test-dotnet-tool.ps1 | 26 ++- doc/ReleaseManagement.md | 5 + 5 files changed, 226 insertions(+), 19 deletions(-) diff --git a/.github/workflows/dotnet-tool.yml b/.github/workflows/dotnet-tool.yml index e7189d86776..922d31d8f7c 100644 --- a/.github/workflows/dotnet-tool.yml +++ b/.github/workflows/dotnet-tool.yml @@ -19,8 +19,21 @@ # Publishes the Windows Thrift compiler to NuGet as a .NET tool, so that it can # be installed with "dotnet tool install --global Apache.Thrift.Compiler". # -# The compiler build mirrors the compiler-windows job in cmake.yml, which is -# what actually tests it on every push. +# The package of a release has to carry the thrift.exe the vote covered, not a +# new build of the same source: two builds are not the same bytes. So where +# there is such an executable, this workflow packs that one, as +# windows-packages.yml does for the installer: +# +# - for a GitHub release, the thrift-.exe in the release's +# directory on dist.apache.org; +# - for a manual run with compiler_url, the executable at that URL, such as +# the one a release candidate put on dist/dev. A manual run never +# publishes; it shows that the package works on a Windows runner. +# +# It is used only once its checksum and its signature against KEYS check out. +# Pull requests, and manual runs without compiler_url, build the compiler from +# source instead. That build mirrors the compiler-windows job in cmake.yml, +# which is what actually tests it on every push. name: ".NET tool" @@ -37,6 +50,11 @@ on: - "build/windows/check-compiler-imports.ps1" - "doc/ReleaseManagement.md" workflow_dispatch: + inputs: + compiler_url: + description: "A signed thrift-X.Y.Z.exe on dist.apache.org to pack, for example https://dist.apache.org/repos/dist/release/thrift/0.25.0/thrift-0.25.0.exe. Empty builds the compiler from the branch instead. A manual run never publishes." + required: false + type: string concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -46,8 +64,84 @@ permissions: contents: read jobs: + voted-compiler: + name: Voted compiler + if: ${{ github.event_name == 'release' || inputs.compiler_url != '' }} + # gpg is at hand on Linux. + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + outputs: + version: ${{ steps.fetch.outputs.version }} + url: ${{ steps.fetch.outputs.url }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # windows-packages.yml runs these on every pull request that touches + # them. Here they run before the download they guard, so that a release + # does not rely on checks that no longer work. + - name: Test the download checks + shell: pwsh + run: ./build/windows/get-voted-compiler-tests.ps1 + + - name: Download and check the compiler + id: fetch + shell: pwsh + env: + RELEASE_TAG: ${{ github.event.release.tag_name }} + INPUT_URL: ${{ inputs.compiler_url }} + run: | + $url = $env:INPUT_URL + if (-not $url) { + # A release is promoted to dist/release before its GitHub release + # is published, so the executable is already there. + $version = $env:RELEASE_TAG -replace '^v', '' + $url = "https://dist.apache.org/repos/dist/release/thrift/$version/thrift-$version.exe" + } + "url=$url" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + ./build/windows/get-voted-compiler.ps1 -Url $url -OutputDir voted + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + - name: Collect LICENSE and NOTICE + env: + VERSION: ${{ steps.fetch.outputs.version }} + run: | + # The package carries the LICENSE and NOTICE of the release it + # packs. A release has a tag to take them from. A release + # candidate has none yet, so its files come from the branch this + # runs on, which then has to be at that version. + if git fetch --quiet --depth=1 origin "refs/tags/v${VERSION}:refs/tags/v${VERSION}" 2>/dev/null; then + git show "v${VERSION}:LICENSE" > voted/LICENSE + git show "v${VERSION}:NOTICE" > voted/NOTICE + echo "LICENSE and NOTICE from tag v${VERSION}" + else + branch_version=$(sed -n 's/^ *set(thrift_VERSION "\([0-9.]*\)").*/\1/p' CMakeLists.txt) + if [ "$branch_version" != "$VERSION" ]; then + echo "::error::There is no tag v${VERSION}, and this branch is at ${branch_version}. Run this from the branch the release candidate was made from." + exit 1 + fi + cp LICENSE NOTICE voted/ + echo "LICENSE and NOTICE from ${GITHUB_REF_NAME}" + fi + + - name: Archive the compiler + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: voted-compiler + if-no-files-found: error + path: voted/ + retention-days: 3 + pack: name: Pack + needs: voted-compiler + # Runs when the voted compiler is there, and when there was none to fetch + # and voted-compiler was skipped. Not when fetching or checking it failed: + # a release must not fall back to building its own compiler. + if: ${{ !cancelled() && needs.voted-compiler.result != 'failure' }} runs-on: windows-2025 timeout-minutes: 60 permissions: @@ -64,6 +158,7 @@ jobs: shell: pwsh env: RELEASE_TAG: ${{ github.event.release.tag_name }} + VOTED_VERSION: ${{ needs.voted-compiler.outputs.version }} run: | $sourceVersion = '' foreach ($line in Get-Content CMakeLists.txt) { @@ -90,16 +185,36 @@ jobs: $version = $tagVersion } + if ($env:VOTED_VERSION) { + # On a release the voted executable was fetched for the tag's + # version. A manual run packs whatever release compiler_url names, + # which need not be the one this branch is at. + if ($env:RELEASE_TAG -and $env:VOTED_VERSION -ne $version) { + Write-Error "Version mismatch: voted compiler=$env:VOTED_VERSION, tag=$env:RELEASE_TAG" + exit 1 + } + $version = $env:VOTED_VERSION + } + Write-Host "Version: $version" "version=$version" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + - name: Download the voted compiler + if: ${{ needs.voted-compiler.result == 'success' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: voted-compiler + path: voted + - name: Install dependencies + if: ${{ needs.voted-compiler.result != 'success' }} shell: pwsh timeout-minutes: 10 # Pinned to the version build/docker/msvc/Dockerfile uses. run: choco install winflexbison3 --version=2.5.24.20210105 -y --no-progress - name: Build the compiler + if: ${{ needs.voted-compiler.result != 'success' }} shell: pwsh timeout-minutes: 30 run: | @@ -110,32 +225,83 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } cmake --install cmake_build --config Release --prefix cmake_install + - name: Choose the compiler to pack + id: compiler + shell: pwsh + env: + VERSION: ${{ steps.version.outputs.version }} + VOTED: ${{ needs.voted-compiler.result == 'success' }} + run: | + if ($env:VOTED -eq 'true') { + $compiler = "voted/thrift-$env:VERSION.exe" + # The LICENSE and NOTICE of that release, collected with it. + $sourceRoot = 'voted' + } + else { + $compiler = 'cmake_install/bin/thrift.exe' + $sourceRoot = '' + } + $sha256 = (Get-FileHash -Algorithm SHA256 -LiteralPath $compiler).Hash.ToLowerInvariant() + Write-Host "Packing $compiler, sha256 $sha256" + "path=$compiler" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + "source_root=$sourceRoot" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + "sha256=$sha256" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + - name: Check what the compiler imports shell: pwsh + env: + COMPILER: ${{ steps.compiler.outputs.path }} # A published NuGet version cannot be replaced, so this has to hold # before the package is built, not only in cmake.yml afterwards. - run: ./build/windows/check-compiler-imports.ps1 -Path cmake_install/bin/thrift.exe + run: ./build/windows/check-compiler-imports.ps1 -Path $env:COMPILER - name: Pack the tool shell: pwsh env: VERSION: ${{ steps.version.outputs.version }} + COMPILER: ${{ steps.compiler.outputs.path }} + SOURCE_ROOT: ${{ steps.compiler.outputs.source_root }} run: | - ./build/windows/build-dotnet-tool.ps1 ` - -Version $env:VERSION ` - -Compiler cmake_install/bin/thrift.exe ` - -OutputDir nupkg + $toolArgs = @{ Version = $env:VERSION; Compiler = $env:COMPILER; OutputDir = 'nupkg' } + if ($env:SOURCE_ROOT) { + $toolArgs['SourceRoot'] = $env:SOURCE_ROOT + } + ./build/windows/build-dotnet-tool.ps1 @toolArgs - name: Test the package shell: pwsh timeout-minutes: 15 env: VERSION: ${{ steps.version.outputs.version }} + COMPILER_SHA256: ${{ steps.compiler.outputs.sha256 }} run: | + # The hash shows that the package carries the very file it was + # given - for a release, the one the vote covered. ./build/windows/dotnet-tool/test-dotnet-tool.ps1 ` -Package "nupkg/Apache.Thrift.Compiler.$env:VERSION.nupkg" ` -Version $env:VERSION ` - -ExpectedCompilerOutput "Thrift version $env:VERSION" + -ExpectedCompilerOutput "Thrift version $env:VERSION" ` + -ExpectedSha256 $env:COMPILER_SHA256 + + - name: Report the compiler + shell: pwsh + env: + VERSION: ${{ steps.version.outputs.version }} + COMPILER_SHA256: ${{ steps.compiler.outputs.sha256 }} + COMPILER_URL: ${{ needs.voted-compiler.outputs.url }} + run: | + $package = "nupkg/Apache.Thrift.Compiler.$env:VERSION.nupkg" + $hash = (Get-FileHash -Algorithm SHA256 $package).Hash.ToLowerInvariant() + @( + '### .NET tool', + '', + "| | |", + "|---|---|", + "| package | ``$([System.IO.Path]::GetFileName($package))`` |", + "| package sha256 | ``$hash`` |", + "| compiler | $(if ($env:COMPILER_URL) { $env:COMPILER_URL } else { 'built from source' }) |", + "| compiler sha256 | ``$env:COMPILER_SHA256`` |" + ) | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append - name: Archive the package uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/build/windows/README.md b/build/windows/README.md index a4768d6d16e..a40fa523abc 100644 --- a/build/windows/README.md +++ b/build/windows/README.md @@ -78,11 +78,12 @@ are accepted. PS C:\thrift> .\build\windows\get-voted-compiler.ps1 -Url https://dist.apache.org/repos/dist/release/thrift/0.25.0/thrift-0.25.0.exe -OutputDir voted ``` -The installer of a release has to contain the executable the vote covered, and -a new build of the same source is not the same bytes. The -[`Windows packages`](../../.github/workflows/windows-packages.yml) workflow uses -this to get that executable. `-Path` and `-KeysPath` run the same checks on -files that are already local. It needs `gpg`. +The installer and the .NET tool of a release have to contain the executable the +vote covered, and a new build of the same source is not the same bytes. The +[`Windows packages`](../../.github/workflows/windows-packages.yml) and +[`.NET tool`](../../.github/workflows/dotnet-tool.yml) workflows use this to get +that executable. `-Path` and `-KeysPath` run the same checks on files that are +already local. It needs `gpg`. ## `get-voted-compiler-tests.ps1` @@ -164,7 +165,14 @@ PS C:\thrift> .\build\windows\build-dotnet-tool.ps1 -Version 0.26.0 -Compiler C: As with the installer, the version is passed in rather than stored in the project file, so [`build/veralign.sh`](../veralign.sh) gains nothing to keep in step. Building without `-Compiler` fails rather than producing a package with no -compiler in it. +compiler in it. `LICENSE` and `NOTICE` come from the checkout unless +`-SourceRoot` names another directory. + +For a release, the [`.NET tool`](../../.github/workflows/dotnet-tool.yml) +workflow packs the voted `thrift-.exe` from `dist/release`, fetched +and checked with `get-voted-compiler.ps1`, together with the `LICENSE` and +`NOTICE` of the release tag. Started by hand with `compiler_url`, it does the +same with the executable at that URL and publishes nothing. ## `dotnet-tool/` @@ -199,7 +207,8 @@ in there and are not empty - and then proves the installed command works. It runs anywhere PowerShell and the .NET SDK do. Off Windows it checks the launcher's refusal instead of the compiler's output, which is the behaviour that -keeps a Linux user from a confusing failure. +keeps a Linux user from a confusing failure. `-ExpectedSha256` checks that the +bundled `thrift.exe` is the very file the package was built from. ## `build-chocolatey-package.ps1` and `chocolatey/` diff --git a/build/windows/dotnet-tool/README.md b/build/windows/dotnet-tool/README.md index 6c3e21e3277..7ffa896cbdc 100644 --- a/build/windows/dotnet-tool/README.md +++ b/build/windows/dotnet-tool/README.md @@ -57,6 +57,9 @@ distributions package Thrift, and Homebrew has it as `thrift`. ## Provenance -The executable in this package is built from the Apache Thrift source release of -the same version. Releases are voted on by the Apache Thrift PMC; the source -release is the official artifact, and this package is a convenience build of it. +The executable in this package is the Windows compiler that the Apache Thrift +PMC voted on with the source release of the same version, the +`thrift-.exe` on the [Download](https://thrift.apache.org/download) +page. Its checksums and its signature are checked against the project's `KEYS` +before it is packed. The source release is the official artifact, and this +package is a convenience package of it. diff --git a/build/windows/dotnet-tool/test-dotnet-tool.ps1 b/build/windows/dotnet-tool/test-dotnet-tool.ps1 index c0fece88b36..905ef687b5d 100644 --- a/build/windows/dotnet-tool/test-dotnet-tool.ps1 +++ b/build/windows/dotnet-tool/test-dotnet-tool.ps1 @@ -44,6 +44,11 @@ Text the bundled executable should print when run. Only checked on Windows. Optional. +.PARAMETER ExpectedSha256 + The SHA-256 the bundled thrift.exe should have, to show that the package + carries the very file it was built from - for a release, the one the vote + covered. Checked on every platform. Optional. + .EXAMPLE pwsh build\windows\dotnet-tool\test-dotnet-tool.ps1 -Package nupkg\Apache.Thrift.Compiler.0.26.0.nupkg -Version 0.26.0 #> @@ -54,7 +59,8 @@ param( [string] $Package, [Parameter(Mandatory = $true)] [string] $Version, - [string] $ExpectedCompilerOutput = '' + [string] $ExpectedCompilerOutput = '', + [string] $ExpectedSha256 = '' ) $ErrorActionPreference = 'Stop' @@ -117,6 +123,24 @@ try { Assert-True 'the compiler appears exactly once' ($payloads.Count -eq 1) ` ("found: " + ($payloads -join ', ')) + if ($ExpectedSha256) { + $entry = $archive.GetEntry("$toolDir/thrift.exe") + $actual = '' + if ($entry) { + $stream = $entry.Open() + $sha = [System.Security.Cryptography.SHA256]::Create() + try { + $actual = [System.BitConverter]::ToString($sha.ComputeHash($stream)).Replace('-', '').ToLowerInvariant() + } + finally { + $sha.Dispose() + $stream.Dispose() + } + } + Assert-True 'the bundled compiler is the expected file' ` + ($actual -eq $ExpectedSha256.ToLowerInvariant()) "sha256 $actual, expected $ExpectedSha256" + } + function Read-Entry { param([string] $Name) diff --git a/doc/ReleaseManagement.md b/doc/ReleaseManagement.md index c9a073bc4b6..e9d2a501fb1 100644 --- a/doc/ReleaseManagement.md +++ b/doc/ReleaseManagement.md @@ -619,6 +619,11 @@ See https://thrift.apache.org/lib/ for the current status of each external packa can be installed with `dotnet tool install --global Apache.Thrift.Compiler`. It is not the `ApacheThrift` runtime library, which is still published by hand following the instructions in `ApacheThrift.nuspec`. + * The package carries the voted `thrift-.exe` from `dist/release`, not a new build. + The workflow downloads it and checks its checksums and its signature against `KEYS` first; + if that fails, nothing is packed or published. To check the package beforehand, for example + around a release candidate's executable, start the workflow from the Actions tab with + `compiler_url` set to that executable on `dist.apache.org`. A manual run never publishes. * Before publishing, verify that nuget.org has a trusted publishing policy for the `apache/thrift` repository, the `.github/workflows/dotnet-tool.yml` workflow and the `release` environment, and that the repository variable `NUGET_USER` names the nuget.org