diff --git a/.github/workflows/windows-packages.yml b/.github/workflows/windows-packages.yml index f4e5d837d21..3c4f458206c 100644 --- a/.github/workflows/windows-packages.yml +++ b/.github/workflows/windows-packages.yml @@ -18,15 +18,25 @@ # # Windows packages for the Thrift compiler. # -# The compiler build here mirrors the compiler-windows job in cmake.yml, minus -# the tests, which that job already runs on every push. This workflow exists to -# build and test what gets packaged. +# The installer of a release has to contain the thrift.exe the vote covered, +# not a new build of the same source: two builds are not the same bytes. So +# where there is such an executable, this workflow packages that one: +# +# - for a GitHub release, the thrift-.exe in the release's +# directory on dist.apache.org; +# - for a manual run with compiler_url, the executable at that URL, such as +# the one a release candidate put on dist/dev. +# +# It is used only once its checksum and its signature against KEYS check out. +# Pull requests, and manual runs without compiler_url, build the compiler from +# source instead. That build mirrors the compiler-windows job in cmake.yml, +# minus the tests, which that job already runs on every push. # # The installer that counts is the one on dist.apache.org, signed by a release -# manager before the vote - long before a GitHub release exists. What this -# workflow attaches to the GitHub release is an unsigned convenience copy, so -# that the link is there straight away; the release manager overwrites it with -# the signed file afterwards. See doc/ReleaseManagement.md. +# manager. What this workflow attaches to the GitHub release is an unsigned +# convenience copy, so that the link is there straight away; the release +# manager overwrites it with the signed file afterwards. See +# doc/ReleaseManagement.md. name: "Windows packages" @@ -40,10 +50,16 @@ on: - ".github/workflows/windows-packages.yml" - "build/windows/**" - "doc/ReleaseManagement.md" - # Run by the release manager against the release branch while preparing a - # release candidate. The artifact this produces is what gets signed and - # uploaded to dist.apache.org. + # Run by the release manager from the release branch while preparing a + # release candidate, with compiler_url pointing at the candidate's signed + # executable on dist/dev. The installer this produces is what gets signed + # and uploaded next to it. workflow_dispatch: + inputs: + compiler_url: + description: "A signed thrift-X.Y.Z.exe on dist.apache.org to package, for example https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0.exe. Empty builds the compiler from the branch instead." + required: false + type: string concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -53,8 +69,97 @@ permissions: contents: read jobs: + # Tests the checks get-voted-compiler.ps1 makes on every run, pull requests + # included, so that a release does not find out that they are broken. + download-checks: + name: Download checks + # gpg is at hand on Linux. + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Test the download checks + shell: pwsh + run: ./build/windows/get-voted-compiler-tests.ps1 + + voted-compiler: + name: Voted compiler + needs: download-checks + if: ${{ github.event_name == 'release' || inputs.compiler_url != '' }} + # gpg is at hand on Linux. + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + outputs: + version: ${{ steps.fetch.outputs.version }} + url: ${{ steps.fetch.outputs.url }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Download and check the compiler + id: fetch + shell: pwsh + env: + RELEASE_TAG: ${{ github.event.release.tag_name }} + INPUT_URL: ${{ inputs.compiler_url }} + run: | + $url = $env:INPUT_URL + if (-not $url) { + # A release is promoted to dist/release before its GitHub release + # is published, so the executable is already there. + $version = $env:RELEASE_TAG -replace '^v', '' + $url = "https://dist.apache.org/repos/dist/release/thrift/$version/thrift-$version.exe" + } + "url=$url" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + ./build/windows/get-voted-compiler.ps1 -Url $url -OutputDir voted + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + - name: Collect LICENSE and NOTICE + env: + VERSION: ${{ steps.fetch.outputs.version }} + run: | + # The installer carries the LICENSE and NOTICE of the release it + # installs. A release has a tag to take them from. A release + # candidate has none yet, so its files come from the branch this + # runs on, which then has to be at that version. + if git fetch --quiet --depth=1 origin "refs/tags/v${VERSION}:refs/tags/v${VERSION}" 2>/dev/null; then + git show "v${VERSION}:LICENSE" > voted/LICENSE + git show "v${VERSION}:NOTICE" > voted/NOTICE + echo "LICENSE and NOTICE from tag v${VERSION}" + else + branch_version=$(sed -n 's/^ *set(thrift_VERSION "\([0-9.]*\)").*/\1/p' CMakeLists.txt) + if [ "$branch_version" != "$VERSION" ]; then + echo "::error::There is no tag v${VERSION}, and this branch is at ${branch_version}. Run this from the branch the release candidate was made from." + exit 1 + fi + cp LICENSE NOTICE voted/ + echo "LICENSE and NOTICE from ${GITHUB_REF_NAME}" + fi + + - name: Archive the compiler + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: voted-compiler + if-no-files-found: error + path: voted/ + retention-days: 3 + installer: name: Installer + needs: [download-checks, voted-compiler] + # Runs when the voted compiler is there, and when there was none to fetch + # and voted-compiler was skipped. Not when fetching or checking it failed, + # and not when the checks themselves failed their tests: a release must + # not fall back to building its own compiler. + if: ${{ !cancelled() && needs.download-checks.result == 'success' && needs.voted-compiler.result != 'failure' }} runs-on: windows-2025 timeout-minutes: 60 permissions: @@ -69,7 +174,15 @@ jobs: - name: Determine the version id: version shell: pwsh + env: + VOTED_VERSION: ${{ needs.voted-compiler.outputs.version }} run: | + if ($env:VOTED_VERSION) { + Write-Host "Version: $env:VOTED_VERSION" + "version=$env:VOTED_VERSION" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + exit 0 + } + $sourceVersion = '' foreach ($line in Get-Content CMakeLists.txt) { if ($line -match '^\s*set\(thrift_VERSION\s+"(\d+\.\d+\.\d+)"\)') { @@ -86,13 +199,22 @@ jobs: Write-Host "Version: $version" "version=$version" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + - name: Download the voted compiler + if: ${{ needs.voted-compiler.result == 'success' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: voted-compiler + path: voted + - name: Install dependencies + if: ${{ needs.voted-compiler.result != 'success' }} shell: pwsh timeout-minutes: 10 # Pinned to the version build/docker/msvc/Dockerfile uses. run: choco install winflexbison3 --version=2.5.24.20210105 -y --no-progress - name: Build the compiler + if: ${{ needs.voted-compiler.result != 'success' }} shell: pwsh timeout-minutes: 30 run: | @@ -104,34 +226,67 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } cmake --install cmake_build --config Release --prefix cmake_install + - name: Choose the compiler to package + id: compiler + shell: pwsh + env: + VERSION: ${{ steps.version.outputs.version }} + VOTED: ${{ needs.voted-compiler.result == 'success' }} + run: | + if ($env:VOTED -eq 'true') { + $compiler = "voted/thrift-$env:VERSION.exe" + # The LICENSE and NOTICE of that release, collected with it. + $sourceRoot = 'voted' + } + else { + $compiler = 'cmake_install/bin/thrift.exe' + $sourceRoot = '' + } + $sha256 = (Get-FileHash -Algorithm SHA256 -LiteralPath $compiler).Hash.ToLowerInvariant() + Write-Host "Packaging $compiler, sha256 $sha256" + "path=$compiler" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + "source_root=$sourceRoot" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + "sha256=$sha256" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + - name: Check what the compiler imports shell: pwsh - run: ./build/windows/check-compiler-imports.ps1 -Path cmake_install/bin/thrift.exe + env: + COMPILER: ${{ steps.compiler.outputs.path }} + run: ./build/windows/check-compiler-imports.ps1 -Path $env:COMPILER - name: Build the installer shell: pwsh env: VERSION: ${{ steps.version.outputs.version }} + COMPILER: ${{ steps.compiler.outputs.path }} + SOURCE_ROOT: ${{ steps.compiler.outputs.source_root }} run: | - ./build/windows/build-installer.ps1 ` - -Version $env:VERSION ` - -Compiler cmake_install/bin/thrift.exe ` - -OutputDir dist + $installerArgs = @{ Version = $env:VERSION; Compiler = $env:COMPILER; OutputDir = 'dist' } + if ($env:SOURCE_ROOT) { + $installerArgs['SourceRoot'] = $env:SOURCE_ROOT + } + ./build/windows/build-installer.ps1 @installerArgs - name: Test the installer shell: pwsh timeout-minutes: 15 env: VERSION: ${{ steps.version.outputs.version }} + COMPILER_SHA256: ${{ steps.compiler.outputs.sha256 }} run: | + # The hash shows that the installer carries the very file it was + # given - for a release, the one the vote covered. ./build/windows/installer/test-installer.ps1 ` -Installer "dist/thrift-$env:VERSION-setup.exe" ` - -ExpectedVersion $env:VERSION + -ExpectedVersion $env:VERSION ` + -ExpectedSha256 $env:COMPILER_SHA256 - name: Report the checksum shell: pwsh env: VERSION: ${{ steps.version.outputs.version }} + COMPILER_SHA256: ${{ steps.compiler.outputs.sha256 }} + COMPILER_URL: ${{ needs.voted-compiler.outputs.url }} run: | $installer = "dist/thrift-$env:VERSION-setup.exe" $hash = (Get-FileHash -Algorithm SHA256 $installer).Hash.ToLowerInvariant() @@ -145,7 +300,9 @@ jobs: "|---|---|", "| file | ``$([System.IO.Path]::GetFileName($installer))`` |", "| size | $size bytes |", - "| sha256 | ``$hash`` |" + "| sha256 | ``$hash`` |", + "| compiler | $(if ($env:COMPILER_URL) { $env:COMPILER_URL } else { 'built from source' }) |", + "| compiler sha256 | ``$env:COMPILER_SHA256`` |" ) | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append - name: Archive the installer @@ -183,7 +340,7 @@ jobs: installer="dist/thrift-${VERSION}-setup.exe" test -f "$installer" sha256sum "$installer" - # This is the unsigned convenience copy. The release manager replaces - # it with the signed file from dist.apache.org after the vote, which - # is also a --clobber upload. + # This is the unsigned convenience copy, around the voted compiler. + # The release manager replaces it with the signed file from + # dist.apache.org after the vote, which is also a --clobber upload. gh release upload "$RELEASE_TAG" "$installer" --clobber diff --git a/build/windows/README.md b/build/windows/README.md index f3d4c00233d..a4768d6d16e 100644 --- a/build/windows/README.md +++ b/build/windows/README.md @@ -65,6 +65,40 @@ grew far enough rejects nothing. The tests pin down that a release build passes, that a static build and a third-party dependency are both rejected, and that the wildcard matching does what the default allow list relies on. +## `get-voted-compiler.ps1` + +Fetches the `thrift-.exe` of a release candidate or a release from +`dist.apache.org` and checks it before anything is built from it: every +checksum file next to it (`.sha512`, `.sha256`) must match, at least one must be +there, and its detached signature must be good and made with a key from the +project's `KEYS`. Only `dist.apache.org` URLs naming a `thrift-.exe` +are accepted. + +```powershell +PS C:\thrift> .\build\windows\get-voted-compiler.ps1 -Url https://dist.apache.org/repos/dist/release/thrift/0.25.0/thrift-0.25.0.exe -OutputDir voted +``` + +The installer of a release has to contain the executable the vote covered, and +a new build of the same source is not the same bytes. The +[`Windows packages`](../../.github/workflows/windows-packages.yml) workflow uses +this to get that executable. `-Path` and `-KeysPath` run the same checks on +files that are already local. It needs `gpg`. + +## `get-voted-compiler-tests.ps1` + +Tests for the above. They sign stand-in executables with a throwaway GPG key, so +they need `gpg` but no network: + +```bash +$ pwsh build/windows/get-voted-compiler-tests.ps1 +``` + +A signed executable with matching checksums passes. Refused are: one changed +after signing, also when its checksums were redone; one with a wrong checksum; +one signed by a key that is not in `KEYS`; one without a signature or without a +checksum; and URLs that are not on `dist.apache.org` or do not name a +`thrift-.exe`. + ## `build-installer.ps1` Packages a built `thrift.exe` into a Windows installer with @@ -76,7 +110,9 @@ PS C:\thrift> .\build\windows\build-installer.ps1 -Version 0.26.0 -Compiler C:\i ``` It writes `thrift--setup.exe` into the output directory and prints its -path. `ISCC.exe` is located automatically; pass `-Iscc` to override. +path. `ISCC.exe` is located automatically; pass `-Iscc` to override. The +installer also carries `LICENSE` and `NOTICE`, taken from the checkout unless +`-SourceRoot` names another directory. ## `installer/thrift.iss` @@ -112,6 +148,10 @@ machine. PS C:\thrift> .\build\windows\installer\test-installer.ps1 -Installer dist\thrift-0.26.0-setup.exe ``` +`-ExpectedVersion` checks what the installed compiler reports, and +`-ExpectedSha256` that the installed `thrift.exe` is the very file the installer +was built from. + ## `build-dotnet-tool.ps1` Packs a built `thrift.exe` as a [.NET tool](https://learn.microsoft.com/en-us/dotnet/core/tools/global-tools-how-to-create), diff --git a/build/windows/build-installer.ps1 b/build/windows/build-installer.ps1 index e343552ed2c..2648d8323fc 100644 --- a/build/windows/build-installer.ps1 +++ b/build/windows/build-installer.ps1 @@ -94,8 +94,11 @@ if (-not (Test-Path -LiteralPath $Compiler)) { } if (-not $SourceRoot) { - $SourceRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path + $SourceRoot = Join-Path $PSScriptRoot '..\..' } +# Inno Setup resolves a relative path against thrift.iss, not against the +# current directory, so it gets an absolute one. +$SourceRoot = (Resolve-Path -LiteralPath $SourceRoot).Path foreach ($required in @('LICENSE', 'NOTICE')) { if (-not (Test-Path -LiteralPath (Join-Path $SourceRoot $required))) { throw "$required not found below -SourceRoot $SourceRoot" diff --git a/build/windows/get-voted-compiler-tests.ps1 b/build/windows/get-voted-compiler-tests.ps1 new file mode 100644 index 00000000000..1ab62442399 --- /dev/null +++ b/build/windows/get-voted-compiler-tests.ps1 @@ -0,0 +1,237 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +<# +.SYNOPSIS + Tests for get-voted-compiler.ps1. + +.DESCRIPTION + Signs stand-in executables with a throwaway GPG key and lets + get-voted-compiler.ps1 check them the way it checks a download from + dist.apache.org. Nothing is downloaded: the URL checks are exercised with + URLs the script has to refuse before it fetches anything. + + The point of these tests is that the checks must be able to *fail*. A + signature check that passes everything would package whatever happens to + be at the URL. + + Needs gpg on PATH, or pass -Gpg. + +.EXAMPLE + pwsh build/windows/get-voted-compiler-tests.ps1 +#> + +[CmdletBinding()] +param( + [string] $Gpg = 'gpg' +) + +$ErrorActionPreference = 'Stop' + +$scriptUnderTest = Join-Path $PSScriptRoot 'get-voted-compiler.ps1' +if (-not (Test-Path -LiteralPath $scriptUnderTest)) { + throw "get-voted-compiler.ps1 not found next to $PSCommandPath" +} + +$workDir = Join-Path ([System.IO.Path]::GetTempPath()) ("thrift-voted-compiler-tests-" + [System.Guid]::NewGuid().ToString('N')) +New-Item -Path $workDir -ItemType Directory -Force | Out-Null + +$failures = [System.Collections.Generic.List[string]]::new() +$passed = 0 +$homes = [System.Collections.Generic.List[string]]::new() + +function Invoke-Gpg { + param([string] $GpgHome, [string[]] $Arguments) + + $output = & $Gpg --batch --quiet --no-permission-warning --homedir $GpgHome --pinentry-mode loopback --passphrase '' @Arguments 2>&1 + if ($LASTEXITCODE -ne 0) { + throw "gpg $($Arguments -join ' ') failed: $output" + } +} + +function New-Signer { + param([string] $Name) + + $gpgHome = Join-Path $workDir "gnupg-$Name" + New-Item -Path $gpgHome -ItemType Directory -Force | Out-Null + $script:homes.Add($gpgHome) + Invoke-Gpg $gpgHome @('--quick-gen-key', "$Name <$Name@example.invalid>", 'ed25519', 'sign', 'never') + return $gpgHome +} + +function New-Candidate { + # Lays out what a release directory on dist.apache.org holds for the + # compiler: the executable, its detached signature and its checksums. + param( + [string] $Case, + [string] $Signer, + [string] $FileName = 'thrift-1.2.3.exe', + [string[]] $Checksums = @('SHA512', 'SHA256'), + [switch] $NoSignature + ) + + $dir = Join-Path $workDir $Case + New-Item -Path $dir -ItemType Directory -Force | Out-Null + $exe = Join-Path $dir $FileName + Set-Content -LiteralPath $exe -Value "stand-in for the compiler, case $Case" -NoNewline + if (-not $NoSignature) { + Invoke-Gpg $Signer @('--armor', '--detach-sign', '--output', "$exe.asc", $exe) + } + foreach ($algorithm in $Checksums) { + $hash = (Get-FileHash -Algorithm $algorithm -LiteralPath $exe).Hash.ToLowerInvariant() + # The format sha512sum writes in binary mode, which is what the + # release instructions produce. + Set-Content -LiteralPath "$exe.$($algorithm.ToLowerInvariant())" -Value "$hash *$FileName" + } + return $exe +} + +function Invoke-Script { + param([hashtable] $ScriptArgs) + + # *>&1 folds every stream into the pipeline so the script under test does + # not print over the test report, and so its error records cannot become + # terminating errors here. + $transcript = & $scriptUnderTest @ScriptArgs -Gpg $Gpg *>&1 | Out-String + Write-Verbose $transcript + return $LASTEXITCODE +} + +function Assert-True { + param([string] $Name, [bool] $Condition, [string] $Detail = '') + + if ($Condition) { + Write-Host " PASS $Name" + $script:passed++ + } + else { + Write-Host " FAIL $Name $Detail" + $script:failures.Add($Name) + } +} + +function Assert-Rejected { + param([string] $Name, [hashtable] $ScriptArgs) + + $outputDir = Join-Path $workDir ("out-" + [System.Guid]::NewGuid().ToString('N')) + $ScriptArgs['OutputDir'] = $outputDir + $exitCode = Invoke-Script $ScriptArgs + $leftBehind = @(Get-ChildItem -LiteralPath $outputDir -Filter '*.exe' -ErrorAction SilentlyContinue).Count + Assert-True $Name (($exitCode -ne 0) -and ($leftBehind -eq 0)) "(exit $exitCode, $leftBehind executable(s) written)" +} + +try { + $releaseManager = New-Signer 'release-manager' + $outsider = New-Signer 'outsider' + + # KEYS holds the release manager's key only. + $keys = Join-Path $workDir 'KEYS' + $exported = & $Gpg --batch --no-permission-warning --homedir $releaseManager --armor --export + Set-Content -LiteralPath $keys -Value $exported + + Write-Host 'Executables that must pass' + + $good = New-Candidate -Case 'good' -Signer $releaseManager + $outputDir = Join-Path $workDir 'out-good' + $githubOutput = Join-Path $workDir 'github-output' + $env:GITHUB_OUTPUT = $githubOutput + try { + $exitCode = Invoke-Script @{ Path = $good; KeysPath = $keys; OutputDir = $outputDir } + } + finally { + Remove-Item Env:GITHUB_OUTPUT + } + $copied = Join-Path $outputDir 'thrift-1.2.3.exe' + $goodHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $good).Hash.ToLowerInvariant() + Assert-True 'a signed executable with matching checksums passes' ($exitCode -eq 0) "(exit $exitCode)" + Assert-True 'it is copied to the output directory unchanged' ` + ((Test-Path -LiteralPath $copied) -and ((Get-FileHash -Algorithm SHA256 -LiteralPath $copied).Hash.ToLowerInvariant() -eq $goodHash)) + $outputs = if (Test-Path -LiteralPath $githubOutput) { Get-Content -LiteralPath $githubOutput } else { @() } + Assert-True 'the version goes to GITHUB_OUTPUT' ($outputs -contains 'version=1.2.3') "(got: $($outputs -join ', '))" + Assert-True 'the sha256 goes to GITHUB_OUTPUT' ($outputs -contains "sha256=$goodHash") "(got: $($outputs -join ', '))" + + # 0.25.0 was published with .sha256 next to the executable, and no .sha512. + $sha256Only = New-Candidate -Case 'sha256-only' -Signer $releaseManager -Checksums @('SHA256') + Assert-True 'a .sha256 alone is enough' ` + ((Invoke-Script @{ Path = $sha256Only; KeysPath = $keys; OutputDir = (Join-Path $workDir 'out-sha256') }) -eq 0) + + Write-Host 'Executables that must be refused' + + $tampered = New-Candidate -Case 'tampered' -Signer $releaseManager + Add-Content -LiteralPath $tampered -Value ' changed after signing' -NoNewline + Assert-Rejected 'changed after signing' @{ Path = $tampered; KeysPath = $keys } + + # Whoever can replace the executable can usually replace its checksums too; + # only the signature catches that. + $resummed = New-Candidate -Case 'resummed' -Signer $releaseManager + Add-Content -LiteralPath $resummed -Value ' changed after signing' -NoNewline + foreach ($algorithm in @('SHA512', 'SHA256')) { + $hash = (Get-FileHash -Algorithm $algorithm -LiteralPath $resummed).Hash.ToLowerInvariant() + Set-Content -LiteralPath "$resummed.$($algorithm.ToLowerInvariant())" -Value "$hash *thrift-1.2.3.exe" + } + Assert-Rejected 'changed after signing, checksums redone' @{ Path = $resummed; KeysPath = $keys } + + $wrongSha512 = New-Candidate -Case 'wrong-sha512' -Signer $releaseManager + Set-Content -LiteralPath "$wrongSha512.sha512" -Value (('0' * 128) + ' *thrift-1.2.3.exe') + Assert-Rejected 'one of two checksums is wrong' @{ Path = $wrongSha512; KeysPath = $keys } + + $foreign = New-Candidate -Case 'foreign' -Signer $outsider + Assert-Rejected 'signed by a key that is not in KEYS' @{ Path = $foreign; KeysPath = $keys } + + $unsigned = New-Candidate -Case 'unsigned' -Signer $releaseManager -NoSignature + Assert-Rejected 'no signature' @{ Path = $unsigned; KeysPath = $keys } + + $unsummed = New-Candidate -Case 'unsummed' -Signer $releaseManager -Checksums @() + Assert-Rejected 'no checksum' @{ Path = $unsummed; KeysPath = $keys } + + $installer = New-Candidate -Case 'installer' -Signer $releaseManager -FileName 'thrift-1.2.3-setup.exe' + Assert-Rejected 'not named thrift-.exe' @{ Path = $installer; KeysPath = $keys } + + Write-Host 'URLs that must be refused before anything is downloaded' + + foreach ($url in @( + 'https://example.com/repos/dist/release/thrift/1.2.3/thrift-1.2.3.exe', + 'http://dist.apache.org/repos/dist/release/thrift/1.2.3/thrift-1.2.3.exe', + 'https://dist.apache.org/repos/dist/release/other/1.2.3/thrift-1.2.3.exe', + 'https://dist.apache.org/repos/dist/release/thrift/1.2.3/thrift-1.2.3-setup.exe', + 'https://dist.apache.org/repos/dist/release/thrift/1.2.3/thrift-1.2.3.exe?x=1', + 'https://dist.apache.org.example.com/repos/dist/release/thrift/1.2.3/thrift-1.2.3.exe' + )) { + Assert-Rejected "URL $url" @{ Url = $url } + } +} +finally { + # Each home started an agent of its own. + if (Get-Command 'gpgconf' -ErrorAction SilentlyContinue) { + foreach ($gpgHome in $homes) { + & gpgconf --homedir $gpgHome --kill gpg-agent 2>$null + } + } + Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue +} + +Write-Host '' +if ($failures.Count -gt 0) { + Write-Host "$($failures.Count) of $($failures.Count + $passed) checks failed:" + foreach ($failure in $failures) { Write-Host " $failure" } + exit 1 +} + +Write-Host "$passed checks passed." +exit 0 diff --git a/build/windows/get-voted-compiler.ps1 b/build/windows/get-voted-compiler.ps1 new file mode 100644 index 00000000000..e90c7d64eac --- /dev/null +++ b/build/windows/get-voted-compiler.ps1 @@ -0,0 +1,229 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +<# +.SYNOPSIS + Fetches the Windows Thrift compiler a vote covered, and checks it. + +.DESCRIPTION + The Windows installer of a release has to contain the thrift.exe the vote + covered, not a new build of the same source: two builds are not the same + bytes. This downloads that executable from the release candidate's or the + release's directory on dist.apache.org, together with its detached GPG + signature, its checksum files and the project's KEYS file, and checks all + of them before anything is built from it: + + - every checksum file present (.sha512, .sha256) must match, and at least + one must be there; + - the signature must be good, and made with a key from KEYS. + + Only https://dist.apache.org/repos/dist/{dev,release}/thrift/ URLs naming a + thrift-...exe are accepted, so that a typo cannot + package something that was never voted on. + + With -Path instead of -Url, the same checks run on files that are already + local: the executable, with its .asc and checksum files next to it, and + -KeysPath. + + The executable is copied into -OutputDir under its own name, and the + version is printed. In GitHub Actions, version and sha256 also go to + GITHUB_OUTPUT. + +.PARAMETER Url + The executable on dist.apache.org, for example + https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0.exe. + +.PARAMETER Path + A local executable to check instead. + +.PARAMETER KeysUrl + Where to get KEYS when downloading. + +.PARAMETER KeysPath + The KEYS file to check a local executable against. + +.PARAMETER OutputDir + Where to put the executable once it has passed. Created when it does not + exist. + +.PARAMETER Gpg + The gpg executable. Defaults to gpg on PATH. + +.EXAMPLE + pwsh build/windows/get-voted-compiler.ps1 -Url https://dist.apache.org/repos/dist/release/thrift/0.25.0/thrift-0.25.0.exe -OutputDir voted +#> + +[CmdletBinding(DefaultParameterSetName = 'Url')] +param( + [Parameter(Mandatory = $true, ParameterSetName = 'Url')] + [string] $Url, + [Parameter(ParameterSetName = 'Url')] + [string] $KeysUrl = 'https://downloads.apache.org/thrift/KEYS', + + [Parameter(Mandatory = $true, ParameterSetName = 'Path')] + [string] $Path, + [Parameter(Mandatory = $true, ParameterSetName = 'Path')] + [string] $KeysPath, + + [string] $OutputDir = 'voted', + [string] $Gpg = 'gpg' +) + +$ErrorActionPreference = 'Stop' + +$NamePattern = '^thrift-(\d+\.\d+\.\d+)\.exe$' +$UrlPattern = '^https://dist\.apache\.org/repos/dist/(dev|release)/thrift/[A-Za-z0-9._-]+/(thrift-\d+\.\d+\.\d+\.exe)$' +$ChecksumAlgorithms = @('SHA512', 'SHA256') + +function Save-Download { + param([string] $Uri, [string] $OutFile, [switch] $Optional) + + try { + Invoke-WebRequest -Uri $Uri -OutFile $OutFile -UseBasicParsing + return $true + } + catch { + $status = $_.Exception.Response.StatusCode.value__ + if ($Optional -and $status -eq 404) { + return $false + } + throw "Could not download $Uri : $($_.Exception.Message)" + } +} + +function Test-Checksums { + param([string] $File) + + $checked = 0 + foreach ($algorithm in $ChecksumAlgorithms) { + $sumFile = "$File.$($algorithm.ToLowerInvariant())" + if (-not (Test-Path -LiteralPath $sumFile)) { + continue + } + # sha512sum writes " *" or " "; only the + # hash matters. + $expected = ((Get-Content -LiteralPath $sumFile -Raw).Trim() -split '\s+')[0].ToLowerInvariant() + $actual = (Get-FileHash -Algorithm $algorithm -LiteralPath $File).Hash.ToLowerInvariant() + if ($expected -ne $actual) { + throw "$algorithm mismatch for $(Split-Path -Leaf $File): the checksum file says $expected, the file has $actual." + } + Write-Host "$algorithm matches: $actual" + $checked++ + } + if ($checked -eq 0) { + throw "No checksum file next to $(Split-Path -Leaf $File); expected one of: $(($ChecksumAlgorithms | ForEach-Object { '.' + $_.ToLowerInvariant() }) -join ', ')." + } +} + +function Test-Signature { + param([string] $File, [string] $Keys) + + $signature = "$File.asc" + if (-not (Test-Path -LiteralPath $signature)) { + throw "No signature $(Split-Path -Leaf $signature) next to the executable." + } + + # A keyring of its own, holding nothing but KEYS, so that no key from the + # machine this runs on can vouch for the file. + $gpgHome = Join-Path ([System.IO.Path]::GetTempPath()) ("thrift-voted-compiler-gnupg-" + [System.Guid]::NewGuid().ToString('N')) + New-Item -Path $gpgHome -ItemType Directory -Force | Out-Null + try { + # A KEYS file can hold keys gpg no longer imports. That is not an + # error here as long as the key that signed the file is among the rest. + $null = & $Gpg --batch --quiet --no-permission-warning --homedir $gpgHome --import $Keys 2>&1 + + $status = & $Gpg --batch --no-permission-warning --homedir $gpgHome --status-fd 1 --verify $signature $File 2>$null + $verifyExit = $LASTEXITCODE + # GOODSIG only comes for a good signature by a key that has neither + # expired nor been revoked; BADSIG, EXPKEYSIG, REVKEYSIG and ERRSIG + # all mean no. + $good = @($status | Where-Object { $_ -match '^\[GNUPG:\] GOODSIG ' }) + if ($verifyExit -ne 0 -or $good.Count -eq 0) { + throw "The signature on $(Split-Path -Leaf $File) does not check out against KEYS: $($status -join ' | ')" + } + Write-Host "Signature good: $(($good[0] -split ' ', 4)[3])" + } + finally { + if (Get-Command 'gpgconf' -ErrorAction SilentlyContinue) { + & gpgconf --homedir $gpgHome --kill gpg-agent 2>$null + } + Remove-Item -LiteralPath $gpgHome -Recurse -Force -ErrorAction SilentlyContinue + } +} + +$downloadDir = $null +try { + if ($PSCmdlet.ParameterSetName -eq 'Url') { + if ($Url -notmatch $UrlPattern) { + throw "Not a Windows compiler on dist.apache.org: $Url. Expected https://dist.apache.org/repos/dist/dev/thrift//thrift-.exe or .../release/thrift//thrift-.exe." + } + $name = $Matches[2] + + $downloadDir = Join-Path ([System.IO.Path]::GetTempPath()) ("thrift-voted-compiler-" + [System.Guid]::NewGuid().ToString('N')) + New-Item -Path $downloadDir -ItemType Directory -Force | Out-Null + $file = Join-Path $downloadDir $name + $keys = Join-Path $downloadDir 'KEYS' + + Write-Host "Downloading $Url" + $null = Save-Download -Uri $Url -OutFile $file + $null = Save-Download -Uri "$Url.asc" -OutFile "$file.asc" -Optional + foreach ($algorithm in $ChecksumAlgorithms) { + $extension = '.' + $algorithm.ToLowerInvariant() + $null = Save-Download -Uri "$Url$extension" -OutFile "$file$extension" -Optional + } + $null = Save-Download -Uri $KeysUrl -OutFile $keys + } + else { + if (-not (Test-Path -LiteralPath $Path)) { + throw "No such file: $Path" + } + $file = (Resolve-Path -LiteralPath $Path).Path + $keys = $KeysPath + } + + $name = Split-Path -Leaf $file + if ($name -notmatch $NamePattern) { + throw "$name is not named thrift-...exe." + } + $version = $Matches[1] + + Test-Checksums -File $file + Test-Signature -File $file -Keys $keys + + New-Item -Path $OutputDir -ItemType Directory -Force | Out-Null + $target = Join-Path $OutputDir $name + Copy-Item -LiteralPath $file -Destination $target -Force + $sha256 = (Get-FileHash -Algorithm SHA256 -LiteralPath $target).Hash.ToLowerInvariant() + + if ($env:GITHUB_OUTPUT) { + "version=$version" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + "sha256=$sha256" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + } + Write-Host "Checked $name, version $version, sha256 $sha256" + Write-Output $version +} +catch { + Write-Host "::error::$($_.Exception.Message)" + exit 1 +} +finally { + if ($downloadDir) { + Remove-Item -LiteralPath $downloadDir -Recurse -Force -ErrorAction SilentlyContinue + } +} diff --git a/build/windows/installer/test-installer.ps1 b/build/windows/installer/test-installer.ps1 index fdfabb95a10..d3bca1cba60 100644 --- a/build/windows/installer/test-installer.ps1 +++ b/build/windows/installer/test-installer.ps1 @@ -41,6 +41,11 @@ .PARAMETER ExpectedVersion Version thrift.exe is expected to report. Optional. +.PARAMETER ExpectedSha256 + SHA-256 the installed thrift.exe is expected to have, which shows that the + installer carries that very file - for a release, the one the vote + covered. Optional. + .EXAMPLE pwsh build\windows\installer\test-installer.ps1 -Installer dist\thrift-0.26.0-setup.exe #> @@ -49,7 +54,8 @@ param( [Parameter(Mandatory = $true)] [string] $Installer, - [string] $ExpectedVersion = '' + [string] $ExpectedVersion = '', + [string] $ExpectedSha256 = '' ) $ErrorActionPreference = 'Stop' @@ -126,6 +132,11 @@ try { $compiler = Join-Path $installDir 'thrift.exe' Assert-True 'thrift.exe is installed' (Test-Path -LiteralPath $compiler) + if ($ExpectedSha256 -and (Test-Path -LiteralPath $compiler)) { + $installedHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $compiler).Hash.ToLowerInvariant() + Assert-True 'the installed thrift.exe is the expected file' ` + ($installedHash -eq $ExpectedSha256.ToLowerInvariant()) "sha256 $installedHash" + } Assert-True 'LICENSE is installed' (Test-Path -LiteralPath (Join-Path $installDir 'LICENSE.txt')) Assert-True 'NOTICE is installed' (Test-Path -LiteralPath (Join-Path $installDir 'NOTICE.txt')) diff --git a/doc/ReleaseManagement.md b/doc/ReleaseManagement.md index b739f04a1fe..98f5681f844 100644 --- a/doc/ReleaseManagement.md +++ b/doc/ReleaseManagement.md @@ -214,11 +214,16 @@ All Apache Thrift releases go through a 72-hour final release candidate voting p This needs [Inno Setup](https://jrsoftware.org/isdl.php) 6.3 or later. The result is `dist\thrift-1.0.0-setup.exe`. - Rather than installing Inno Setup, you can run the - [`Windows packages`](../.github/workflows/windows-packages.yml) workflow against the - release branch from the Actions tab and download its `windows-installer` artifact. The - workflow builds the compiler the same way and also installs and uninstalls the result to - check it. + Rather than installing Inno Setup, you can have the + [`Windows packages`](../.github/workflows/windows-packages.yml) workflow build it around + the signed executable. Upload the executable with its signature and checksums to the + release candidate's directory first (next step), then run the workflow from the release + branch on the Actions tab, with `compiler_url` set to the executable's URL there, for + example `https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0.exe`. It + checks the checksum and the signature against `KEYS`, packages exactly that file, + installs and uninstalls the result to check it, and leaves the installer as its + `windows-installer` artifact. Its own build of the compiler, which it uses without + `compiler_url`, will not do: that is not the executable you signed. 1. Copy the installer to your linux system and sign and checksum it the same way you signed the tarball and the executable. @@ -262,14 +267,14 @@ All Apache Thrift releases go through a 72-hour final release candidate voting p I propose that we accept the following release candidate as the official Apache Thrift 1.0.0 release: - https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0-rc0.tar.gz + https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0.tar.gz The release candidate was created from the release/1.0.0 branch and can be cloned using: git clone -b release/1.0.0 https://github.com/apache/thrift.git The release candidates GPG signature can be found at: - https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0-rc0.tar.gz.asc + https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0.tar.gz.asc The release candidates checksums are: sha256: @@ -277,10 +282,10 @@ All Apache Thrift releases go through a 72-hour final release candidate voting p A prebuilt Windows compiler is available at: - https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0-rc0.exe + https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0.exe Prebuilt Windows compiler GPG signature: - https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0-rc0.exe.asc + https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0.exe.asc Prebuilt Windows compiler checksums are: sha256: @@ -288,10 +293,10 @@ All Apache Thrift releases go through a 72-hour final release candidate voting p A Windows installer for the compiler is available at: - https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0-rc0-setup.exe + https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0-setup.exe Windows installer GPG signature: - https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0-rc0-setup.exe.asc + https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0-setup.exe.asc Windows installer checksums are: sha256: @@ -299,10 +304,10 @@ All Apache Thrift releases go through a 72-hour final release candidate voting p The source tree as ZIP file to be published via Github releases: - https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0-rc0.zip + https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0.zip ZIP source tree GPG signature: - https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0-rc0.zip.asc + https://dist.apache.org/repos/dist/dev/thrift/1.0.0-rc0/thrift-1.0.0.zip.asc ZIP source tree checksums are: sha256: @@ -489,18 +494,28 @@ from `dist.apache.org`. The file the vote covers has to carry a release manager's signature, so it cannot be produced by a workflow at release time. What *is* automated is building and testing it. The -[`Windows packages`](../.github/workflows/windows-packages.yml) workflow builds -the compiler on a Windows runner the way a release build does, checks what the -executable imports, packages it with Inno Setup, and then installs and -uninstalls the result to check that the compiler runs, that the install -directory lands on `PATH`, and that uninstalling takes that one `PATH` entry -away and leaves the others alone. It runs on every pull request that touches -the packaging, and on demand from the Actions tab - which is how you get the -artifact to sign while preparing the release candidate, described above under -*Generate the Windows Thrift Compiler*. - -When the GitHub release is published, the same workflow attaches an installer to -it as an unsigned convenience copy, so that the download link is there +[`Windows packages`](../.github/workflows/windows-packages.yml) workflow checks +what the executable imports, packages it with Inno Setup, and then installs and +uninstalls the result to check that the compiler runs, that the installed +`thrift.exe` is the very file it was given, that the install directory lands on +`PATH`, and that uninstalling takes that one `PATH` entry away and leaves the +others alone. Which executable it packages depends on how it runs: + +* For a pull request that touches the packaging, and from the Actions tab + without `compiler_url`, it builds the compiler on a Windows runner the way a + release build does. +* From the Actions tab with `compiler_url`, it packages the signed executable + at that URL on `dist.apache.org` - which is how you get the installer to sign + while preparing the release candidate, described above under *Generate the + Windows Thrift Compiler*. +* When the GitHub release is published, it packages the voted executable from + `dist/release`. + +A downloaded executable is used only once its checksums match and its +signature checks out against `KEYS`. + +When the GitHub release is published, the workflow attaches the installer to it +as an unsigned convenience copy, so that the download link is there immediately. **One thing is left to do after the release:** overwrite that asset with the signed file from `dist/release`, so that what people download from GitHub is what the signatures on `dist.apache.org` cover. @@ -513,15 +528,17 @@ If the installer was not part of the release candidate, it is missing from `dist/release` too, and the Chocolatey and WinGet runs of the release fail to download it. To add it afterwards: -1. Build it from the voted compiler, with Inno Setup 6.3 or later, in a checkout - of the release tag, which provides the `LICENSE` and `NOTICE` it installs: +1. Build it around the voted compiler. Run the `Windows packages` workflow from + the Actions tab with `compiler_url` set to the executable in `dist/release`, + for example `https://dist.apache.org/repos/dist/release/thrift/1.0.0/thrift-1.0.0.exe`, + and take its `windows-installer` artifact. It installs the `LICENSE` and + `NOTICE` of the release tag. To build it yourself instead, use Inno Setup + 6.3 or later in a checkout of the release tag: ```powershell PS C:\thrift> .\build\windows\build-installer.ps1 -Version 1.0.0 -Compiler C:\dist\thrift-1.0.0.exe -OutputDir dist ``` - The `windows-installer` artifact of the `Windows packages` workflow will not - do: it contains a compiler that the workflow built itself, not the voted one. 1. Sign and checksum it the same way as the other release artifacts, and commit it with its signature and checksums to `dist/release/thrift/1.0.0`. 1. Overwrite the asset on the GitHub release with it, as shown above.